CINXE.COM

<!DOCTYPE html> <html class="client-nojs vector-feature-language-in-header-enabled vector-feature-language-in-main-page-header-disabled vector-feature-sticky-header-disabled vector-feature-page-tools-pinned-disabled vector-feature-toc-pinned-clientpref-1 vector-feature-main-menu-pinned-disabled vector-feature-limited-width-clientpref-1 vector-feature-limited-width-content-enabled vector-feature-custom-font-size-clientpref-1 vector-feature-appearance-pinned-clientpref-1 vector-feature-night-mode-disabled skin-theme-clientpref-day vector-toc-available" lang="en" dir="ltr"> <head> <meta charset="UTF-8"> <title>SLO/Lift Wing - Wikitech</title> <script>(function(){var className="client-js vector-feature-language-in-header-enabled vector-feature-language-in-main-page-header-disabled vector-feature-sticky-header-disabled vector-feature-page-tools-pinned-disabled vector-feature-toc-pinned-clientpref-1 vector-feature-main-menu-pinned-disabled vector-feature-limited-width-clientpref-1 vector-feature-limited-width-content-enabled vector-feature-custom-font-size-clientpref-1 vector-feature-appearance-pinned-clientpref-1 vector-feature-night-mode-disabled skin-theme-clientpref-day vector-toc-available";var cookie=document.cookie.match(/(?:^|; )labswikimwclientpreferences=([^;]+)/);if(cookie){cookie[1].split('%2C').forEach(function(pref){className=className.replace(new RegExp('(^| )'+pref.replace(/-clientpref-\w+$|[^\w-]+/g,'')+'-clientpref-\\w+( |$)'),'$1'+pref+'$2');});}document.documentElement.className=className;}());RLCONF={"wgBreakFrames":false,"wgSeparatorTransformTable":["",""],"wgDigitTransformTable":["",""],"wgDefaultDateFormat": "dmy","wgMonthNames":["","January","February","March","April","May","June","July","August","September","October","November","December"],"wgRequestId":"70cefad4-c2c4-468f-8560-06ddc62bfd8b","wgCanonicalNamespace":"","wgCanonicalSpecialPageName":false,"wgNamespaceNumber":0,"wgPageName":"SLO/Lift_Wing","wgTitle":"SLO/Lift Wing","wgCurRevisionId":2108688,"wgRevisionId":2108688,"wgArticleId":452512,"wgIsArticle":true,"wgIsRedirect":false,"wgAction":"view","wgUserName":null,"wgUserGroups":["*"],"wgCategories":[],"wgPageViewLanguage":"en","wgPageContentLanguage":"en","wgPageContentModel":"wikitext","wgRelevantPageName":"SLO/Lift_Wing","wgRelevantArticleId":452512,"wgIsProbablyEditable":false,"wgRelevantPageIsProbablyEditable":false,"wgRestrictionEdit":[],"wgRestrictionMove":[],"wgNoticeProject":"wikitech","wgCiteReferencePreviewsActive":true,"wgMediaViewerOnClick":true,"wgMediaViewerEnabledByDefault":true,"wgVisualEditor":{"pageLanguageCode":"en","pageLanguageDir":"ltr","pageVariantFallbacks" :"en"},"wgMFDisplayWikibaseDescriptions":{"search":true,"watchlist":true,"tagline":false,"nearby":true},"wgWMESchemaEditAttemptStepOversample":false,"wgWMEPageLength":20000,"wgEditSubmitButtonLabelPublish":true,"wgDiscussionToolsFeaturesEnabled":{"replytool":true,"newtopictool":true,"sourcemodetoolbar":true,"topicsubscription":false,"autotopicsub":false,"visualenhancements":false,"visualenhancements_reply":false,"visualenhancements_pageframe":false},"wgDiscussionToolsFallbackEditMode":"visual","wgULSPosition":"personal","wgULSisCompactLinksEnabled":false,"wgVector2022LanguageInHeader":true,"wgULSisLanguageSelectorEmpty":false,"wgCheckUserClientHintsHeadersJsApi":["brands","architecture","bitness","fullVersionList","mobile","model","platform","platformVersion"],"wgSiteNoticeId":"2.0"};RLSTATE={"ext.globalCssJs.user.styles":"ready","site.styles":"ready","user.styles":"ready","ext.globalCssJs.user":"ready","user":"ready","user.options":"loading","ext.discussionTools.init.styles":"ready", "oojs-ui-core.styles":"ready","oojs-ui.styles.indicators":"ready","mediawiki.widgets.styles":"ready","oojs-ui-core.icons":"ready","skins.vector.search.codex.styles":"ready","skins.vector.styles":"ready","skins.vector.icons":"ready","ext.wikimediamessages.styles":"ready","ext.visualEditor.desktopArticleTarget.noscript":"ready","ext.uls.pt":"ready","ext.dismissableSiteNotice.styles":"ready"};RLPAGEMODULES=["site","mediawiki.page.ready","mediawiki.toc","skins.vector.js","ext.centralNotice.geoIP","ext.centralNotice.startUp","ext.gadget.site","ext.urlShortener.toolbar","ext.centralauth.centralautologin","mmv.bootstrap","ext.visualEditor.desktopArticleTarget.init","ext.visualEditor.targetLoader","ext.echo.centralauth","ext.discussionTools.init","ext.eventLogging","ext.wikimediaEvents","ext.uls.interface","ext.checkUser.clientHints","ext.dismissableSiteNotice"];</script> <script>(RLQ=window.RLQ||[]).push(function(){mw.loader.impl(function(){return["user.options@12s5i",function($,jQuery,require,module){mw.user.tokens.set({"patrolToken":"+\\","watchToken":"+\\","csrfToken":"+\\"}); }];});});</script> <link rel="stylesheet" href="/w/load.php?lang=en&modules=ext.discussionTools.init.styles%7Cext.dismissableSiteNotice.styles%7Cext.uls.pt%7Cext.visualEditor.desktopArticleTarget.noscript%7Cext.wikimediamessages.styles%7Cmediawiki.widgets.styles%7Coojs-ui-core.icons%2Cstyles%7Coojs-ui.styles.indicators%7Cskins.vector.icons%2Cstyles%7Cskins.vector.search.codex.styles&only=styles&skin=vector-2022"> <script async="" src="/w/load.php?lang=en&modules=startup&only=scripts&raw=1&skin=vector-2022"></script> <meta name="ResourceLoaderDynamicStyles" content=""> <link rel="stylesheet" href="/w/load.php?lang=en&modules=site.styles&only=styles&skin=vector-2022"> <meta name="generator" content="MediaWiki 1.44.0-wmf.5"> <meta name="referrer" content="origin"> <meta name="referrer" content="origin-when-cross-origin"> <meta name="robots" content="max-image-preview:standard"> <meta name="format-detection" content="telephone=no"> <meta name="viewport" content="width=1120"> <meta property="og:title" content="SLO/Lift Wing - Wikitech"> <meta property="og:type" content="website"> <link rel="preconnect" href="//upload.wikimedia.org"> <link rel="icon" href="/static/favicon/wikitech.ico"> <link rel="search" type="application/opensearchdescription+xml" href="/w/rest.php/v1/search" title="Wikitech (en)"> <link rel="EditURI" type="application/rsd+xml" href="//wikitech.wikimedia.org/w/api.php?action=rsd"> <link rel="canonical" href="https://wikitech.wikimedia.org/wiki/SLO/Lift_Wing"> <link rel="license" href="https://creativecommons.org/licenses/by-sa/4.0/"> <link rel="alternate" type="application/atom+xml" title="Wikitech Atom feed" href="/w/index.php?title=Special:RecentChanges&feed=atom"> <link rel="dns-prefetch" href="//meta.wikimedia.org" /> <link rel="dns-prefetch" href="//login.wikimedia.org"> </head> <body class="ext-discussiontools-replytool-enabled ext-discussiontools-newtopictool-enabled ext-discussiontools-sourcemodetoolbar-enabled skin--responsive skin-vector skin-vector-search-vue mediawiki ltr sitedir-ltr mw-hide-empty-elt ns-0 ns-subject page-SLO_Lift_Wing rootpage-SLO skin-vector-2022 action-view"><a class="mw-jump-link" href="#bodyContent">Jump to content</a> <div class="vector-header-container"> <header class="vector-header mw-header"> <div class="vector-header-start"> <nav class="vector-main-menu-landmark" aria-label="Site"> <div id="vector-main-menu-dropdown" class="vector-dropdown vector-main-menu-dropdown vector-button-flush-left vector-button-flush-right" > <input type="checkbox" id="vector-main-menu-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-main-menu-dropdown" class="vector-dropdown-checkbox " aria-label="Main menu" > <label id="vector-main-menu-dropdown-label" for="vector-main-menu-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-menu mw-ui-icon-wikimedia-menu"></span> <span class="vector-dropdown-label-text">Main menu</span> </label> <div class="vector-dropdown-content"> <div id="vector-main-menu-unpinned-container" class="vector-unpinned-container"> <div id="vector-main-menu" class="vector-main-menu vector-pinnable-element"> <div class="vector-pinnable-header vector-main-menu-pinnable-header vector-pinnable-header-unpinned" data-feature-name="main-menu-pinned" data-pinnable-element-id="vector-main-menu" data-pinned-container-id="vector-main-menu-pinned-container" data-unpinned-container-id="vector-main-menu-unpinned-container" > <div class="vector-pinnable-header-label">Main menu</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-main-menu.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-main-menu.unpin">hide</button> </div> <div id="p-navigation" class="vector-menu mw-portlet mw-portlet-navigation" > <div class="vector-menu-heading"> Navigation </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="n-mainpage-description" class="mw-list-item"><a href="/wiki/Main_Page" title="Visit the main page [z]" accesskey="z"><span>Main page</span></a></li><li id="n-recentchanges" class="mw-list-item"><a href="/wiki/Special:RecentChanges" title="A list of recent changes in the wiki [r]" accesskey="r"><span>Recent changes</span></a></li><li id="n-Server-admin-log:-Prod" class="mw-list-item"><a href="/wiki/Server_Admin_Log"><span>Server admin log: Prod</span></a></li><li id="n-Admin-log:-RelEng" class="mw-list-item"><a href="/wiki/Release_Engineering/SAL"><span>Admin log: RelEng</span></a></li><li id="n-Incident-status" class="mw-list-item"><a href="/wiki/Incident_status"><span>Incident status</span></a></li><li id="n-Deployments" class="mw-list-item"><a href="/wiki/Deployments"><span>Deployments</span></a></li><li id="n-SRE-Team-Help" class="mw-list-item"><a href="/wiki/SRE/SRE_Team_requests"><span>SRE Team Help</span></a></li> </ul> </div> </div> <div id="p-Cloud_VPS_&_Toolforge" class="vector-menu mw-portlet mw-portlet-Cloud_VPS_Toolforge" > <div class="vector-menu-heading"> Cloud VPS & Toolforge </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="n-Cloud-VPS-portal" class="mw-list-item"><a href="/wiki/Portal:Cloud_VPS"><span>Cloud VPS portal</span></a></li><li id="n-Toolforge-portal" class="mw-list-item"><a href="/wiki/Portal:Toolforge"><span>Toolforge portal</span></a></li><li id="n-Request-VPS-project" class="mw-list-item"><a href="https://phabricator.wikimedia.org/project/view/2875/"><span>Request VPS project</span></a></li><li id="n-Admin-log:-Cloud-VPS" class="mw-list-item"><a href="/wiki/Cloud_VPS_Server_Admin_Log"><span>Admin log: Cloud VPS</span></a></li> </ul> </div> </div> </div> </div> </div> </div> </nav> <a href="/wiki/Main_Page" class="mw-logo"> <img class="mw-logo-icon" src="/static/images/icons/wikitech.svg" alt="" aria-hidden="true" height="50" width="50"> <span class="mw-logo-container skin-invert"> <img class="mw-logo-wordmark" alt="Wikitech" src="/static/images/mobile/copyright/wikitech-wordmark.svg" style="width: 8.75em; height: 1.6875em;"> </span> </a> </div> <div class="vector-header-end"> <div id="p-search" role="search" class="vector-search-box-vue vector-search-box-collapses vector-search-box-show-thumbnail vector-search-box-auto-expand-width vector-search-box"> <a href="/wiki/Special:Search" class="cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only search-toggle" title="Search Wikitech [f]" accesskey="f"><span class="vector-icon mw-ui-icon-search mw-ui-icon-wikimedia-search"></span> <span>Search</span> </a> <div class="vector-typeahead-search-container"> <div class="cdx-typeahead-search cdx-typeahead-search--show-thumbnail cdx-typeahead-search--auto-expand-width"> <form action="/w/index.php" id="searchform" class="cdx-search-input cdx-search-input--has-end-button"> <div id="simpleSearch" class="cdx-search-input__input-wrapper" data-search-loc="header-moved"> <div class="cdx-text-input cdx-text-input--has-start-icon"> <input class="cdx-text-input__input" type="search" name="search" placeholder="Search Wikitech" aria-label="Search Wikitech" autocapitalize="sentences" title="Search Wikitech [f]" accesskey="f" id="searchInput" > <span class="cdx-text-input__icon cdx-text-input__start-icon"></span> </div> <input type="hidden" name="title" value="Special:Search"> </div> <button class="cdx-button cdx-search-input__end-button">Search</button> </form> </div> </div> </div> <nav class="vector-user-links vector-user-links-wide" aria-label="Personal tools"> <div class="vector-user-links-main"> <div id="p-vector-user-menu-preferences" class="vector-menu mw-portlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-uls" class="mw-list-item active user-links-collapsible-item"><a data-mw="interface" href="#" class="uls-trigger cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet"><span class="vector-icon mw-ui-icon-wikimedia-language mw-ui-icon-wikimedia-wikimedia-language"></span> <span>English</span></a> </li> </ul> </div> </div> <div id="p-vector-user-menu-userpage" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <nav class="vector-appearance-landmark" aria-label="Appearance"> <div id="vector-appearance-dropdown" class="vector-dropdown " title="Change the appearance of the page's font size, width, and color" > <input type="checkbox" id="vector-appearance-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-appearance-dropdown" class="vector-dropdown-checkbox " aria-label="Appearance" > <label id="vector-appearance-dropdown-label" for="vector-appearance-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-appearance mw-ui-icon-wikimedia-appearance"></span> <span class="vector-dropdown-label-text">Appearance</span> </label> <div class="vector-dropdown-content"> <div id="vector-appearance-unpinned-container" class="vector-unpinned-container"> </div> </div> </div> </nav> <div id="p-vector-user-menu-notifications" class="vector-menu mw-portlet emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> <div id="p-vector-user-menu-overflow" class="vector-menu mw-portlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-sitesupport-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="https://donate.wikimedia.org/?wmf_source=donate&wmf_medium=sidebar&wmf_campaign=wikitech.wikimedia.org&uselang=en" class=""><span>Donate</span></a> </li> <li id="pt-login-2" class="user-links-collapsible-item mw-list-item user-links-collapsible-item"><a data-mw="interface" href="/w/index.php?title=Special:UserLogin&returnto=SLO%2FLift+Wing" title="You are encouraged to log in; however, it is not mandatory [o]" accesskey="o" class=""><span>Log in</span></a> </li> </ul> </div> </div> </div> <div id="vector-user-links-dropdown" class="vector-dropdown vector-user-menu vector-button-flush-right vector-user-menu-logged-out user-links-collapsible-item" title="More options" > <input type="checkbox" id="vector-user-links-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-user-links-dropdown" class="vector-dropdown-checkbox " aria-label="Personal tools" > <label id="vector-user-links-dropdown-label" for="vector-user-links-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-ellipsis mw-ui-icon-wikimedia-ellipsis"></span> <span class="vector-dropdown-label-text">Personal tools</span> </label> <div class="vector-dropdown-content"> <div id="p-personal" class="vector-menu mw-portlet mw-portlet-personal user-links-collapsible-item" title="User menu" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="pt-sitesupport" class="user-links-collapsible-item mw-list-item"><a href="https://donate.wikimedia.org/?wmf_source=donate&wmf_medium=sidebar&wmf_campaign=wikitech.wikimedia.org&uselang=en"><span>Donate</span></a></li><li id="pt-login" class="user-links-collapsible-item mw-list-item"><a href="/w/index.php?title=Special:UserLogin&returnto=SLO%2FLift+Wing" title="You are encouraged to log in; however, it is not mandatory [o]" accesskey="o"><span class="vector-icon mw-ui-icon-logIn mw-ui-icon-wikimedia-logIn"></span> <span>Log in</span></a></li> </ul> </div> </div> </div> </div> </nav> </div> </header> </div> <div class="mw-page-container"> <div class="mw-page-container-inner"> <div class="vector-sitenotice-container"> <div id="siteNotice"><div id="mw-dismissablenotice-anonplace"></div><script>(function(){var node=document.getElementById("mw-dismissablenotice-anonplace");if(node){node.outerHTML="\u003Cdiv class=\"mw-dismissable-notice\"\u003E\u003Cdiv class=\"mw-dismissable-notice-close\"\u003E[\u003Ca tabindex=\"0\" role=\"button\"\u003Edismiss\u003C/a\u003E]\u003C/div\u003E\u003Cdiv class=\"mw-dismissable-notice-body\"\u003E\u003C!-- CentralNotice --\u003E\u003Cdiv id=\"localNotice\" data-nosnippet=\"\"\u003E\u003Cdiv class=\"sitenotice\" lang=\"en\" dir=\"ltr\"\u003E\u003Ctable style=\"width: 75%; background-color: var(--background-color-warning-subtle, #fdf2d5); border: var(--border-subtle, 1px solid #987027); color: var(--color-base, #202122); border-radius: 10px; padding: 5px; margin: 0 auto;\"\u003E\n\u003Ctbody\u003E\u003Ctr\u003E\n\u003Ctd style=\"width:40px; height:40px; text-align:center; vertical-align:middle; padding: 2px;\"\u003E\u003Cspan typeof=\"mw:File\"\u003E\u003Ca href=\"/wiki/File:OOjs_UI_icon_alert-warning.svg\" class=\"mw-file-description\"\u003E\u003Cimg src=\"//upload.wikimedia.org/wikipedia/commons/thumb/3/3b/OOjs_UI_icon_alert-warning.svg/30px-OOjs_UI_icon_alert-warning.svg.png\" decoding=\"async\" width=\"30\" height=\"30\" class=\"mw-file-element\" srcset=\"//upload.wikimedia.org/wikipedia/commons/thumb/3/3b/OOjs_UI_icon_alert-warning.svg/45px-OOjs_UI_icon_alert-warning.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/3/3b/OOjs_UI_icon_alert-warning.svg/60px-OOjs_UI_icon_alert-warning.svg.png 2x\" data-file-width=\"20\" data-file-height=\"20\" /\u003E\u003C/a\u003E\u003C/span\u003E\n\u003C/td\u003E\n\u003Ctd style=\"text-align:center; vertical-align: middle; padding: 4px; max-height: 60px;\"\u003E\u003Cb\u003EWe are migrating Wikitech to \u003Ca href=\"/wiki/Wikitech/SUL-migration\" title=\"Wikitech/SUL-migration\"\u003ESUL\u003C/a\u003E!\u003C/b\u003E\n\u003Cp\u003E\u003Cb\u003EAction may be required for your \u003Ca href=\"/wiki/Wikitech/SUL-migration#What_You_Should_Do\" title=\"Wikitech/SUL-migration\"\u003E account\u003C/a\u003E!\u003C/b\u003E\n\u003C/p\u003E\u003Cp\u003E\u003Cb\u003ETrouble logging in? Please visit \u003Ca href=\"https://phabricator.wikimedia.org/T376267\" class=\"extiw\" title=\"phab:T376267\"\u003ET376267\u003C/a\u003E\u003C/b\u003E\n\u003C/p\u003E\n\u003C/td\u003E\u003C/tr\u003E\u003C/tbody\u003E\u003C/table\u003E\u003C/div\u003E\u003C/div\u003E\u003C/div\u003E\u003C/div\u003E";}}());</script></div> </div> <div class="vector-column-start"> <div class="vector-main-menu-container"> <div id="mw-navigation"> <nav id="mw-panel" class="vector-main-menu-landmark" aria-label="Site"> <div id="vector-main-menu-pinned-container" class="vector-pinned-container"> </div> </nav> </div> </div> <div class="vector-sticky-pinned-container"> <nav id="mw-panel-toc" aria-label="Contents" data-event-name="ui.sidebar-toc" class="mw-table-of-contents-container vector-toc-landmark"> <div id="vector-toc-pinned-container" class="vector-pinned-container"> <div id="vector-toc" class="vector-toc vector-pinnable-element"> <div class="vector-pinnable-header vector-toc-pinnable-header vector-pinnable-header-pinned" data-feature-name="toc-pinned" data-pinnable-element-id="vector-toc" > <h2 class="vector-pinnable-header-label">Contents</h2> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-toc.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-toc.unpin">hide</button> </div> <ul class="vector-toc-contents" id="mw-panel-toc-list"> <li id="toc-mw-content-text" class="vector-toc-list-item vector-toc-level-1"> <a href="#" class="vector-toc-link"> <div class="vector-toc-text">Beginning</div> </a> </li> <li id="toc-Organizational" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Organizational"> <div class="vector-toc-text"> <span class="vector-toc-numb">1</span> <span>Organizational</span> </div> </a> <button aria-controls="toc-Organizational-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> <span class="vector-icon mw-ui-icon-wikimedia-expand"></span> <span>Toggle Organizational subsection</span> </button> <ul id="toc-Organizational-sublist" class="vector-toc-list"> <li id="toc-What_is_the_service?" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#What_is_the_service?"> <div class="vector-toc-text"> <span class="vector-toc-numb">1.1</span> <span>What is the service?</span> </div> </a> <ul id="toc-What_is_the_service?-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Service_structure_and_frameworks" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Service_structure_and_frameworks"> <div class="vector-toc-text"> <span class="vector-toc-numb">1.2</span> <span>Service structure and frameworks</span> </div> </a> <ul id="toc-Service_structure_and_frameworks-sublist" class="vector-toc-list"> <li id="toc-Glossary" class="vector-toc-list-item vector-toc-level-3"> <a class="vector-toc-link" href="#Glossary"> <div class="vector-toc-text"> <span class="vector-toc-numb">1.2.1</span> <span>Glossary</span> </div> </a> <ul id="toc-Glossary-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Who_are_the_responsible_teams?" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Who_are_the_responsible_teams?"> <div class="vector-toc-text"> <span class="vector-toc-numb">1.3</span> <span>Who are the responsible teams?</span> </div> </a> <ul id="toc-Who_are_the_responsible_teams?-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Architectural" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Architectural"> <div class="vector-toc-text"> <span class="vector-toc-numb">2</span> <span>Architectural</span> </div> </a> <button aria-controls="toc-Architectural-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> <span class="vector-icon mw-ui-icon-wikimedia-expand"></span> <span>Toggle Architectural subsection</span> </button> <ul id="toc-Architectural-sublist" class="vector-toc-list"> <li id="toc-What_are_the_service's_dependencies?" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#What_are_the_service's_dependencies?"> <div class="vector-toc-text"> <span class="vector-toc-numb">2.1</span> <span>What are the service's dependencies?</span> </div> </a> <ul id="toc-What_are_the_service's_dependencies?-sublist" class="vector-toc-list"> <li id="toc-Hard_dependencies" class="vector-toc-list-item vector-toc-level-3"> <a class="vector-toc-link" href="#Hard_dependencies"> <div class="vector-toc-text"> <span class="vector-toc-numb">2.1.1</span> <span>Hard dependencies</span> </div> </a> <ul id="toc-Hard_dependencies-sublist" class="vector-toc-list"> <li id="toc-ML-Serve_Kubernetes" class="vector-toc-list-item vector-toc-level-4"> <a class="vector-toc-link" href="#ML-Serve_Kubernetes"> <div class="vector-toc-text"> <span class="vector-toc-numb">2.1.1.1</span> <span>ML-Serve Kubernetes</span> </div> </a> <ul id="toc-ML-Serve_Kubernetes-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Inference_Services" class="vector-toc-list-item vector-toc-level-4"> <a class="vector-toc-link" href="#Inference_Services"> <div class="vector-toc-text"> <span class="vector-toc-numb">2.1.1.2</span> <span>Inference Services</span> </div> </a> <ul id="toc-Inference_Services-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Soft_dependencies" class="vector-toc-list-item vector-toc-level-3"> <a class="vector-toc-link" href="#Soft_dependencies"> <div class="vector-toc-text"> <span class="vector-toc-numb">2.1.2</span> <span>Soft dependencies</span> </div> </a> <ul id="toc-Soft_dependencies-sublist" class="vector-toc-list"> <li id="toc-ML-Serve_Kubernetes_2" class="vector-toc-list-item vector-toc-level-4"> <a class="vector-toc-link" href="#ML-Serve_Kubernetes_2"> <div class="vector-toc-text"> <span class="vector-toc-numb">2.1.2.1</span> <span>ML-Serve Kubernetes</span> </div> </a> <ul id="toc-ML-Serve_Kubernetes_2-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Inference_Services_2" class="vector-toc-list-item vector-toc-level-4"> <a class="vector-toc-link" href="#Inference_Services_2"> <div class="vector-toc-text"> <span class="vector-toc-numb">2.1.2.2</span> <span>Inference Services</span> </div> </a> <ul id="toc-Inference_Services_2-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> </ul> </li> </ul> </li> <li id="toc-Client-facing" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Client-facing"> <div class="vector-toc-text"> <span class="vector-toc-numb">3</span> <span>Client-facing</span> </div> </a> <button aria-controls="toc-Client-facing-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> <span class="vector-icon mw-ui-icon-wikimedia-expand"></span> <span>Toggle Client-facing subsection</span> </button> <ul id="toc-Client-facing-sublist" class="vector-toc-list"> <li id="toc-Who_are_the_service's_clients?" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Who_are_the_service's_clients?"> <div class="vector-toc-text"> <span class="vector-toc-numb">3.1</span> <span>Who are the service's clients?</span> </div> </a> <ul id="toc-Who_are_the_service's_clients?-sublist" class="vector-toc-list"> <li id="toc-External_users" class="vector-toc-list-item vector-toc-level-3"> <a class="vector-toc-link" href="#External_users"> <div class="vector-toc-text"> <span class="vector-toc-numb">3.1.1</span> <span>External users</span> </div> </a> <ul id="toc-External_users-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Internal_users" class="vector-toc-list-item vector-toc-level-3"> <a class="vector-toc-link" href="#Internal_users"> <div class="vector-toc-text"> <span class="vector-toc-numb">3.1.2</span> <span>Internal users</span> </div> </a> <ul id="toc-Internal_users-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> </ul> </li> <li id="toc-Write_the_service_level_indicators" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Write_the_service_level_indicators"> <div class="vector-toc-text"> <span class="vector-toc-numb">4</span> <span>Write the service level indicators</span> </div> </a> <ul id="toc-Write_the_service_level_indicators-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Operational" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Operational"> <div class="vector-toc-text"> <span class="vector-toc-numb">5</span> <span>Operational</span> </div> </a> <button aria-controls="toc-Operational-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> <span class="vector-icon mw-ui-icon-wikimedia-expand"></span> <span>Toggle Operational subsection</span> </button> <ul id="toc-Operational-sublist" class="vector-toc-list"> <li id="toc-How_is_the_service_monitored?" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#How_is_the_service_monitored?"> <div class="vector-toc-text"> <span class="vector-toc-numb">5.1</span> <span>How is the service monitored?</span> </div> </a> <ul id="toc-How_is_the_service_monitored?-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-How_complex_is_the_service_to_troubleshoot?" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#How_complex_is_the_service_to_troubleshoot?"> <div class="vector-toc-text"> <span class="vector-toc-numb">5.2</span> <span>How complex is the service to troubleshoot?</span> </div> </a> <ul id="toc-How_complex_is_the_service_to_troubleshoot?-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-How_is_the_service_deployed?" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#How_is_the_service_deployed?"> <div class="vector-toc-text"> <span class="vector-toc-numb">5.3</span> <span>How is the service deployed?</span> </div> </a> <ul id="toc-How_is_the_service_deployed?-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-Write_the_service_level_objectives" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#Write_the_service_level_objectives"> <div class="vector-toc-text"> <span class="vector-toc-numb">6</span> <span>Write the service level objectives</span> </div> </a> <button aria-controls="toc-Write_the_service_level_objectives-sublist" class="cdx-button cdx-button--weight-quiet cdx-button--icon-only vector-toc-toggle"> <span class="vector-icon mw-ui-icon-wikimedia-expand"></span> <span>Toggle Write the service level objectives subsection</span> </button> <ul id="toc-Write_the_service_level_objectives-sublist" class="vector-toc-list"> <li id="toc-Calculate_the_realistic_targets" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Calculate_the_realistic_targets"> <div class="vector-toc-text"> <span class="vector-toc-numb">6.1</span> <span>Calculate the realistic targets</span> </div> </a> <ul id="toc-Calculate_the_realistic_targets-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Calculate_the_ideal_targets" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Calculate_the_ideal_targets"> <div class="vector-toc-text"> <span class="vector-toc-numb">6.2</span> <span>Calculate the ideal targets</span> </div> </a> <ul id="toc-Calculate_the_ideal_targets-sublist" class="vector-toc-list"> </ul> </li> <li id="toc-Reconcile_the_realistic_vs_ideal_targets" class="vector-toc-list-item vector-toc-level-2"> <a class="vector-toc-link" href="#Reconcile_the_realistic_vs_ideal_targets"> <div class="vector-toc-text"> <span class="vector-toc-numb">6.3</span> <span>Reconcile the realistic vs ideal targets</span> </div> </a> <ul id="toc-Reconcile_the_realistic_vs_ideal_targets-sublist" class="vector-toc-list"> </ul> </li> </ul> </li> <li id="toc-References" class="vector-toc-list-item vector-toc-level-1 vector-toc-list-item-expanded"> <a class="vector-toc-link" href="#References"> <div class="vector-toc-text"> <span class="vector-toc-numb">7</span> <span>References</span> </div> </a> <ul id="toc-References-sublist" class="vector-toc-list"> </ul> </li> </ul> </div> </div> </nav> </div> </div> <div class="mw-content-container"> <main id="content" class="mw-body"> <header class="mw-body-header vector-page-titlebar"> <nav aria-label="Contents" class="vector-toc-landmark"> <div id="vector-page-titlebar-toc" class="vector-dropdown vector-page-titlebar-toc vector-button-flush-left" > <input type="checkbox" id="vector-page-titlebar-toc-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-page-titlebar-toc" class="vector-dropdown-checkbox " aria-label="Toggle the table of contents" > <label id="vector-page-titlebar-toc-label" for="vector-page-titlebar-toc-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet cdx-button--icon-only " aria-hidden="true" ><span class="vector-icon mw-ui-icon-listBullet mw-ui-icon-wikimedia-listBullet"></span> <span class="vector-dropdown-label-text">Toggle the table of contents</span> </label> <div class="vector-dropdown-content"> <div id="vector-page-titlebar-toc-unpinned-container" class="vector-unpinned-container"> </div> </div> </div> </nav> <h1 id="firstHeading" class="firstHeading mw-first-heading"><span class="mw-page-title-main">SLO/Lift Wing</span></h1> </header> <div class="vector-page-toolbar"> <div class="vector-page-toolbar-container"> <div id="left-navigation"> <nav aria-label="Namespaces"> <div id="p-associated-pages" class="vector-menu vector-menu-tabs mw-portlet mw-portlet-associated-pages" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-nstab-main" class="selected vector-tab-noicon mw-list-item"><a href="/wiki/SLO/Lift_Wing" title="View the content page [c]" accesskey="c"><span>Page</span></a></li><li id="ca-talk" class="new vector-tab-noicon mw-list-item"><a href="/w/index.php?title=Talk:SLO/Lift_Wing&action=edit&redlink=1" rel="discussion" class="new" title="Discussion about the content page (page does not exist) [t]" accesskey="t"><span>Discussion</span></a></li> </ul> </div> </div> <div id="vector-variants-dropdown" class="vector-dropdown emptyPortlet" > <input type="checkbox" id="vector-variants-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-variants-dropdown" class="vector-dropdown-checkbox " aria-label="Change language variant" > <label id="vector-variants-dropdown-label" for="vector-variants-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet" aria-hidden="true" ><span class="vector-dropdown-label-text">English</span> </label> <div class="vector-dropdown-content"> <div id="p-variants" class="vector-menu mw-portlet mw-portlet-variants emptyPortlet" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> </ul> </div> </div> </div> </div> </nav> </div> <div id="right-navigation" class="vector-collapsible"> <nav aria-label="Views"> <div id="p-views" class="vector-menu vector-menu-tabs mw-portlet mw-portlet-views" > <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-view" class="selected vector-tab-noicon mw-list-item"><a href="/wiki/SLO/Lift_Wing"><span>Read</span></a></li><li id="ca-viewsource" class="vector-tab-noicon mw-list-item"><a href="/w/index.php?title=SLO/Lift_Wing&action=edit" title="This page is protected.
You can view its source [e]" accesskey="e"><span>View source</span></a></li><li id="ca-history" class="vector-tab-noicon mw-list-item"><a href="/w/index.php?title=SLO/Lift_Wing&action=history" title="Past revisions of this page [h]" accesskey="h"><span>View history</span></a></li> </ul> </div> </div> </nav> <nav class="vector-page-tools-landmark" aria-label="Page tools"> <div id="vector-page-tools-dropdown" class="vector-dropdown vector-page-tools-dropdown" > <input type="checkbox" id="vector-page-tools-dropdown-checkbox" role="button" aria-haspopup="true" data-event-name="ui.dropdown-vector-page-tools-dropdown" class="vector-dropdown-checkbox " aria-label="Tools" > <label id="vector-page-tools-dropdown-label" for="vector-page-tools-dropdown-checkbox" class="vector-dropdown-label cdx-button cdx-button--fake-button cdx-button--fake-button--enabled cdx-button--weight-quiet" aria-hidden="true" ><span class="vector-dropdown-label-text">Tools</span> </label> <div class="vector-dropdown-content"> <div id="vector-page-tools-unpinned-container" class="vector-unpinned-container"> <div id="vector-page-tools" class="vector-page-tools vector-pinnable-element"> <div class="vector-pinnable-header vector-page-tools-pinnable-header vector-pinnable-header-unpinned" data-feature-name="page-tools-pinned" data-pinnable-element-id="vector-page-tools" data-pinned-container-id="vector-page-tools-pinned-container" data-unpinned-container-id="vector-page-tools-unpinned-container" > <div class="vector-pinnable-header-label">Tools</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-page-tools.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-page-tools.unpin">hide</button> </div> <div id="p-cactions" class="vector-menu mw-portlet mw-portlet-cactions emptyPortlet vector-has-collapsible-items" title="More options" > <div class="vector-menu-heading"> Actions </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="ca-more-view" class="selected vector-more-collapsible-item mw-list-item"><a href="/wiki/SLO/Lift_Wing"><span>Read</span></a></li><li id="ca-more-viewsource" class="vector-more-collapsible-item mw-list-item"><a href="/w/index.php?title=SLO/Lift_Wing&action=edit"><span>View source</span></a></li><li id="ca-more-history" class="vector-more-collapsible-item mw-list-item"><a href="/w/index.php?title=SLO/Lift_Wing&action=history"><span>View history</span></a></li> </ul> </div> </div> <div id="p-tb" class="vector-menu mw-portlet mw-portlet-tb" > <div class="vector-menu-heading"> General </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="t-whatlinkshere" class="mw-list-item"><a href="/wiki/Special:WhatLinksHere/SLO/Lift_Wing" title="A list of all wiki pages that link here [j]" accesskey="j"><span>What links here</span></a></li><li id="t-recentchangeslinked" class="mw-list-item"><a href="/wiki/Special:RecentChangesLinked/SLO/Lift_Wing" rel="nofollow" title="Recent changes in pages linked from this page [k]" accesskey="k"><span>Related changes</span></a></li><li id="t-specialpages" class="mw-list-item"><a href="/wiki/Special:SpecialPages" title="A list of all special pages [q]" accesskey="q"><span>Special pages</span></a></li><li id="t-permalink" class="mw-list-item"><a href="/w/index.php?title=SLO/Lift_Wing&oldid=2108688" title="Permanent link to this revision of this page"><span>Permanent link</span></a></li><li id="t-info" class="mw-list-item"><a href="/w/index.php?title=SLO/Lift_Wing&action=info" title="More information about this page"><span>Page information</span></a></li><li id="t-cite" class="mw-list-item"><a href="/w/index.php?title=Special:CiteThisPage&page=SLO%2FLift_Wing&id=2108688&wpFormIdentifier=titleform" title="Information on how to cite this page"><span>Cite this page</span></a></li><li id="t-urlshortener" class="mw-list-item"><a href="/w/index.php?title=Special:UrlShortener&url=https%3A%2F%2Fwikitech.wikimedia.org%2Fwiki%2FSLO%2FLift_Wing"><span>Get shortened URL</span></a></li><li id="t-urlshortener-qrcode" class="mw-list-item"><a href="/w/index.php?title=Special:QrCode&url=https%3A%2F%2Fwikitech.wikimedia.org%2Fwiki%2FSLO%2FLift_Wing"><span>Download QR code</span></a></li> </ul> </div> </div> <div id="p-coll-print_export" class="vector-menu mw-portlet mw-portlet-coll-print_export" > <div class="vector-menu-heading"> Print/export </div> <div class="vector-menu-content"> <ul class="vector-menu-content-list"> <li id="coll-create_a_book" class="mw-list-item"><a href="/w/index.php?title=Special:Book&bookcmd=book_creator&referer=SLO%2FLift+Wing"><span>Create a book</span></a></li><li id="coll-download-as-rl" class="mw-list-item"><a href="/w/index.php?title=Special:DownloadAsPdf&page=SLO%2FLift_Wing&action=show-download-screen"><span>Download as PDF</span></a></li><li id="t-print" class="mw-list-item"><a href="/w/index.php?title=SLO/Lift_Wing&printable=yes" title="Printable version of this page [p]" accesskey="p"><span>Printable version</span></a></li> </ul> </div> </div> </div> </div> </div> </div> </nav> </div> </div> </div> <div class="vector-column-end"> <div class="vector-sticky-pinned-container"> <nav class="vector-page-tools-landmark" aria-label="Page tools"> <div id="vector-page-tools-pinned-container" class="vector-pinned-container"> </div> </nav> <nav class="vector-appearance-landmark" aria-label="Appearance"> <div id="vector-appearance-pinned-container" class="vector-pinned-container"> <div id="vector-appearance" class="vector-appearance vector-pinnable-element"> <div class="vector-pinnable-header vector-appearance-pinnable-header vector-pinnable-header-pinned" data-feature-name="appearance-pinned" data-pinnable-element-id="vector-appearance" data-pinned-container-id="vector-appearance-pinned-container" data-unpinned-container-id="vector-appearance-unpinned-container" > <div class="vector-pinnable-header-label">Appearance</div> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-pin-button" data-event-name="pinnable-header.vector-appearance.pin">move to sidebar</button> <button class="vector-pinnable-header-toggle-button vector-pinnable-header-unpin-button" data-event-name="pinnable-header.vector-appearance.unpin">hide</button> </div> </div> </div> </nav> </div> </div> <div id="bodyContent" class="vector-body" aria-labelledby="firstHeading" data-mw-ve-target-container> <div class="vector-body-before-content"> <div class="mw-indicators"> </div> <div id="siteSub" class="noprint">From Wikitech</div> </div> <div id="contentSub"><div id="mw-content-subtitle"><div class="subpages">< <bdi dir="ltr"><a href="/wiki/SLO" title="SLO">SLO</a></bdi></div></div></div> <div id="mw-content-text" class="mw-body-content"><div class="mw-content-ltr mw-parser-output" lang="en" dir="ltr"><style data-mw-deduplicate="TemplateStyles:r2211903">.mw-parser-output .note{background-position:left 7px top 50%;padding:0.5em 0.5em 0.5em 40px;margin:0.5em 0;overflow:hidden;background-color:#f8f9fa;color:#333;background-repeat:no-repeat;border:1px solid #ddd}.mw-parser-output .note-inline{display:inline-block;vertical-align:middle}.mw-parser-output .note-info{background-color:#eaf3ff;color:#333;background-image:url("https://upload.wikimedia.org/wikipedia/commons/e/ec/OOjs_UI_icon_information-progressive.svg");background-size:25px;border-color:#a3caff;padding-left:40px;min-height:25px}.mw-parser-output .note-reminder{background-color:#fff9ea;color:#333;background-image:url("https://upload.wikimedia.org/wikipedia/commons/a/a8/OOjs_UI_icon_lightbulb-yellow.svg");background-size:25px;border-color:#fc3;min-height:25px}.mw-parser-output .note-warn{background-color:#fff9ea;color:#333;background-image:url("https://upload.wikimedia.org/wikipedia/commons/3/3b/OOjs_UI_icon_alert-warning.svg");background-size:25px;border-color:#fc3;min-height:25px}.mw-parser-output .note-error{background-color:#fee7e6;color:#333;background-image:url("https://upload.wikimedia.org/wikipedia/commons/b/bf/OOjs_UI_icon_notice-destructive.svg");background-size:25px;border-color:#c33;min-height:25px}@media screen{html.skin-theme-clientpref-night .mw-parser-output .note{background-color:transparent;color:inherit}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .note{background-color:transparent;color:inherit}}</style><div role="note" class="note note-info"><b>This page is currently a draft.</b><br/> Material may not yet be complete, information may presently be omitted, and certain parts of the content may be subject to radical, rapid alteration. More information pertaining to this may be available on the <a href="/w/index.php?title=Talk:SLO/Lift_Wing&action=edit&redlink=1" class="new" title="Talk:SLO/Lift Wing (page does not exist)">talk page</a>.</div> <meta property="mw:PageProp/toc"/> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="Organizational" data-mw-thread-id="h-Organizational"><span data-mw-comment-start="" id="h-Organizational"></span>Organizational<span data-mw-comment-end="h-Organizational"></span></h2></div> <div class="mw-heading mw-heading3"><h3 id="What_is_the_service?" data-mw-thread-id="h-What_is_the_service?-Organizational"><span id="What_is_the_service.3F"></span><span data-mw-comment-start="" id="h-What_is_the_service?-Organizational"></span>What is the service?<span data-mw-comment-end="h-What_is_the_service?-Organizational"></span></h3></div> <p><a href="/wiki/Machine_Learning/LiftWing" title="Machine Learning/LiftWing">Machine Learning/LiftWing</a> is a set of services run by the <a href="/wiki/Machine_Learning" title="Machine Learning">Machine Learning</a> team. While these services share common libraries/frameworks and all are topically related (Machine Learning and Interference for WMF-related data), they are somewhat independent: LW is not a cohesive API of complementing REST endpoints, but more like a project that covers topically-similar services. </p><p>The base infrastructure is a <a href="/wiki/Kubernetes/Clusters#ml-serve" title="Kubernetes/Clusters">Kubernetes</a> cluster that is spread across the <code>eqiad</code> and <code>codfw</code> data centers. On this cluster runs a collection of ML services and some supporting infrastructure. The two locations are independent and mirrored, with the intent that even with the loss of an entire data center, the services running in the other data center are sufficient to handle all expected traffic. There also is a smaller staging cluster in codfw. Note that this SLO does not apply to the staging cluster nor the services running there. </p><p>The services running on the cluster are currently mostly inherited from <a href="/wiki/ORES" title="ORES">ORES</a> and they are intended to provide the same (or better) functionality as that project does at the moment, while being more maintainable than the custom/bespoke ORES setup. </p><p>In addition, there are ML services on the cluster that in a sense are <i>post-ORES,</i> in that they have no ORES equivalent and were created with LW in mind, taking advantage of the added features of this system. Over time, the fraction of newer services is expected to increase, while eventually, the old ORES services will be turned off. </p><p>Finally, there are auxiliary services, that either provide functionality for other services on the cluster, or are providing legacy functionality for former clients of ORES. </p><p>For the purposes of the SLIs and SLOs discussed here, we will consider the services to be in classes which have similar SLOs, and the Kubernetes infrastructure they run on, separately. More on this below. </p> <div class="mw-heading mw-heading3"><h3 id="Service_structure_and_frameworks" data-mw-thread-id="h-Service_structure_and_frameworks-Organizational"><span data-mw-comment-start="" id="h-Service_structure_and_frameworks-Organizational"></span>Service structure and frameworks<span data-mw-comment-end="h-Service_structure_and_frameworks-Organizational"></span></h3></div> <p>The Inference Services (ISes) are usually stateless microservices that use the MediaWiki API (MWAPI) as a backend. Each of these services has its own endpoint, reachable through Discovery and the API Gateway using distinct endpoints. Here are two examples of such endpoints: </p> <ul><li><code> <a rel="nofollow" class="external free" href="https://inference.discovery.wmnet:30443/v1/models/wikidatawiki-itemquality:predict">https://inference.discovery.wmnet:30443/v1/models/wikidatawiki-itemquality:predict</a> </code></li> <li><code> <a rel="nofollow" class="external free" href="https://inference.discovery.wmnet:30443/v1/models/bloom-560m:predict">https://inference.discovery.wmnet:30443/v1/models/bloom-560m:predict</a> </code></li></ul> <p>On the API gateway, the URL structure is slightly different for technical reasons. More information can be found in the <a href="/wiki/Machine_Learning/LiftWing/Usage" title="Machine Learning/LiftWing/Usage">LiftWing usage documentation</a>. </p><p>Most of these services use a framework to simplify request handling and integration with Kubernetes, typically <a href="/wiki/Machine_Learning/LiftWing/KServe" title="Machine Learning/LiftWing/KServe">Kserve</a>. </p><p>For the purposes of the SLO discussion, we will consider the ORES services and the newer ML services written and designed for LW separately. </p> <div class="mw-heading mw-heading4"><h4 id="Glossary" data-mw-thread-id="h-Glossary-Service_structure_and_frameworks"><span data-mw-comment-start="" id="h-Glossary-Service_structure_and_frameworks"></span>Glossary<span data-mw-comment-end="h-Glossary-Service_structure_and_frameworks"></span></h4></div> <p>All these names can be confusing, so here is a short summary: </p> <ul><li><b>Inference Service:</b> A microservice that uses a machine learning model to make predictions on revisions, articles etc.</li> <li><b>Kserve:</b> A Python framework that helps with wrapping an ML model in a HTTP service, and integration with Kubernetes.</li> <li><b>LiftWing:</b> A set of typically Kserve-based microservices that run on the ML-Serve Kubernetes cluster.</li> <li><b>ORES Legacy:</b> A service running on ML-Serve that facilitates the migration of clients that use ORES to get the same (or very similar) API from LiftWing services running on ML-Serve. This service is only a relay that queries LiftWing services (and a Redis cache).</li> <li><b>ORES:</b> An older, custom-built cluster of inference services that ML-Serve and LiftWing are meant to supplant. Hardware and most of the stack (except the ML models) are completely different from LW.</li></ul> <div class="mw-heading mw-heading3"><h3 id="Who_are_the_responsible_teams?" data-mw-thread-id="h-Who_are_the_responsible_teams?-Organizational"><span id="Who_are_the_responsible_teams.3F"></span><span data-mw-comment-start="" id="h-Who_are_the_responsible_teams?-Organizational"></span>Who are the responsible teams?<span data-mw-comment-end="h-Who_are_the_responsible_teams?-Organizational"></span></h3></div> <p>Lift Wing is primarily run by the Machine Learning team, who both take care of the Kubernetes setup, as well as the ML services running on top of it. The services themselves can be fully maintained by the Machine Learning team (e.g. the services that provide ORES-like functionality), or they are provided by other teams (e.g. the Research team), but the day-to-day maintenance of them is handled by the ML Team. </p><p>Since ML team is the main team responsible for the operation of Lift Wing, they have to sign off on services running on LW. The staging cluster allows for iterating on new services until they meet the requirements needed before they can be run on the production cluster. </p> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="Architectural" data-mw-thread-id="h-Architectural"><span data-mw-comment-start="" id="h-Architectural"></span>Architectural<span data-mw-comment-end="h-Architectural"></span></h2></div> <div class="mw-heading mw-heading3"><h3 id="What_are_the_service's_dependencies?" data-mw-thread-id="h-What_are_the_service's_dependencies?-Architectural"><span id="What_are_the_service.27s_dependencies.3F"></span><span data-mw-comment-start="" id="h-What_are_the_service's_dependencies?-Architectural"></span>What are the service's dependencies?<span data-mw-comment-end="h-What_are_the_service's_dependencies?-Architectural"></span></h3></div> <p>Dependencies of the LW/Inference system fall into two categories: </p> <ul><li>Dependencies the Kubernetes framework has.</li> <li>Dependencies of the individual inference services themselves.</li></ul> <p>The inference services implicitly depend on Kubernetes of course, but the latency, outage budgets etc of the individual services include the portion caused/offered by Kubernetes. Nevertheless, we will also define an SLO for the Kubernetes portion eventually, which likely will become a separate document eventually. </p> <div class="mw-heading mw-heading4"><h4 id="Hard_dependencies" data-mw-thread-id="h-Hard_dependencies-What_are_the_service's_dependencies?"><span data-mw-comment-start="" id="h-Hard_dependencies-What_are_the_service's_dependencies?"></span>Hard dependencies<span data-mw-comment-end="h-Hard_dependencies-What_are_the_service's_dependencies?"></span></h4></div> <div class="mw-heading mw-heading5"><h5 id="ML-Serve_Kubernetes" data-mw-thread-id="h-ML-Serve_Kubernetes-Hard_dependencies"><span data-mw-comment-start="" id="h-ML-Serve_Kubernetes-Hard_dependencies"></span>ML-Serve Kubernetes<span data-mw-comment-end="h-ML-Serve_Kubernetes-Hard_dependencies"></span></h5></div> <p>The ML-Serve Kubernetes cluster runs the Lift Wing Inference Services (typically based on Kserve), plus auxiliary services such as the ORES Legacy endpoint. </p><p>This cluster naturally depends on the base infrastructure of the data centers it is hosted in: power, networking etc. </p><p>The more specific hard dependencies are: </p> <ul><li>PyBal and DNS Discovery to route traffic towards Lift Wing</li> <li>The API Gateway to route external traffic</li> <li>A set of etcd servers run by the ML team for leader election in the Kubernetes layer</li> <li>Swift (Thanos) for the storage of model binaries</li> <li>The WMF Docker Registry to store and fetch the Docker images of services running on the cluster</li></ul> <div class="mw-heading mw-heading5"><h5 id="Inference_Services" data-mw-thread-id="h-Inference_Services-Hard_dependencies"><span data-mw-comment-start="" id="h-Inference_Services-Hard_dependencies"></span>Inference Services<span data-mw-comment-end="h-Inference_Services-Hard_dependencies"></span></h5></div> <p>Most if not all Inference services will depend on the Media Wiki API (MWAPI) and some logging service like Logstash, but may have additional dependencies, like Feature Stores, Score Caches and the like. Since the shape of degradation of a particular Inference Service is highly variable, distinguishing hard and soft dependencies broadly is difficult. For now, all of the services running on the LW Kubernetes cluster have these hard dependencies: </p> <ol><li>Kubernetes itself</li> <li>Swift (Thanos)</li> <li>The WMF Docker registry</li> <li>MW API (to fetch revisions)</li> <li>Some future services may use a Feature Store service</li></ol> <p>Swift and Docker are needed during service (re)starts, and so are less critical for the steady state than the rest. </p> <div class="mw-heading mw-heading4"><h4 id="Soft_dependencies" data-mw-thread-id="h-Soft_dependencies-What_are_the_service's_dependencies?"><span data-mw-comment-start="" id="h-Soft_dependencies-What_are_the_service's_dependencies?"></span>Soft dependencies<span data-mw-comment-end="h-Soft_dependencies-What_are_the_service's_dependencies?"></span></h4></div> <div class="mw-heading mw-heading5"><h5 id="ML-Serve_Kubernetes_2" data-mw-thread-id="h-ML-Serve_Kubernetes_2-Soft_dependencies"><span data-mw-comment-start="" id="h-ML-Serve_Kubernetes_2-Soft_dependencies"></span>ML-Serve Kubernetes<span data-mw-comment-end="h-ML-Serve_Kubernetes_2-Soft_dependencies"></span></h5></div> <p>Lift Wing is monitored via Icinga, Prometheus and Grafana. As such, normal, controlled operation requires these services to be up, but Lift Wing can operate without them for short times. </p> <div class="mw-heading mw-heading5"><h5 id="Inference_Services_2" data-mw-thread-id="h-Inference_Services_2-Soft_dependencies"><span data-mw-comment-start="" id="h-Inference_Services_2-Soft_dependencies"></span>Inference Services<span data-mw-comment-end="h-Inference_Services_2-Soft_dependencies"></span></h5></div> <p>In addition to the monitoring infrastructure as mentioned above: </p> <ul><li>Inference Services based on Kserve use Logstash for logging requests and debug messages.</li> <li>Some services may use Score Caches (e.g. Redis or Cassandra) to improve response times. Depending on the users of the service, such a cache may become a hard dependency.</li></ul> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="Client-facing" data-mw-thread-id="h-Client-facing"><span data-mw-comment-start="" id="h-Client-facing"></span>Client-facing<span data-mw-comment-end="h-Client-facing"></span></h2></div> <div class="mw-heading mw-heading3"><h3 id="Who_are_the_service's_clients?" data-mw-thread-id="h-Who_are_the_service's_clients?-Client-facing"><span id="Who_are_the_service.27s_clients.3F"></span><span data-mw-comment-start="" id="h-Who_are_the_service's_clients?-Client-facing"></span>Who are the service's clients?<span data-mw-comment-end="h-Who_are_the_service's_clients?-Client-facing"></span></h3></div> <p>Clients fall into two classes: </p> <ul><li>External users, via the API GW (this includes applications on Toolforge or Cloud VPSes)</li> <li>Internal users, via the Discovery endpoint or the API GW</li></ul> <div class="mw-heading mw-heading4"><h4 id="External_users" data-mw-thread-id="h-External_users-Who_are_the_service's_clients?"><span data-mw-comment-start="" id="h-External_users-Who_are_the_service's_clients?"></span>External users<span data-mw-comment-end="h-External_users-Who_are_the_service's_clients?"></span></h4></div> <p>These are external users that call Lift Wing endpoints. Examples are researchers that analyze Wiki edits, bots that surface metadata, moderation tools and the like. Some of these were formerly using ORES and will have to migrate to using Lift Wing services (either ISes or ORES Legacy) before September 2023. </p> <div class="mw-heading mw-heading4"><h4 id="Internal_users" data-mw-thread-id="h-Internal_users-Who_are_the_service's_clients?"><span data-mw-comment-start="" id="h-Internal_users-Who_are_the_service's_clients?"></span>Internal users<span data-mw-comment-end="h-Internal_users-Who_are_the_service's_clients?"></span></h4></div> <p>One of the main internal uses of Lift Wing is the population of Kafka topics with LW-generated content (scores) computed from the stream of changes of various wikis. Lift Wing itself does not create the topics for Kafka (in contrast to how ORES achieves the same effect), but only generates the scores and hands them back to a service that then puts them in the correct Kafka streams. </p><p>The ORES extension is an example of an application that uses (or will use) Lift Wing to provide additional data on Wiki pages (e.g. surfacing whether a particular edit was likely damaging). </p><p>Other clients may not be stream-based/stream-triggered, querying a service for a prediction on behalf of a user, or a series of queries that are based on some other criteria. </p> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="Write_the_service_level_indicators" data-mw-thread-id="h-Write_the_service_level_indicators"><span data-mw-comment-start="" id="h-Write_the_service_level_indicators"></span>Write the service level indicators<span data-mw-comment-end="h-Write_the_service_level_indicators"></span></h2></div> <p>All the ISes and auxiliary services have these two main SLIs: </p> <ol><li>Latency of requests</li> <li>Availability</li></ol> <p>These SLIs are complex in that from the user perspective, the latency/availability of the service itself is combined with the latency and availability of Lift Wing. That is, the overall latency experienced by the user is the combination of Lift Wing's processing and routing of the request as well as the latency the service itself. Since services running on LW can be nearly arbitrarily complex, a lot of the latency and error budget (as seen by the user) will be allocated towards the service rather than Lift Wing itself. Still, for the purposes of the SLO, we consider services running on LW as "black boxes", that is we for example do not subtract the backend latency. </p><p>Telemetry on inference services should expose the backend-added latency. In a similar vein, errors caused by the backend services must be exported as separate counts. Ideally, this distinction (a server error inside the Inference Service vs an error returned by a backend) should be exposed to the user as well. While it is impossible to distinguish these perfectly (is an unreachable backend a problem of the IS configuration, or is the back end service down?), a best effort should be made. </p> <ul><li><b>Request Latency SLI, acceptable fraction:</b> The percentage of all requests where the latency is higher than the chosen threshold.</li> <li><b>Service Availability SLI:</b> The percentage of all requests receiving a non-error response (note: client-caused errors like 404, 429 etc. are not counted here).</li></ul> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="Operational" data-mw-thread-id="h-Operational"><span data-mw-comment-start="" id="h-Operational"></span>Operational<span data-mw-comment-end="h-Operational"></span></h2></div> <p>Every service experiences an outage sometimes, so the SLO should reflect its expected time to recovery. If the expected duration of a single outage exceeds the error budget, then the SLO reduces to "we promise not to make any mistakes." Relying on such an SLO is untenable. </p><p>Answer these questions for the service <i>as it is</i>, not as it ought to be, in order to arrive at a realistically supportable SLO. Alternatively, you may be able to make incremental improvements to the service as you progress through the worksheet. Resist the temptation to publish a more ambitious SLO than you can actually support immediately, even if it feels like you <i>should</i> be able to support it. </p> <div class="mw-heading mw-heading3"><h3 id="How_is_the_service_monitored?" data-mw-thread-id="h-How_is_the_service_monitored?-Operational"><span id="How_is_the_service_monitored.3F"></span><span data-mw-comment-start="" id="h-How_is_the_service_monitored?-Operational"></span>How is the service monitored?<span data-mw-comment-end="h-How_is_the_service_monitored?-Operational"></span></h3></div> <p>Lift Wing is monitored via Prometheus+Grafana and Icinga, sharing configuration and infrastructure to do so with the rest of WMF. </p> <div class="mw-heading mw-heading3"><h3 id="How_complex_is_the_service_to_troubleshoot?" data-mw-thread-id="h-How_complex_is_the_service_to_troubleshoot?-Operational"><span id="How_complex_is_the_service_to_troubleshoot.3F"></span><span data-mw-comment-start="" id="h-How_complex_is_the_service_to_troubleshoot?-Operational"></span>How complex is the service to troubleshoot?<span data-mw-comment-end="h-How_complex_is_the_service_to_troubleshoot?-Operational"></span></h3></div> <p>Lift Wing itself uses Kubernetes (and assorted related services) to host services. It is a fairly complex stack of technologies, and especially routing and TLS can be hard to debug. Fortunately, the wider team of SREs at WMF have experience with operating and debugging Kubernetes issues. As a result, if the ML team can't quickly solve a problem themselves, external (to the team) help may be available. </p><p>Issues within the services running on Lift Wing may be debugged and resolved by ML Team, the team who provided the service themselves, or a combination of the two. </p><p>Regarding "who is responsible for the debugging and ongoing maintenance of a service running on Lift Wing", the ML Team strives to only onboard services where a clear mutual understanding exists regarding ownership. While the team does runs services it also owns, it has finite resources and thus can not completely maintain many more such services. Of particular concern is also the re-training of models to avoid drift and deterioration of predictions. </p> <div class="mw-heading mw-heading3"><h3 id="How_is_the_service_deployed?" data-mw-thread-id="h-How_is_the_service_deployed?-Operational"><span id="How_is_the_service_deployed.3F"></span><span data-mw-comment-start="" id="h-How_is_the_service_deployed?-Operational"></span>How is the service deployed?<span data-mw-comment-end="h-How_is_the_service_deployed?-Operational"></span></h3></div> <p>Lift Wing services are deployed using Helm charts, similar to other Kubernetes installations at WMF, using the same pipeline as the Wikikube clusters. </p> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="Write_the_service_level_objectives" data-mw-thread-id="h-Write_the_service_level_objectives"><span data-mw-comment-start="" id="h-Write_the_service_level_objectives"></span>Write the service level objectives<span data-mw-comment-end="h-Write_the_service_level_objectives"></span></h2></div> <p>The four SLO reporting quarters are: </p> <ul><li>December 1 - February 28 (or 29)</li> <li>March 1 - May 31</li> <li>June 1 - August 31</li> <li>September 1 - November 30</li></ul> <div class="mw-heading mw-heading3"><h3 id="Calculate_the_realistic_targets" data-mw-thread-id="h-Calculate_the_realistic_targets-Write_the_service_level_objectives"><span data-mw-comment-start="" id="h-Calculate_the_realistic_targets-Write_the_service_level_objectives"></span>Calculate the realistic targets<span data-mw-comment-end="h-Calculate_the_realistic_targets-Write_the_service_level_objectives"></span></h3></div> <p><b>Note:</b> We don't have any historical data here. Or at least not at a volume that is useful to make predictions about what is feasible. </p><p>As mentioned before, we have four broad classes: </p> <ol><li>Inference Services derived from ORES model (articlequality, damaging, drafttopic etc.) <ul><li><b>Service Availability SLI:</b> 98% of all requests are answered with non-errors.</li> <li><b>Request Latency SLI, acceptable fraction:</b> <=5 seconds, 98%</li></ul></li> <li>Stable and reliable Inference Services (revertrisk language agnostic, etc) <ul><li><b>Service Availability SLI:</b> 98% of all requests are answered with non-errors.</li> <li><b>Request Latency SLI, acceptable fraction:</b> <=500 milliseconds, 98%</li></ul></li> <li>Experimental Inference Services (revertrisk multi lingual, outlink, etc) <ul><li><b>Service Availability SLI:</b> 95% of all requests are answered with non-errors.</li> <li><b>Request Latency SLI, acceptable fraction:</b> <=500 (or 5000 depending on the model) milliseconds, 95%</li></ul></li> <li>Auxiliary services (ORES Legacy etc.) <ul><li><b>Service Availability SLI:</b> 95% of all requests are answered with non-errors.</li> <li><b>Request Latency SLI, acceptable fraction:</b> <=500 milliseconds, 95%</li></ul></li> <li>Lift Wing Kubernetes API <ul><li><b>Service Availability SLI:</b> 99% of all requests are answered with non-errors.</li> <li><b>Request Latency SLI, acceptable fraction:</b> <=500 milliseconds, 99%</li></ul></li></ol> <p>Note that the bare ORES services have a much higher latency SLO than what ORES currently delivers. This is due to ORES having a caching layer that hides the actual latency of the models running there. For the services running on Lift Wing, the OREs Legacy service provides this caching functionality (using the same Redis infrastructure that ORES uses for this). </p> <div class="mw-heading mw-heading3"><h3 id="Calculate_the_ideal_targets" data-mw-thread-id="h-Calculate_the_ideal_targets-Write_the_service_level_objectives"><span data-mw-comment-start="" id="h-Calculate_the_ideal_targets-Write_the_service_level_objectives"></span>Calculate the ideal targets<span data-mw-comment-end="h-Calculate_the_ideal_targets-Write_the_service_level_objectives"></span></h3></div> <p><b>TODO:</b> For a busy service and a smooth-running LW, the targets above are too defensive. A decent service running on LW shouldn't really have 5% of failures. With very low traffic numbers, there is a granularity problem of course, even at the one-quarter scale. </p> <div class="mw-heading mw-heading3"><h3 id="Reconcile_the_realistic_vs_ideal_targets" data-mw-thread-id="h-Reconcile_the_realistic_vs_ideal_targets-Write_the_service_level_objectives"><span data-mw-comment-start="" id="h-Reconcile_the_realistic_vs_ideal_targets-Write_the_service_level_objectives"></span>Reconcile the realistic vs ideal targets<span data-mw-comment-end="h-Reconcile_the_realistic_vs_ideal_targets-Write_the_service_level_objectives"></span></h3></div> <p>Now that you've worked out what SLO targets you'd like to offer, and what targets you can actually support, compare them. If you're lucky, the realistic values are the same or better than the ideal ones: that's great news. Publish the ideal values as your SLO, or choose a value in between. (Resist the urge to set a stricter SLO just because you can; it will constrain your options later.) </p><p>If you're less lucky, there's some distance between the SLO you'd like to offer and the one you can support. This is an uncomfortable situation, but it's also a natural one for a network of dependent services establishing their SLOs for the first time. Here, you'll need to make some decisions to close the gap. (Resist, even more strongly, the urge to set a stricter SLO just because you wish you could.) </p><p>One approach is to make the same decisions you would make if you already had an SLO and you were violating it. (In some sense, that's effectively the case: your service isn't reliable enough to meet its clients' expectations, you just didn't know it yet.) That means it's time to refocus engineering work onto the kind of projects that will bolster the affected SLIs. Publish an SLO that reflects the promises you can keep right now, but continue to tighten it over time as you complete reliability work. </p><p>The other approach is to do engineering work to relax clients' expectations. If they're relying on you for a level of service that you can't provide, there may be a way to make that level of service unnecessary. If your tail latency is high but you have spare capacity, they can use request hedging to avoid the tail. If they can't tolerate your rate of outages in a hard dependency, maybe they can rely on you as a soft dependency by adding a degraded mode. </p> <style data-mw-deduplicate="TemplateStyles:r2232765">.mw-parser-output .ambox{border:1px solid #a2a9b1;border-left:10px solid #36c;background-color:#fbfbfb;box-sizing:border-box}.mw-parser-output .ambox+link+.ambox,.mw-parser-output .ambox+link+style+.ambox,.mw-parser-output .ambox+link+link+.ambox,.mw-parser-output .ambox+.mw-empty-elt+link+.ambox,.mw-parser-output .ambox+.mw-empty-elt+link+style+.ambox,.mw-parser-output .ambox+.mw-empty-elt+link+link+.ambox{margin-top:-1px}html body.mediawiki .mw-parser-output .ambox.mbox-small-left{margin:4px 1em 4px 0;overflow:hidden;width:238px;border-collapse:collapse;font-size:88%;line-height:1.25em}.mw-parser-output .ambox-speedy{border-left:10px solid #b32424;background-color:#fee7e6}.mw-parser-output .ambox-delete{border-left:10px solid #b32424}.mw-parser-output .ambox-content{border-left:10px solid #f28500}.mw-parser-output .ambox-style{border-left:10px solid #fc3}.mw-parser-output .ambox-move{border-left:10px solid #9932cc}.mw-parser-output .ambox-protection{border-left:10px solid #a2a9b1}.mw-parser-output .ambox .mbox-text{border:none;padding:0.25em 0.5em;width:100%}.mw-parser-output .ambox .mbox-image{border:none;padding:2px 0 2px 0.5em;text-align:center}.mw-parser-output .ambox .mbox-imageright{border:none;padding:2px 0.5em 2px 0;text-align:center}.mw-parser-output .ambox .mbox-empty-cell{border:none;padding:0;width:1px}.mw-parser-output .ambox .mbox-image-div{width:52px}html.client-js body.skin-minerva .mw-parser-output .mbox-text-span{margin-left:23px!important}@media(min-width:720px){.mw-parser-output .ambox{margin:0 10%}}</style><table class="box-Notice plainlinks metadata ambox ambox-notice" role="presentation"><tbody><tr><td class="mbox-image"><div class="mbox-image-div"><span typeof="mw:File"><span><img alt="" src="//upload.wikimedia.org/wikipedia/commons/thumb/1/1d/Information_icon4.svg/40px-Information_icon4.svg.png" decoding="async" width="40" height="40" class="mw-file-element" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/1/1d/Information_icon4.svg/60px-Information_icon4.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/1/1d/Information_icon4.svg/80px-Information_icon4.svg.png 2x" data-file-width="620" data-file-height="620"/></span></span></div></td><td class="mbox-text"><div class="mbox-text-span">Request hedging: For an idempotent request such as a read, the client sends duplicate requests to multiple replicas of your service, then uses whichever response comes back first. By waiting to send the extra requests until, say, the 90th percentile latency has already elapsed, the client can reliably avoid the rest of the latency tail(!) while keeping the extra request load to only an additional 10% of traffic. Depending on your service's latency distribution, the result can be a sharp improvement in tail latency at relatively little cost.</div></td></tr></tbody></table> <p>Despite the use of "you" and "they" in the last couple of paragraphs, this is collaborative work toward a shared goal. The decision of which approach to take doesn't need to be adversarial or defensive. </p><p>You should also expect this work to comprise the majority of the effort involved in the SLO process. Where the earlier steps were characterized by documentation and gathering, here your work is directed at improving the practical reality of your software in production. </p><p>Regardless of the approach you take to reconciliation, you should publish a currently-realistic SLO, and begin measuring your performance against it, sooner rather than later. You can publish your aspirational targets too (as long as it's clearly marked that you don't currently guarantee to meet them) so that other teams can consider them in their longer-term planning. In the meantime, you'll be able to prioritize work to keep from backsliding on the progress you've already made. </p> <table class="wikitable"> <tbody><tr> <td>📝 Clearly document any decisions you made during reconciliation. Finally, clearly list the agreed SLOs -- that is, SLIs and associated targets. There should be as many SLOs as the number of SLIs multiplied by the number of request classes -- or, if some request classes are ineligible for any guarantee, say which. </td></tr></tbody></table> <div class="mw-heading mw-heading2 ext-discussiontools-init-section"><h2 id="References" data-mw-thread-id="h-References"><span data-mw-comment-start="" id="h-References"></span>References<span data-mw-comment-end="h-References"></span></h2></div> <ul><li>Jones, Wilkes, and Murphy with Smith, "Service Level Objectives" in <i>Site Reliability Engineering</i>, O'Reilly 2016 <a rel="nofollow" class="external text" href="https://sre.google/sre-book/service-level-objectives">(free online)</a></li> <li>Alex Hidalgo, <i>Implementing Service Level Objectives</i>, O'Reilly 2020 <a rel="nofollow" class="external text" href="https://drive.google.com/file/d/1QqRjKwBQj7NeKOhiHWm3kepVWmfmeZHJ/view?usp=sharing">(WMF Tech copy)</a></li></ul>    </div><noscript><img src="https://login.wikimedia.org/wiki/Special:CentralAutoLogin/start?type=1x1&useformat=desktop" alt="" width="1" height="1" style="border: none; position: absolute;"></noscript> <div class="printfooter" data-nosnippet="">Retrieved from "<a dir="ltr" href="https://wikitech.wikimedia.org/w/index.php?title=SLO/Lift_Wing&oldid=2108688">https://wikitech.wikimedia.org/w/index.php?title=SLO/Lift_Wing&oldid=2108688</a>"</div></div> <div id="catlinks" class="catlinks catlinks-allhidden" data-mw="interface"></div> </div> </main> </div> <div class="mw-footer-container"> <footer id="footer" class="mw-footer" > <ul id="footer-info"> <li id="footer-info-lastmod"> This page was last edited on 4 September 2023, at 14:51.</li> <li id="footer-info-copyright">Text is available under the <a rel="nofollow" class="external text" href="https://creativecommons.org/licenses/by-sa/4.0/deed.en">Creative Commons Attribution-ShareAlike License</a>; additional terms may apply. See <a class="external text" href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Terms_of_Use">Terms of Use</a> for details.</li> </ul> <ul id="footer-places"> <li id="footer-places-privacy"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Privacy_policy">Privacy policy</a></li> <li id="footer-places-about"><a href="/wiki/Main_Page">About Wikitech</a></li> <li id="footer-places-disclaimers"><a href="https://foundation.wikimedia.org/wiki/General_disclaimer">Disclaimers</a></li> <li id="footer-places-wm-codeofconduct"><a href="https://www.mediawiki.org/wiki/Special:MyLanguage/Code_of_Conduct">Code of Conduct</a></li> <li id="footer-places-developers"><a href="https://developer.wikimedia.org">Developers</a></li> <li id="footer-places-statslink"><a href="https://stats.wikimedia.org/#/wikitech.wikimedia.org">Statistics</a></li> <li id="footer-places-cookiestatement"><a href="https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Cookie_statement">Cookie statement</a></li> <li id="footer-places-mobileview"><a href="//wikitech.wikimedia.org/w/index.php?title=SLO/Lift_Wing&mobileaction=toggle_view_mobile" class="noprint stopMobileRedirectToggle">Mobile view</a></li> </ul> <ul id="footer-icons" class="noprint"> <li id="footer-copyrightico"><a href="https://wikimediafoundation.org/" class="cdx-button cdx-button--fake-button cdx-button--size-large cdx-button--fake-button--enabled"><img src="/static/images/footer/wikimedia-button.svg" width="84" height="29" alt="Wikimedia Foundation" loading="lazy"></a></li> <li id="footer-poweredbyico"><a href="https://www.mediawiki.org/" class="cdx-button cdx-button--fake-button cdx-button--size-large cdx-button--fake-button--enabled"><img src="/w/resources/assets/poweredby_mediawiki.svg" alt="Powered by MediaWiki" width="88" height="31" loading="lazy"></a></li> </ul> </footer> </div> </div> </div> <div class="vector-settings" id="p-dock-bottom"> <ul></ul> </div><script>(RLQ=window.RLQ||[]).push(function(){mw.config.set({"wgHostname":"mw-web.codfw.main-6dfcdd5ff5-5mt7c","wgBackendResponseTime":137,"wgDiscussionToolsPageThreads":[{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Organizational","replies":[{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-What_is_the_service?-Organizational","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Service_structure_and_frameworks-Organizational","replies":[{"headingLevel":4,"name":"h-","type":"heading","level":0,"id":"h-Glossary-Service_structure_and_frameworks","replies":[]}]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Who_are_the_responsible_teams?-Organizational","replies":[]}]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Architectural","replies":[{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-What_are_the_service's_dependencies?-Architectural","replies":[{"headingLevel":4,"name":"h-","type":"heading","level":0,"id":"h-Hard_dependencies-What_are_the_service's_dependencies?","replies":[{"headingLevel":5,"name":"h-","type":"heading","level":0,"id":"h-ML-Serve_Kubernetes-Hard_dependencies","replies":[]},{"headingLevel":5,"name":"h-","type":"heading","level":0,"id":"h-Inference_Services-Hard_dependencies","replies":[]}]},{"headingLevel":4,"name":"h-","type":"heading","level":0,"id":"h-Soft_dependencies-What_are_the_service's_dependencies?","replies":[{"headingLevel":5,"name":"h-","type":"heading","level":0,"id":"h-ML-Serve_Kubernetes_2-Soft_dependencies","replies":[]},{"headingLevel":5,"name":"h-","type":"heading","level":0,"id":"h-Inference_Services_2-Soft_dependencies","replies":[]}]}]}]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Client-facing","replies":[{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Who_are_the_service's_clients?-Client-facing","replies":[{"headingLevel":4,"name":"h-","type":"heading","level":0,"id":"h-External_users-Who_are_the_service's_clients?","replies":[]},{"headingLevel":4,"name":"h-","type":"heading","level":0,"id":"h-Internal_users-Who_are_the_service's_clients?","replies":[]}]}]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Write_the_service_level_indicators","replies":[]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Operational","replies":[{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-How_is_the_service_monitored?-Operational","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-How_complex_is_the_service_to_troubleshoot?-Operational","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-How_is_the_service_deployed?-Operational","replies":[]}]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-Write_the_service_level_objectives","replies":[{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Calculate_the_realistic_targets-Write_the_service_level_objectives","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Calculate_the_ideal_targets-Write_the_service_level_objectives","replies":[]},{"headingLevel":3,"name":"h-","type":"heading","level":0,"id":"h-Reconcile_the_realistic_vs_ideal_targets-Write_the_service_level_objectives","replies":[]}]},{"headingLevel":2,"name":"h-","type":"heading","level":0,"id":"h-References","replies":[]}],"wgPageParseReport":{"discussiontools":{"limitreport-timeusage":"0.012"},"limitreport":{"cputime":"0.085","walltime":"0.100","ppvisitednodes":{"value":170,"limit":1000000},"postexpandincludesize":{"value":3733,"limit":2097152},"templateargumentsize":{"value":1118,"limit":2097152},"expansiondepth":{"value":7,"limit":100},"expensivefunctioncount":{"value":0,"limit":500},"unstrip-depth":{"value":0,"limit":20},"unstrip-size":{"value":3247,"limit":5000000},"timingprofile":["100.00% 62.733 1 -total"," 73.69% 46.231 1 Template:Notice"," 69.08% 43.336 1 Template:Mbox"," 25.89% 16.239 1 Template:Draft"," 22.96% 14.402 1 Template:Note"]},"scribunto":{"limitreport-timeusage":{"value":"0.026","limit":"10.000"},"limitreport-memusage":{"value":635893,"limit":52428800}},"cachereport":{"origin":"mw-api-int.codfw.main-6fdd9f9b88-zmrj6","timestamp":"20241129124214","ttl":2592000,"transientcontent":false}}});});</script> </body> </html>