Typo blamed for Microsoft Azure DevOps outage in Brazil • The Register

<!doctype html> <html lang="en"> <head> <meta content="text/html; charset=utf-8" http-equiv="Content-Type"> <title>Typo blamed for Microsoft Azure DevOps outage in Brazil • The Register</title> <meta name="robots" content="max-snippet:-1, max-image-preview:standard, max-video-preview:0"> <meta name="viewport" content="initial-scale=1.0, width=device-width"/> <meta property="og:image" content="https://regmedia.co.uk/2023/06/02/shutterstock_rio.jpg"/> <meta property="og:type" content="article" /> <meta property="og:url" content="https://www.theregister.com/2023/06/03/microsoft_azure_outage_brazil/" /> <meta property="og:title" content="Typo blamed for Microsoft Azure DevOps outage in Brazil" /> <meta property="og:description" content="Errant code fix deleted entire servers rather than snapshots of database" /> <meta name="twitter:card" content="summary_large_image"> <meta name="twitter:site" content="@TheRegister"> <script type="application/ld+json"> { "@context":"http://schema.org", "@type":"NewsArticle", "mainEntityOfPage":{"@type":"WebPage","@id":"https://www.theregister.com/2023/06/03/microsoft_azure_outage_brazil/"}, "headline":"This typo sparked a Microsoft Azure outage", "datePublished":"2023-06-03T19:58:14Z", "dateModified":"2023-06-02T23:58:15Z", "image":{"@type":"ImageObject","url":"https://regmedia.co.uk/2023/06/02/shutterstock_rio.jpg","width":"1000","height":"667"}, "author":{"@type":"Person","name":"Thomas Claburn"}, "publisher":{"@type":"Organization","name":"The Register","url":"https://www.theregister.com/","logo":{"@type":"ImageObject","url":"https://www.theregister.com/design_picker/1fea2ae01c5036112a295123c3cc9c56eb28836a/graphics/std/red_logo_sans_strapline.png","width":330,"height":55}} } </script> <script> var RegZoot = { }; var RegCC = [ ]; var RegPageType = 'Story'; var RegTruePageType = 'www story'; </script> <link rel="canonical" href="https://www.theregister.com/2023/06/03/microsoft_azure_outage_brazil/"> <link rel="amphtml" href="https://www.theregister.com/AMP/2023/06/03/microsoft_azure_outage_brazil/"> <script src="/Design/javascript/html5shiv.min.js"></script> <script> // IE8 only polyfilly for eventListener // source: https://developer.mozilla.org/en-US/docs/Web/API/EventTarget/addEventListener#Compatibility !function(){if(Event.prototype.preventDefault||(Event.prototype.preventDefault=function(){this.returnValue=!1}),Event.prototype.stopPropagation||(Event.prototype.stopPropagation=function(){this.cancelBubble=!0}),!Element.prototype.addEventListener){var e=[],t=function(t,n){var o=this,r=function(e){e.target=e.srcElement,e.currentTarget=o,void 0!==n.handleEvent?n.handleEvent(e):n.call(o,e)};if("DOMContentLoaded"==t){var a=function(e){"complete"==document.readyState&&r(e)};if(document.attachEvent("onreadystatechange",a),e.push({object:this,type:t,listener:n,wrapper:a}),"complete"==document.readyState){var p=new Event;p.srcElement=window,a(p)}}else this.attachEvent("on"+t,r),e.push({object:this,type:t,listener:n,wrapper:r})},n=function(t,n){for(var o=0;o<e.length;){var r=e[o];if(r.object==this&&r.type==t&&r.listener==n){"DOMContentLoaded"==t?this.detachEvent("onreadystatechange",r.wrapper):this.detachEvent("on"+t,r.wrapper),e.splice(o,1);break}++o}};Element.prototype.addEventListener=t,Element.prototype.removeEventListener=n,HTMLDocument&&(HTMLDocument.prototype.addEventListener=t,HTMLDocument.prototype.removeEventListener=n),Window&&(Window.prototype.addEventListener=t,Window.prototype.removeEventListener=n)}}(); document.attachEvent("onreadystatechange", function() { if (document.readyState === "complete") { // list of icons we want <= IE8 to replace with their png equivalents var svg_icons_png_equiv = [ // masthead icons (twitter + facebook are also shared for footer): 'reg_logo.svg', 'twitter.svg', 'facebook.svg', 'linkedin.svg', // navigation bar icons: 'vulture.svg', 'vulture_white.svg', 'search.svg', 'search_white.svg', // footer icons: 'sitpub_footer.svg', 'linkedin_white.svg', 'rss.svg', // lectures section icons: 'reglecture_logo.svg', // story template icons: 'reddit.svg', 'linkedin_alt.svg', 'linkedin.svg', 'calendar.svg', 'location.svg', 'rect_comment_bubble_white.svg', 'rect_comment_bubble_black.svg', 'envelope.svg', 'polls_unit_arrow.svg' ]; for (i = 0; i <= svg_icons_png_equiv.length - 1; i++) { var svg_icon = svg_icons_png_equiv[i]; var img_svg_icons = $('img[src$="' + svg_icon + '"]'); img_svg_icons.each(function() { $(this).attr('src', $(this).attr('src').replace('.svg','.png')); }); } var ad_params = { src: 'https://regmedia.co.uk/2018/06/15/gg2b_book.png', href: 'https://forms.theregister.com/gg2b/?td=iaomwtkie78' }; bird_alternative('ad_wp_top', ad_params); } }); </script> <script> var RegArticle={id:228037,pf:0,af:0,bms:0,sec:'software/devops',cat:'update_me',ec:['microsoft'],kw:[["azure",'Azure'],["devops",'Devops'],["microsoft",'Microsoft'],["outage",'Outage']],kwp:[["bill gates",'Bill Gates'],["cloud computing",'Cloud Computing'],["development",'Development']],short_url:'https://reg.cx/48HC',cp:0,noads:[],author:'Thomas Claburn'} </script> <link rel=stylesheet type="text/css" href="/css/e5c206ed408f082870465a2c478e657ff0db3937/scaffolding.css"> <link rel=stylesheet type="text/css" href="/css/e5c206ed408f082870465a2c478e657ff0db3937/design.css"> <style> #nav-software, #nav-software-devops { text-decoration: underline !important; } </style> <link rel='stylesheet' type='text/css' href='/css/e5c206ed408f082870465a2c478e657ff0db3937/story_only.css'> <link rel=stylesheet type="text/css" href="/css/e5c206ed408f082870465a2c478e657ff0db3937/rows_basic.css"> <link rel=alternate type="application/atom+xml" href="/headlines.atom" title="The Register: whole site"> <link rel=alternate type="application/atom+xml" href="/software/devops/headlines.atom" title="The Register: Devops section"> <script> var RegCR = false; </script> <script src="/design_picker/14513432720673f1c1ee02761ba265b674b7bee1/javascript/_.js"></script> <script> RegGPT('reg_software/devops','0df13fad2ea597c71ae99fa84c3f976d','0df13fad2ea597c71ae99fa84c3f976d'); </script> <script async src="https://www.googletagmanager.com/gtag/js"></script> <link rel=search href="https://search.theregister.com/"> <link rel=search type="application/opensearchdescription+xml" title="El Reg Search" href="/Design/page/search.osd"> <link rel="icon" href="/design_picker/13249a2e80709c7ff2e57dd3d49801cd534f2094/graphics/favicons/favicon.ico" sizes="any"> <link rel="icon" href="/design_picker/13249a2e80709c7ff2e57dd3d49801cd534f2094/graphics/favicons/favicon.svg" type="image/svg+xml"> <link rel="apple-touch-icon" href="/design_picker/13249a2e80709c7ff2e57dd3d49801cd534f2094/graphics/favicons/apple-touch-icon.png"> <link rel="manifest" href="/design_picker/13249a2e80709c7ff2e57dd3d49801cd534f2094/graphics/favicons/site.webmanifest"> <meta name="msapplication-TileColor" content="#ff0000"> <meta name="msapplication-config" content="/design_picker/13249a2e80709c7ff2e57dd3d49801cd534f2094/graphics/favicons/browserconfig.xml"> <meta name="theme-color" content="#ff0000"> <script src="/Design/javascript/respond.min.js"></script> </head> <body class="fullwidth" data-pagetype='Story' data-iebrowser='7' data-pagenum="0"> <div id="page"> <div data-oop="1" data-pos="top" data-raptor="kite" aria-hidden="true" class="adun"></div> <div id="masthead"> <div class="los_amigos"> <div class="left_nav"> <a id="mob_user_link" href="https://account.theregister.com/register/" aria-label="Your Account"> <img class="account_icon" width="16" height="16" src="/design_picker/ae01b183a707a7db8cd5f2c947715ed56d335138/graphics/std/user_icon_white_extents_16x16.png" srcset="/design_picker/ae01b183a707a7db8cd5f2c947715ed56d335138/graphics/std/user_icon_white_extents.svg" alt=""> <img class="filled_icon" width="16" height="16" src="/design_picker/ae01b183a707a7db8cd5f2c947715ed56d335138/graphics/std/user_icon_white_filled_extents_16x16.png" srcset="/design_picker/ae01b183a707a7db8cd5f2c947715ed56d335138/graphics/std/user_icon_filled_white_extents.svg" alt=""> <span id="mob_user_text"><span>Sign in / up</span></span> </a> </div> <div class="center_nav"> <a href="https://www.theregister.com/" id="logo"> <img src="/design_picker/fa16d26efb42e6ba1052f1d387470f643c5aa18d/graphics/std/reg_logo_no_strapline.png" srcset="/design_picker/fa16d26efb42e6ba1052f1d387470f643c5aa18d/graphics/std/reg_logo_no_strapline.svg" width="190" height="35" alt="The Register® — Biting the hand that feeds IT"> </a> </div> <div class="right_nav"> <a href="https://search.theregister.com/" class="nav_search topnav_elem" data-name="Search" aria-label="Search"> <img width="16" height="16" src="/design_picker/ae01b183a707a7db8cd5f2c947715ed56d335138/graphics/std/magnifying_glass_white_extents_16x16.png" srcset="/design_picker/ae01b183a707a7db8cd5f2c947715ed56d335138/graphics/std/magnifying_glass_white_extents.svg" alt=""> </a> <div id="site_nav_mobile"> <noscript><div id="site_nav_mobile_hiding_stamp"></div></noscript> <button id="mobile_menu_toggle" aria-label="Open menu" type="button"> <img width="16" height="16" src="/design_picker/ae01b183a707a7db8cd5f2c947715ed56d335138/graphics/icon/burger_menu_white_16x16.png" srcset="/design_picker/ae01b183a707a7db8cd5f2c947715ed56d335138/graphics/icon/burger_menu_white_extents.svg" alt=""> <img width="16" height="16" src="/design_picker/ae01b183a707a7db8cd5f2c947715ed56d335138/graphics/icon/burger_menu_white_close_16x16.png" srcset="/design_picker/ae01b183a707a7db8cd5f2c947715ed56d335138/graphics/icon/burger_menu_white_close_extents.svg" alt=""> </button> </div> </div> </div> <div id="top_panel_wrapper"> <div id="top_panel"> <div class="block_section nav"> <div class="nav_col first_col"> <div class="nav_top_group"> <div class="nav_topics"> <div class="nav_head_bk"> <h2 class="main_head">Topics</h2> </div> <div> <nav> <div class="nav_elem"> <div class="cat_header"> <div id="nav-security"> <a href="#subnav-box-nav-security" data-toggle-for="subnav-box-nav-security" class="topnav_elem mob_only">Security</a> <h2 class="desk_only section_nav-security"> <a href="#subnav-box-nav-security" data-toggle-for="subnav-box-nav-security" class="topnav_elem desk_only">Security</a> </h2> </div> </div><div id="subnav-box-nav-security" class="subnav_box"><a href="https://www.theregister.com/security/" class="subnav_elem" id="nav-security-all"><span class="prefix_all">All </span>Security</a><a href="https://www.theregister.com/security/cyber_crime/" class="subnav_elem" id="nav-security-cyber_crime">Cyber-crime</a><a href="https://www.theregister.com/security/patches/" class="subnav_elem" id="nav-security-patches">Patches</a><a href="https://www.theregister.com/security/research/" class="subnav_elem" id="nav-security-research">Research</a><a href="https://www.theregister.com/security/cso/" class="subnav_elem" id="nav-security-cso">CSO</a> <noscript><a href="#masthead" class="subnav_elem close_box" aria-label="Top navigation">(X)</a></noscript> </div> </div><div class="nav_elem"> <div class="cat_header"> <div id="nav-off_prem"> <a href="#subnav-box-nav-off_prem" data-toggle-for="subnav-box-nav-off_prem" class="topnav_elem mob_only">Off-Prem</a> <h2 class="desk_only section_nav-off_prem"> <a href="#subnav-box-nav-off_prem" data-toggle-for="subnav-box-nav-off_prem" class="topnav_elem desk_only">Off-Prem</a> </h2> </div> </div><div id="subnav-box-nav-off_prem" class="subnav_box"><a href="https://www.theregister.com/off_prem/" class="subnav_elem" id="nav-off_prem-all"><span class="prefix_all">All </span>Off-Prem</a><a href="https://www.theregister.com/off_prem/edge_iot/" class="subnav_elem" id="nav-off_prem-edge_iot">Edge + IoT</a><a href="https://www.theregister.com/off_prem/channel/" class="subnav_elem" id="nav-off_prem-channel">Channel</a><a href="https://www.theregister.com/off_prem/paas_iaas/" class="subnav_elem" id="nav-off_prem-paas_iaas">PaaS + IaaS</a><a href="https://www.theregister.com/off_prem/saas/" class="subnav_elem" id="nav-off_prem-saas">SaaS</a> <noscript><a href="#masthead" class="subnav_elem close_box" aria-label="Top navigation">(X)</a></noscript> </div> </div><div class="nav_elem"> <div class="cat_header"> <div id="nav-on_prem"> <a href="#subnav-box-nav-on_prem" data-toggle-for="subnav-box-nav-on_prem" class="topnav_elem mob_only">On-Prem</a> <h2 class="desk_only section_nav-on_prem"> <a href="#subnav-box-nav-on_prem" data-toggle-for="subnav-box-nav-on_prem" class="topnav_elem desk_only">On-Prem</a> </h2> </div> </div><div id="subnav-box-nav-on_prem" class="subnav_box"><a href="https://www.theregister.com/on_prem/" class="subnav_elem" id="nav-on_prem-all"><span class="prefix_all">All </span>On-Prem</a><a href="https://www.theregister.com/on_prem/systems/" class="subnav_elem" id="nav-on_prem-systems">Systems</a><a href="https://www.theregister.com/on_prem/storage/" class="subnav_elem" id="nav-on_prem-storage">Storage</a><a href="https://www.theregister.com/on_prem/networks/" class="subnav_elem" id="nav-on_prem-networks">Networks</a><a href="https://www.theregister.com/on_prem/hpc/" class="subnav_elem" id="nav-on_prem-hpc">HPC</a><a href="https://www.theregister.com/on_prem/personal_tech/" class="subnav_elem" id="nav-on_prem-personal_tech">Personal Tech</a><a href="https://www.theregister.com/on_prem/cxo/" class="subnav_elem" id="nav-on_prem-cxo">CxO</a><a href="https://www.theregister.com/on_prem/public_sector/" class="subnav_elem" id="nav-on_prem-public_sector">Public Sector</a> <noscript><a href="#masthead" class="subnav_elem close_box" aria-label="Top navigation">(X)</a></noscript> </div> </div><div class="nav_elem"> <div class="cat_header"> <div id="nav-software"> <a href="#subnav-box-nav-software" data-toggle-for="subnav-box-nav-software" class="topnav_elem mob_only">Software</a> <h2 class="desk_only section_nav-software"> <a href="#subnav-box-nav-software" data-toggle-for="subnav-box-nav-software" class="topnav_elem desk_only">Software</a> </h2> </div> </div><div id="subnav-box-nav-software" class="subnav_box"><a href="https://www.theregister.com/software/" class="subnav_elem" id="nav-software-all"><span class="prefix_all">All </span>Software</a><a href="https://www.theregister.com/software/ai_ml/" class="subnav_elem" id="nav-software-ai_ml">AI + ML</a><a href="https://www.theregister.com/software/applications/" class="subnav_elem" id="nav-software-applications">Applications</a><a href="https://www.theregister.com/software/databases/" class="subnav_elem" id="nav-software-databases">Databases</a><a href="https://www.theregister.com/software/devops/" class="subnav_elem" id="nav-software-devops">DevOps</a><a href="https://www.theregister.com/software/oses/" class="subnav_elem" id="nav-software-oses">OSes</a><a href="https://www.theregister.com/software/virtualization/" class="subnav_elem" id="nav-software-virtualization">Virtualization</a> <noscript><a href="#masthead" class="subnav_elem close_box" aria-label="Top navigation">(X)</a></noscript> </div> </div><div class="nav_elem"> <div class="cat_header"> <div id="nav-offbeat"> <a href="#subnav-box-nav-offbeat" data-toggle-for="subnav-box-nav-offbeat" class="topnav_elem mob_only">Offbeat</a> <h2 class="desk_only section_nav-offbeat"> <a href="#subnav-box-nav-offbeat" data-toggle-for="subnav-box-nav-offbeat" class="topnav_elem desk_only">Offbeat</a> </h2> </div> </div><div id="subnav-box-nav-offbeat" class="subnav_box"><a href="https://www.theregister.com/offbeat/" class="subnav_elem" id="nav-offbeat-all"><span class="prefix_all">All </span>Offbeat</a><a href="https://www.theregister.com/Debates/" class="subnav_elem" id="nav-offbeat-debates">Debates</a><a href="https://www.theregister.com/offbeat/columnists/" class="subnav_elem" id="nav-offbeat-columnists">Columnists</a><a href="https://www.theregister.com/offbeat/science/" class="subnav_elem" id="nav-offbeat-science">Science</a><a href="https://www.theregister.com/offbeat/geeks_guide/" class="subnav_elem" id="nav-offbeat-geeks_guide">Geek's Guide</a><a href="https://www.theregister.com/offbeat/bofh/" class="subnav_elem" id="nav-offbeat-bofh">BOFH</a><a href="https://www.theregister.com/offbeat/legal/" class="subnav_elem" id="nav-offbeat-legal">Legal</a><a href="https://www.theregister.com/offbeat/bootnotes/" class="subnav_elem" id="nav-offbeat-bootnotes">Bootnotes</a><a href="https://www.theregister.com/offbeat/site_news/" class="subnav_elem" id="nav-offbeat-site_news">Site News</a><a href="https://www.theregister.com/offbeat/about_us/" class="subnav_elem" id="nav-offbeat-about_us">About Us</a> <noscript><a href="#masthead" class="subnav_elem close_box" aria-label="Top navigation">(X)</a></noscript> </div> </div> </nav> </div> </div> </div> <div class="nav_bottom_group"> <div class="nav_bottom_section nav_special_features"> <div class="nav_head_bk"> <a href="#subnav-box-nav-special_features" data-toggle-for="subnav-box-nav-special_features" id="nav-special_features" class="topnav_elem mob_only">Special Features</a> <h2 class="main_head"> <span class="topnav_elem desk_only">Special Features</span> </h2> </div> <nav> <div class="nav_elem"> <div id="subnav-box-nav-special_features" class="subnav_box"> <a href="https://www.theregister.com/special_features">All Special Features</a> <a href="https://www.theregister.com/special_features/cybersecurity_month">Cybersecurity Month</a> <a href="https://www.theregister.com/special_features/vmware_explore">VMware Explore</a> <a href="https://www.theregister.com/special_features/blackhat_and_defcon">Blackhat and DEF CON</a> <a href="https://www.theregister.com/special_features/cloud_infrastructure_month">Cloud Infrastructure Month</a> <a href="https://www.theregister.com/special_features/malware_month">Malware Month</a> <a href="https://www.theregister.com/special_features/the_reg_in_space">The Reg in Space</a> <a href="https://www.theregister.com/special_features/spotlight_on_rsa">Spotlight on RSA</a> </div> </div> </nav> </div> <div class="nav_bottom_section nav_elem nav_vendor_voice"> <div class="nav_head_bk"> <h2 class="main_head"> <span class="topnav_elem desk_only">Vendor Voice</span> </h2> </div> <nav> <div class="nav_elem"> <div class="cat_header"> <div id="nav-tag-vendor-voice"> <a href="#subnav-box-nav-tag-vendor-voice" data-toggle-for="subnav-box-nav-tag-vendor-voice" class="topnav_elem mob_only">Vendor Voice</a> <h2 class="desk_only section_nav-tag-vendor-voice"> <a href="#subnav-box-nav-tag-vendor-voice" data-toggle-for="subnav-box-nav-tag-vendor-voice" class="topnav_elem desk_only">Vendor Voice</a> </h2> </div> </div> <div id="subnav-box-nav-tag-vendor-voice" class="subnav_box"> <a href="https://www.theregister.com/VendorVoice/" class="subnav_elem" id="nav-tag-vendor-voice-all"> <span class="prefix_all">All </span>Vendor Voice </a> <a href="https://www.theregister.com/VendorVoice/aws_here/" class="subnav_elem" id="nav-tag-vendor-voice-vv_aws_here"> HERE and AWS </a> <a href="https://www.theregister.com/VendorVoice/aws_vonage/" class="subnav_elem" id="nav-tag-vendor-voice-vv_aws_vonage"> Vonage </a> <a href="https://www.theregister.com/VendorVoice/aws_amdocs/" class="subnav_elem" id="nav-tag-vendor-voice-vv_aws_amdocs"> Amdocs </a> <a href="https://www.theregister.com/VendorVoice/aws_ge_vernova_manufacturing/" class="subnav_elem" id="nav-tag-vendor-voice-vv_aws_ge_vernova_manufacturing"> GE Vernova with AWS </a> <a href="https://www.theregister.com/VendorVoice/aws_ge_vernova/" class="subnav_elem" id="nav-tag-vendor-voice-vv_aws_ge_vernova"> GE Vernova with AWS </a> <a href="https://www.theregister.com/VendorVoice/siemens_aws/" class="subnav_elem" id="nav-tag-vendor-voice-vv_siemens_aws"> Siemens and AWS Gen AI </a> <a href="https://www.theregister.com/VendorVoice/siemens_aws_itot/" class="subnav_elem" id="nav-tag-vendor-voice-vv_siemens_aws_itot"> Siemens and AWS IT/OT </a> <a href="https://www.theregister.com/VendorVoice/aws_new_horizon_solutions/" class="subnav_elem" id="nav-tag-vendor-voice-vv_aws_new_horizon_solutions"> Amazon Web Services (AWS) New Horizon in Cloud Computing </a> <a href="https://www.theregister.com/VendorVoice/ddn/" class="subnav_elem" id="nav-tag-vendor-voice-vv_ddn"> DDN </a> <a href="https://www.theregister.com/VendorVoice/google_cloud_data_transformation/" class="subnav_elem" id="nav-tag-vendor-voice-vv_google_cloud_data_transformation"> Google Cloud Data Transformation </a> <a href="https://www.theregister.com/VendorVoice/google_gemini/" class="subnav_elem" id="nav-tag-vendor-voice-vv_google_gemini"> Google Gemini </a> <a href="https://www.theregister.com/VendorVoice/hpe_greenlake/" class="subnav_elem" id="nav-tag-vendor-voice-vv_hpe_greenlake"> Hewlett Packard Enterprise: Edge-to-Cloud Platform </a> <a href="https://www.theregister.com/VendorVoice/intelvpro/" class="subnav_elem" id="nav-tag-vendor-voice-vv_intelvpro"> Intel vPro </a> <a href="https://www.theregister.com/VendorVoice/vmware/" class="subnav_elem" id="nav-tag-vendor-voice-vv_vmware"> VMware </a> <noscript> <a href="#masthead" class="subnav_elem close_box" aria-label="Top navigation">(X)</a> </noscript> </div> </div> </nav> </div> <div class="nav_bottom_section nav_resources"> <div class="nav_head_bk"> <a href="#subnav-box-nav-resources" data-toggle-for="subnav-box-nav-resources" id="nav-resources" class="topnav_elem mob_only">Resources</a> <h2 class="main_head"> <span class="topnav_elem desk_only">Resources</span> </h2> </div> <nav id="top_nav"> <div class="nav_elem"> <div id="subnav-box-nav-resources" class="subnav_box"> <a href="https://whitepapers.theregister.com/">Whitepapers</a> <a href="https://whitepapers.theregister.com/events/list/">Webinars & Events</a> <a href="https://account.theregister.com/edit/newsletter/">Newsletters</a> </div> </div> </nav> </div> </div> </div> </div> </div> </div> </div> <div aria-hidden="true" class="adun" data-pos="top" data-raptor="condor" data-xmd=",fluid,leaderboard," data-lg=",fluid,leaderboard," data-xlg=",fluid,superleaderboard,billboard,leaderboard," data-xxlg=",fluid,superleaderboard,billboard,brandwidth,leaderboard,"> <noscript> <a href="https://pubads.g.doubleclick.net/gampad/jump?co=1&iu=/6978/reg_software/devops&sz=300x50%7C300x100%7C300x250%7C300x251%7C300x252%7C300x600%7C300x601&tile=2&c=2Z0OP5VPLBgOPLAjC-o52nwAAAFQ&t=ct%3Dns%26unitnum%3D2%26raptor%3Dcondor%26pos%3Dtop%26test%3D0" target="_blank"> <img src="https://pubads.g.doubleclick.net/gampad/ad?co=1&iu=/6978/reg_software/devops&sz=300x50%7C300x100%7C300x250%7C300x251%7C300x252%7C300x600%7C300x601&tile=2&c=2Z0OP5VPLBgOPLAjC-o52nwAAAFQ&t=ct%3Dns%26unitnum%3D2%26raptor%3Dcondor%26pos%3Dtop%26test%3D0" alt=""> </a> </noscript> </div> <article> <div id=top-col-story> <div class="header_left"> <div class="cat_header"> <h4 class="dcl"> <a href="/software/devops/" aria-label="Devops">Devops</a> </h4> </div> <div class="comments_wrap mobile_only"> <a class="comment_count" aria-label="Read comments on this article, currently there are 89 comments" title="View comments on this article" href="https://forums.theregister.com/forum/all/2023/06/03/microsoft_azure_outage_brazil/"> <strong aria-hidden="true">89</strong> <img aria-hidden="true" width="18" height="16" alt="comment bubble on white" src="/design_picker/f5daacc84b9722c1e31ba85f836c37e4ad993fc4/graphics/icons/bubble_comment_white.png" srcset="/design_picker/f5daacc84b9722c1e31ba85f836c37e4ad993fc4/graphics/icons/bubble_comment_white.svg"> </a> </div> </div> <div class="header_right"> <div class="old_article"> <p>This article is more than <strong>1 year old</strong></p> </div> <h1>This typo sparked a Microsoft Azure outage</h1> </div> <div class="header_left"> <div class="comments_wrap desktop_only"> <a class="comment_count" aria-label="Read comments on this article, currently there are 89 comments" title="View comments on this article" href="https://forums.theregister.com/forum/all/2023/06/03/microsoft_azure_outage_brazil/"> <strong aria-hidden="true">89</strong> <img aria-hidden="true" width="18" height="16" alt="comment bubble on white" src="/design_picker/f5daacc84b9722c1e31ba85f836c37e4ad993fc4/graphics/icons/bubble_comment_white.png" srcset="/design_picker/f5daacc84b9722c1e31ba85f836c37e4ad993fc4/graphics/icons/bubble_comment_white.svg"> </a> </div> </div> <div class="header_right"> <h2>Errant code fix deleted entire servers rather than snapshots of database</h2> <div class="byline_and_dateline_and_share_and_comments"> <div class="byline_wrap"> <img class="vulture_icon" src="/design_picker/d518b499f8a6e2c65d4d8c49aca8299d54b03012/graphics/icon/vulture_red.svg" alt="icon"> <a class="byline" href="/Author/Thomas-Claburn" title="Read more by this author"> Thomas Claburn </a> </div> <div class="dateline_wrap"> <span class="dateline"> Sat 3 Jun 2023 <span class="slashes"> // </span> 19:58 UTC </span> </div> </div> </div> </div> <div id=main-col> <div id="article-wrapper" class="article_wrap"> <div class="left_col"> <div class="floating_bar"> <div class="sharing_widget_story_desktop uses_overlay"> <button class="top_blob" aria-label="Share this story" title="Share this story"> <img width="25" height="25" src="/design_picker/d2e337b97204af4aa34dda04c4e5d56d954b216f/graphics/icons/social_share_icon.svg" alt=""> </button> <div class="sharing_widget_overlay" id="sharing_widget_overlay_2"> <div class="sharing_box"> <a data-social="reddit" href="https://www.reddit.com/submit?url=https://www.theregister.com/2023/06/03/microsoft_azure_outage_brazil/%3futm_medium%3dshare%26utm_content%3darticle%26utm_source%3dreddit&title=This%20typo%20sparked%20a%20Microsoft%20Azure%20outage" target="_blank"> </a> <a data-social="twitter" class="twit" href="https://twitter.com/intent/tweet?text=This%20typo%20sparked%20a%20Microsoft%20Azure%20outage&url=https://www.theregister.com/2023/06/03/microsoft_azure_outage_brazil/%3futm_medium%3dshare%26utm_content%3darticle%26utm_source%3dtwitter&via=theregister" target="_blank"> </a> <a data-social="facebook" class="faceb_dialog" href="https://www.facebook.com/dialog/feed?app_id=1404095453459035&display=popup&link=https://www.theregister.com/2023/06/03/microsoft_azure_outage_brazil/%3futm_medium%3dshare%26utm_content%3darticle%26utm_source%3dfacebook" target="_blank"> </a> <br class="hide_after_sm"> <a data-social="linkedin" class="linkedin_social" href="https://www.linkedin.com/shareArticle?mini=true&url=https://www.theregister.com/2023/06/03/microsoft_azure_outage_brazil/%3futm_medium%3dshare%26utm_content%3darticle%26utm_source%3dlinkedin&title=This%20typo%20sparked%20a%20Microsoft%20Azure%20outage&summary=Errant%20code%20fix%20deleted%20entire%20servers%20rather%20than%20snapshots%20of%20database" target="_blank"> </a> <a data-social="whatsapp" href="https://api.whatsapp.com/send?text=https://www.theregister.com/2023/06/03/microsoft_azure_outage_brazil/%3futm_medium%3dshare%26utm_content%3darticle%26utm_source%3dwhatsapp" target="_blank"> </a> </div> </div> </div> </div> <div class="promo_advert"> </div> </div> <div class="centre_col"> <div id="article"> <div id="body"> <p>Microsoft Azure DevOps, a suite of application lifecycle services, stopped working in the South Brazil region for about ten hours on Wednesday due to a basic code error.</p> <p>On Friday Eric Mattingly, principal software engineering manager, offered an apology for the disruption and revealed the cause of the outage: <a target="_blank" rel="nofollow" href="https://status.dev.azure.com/_event/392143683/post-mortem">a simple typo</a> that deleted seventeen production databases.</p> <p>Mattingly explained that Azure DevOps engineers occasionally take snapshots of production databases to look into reported problems or test performance improvements. And they rely on a background system that runs daily and deletes old snapshots after a set period of time.</p> <div aria-hidden="true" class="adun" data-pos="top" data-raptor="condor" data-xsm=",fluid,mpu," data-sm=",fluid,mpu," data-md=",fluid,mpu,"> <noscript> <a href="https://pubads.g.doubleclick.net/gampad/jump?co=1&iu=/6978/reg_software/devops&sz=300x50%7C300x100%7C300x250%7C300x251%7C300x252%7C300x600%7C300x601&tile=2&c=2Z0OP5VPLBgOPLAjC-o52nwAAAFQ&t=ct%3Dns%26unitnum%3D2%26raptor%3Dcondor%26pos%3Dtop%26test%3D0" target="_blank"> <img src="https://pubads.g.doubleclick.net/gampad/ad?co=1&iu=/6978/reg_software/devops&sz=300x50%7C300x100%7C300x250%7C300x251%7C300x252%7C300x600%7C300x601&tile=2&c=2Z0OP5VPLBgOPLAjC-o52nwAAAFQ&t=ct%3Dns%26unitnum%3D2%26raptor%3Dcondor%26pos%3Dtop%26test%3D0" alt=""> </a> </noscript> </div> <p>During a recent <a target="_blank" rel="nofollow" href="https://learn.microsoft.com/en-us/azure/devops/boards/sprints/assign-work-sprint?view=azure-devops">sprint</a> – a group project in Agile jargon – Azure DevOps engineers performed a code upgrade, replacing deprecated Microsoft.Azure.Managment.* packages with supported Azure.ResourceManager.* NuGet packages.</p> <div aria-hidden="true" class="adun" data-pos="top" data-raptor="falcon" data-xmd=",fluid,mpu,leaderboard," data-lg=",fluid,mpu,leaderboard," data-xlg=",fluid,billboard,superleaderboard,mpu,leaderboard," data-xxlg=",fluid,billboard,superleaderboard,brandwidth,brandimpact,leaderboard,mpu,"> <noscript> <a href="https://pubads.g.doubleclick.net/gampad/jump?co=1&iu=/6978/reg_software/devops&sz=300x50%7C300x100%7C300x250%7C300x251%7C300x252%7C300x600%7C300x601&tile=4&c=44Z0OP5VPLBgOPLAjC-o52nwAAAFQ&t=ct%3Dns%26unitnum%3D4%26raptor%3Dfalcon%26pos%3Dmid%26test%3D0" target="_blank"> <img src="https://pubads.g.doubleclick.net/gampad/ad?co=1&iu=/6978/reg_software/devops&sz=300x50%7C300x100%7C300x250%7C300x251%7C300x252%7C300x600%7C300x601&tile=4&c=44Z0OP5VPLBgOPLAjC-o52nwAAAFQ&t=ct%3Dns%26unitnum%3D426raptor%3Dfalcon%26pos%3Dmid%26test%3D0" alt=""> </a> </noscript> </div> <div class="adun_eagle_desktop_story_wrapper"> <div aria-hidden="true" class="adun" data-pos="mid" data-raptor="eagle" data-xxlg=",mpu,dmpu,"> <noscript> <a href="https://pubads.g.doubleclick.net/gampad/jump?co=1&iu=/6978/reg_software/devops&sz=300x50%7C300x100%7C300x250%7C300x251%7C300x252%7C300x600%7C300x601&tile=3&c=33Z0OP5VPLBgOPLAjC-o52nwAAAFQ&t=ct%3Dns%26unitnum%3D3%26raptor%3Deagle%26pos%3Dmid%26test%3D0" target="_blank"> <img src="https://pubads.g.doubleclick.net/gampad/ad?co=1&iu=/6978/reg_software/devops&sz=300x50%7C300x100%7C300x250%7C300x251%7C300x252%7C300x600%7C300x601&tile=3&c=33Z0OP5VPLBgOPLAjC-o52nwAAAFQ&t=ct%3Dns%26unitnum%3D3%26raptor%3Deagle%26pos%3Dmid%26test%3D0" alt=""> </a> </noscript> </div> </div> <p>The result was a large pull request of changes that swapped API calls in the old packages for those in the newer packages. The typo occurred in the pull request – a code change that has to be reviewed and merged into the applicable project. And it led the background snapshot deletion job to delete the entire server.</p> <p>"Hidden within this pull request was a typo bug in the snapshot deletion job which swapped out a call to delete the Azure SQL Database to one that deletes the Azure SQL Server that hosts the database," said Mattingly.</p> <ul class="listinks"> <li><a href="https://www.theregister.com/2023/04/24/microsoft_365_search_outage/">Where are we now – Microsoft 363? Cloud suite suffers another outage</a></li> <li><a href="https://www.theregister.com/2023/04/04/microsoft_app_center_outage/">Microsoft App Center has been down for nearly a day, and no word on when it'll end</a></li> <li><a href="https://www.theregister.com/2023/04/20/microsoft_365_services_outage/">Microsoft not a Teams player as admin center, 365 service suffer partial outage</a></li> <li><a href="https://www.theregister.com/2023/03/24/microsoft_geolocation_fail_uzbekistan/">Microsoft breaks geolocation, locking users out of Azure and M365</a></li> </ul> <p>Azure DevOps has tests to catch such issues, but according to Mattingly, the errant code only runs under certain conditions and thus isn't well covered under existing tests. Those conditions, presumably, require the presence of a database snapshot that is old enough to be caught by the deletion script.</p> <p>Mattingly said Sprint 222 was deployed internally (Ring 0) without incident due to the absence of any snapshot databases. Several days later, the software changes were deployed to the customer environment (Ring 1) for the South Brazil scale unit (a cluster of servers for a specific role). That environment had a snapshot database old enough to trigger the bug, which led the background job to delete the "entire Azure SQL Server and all seventeen production databases" for the scale unit.</p> <div aria-hidden="true" class="adun" data-pos="top" data-raptor="falcon" data-xsm=",fluid,mpu," data-sm=",fluid,mpu," data-md=",fluid,mpu,"> <noscript> <a href="https://pubads.g.doubleclick.net/gampad/jump?co=1&iu=/6978/reg_software/devops&sz=300x50%7C300x100%7C300x250%7C300x251%7C300x252%7C300x600%7C300x601&tile=4&c=44Z0OP5VPLBgOPLAjC-o52nwAAAFQ&t=ct%3Dns%26unitnum%3D4%26raptor%3Dfalcon%26pos%3Dmid%26test%3D0" target="_blank"> <img src="https://pubads.g.doubleclick.net/gampad/ad?co=1&iu=/6978/reg_software/devops&sz=300x50%7C300x100%7C300x250%7C300x251%7C300x252%7C300x600%7C300x601&tile=4&c=44Z0OP5VPLBgOPLAjC-o52nwAAAFQ&t=ct%3Dns%26unitnum%3D426raptor%3Dfalcon%26pos%3Dmid%26test%3D0" alt=""> </a> </noscript> </div> <p>The data has all been recovered, but it took more than ten hours. There are several reasons for that, said Mattingly.</p> <p>One is that since customers can't revive Azure SQL Servers themselves, on-call Azure engineers had to handle that, a process that took about an hour for many.</p> <p>Another reason is that the databases had different backup configurations: some were configured for Zone-redundant backup and others were set up for the more recent Geo-zone-redundant backup. Reconciling this mismatch added many hours to the recovery process.</p> <div aria-hidden="true" class="adun" id="story_eagle_xsm_sm_md_xmd_lg_xlg" data-pos="mid" data-raptor="eagle" data-xsm=",mpu,dmpu," data-sm=",mpu,dmpu," data-md=",mpu,dmpu," data-xmd=",mpu,dmpu," data-lg=",mpu,dmpu," data-xlg=",mpu,dmpu,"> <noscript> <a href="https://pubads.g.doubleclick.net/gampad/jump?co=1&iu=/6978/reg_software/devops&sz=300x50%7C300x100%7C300x250%7C300x251%7C300x252%7C300x600%7C300x601&tile=3&c=33Z0OP5VPLBgOPLAjC-o52nwAAAFQ&t=ct%3Dns%26unitnum%3D3%26raptor%3Deagle%26pos%3Dmid%26test%3D0" target="_blank"> <img src="https://pubads.g.doubleclick.net/gampad/ad?co=1&iu=/6978/reg_software/devops&sz=300x50%7C300x100%7C300x250%7C300x251%7C300x252%7C300x600%7C300x601&tile=3&c=33Z0OP5VPLBgOPLAjC-o52nwAAAFQ&t=ct%3Dns%26unitnum%3D3%26raptor%3Deagle%26pos%3Dmid%26test%3D0" alt=""> </a> </noscript> </div> <p>"Finally," said Mattingly, "Even after databases began coming back online, the entire scale unit remained inaccessible even to customers whose data was in those databases due to a complex set of issues with our web servers."</p> <p>These issues arose from a server warmup task that iterated through the list of available databases with a test call. Databases in the process of being recovered chucked up an error that led the warm-up test "to perform an exponential backoff retry resulting in warmup taking ninety minutes on average, versus sub-second in a normal situation."</p> <p>Further complicating matters, this recovery process was staggered and once one or two of the servers started taking customer traffic again, they'd get overloaded, and go down. Ultimately, restoring service required blocking all traffic to the South Brazil scale unit until everything was sufficiently ready to rejoin the load balancer and handle traffic.</p> <p>Various fixes and reconfigurations have been put in place to prevent the issue from recurring.</p> <p>"Once again, we apologize to all the customers impacted by this outage," said Mattingly. ®</p> <div class="wptl btm"> <noscript><strong>Get our</strong> <a href="https://whitepapers.theregister.com/" style="text-transform:uppercase">Tech Resources</a></noscript> </div> </div> <div class="article_body_btm mobile_only"> <div class="sharing_widget_story_desktop uses_overlay"> <button class="top_blob" aria-label="Share this story" title="Share this story"> <img width="25" height="25" src="/design_picker/d2e337b97204af4aa34dda04c4e5d56d954b216f/graphics/icons/social_share_icon.svg" alt=""> <span>Share</span> </button> <div class="sharing_widget_overlay" id="sharing_widget_overlay_3"> <div class="sharing_box"> <a data-social="reddit" href="https://www.reddit.com/submit?url=https://www.theregister.com/2023/06/03/microsoft_azure_outage_brazil/%3futm_medium%3dshare%26utm_content%3darticle%26utm_source%3dreddit&title=This%20typo%20sparked%20a%20Microsoft%20Azure%20outage" target="_blank"> </a> <a data-social="twitter" class="twit" href="https://twitter.com/intent/tweet?text=This%20typo%20sparked%20a%20Microsoft%20Azure%20outage&url=https://www.theregister.com/2023/06/03/microsoft_azure_outage_brazil/%3futm_medium%3dshare%26utm_content%3darticle%26utm_source%3dtwitter&via=theregister" target="_blank"> </a> <a data-social="facebook" class="faceb_dialog" href="https://www.facebook.com/dialog/feed?app_id=1404095453459035&display=popup&link=https://www.theregister.com/2023/06/03/microsoft_azure_outage_brazil/%3futm_medium%3dshare%26utm_content%3darticle%26utm_source%3dfacebook" target="_blank"> </a> <br class="hide_after_sm"> <a data-social="linkedin" class="linkedin_social" href="https://www.linkedin.com/shareArticle?mini=true&url=https://www.theregister.com/2023/06/03/microsoft_azure_outage_brazil/%3futm_medium%3dshare%26utm_content%3darticle%26utm_source%3dlinkedin&title=This%20typo%20sparked%20a%20Microsoft%20Azure%20outage&summary=Errant%20code%20fix%20deleted%20entire%20servers%20rather%20than%20snapshots%20of%20database" target="_blank"> </a> <a data-social="whatsapp" href="https://api.whatsapp.com/send?text=https://www.theregister.com/2023/06/03/microsoft_azure_outage_brazil/%3futm_medium%3dshare%26utm_content%3darticle%26utm_source%3dwhatsapp" target="_blank"> </a> </div> </div> </div> </div> </div> </div> <div class="right_col desktop_only"> <div class="similar_topics"> <div class="similar_topics"> <h4>More about</h4> <ul class="keywords"> <li> <a href="/Tag/Azure/" > <span class="keyword_name"> Azure </span> </a> </li> <li> <a href="/Tag/Devops/" > <span class="keyword_name"> Devops </span> </a> </li> <li> <a href="/Tag/Microsoft/" > <span class="keyword_name"> Microsoft </span> </a> </li> </ul> </div> <div class="keyword_wrap" style="display: none;"> <div class="keyword_trigger">More like these</div> </div> <div class="lightbox_overlay"> <div class="keyword_popup more_topics"> <div class="close">×</div> <div class="keyword_group similar_topics"> <h3>More about</h3> <ul class="keywords"> <li> <a href="/Tag/Azure/" > <span class="keyword_name"> Azure </span> </a> </li> <li> <a href="/Tag/Devops/" > <span class="keyword_name"> Devops </span> </a> </li> <li> <a href="/Tag/Microsoft/" > <span class="keyword_name"> Microsoft </span> </a> </li> <li> <a href="/Tag/Outage/" > <span class="keyword_name"> Outage </span> </a> </li> </ul> </div> <div class="keyword_group child_topics"> <h3>Narrower topics</h3> <ul class="keywords"> <li> <a href="/Tag/Active%20Directory/" > <span class="keyword_name"> Active Directory </span> </a> </li> <li> <a href="/Tag/API/" > <span class="keyword_name"> API </span> </a> </li> <li> <a href="/Tag/Azure%20Stack/" > <span class="keyword_name"> Azure Stack </span> </a> </li> <li> <a href="/Tag/Bing/" > <span class="keyword_name"> Bing </span> </a> </li> <li> <a href="/Tag/BSoD/" > <span class="keyword_name"> BSoD </span> </a> </li> <li> <a href="/Tag/Cloud%20native/" > <span class="keyword_name"> Cloud native </span> </a> </li> <li> <a href="/Tag/Excel/" > <span class="keyword_name"> Excel </span> </a> </li> <li> <a href="/Tag/Exchange%20Server/" > <span class="keyword_name"> Exchange Server </span> </a> </li> <li> <a href="/Tag/FinOps/" > <span class="keyword_name"> FinOps </span> </a> </li> <li> <a href="/Tag/HoloLens/" > <span class="keyword_name"> HoloLens </span> </a> </li> <li> <a href="/Tag/Internet%20Explorer/" > <span class="keyword_name"> Internet Explorer </span> </a> </li> <li> <a href="/Tag/LinkedIn/" > <span class="keyword_name"> LinkedIn </span> </a> </li> <li> <a href="/Tag/Microsoft%20365/" > <span class="keyword_name"> Microsoft 365 </span> </a> </li> <li> <a href="/Tag/Microsoft%20Build/" > <span class="keyword_name"> Microsoft Build </span> </a> </li> <li> <a href="/Tag/Microsoft%20Edge/" > <span class="keyword_name"> Microsoft Edge </span> </a> </li> <li> <a href="/Tag/Microsoft%20Ignite/" > <span class="keyword_name"> Microsoft Ignite </span> </a> </li> <li> <a href="/Tag/Microsoft%20Office/" > <span class="keyword_name"> Microsoft Office </span> </a> </li> <li> <a href="/Tag/Microsoft%20Surface/" > <span class="keyword_name"> Microsoft Surface </span> </a> </li> <li> <a href="/Tag/Microsoft%20Teams/" > <span class="keyword_name"> Microsoft Teams </span> </a> </li> <li> <a href="/Tag/.NET/" > <span class="keyword_name"> .NET </span> </a> </li> <li> <a href="/Tag/Office%20365/" > <span class="keyword_name"> Office 365 </span> </a> </li> <li> <a href="/Tag/OS%2F2/" > <span class="keyword_name"> OS/2 </span> </a> </li> <li> <a href="/Tag/Outlook/" > <span class="keyword_name"> Outlook </span> </a> </li> <li> <a href="/Tag/Patch%20Tuesday/" > <span class="keyword_name"> Patch Tuesday </span> </a> </li> <li> <a href="/Tag/Pluton/" > <span class="keyword_name"> Pluton </span> </a> </li> <li> <a href="/Tag/SharePoint/" > <span class="keyword_name"> SharePoint </span> </a> </li> <li> <a href="/Tag/Skype/" > <span class="keyword_name"> Skype </span> </a> </li> <li> <a href="/Tag/SQL%20Server/" > <span class="keyword_name"> SQL Server </span> </a> </li> <li> <a href="/Tag/Visual%20Studio/" > <span class="keyword_name"> Visual Studio </span> </a> </li> <li> <a href="/Tag/Visual%20Studio%20Code/" > <span class="keyword_name"> Visual Studio Code </span> </a> </li> <li> <a href="/Tag/Windows/" > <span class="keyword_name"> Windows </span> </a> </li> <li> <a href="/Tag/Windows%2010/" > <span class="keyword_name"> Windows 10 </span> </a> </li> <li> <a href="/Tag/Windows%2011/" > <span class="keyword_name"> Windows 11 </span> </a> </li> <li> <a href="/Tag/Windows%207/" > <span class="keyword_name"> Windows 7 </span> </a> </li> <li> <a href="/Tag/Windows%208/" > <span class="keyword_name"> Windows 8 </span> </a> </li> <li> <a href="/Tag/Windows%20Server/" > <span class="keyword_name"> Windows Server </span> </a> </li> <li> <a href="/Tag/Windows%20Server%202003/" > <span class="keyword_name"> Windows Server 2003 </span> </a> </li> <li> <a href="/Tag/Windows%20Server%202008/" > <span class="keyword_name"> Windows Server 2008 </span> </a> </li> <li> <a href="/Tag/Windows%20Server%202012/" > <span class="keyword_name"> Windows Server 2012 </span> </a> </li> <li> <a href="/Tag/Windows%20Server%202013/" > <span class="keyword_name"> Windows Server 2013 </span> </a> </li> <li> <a href="/Tag/Windows%20Server%202016/" > <span class="keyword_name"> Windows Server 2016 </span> </a> </li> <li> <a href="/Tag/Windows%20Subsystem%20for%20Linux/" > <span class="keyword_name"> Windows Subsystem for Linux </span> </a> </li> <li> <a href="/Tag/Windows%20XP/" > <span class="keyword_name"> Windows XP </span> </a> </li> <li> <a href="/Tag/Xbox/" > <span class="keyword_name"> Xbox </span> </a> </li> <li> <a href="/Tag/Xbox%20360/" > <span class="keyword_name"> Xbox 360 </span> </a> </li> </ul> </div> <div class="keyword_group parent_topics"> <h3>Broader topics</h3> <ul class="keywords"> <li> <a href="/Tag/Bill%20Gates/" > <span class="keyword_name"> Bill Gates </span> </a> </li> <li> <a href="/Tag/Cloud%20Computing/" > <span class="keyword_name"> Cloud Computing </span> </a> </li> <li> <a href="/Tag/Development/" > <span class="keyword_name"> Development </span> </a> </li> </ul> </div> </div> </div> </div> </div> <div class="right_col mobile_only"> <div class="similar_topics"> <h4>More about</h4> </div> </div> <div class="left_col main_content"> <div class="sharing_block"> <div class=article_body_btm> <div class="sharing_widget_story_desktop uses_overlay"> <button class="top_blob" aria-label="Share this story" title="Share this story"> <img width="25" height="25" src="/design_picker/d2e337b97204af4aa34dda04c4e5d56d954b216f/graphics/icons/social_share_icon.svg" alt=""> <span>Share</span> </button> <div class="sharing_widget_overlay" id="sharing_widget_overlay_4"> <div class="sharing_box"> <a data-social="reddit" href="https://www.reddit.com/submit?url=https://www.theregister.com/2023/06/03/microsoft_azure_outage_brazil/%3futm_medium%3dshare%26utm_content%3darticle%26utm_source%3dreddit&title=This%20typo%20sparked%20a%20Microsoft%20Azure%20outage" target="_blank"> </a> <a data-social="twitter" class="twit" href="https://twitter.com/intent/tweet?text=This%20typo%20sparked%20a%20Microsoft%20Azure%20outage&url=https://www.theregister.com/2023/06/03/microsoft_azure_outage_brazil/%3futm_medium%3dshare%26utm_content%3darticle%26utm_source%3dtwitter&via=theregister" target="_blank"> </a> <a data-social="facebook" class="faceb_dialog" href="https://www.facebook.com/dialog/feed?app_id=1404095453459035&display=popup&link=https://www.theregister.com/2023/06/03/microsoft_azure_outage_brazil/%3futm_medium%3dshare%26utm_content%3darticle%26utm_source%3dfacebook" target="_blank"> </a> <br class="hide_after_sm"> <a data-social="linkedin" class="linkedin_social" href="https://www.linkedin.com/shareArticle?mini=true&url=https://www.theregister.com/2023/06/03/microsoft_azure_outage_brazil/%3futm_medium%3dshare%26utm_content%3darticle%26utm_source%3dlinkedin&title=This%20typo%20sparked%20a%20Microsoft%20Azure%20outage&summary=Errant%20code%20fix%20deleted%20entire%20servers%20rather%20than%20snapshots%20of%20database" target="_blank"> </a> <a data-social="whatsapp" href="https://api.whatsapp.com/send?text=https://www.theregister.com/2023/06/03/microsoft_azure_outage_brazil/%3futm_medium%3dshare%26utm_content%3darticle%26utm_source%3dwhatsapp" target="_blank"> </a> </div> </div> </div> </div> </div> </div> <div class="centre_col main_content"> <div class="comments "> <a class="comment_count" aria-label="Read comments on this article, currently there are 89 comments" title="View comments on this article" href="https://forums.theregister.com/forum/all/2023/06/03/microsoft_azure_outage_brazil/"> <strong aria-hidden="true">89</strong> <img aria-hidden="true" width="18" height="16" alt="comment bubble on white" src="/design_picker/f5daacc84b9722c1e31ba85f836c37e4ad993fc4/graphics/icons/bubble_comment_white.png" srcset="/design_picker/f5daacc84b9722c1e31ba85f836c37e4ad993fc4/graphics/icons/bubble_comment_white.svg"> COMMENTS </a> </div> </div> <div class="hidden_col mobile_only"> <div class="similar_topics"> <h4>More about</h4> <ul class="keywords"> <li> <a href="/Tag/Azure/" > <span class="keyword_name"> Azure </span> </a> </li> <li> <a href="/Tag/Devops/" > <span class="keyword_name"> Devops </span> </a> </li> <li> <a href="/Tag/Microsoft/" > <span class="keyword_name"> Microsoft </span> </a> </li> </ul> </div> <div class="keyword_wrap" style="display: none;"> <div class="keyword_trigger">More like these</div> </div> <div class="lightbox_overlay"> <div class="keyword_popup more_topics"> <div class="close">×</div> <div class="keyword_group similar_topics"> <h3>More about</h3> <ul class="keywords"> <li> <a href="/Tag/Azure/" > <span class="keyword_name"> Azure </span> </a> </li> <li> <a href="/Tag/Devops/" > <span class="keyword_name"> Devops </span> </a> </li> <li> <a href="/Tag/Microsoft/" > <span class="keyword_name"> Microsoft </span> </a> </li> <li> <a href="/Tag/Outage/" > <span class="keyword_name"> Outage </span> </a> </li> </ul> </div> <div class="keyword_group child_topics"> <h3>Narrower topics</h3> <ul class="keywords"> <li> <a href="/Tag/Active%20Directory/" > <span class="keyword_name"> Active Directory </span> </a> </li> <li> <a href="/Tag/API/" > <span class="keyword_name"> API </span> </a> </li> <li> <a href="/Tag/Azure%20Stack/" > <span class="keyword_name"> Azure Stack </span> </a> </li> <li> <a href="/Tag/Bing/" > <span class="keyword_name"> Bing </span> </a> </li> <li> <a href="/Tag/BSoD/" > <span class="keyword_name"> BSoD </span> </a> </li> <li> <a href="/Tag/Cloud%20native/" > <span class="keyword_name"> Cloud native </span> </a> </li> <li> <a href="/Tag/Excel/" > <span class="keyword_name"> Excel </span> </a> </li> <li> <a href="/Tag/Exchange%20Server/" > <span class="keyword_name"> Exchange Server </span> </a> </li> <li> <a href="/Tag/FinOps/" > <span class="keyword_name"> FinOps </span> </a> </li> <li> <a href="/Tag/HoloLens/" > <span class="keyword_name"> HoloLens </span> </a> </li> <li> <a href="/Tag/Internet%20Explorer/" > <span class="keyword_name"> Internet Explorer </span> </a> </li> <li> <a href="/Tag/LinkedIn/" > <span class="keyword_name"> LinkedIn </span> </a> </li> <li> <a href="/Tag/Microsoft%20365/" > <span class="keyword_name"> Microsoft 365 </span> </a> </li> <li> <a href="/Tag/Microsoft%20Build/" > <span class="keyword_name"> Microsoft Build </span> </a> </li> <li> <a href="/Tag/Microsoft%20Edge/" > <span class="keyword_name"> Microsoft Edge </span> </a> </li> <li> <a href="/Tag/Microsoft%20Ignite/" > <span class="keyword_name"> Microsoft Ignite </span> </a> </li> <li> <a href="/Tag/Microsoft%20Office/" > <span class="keyword_name"> Microsoft Office </span> </a> </li> <li> <a href="/Tag/Microsoft%20Surface/" > <span class="keyword_name"> Microsoft Surface </span> </a> </li> <li> <a href="/Tag/Microsoft%20Teams/" > <span class="keyword_name"> Microsoft Teams </span> </a> </li> <li> <a href="/Tag/.NET/" > <span class="keyword_name"> .NET </span> </a> </li> <li> <a href="/Tag/Office%20365/" > <span class="keyword_name"> Office 365 </span> </a> </li> <li> <a href="/Tag/OS%2F2/" > <span class="keyword_name"> OS/2 </span> </a> </li> <li> <a href="/Tag/Outlook/" > <span class="keyword_name"> Outlook </span> </a> </li> <li> <a href="/Tag/Patch%20Tuesday/" > <span class="keyword_name"> Patch Tuesday </span> </a> </li> <li> <a href="/Tag/Pluton/" > <span class="keyword_name"> Pluton </span> </a> </li> <li> <a href="/Tag/SharePoint/" > <span class="keyword_name"> SharePoint </span> </a> </li> <li> <a href="/Tag/Skype/" > <span class="keyword_name"> Skype </span> </a> </li> <li> <a href="/Tag/SQL%20Server/" > <span class="keyword_name"> SQL Server </span> </a> </li> <li> <a href="/Tag/Visual%20Studio/" > <span class="keyword_name"> Visual Studio </span> </a> </li> <li> <a href="/Tag/Visual%20Studio%20Code/" > <span class="keyword_name"> Visual Studio Code </span> </a> </li> <li> <a href="/Tag/Windows/" > <span class="keyword_name"> Windows </span> </a> </li> <li> <a href="/Tag/Windows%2010/" > <span class="keyword_name"> Windows 10 </span> </a> </li> <li> <a href="/Tag/Windows%2011/" > <span class="keyword_name"> Windows 11 </span> </a> </li> <li> <a href="/Tag/Windows%207/" > <span class="keyword_name"> Windows 7 </span> </a> </li> <li> <a href="/Tag/Windows%208/" > <span class="keyword_name"> Windows 8 </span> </a> </li> <li> <a href="/Tag/Windows%20Server/" > <span class="keyword_name"> Windows Server </span> </a> </li> <li> <a href="/Tag/Windows%20Server%202003/" > <span class="keyword_name"> Windows Server 2003 </span> </a> </li> <li> <a href="/Tag/Windows%20Server%202008/" > <span class="keyword_name"> Windows Server 2008 </span> </a> </li> <li> <a href="/Tag/Windows%20Server%202012/" > <span class="keyword_name"> Windows Server 2012 </span> </a> </li> <li> <a href="/Tag/Windows%20Server%202013/" > <span class="keyword_name"> Windows Server 2013 </span> </a> </li> <li> <a href="/Tag/Windows%20Server%202016/" > <span class="keyword_name"> Windows Server 2016 </span> </a> </li> <li> <a href="/Tag/Windows%20Subsystem%20for%20Linux/" > <span class="keyword_name"> Windows Subsystem for Linux </span> </a> </li> <li> <a href="/Tag/Windows%20XP/" > <span class="keyword_name"> Windows XP </span> </a> </li> <li> <a href="/Tag/Xbox/" > <span class="keyword_name"> Xbox </span> </a> </li> <li> <a href="/Tag/Xbox%20360/" > <span class="keyword_name"> Xbox 360 </span> </a> </li> </ul> </div> <div class="keyword_group parent_topics"> <h3>Broader topics</h3> <ul class="keywords"> <li> <a href="/Tag/Bill%20Gates/" > <span class="keyword_name"> Bill Gates </span> </a> </li> <li> <a href="/Tag/Cloud%20Computing/" > <span class="keyword_name"> Cloud Computing </span> </a> </li> <li> <a href="/Tag/Development/" > <span class="keyword_name"> Development </span> </a> </li> </ul> </div> </div> </div> </div> <div class="right_col main_content"> <div class="tip_off_widget"> <h4>TIP US OFF</h4> <p><a href="https://www.theregister.com/Profile/contact/" target="_blank">Send us news</a></p> </div> </div> </div> </div> </article> <hr id=story_section_break> <div id=story-bot-col> <h3 style="position:absolute;color:transparent;z-index:-1;">Other stories you might like</h3> <div id="aua" data-unit-type="aua" class="keepreading"> <div class=headlines> <div class="img_lite_srow img_lite_rt-1b"> <article> <a href="/2024/11/20/microsoft_flight_simulator_2024_launch/?td=keepreading" class=story_link> <div class="article_text_elements"> <h4>Microsoft Flight Simulator 2024 struggles to take off</h4> <div class=standfirst>If only the company in the title knew how to scale servers in the cloud</div> <div class=time_comments> <span class="section_name">Personal Tech</span><span class="time_stamp" title="20 Nov 2024 11:28" data-epoch="1732102093">20 Nov 2024</span> | <span class="comment light_bg_comments">41</span></div> </div> </a> </article> <article> <a href="/2024/11/20/european_cloud_competition_observatory/?td=keepreading" class=story_link> <div class="article_text_elements"> <h4>European Cloud Competition Observatory created to keep an eye on software licensing</h4> <div class=standfirst> <span class="label">Exclusive</span> Initiative follows Microsoft settlement with CISPE consortium</div> <div class=time_comments> <span class="section_name">PaaS + IaaS</span><span class="time_stamp" title="20 Nov 2024 6:1" data-epoch="1732082473">20 Nov 2024</span> | <span class="comment light_bg_comments">3</span></div> </div> </a> </article> <article> <a href="/2024/11/19/microsoft_autonomous_copilot_ai/?td=keepreading" class=story_link> <div class="article_text_elements"> <h4>Microsoft unleashes autonomous Copilot AI agents in public preview</h4> <div class=standfirst> <span class="label">Ignite</span> They can learn, adapt, and make decisions – but don't worry, they're not coming for <i>your</i> job</div> <div class=time_comments> <span class="section_name">PaaS + IaaS</span><span class="time_stamp" title="19 Nov 2024 13:30" data-epoch="1732023007">19 Nov 2024</span> | <span class="comment light_bg_comments">9</span></div> </div> </a> </article> <article> <a href="/2024/10/07/an_easy_route_to_aienhanced/?td=keepreading" class=story_link> <div class="article_text_elements"> <h4>An easy route to AI-enhanced productivity</h4> <div class=standfirst>How the integration of Google Gemini across Google Workspace turbo charges existing apps with AI power</div> <div class=time_comments><span class="section_name">Sponsored Feature</span></div> </div> </a> </article> </div> <div aria-hidden="true" class="adun" data-pos="btm" data-raptor="hawk" data-xsm=",fluid,mpu," data-sm=",fluid,mpu," data-md=",fluid,mpu," data-xmd=",fluid,leaderboard,mpu," data-lg=",fluid,mpu,leaderboard," data-xlg=",fluid,billboard,superleaderboard,mpu,leaderboard," data-xxlg=",fluid,billboard,superleaderboard,brandwidth,brandimpact,mpu,leaderboard,"> <noscript> <a href="https://pubads.g.doubleclick.net/gampad/jump?co=1&iu=/6978/reg_software/devops&sz=300x50%7C300x100%7C300x250%7C300x251%7C300x252%7C300x600%7C300x601&tile=6&c=66Z0OP5VPLBgOPLAjC-o52nwAAAFQ&t=ct%3Dns%26unitnum%3D6%26raptor%3Dhawk%26pos%3Dbtm%26test%3D0" target="_blank"> <img src="https://pubads.g.doubleclick.net/gampad/ad?co=1&iu=/6978/reg_software/devops&sz=300x50%7C300x100%7C300x250%7C300x251%7C300x252%7C300x600%7C300x601&tile=6&c=66Z0OP5VPLBgOPLAjC-o52nwAAAFQ&t=ct%3Dns%26unitnum%3D6%26raptor%3Dhawk%26pos%3Dbtm%26test%3D0" alt=""> </a> </noscript> </div> <div class="img_lite_srow img_lite_rt-1b"> <article> <a href="/2024/11/14/microsoft_365_copilot_monthly/?td=keepreading" class=story_link> <div class="article_text_elements"> <h4>Microsoft 365 Copilot goes monthly for a 5% premium and annual commitment</h4> <div class=standfirst>Other subscriptions also set for updates in the name of 'cash flow flexibility'</div> <div class=time_comments> <span class="section_name">Software</span><span class="time_stamp" title="14 Nov 2024 13:28" data-epoch="1731590889">14 Nov 2024</span> | <span class="comment light_bg_comments">26</span></div> </div> </a> </article> <article> <a href="/2024/11/14/qualcomm_pc_strategy/?td=keepreading" class=story_link> <div class="article_text_elements"> <h4>Qualcomm's Windows on Arm push would be great – if only it ran all your software</h4> <div class=standfirst> <span class="label">Analysis</span> Until compatibility issues are properly addressed, it'll never stand up to x86</div> <div class=time_comments> <span class="section_name">Personal Tech</span><span class="time_stamp" title="14 Nov 2024 16:27" data-epoch="1731601632">14 Nov 2024</span> | <span class="comment light_bg_comments">51</span></div> </div> </a> </article> <article> <a href="/2024/11/19/microsoft_windows_365_link/?td=keepreading" class=story_link> <div class="article_text_elements"> <h4>Microsoft goes thin client with $349 Windows 365 Link mini PC</h4> <div class=standfirst> <span class="label">Ignite</span> Just as good as Apple at making squat boxes, but this one doesn't do very much</div> <div class=time_comments> <span class="section_name">Personal Tech</span><span class="time_stamp" title="19 Nov 2024 14:44" data-epoch="1732027454">19 Nov 2024</span> | <span class="comment light_bg_comments">59</span></div> </div> </a> </article> <article> <a href="/2024/11/22/microsoft_recall_release/?td=keepreading" class=story_link> <div class="article_text_elements"> <h4>Now’s your chance to try Microsoft’s controversial Windows Recall ... maybe</h4> <div class=standfirst>Like its AI, this automated screenshotter and logger is a feature not exactly everyone wanted</div> <div class=time_comments> <span class="section_name">OSes</span><span class="time_stamp" title="22 Nov 2024 23:54" data-epoch="1732319654">22 Nov 2024</span> | <span class="comment light_bg_comments">29</span></div> </div> </a> </article> </div> <div class="img_lite_srow img_lite_rt-1b"> <article> <a href="/2024/11/15/microsoft_exchange_buggy_patch/?td=keepreading" class=story_link> <div class="article_text_elements"> <h4>Microsoft Exchange update fixes security flaws, breaks other stuff</h4> <div class=standfirst>Flawed patch stops on-premises, hybrid server transport rules in their tracks for some</div> <div class=time_comments> <span class="section_name">Applications</span><span class="time_stamp" title="15 Nov 2024 12:29" data-epoch="1731673755">15 Nov 2024</span> | <span class="comment light_bg_comments">33</span></div> </div> </a> </article> <article> <a href="/2024/11/19/microsoft_365_copilot_symbol/?td=keepreading" class=story_link> <div class="article_text_elements"> <h4>Microsoft 365 Copilot trips over angle brackets, frustrating coders</h4> <div class=standfirst> <span class="label">updated</span> Chatbot seems to choke when fed tricky less-than character</div> <div class=time_comments> <span class="section_name">AI + ML</span><span class="time_stamp" title="19 Nov 2024 21:3" data-epoch="1732050224">19 Nov 2024</span> | <span class="comment light_bg_comments">22</span></div> </div> </a> </article> <article> <a href="/2024/11/19/microsoft_breaks_timezones_in_settings/?td=keepreading" class=story_link> <div class="article_text_elements"> <h4>Microsoft breaks timezones in Settings and calls on an unlikely ally for help</h4> <div class=standfirst>Control Panel hobbles to the rescue!</div> <div class=time_comments> <span class="section_name">Applications</span><span class="time_stamp" title="19 Nov 2024 18:25" data-epoch="1732040709">19 Nov 2024</span> | <span class="comment light_bg_comments">30</span></div> </div> </a> </article> <article> <a href="/2024/11/04/black_screens_windows/?td=keepreading" class=story_link> <div class="article_text_elements"> <h4>Black screens still plague Windows 10 Azure Virtual Desktop users</h4> <div class=standfirst>Meanwhile, even Task Manager is no match for the broken patch</div> <div class=time_comments> <span class="section_name">OSes</span><span class="time_stamp" title="4 Nov 2024 18:0" data-epoch="1730743210">4 Nov 2024</span> | <span class="comment light_bg_comments">8</span></div> </div> </a> </article> </div> </div> <div aria-hidden="true" class="adun" data-pos="btm" data-raptor="owl" data-xsm=",fluid,mpu,dmpu," data-sm=",fluid,mpu,dmpu," data-md=",fluid,mpu,dmpu," data-xmd=",fluid,leaderboard,mpu," data-lg=",fluid,mpu,leaderboard," data-xlg=",fluid,billboard,superleaderboard,mpu,leaderboard," data-xxlg=",fluid,billboard,superleaderboard,brandwidth,brandimpact,mpu,leaderboard,"></div> </div> </div><div id=footer> <div class="footer_slogan"> <div class="footer_wrapper"> <p>The Register <img class="vulture_icon" src="/design_picker/d518b499f8a6e2c65d4d8c49aca8299d54b03012/graphics/icon/vulture_white.png" alt="icon"> Biting the hand that feeds IT</p> </div> </div> <div class="footer_wrapper"> <div class=foot_wrapper> <div class="left_block"> <div class="foot_list"> <h4>About Us<img loading="lazy" width="7" height="11" alt="" src="/design_picker/d2e337b97204af4aa34dda04c4e5d56d954b216f/graphics/icon/footer_mob_nav_arrow_black.svg" class="expand_arrow"></h4> <ul> <li><a href="https://www.theregister.com/Profile/contact/">Contact us</a></li> <li><a target=_blank rel=noopener href="https://www.theregister.com/AdvertiseWithUs/">Advertise with us</a></li> <li><a href="https://www.theregister.com/Profile/about_the_register/">Who we are</a></li> </ul> </div> <div class="foot_list more_us"> <h4>Our Websites<img loading="lazy" width="7" height="11" alt="" src="/design_picker/d2e337b97204af4aa34dda04c4e5d56d954b216f/graphics/icon/footer_mob_nav_arrow_black.svg" class="expand_arrow"></h4> <ul> <li><a href="https://www.nextplatform.com/">The Next Platform</a></li> <li><a href="https://devclass.com/">DevClass</a></li> <li><a href="https://blocksandfiles.com/">Blocks and Files</a></li> </ul> </div> <div class="foot_list privacy"> <h4>Your Privacy<img loading="lazy" width="7" height="11" alt="" src="/design_picker/d2e337b97204af4aa34dda04c4e5d56d954b216f/graphics/icon/footer_mob_nav_arrow_black.svg" class="expand_arrow"></h4> <ul> <li><a href="https://www.theregister.com/Profile/cookies/">Cookies Policy</a></li> <li><a href="https://www.theregister.com/Profile/privacy/">Privacy Policy</a></li> <li><a href="https://www.theregister.com/Profile/terms_and_conditions_of_use/">Ts & Cs</a></li> </ul> </div> </div> <div class="right_block"> <div class="foot_list"> <a href="https://situationpublishing.com/" id="sitpub_logo"> <img loading="lazy" width="250" alt="Situation Publishing" src="/design_picker/d2e337b97204af4aa34dda04c4e5d56d954b216f/graphics/std/sitpublogo_2022.png"> </a> <p> Copyright. All rights reserved © 1998–2024 </p> </div> </div> <noscript><img width="1" height="1" src="/Design/graphics/std/transparent_pixel.png" alt="no-js"></noscript> </div> </div> </div> <div id=end_scripts> <script> if (typeof(ElReg.Ga.sendPageView) === 'function') { ElReg.Ga.sendPageView('reg_software/devops','0df13fad2ea597c71ae99fa84c3f976d','0df13fad2ea597c71ae99fa84c3f976d'); } </script> <script> $(function() { RegUtils.set_bucket_group(986) }); </script> </div> </div> </body> </html>

CINXE.COM

Typo blamed for Microsoft Azure DevOps outage in Brazil • The Register