CINXE.COM

<!DOCTYPE html>     <html lang="en-US">  <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width" /> <title>Ethernet at NANOG 92 | blabs</title> <link rel="profile" href="http://gmpg.org/xfn/11" /> <link rel="stylesheet" href="//maxcdn.bootstrapcdn.com/bootstrap/3.3.2/css/bootstrap.min.css"> <link rel="stylesheet" type="text/css" media="all" href="https://labs.apnic.net/blabs/wp-content/themes/apnic/style.css" /> <style> /* "nojs" css is embedded so it never fails to be applied. */ .js-off{display:none;} .nojs .js-on{display:none;} /* .js-on = show me if js is on/enabled */ .nojs .js-off{display:inherit;}/* .js-off = show me if js is off/disabled */ </style> <link rel="pingback" href="https://labs.apnic.net/blabs/xmlrpc.php" /> <meta name='robots' content='max-image-preview:large' /> <link rel="alternate" type="application/rss+xml" title="blabs » Feed" href="https://labs.apnic.net/index.php/feed/" /> <link rel="alternate" type="application/rss+xml" title="blabs » Comments Feed" href="https://labs.apnic.net/index.php/comments/feed/" /> <script type="text/javascript"> /* <![CDATA[ */ window._wpemojiSettings = {"baseUrl":"https:\/\/s.w.org\/images\/core\/emoji\/15.0.3\/72x72\/","ext":".png","svgUrl":"https:\/\/s.w.org\/images\/core\/emoji\/15.0.3\/svg\/","svgExt":".svg","source":{"concatemoji":"https:\/\/labs.apnic.net\/blabs\/wp-includes\/js\/wp-emoji-release.min.js?ver=6.7"}}; /*! This file is auto-generated */ !function(i,n){var o,s,e;function c(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(o,JSON.stringify(t))}catch(e){}}function p(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data),r=(e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0),new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data));return t.every(function(e,t){return e===r[t]})}function u(e,t,n){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\uddfa\ud83c\uddf3","\ud83c\uddfa\u200b\ud83c\uddf3")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!n(e,"\ud83d\udc26\u200d\u2b1b","\ud83d\udc26\u200b\u2b1b")}return!1}function f(e,t,n){var r="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):i.createElement("canvas"),a=r.getContext("2d",{willReadFrequently:!0}),o=(a.textBaseline="top",a.font="600 32px Arial",{});return e.forEach(function(e){o[e]=t(a,e,n)}),o}function t(e){var t=i.createElement("script");t.src=e,t.defer=!0,i.head.appendChild(t)}"undefined"!=typeof Promise&&(o="wpEmojiSettingsSupports",s=["flag","emoji"],n.supports={everything:!0,everythingExceptFlag:!0},e=new Promise(function(e){i.addEventListener("DOMContentLoaded",e,{once:!0})}),new Promise(function(t){var n=function(){try{var e=JSON.parse(sessionStorage.getItem(o));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(s),u.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"}),a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=function(e){c(n=e.data),a.terminate(),t(n)})}catch(e){}c(n=f(s,u,p))}t(n)}).then(function(e){for(var t in e)n.supports[t]=e[t],n.supports.everything=n.supports.everything&&n.supports[t],"flag"!==t&&(n.supports.everythingExceptFlag=n.supports.everythingExceptFlag&&n.supports[t]);n.supports.everythingExceptFlag=n.supports.everythingExceptFlag&&!n.supports.flag,n.DOMReady=!1,n.readyCallback=function(){n.DOMReady=!0}}).then(function(){return e}).then(function(){var e;n.supports.everything||(n.readyCallback(),(e=n.source||{}).concatemoji?t(e.concatemoji):e.wpemoji&&e.twemoji&&(t(e.twemoji),t(e.wpemoji)))}))}((window,document),window._wpemojiSettings); /* ]]> */ </script> <style id='wp-emoji-styles-inline-css' type='text/css'> img.wp-smiley, img.emoji { display: inline !important; border: none !important; box-shadow: none !important; height: 1em !important; width: 1em !important; margin: 0 0.07em !important; vertical-align: -0.1em !important; background: none !important; padding: 0 !important; } </style> <link rel='stylesheet' id='wp-block-library-css' href='https://labs.apnic.net/blabs/wp-includes/css/dist/block-library/style.min.css?ver=6.7' type='text/css' media='all' /> <style id='classic-theme-styles-inline-css' type='text/css'> /*! This file is auto-generated */ .wp-block-button__link{color:#fff;background-color:#32373c;border-radius:9999px;box-shadow:none;text-decoration:none;padding:calc(.667em + 2px) calc(1.333em + 2px);font-size:1.125em}.wp-block-file__button{background:#32373c;color:#fff;text-decoration:none} </style> <style id='global-styles-inline-css' type='text/css'> :root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #ffffff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgba(6,147,227,1) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgba(252,185,0,1) 0%,rgba(255,105,0,1) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgba(255,105,0,1) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgba(255, 255, 255, 1), 6px 6px rgba(0, 0, 0, 1);--wp--preset--shadow--crisp: 6px 6px 0px rgba(0, 0, 0, 1);}:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}:where(.wp-block-columns.is-layout-flex){gap: 2em;}:where(.wp-block-columns.is-layout-grid){gap: 2em;}:where(.wp-block-post-template.is-layout-flex){gap: 1.25em;}:where(.wp-block-post-template.is-layout-grid){gap: 1.25em;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;} :where(.wp-block-post-template.is-layout-flex){gap: 1.25em;}:where(.wp-block-post-template.is-layout-grid){gap: 1.25em;} :where(.wp-block-columns.is-layout-flex){gap: 2em;}:where(.wp-block-columns.is-layout-grid){gap: 2em;} :root :where(.wp-block-pullquote){font-size: 1.5em;line-height: 1.6;} </style> <link rel="https://api.w.org/" href="https://labs.apnic.net/index.php/wp-json/" /><link rel="alternate" title="JSON" type="application/json" href="https://labs.apnic.net/index.php/wp-json/wp/v2/posts/1936" /><link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://labs.apnic.net/blabs/xmlrpc.php?rsd" /> <meta name="generator" content="WordPress 6.7" /> <link rel="canonical" href="https://labs.apnic.net/index.php/2024/10/31/ethernet-at-nanog-92/" /> <link rel='shortlink' href='https://labs.apnic.net/?p=1936' /> <link rel="alternate" title="oEmbed (JSON)" type="application/json+oembed" href="https://labs.apnic.net/index.php/wp-json/oembed/1.0/embed?url=https%3A%2F%2Flabs.apnic.net%2Findex.php%2F2024%2F10%2F31%2Fethernet-at-nanog-92%2F" /> <link rel="alternate" title="oEmbed (XML)" type="text/xml+oembed" href="https://labs.apnic.net/index.php/wp-json/oembed/1.0/embed?url=https%3A%2F%2Flabs.apnic.net%2Findex.php%2F2024%2F10%2F31%2Fethernet-at-nanog-92%2F&format=xml" /> <style type="text/css">.recentcomments a{display:inline !important;padding:0 !important;margin:0 !important;}</style></head> <body class="post-template-default single single-post postid-1936 single-format-standard singular one-column content"> <div id="wrapper" class="nojs"> <script> // Embedded and no libs so it never fails to run (unless js is switched off). // Located here so there is no flicker in stuff being shown/hidden. document.getElementById('wrapper').className = document.getElementById('wrapper').className.replace(/nojs/gi,''); </script> <div class="navbar navbar-inverse navbar-fixed-top" role="navigation"> <div class="container"> <div class="navbar-header"> <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse"> <span class="sr-only">Toggle navigation</span> <span class="icon-bar"></span> <span class="icon-bar"></span> <span class="icon-bar"></span> </button> </div> <div class="collapse navbar-collapse"> <ul class="nav navbar-nav"> <li id="global-apnic-navbar-link-blog"><a href="//blabs.apnic.net">Labs Blog Posts</a></li> <li id="global-apnic-navbar-link-presentations"><a href="//labs.apnic.net/presentations">Labs Presentations</a></li> <li id="global-apnic-navbar-link-measurement"><a href="//labs.apnic.net/measurements">Measurements and Data</a></li> </ul> </div> </div> </div> <header id="site-banner" class="container"> <h2 class="apnic"><span>APNIC</span></h2> <h2 class="labs"><span>Labs</span></h2> </header> <div id="page-wrapper" class="container"> <div id="page"> <div id="contents"> <div class="row"> <div class="col-md-offset-1 col-md-10"> <div> <div class="menu-main-container"><ul id="menu-main" class="nav nav-tabs"><li id="menu-item-9" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-9"><a href="/">Home</a></li> <li id="menu-item-6" class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-6"><a href="https://labs.apnic.net/index.php/category/routing/">Routing</a></li> <li id="menu-item-7" class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-7"><a href="https://labs.apnic.net/index.php/category/security/">Security</a></li> <li id="menu-item-8" class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-8"><a href="https://labs.apnic.net/index.php/category/dns/">DNS</a></li> <li id="menu-item-10" class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-10"><a href="https://labs.apnic.net/index.php/category/ipv6/">IPv6</a></li> <li id="menu-item-11" class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-11"><a href="https://labs.apnic.net/index.php/category/policy/">Policy</a></li> <li id="menu-item-332" class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-332"><a href="https://labs.apnic.net/index.php/category/ip-addresses/">IP Addresses</a></li> </ul></div> </div> </div> </div> <div class="js-off alert alert-error"> <h3>Javascript is disabled</h3> <p>We would like to provide you with a better user experience. Please re-enable Javascript in your web browser.</p> </div> <div class="row"> <div class="col-md-offset-1 col-md-10"> <div id="primary"> <div id="content" role="main"> <nav id="nav-single"> <h3 class="assistive-text">Post navigation</h3> <span class="nav-previous"><a href="https://labs.apnic.net/index.php/2024/10/19/the-ipv6-transition/" rel="prev"><span class="meta-nav">←</span> Previous</a></span> <span class="nav-next"><a href="https://labs.apnic.net/index.php/2024/11/20/dns-oarc-43/" rel="next">Next <span class="meta-nav">→</span></a></span> </nav> <article id="post-1936" class="post-1936 post type-post status-publish format-standard hentry category-technology"> <header class="entry-header"> <h1 class="entry-title">Ethernet at NANOG 92</h1> <p class="entry-meta"> <span class="sep"></span><time class="entry-date" datetime="2024-10-31T00:16:37+00:00"> 31 Oct 2024</time> <span class="cat-links"> <span class="entry-utility-prep entry-utility-prep-cat-links">in</span> <a href="https://labs.apnic.net/index.php/category/technology/" rel="category tag">Technology</a> </span> by <span class="sep"></span><span class="author-name">Geoff Huston</span> </header> <div class="entry-content"> <p><meta http-equiv="Content-Type" content="text/html; charset=utf-8"></p> <style type="text/css">.small {font-size: smaller}.small5 {font-size: smaller; margin-left: 5em; text-align: left; font-family: monospace; }p {text-align: justify; }.quote {margin-left: 3em; text-align: justify; font-family: Verdana, Arial, Helvetica, sans-serif; }.sidenote2 { color: rgb(80,80,80); background-color: #F5F0DA; margin-right: 5px; margin-left: 140px; padding: 10px; border: 1px #DFCE9D solid; font-size: smaller; }pre {margin-left: 2em; padding: 3px; font-family: Menlo,"Courier New", Courier, mono; font-size: smaller; }.indent5 { margin-left: 5em; }.i5 { margin-left: 5em; font-size: smaller; }.indent5 { margin-left: 5em;}.indent3 { margin-left: 3em;}.hang5 { margin-left: 10em; text-indent: -5em; }p.caption5 {text-align: left; margin-left: 5em; font-family: Verdana, Arial, Helvetica, sans-serif; font-size: smaller; font-style:italic;}p.caption {text-align: center; font-family: Verdana, Arial, Helvetica, sans-serif; font-size: smaller; font-style:italic;}table.inline { border-width: 1px; border-spacing: ; border-style: none; border-color: gray; border-collapse: collapse;background-color: white; text-align: right ;background-color: white; }table.inline th { border-width: 1px; padding: 2px; border-style: inset; border-color: gray; background-color: white; -moz-border-radius: ; font-style: bold; text-align: left; font-size: smaller;}table.inline td { border-width: 1px; padding: 2px; border-style: inset; border-color: gray; background-color: white; -moz-border-radius: ; vertical-align: top ; text-align: right; font-size: smaller; }table.nonline { border-width: 0px; border-spacing: ; border-style: none; border-color: gray; border-collapse: collapse;background-color: white; text-align: left; background-color: white;}table.nonline td { vertical-align: top ; text-align: left; font-size: smaller; }.ttc { color: rgb(100,100,100); margin-left: 5em; font-family: "Lucida Console", "Courier New", Courier; text-align: left; }.tx { margin-left: 0em; font-family: "Lucida Console", "Courier New", Courier, mono; text-align: left; }.ttb { margin-left: 5em; font-family: "Courier New", Courier; text-align: left; font-size: smaller; }iframe {margin: 0; padding: 0; border: none;}</style> <p>Ethernet has been the mainstay of much of the networking environment for almost 50 years now, but that doesn’t mean that it’s remained unchanged over that period. The evolution of this technology has featured continual increases in the scale of Ethernet networks, increasing in capacity, reach and connections. I’d like to report on a couple of Ether-related presentations that took place at the recent NANOG 92 meeting, held in Toronto in October 2024 that described some recent developments in Ethernet.</p> <h3>Towards 800Ge</h3> <p>The original 1982 10Mbps Ethernet is a distant memory these days. A little over a decade later we saw a 100Mbs standard, and five years after that, in 1999, there was Gigabit Ethernet (1GbE). 10GbE followed after a further five years, and 100GbE in 2010. 400GbE was defined in 2017.</p> <p>Much of this has occurred by realising the potential from increasing gate density on silicon chips. Aggregate Switching capacity has increased by a factor of 16 in the past decade, from 3.2Tbps to 51.2Tbps. This has been accompanied by an increase in the number of ports per device and the individual port capacity. (Figure 1)</p> <p class="caption5"><img decoding="async" src="https://www.potaroo.net/ispcol/2024-10/ethernet-fig1.png" width=90%><br />Figure 1 – Switch Capacity over Time – From “Progress update on IEEE’s next generation Ethernet project (P802.3dj – 800 GbE & 1.6 TbE)” by Mark Nowell – Cisco Systems, NANOG 92</p> <p>Today’s Ethernet systems achieve high capacity by using a parallel approach. The initial 100Gb Ethernet systems used 10 lanes each running at 10Gbps. When the standard for 25Gbps Ethernet was completed, the same 100Gbps system could be constructed using 4 x 25Gbps lanes. This generic approach of using parallel lanes to achieve higher aggregate network capacity is shown in Figure 2.</p> <p class="caption5"><img decoding="async" src="https://www.potaroo.net/ispcol/2024-10/ethernet-fig2.png" width=90%><br />Figure 2 – Ethernet Lanes and Capacity over Time – From “Progress update on IEEE’s next generation Ethernet project (P802.3dj – 800 GbEto 1.6 TbE)” by Mark Nowell – Cisco Systems, NANOG 92</p> <p>The foundation of today’s production Ethernet networks lies in 100Gbps lanes. When paired up they provider a 200Gbps service, and in groups of four they provide today’s 400Gbs Ethernet service.</p> <p>The IEEE’s P802.3dj work is to define a standard specification for a base capacity of a 200Gbps Ethernet lane, and define the combining of 2, 4, and 8 such lanes, to provide 400GbE, 800GbE and 1.6TbE Ethernet interfaces. The associated electrical interfaces are specifications of a 200Gbs serial interface, define for copper cables, backplanes and chip-to-module (AUI) interfaces. For optical cable a 200Gbps interface is only defined for single mode cable. For short spans between 500m and 10km a 200Gbps signal is carried using IMDD (Intensity Modulation/Direct Detection) using on-off keying (OOK) modulation. Longer distances (10km to 40km) are specified in the standard specification using 800Gps per lane with coherent signalling with 16QAM (phase amplitude keying).</p> <p>This P802.3dj specification covers most of the current specifications for 100Gbps per lane Ethernet with provision for up to four concurrent lanes, operating over copper cables and fibre cable plant, and is intended to be compatible with current systems.</p> <p>One of the major changes in the 200Gbps per lane specification is the addition of new (larger) Forward Error Correction (DEC) schemes. FEC schemes used for 100Gbps and slower is retained, and a new FEC scheme is defined for 200Gbps per lane.</p> <p>Much effort has been spent on maintaining backward compatibility with existing Ethernet plane, as shown in Figure 3.</p> <p class="caption5"><img decoding="async" src="https://www.potaroo.net/ispcol/2024-10/ethernet-fig3.png" width=90%><br />Figure 3 – Key Elements of 200 GbE – From “Progress update on IEEE’s next generation Ethernet project (P802.3dj – 800 GbE to 1.6 TbE)” by Mark Nowell – Cisco Systems, NANOG 92</p> <p>Where does it go from here?</p> <p>Increasing the number of lanes provides a performance gain, but it comes at a linear cost multiplier. While multiple lanes increase the aggregate capacity of the network, individual streams still operate within the capacity constraints of an individual lane.</p> <p>Increasing the per lane capacity of a system offers the potential of greater cost performance of the system. These days the increasing per lane speeds require faster signal modulation drivers and more capable signal detector systems what operate reliably at lower signal to noise levels. This is achieved by using more complex digital signal processors, which require increased gate counts on the asic chips and also generally need more power to operate, which in turn imposes some constraints on the physical packaging of the network interface devices. It appears to come down to silicon capabilities (Figure 4).</p> <p class="caption5"><img decoding="async" src="https://www.potaroo.net/ispcol/2024-10/ethernet-fig4.png" width=90%><br />Figure 4 – Evolution of Fibre Capacity</p> <p>There is still some potential to increase the base speed by extending the signal encoding to use more discrete points in the phase/amplitude space, but it appears that to do so we need to use digital signal processors in chips built using a 3nm process, and this level of size reduction also implies the use of more than simple planar gate design on the chip, such as GAAS and other related approaches that increase the junction areas within each gate within the constraints of a narrow tack width.</p> <p>At this stage is not does appear to be outlandish to expect per-lane capacities to increase from 200G to 400G in the coming years. Even higher speeds appear to rely on the use of as-yet undefined silicon chip technologies that can offer the combination of high yield, reliable operation, increased gate density and more efficient power management.</p> <h3>Ultra Ethernet for AI</h3> <p>There is more to designing performance for very high capacity local network environments than just the headline number of per-lane speeds. Understanding the nature of the workload that the network has to support, and tailoring the network to match that workload is also important. Much of the scaling effort in today’s network infrastructure is focussed on the demands being made by large-scale AI platforms.</p> <p>In very general terms, a local AI network is a large set of GPUs and a set of memory banks, and the network is cast into the role of a distributed common bus to interconnect GPUs to memory in a similar vein that the backplane of the old mainframe computer design was used to connect a number of processing engines to a set of storage banks (Figure 5). Here Remote Memory Access performance is critical, and the approach, in very general terms, is to enable the network to pass packet payloads direct to the hardware modules that write to memory, and similarly assemble packets to send by reading directly from memory. The current way to achieve this is by RoCE, or Remote Direct Memory Access (RDMA) over Converged Ethernet.</p> <p class="caption5"><img decoding="async" src="https://www.potaroo.net/ispcol/2024-10/ethernet-fig5.png" width=90%><br />Figure 5 – Ultra Ethernet Reference network – from “Networking for AI and HPC, and Ultra Ethernet”, By Hugh Holbrook, Arista Networks</p> <p>Network-intensive applications like AI, networked storage or cluster computing need a network infrastructure with a high bandwidth and low latency. The advantages of RoCE over other network application programming interfaces such as the socket abstraction are lower latency, lower processing overheads and higher bandwidth. However, the concept of using the network for remote memory (Remote Memory Access, or RMA) where it is possible to transfer from one device’s memory to another without involving the operating systems of the two devices is not a new one, and RMA has been around for some 25 years, often seen in parallel computing clusters. However, these approaches had their shortcomings, including the lack of explicit support for multi-pathing within the network (which constrained the network service to strict in-order delivery), poor recovery from single packet failure and unwieldy congestion control mechanisms.</p> <p>Ultra Ethernet is not a new media layer for Ethernet, but is intended to be an evolution for the RMA protocol for larger and more demanding network clusters. The key changes are the support for multi-pathing in the network and an associated relaxing of the strict in-der packet delivery requirement. It also uses rapid loss recovery and improved congestion control. In short, it appears to apply the behaviours of today’s high performance multi-path transport-level protocols to the RMA world. The idea is to use a protocol that can make effective use of the mesh topology used in high performance data centre networks to allow the scaling of the interconnection of GPUs with storage, computation and distribution networks.</p> <p>A critical part of this approach lies in multi-path support which is based on dropping the requirement for strict in-order packet delivery. Each packet is tagged with its ultimate memory address, allowing arriving packets to be placed directly into memory without any order-based blocking. However, this does place a higher burden on loss detection within the overall RMA architecture. Ultra Ethernet replaces silent packet discard with a form of signalled discard, similar to the ECN signalling mechanisms used in L4S in TCP. If a packet cannot be queued in a switch because the queue is fully occupied, the packet is trimmed to a 64-octet header snippet and this snippet is placed into a high priority send queue. Reception of such a trimmed packet causes the receiver to explicitly request re-transmission from the sender for the missing packet, which is analogous to selective acknowledgment (SACK) signalling used in TCP.</p> <p>UE also proposes the use of fast startup, including adding data payloads to the initial handshake used to establish a flow. This approach eliminates the delay of a round-trip handshake before transmitting, as the connection is established on-demand by the first data packet.</p> <p>The challenge for many networking protocols is to be sufficiently generic to be useful in a broad diversity of environments. If you are permitted to look at optimising the performance of the protocol in a far more limited scenario then it’s possible to introduce further optimisations for the protocol. That is what is going on for UE, which takes the more generic approach of RMA over IP and makes some specific assumptions about the environment and the payload profile that are specific to high performance data centres used for data-intensive processing, as seen in AI applications. It’s an interesting approach to scaling in data centres, as it does not attempt to alter the underlying Ethernet behaviours, but pulls in much of the experience gained from high performance TCP and applying it directly to an Ethernet packet RMA management library.</p> </div> <footer class="entry-meta"> This entry was posted in <a href="https://labs.apnic.net/index.php/category/technology/" rel="category tag">Technology</a> by <a href="https://labs.apnic.net/index.php/author/gih/">Geoff Huston</a>. Bookmark the <a href="https://labs.apnic.net/index.php/2024/10/31/ethernet-at-nanog-92/" title="Permalink to Ethernet at NANOG 92" rel="bookmark">permalink</a>. </footer> </article> </div> </div> </div> </div> </div> </div> </div>  <div id="footer"> <div class="container"> <div class="row"> <div class="collapse navbar-collapse"> <ul class="nav navbar-nav"> <li id="global-apnic-navbar-link-apnic"><a href="//www.apnic.net/">APNIC</a></li> <li id="global-apnic-navbar-link-myapnic"><a href="//myapnic.net/">MyAPNIC</a></li> <li id="global-apnic-navbar-link-training"><a href="//training.apnic.net/">Training</a></li> <li id="global-apnic-navbar-link-conferences"><a href="//conference.apnic.net/">Conferences</a></li> <li id="global-apnic-navbar-link-icons" class="active"><a href="https://labs.apnic.net/">Labs</a></li> </ul> </div> <div class="col-sm-5 col-xs-6 col-left"> <div id="footer-stamp"> <p><strong>APNIC</strong><br/> Asia Pacific Network<br/> Information Centre</p> </div> </div> <div class="col-sm-3 col-xs-6 col-middle"> <div id="footer-contact"> <h2 class="h4">Contact us</h2> <p><a href="mailto:research@apnic.net">research@apnic.net</a><br/>Tel: +61 7 3858 3188</p> </div> </div> <div class="col-sm-4 text-right col-right"> <ul class="list-inline"> <li><a href="//www.youtube.com/user/apnicmultimedia"><img src="//labs.apnic.net/template-files/social-media/32/youtube.png" width="25" alt="YouTube" title="YouTube"/></a></li><li><a href="//www.flickr.com/photos/apnictraining/"><img src="//labs.apnic.net/template-files/social-media/32/flickr.png" width="25" alt="Flickr" title="Flickr"/></a></li><li><a href="//www.facebook.com/APNIC"><img src="//labs.apnic.net/template-files/social-media/32/facebook.png" width="25" alt="Facebook" title="Facebook"/></a></li><li><a href="//twitter.com/apnic"><img src="//labs.apnic.net/template-files/social-media/32/twitter.png" width="25" alt="Twitter" title="Twitter"/></a></li><li><a href="//www.linkedin.com/company/apnic"><img src="//labs.apnic.net/template-files/social-media/32/linkedin.png" width="25" alt="LinkedIn" title="LinkedIn"/></a> </li><li><a href="https://www.apnic.net/apnic-info/rss/apnic-news"><img src="https://labs.apnic.net/template-files/social-media/32/rss.png" width="25" alt="RSS" title="RSS"/></a></li> </ul> <div id="footer-copyright"> <p class="text-right">© 2015 APNIC | <a href="https://www.apnic.net/apnic-info/privacy">Privacy</a></p> </div> </div> </div> </div> </div> </div>  </div> </body> </html>