Compute Trends Across Three Eras of Machine Learning

<!DOCTYPE html> <html lang="en"><head>  <script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= 'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-TB9Q2NN5');</script>  <meta charset="utf-8"> <meta http-equiv="X-UA-Compatible" content="IE=edge"> <meta name="viewport" content="width=device-width, initial-scale=1">  <meta name="generator" content="Jekyll v4.2.2" /> <meta property="og:title" content="Compute Trends Across Three Eras of Machine Learning" /> <meta name="author" content="Jaime Sevilla" /> <meta property="og:locale" content="en_US" /> <meta name="description" content="We’ve compiled a comprehensive dataset of the training compute of AI models, providing key insights into AI development." /> <meta property="og:description" content="We’ve compiled a comprehensive dataset of the training compute of AI models, providing key insights into AI development." /> <link rel="canonical" href="https://epoch.ai/blog/compute-trends" /> <meta property="og:url" content="https://epoch.ai/blog/compute-trends" /> <meta property="og:site_name" content="Epoch AI" /> <meta property="og:image" content="https://epoch.ai/assets/images/posts/2022/compute-trends.png" /> <meta property="og:type" content="article" /> <meta property="article:published_time" content="2022-02-16T00:00:00+00:00" /> <meta name="twitter:card" content="summary_large_image" /> <meta property="twitter:image" content="https://epoch.ai/assets/images/posts/2022/compute-trends.png" /> <meta property="twitter:title" content="Compute Trends Across Three Eras of Machine Learning" /> <script type="application/ld+json"> {"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"Jaime Sevilla"},"dateModified":"2022-02-16T00:00:00+00:00","datePublished":"2022-02-16T00:00:00+00:00","description":"We’ve compiled a comprehensive dataset of the training compute of AI models, providing key insights into AI development.","headline":"Compute Trends Across Three Eras of Machine Learning","image":"https://epoch.ai/assets/images/posts/2022/compute-trends.png","mainEntityOfPage":{"@type":"WebPage","@id":"https://epoch.ai/blog/compute-trends"},"url":"https://epoch.ai/blog/compute-trends"}</script>  <meta property="article:tag" content="Trends"> <meta property="article:tag" content="Compute"> <meta property="article:author" content="Jaime Sevilla"> <meta property="article:author" content="Lennart Heim"> <meta property="article:author" content="Anson Ho"> <meta property="article:author" content="Tamay Besiroglu"> <meta property="article:author" content="Marius Hobbhahn"> <meta property="article:author" content="Pablo Villalobos"> <title>Compute Trends Across Three Eras of Machine Learning | Epoch AI</title> <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.8.1/font/bootstrap-icons.css" crossorigin="anonymous"> <link rel="stylesheet" href="/assets/css/micromodal.css"> <link rel="shortcut icon" type="image/png" href="/assets/images/favicon.png" > <link rel="icon" type="image/x-icon" href="/assets/images/favicon.ico" > <link rel="icon" type="image/svg+xml" href="/assets/images/favicon.svg" > <link rel="alternate" type="application/rss+xml" title="Epoch AI" href="/atom.xml" />  <script type="text/x-mathjax-config"> MathJax.Hub.Config({ tex2jax: { inlineMath: [ ['$','$'], ["\$","\$"] ], processEscapes: true }, }); </script> <script id="MathJax-script" async src="https://cdn.jsdelivr.net/npm/mathjax@3.0.1/es5/tex-mml-chtml.js"></script> <script async src="https://ga.jspm.io/npm:es-module-shims@1.8.2/dist/es-module-shims.js"></script> <link rel="stylesheet" href="/assets/css/main.css"> <script src="/assets/js/micromodal.min.js"></script>  <script src="/assets/js/clipboard.min.js"></script> <script> 'use strict'; // TODO Temporary, compress function addCopyButton(node) { let copyButton; if (node.tagName == 'PRE') { copyButton = document.createElement('button'); node.parentElement.appendChild(copyButton); node.parentElement.classList.add('copiable-wrapper'); node.classList.add('copy-target'); copyButton.classList.add('copy-button'); //copyButton.innerHTML = '<i class="bi-clipboard copy-icon"></i>'; copyButton.innerHTML = '<img class="copy-icon" src="/assets/images/icons/copy-icon.svg">'; } else { let wrapper = document.createElement('div'); wrapper.classList.add('copiable-wrapper'); node.parentNode.insertBefore(wrapper, node); wrapper.appendChild(node); copyButton = document.createElement('button'); node.parentElement.appendChild(copyButton); node.parentElement.classList.add('copiable-wrapper'); node.classList.add('copy-target'); copyButton.classList.add('copy-button'); //copyButton.innerHTML = '<i class="bi-clipboard copy-icon"></i>'; copyButton.innerHTML = '<img class="copy-icon" src="/assets/images/icons/copy-icon.svg">'; } copyButton.querySelector('.copy-icon').style.backgroundColor = node.style.backgroundColor; let clipboard = new ClipboardJS(copyButton, { target: function(trigger) { return trigger.parentElement.querySelector('.copy-target'); }, }); let tooltip = tippy(copyButton, { content: 'Copied', theme: 'epoch-default', trigger: 'manual', placement: 'left', appendTo: copyButton, arrow: false, offset: [1, -1], }); clipboard.on('success', function(e) { e.clearSelection(); let icon = e.trigger.querySelector('.copy-icon'); icon.classList.remove('bi-clipboard'); icon.classList.add('bi-clipboard-check'); tooltip.show(); setTimeout(() => { tooltip.hide(); icon.classList.remove('bi-clipboard-check'); icon.classList.add('bi-clipboard'); }, 1200); }); clipboard.on('error', function(e) { e.clearSelection(); }); } window.addEventListener('DOMContentLoaded', () => { for (let element of document.querySelectorAll('pre, .boxed-text')) { addCopyButton(element); } }); </script> <script> 'use strict'; let today = new Date(); let currentDateStr = today.toISOString().split('T')[0]; let currentYearStr = today.getUTCFullYear().toString(); // Format: 31 Jul 2021 let currentDateCitationStr = today.getUTCDate() + ' ' + [ 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec' ][today.getUTCMonth()] + ' ' + currentYearStr; document.addEventListener('DOMContentLoaded', function() { injectCurrentDates(); }); function injectCurrentDates(container=document) { for (let node of container.querySelectorAll('.current-date-unfilled')) { if (node.dataset.citationFormat == 'true') { node.innerHTML = currentDateCitationStr; } else { node.innerHTML = currentDateStr; } node.classList.remove('current-date-unfilled'); } for (let node of container.querySelectorAll('.current-year-unfilled')) { node.innerHTML = currentYearStr; node.classList.remove('current-year-unfilled'); } } </script> <script> 'use strict'; let handleSelectableLinks; document.addEventListener('DOMContentLoaded', function() { let onTouchDevice = matchMedia('(hover: none)').matches; // // Jump on click const delta = 10; let mouseAnchor = {x: null, y: null}; handleSelectableLinks = function(container) { for (let node of container.querySelectorAll('[data-selectable-link]')) { node.addEventListener('mousedown', (e) => { mouseAnchor.x = e.pageX; mouseAnchor.y = e.pageY; }); node.addEventListener('mouseup', (e) => { let dist = (mouseAnchor.x - e.pageX)**2 + (mouseAnchor.y - e.pageY)**2; if (dist > delta) { return; } let clickEvent = new MouseEvent("click", e) node.dispatchEvent(clickEvent); if (e.target.tagName == 'A' || clickEvent.defaultPrevented) { return; } if (e.target.classList.contains('uncertainty') || e.target.classList.contains('info-icon')) { return } // True click! Follow the link. if (node.dataset.newTab || e.ctrlKey) { window.open(node.dataset.selectableLink, '_blank'); } else { window.location.href = node.dataset.selectableLink; } }); } } handleSelectableLinks(document); }); </script> <script> 'use strict'; document.addEventListener('DOMContentLoaded', () => { document.body.addEventListener('click', (e) => { if (e.target.tagName === 'A' && e.target.href.startsWith(window.location.origin)) { // Freeze the navigation bar until we stop scrolling freezeNavBar(); unfreezeNavBarOnScrollStop(); } }); }); </script>    <script> /*This function will load script and call the callback once the script has loaded*/ function loadScriptAsync(scriptSrc, callback) { if (typeof callback !== 'function') { throw new Error('Not a valid callback for async script load'); } var script = document.createElement('script'); script.onload = callback; script.src = scriptSrc; document.head.appendChild(script); } /* This is the part where you call the above defined function and "call back" your code which gets executed after the script has loaded */ loadScriptAsync('https://www.googletagmanager.com/gtag/js?id=G-2JLB2RT2V0', function () { window.dataLayer = window.dataLayer || []; function gtag() { dataLayer.push(arguments); } gtag('consent', 'default', { 'ad_storage': 'denied', 'analytics_storage': 'granted', 'ad_user_data': 'denied', 'ad_personalization': 'denied', 'wait_for_update': 500, }); gtag('js', new Date()); gtag('config', 'G-2JLB2RT2V0', { 'anonymize_ip': true }); }) // Handle user consent function setConsent(consent) { window.dataLayer = window.dataLayer || []; function gtag() { dataLayer.push(arguments); } gtag('consent', 'update', { 'ad_storage': consent ? 'granted' : 'denied', 'analytics_storage': consent ? 'granted' : 'denied', 'ad_user_data': consent ? 'granted' : 'denied', 'ad_personalization': consent ? 'granted' : 'denied', }); } </script>  <script type="text/javascript"> (function(c,l,a,r,i,t,y){ c[a]=c[a]||function(){(c[a].q=c[a].q||[]).push(arguments)}; t=l.createElement(r);t.async=1;t.src="https://www.clarity.ms/tag/"+i; y=l.getElementsByTagName(r)[0];y.parentNode.insertBefore(t,y); })(window, document, "clarity", "script", "mrxdks5ty9"); </script> <script> 'use strict'; // Deobfuscate email addresses function addMailDeobfuscator(element) { element.addEventListener('mouseover', () => element.href = 'mailto:' + atob(element.dataset.contact)); element.addEventListener('focus', () => element.href = 'mailto:' + atob(element.dataset.contact)); } window.addEventListener('load', () => { for (let element of document.querySelectorAll("[data-contact]")) { addMailDeobfuscator(element); if (element.dataset.contactSetText == "true") { element.innerHTML = atob(element.dataset.contact); } } }); </script> <script src="https://unpkg.com/@popperjs/core@2.11.2"></script> <script src="https://unpkg.com/tippy.js@6.2.6"></script> <link rel="stylesheet" href="/assets/css/handorgel.min.css"> <script src="/assets/js/handorgel.min.js"></script> <script> "use strict"; let getAccordionByNode; let makeFoldable; let makeUnfoldable; let followInternalLink; { let accordions = []; let nodeToAccordion = new Map(); getAccordionByNode = function(node) { return nodeToAccordion.get(node); }; makeUnfoldable = function(container) { if (!container.classList.contains('fold-container')) { return; } container.classList.remove('fold-container'); for (let accordion of accordions) { if (accordion.element === container) { accordion.destroy(); accordions.splice(accordions.indexOf(accordion), 1); break; } } nodeToAccordion.delete(handorgelDiv); let nodes = []; for (let handorgelHeader of container.querySelectorAll('.handorgel__header')) { let header = document.createElement(handorgelHeader.dataset.headerType); handorgelHeader.querySelector('handorgel__header__icon').remove(); header.innerHTML = handorgelHeader.querySelector('.handorgel__header__button').innerHTML; header.id = handorgelHeader.id; let content = handorgelHeader.nextElementSibling; let section = document.createElement('div'); section.appendChild(header); for (let child of content.children) { section.appendChild(child); } section.classList.add('folded-section'); nodes.push(section); } container.innerHTML = ''; for (let node of nodes) { container.appendChild(node); } }; makeFoldable = function(container, openByDefault=false) { if (container.classList.contains('fold-container')) { return; } container.classList.add('fold-container'); let handorgelDiv = document.createElement('div'); handorgelDiv.classList.add('handorgel'); let openByDefaultArray = []; // fold-wise for (let section of container.querySelectorAll('.folded-section')) { let header = section.querySelector('h1,h2,h3,h4,h5,h6'); let button = document.createElement('button'); if (container.dataset.foldOpenIcon) { button.innerHTML = header.innerHTML + ' <img class="handorgel__header__icon" src="' + container.dataset.foldOpenIcon + '">'; } else { button.innerHTML = header.innerHTML + ' <img class="handorgel__header__icon" src="/assets/images/icons/chevron-down.svg">'; } openByDefaultArray.push(section.dataset.openByDefault); button.classList.add('handorgel__header__button'); button.id = header.id; // Handorgel overwrites the h* id :_( header.classList.add('handorgel__header'); header.innerHTML = ''; header.dataset.headerType = header.tagName; header.appendChild(button); handorgelDiv.appendChild(header); let content = document.createElement('div'); content.classList.add('handorgel__content'); for (let clazz of section.classList) { content.classList.add(clazz); } if (openByDefault) { content.dataset.open = 'true'; } content.dataset.close = true; let innerDiv = document.createElement('div'); content.appendChild(innerDiv); for (let element of [...section.children]) { innerDiv.appendChild(element); } handorgelDiv.appendChild(content); button.addEventListener('click', () => { // If the section header goes out of view, scroll it back into view setTimeout(() => { if (header.getBoundingClientRect().top < 0) { // Get scroll margin let margin = parseInt(getComputedStyle(button).scrollMarginTop); //header.scrollIntoView({behavior: 'smooth'}); window.scrollBy({ top: header.getBoundingClientRect().top - margin, behavior: 'smooth', }); } }, 400); }); } container.innerHTML = ''; container.appendChild(handorgelDiv); let options = { multiSelectable: container.dataset.multiSelectable != 'false', collapsible: container.dataset.collapsible != 'false', initialOpenTransition: false, }; let accordion = new handorgel(handorgelDiv, options); nodeToAccordion.set(handorgelDiv, accordion); accordions.push(accordion); for (let content of document.querySelectorAll('.handorgel .handorgel__content')) { // To remove margins markTopNodes(content.children[0]); markBottomNodes(content.children[0]); } accordion.on('fold:open', function (e) { setTimeout(() => { // The browser won't play the animation for the margin or padding bottom // starting from height 0, so we need to wait for the height to change // before we can set the margin/padding. e.content.classList.add('expanded'); if (container.dataset.foldCloseIcon) { e.button.querySelector('.handorgel__header__icon').src = container.dataset.foldCloseIcon; } }, 0); }); accordion.on('fold:close', function (e) { e.content.classList.remove('expanded'); if (container.dataset.foldOpenIcon) { e.button.querySelector('.handorgel__header__icon').src = container.dataset.foldOpenIcon; } }); // Open the first one for (let i = 0; i < accordion.folds.length; i++) { let fold = accordion.folds[i]; if (openByDefaultArray[i]) { fold.open({transition: false}); } } }; function markTopNodes(node) { node.classList.add('top-node'); if (node.children.length > 0) { markTopNodes(node.children[0]); } } function markBottomNodes(node) { node.classList.add('bottom-node'); if (node.children.length > 0) { markBottomNodes(node.children[node.children.length - 1]); } } function scrollTo(node) { node.scrollIntoView({ behavior: 'smooth', }); } function openFold(accordion, fold, scrollTarget) { if (!scrollTarget) scrollTarget = fold.header; if (!fold.expanded) { accordion.once('fold:opened', (f) => { }); fold.open({transition: false}); } } followInternalLink = function(href) { let id = href.slice(1); // Is it a fold? for (let accordion of accordions) { for (let fold of accordion.folds) { if (fold.button.id == id) { openFold(accordion, fold); return true; } } } // Is it _inside_ a fold? let element = document.getElementById(id); if (!element) return false; let node = element; while (node) { for (let accordion of accordions) { for (let fold of accordion.folds) { if (fold.button == node || fold.content == node) { openFold(accordion, fold, element); return true; } } } node = node.parentElement; } return false; }; document.addEventListener('click', (e) => { if (e.target.tagName == 'A' && e.target.hasAttribute('href') && !e.shiftKey && !e.ctrlKey && !e.metaKey && !e.altKey) { const newUrl = new URL(e.target.href); const currentUrl = new URL(window.location.href); if (newUrl.pathname == currentUrl.pathname && newUrl.hash) { followInternalLink(newUrl.hash); } } }); // On state restoration, follow the link window.addEventListener('popstate', (e) => { const newUrl = new URL(window.location.href); if (newUrl.hash) { followInternalLink(newUrl.hash); } }); } </script> <script src="/assets/js/footnotes.js"></script> <link rel="stylesheet" href="/assets/css/post.css"> <link rel="stylesheet" href="/assets/components/update-note/update-note.css"> <link rel="stylesheet" href="/assets/components/toc/toc.css"> <link rel="stylesheet" href="/assets/components/shares-div/shares-div.css"> <link rel="stylesheet" href="/assets/components/related-work-card/related-work-card.css"> <link rel="stylesheet" href="/assets/components/citation-modal/citation-modal.css"> <script src="/assets/components/toc/toc.js"></script> <script src="/assets/components/citation-modal/citation-modal.js"></script> </head> <noscript> <style> body { opacity: 1 !important; } </style> </noscript> <body style="opacity: 0"><script src="/assets/js/headroom.min.js"></script> <script src="/assets/js/dist/epoch-ui.js"></script> <div id="top-nav-bar"> <div class="top-strip"> <div class="navigation"> <div class="logo-container"> <a class="logo" href="/"> <div class="round"> <img class="logo-normal" src="/assets/images/epoch-full-standard.svg" alt="Epoch AI's logo"> <img class="logo-white" src="/assets/images/epoch-full-white.svg" alt="Epoch AI's logo"> </div> </a> </div> <div class="wide-menu"> <div class="menu-item "> <a href="/blog" data-submenu-id="Publications" class="menu-item-link"> <span class="menu-item-label">Publications </span> </a> </div> <div class="menu-item "> <a href="/gradient-updates" data-submenu-id="Gradient Updates" class="menu-item-link"> <span class="menu-item-label">Gradient Updates </span> </a> </div> <div class="menu-item has-dropdown"> <a href="#" data-submenu-id="Data & Resources" class="menu-item-link"> <span class="menu-item-label">Data & Resources <svg width="13" height="9" class="chevron" viewBox="0 0 13 9" fill="none" xmlns="http://www.w3.org/2000/svg"> <path d="M0.5 1L6.5 7.5L12.5 1" stroke="#07171A" stroke-linecap="round" /> </svg></span> </a> <div class="submenu" data-submenu-id="Data & Resources"> <a href="/data" class="submenu-item">Data on AI</a> <a href="/trends" class="submenu-item">Key Trends & Insights</a> </div> </div> <div class="menu-item has-dropdown"> <a href="#" data-submenu-id="Projects" class="menu-item-link"> <span class="menu-item-label">Projects <svg width="13" height="9" class="chevron" viewBox="0 0 13 9" fill="none" xmlns="http://www.w3.org/2000/svg"> <path d="M0.5 1L6.5 7.5L12.5 1" stroke="#07171A" stroke-linecap="round" /> </svg></span> </a> <div class="submenu" data-submenu-id="Projects"> <a href="/frontiermath" class="submenu-item">FrontierMath</a> <a href="/tools/distributed-training" class="submenu-item">Distributed Training</a> </div> </div> <div class="menu-item has-dropdown"> <a href="#" data-submenu-id="About" class="menu-item-link"> <span class="menu-item-label">About <svg width="13" height="9" class="chevron" viewBox="0 0 13 9" fill="none" xmlns="http://www.w3.org/2000/svg"> <path d="M0.5 1L6.5 7.5L12.5 1" stroke="#07171A" stroke-linecap="round" /> </svg></span> </a> <div class="submenu" data-submenu-id="About"> <a href="/about" class="submenu-item">About Us</a> <a href="/research" class="submenu-item">About Our Research</a> <a href="/careers" class="submenu-item">Careers</a> <a href="/support-us" class="submenu-item">Support Us</a> </div> </div> <div class="menu-item "> <a href="mailto:info@epoch.ai" data-submenu-id="Contact" class="menu-item-link"> <span class="menu-item-label">Contact </span> </a> </div> <a href="/search" class="menu-item search-menu-item"> <div class="loupe-container"> <svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"> <path d="M14.1931 5.58187C16.525 7.91369 16.525 11.6943 14.1931 14.0261C11.8613 16.358 8.08069 16.358 5.74887 14.0261C3.41704 11.6943 3.41704 7.91369 5.74887 5.58187C8.08069 3.25005 11.8613 3.25005 14.1931 5.58187" stroke="#091D21" stroke-linecap="round" stroke-linejoin="round"/> <path d="M14.1501 14.06L20.0001 19.99" stroke="#091D21" stroke-linecap="round" stroke-linejoin="round"/> </svg> </div> </a> </div> <div class="narrow-menu"> <div class="narrow-menu-contents"> <div class="narrow-menu-bundle"> <a href="/search" class="menu-item search-menu-item"> <div class="loupe-container"> <svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"> <path d="M14.1931 5.58187C16.525 7.91369 16.525 11.6943 14.1931 14.0261C11.8613 16.358 8.08069 16.358 5.74887 14.0261C3.41704 11.6943 3.41704 7.91369 5.74887 5.58187C8.08069 3.25005 11.8613 3.25005 14.1931 5.58187" stroke="#091D21" stroke-linecap="round" stroke-linejoin="round"/> <path d="M14.1501 14.06L20.0001 19.99" stroke="#091D21" stroke-linecap="round" stroke-linejoin="round"/> </svg> </div> </a> <div id="open-menu-button" class=" narrow-menu-container"> <svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"> <path d="M3 12H21M3 6H21M3 18H21" stroke="#162B32" stroke-linecap="round" stroke-linejoin="round"/> </svg> </div> </div> </div> </div> </div> </div> </div> <div class="sidebar"> <div class="menu-header"> <div class="logo-container"> <a class="logo" href="/"> <div class="round"> <img class="logo-normal" src="/assets/images/epoch-full-standard.svg" alt="Epoch AI's logo"> <img class="logo-white" src="/assets/images/epoch-full-white.svg" alt="Epoch AI's logo"> </div> </a> </div> <button class="close-button"> <svg width="14" height="14" viewBox="0 0 14 14" fill="none" xmlns="http://www.w3.org/2000/svg"> <path d="M13 1L1 13M1 1L13 13" stroke="#162B32" stroke-linecap="round" stroke-linejoin="round"/> </svg> </button> </div> <div class="menu-list-container"> </div> </div> <div class="sidebar-overlay"></div> <script> 'use strict'; let addNavBarAnimationListener; let addNavBarDisplacement; let freezeNavBar = null; let unfreezeNavBar = null; let unfreezeNavBarOnScrollStop = null; document.documentElement.style.setProperty('--current-header-height', '64px'); let navBarHeight = 64; { // Animates an element between two points, for the nav bar // TODO: Maybe we should have used a library class RangeAnimator { constructor(config) { let defaults = { range: [0, 1], durationMs: 5000, onUpdate: (offset) => {}, }; this.config = {...defaults, ...config}; this.alpha = 0; // current position in the range, from 0 to 1 this.targetAlpha = 0; } // Move smoothly to the start position moveToStart() { this.targetAlpha = 0; this.move(); } // Move smoothly to the end position moveToEnd() { this.targetAlpha = 1; this.move(); } // Move instantaneously to the start position setToStart() { this.targetAlpha = 0; this.setAlpha(0); } // Move instantaneously to the end position setToEnd() { this.targetAlpha = 1; this.setAlpha(1); } setAlpha(alpha) { if (alpha < 0) alpha = 0; if (alpha > 1) alpha = 1; this.alpha = alpha; let offset = this.config.range[0] + (this.config.range[1] - this.config.range[0]) * this.alpha; this.config.onUpdate(offset); } // Move alpha towards targetAlpha move() { if (!this.animating) { this.animating = true; let prevT = Date.now(); function update(t) { let dt = t - prevT; prevT = t; let newAlpha = this.alpha + dt / this.config.durationMs * Math.sign(this.targetAlpha - this.alpha); this.setAlpha(newAlpha); if (this.alpha != this.targetAlpha) { requestAnimationFrame(update.bind(this)); } else { this.animating = false; } } requestAnimationFrame(update.bind(this)); } } } let animationNavBarCallbacks = []; addNavBarAnimationListener = (callback) => { animationNavBarCallbacks.push(callback); }; addNavBarAnimationListener((offset) => { let height = navBarHeight + offset; document.documentElement.style.setProperty('--current-header-height', `${height}px`); }) addNavBarDisplacement = (node) => { function update() { let navBar = document.getElementById('top-nav-bar'); let container = node.parentElement; let navBarBounds = navBar.getBoundingClientRect(); let containerBounds = container.getBoundingClientRect(); let effectiveNavBarHeight = navBarBounds.bottom; let displacement = navBarBounds.bottom - containerBounds.top; if (displacement < 0) displacement = 0; if (displacement > effectiveNavBarHeight) displacement = effectiveNavBarHeight; node.style.setProperty('--displacement', `${displacement}px`); } addNavBarAnimationListener(update); update(); }; document.addEventListener("DOMContentLoaded", (event) => { let navBar = document.getElementById('top-nav-bar'); let animator = new RangeAnimator({ range: [0, -navBarHeight], durationMs: 200, onUpdate: (offset) => { navBar.style.transform = `translateY(${offset}px)`; for (let callback of animationNavBarCallbacks) { callback(offset); } }, }); let innerNavBar = document.getElementById('inner-nav-bar'); if (innerNavBar) { function updateInnerNavBar() { const navBarBottom = navBar.getBoundingClientRect().bottom; innerNavBar.style.top = `${navBarBottom}px`; } // Move sub navigation bars when the main nav bar moves up and down addNavBarAnimationListener((offset) => { updateInnerNavBar(); }); // Listen for changes in the nav bar position document.addEventListener('scroll', () => { updateInnerNavBar(); }); } let toc = document.querySelector('.toc'); if (toc) { let tocNavContainer = document.querySelector('.nav-container'); let tocListContainer = document.querySelector('.toc-list-container'); let baseTop = parseFloat(getComputedStyle(tocNavContainer).top); function update() { let navBarBottom = navBar.getBoundingClientRect().bottom; if (innerNavBar) { navBarBottom = Math.max(navBarBottom, innerNavBar.getBoundingClientRect().bottom); } tocNavContainer.style.top = `${baseTop + navBarBottom}px`; tocNavContainer.style.setProperty('--displacement', `${navBarBottom}px`); // Instantaneous transition tocListContainer.style.transition = "none"; setTimeout(() => { tocListContainer.style.transition = null; }); } addNavBarAnimationListener(update); if (innerNavBar) { document.addEventListener('scroll', update); } update(); } let fixed = false; let headroom = new Headroom(navBar, { offset: 81, tolerance: { up: 50, down: 0, }, onPin: () => { if (!fixed) { animator.moveToStart(); } }, onUnpin: () => { if (!fixed) { animator.moveToEnd(); } }, onTop : () => { animator.setToStart(); fixed = true; }, onNotTop : () => { fixed = false; }, }); headroom.init(); freezeNavBar = () => { headroom.freeze(); }; unfreezeNavBar = () => { headroom.unfreeze(); }; // Burger menu let narrowMenuContents = navBar.querySelector('.narrow-menu-contents'); let closeMenu = document.getElementById('close-menu-button'); // Handle opening and closing the top navigation bar let topNavBar = document.getElementById("top-nav-bar"); let searchOverlay = document.getElementById("search-overlay"); searchOverlay.addEventListener('enter-fullscreen', () => { headroom.unfreeze(); document.querySelector('#top-nav-bar .narrow-menu').classList.remove('open'); }); searchOverlay.addEventListener('closed', () => { onSearchClosed(); }); topNavBar.querySelectorAll('.search-menu-item').forEach(searchItem => searchItem.addEventListener("click", (e) => { let modifier = (e.shiftKey || e.ctrlKey || e.metaKey || e.altKey); if (!searchOverlay.classList.contains('fullscreen') && !modifier) { e.preventDefault(); let opened = searchOverlay.classList.contains("opened"); if (opened) { closeSearch(); } else { openSearch(); } } })); document.body.addEventListener('keydown', (e) => { if (e.key == "Escape") { if (!searchOverlay.classList.contains("fullscreen")) { closeSearch(); } } }); function openSearch() { headroom.pin(); headroom.freeze(); searchOverlay.dispatchEvent(new CustomEvent("open")); } function closeSearch() { searchOverlay.dispatchEvent(new CustomEvent("close")); onSearchClosed(); } }); let prevScroll = 0; unfreezeNavBarOnScrollStop = function() { prevScroll = window.scrollY; setTimeout(unfreezeNavBarOnScrollStopAux, 100); } function unfreezeNavBarOnScrollStopAux() { let currentScroll = window.scrollY; if (prevScroll == currentScroll) { unfreezeNavBar(); } else { prevScroll = currentScroll; setTimeout(unfreezeNavBarOnScrollStopAux, 100); } } } </script> <script> // Store navigation data in a JavaScript object const menuData = [{"label":"Publications","url":"/blog"},{"label":"Gradient Updates","url":"/gradient-updates"},{"label":"Data & Resources","subitems":[{"label":"Data on AI","url":"/data"},{"label":"Key Trends & Insights","url":"/trends"}]},{"label":"Projects","subitems":[{"label":"FrontierMath","url":"/frontiermath","subitems":[{"label":"Overview","url":"/frontiermath"},{"label":"The Benchmark","url":"/frontiermath/the-benchmark"},{"label":"Expert Perspectives","url":"/frontiermath/expert-perspectives"},{"label":"Benchmark Problems","url":"/frontiermath/benchmark-problems"},{"label":"Paper","external":true,"url":"https://arxiv.org/abs/2411.04872"}]},{"label":"Distributed Training","url":"/tools/distributed-training"}]},{"label":"About","subitems":[{"label":"About Us","url":"/about"},{"label":"About Our Research","url":"/research"},{"label":"Careers","url":"/careers"},{"label":"Support Us","url":"/support-us"}]},{"label":"Contact","url":"mailto:info@epoch.ai"}]; document.addEventListener("DOMContentLoaded", () => { const menuContainer = document.querySelector(".menu-list-container"); const sidebar = document.querySelector(".sidebar"); const sidebarOverlay = document.querySelector(".sidebar-overlay"); const openMenuBtn = document.querySelector("#open-menu-button"); const closeMenuBtn = document.querySelector(".sidebar .close-button"); const searchOverlay = document.getElementById("search-overlay"); const searchButton = document.querySelector(".wide-menu .search-menu-item"); const currentPath = window.location.pathname; // Function to show sidebar function showSidebar() { // Close search if it's open if (searchOverlay && document.location.pathname != "/search") { // Remove fullscreen class searchOverlay.classList.remove("fullscreen"); // Dispatch close event searchOverlay.dispatchEvent(new CustomEvent("close")); // Reset current menu item const currentDir = document.querySelector(".current-dir"); if (currentDir) { currentDir.classList.add("current-menu-item"); } document.querySelector(".search-menu-item")?.classList.remove("current-menu-item"); } sidebar.style.transform = "translateX(0)"; sidebarOverlay.style.display = "block"; document.body.style.overflow = "hidden"; } // Function to hide sidebar function hideSidebar() { sidebar.style.transform = "translateX(100%)"; sidebarOverlay.style.display = "none"; document.body.style.overflow = ""; } // Function to check window width and hide sidebar if needed function checkWindowWidth() { if (window.innerWidth >= 1120) { hideSidebar(); } } // Event listeners for sidebar controls openMenuBtn?.addEventListener("click", showSidebar); closeMenuBtn?.addEventListener("click", hideSidebar); sidebarOverlay?.addEventListener("click", hideSidebar); // Watch window resize window.addEventListener("resize", checkWindowWidth); // Initial check checkWindowWidth(); function extractMenuSteps(menu, parentStep = null, steps = {}, stepTitle = "Root") { const currentStep = { title: stepTitle, parent: parentStep, items: menu.map(item => ({ label: item.label, url: item.url || null, external: item.external || false, hasSubitems: !!item.subitems })) }; const stepId = parentStep ? `${parentStep}-${stepTitle}` : stepTitle; steps[stepId] = currentStep; menu.forEach(item => { if (item.subitems) { extractMenuSteps(item.subitems, stepId, steps, item.label); } }); return steps; } const steps = extractMenuSteps(menuData); function renderMenu(steps) { Object.entries(steps).forEach(([key, step]) => { const menuList = document.createElement("div"); menuList.classList.add("menu-list"); menuList.dataset.parentId = step.parent; menuList.dataset.levelId = key; if (step.parent) { const menuTitle = document.createElement("div"); menuTitle.classList.add("menu-list-title"); menuTitle.innerHTML = ` <a href="javascript:void(0)" class="menu-item"> <svg class="chevron" width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"> <path d="M9 18L15 12L9 6" stroke="#07171A" stroke-linecap="round" stroke-linejoin="round"/> </svg> <span class="menu-item-label">${step.title}</span> </a> `; menuList.appendChild(menuTitle); } const navUl = document.createElement("ul"); step.items.forEach(item => { let submenu = null let menuItem = `` if (item.hasSubitems) { if (step.parent) { submenu = step.parent + "-" + step.title + "-" + item.label; } else { submenu = step.title + "-" + item.label; } menuItem = ` <li> <a href="javascript:void(0)" class="menu-item" data-submenu-id="${submenu}"> <span class="menu-item-label">${item.label}</span> <svg class="chevron" width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"> <path d="M9 18L15 12L9 6" stroke="#07171A" stroke-linecap="round" stroke-linejoin="round"/> </svg> </a> </li> ` } else { menuItem = ` <li> <a href="${item.url}" class="menu-item"> <span class="menu-item-label">${item.label}</span> </a> </li> ` } navUl.innerHTML += menuItem; }); menuList.appendChild(navUl); menuContainer.appendChild(menuList); }); } renderMenu(steps); const currentDir = "/blog"; const currentPage = "/blog/compute-trends"; const menus = document.querySelectorAll(".menu-list"); // Function to find menu ID containing a specific URL function findMenuIdForUrl(url, menuData, path = ["Root"]) { // Check if this is a directory URL (ends with /) const isDirectory = url.endsWith('/'); const urlToMatch = isDirectory ? url : url + '/'; // Check direct items in this level for (const item of menuData) { const itemUrl = item.url?.endsWith('/') ? item.url : item.url + '/'; // If URL matches this item if (itemUrl === urlToMatch && !item.subitems) { return path.join("-"); } // If this item has subitems, recursively check them if (item.subitems) { const newPath = [...path, item.label]; const foundId = findMenuIdForUrl(url, item.subitems, newPath); if (foundId) return foundId; } } return null; } // Find the correct menu ID based on current page or directory let currentMenuId = findMenuIdForUrl(currentPage, menuData) || "Root"; // Function to reset menu classes function resetMenuClasses() { menus.forEach(menu => { menu.classList.remove("active", "prev", "next"); }); } function activateMenu(menuId) { resetMenuClasses(); const activateMenu = steps[menuId]; menus.forEach(menu => { if (menu.dataset.levelId === menuId) { menu.classList.add("active"); } else if (menu.dataset.levelId === activateMenu.parent) { menu.classList.add("prev"); } else if (menu.dataset.parentId === menuId) { menu.classList.add("next"); } else { menu.classList.add("next"); } }); currentMenuId = menuId; } menuContainer.addEventListener("click", event => { const submenuLink = event.target.closest(".menu-item[data-submenu-id]"); if (submenuLink) { const submenuId = submenuLink.dataset.submenuId; activateMenu(submenuId); } const backButton = event.target.closest(".menu-list-title"); if (backButton && backButton.parentElement.dataset.parentId) { const parentId = backButton.parentElement.dataset.parentId; activateMenu(parentId); } }); activateMenu(currentMenuId); }); </script> <div class="body-filling"> <script src="https://cdn.jsdelivr.net/gh/nextapps-de/flexsearch@0.7.31/dist/flexsearch.bundle.js"></script> <script src="/assets/js/make-clickable.js"></script> <script src="https://cdn.jsdelivr.net/npm/algoliasearch@4.19.1/dist/algoliasearch-lite.umd.js" integrity="sha256-qzlNbRtZWHoUV5I2mI2t9QR7oYXlS9oNctX+0pECXI0=" crossorigin="anonymous"></script> <script src="/assets/components/search-bar/search-bar.js"></script> <link rel="stylesheet" href="/assets/components/search-bar/search-bar.css"> <script> document.addEventListener('DOMContentLoaded', function() { for (let link of document.querySelectorAll('a')) { // Skip javascript:void(0) links if (link.href === 'javascript:void(0)' || link.getAttribute('href') === 'javascript:void(0)') { continue; } // Handle external links if (link.hostname !== window.location.hostname) { if (!link.target) { link.target = '_blank'; link.rel = 'noopener'; } } } }); </script> <div id="search-overlay" class="hidden"> <div class="search-results-pan-container"> <div class="search-bar-container"> <div class="search-bar-placeholder"> <img class="loupe-icon" src="/assets/images/icons/Search-Loupe-2.svg"> Search epoch.ai </div> <input class="search-bar"> <a class="search-button button filled-light-green-button" href="/search">Search</a> </div> <div class="search-results-container-wrapper hidden"> <div class="search-results-container"> <div class="results-header no-results"> Enter a query to search for results </div> <div class="search-results-wrapper hidden"> <div class="search-results"> </div> <script src="/assets/components/page-selector/page-selector.js"></script> <link rel="stylesheet" href="/assets/components/page-selector/page-selector.css"> <div class="no-search-index page-selector not-inited" data-items-per-page=10 data-num-items="30" data-current-page="1" data-num-pages=""> <button role="button" class="button light-button prev-button">Placeholder</button> </div> </div> </div> </div> </div> </div> <div class="page-content"> <link rel="stylesheet" href="https://unpkg.com/handorgel@1.0.0/lib/css/handorgel.min.css"> <script src="https://unpkg.com/handorgel@1.0.0/lib/js/umd/handorgel.min.js"></script> <link rel="stylesheet" href="/assets/css/handorgel.css"> <style> :root { --header-background-color: #09323A; --header-color: white; --post-type-color: #95A6A9; --post-type-opacity: 1; --post-type-mix-blend-mode: "unset"; --background-image: url("/assets/images/dot-light-crimson-background.svg"); } #top-nav-bar { /* show top nav bar after setting its color */ opacity: 1; } </style> <div class="post-container"> <div class="post post-paper"> <div class="post-head dark-background"> <div class="breadcrumb standard-max-width no-saarch-index"> <a class="root path-component" href="/blog">Articles</a> <svg class="chevron-right" xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16" fill="none"> <g> <path d="M6 12L10 8L6 4" stroke="#034752" stroke-width="1.33333" stroke-linecap="round" stroke-linejoin="round"/> </g> </svg>  <a class="path-component" href="/blog/compute-trends">Compute Trends Across Three Eras of Machine Learning</a> </div> <div class="post-presentation"> <div class="post-type post-type-blend">paper</div> <h1 class="post-title">Compute Trends Across Three Eras of Machine Learning</h1> <p class="post-description">We’ve compiled a dataset of the training compute for over 120 machine learning models, highlighting novel trends and insights into the development of AI since 1952, and what to expect going forward."</p> </div> <div class="post-thumbnails-container"> <div class="post-thumbnails"> <div class="post-thumbnail-content"> <img class="post-thumbnail" src="/assets/images/posts/2022/compute-trends.png"> </div> </div> </div> </div> <div class="post-body standard-max-width"> <div class="post-properties"> <div class="post-publication-date-container"> <h3>Published</h3> <div class="post-publication-date">Feb 16, 2022</div> <div class="post-publication-date-label">Last updated May 02, 2022</div> </div> <div class="post-authors"> <h3>Authors</h3> <button type="button" class="post-author" data-author-id="Jaime Sevilla" disabled-data-micromodal-trigger="author-page-not-implemented"> Jaime Sevilla </button> <button type="button" class="post-author" data-author-id="Lennart Heim" disabled-data-micromodal-trigger="author-page-not-implemented"> Lennart Heim </button> <button type="button" class="post-author" data-author-id="Anson Ho" disabled-data-micromodal-trigger="author-page-not-implemented"> Anson Ho </button> <button type="button" class="post-author" data-author-id="Tamay Besiroglu" disabled-data-micromodal-trigger="author-page-not-implemented"> Tamay Besiroglu </button> <button type="button" class="post-author" data-author-id="Marius Hobbhahn" disabled-data-micromodal-trigger="author-page-not-implemented"> Marius Hobbhahn </button> <button type="button" class="post-author" data-author-id="Pablo Villalobos" disabled-data-micromodal-trigger="author-page-not-implemented"> Pablo Villalobos </button> </div> <div class="post-resources-container"> <h3>Resources</h3> <div class="post-resources"> <a class="post-resource button filled-light-green-button no-decorations-link" href="https://arxiv.org/abs/2202.05924"> <img src="/assets/images/icons/document-paper-angle.svg"> Paper </a> <a class="post-resource button filled-light-green-button no-decorations-link" href="/data/notable-ai-models"> <img src="/assets/images/icons/item-list-columns.svg"> Data </a> <button class="post-resource button filled-light-green-button no-decorations-link" data-micromodal-trigger="citation-modal"> <img src="/assets/images/icons/citation.svg"> Cite </button> </div> </div> </div> <div class="section-divider top-section-divider"></div> <div class="post-content-container"> <nav class="toc" class="no-scrollbars"> <div class="nav-container no-toc-section"> <div markdown="1" class="folded-section"> <h3 class="toc-header"> Contents <svg class="toc-chevron" xmlns="http://www.w3.org/2000/svg" width="16" height="17" viewBox="0 0 16 17" fill="none"> <path d="M12 10.5L8 6.5L4 10.5" stroke="#09323A" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/> </svg> </h3> <div class="toc-list-container"> <div class="toc-list"> <div class="toc-bar"> <div class="toc-bar-active"></div> </div><div> <ul id="toc" class="section-nav"> <li class="toc-entry toc-h2"><a href="#introduction">Introduction</a></li> <li class="toc-entry toc-h2"><a href="#methodology">Methodology</a></li> <li class="toc-entry toc-h2"><a href="#results">Results</a> <ul> <li class="toc-entry toc-h3"><a href="#compute-trends-are-slower-than-previously-reported">Compute trends are slower than previously reported</a></li> <li class="toc-entry toc-h3"><a href="#three-eras-of-machine-learning">Three eras of machine learning</a></li> </ul> </li> <li class="toc-entry toc-h2"><a href="#implications-and-further-work">Implications and further work</a></li> </ul> </div></div> </div> </div> </div> </nav> <div class="post-content regular-content"> <style> #trends-graph-note { margin-bottom: 46px; display: flex; width: 100%; padding: 24px; flex-direction: column; align-items: flex-start; gap: 16px; border-radius: 12px; border: 1px solid var(--Epoch-Website-Gray-300, #CCD8D9); background: var(--Epoch-Website-Gray-50, #F2FAF9); } #trends-graph-note .note-header { color: var(--Epoch-Website-Gray-900, #07171A); font-size: 18px; font-weight: 600; line-height: 160%; /* 28.8px */ letter-spacing: 0.09px; margin-bottom: 0; } #trends-graph-note .note-content > :last-child { margin-bottom: 0; } figure { padding: 24px; } </style> <div class="update-note"> <div class="note-header">Update</div> <div class="note-content"> <p>This paper was originally published on Feb 16, 2022. For the latest research and updates on this subject, please see: <a href="/blog/training-compute-of-frontier-ai-models-grows-by-4-5x-per-year">Training Compute of Frontier AI Models Grows by 4-5x per Year</a>.</p> </div> </div> <p><em>Summary: We have collected a dataset and analysed key trends in the training compute of machine learning models since 1950. We identify three major eras of training compute - the pre-Deep Learning Era, the Deep Learning Era, and the Large-Scale Era. Furthermore, we find that the training compute has grown by a factor of 10 billion since 2010, with a doubling rate of around 5-6 months. See our recent paper, <a href="https://arxiv.org/abs/2202.05924">Compute Trends Across Three Eras of Machine Learning</a>, for more details.</em></p> <h2 id="introduction">Introduction</h2> <p>It is well known that progress in machine learning (ML) is driven by three primary factors - algorithms, data, and compute. This makes intuitive sense - the development of algorithms like backpropagation transformed the way that machine learning models were trained, leading to significantly improved efficiency compared to previous optimisation techniques (<a href="https://www.deeplearningbook.org/contents/mlp.html#pf25">Goodfellow <em>et al.</em>, 2016</a>; <a href="http://www.cs.toronto.edu/~hinton/absps/naturebp.pdf">Rumelhart <em>et al.</em>, 1986</a>). Data has been becoming increasingly available, particularly with the advent of “<a href="https://en.wikipedia.org/wiki/Big_data">big data</a>” in recent years. At the same time, progress in computing hardware has been rapid, with increasingly powerful and specialised AI hardware (<a href="https://forum.effectivealtruism.org/s/4yLbeJ33fYrwnfDev/p/YNB39RyJ7iAQKGJvq">Heim, 2021</a>; <a href="https://cset.georgetown.edu/wp-content/uploads/AI-Chips%E2%80%94What-They-Are-and-Why-They-Matter.pdf">Khan and Mann, 2020</a>).</p> <p>What is less obvious is the <em>relative</em> importance of these factors, and what this implies for the future of AI. <a href="https://arxiv.org/abs/2001.08361">Kaplan <em>et al.</em> (2020)</a> studied these developments through the lens of <strong>scaling laws</strong>, identifying three key variables:</p> <ul> <li>Number of parameters of a machine learning model</li> <li>Training dataset size</li> <li>Compute required for the final training run of a machine learning model (henceforth referred to as <strong>training compute</strong>)</li> </ul> <p>Trying to understand the relative importance of these is challenging because our theoretical understanding of them is insufficient - instead, we need to take large quantities of data and analyse the resulting trends. Previously, we looked at trends in parameter counts of ML models - in this paper, we try to understand how training compute has evolved over time.</p> <p><a href="https://openai.com/blog/ai-and-compute/">Amodei and Hernandez (2018)</a> laid the groundwork for this, finding a $300,000\times$ increase in training compute from 2012 to 2018, doubling every 3.4 months. However, this investigation had only around $15$ datapoints, and does not include some of the most impressive recent ML models like GPT-3 (<a href="https://arxiv.org/abs/2005.14165">Brown, 2020</a>).</p> <p>Motivated by these problems, we have curated the largest ever dataset containing the training compute of machine learning models, with over 120 datapoints. Using this data, we have drawn several novel insights into the significance of compute as an input to ML models.</p> <p>These findings have implications for the future of AI development, and how governments should orient themselves to compute governance and a future with powerful AI systems.</p> <h2 id="methodology">Methodology</h2> <p>Following the approach taken by OpenAI (<a href="https://openai.com/blog/ai-and-compute/">Amodei and Hernandez, 2018</a>), we use two main approaches to determining the training compute<sup id="fnref:1" role="doc-noteref"><a href="#fn:1" class="footnote" rel="footnote">1</a></sup> of ML systems:</p> <ol> <li><strong>Counting the number of operations</strong>: The training compute can be determined from the number of arithmetic operations that is performed by the machine. By looking at the model architecture and closely monitoring the training process, we can directly calculate the total number of multiplications and additions, yielding the training compute. As ML models become significantly more complex (as continues to be the case), this approach becomes significantly more tedious and tricky. Doing this also requires knowledge of key details of the training process<sup id="fnref:2" role="doc-noteref"><a href="#fn:2" class="footnote" rel="footnote">2</a></sup>, which may not always be accessible.</li> <li><strong>GPU-time</strong>: A second approach, which is independent of the model architecture, uses the information about the total training time and hardware to estimate the training compute. This method typically requires making several assumptions about the training process, which leads to a greater uncertainty in the final value.<sup id="fnref:3" role="doc-noteref"><a href="#fn:3" class="footnote" rel="footnote">3</a></sup></li> </ol> <p>Most of the time, we were able to use either of the above approaches to estimate the training compute for a particular ML model. In practice this process involved many difficulties, since authors would often not publish key information about the hardware used or training time.</p> <p>Of course, it would be infeasible for us to gather this data for <em>all</em> ML systems since 1950. Instead, we focus on <em>milestone</em> systems, based on the following criteria:</p> <ul> <li><strong>Clear importance</strong>: These are systems that have major historical influence, significantly improve on the state-of-the-art, or have over 1000 citations<sup id="fnref:4" role="doc-noteref"><a href="#fn:4" class="footnote" rel="footnote">4</a></sup></li> <li><strong>Relevance</strong>: We only include papers which include <em>experimental results</em> and a key machine learning component, and have a goal of pushing the existing state-of-the-art</li> <li><strong>Uniqueness</strong>: If another paper describing the same system is more influential, then the paper is excluded from our dataset</li> </ul> <p>This selection process lets us focus on the most important systems, helping us understand the key drivers of the state-of-the-art.</p> <h2 id="results">Results</h2> <p>Using these techniques, we yielded a <a href="/data/notable-ai-models?view=table#explore-the-data">dataset</a> with training compute for over 120 milestone ML systems, the largest such dataset yet. We have chosen to make this and our <a href="/data/notable-ai-models#explore-the-data">interactive data visualisation</a> publicly available, in order to facilitate further research along the same lines.</p> <figure class="no-toc-section active epoch-figure"> <div id="embedded-viz"> </div> </figure> <div id="trends-graph-note"> <div class="note-header">Update</div> <div class="note-content"> <p>This visualization is updated as we collect further information on notable ML systems. As a result, the showcased trends differ from the time of our original publication.</p> <p>The interactive visualization originally featured in this article now has its own dedicated page for a more comprehensive experience.</p> <p><a href="/data/notable-ai-models#explore-the-data">Explore the updated visualization here.</a></p> </div> </div> <p>When analysing the gathered data, we draw two main conclusions.</p> <ul> <li>Trends in training compute are slower than previously reported</li> <li>We identify three eras of training compute usage across machine learning</li> </ul> <h3 id="compute-trends-are-slower-than-previously-reported">Compute trends are slower than previously reported</h3> <p>In the previous investigation by <a href="https://openai.com/blog/ai-and-compute/">Amodei and Hernandez (2018)</a>, the authors found that the training compute used was growing extremely rapidly - doubling every 3.4 months. With approximately 10 times more data than the original study, we find a doubling time closer to 6 months. This is still extraordinarily fast! Since 2010, the amount of training compute for machine learning models has grown by a factor of 10 billion, significantly exceeding a naive extrapolation of Moore’s Law.</p> <p>This suggests that many previous analyses based on OpenAI’s paper were biased towards rapid developments, approximately by a factor of two.</p> <h3 id="three-eras-of-machine-learning">Three eras of machine learning</h3> <p>One of the more speculative contributions of our paper is that we argue for the presence of three eras of machine learning. This is in contrast to prior work, which identifies two trends separated by the start of the Deep Learning revolution (<a href="https://openai.com/blog/ai-and-compute/">Amodei and Hernandez, 2018</a>). Instead, we split the history of ML compute into three eras:</p> <ol> <li>The <strong>Pre-Deep Learning Era</strong>: Prior to Deep Learning, training compute approximately follows Moore’s Law, with a doubling time of approximately every 20 months.</li> <li>The <strong>Deep Learning Era</strong>: This starts somewhere between 2010 and 2012<sup id="fnref:5" role="doc-noteref"><a href="#fn:5" class="footnote" rel="footnote">5</a></sup>, and displays a doubling time of approximately 6 months.</li> <li>The <strong>Large-Scale Era</strong>: Arguably, a separate trend of of models breaks off the main trend between 2015 and 2016. These systems are characteristic in that they are run by large corporations, and use training compute 2-3 orders of magnitude larger than systems that follow the Deep Learning Era trend in the same year. Interestingly, the growth of compute in these Large-Scale models seems slower, with a doubling time of about 10 months.</li> </ol> <p>A key benefit of this framing is that it helps make sense of developments over the last two decades of ML research. Deep Learning marked a major paradigm shift in ML, with an increased focus on training larger models, using larger datasets, and using more compute. The bifurcation of the Deep Learning trend coincides with the shift in focus towards major projects at large corporations, such as DeepMind and OpenAI.</p> <p>However, there is a fair bit of ambiguity with this framing. For instance, how do we know exactly which models can be considered large-scale? How can we be sure that this “large-scale” trend isn’t just due to noise? To test these questions, we used different statistical thresholds for what counts as “large-scale”, and the resulting trend does not change very much, thus the findings are at least somewhat robust to different selection criteria. Of course, the exact threshold that we use is still debatable, and it is hard to be certain about the observed trends without more data.</p> <h2 id="implications-and-further-work">Implications and further work</h2> <p>We expect that future work will build upon this research project. Using the aforementioned compute estimation techniques, more training compute data can be gathered, offering the potential for more conclusive analyses. We can also make the data gathering process easier, such as by:</p> <ul> <li>Developing tools for automatically measuring training compute usage (as well as inference compute)</li> <li>Publishing key details about the training process, such as the GPU model used</li> </ul> <p>Taking these steps helps key actors obtain valuable information in the future.</p> <p>Naturally, we will also be looking at trends in dataset sizes, and comparing the relative importance of data and compute for increased performance. We can also look how factors like funding and talent influence the primary inputs of a ML system, like data and compute.</p> <p>Answering questions like these is crucial for understanding how the future of AI will look like. At Epoch AI, we’re particularly concerned about ensuring that AI is developed in a beneficial way, with appropriate governance intervention to ensure safety. Better understanding the progress of compute capabilities can help us better navigate a future with powerful AI systems.</p> <p><em><a href="https://arxiv.org/abs/2202.05924">Read the full paper now on the arXiv</a>.</em></p> <hr /> <style> #embedded-viz { height: 90vh; max-height: 700px; } /* Undo the default styling */ #embedded-viz .regular-content h3 { margin: 0; } </style> <script src="/assets/js/dist/notable-models-viz.js"></script> <script> renderNotableModelsViz('#embedded-viz', true); </script> <div class="footnotes" role="doc-endnotes"> <ol> <li id="fn:1" role="doc-endnote"> <p>Specifically, we focus on the <em>final</em> training run of a ML system. This is primarily due to measurability - researchers generally do not mention the total compute or training time that does not directly contribute to the final machine learning model. We simply do not have sufficient information to determine the total compute through the entire experimentation process. <a href="#fnref:1" class="reversefootnote" role="doc-backlink">↩</a></p> </li> <li id="fn:2" role="doc-endnote"> <p>i.e. the total number of iterations during training. <a href="#fnref:2" class="reversefootnote" role="doc-backlink">↩</a></p> </li> <li id="fn:3" role="doc-endnote"> <p>For instance, we often assumed that the GPU utilisation rate was 10%. We often also had to guess which GPU model was in use based on the year in which the paper was published, in the event that this information was not disclosed in the paper of interest and the authors were not able to provide an answer. <a href="#fnref:3" class="reversefootnote" role="doc-backlink">↩</a></p> </li> <li id="fn:4" role="doc-endnote"> <p>These criteria are ambiguous and can vary on a case-by-case basis. For instance, new papers (published within the last year or two) can be very influential without having had the time to gather many citations. In such cases we make relatively subjective decisions of the importance of these ML models. <a href="#fnref:4" class="reversefootnote" role="doc-backlink">↩</a></p> </li> <li id="fn:5" role="doc-endnote"> <p>We discuss this more in Appendix D of the <a href="https://arxiv.org/pdf/2202.05924.pdf">paper</a>. While AlexNet (<a href="https://proceedings.neurips.cc/paper/2012/file/c399862d3b9d6b76c8436e924a68c45b-Paper.pdf">Krizhevsky <em>et al.</em>, 2012</a>) is commonly associated with the start of Deep Learning, we argue that models before AlexNet have key features commonly associated with Deep Learning, and that 2010 is most consistent with evidence. <a href="#fnref:5" class="reversefootnote" role="doc-backlink">↩</a></p> </li> </ol> </div> </div> </div> <div class="post-bottom"> <div class="section-divider bottom-section-divider"></div> <div class="about-the-authors"> <h3>About the authors</h3> <div class="author-cards"> <div class="author-card"> <div class="avatar-container"><img class="avatar" src="/assets/images/team/square-small/jaime-sevilla.jpg"></div> <div class="description"> <div class="author-description"><span class="description-name">Jaime Sevilla</span> is the director of Epoch AI. His research is focused on technological forecasting and the trajectory of AI. He has a background in Mathematics and Computer Science.</div> </div> </div> <div class="author-card"> <div class="avatar-container"><img class="avatar" src="/assets/images/team/square-small/lennart-heim.jpg"></div> <div class="description"> <div class="author-description"><span class="description-name">Lennart Heim</span> supports Epoch AI’s research and strategy. He extracts the strategy and policy implications of Epoch AI’s investigation for the AI governance community. <br> In his primary role, Lennart is a researcher at <a href="https://www.rand.org/global-and-emerging-risks.html">RAND</a>. He leads the compute research in the Technology and Security Policy Center within RAND Global and Emerging Risks. His research interests include the role of compute in the production of AI, the compute supply chain, forecasting emerging technologies, and the security of AI systems.</div> </div> </div> <div class="author-card"> <div class="avatar-container"><img class="avatar" src="/assets/images/team/square-small/anson-ho.jpg"></div> <div class="description"> <div class="author-description"><span class="description-name">Anson Ho</span> is a researcher at Epoch AI. He is interested in helping develop a more rigorous understanding of future developments in AI and its societal impacts. Prior to this, he completed his BSc in physics at the University of St Andrews.</div> </div> </div> <div class="author-card"> <div class="avatar-container"><img class="avatar" src="/assets/images/team/square-small/tamay-besiroglu.jpg"></div> <div class="description"> <div class="author-description"><span class="description-name">Tamay Besiroglu</span> is the associate director at Epoch AI. His work focuses on the economics of computing and big-picture trends in machine learning. Previously, he was a researcher at the Future Tech Lab at MIT, led strategy for Metaculus, consulted for the UK Government, and worked at the Future of Humanity Institute.</div> </div> </div> <div class="author-card"> <div class="avatar-container"><img class="avatar" src="/assets/images/team/square-small/marius-hobbhahn.jpg"></div> <div class="description"> <div class="author-type">Former employee</div> <div class="author-description"><span class="description-name">Marius Hobbhahn</span> builds models for AI timelines and takeoff using historical trends and his best understanding of the future.</div> </div> </div> <div class="author-card"> <div class="avatar-container"><img class="avatar" src="/assets/images/team/square-small/pablo-villalobos.jpg"></div> <div class="description"> <div class="author-description"><span class="description-name">Pablo Villalobos</span> has a background in Mathematics and Computer Science. After spending some time as a software engineer, he decided to pivot towards AI. His interests include the economic consequences of advanced AI systems and the role of algorithmic improvements in AI progress.</div> </div> </div> </div> </div> <div class="section-divider bottom-section-divider"></div> <div class="shares-and-tags"> <div class="shares-container"> <h3>Share</h3> <div class="shares"> <a class="button filled-light-green-button no-decoration-link" href="https://twitter.com/intent/tweet?url=https://epoch.ai/blog/compute-trends" data-size="large"> <svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 20 20" fill="none"> <path d="M7.36796 17.5C5.34609 17.5 3.46136 16.9119 1.875 15.8973C3.22187 15.9844 5.59879 15.7757 7.07724 14.3655C4.85317 14.2634 3.85016 12.5577 3.71934 11.8287C3.90831 11.9016 4.80956 11.9891 5.31834 11.785C2.75994 11.1435 2.36745 8.89833 2.45467 8.21311C2.93437 8.54843 3.74841 8.66506 4.06821 8.6359C1.68425 6.93015 2.54189 4.36422 2.96345 3.81022C4.67426 6.18042 7.23825 7.51161 10.4102 7.58565C10.3504 7.32335 10.3188 7.05026 10.3188 6.76978C10.3188 4.75682 11.9459 3.125 13.9529 3.125C15.0016 3.125 15.9465 3.57047 16.6098 4.28302C17.3106 4.11882 18.3652 3.73442 18.8808 3.402C18.6209 4.33507 17.8118 5.11343 17.3224 5.40192C17.3185 5.39207 17.3265 5.41174 17.3224 5.40192C17.7523 5.3369 18.9155 5.11336 19.375 4.8016C19.1478 5.32578 18.29 6.19733 17.5861 6.68526C17.717 12.4613 13.2978 17.5 7.36796 17.5Z" fill="#034752"/> </svg> Twitter </a> <a class="button filled-light-green-button no-decoration-link" href="https://www.linkedin.com/sharing/share-offsite/?url=https://epoch.ai/blog/compute-trends"> <svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 18 18" fill="none"> <path d="M15.3373 15.3371H12.6702V11.1603C12.6702 10.1643 12.6524 8.88217 11.283 8.88217C9.89389 8.88217 9.68135 9.96735 9.68135 11.0878V15.3369H7.01423V6.74764H9.57466V7.92145H9.61051C9.86674 7.48333 10.237 7.12292 10.6819 6.87859C11.1268 6.63426 11.6296 6.51517 12.1368 6.534C14.84 6.534 15.3384 8.31207 15.3384 10.6252L15.3373 15.3371ZM4.00484 5.57356C3.69872 5.57361 3.39945 5.48289 3.14489 5.31287C2.89033 5.14284 2.69193 4.90115 2.57473 4.61835C2.45753 4.33556 2.42682 4.02436 2.48648 3.72411C2.54615 3.42387 2.69352 3.14806 2.90994 2.93156C3.12635 2.71506 3.40211 2.5676 3.70234 2.50783C4.00256 2.44805 4.31377 2.47865 4.5966 2.59574C4.87944 2.71284 5.1212 2.91118 5.29132 3.16567C5.46143 3.42017 5.55227 3.7194 5.55232 4.02552C5.55236 4.22877 5.51235 4.43004 5.43461 4.61784C5.35686 4.80564 5.24289 4.97628 5.0992 5.12003C4.9555 5.26378 4.78488 5.37782 4.59711 5.45563C4.40934 5.53345 4.20809 5.57352 4.00484 5.57356ZM5.33839 15.3371H2.6685V6.74764H5.33839V15.3371ZM16.667 0.00122636H1.32828C0.980136 -0.00270245 0.644657 0.131747 0.395592 0.37503C0.146526 0.618313 0.00424892 0.950526 0 1.29867V16.701C0.0041035 17.0493 0.146297 17.3817 0.395354 17.6253C0.644411 17.8688 0.979964 18.0035 1.32828 17.9998H16.667C17.016 18.0042 17.3525 17.8699 17.6025 17.6263C17.8526 17.3828 17.9957 17.05 18.0005 16.701V1.29756C17.9955 0.948724 17.8523 0.616134 17.6023 0.372856C17.3522 0.129579 17.0158 -0.00448605 16.667 0.000114616" fill="#034752"/> </svg> LinkedIn </a> </div> </div> <div class="tags-container"> <h3>Tags</h3> <div class="tags"> <a class="button outline-thin-button" href="/blog?tag=trends">Trends</a> <a class="button outline-thin-button" href="/blog?tag=compute">Compute</a> </div> </div> </div> </div> </div> <div class="related-research"> <div class="related-research-content standard-max-width"> <h2>Related work</h2> <div class="related-research-posts"> <a href="/blog/training-compute-of-frontier-ai-models-grows-by-4-5x-per-year" class="related-work-card " data-filter-tags="Trends;Compute;report"> <div class="work-img-container"> <img class="work-img" src="/assets/images/posts/2024/training-compute-of-frontier-ai-models-grows-by-4-5x-per-year/summary_figure.png" style="--image-fit: contain; --image-position: center"> </div> <div class="work-body"> <div class="work-tag-and-reading-time"> <span class="work-type"> report </span> <span class="work-reading-time reading-time" data-tippy-content="5121 words"> · 20 min read </span> </div> <div class="work-title"> Training Compute of Frontier AI Models Grows by 4-5x per Year </div> <div class="work-description"> Our expanded AI model database shows that training compute grew 4-5x/year from 2010 to 2024, with similar trends in frontier and large language models. </div> <div class="work-date-and-authors"> May 28, 2024 · By Jaime Sevilla and Edu Roldán </div> </div> </a> <a href="/blog/trading-off-compute-in-training-and-inference" class="related-work-card " data-filter-tags="Compute;Scaling laws;report"> <div class="work-img-container"> <img class="work-img" src="/assets/images/posts/2023/trading-off-compute-in-training-and-inference/trading-off-compute-in-training-and-inference-banner.png" style="--image-fit: contain; --image-position: center"> </div> <div class="work-body"> <div class="work-tag-and-reading-time"> <span class="work-type"> report </span> <span class="work-reading-time reading-time" data-tippy-content="6808 words"> · 27 min read </span> </div> <div class="work-title"> Trading Off Compute in Training and Inference </div> <div class="work-description"> We characterize techniques that induce a tradeoff between spending resources on training and inference, outlining their implications for AI governance. </div> <div class="work-date-and-authors"> Jul 28, 2023 · By Pablo Villalobos and David Atkinson </div> </div> </a> <a href="/blog/announcing-expanded-biology-ai-coverage" class="related-work-card " data-filter-tags="Compute;Announcements;Trends;announcement"> <div class="work-img-container"> <img class="work-img" src="/assets/images/posts/2025/announcing-expanded-biology-ai-coverage/biology-ai-models-compute-banner.png" style="--image-fit: contain; --image-position: center"> </div> <div class="work-body"> <div class="work-tag-and-reading-time"> <span class="work-type"> announcement </span> <span class="work-reading-time reading-time" data-tippy-content="381 words"> · 2 min read </span> </div> <div class="work-title"> Announcing our Expanded Biology AI Coverage </div> <div class="work-description"> We've expanded our Biology AI Dataset, now covering 360+ models. Our analysis reveals rapid scaling from 2017-2021, followed by a notable slowdown in biological model development. </div> <div class="work-date-and-authors"> Jan 29, 2025 · By Pablo Villalobos and David Atanasov </div> </div> </a> </div> </div> </div> </div> </div> <div class="profile-modal micromodal-slide no-toc-section citation-modal" id="citation-modal" aria-hidden="true"> <div class="modal-overlay" tabindex="-1" data-micromodal-close> <div class="modal-container" role="dialog" aria-modal="true" aria-labelledby="modal-title"> <header class="modal-header"> <div> <h2 class="modal-title"></h2> </div> <button class="modal-close" aria-label="Close modal" data-micromodal-close> <svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 20 20" fill="none"> <path d="M15 5L5 15M5 5L15 15" stroke="#3E555E" stroke-width="1.66667" stroke-linecap="round" stroke-linejoin="round"/> </svg> </button> </header> <div class="modal-content-content"> <div class="modal-content"> <div class="citation-block"> <h3>Cite this work as </h3> <div class="citation-code language-plaintext highlighter-rouge wrappable-pre"> <div class="highlight"> <pre class="highlight"><code>J. Sevilla, L. Heim, A. Ho, T. Besiroglu, M. Hobbhahn and P. Villalobos, “Compute Trends Across Three Eras of Machine Learning,” 2022 International Joint Conference on Neural Networks (IJCNN), Padua, Italy, 2022, pp. 1-8, doi: 10.1109/IJCNN55064.2022.9891914. </code></pre> </div> </div> </div> <div class="citation-block"> <h3>BibTeX citation</h3> <div class="citation-code language-plaintext highlighter-rouge"><div class="highlight"> <pre class="highlight"><code>@INPROCEEDINGS{9891914, author={Sevilla, Jaime and Heim, Lennart and Ho, Anson and Besiroglu, Tamay and Hobbhahn, Marius and Villalobos, Pablo}, booktitle={2022 International Joint Conference on Neural Networks (IJCNN)}, title={Compute Trends Across Three Eras of Machine Learning}, year={2022}, volume={}, number={}, pages={1-8}, keywords={Deep learning;Training;Machine learning algorithms;Computational modeling;Neural networks;Market research;History;machine learning;artificial intelligence;deep learning;computational efficiency;AI accelerators;backpropagation;high performance computing}, doi={10.1109/IJCNN55064.2022.9891914} }</code></pre> </div></div> </div> </div> </div> <footer class="modal-footer"> </footer> </div> </div> </div> <style> #author-page-not-implemented .modal-close::before { content: ''; } .post-author { background: none; border: none; } </style> <div class="profile-modal micromodal-slide no-toc-section" id="author-page-not-implemented" aria-hidden="true"> <div class="modal-overlay" tabindex="-1" data-micromodal-close> <div class="modal-container" role="dialog" aria-modal="true" aria-labelledby="modal-title"> <header class="modal-header"> <div> <h2 class="modal-title">Not implemented yet</h2> </div> <button class="modal-close" aria-label="Close modal" data-micromodal-close> <svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 20 20" fill="none"> <path d="M15 5L5 15M5 5L15 15" stroke="#3E555E" stroke-width="1.66667" stroke-linecap="round" stroke-linejoin="round"/> </svg> </button> </header> <div class="modal-content-content"> <div class="modal-content"> We will redirect to a new page listing all the posts by this author. </div> <footer class="modal-footer"> </footer> </div> </div> </div> </div> <script> // Open changelog on link click for (let link of document.querySelectorAll('[href="#updates"]')) { link.addEventListener('click', function(e) { let changelog = document.querySelector('#updates .changelog .handorgel'); let accordion = getAccordionByNode(changelog); accordion.folds[0].open() }); } { // Open changelog on page load let hash = window.location.hash; if (hash == '#updates') { setTimeout(() => { let changelog = document.querySelector('#updates .changelog .handorgel'); console.log('changelog:', changelog); let accordion = getAccordionByNode(changelog); accordion.folds[0].open() }, 300); } } </script> <script> { let canHover = window.matchMedia('(hover: hover)').matches; if (canHover) { let authorInfo = { "Jaime Sevilla": { name: "Jaime Sevilla", position: 'Director', mug: '/assets/images/team/square-small/jaime-sevilla.jpg', resources: [ {name: 'mail', url: 'amFpbWVAZXBvY2guYWk=', icon: 'envelope'}, {name: 'twitter', url: 'https://twitter.com/Jsevillamol', icon: 'twitter'}, {name: 'website', url: 'https://jaimesevilla.me/', icon: 'globe'}, ] }, "Lennart Heim": { name: "Lennart Heim", position: 'Strategy Specialist', mug: '/assets/images/team/square-small/lennart-heim.jpg', resources: [ {name: 'mail', url: 'bGVubmFydEBlcG9jaC5haQ==', icon: 'envelope'}, {name: 'twitter', url: 'https://twitter.com/ohlennart', icon: 'twitter'}, {name: 'website', url: 'https://heim.xyz', icon: 'globe'}, ] }, "Anson Ho": { name: "Anson Ho", position: 'Staff Researcher', mug: '/assets/images/team/square-small/anson-ho.jpg', resources: [ {name: 'mail', url: 'YW5zb25AZXBvY2guYWk=', icon: 'envelope'}, {name: 'website', url: 'https://ansonwhho.github.io/', icon: 'globe'}, ] }, "Tamay Besiroglu": { name: "Tamay Besiroglu", position: 'Associate Director', mug: '/assets/images/team/square-small/tamay-besiroglu.jpg', resources: [ {name: 'mail', url: 'dGFtYXlAZXBvY2guYWk=', icon: 'envelope'}, {name: 'twitter', url: 'https://x.com/tamaybes', icon: 'twitter'}, {name: 'website', url: 'https://tamaybesiroglu.com/', icon: 'globe'}, ] }, "Marius Hobbhahn": { name: "Marius Hobbhahn", position: 'Former Research Fellow', mug: '/assets/images/team/square-small/marius-hobbhahn.jpg', resources: [ {name: 'mail', url: 'bWFyaXVzQGVwb2NoLmFp', icon: 'envelope'}, ] }, "Pablo Villalobos": { name: "Pablo Villalobos", position: 'Staff Researcher', mug: '/assets/images/team/square-small/pablo-villalobos.jpg', resources: [ {name: 'mail', url: 'cGFibG9AZXBvY2guYWk=', icon: 'envelope'}, ] }, }; let tippyInstances = []; for (let authorNode of document.querySelectorAll('.post-properties .post-author')) { let author = authorInfo[authorNode.dataset.authorId]; if (!author) continue; let resourcesHtml = []; if (author.resources) { for (let resource of author.resources) { if (resource.name == 'mail') { resourcesHtml.push(`<a class="resource" data-contact="${resource.url}" href="#"><i class="bi bi-envelope"></i></a>`); } else { resourcesHtml.push(`<a class="resource" href="${resource.url}"><i class="bi bi-${resource.icon}"></i></a>`); } } } let instance = tippy(authorNode, { allowHTML: true, placement: 'bottom-start', theme: 'round-light', interactive: true, trigger: 'mouseenter click', offset: [20, 4], arrow: false, appendTo: document.body, onShow: (instance) => { tippy.hideAll(); /* if (window.innerWidth < 1000) { return false; } */ for (let element of instance.popper.querySelectorAll('[data-contact]')) { addMailDeobfuscator(element); } for (let element of instance.popper.querySelectorAll('[data-micromodal-trigger]')) { element.addEventListener('click', () => { MicroModal.show(element.dataset.micromodalTrigger); }); } }, content: ` <div class="author-card-container"> <div class="left" style="--mug-url: url(${author.mug})"> </div> <div class="right"> <div class="top"> <div class="name">${author.name}</div> <div class="position">${author.position}</div> </div> <div class="bottom"> <div class="resources"> ${resourcesHtml.join('\n')} </div>  </div> </div> </div> `, }); tippyInstances.push(instance); } } } </script> </div> </div><div id="footer-container"> <div class="banner"> <span class="banner-text">Excited about our work?</span> <div class="banner-buttons"> <a role="button" class="button banner-button outline-light-medium-button" data-contact="aW5mb0BlcG9jaGFpLm9yZw==" href="#">Talk to us</a> <a href="/support-us" role="button" class="button banner-button outline-light-medium-button">Support our research</a> </div> </div> <div class="footer"> <div class="top"> <div class="logo-newsletter"> <div class="logo-wrapper"> <img class="logo" src="/assets/images/epoch-full-white.svg" alt="Epoch AI's logo"> </div> <div class="newsletter"> <span class="newsletter-text">Sign up for our newsletter to read weekly commentary on AI news and developments and the latest updates on our research.</span> <a class="button banner-button outline-light-medium-button" href="https://epochai.substack.com/subscribe">Subscribe to our newsletter</a> </div> <div class="social"> <a class="social-item" href="https://twitter.com/EpochAIResearch"> <svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg"> <path d="M29.9441 8.00212C28.9174 8.46878 27.8108 8.77545 26.6641 8.92212C27.8374 8.21545 28.7441 7.09545 29.1708 5.74878C28.0641 6.41545 26.8374 6.88212 25.5441 7.14878C24.4908 6.00212 23.0108 5.33545 21.3308 5.33545C18.1974 5.33545 15.6374 7.89545 15.6374 11.0555C15.6374 11.5088 15.6908 11.9488 15.7841 12.3621C11.0374 12.1221 6.81078 9.84212 3.99745 6.38878C3.50411 7.22878 3.22411 8.21545 3.22411 9.25545C3.22411 11.2421 4.22411 13.0021 5.77078 14.0021C4.82411 14.0021 3.94411 13.7355 3.17078 13.3355V13.3755C3.17078 16.1488 5.14411 18.4688 7.75745 18.9888C6.91856 19.2194 6.0375 19.2513 5.18411 19.0821C5.54626 20.2188 6.2555 21.2133 7.21214 21.926C8.16878 22.6387 9.32471 23.0337 10.5174 23.0555C8.49567 24.6561 5.98946 25.5213 3.41078 25.5088C2.95745 25.5088 2.50411 25.4821 2.05078 25.4288C4.58411 27.0555 7.59745 28.0021 10.8241 28.0021C21.3308 28.0021 27.1041 19.2821 27.1041 11.7221C27.1041 11.4688 27.1041 11.2288 27.0908 10.9754C28.2108 10.1754 29.1708 9.16212 29.9441 8.00212Z" fill="#CCD8D9"/> </svg> </a> <a class="social-item" href="https://github.com/epoch-research"> <svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg"> <path d="M15.9996 2.66724C14.2486 2.66724 12.5148 3.01211 10.8971 3.68218C9.27947 4.35224 7.80962 5.33436 6.5715 6.57248C4.07102 9.07296 2.66626 12.4643 2.66626 16.0006C2.66626 21.8939 6.49293 26.8939 11.7863 28.6672C12.4529 28.7739 12.6663 28.3606 12.6663 28.0006V25.7472C8.97293 26.5472 8.18626 23.9606 8.18626 23.9606C7.57293 22.4139 6.70626 22.0006 6.70626 22.0006C5.49293 21.1739 6.79959 21.2006 6.79959 21.2006C8.13293 21.2939 8.83959 22.5739 8.83959 22.5739C9.99959 24.6006 11.9596 24.0006 12.7196 23.6806C12.8396 22.8139 13.1863 22.2272 13.5596 21.8939C10.5996 21.5606 7.49293 20.4139 7.49293 15.3339C7.49293 13.8539 7.99959 12.6672 8.86626 11.7206C8.73293 11.3872 8.26626 10.0006 8.99959 8.20057C8.99959 8.20057 10.1196 7.84057 12.6663 9.56057C13.7196 9.26723 14.8663 9.12057 15.9996 9.12057C17.1329 9.12057 18.2796 9.26723 19.3329 9.56057C21.8796 7.84057 22.9996 8.20057 22.9996 8.20057C23.7329 10.0006 23.2663 11.3872 23.1329 11.7206C23.9996 12.6672 24.5063 13.8539 24.5063 15.3339C24.5063 20.4272 21.3863 21.5472 18.4129 21.8806C18.8929 22.2939 19.3329 23.1072 19.3329 24.3472V28.0006C19.3329 28.3606 19.5463 28.7872 20.2263 28.6672C25.5196 26.8806 29.3329 21.8939 29.3329 16.0006C29.3329 14.2496 28.9881 12.5158 28.318 10.8981C27.6479 9.28045 26.6658 7.81059 25.4277 6.57248C24.1896 5.33436 22.7197 4.35224 21.102 3.68218C19.4844 3.01211 17.7506 2.66724 15.9996 2.66724Z" fill="#CCD8D9"/> </svg> </a> <a class="social-item linkedin-social-item" href="https://www.linkedin.com/company/epochai"> <svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg"> <path d="M25.3336 3.99988C26.0408 3.99988 26.7191 4.28083 27.2192 4.78093C27.7193 5.28102 28.0002 5.9593 28.0002 6.66654V25.3332C28.0002 26.0405 27.7193 26.7187 27.2192 27.2188C26.7191 27.7189 26.0408 27.9999 25.3336 27.9999H6.66691C5.95967 27.9999 5.28139 27.7189 4.78129 27.2188C4.2812 26.7187 4.00024 26.0405 4.00024 25.3332V6.66654C4.00024 5.9593 4.2812 5.28102 4.78129 4.78093C5.28139 4.28083 5.95967 3.99988 6.66691 3.99988H25.3336ZM24.6669 24.6665V17.5999C24.6669 16.4471 24.209 15.3415 23.3938 14.5263C22.5786 13.7112 21.4731 13.2532 20.3202 13.2532C19.1869 13.2532 17.8669 13.9465 17.2269 14.9865V13.5065H13.5069V24.6665H17.2269V18.0932C17.2269 17.0665 18.0536 16.2265 19.0802 16.2265C19.5753 16.2265 20.0501 16.4232 20.4002 16.7733C20.7502 17.1233 20.9469 17.5981 20.9469 18.0932V24.6665H24.6669ZM9.17358 11.4132C9.76766 11.4132 10.3374 11.1772 10.7575 10.7571C11.1776 10.337 11.4136 9.7673 11.4136 9.17321C11.4136 7.93321 10.4136 6.91988 9.17358 6.91988C8.57596 6.91988 8.00281 7.15728 7.58023 7.57986C7.15765 8.00245 6.92024 8.57559 6.92024 9.17321C6.92024 10.4132 7.93358 11.4132 9.17358 11.4132ZM11.0269 24.6665V13.5065H7.33358V24.6665H11.0269Z" fill="#CCD8D9"/> </svg> </a> </div> </div> <div class="map-wrapper"> <div class="map"> <div class="map-group"> <span class="map-header">Research</span> <ul class="map-list"> <li class="map-list-item"><a href="/blog">Publications</a></li> <li class="map-list-item"><a href="/data">Data on AI</a></li> <li class="map-list-item"><a href="/trends">Key Trends & Insights</a></li> <li class="map-list-item"><a href="/gradient-updates">Gradient Updates</a></li> </ul> </div> <div class="map-group"> <span class="map-header">Projects</span> <ul class="map-list"> <li class="map-list-item"><a href="/frontiermath">FrontierMath</a></li> <li class="map-list-item"><a href="/tools/distributed-training#careers">Distributed Training</a></li> </ul> </div> <div class="map-group"> <span class="map-header">Company</span> <ul class="map-list"> <li class="map-list-item"><a href="/about">About Us</a></li> <li class="map-list-item"><a href="/research">About Our Research</a></li> <li class="map-list-item"><a href="/careers#careers"> Careers</a></li> <li class="map-list-item"><a href="/support-us">Support Us</a></li> <li class="map-list-item"><a data-contact="aW5mb0BlcG9jaGFpLm9yZw==" href="#">Contact</a></li> </ul> </div> </div> </div> </div> <div class="bottom"> <span class="copyright">@ 2025 Epoch AI</span> <div style="display: flex; gap: 1em"> <a href="/privacy">Privacy Notice</a> <a href="/cookies">Cookie Policy</a> </div> </div> </div> </div> <script> "use strict"; for (let container of document.querySelectorAll('.auto-fold')) { makeFoldable(container); } if (location.hash) { followInternalLink(location.hash); } </script> <script> if (tippy) { tippy('[data-tippy-content]', { theme: 'epoch-default', allowHTML: true, zIndex: 999999, }); } </script> <style> #cookie-notice { padding: 1.5rem; display: none; text-align: left; position: fixed; bottom: 1.5rem; left: 1.5rem; max-width: 560px; background: #102930; color: #F7FCFB; z-index: 9999999 !important; font-family: Messina Sans; border-radius: 8px; } .cookie-notice-content h4 { font-family: Messina Serif; font-weight: 170; line-height: 40.6px; font-size: 29px; letter-spacing: -0.02em; text-align: left; margin-bottom: 0.5rem; } .cookie-notice-content span { font-family: Messina Sans; font-size: 16px; font-weight: 400; line-height: 25.6px; letter-spacing: 0.005em; text-align: left; } .cookie-notice-btns { display: flex; justify-content: space-between; margin-top: 1rem; gap: 1rem; } .btn { flex: 1 1 auto; padding: 8px 24px; border-radius: 99999px; color: #F7FCFB; cursor: pointer; font-family: Messina Sans; font-size: 16px; line-height: 25.6px; } .btn-primary { background: #34D2B9; border: 1px solid #34D2B9; color: #102930; } .btn-primary:hover, .btn-secondary { background: transparent; border: 1px solid #F7FCFB; color: #F7FCFB; } .btn:active, .btn-secondary:hover { background: #07171A; border: 1px solid #07171A; color: #F7FCFB; } @media (max-width: 608px) { #cookie-notice { left: 0; bottom: 0; width: 100%; border-radius: 8px 8px 0 0; } } </style> <div id="cookie-notice"> <div class="cookie-notice-content"> <h4>We value your privacy</h4> <span>Our website uses cookies to enhance your browsing experience and analyze site traffic. By clicking ‘Accept All,’ you consent to our use of cookies as described in our <a href="/privacy">Privacy Policy</a> and <a href="/cookies">Cookie Policy</a>. If you wish to withdraw your consent, you can contact us at <a href="mailto:infra@epoch.ai">infra@epoch.ai</a>.</span> </div> <div class="cookie-notice-btns"> <button id="cookie-notice-decline" class="btn btn-secondary">Reject all</button> <button id="cookie-notice-accept" class="btn btn-primary">Accept all</button> </div> </div> <script> function createCookie(name, value, days) { var expires = ""; if (days) { var date = new Date(); date.setTime(date.getTime() + (days * 24 * 60 * 60 * 1000)); expires = "; expires=" + date.toUTCString(); } document.cookie = name + "=" + value + expires + "; path=/"; } function readCookie(name) { var nameEQ = name + "="; var ca = document.cookie.split(';'); for (var i = 0; i < ca.length; i++) { var c = ca[i]; while (c.charAt(0) == ' ') c = c.substring(1, c.length); if (c.indexOf(nameEQ) == 0) return c.substring(nameEQ.length, c.length); } return null; } function eraseCookie(name) { createCookie(name, "", -1); } function checkCookieConsent() { var cookieConsent = readCookie('cookie-notice-dismissed'); var consentDate = readCookie('cookie-consent-date'); if (cookieConsent && consentDate) { var currentDate = new Date(); var consentDateObj = new Date(consentDate); if ((currentDate - consentDateObj) / (1000 * 60 * 60 * 24) < 30) { setConsent(cookieConsent === 'true'); // Microsoft Clarity tracking code if (cookieConsent === 'true') { (function (c, l, a, r, i, t, y) { c[a] = c[a] || function () { (c[a].q = c[a].q || []).push(arguments) }; t = l.createElement(r); t.async = 1; t.src = "https://www.clarity.ms/tag/" + i; y = l.getElementsByTagName(r)[0]; y.parentNode.insertBefore(t, y); })(window, document, "clarity", "script", "mrxdks5ty9"); } return true; } else { eraseCookie('cookie-notice-dismissed'); eraseCookie('cookie-consent-date'); } } return false; } if (!checkCookieConsent()) { document.getElementById('cookie-notice').style.display = 'block'; document.getElementById('cookie-notice-accept').addEventListener("click", function () { createCookie('cookie-notice-dismissed', 'true', 31); createCookie('cookie-consent-date', new Date().toISOString(), 31); setConsent(true); document.getElementById('cookie-notice').style.display = 'none'; location.reload(); }); document.getElementById('cookie-notice-decline').addEventListener("click", function () { createCookie('cookie-notice-dismissed', 'false', 31); createCookie('cookie-consent-date', new Date().toISOString(), 31); setConsent(false); var cookies = document.cookie.split(";"); for (var i = 0; i < cookies.length; i++) { var cookie = cookies[i]; var eqPos = cookie.indexOf("="); var name = eqPos > -1 ? cookie.substr(0, eqPos) : cookie; if (name.trim().startsWith('_ga')) { document.cookie = name + "=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=/;"; } } document.getElementById('cookie-notice').style.display = 'none'; }); } </script> </body> <script> // Show page only after it has loaded document.addEventListener("DOMContentLoaded", () => { async function waitForFontReady() { let searchOverlay = document.querySelector("#search-overlay"); let ready = await document.fonts.ready; if (!searchOverlay.dataset.loaded) { searchOverlay.addEventListener("load", () => { document.body.style.opacity = 1; }); } else { document.body.style.opacity = 1; } } waitForFontReady(); }); // Jump to anchor on page load (it's not working by default) document.addEventListener("DOMContentLoaded", () => { let hash = window.location.hash; if (hash) { let element = document.getElementById(hash.slice(1)); if (element) { setTimeout(() => { freezeNavBar(); element.scrollIntoView(); unfreezeNavBarOnScrollStop(); }, 200); } } }); </script> </html>

CINXE.COM

Compute Trends Across Three Eras of Machine Learning | Epoch AI