Get started with Gemma on Ray on Vertex AI

<!DOCTYPE html> <html lang="en"> <head> <meta charset="utf-8" /> <title> Get started with Gemma on Ray on Vertex AI - Google Developers Blog </title> <meta property="og:title" content="Get started with Gemma on Ray on Vertex AI- Google Developers Blog" /> <meta name="viewport" content="width=device-width, initial-scale=1" /> <meta name="description" content="Use Gemma Supervised tuning on Ray on Vertex AI to train and serve machine learning models efficiently and effectively." /> <meta content="summary_large_image" name="twitter:card"/> <meta content="Google for Developers Blog - News about Web, Mobile, AI and Cloud" property="twitter:title"/> <meta property="og:title" content="Get started with Gemma on Ray on Vertex AI" /> <script type="application/ld+json"> { "@context": "https://schema.org", "@type": "BreadcrumbList", "itemListElement": [{ "@type": "ListItem", "position": 1, "name": "Google for Developers Blog", "item": "https://developers.googleblog.com/" },{ "@type": "ListItem", "position": 2, "name": "Get started with Gemma on Ray on Vertex AI", "item": "http://developers.googleblog.com/en/get-started-with-gemma-on-ray-on-vertex-ai/" }] } </script> <script type="application/ld+json"> { "@context": "https://schema.org", "@type": "Article", "headline": "Get started with Gemma on Ray on Vertex AI", "description": "Use Gemma Supervised tuning on Ray on Vertex AI to train and serve machine learning models efficiently and effectively.", "image": "https://storage.googleapis.com/gweb-developer-goog-blog-assets/images/Copy_of_Wagtail_Gemma_Blog_social_.2e16d0ba.fill-800x400.png", "datePublished": "2024-07-09", "author": [ { "@type": "Person", "name": "Ju-yeong Ji", "url": "/en/search/?author=Ju-yeong+Ji" }, { "@type": "Person", "name": "Ivan Nardini", "url": "/en/search/?author=Ivan+Nardini" } ] } </script> <meta content="https://storage.googleapis.com/gweb-developer-goog-blog-assets/images/Copy_of_Wagtail_Gemma_Blog_social.2e16d0ba.fill-1200x600.png" property="og:image"/>  <script type="text/javascript" src="https://storage.googleapis.com/gweb-developer-goog-blog-cms-assets/f677f1b8-8552-4a23-83cc-57c36dbaf782/js/analytics.js"></script>  <link href="//www.gstatic.com/glue/v27_1/glue.min.css" rel="stylesheet"> <link rel="stylesheet" type="text/css" href="https://storage.googleapis.com/gweb-developer-goog-blog-cms-assets/f677f1b8-8552-4a23-83cc-57c36dbaf782/css/dgc_blog.css"> <link rel="icon" href="https://storage.googleapis.com/gweb-developer-goog-blog-cms-assets/f677f1b8-8552-4a23-83cc-57c36dbaf782/images/favicon.ico" type="image/x-icon"> <link rel="preconnect" href="https://fonts.googleapis.com"> <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin> <link rel="preload" href="https://fonts.googleapis.com/css2?family=Product+Sans&family=Google+Sans+Display:ital@0;1&family=Google+Sans:ital,wght@0,400;0,500;0,700;1,400;1,500;1,700&family=Google+Sans+Text:ital,wght@0,400;0,500;0,700;1,400;1,500;1,700&display=swap" as="style"> <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Product+Sans&family=Google+Sans+Display:ital@0;1&family=Google+Sans:ital,wght@0,400;0,500;0,700;1,400;1,500;1,700&family=Google+Sans+Text:ital,wght@0,400;0,500;0,700;1,400;1,500;1,700&display=swap"> <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Roboto:wght@400&display=swap"> <link href="https://www.gstatic.com/glue/cookienotificationbar/cookienotificationbar.min.css" rel="stylesheet"> <link rel="stylesheet" type="text/css" href="https://storage.googleapis.com/gweb-developer-goog-blog-cms-assets/f677f1b8-8552-4a23-83cc-57c36dbaf782/css/blog_detail.css" /> <link rel="stylesheet" href="https://storage.googleapis.com/gweb-developer-goog-blog-cms-assets/f677f1b8-8552-4a23-83cc-57c36dbaf782/pygmentify/css/default.min.css"> </head> <body id="main-content" class="glue-body ">  <noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-WVTLDSL " height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>   <header class="dgc-header"> <div class="dgc-header-inner"> <button class="hamburger" aria-haspopup="true" aria-expanded="false" aria-label="Open Menu"> <svg role="presentation" aria-hidden="true" class="glue-icon"> <use href="/glue-icon/#menu"></use> </svg> </button> <div class="product-name-wrapper"> <a href="https://developers.google.com/" class="site-logo-link" data-label="Site logo"> <img src="https://storage.googleapis.com/gweb-developer-goog-blog-cms-assets/f677f1b8-8552-4a23-83cc-57c36dbaf782/images/g-dev.svg" class="site-logo" alt="Google for Developers"> </a> </div> <div class="desktop-nav-wrapper"> <div class="upper-tabs-wrapper"> <div class="upper-tabs"> <nav class="tabs" aria-label="Upper Tabs"> <div class="tab" dropdown> <a href="//developers.google.com/products" class="top-nav-title"> Products </a> <a href="#" role="button" aria-haspopup="true" aria-expanded="true" aria-label="Dropdown menu for Products" class="dropdown-toggle"> <svg role="presentation" aria-hidden="true" class="glue-icon"> <use href="/glue-icon/#arrow-drop-down"></use> </svg> </a> <div class="tabs-dropdown" aria-label="Submenu"> <div class="tabs-dropdown-content"> <div class="tabs-dropdown-column"> <ul class="tabs-dropdown-section"> <li class="nav-title" aria-label="Develop Menu">Develop</li> <li class="nav-item"> <a href="//developer.android.com"> <div class="nav-item-title"> Android </div> </a> </li> <li class="nav-item"> <a href="//developer.chrome.com"> <div class="nav-item-title"> Chrome </div> </a> </li> <li class="nav-item"> <a href="//chromeos.dev/"> <div class="nav-item-title"> ChromeOS </div> </a> </li> <li class="nav-item"> <a href="//cloud.google.com"> <div class="nav-item-title"> Cloud </div> </a> </li> <li class="nav-item"> <a href="//firebase.google.com"> <div class="nav-item-title"> Firebase </div> </a> </li> <li class="nav-item"> <a href="//flutter.dev"> <div class="nav-item-title"> Flutter </div> </a> </li> <li class="nav-item"> <a href="//developers.google.com/assistant"> <div class="nav-item-title"> Google Assistant </div> </a> </li> <li class="nav-item"> <a href="//developers.google.com/maps"> <div class="nav-item-title"> Google Maps Platform </div> </a> </li> <li class="nav-item"> <a href="//developers.google.com/workspace"> <div class="nav-item-title"> Google Workspace </div> </a> </li> <li class="nav-item"> <a href="//www.tensorflow.org"> <div class="nav-item-title"> TensorFlow </div> </a> </li> <li class="nav-item"> <a href="//developers.google.com/youtube"> <div class="nav-item-title"> YouTube </div> </a> </li> </ul> </div> <div class="tabs-dropdown-column"> <ul class="tabs-dropdown-section"> <li class="nav-title" aria-label="Grow Menu">Grow</li> <li class="nav-item"> <a href="//firebase.google.com"> <div class="nav-item-title"> Firebase </div> </a> </li> <li class="nav-item"> <a href="//developers.google.com/google-ads"> <div class="nav-item-title"> Google Ads </div> </a> </li> <li class="nav-item"> <a href="//developers.google.com/analytics"> <div class="nav-item-title"> Google Analytics </div> </a> </li> <li class="nav-item"> <a href="//developer.android.com/distribute"> <div class="nav-item-title"> Google Play </div> </a> </li> <li class="nav-item"> <a href="//developers.google.com/search"> <div class="nav-item-title"> Search </div> </a> </li> <li class="nav-item"> <a href="//developers.google.com/web/fundamentals/engage-and-retain/push-notifications"> <div class="nav-item-title"> Web Push and Notification APIs </div> </a> </li> </ul> </div> <div class="tabs-dropdown-column"> <ul class="tabs-dropdown-section"> <li class="nav-title" aria-label="Earn Menu">Earn</li> <li class="nav-item"> <a href="//developers.google.com/admob"> <div class="nav-item-title"> AdMob </div> </a> </li> <li class="nav-item"> <a href="//developers.google.com/google-ads/api"> <div class="nav-item-title"> Google Ads API </div> </a> </li> <li class="nav-item"> <a href="//developers.google.com/pay"> <div class="nav-item-title"> Google Pay </div> </a> </li> <li class="nav-item"> <a href="//developer.android.com/google/play/billing/"> <div class="nav-item-title"> Google Play Billing </div> </a> </li> <li class="nav-item"> <a href="//developers.google.com/interactive-media-ads"> <div class="nav-item-title"> Interactive Media Ads </div> </a> </li> </ul> </div> </div> </div> </div> <div class="tab"> <a href="//developers.google.com/solutions/catalog" class="top-nav-title"> Solutions </a> </div> <div class="tab"> <a href="//developers.google.com/events" class="top-nav-title"> Events </a> </div> <div class="tab"> <a href="//developers.google.com/learn" class="top-nav-title"> Learn </a> </div> <div class="tab" dropdown> <a href="//developers.google.com/community" class="top-nav-title"> Community </a> <a href="#" role="button" aria-haspopup="true" aria-expanded="true" aria-label="Dropdown menu for Community" class="dropdown-toggle"> <svg role="presentation" aria-hidden="true" class="glue-icon"> <use href="/glue-icon/#arrow-drop-down"></use> </svg> </a> <div class="tabs-dropdown" aria-label="Submenu"> <div class="tabs-dropdown-content"> <div class="tabs-dropdown-column"> <ul class="tabs-dropdown-section"> <li class="nav-title" aria-label="Groups Menu">Groups</li> <li class="nav-item"> <a href="//developers.google.com/community/gdg"> <div class="nav-item-title"> Google Developer Groups </div> </a> </li> <li class="nav-item"> <a href="//developers.google.com/community/gdsc"> <div class="nav-item-title"> Google Developer Student Clubs </div> </a> </li> <li class="nav-item"> <a href="//developers.google.com/womentechmakers"> <div class="nav-item-title"> Woman Techmakers </div> </a> </li> <li class="nav-item"> <a href="//developers.google.com/community/experts"> <div class="nav-item-title"> Google Developer Experts </div> </a> </li> <li class="nav-item"> <a href="//www.techequitycollective.com/"> <div class="nav-item-title"> Tech Equity Collective </div> </a> </li> </ul> </div> <div class="tabs-dropdown-column"> <ul class="tabs-dropdown-section"> <li class="nav-title" aria-label="Programs Menu">Programs</li> <li class="nav-item"> <a href="//developers.google.com/community/accelerators"> <div class="nav-item-title"> Accelerator </div> </a> </li> <li class="nav-item"> <a href="//developers.google.com/community/gdsc-solution-challenge"> <div class="nav-item-title"> Solution Challenge </div> </a> </li> <li class="nav-item"> <a href="//developers.google.com/community/devfest"> <div class="nav-item-title"> DevFest </div> </a> </li> </ul> </div> <div class="tabs-dropdown-column"> <ul class="tabs-dropdown-section"> <li class="nav-title" aria-label="Stories Menu">Stories</li> <li class="nav-item"> <a href="//developers.google.com/community/stories"> <div class="nav-item-title"> All Stories </div> </a> </li> </ul> </div> </div> </div> </div> <div class="tab"> <a href="//developers.google.com/profile/u/me" class="top-nav-title"> Developer Program </a> </div> <div class="tab"> <a href="//developers.googleblog.com/" class="top-nav-title"> Blog </a> </div> </nav> </div> </div> </div> </div> <div class="dgc-header-search"> <div class="search-wrapper glue-page"> <div class="glue-grid"> <form id="search-form" action="/en/search/" method="get" class="search-content glue-grid__col glue-grid__col--span-4-sm glue-grid__col--span-9-md glue-grid__col--span-7-lg"> <div class="search-input-wrapper"> <svg role="presentation" aria-hidden="true" class="glue-icon search-icon"> <use href="/glue-icon/#search"></use> </svg> <input type="text" name="query" placeholder="Search all articles..." aria-label="Search" class="search-input-field" /> </div> <button class="glue-button glue-button--high-emphasis"> Search </button> </form> <div class="language-selector glue-grid__col glue-grid__col--span-4-sm glue-grid__col--span-3-md glue-grid__col--span-5-lg" aria-label="Select your language preference"> <button type="button" aria-controls="lang-menu-header" aria-haspopup="true"> <svg role="presentation" aria-hidden="true" class="glue-icon lang-icon"> <use href="/glue-icon/#language"></use> </svg> English <svg role="presentation" aria-hidden="true" class="glue-icon lang-icon"> <use href="/glue-icon/#arrow-drop-down"></use> </svg> </button> <div id="lang-menu-header" class="lang-menu hidden" role="menu"> <ul role="presentation"> <li role="presentation"> <a role="menuitem" lang="en" aria-current="true" href="/en/get-started-with-gemma-on-ray-on-vertex-ai/">English</a> </li> <li role="presentation"> <a role="menuitem" lang="es" href="/es/get-started-with-gemma-on-ray-on-vertex-ai/">Español (Latam)</a> </li> <li role="presentation"> <a role="menuitem" lang="id" href="/id/get-started-with-gemma-on-ray-on-vertex-ai/">Bahasa Indonesia</a> </li> <li role="presentation"> <a role="menuitem" lang="ja" href="/ja/get-started-with-gemma-on-ray-on-vertex-ai/">日本語</a> </li> <li role="presentation"> <a role="menuitem" lang="ko" href="/ko/get-started-with-gemma-on-ray-on-vertex-ai/">한국어</a> </li> <li role="presentation"> <a role="menuitem" lang="pt-br" href="/pt-br/get-started-with-gemma-on-ray-on-vertex-ai/">Português (Brasil)</a> </li> <li role="presentation"> <a role="menuitem" lang="zh-hans" href="/zh-hans/get-started-with-gemma-on-ray-on-vertex-ai/">简体中文</a> </li> </ul> </div> </div> </div> </div> </div> </header> <div class="mobile-drawer" top-level-nav> <nav class="nav-content" aria-label="Side menu"> <div class="mobile-header"> <button class="nav-close-btn nav-btn" aria-label="Close navigation"> <svg role="presentation" aria-hidden="true" class="glue-icon"> <use href="/glue-icon/#close"></use> </svg> </button> <button class="nav-back-btn nav-btn hidden" aria-label="Back to Menu"> <svg role="presentation" aria-hidden="true" class="glue-icon"> <use href="/glue-icon/#arrow-back"></use> </svg> </button> <div class="product-name-wrapper"> <a href="https://developers.google.com/" class="site-logo-link" data-label="Site logo"> <img src="https://storage.googleapis.com/gweb-developer-goog-blog-cms-assets/f677f1b8-8552-4a23-83cc-57c36dbaf782/images/g-dev.svg" class="site-logo" alt="Google for Developers"> </a> </div> </div> <div class="nav-wrapper"> <div class="mobile-nav-top"> <ul class="nav-list"> <li class="nav-item"> <a href="//developers.google.com/products" class="nav-title" data-label="Tab: Products"> Products </a> <ul class="nav-responsive-tabs"> <li class="nav-item"> More <svg role="presentation" aria-hidden="true" class="glue-icon"> <use href="/glue-icon/#arrow-forward"></use> </svg> </li> </ul> </li> <li class="nav-item"> <a href="//developers.google.com/solutions/catalog" class="nav-title" data-label="Tab: Solutions"> Solutions </a> </li> <li class="nav-item"> <a href="//developers.google.com/events" class="nav-title" data-label="Tab: Events"> Events </a> </li> <li class="nav-item"> <a href="//developers.google.com/learn" class="nav-title" data-label="Tab: Learn"> Learn </a> </li> <li class="nav-item"> <a href="//developers.google.com/community" class="nav-title" data-label="Tab: Community"> Community </a> <ul class="nav-responsive-tabs"> <li class="nav-item"> More <svg role="presentation" aria-hidden="true" class="glue-icon"> <use href="/glue-icon/#arrow-forward"></use> </svg> </li> </ul> </li> <li class="nav-item"> <a href="//developers.google.com/profile/u/me" class="nav-title" data-label="Tab: Developer Program"> Developer Program </a> </li> <li class="nav-item"> <a href="//developers.googleblog.com/" class="nav-title" data-label="Tab: Blog"> Blog </a> </li> </ul> </div> <div class="mobile-nav-bottom"> <ul class="nav-list products-submenu" menu="Products" aria-label="Side menu"> <li class="nav-item nav-heading"> Develop </li> <li class="nav-item"> <a href="//developer.android.com" class="nav-title"> Android </a> </li> <li class="nav-item"> <a href="//developer.chrome.com" class="nav-title"> Chrome </a> </li> <li class="nav-item"> <a href="//chromeos.dev/" class="nav-title"> ChromeOS </a> </li> <li class="nav-item"> <a href="//cloud.google.com" class="nav-title"> Cloud </a> </li> <li class="nav-item"> <a href="//firebase.google.com" class="nav-title"> Firebase </a> </li> <li class="nav-item"> <a href="//flutter.dev" class="nav-title"> Flutter </a> </li> <li class="nav-item"> <a href="//developers.google.com/assistant" class="nav-title"> Google Assistant </a> </li> <li class="nav-item"> <a href="//developers.google.com/maps" class="nav-title"> Google Maps Platform </a> </li> <li class="nav-item"> <a href="//developers.google.com/workspace" class="nav-title"> Google Workspace </a> </li> <li class="nav-item"> <a href="//www.tensorflow.org" class="nav-title"> TensorFlow </a> </li> <li class="nav-item"> <a href="//developers.google.com/youtube" class="nav-title"> YouTube </a> </li> <li class="nav-item nav-heading"> Grow </li> <li class="nav-item"> <a href="//firebase.google.com" class="nav-title"> Firebase </a> </li> <li class="nav-item"> <a href="//developers.google.com/google-ads" class="nav-title"> Google Ads </a> </li> <li class="nav-item"> <a href="//developers.google.com/analytics" class="nav-title"> Google Analytics </a> </li> <li class="nav-item"> <a href="//developer.android.com/distribute" class="nav-title"> Google Play </a> </li> <li class="nav-item"> <a href="//developers.google.com/search" class="nav-title"> Search </a> </li> <li class="nav-item"> <a href="//developers.google.com/web/fundamentals/engage-and-retain/push-notifications" class="nav-title"> Web Push and Notification APIs </a> </li> <li class="nav-item nav-heading"> Earn </li> <li class="nav-item"> <a href="//developers.google.com/admob" class="nav-title"> AdMob </a> </li> <li class="nav-item"> <a href="//developers.google.com/google-ads/api" class="nav-title"> Google Ads API </a> </li> <li class="nav-item"> <a href="//developers.google.com/pay" class="nav-title"> Google Pay </a> </li> <li class="nav-item"> <a href="//developer.android.com/google/play/billing/" class="nav-title"> Google Play Billing </a> </li> <li class="nav-item"> <a href="//developers.google.com/interactive-media-ads" class="nav-title"> Interactive Media Ads </a> </li> </ul> <ul class="nav-list community-submenu hidden" menu="Community" aria-label="Side menu"> <li class="nav-item nav-heading"> Groups </li> <li class="nav-item"> <a href="//developers.google.com/community/gdg" class="nav-title"> Google Developer Groups </a> </li> <li class="nav-item"> <a href="//developers.google.com/community/gdsc" class="nav-title"> Google Developer Student Clubs </a> </li> <li class="nav-item"> <a href="//developers.google.com/womentechmakers" class="nav-title"> Woman Techmakers </a> </li> <li class="nav-item"> <a href="//developers.google.com/community/experts" class="nav-title"> Google Developer Experts </a> </li> <li class="nav-item"> <a href="//www.techequitycollective.com/" class="nav-title"> Tech Equity Collective </a> </li> <li class="nav-item nav-heading"> Programs </li> <li class="nav-item"> <a href="//developers.google.com/community/accelerators" class="nav-title"> Accelerator </a> </li> <li class="nav-item"> <a href="//developers.google.com/community/gdsc-solution-challenge" class="nav-title"> Solution Challenge </a> </li> <li class="nav-item"> <a href="//developers.google.com/community/devfest" class="nav-title"> DevFest </a> </li> <li class="nav-item nav-heading"> Stories </li> <li class="nav-item"> <a href="//developers.google.com/community/stories" class="nav-title" aria-label="All Stories"> All Stories </a> </li> </ul> </div> </div> </nav> <select aria-label="Change language" name="lang-selector" id="lang-selector-nav" class="glue-form__dropdown glue-mobile-nav__lang-dropdown" onchange="window.location.href=this.value" > <option value="/en/get-started-with-gemma-on-ray-on-vertex-ai/" lang="en" selected > English </option> <option value="/es/get-started-with-gemma-on-ray-on-vertex-ai/" lang="es" > Español (Latam) </option> <option value="/id/get-started-with-gemma-on-ray-on-vertex-ai/" lang="id" > Bahasa Indonesia </option> <option value="/ja/get-started-with-gemma-on-ray-on-vertex-ai/" lang="ja" > 日本語 </option> <option value="/ko/get-started-with-gemma-on-ray-on-vertex-ai/" lang="ko" > 한국어 </option> <option value="/pt-br/get-started-with-gemma-on-ray-on-vertex-ai/" lang="pt-br" > Português (Brasil) </option> <option value="/zh-hans/get-started-with-gemma-on-ray-on-vertex-ai/" lang="zh-hans" > 简体中文 </option> </select> </div> <div class="backdrop"></div> <div class="blog-detail-container"> <section class="tags-container glue-page glue-spacer-5-top"> <div class="glue-eyebrow"><a href="/en/search/?product_categories=Gemma">Gemma</a></div> </section> <section class="heading-container glue-page glue-spacer-1-top"> <h1 class="glue-headline glue-headline--headline-1">Get started with Gemma on Ray on Vertex AI</h1> </section> <section class="summary-container glue-page glue-spacer-4-top"> <div class="date-time"> <div class="published-date glue-font-weight-medium">JUL 09, 2024</div> </div> </section> <section class="glue-page glue-grid glue-spacer-1-top"> <section class="author-container glue-grid__col glue-grid__col--span-4-sm glue-grid__col--span-10-md"> <div class="author-obj"> <a class="glue-font-weight-medium" href="/en/search/?author=Ju-yeong+Ji">Ju-yeong Ji</a> Gemma DevRel </div> <div class="author-obj"> <a class="glue-font-weight-medium" href="/en/search/?author=Ivan+Nardini">Ivan Nardini</a> Customer Engineer Google Cloud </div> </section> <section class="social-container glue-grid__col glue-grid__col--span-4-sm glue-grid__col--span-2-md"> <button id="social-button" class="glue-button glue-button--low-emphasis glue-button--icon" aria-haspopup="true" aria-expanded="false"> <svg role="presentation" aria-hidden="true" class="glue-icon"> <use href="/glue-icon/#share"></use> </svg> Share </button> <ul id="social-menu" class="glue-elevation-level-1" role="menu" aria-labelledby="social-button"> <li> <a href="https://www.facebook.com/sharer/sharer.php?u={url}" title="Share on Facebook" target="_blank" rel="noopener"> <svg role="presentation" aria-hidden="true" class="glue-icon glue-icon--social glue-icon--32px"> <use href="/glue-icon/#post-facebook"></use> </svg> Facebook </a> </li> <li> <a href="https://twitter.com/intent/tweet?text={url}" title="Share on Twitter" target="_blank" rel="noopener"> <svg role="presentation" aria-hidden="true" class="glue-icon glue-icon--social glue-icon--32px"> <use href="/glue-icon/#twitter-x"></use> </svg> Twitter </a> </li> <li> <a href="https://www.linkedin.com/shareArticle?url={url}&mini=true" title="Share on LinkedIn" target="_blank" rel="noopener"> <svg role="presentation" aria-hidden="true" class="glue-icon glue-icon--social glue-icon--32px"> <use href="/glue-icon/#post-linkedin"></use> </svg> LinkedIn </a> </li> <li> <a href="mailto:name@example.com?subject=Check%20out%20this%20site&body=Check%20out%20{url}" title="Send via Email"> <svg role="presentation" aria-hidden="true" class="glue-icon glue-icon--social glue-icon--32px"> <use href="/glue-icon/#email"></use> </svg> Mail </a> </li> <li> <a href="#" title="Get shareable link" data-link="" data-copy-text="Copy Link" data-copied-text="Copied!"> <svg role="presentation" aria-hidden="true" class="glue-icon glue-icon--social glue-icon--32px"> <use href="/glue-icon/#link"></use> </svg> </a> </li> </ul> </section> </section> <section class="blocks-container glue-page glue-spacer-3-top"> <div class="block"> <img class="banner-image" src="https://storage.googleapis.com/gweb-developer-goog-blog-assets/images/Blog_Hero_-_Logo.original.png" alt="Gemma blog hero" /> <div class="inner-block-content rich-content"> <a href="https://ai.google.dev/gemma">Gemma</a> is a family of open models built from the same research and technology used to create the Gemini models. Gemma models are capable of performing a wide range of tasks, including text generation, code completion and generation, fine-tuning for specific tasks, and running on various devices.<a href="https://docs.ray.io/en/latest/index.html">Ray</a> is an open-source framework for scaling AI and Python applications. Ray provides the infrastructure to perform distributed computing and parallel processing for your machine learning (ML) workflow.By the end of this tutorial, you'll have a solid understanding of how to use Gemma Supervised tuning on Ray on Vertex AI to train and serve machine learning models efficiently and effectively.You can explore the "<a href="https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks%2Fcommunity%2Fmodel_garden%2Fmodel_garden_gemma_fine_tuning_batch_deployment_on_rov.ipynb">Get started with Gemma on Ray on Vertex AI</a>" tutorial notebook on GitHub to learn more about Gemma on Ray. All the code below is on this notebook to make your journey easier.<h2 data-block-key="26fr7"> Prerequisite</h2>The following steps are required, regardless of your environment.<ol><li data-block-key="76s7a"><a href="https://colab.research.google.com/corgiredirector?site=https%3A%2F%2Fconsole.cloud.google.com%2Fcloud-resource-manager">Select or create a Google Cloud project</a>.</li></ol>2. <a href="https://colab.research.google.com/corgiredirector?site=https%3A%2F%2Fcloud.google.com%2Fbilling%2Fdocs%2Fhow-to%2Fmodify-project">Make sure that billing is enabled for your project</a>.3. <a href="https://colab.research.google.com/corgiredirector?site=https%3A%2F%2Fconsole.cloud.google.com%2Fflows%2Fenableapi%3Fapiid%3Daiplatform.googleapis.com%2Cartifactregistry.googleapis.com%2Ccloudbuild.googleapis.com">Enable APIs</a>.If you’re running this tutorial locally, you need to install the <a href="https://colab.research.google.com/corgiredirector?site=https%3A%2F%2Fcloud.google.com%2Fsdk">Cloud SDK</a>.<h3 data-block-key="83otv"> Costs</h3>This tutorial uses billable components of Google Cloud:<ul><li data-block-key="c0e1u">Vertex AI</li></ul><ul><li data-block-key="fb6ku">Cloud Build</li></ul><ul><li data-block-key="7e8p4">Artifact Registry</li></ul><ul><li data-block-key="frmi8">Cloud Storage</li></ul>Learn about pricing, use the <a href="https://cloud.google.com/products/calculator/">Pricing Calculator</a> to generate a cost estimate based on your projected usage.<h2 data-block-key="26c4h"> What you need</h2><h3 data-block-key="ddphc">Dataset</h3>We’ll use the <a href="https://huggingface.co/datasets/EdinburghNLP/xsum">Extreme Summarization (XSum) dataset</a>, which is a dataset about abstractive single-document summarization systems.<h3 data-block-key="68amr"> Cloud Storage Bucket</h3>You have to create <a href="https://console.cloud.google.com/storage">a storage bucket</a> to store intermediate artifacts such as datasets. </div> <div class="inner-block-content"> <div class="image-wrapper"> <img class="regular-image" src="https://storage.googleapis.com/gweb-developer-goog-blog-assets/images/image6_VAOPymp.original.png" alt="Create a storage bucket to store intermediate artifacts" /> </div> </div> <div class="inner-block-content rich-content"> Or with Google Cloud CLI </div> <div class="inner-block-content code-block"> <div class="highlight"><pre class="python">gsutil mb -l {REGION} -p {PROJECT_ID} {BUCKET_URI} # for example: gsutil mb -l asia-northeast1 -p test-bebechien gs://test-bebechien-ray-bucket </pre></div> </div> <div class="inner-block-content rich-content"> <h2 data-block-key="vn49f">Docker Image Repository</h2>To store the custom cluster image, create a Docker repository in the <a href="https://console.cloud.google.com/artifacts">Artifact Registry</a>. </div> <div class="inner-block-content"> <div class="image-wrapper"> <img class="regular-image" src="https://storage.googleapis.com/gweb-developer-goog-blog-assets/images/image1_OTNw0dv.original.png" alt="Create a Docker repository in the Artifact Registry." /> </div> </div> <div class="inner-block-content rich-content"> Or with Google Cloud CLI </div> <div class="inner-block-content code-block"> <div class="highlight"><pre class="python">gcloud artifacts repositories create your-repo --repository-format=docker --location=your-region --description="Tutorial repository" </pre></div> </div> <div class="inner-block-content rich-content"> <h2 data-block-key="vn49f">Vertex AI TensorBoard Instance</h2>A TensorBoard instance is for tracking and monitoring your tuning jobs. You can create one from <a href="https://console.cloud.google.com/vertex-ai/experiments/tensorboard-instances">Experiments</a>. </div> <div class="inner-block-content"> <div class="image-wrapper"> <img class="regular-image" src="https://storage.googleapis.com/gweb-developer-goog-blog-assets/images/image3_Gtok2lm.original.png" alt="Create a TensorBoard instance from Experiments" /> </div> </div> <div class="inner-block-content rich-content"> Or with Google Cloud CLI </div> <div class="inner-block-content code-block"> <div class="highlight"><pre class="python">gcloud ai tensorboards create --display-name your-tensorboard --project your-project --region your-region </pre></div> </div> <div class="inner-block-content rich-content"> <h2 data-block-key="vn49f">How to set a Ray cluster on Vertex AI</h2><h3 data-block-key="ba96t">Build the custom cluster image</h3>To get started with Ray on Vertex AI, you can choose to either create a Dockerfile for a custom image from scratch or utilize one of the pre-built Ray base images. One such base image is available <a href="http://us-docker.pkg.dev/vertex-ai/training/ray-gpu.2-9.py310:latest">here</a>.First, prepare the requirements file that includes the dependencies your Ray application needs to run.<ul><li data-block-key="8o9ke"><a href="https://github.com/bebechien/gemma/blob/main/ray/tutorial/build/requirements.txt"><code>tutorial/build/requirements.txt</code></a></li></ul>Then, create the Dockerfile for the custom image by leveraging one of the prebuilt Ray on Vertex AI base images.<ul><li data-block-key="cb0a4"><a href="https://github.com/bebechien/gemma/blob/main/ray/tutorial/build/Dockerfile"><code>tutorial/build/Dockerfile</code></a></li></ul>Finally, build the Ray cluster custom image using Cloud Build. </div> <div class="inner-block-content code-block"> <div class="highlight"><pre class="python">gcloud builds submit --region=your-region --tag=your-region-docker.pkg.dev/your-project/your-repo/train --machine-type=E2_HIGHCPU_32 ./dockerfile-path </pre></div> </div> <div class="inner-block-content rich-content"> If everything goes well, you’ll see the custom image has been successfully pushed to your docker image repository. </div> <div class="inner-block-content"> <div class="image-wrapper"> <img class="regular-image" src="https://storage.googleapis.com/gweb-developer-goog-blog-assets/images/image4_DbF7N1n.original.png" alt="The custom image has been successfully pushed to your docker image repository." /> </div> </div> <div class="inner-block-content rich-content"> Also on your <a href="https://console.cloud.google.com/artifacts">Artifact Registry</a> </div> <div class="inner-block-content"> <div class="image-wrapper"> <img class="regular-image" src="https://storage.googleapis.com/gweb-developer-goog-blog-assets/images/image7_mRT491c.original.png" alt="The custom image has been successfully pushed to your artifact registry" /> </div> </div> <div class="inner-block-content rich-content"> <h2 data-block-key="vn49f">Create the Ray Cluster</h2>You can create the ray cluster from <a href="https://console.cloud.google.com/vertex-ai/ray">Ray on Vertex AI</a>. </div> <div class="inner-block-content"> <div class="image-wrapper"> <img class="regular-image" src="https://storage.googleapis.com/gweb-developer-goog-blog-assets/images/image9_iW9j9bT.original.png" alt="Ray on Vertex AI" /> </div> </div> <div class="inner-block-content rich-content"> Or use the Vertex AI Python SDK to create a Ray cluster with a custom image and to customize the cluster configuration. To learn more about the cluster configuration, see the <a href="https://cloud.google.com/vertex-ai/docs/open-source/ray-on-vertex-ai/create-cluster#ray-on-vertex-ai-sdk">documentation</a>.Below is an example Python code to create the Ray cluster with the predefined custom configuration.NOTE: Creating a cluster can take several minutes, depending on its configuration. </div> <div class="inner-block-content code-block"> <div class="highlight"><pre class="python"># Set up Ray on Vertex AI import vertex_ray from google.cloud import aiplatform as vertex_ai from vertex_ray import NodeImages, Resources # Retrieves an existing managed tensorboard given a tensorboard ID tensorboard = vertex_ai.Tensorboard(your-tensorboard-id, project=your-project, location=your-region) # Initialize the Vertex AI SDK for Python for your project vertex_ai.init(project=your-project, location=your-region, staging_bucket=your-bucket-uri, experiment_tensorboard=tensorboard) HEAD_NODE_TYPE = Resources( machine_type= "n1-standard-16", node_count=1, ) WORKER_NODE_TYPES = [ Resources( machine_type="n1-standard-16", node_count=1, accelerator_type="NVIDIA_TESLA_T4", accelerator_count=2, ) ] CUSTOM_IMAGES = NodeImages( head="your-region-docker.pkg.dev/your-project/your-repo/train", worker="your-region-docker.pkg.dev/your-project/your-repo/train", ) ray_cluster_name = vertex_ray.create_ray_cluster( head_node_type=HEAD_NODE_TYPE, worker_node_types=WORKER_NODE_TYPES, custom_images=CUSTOM_IMAGES, cluster_name=”your-cluster-name”, ) </pre></div> </div> <div class="inner-block-content rich-content"> Now you can get the Ray cluster with <code>get_ray_cluster()</code>. Use <a href="https://github.com/googleapis/python-aiplatform/blob/cd85d8f74d3922de3f871415bacf77c594f0c547/google/cloud/aiplatform/preview/vertex_ray/cluster_init.py#L340"><code>list_ray_clusters()</code></a> if you want to see all clusters associated with your project. </div> <div class="inner-block-content code-block"> <div class="highlight"><pre class="python">ray_clusters = vertex_ray.list_ray_clusters() ray_cluster_resource_name = ray_clusters[-1].cluster_resource_name ray_cluster = vertex_ray.get_ray_cluster(ray_cluster_resource_name) print("Ray cluster on Vertex AI:", ray_cluster_resource_name) </pre></div> </div> <div class="inner-block-content rich-content"> <h2 data-block-key="vn49f">Fine-Tune Gemma with Ray on Vertex AI</h2>To fine-tune Gemma with Ray on Vertex AI, you can use <a href="https://docs.ray.io/en/latest/train/train.html">Ray Train</a> for distributing HuggingFace Transformers with PyTorch training, as you can see below.With Ray Train, you define a training function which contains your HuggingFace Transformers code for tuning Gemma that you want to distribute. Next, you define the scaling configuration to specify the desired number of workers and indicate whether the distributed training process requires GPUs. Additionally, you can define a runtime configuration to specify checkpointing and synchronization behaviors. Finally, you submit the fine-tuning by initiating a TorchTrainer and run it using its fit method.In this tutorial, we’ll fine-tune Gemma 2B (<code>gemma-2b-it</code>) for summarizing newspaper articles using HuggingFace Transformer on Ray on Vertex AI. We wrote a simple Python <code>trainer.py</code> script and will submit it to the Ray cluster.<h3 data-block-key="8tphm"> Prepare Python Scripts</h3>Let’s prepare the train script, below is an example Python script for initializing Gemma fine-tuning using HuggingFace TRL library.<ul><li data-block-key="uith"><a href="https://github.com/bebechien/gemma/blob/main/ray/tutorial/src/train.py"><code>tutorial/src/train.py</code></a></li></ul>Next, prepare the distributed training script. Below is an example Python script for executing the Ray distributed training job.<ul><li data-block-key="5214n"><a href="https://github.com/bebechien/gemma/blob/main/ray/tutorial/src/trainer.py"><code>tutorial/src/trainer.py</code></a></li></ul>Now we submit the script to the Ray cluster using the Ray Jobs API via the Ray dashboard address. You can also find the dashboard address on the <a href="https://console.cloud.google.com/vertex-ai/ray">Cluster details page</a> like below. </div> <div class="inner-block-content"> <div class="image-wrapper"> <img class="regular-image" src="https://storage.googleapis.com/gweb-developer-goog-blog-assets/images/image5_dlnlzMZ.original.png" alt="Cluster details page showing the dashboard address" /> </div> </div> <div class="inner-block-content rich-content"> First, initiate the client to submit the job. </div> <div class="inner-block-content code-block"> <div class="highlight"><pre class="python">import ray from ray.job_submission import JobSubmissionClient client = JobSubmissionClient( address="vertex_ray://{}".format(ray_cluster.dashboard_address) ) </pre></div> </div> <div class="inner-block-content rich-content"> Let’s set some job configuration including model path, job id, prediction entrypoint, and more. </div> <div class="inner-block-content code-block"> <div class="highlight"><pre class="python">import random, string, datasets, transformers from etils import epath from huggingface_hub import login # Initialize some libraries settings login(token=”your-hf-token”) datasets.disable_progress_bar() transformers.set_seed(8) train_experiment_name = “your-experiment-name” train_submission_id = “your-submission-id” train_entrypoint = f"python3 trainer.py --experiment-name={train_experiment_name} --logging-dir=”your-bucket-uri/logs” --num-workers=2 --use-gpu" train_runtime_env = { "working_dir": "your-working-dir", "env_vars": {"HF_TOKEN": ”your-hf-token”, "TORCH_NCCL_ASYNC_ERROR_HANDLING": "3"}, } </pre></div> </div> <div class="inner-block-content rich-content"> Submit the job </div> <div class="inner-block-content code-block"> <div class="highlight"><pre class="python">train_job_id = client.submit_job( submission_id=train_submission_id, entrypoint=train_entrypoint, runtime_env=train_runtime_env, ) </pre></div> </div> <div class="inner-block-content rich-content"> Check the status of the job from the OSS dashboard. </div> <div class="inner-block-content"> <div class="image-wrapper"> <img class="regular-image" src="https://storage.googleapis.com/gweb-developer-goog-blog-assets/images/image8_XHfJIXp.original.png" alt="OSS dashboard showing job status" /> </div> </div> <div class="inner-block-content rich-content"> <h2 data-block-key="vn49f">Check training artifacts and monitor the training</h2>Using Ray on Vertex AI for developing AI/ML applications offers various benefits. In this scenario, you can use Cloud storage to conveniently store model checkpoints, metrics, and more. This allows you to quickly consume the model for AI/ML downstreaming tasks including monitoring the training process using Vertex AI TensorBoard or generating batch predictions using Ray Data.While the Ray training job is running and after it has completed, you see the model artifacts in the Cloud Storage location with Google Cloud CLI. </div> <div class="inner-block-content code-block"> <div class="highlight"><pre class="python">gsutil ls -l your-bucket-uri/your-experiments/your-experiment-name </pre></div> </div> <div class="inner-block-content rich-content"> You can use Vertex AI TensorBoard for validating your training job by logging resulting metrics. </div> <div class="inner-block-content code-block"> <div class="highlight"><pre class="python">vertex_ai.upload_tb_log( tensorboard_id=tensorboard.name, tensorboard_experiment_name=train_experiment_name, logdir=./experiments, ) </pre></div> </div> <div class="inner-block-content"> <div class="image-wrapper"> <img class="regular-image" src="https://storage.googleapis.com/gweb-developer-goog-blog-assets/images/image10_Bv17gCb.original.png" alt="Vertex AI TensorBoard" /> </div> </div> <div class="inner-block-content rich-content"> <h2 data-block-key="vn49f">Validate Gemma training on Vertex AI</h2>Assuming that your training runs successfully, you can generate predictions locally to validate the tuned model.First, download all resulting checkpoints from Ray job with Google Cloud CLI. </div> <div class="inner-block-content code-block"> <div class="highlight"><pre class="python"># copy all artifacts gsutil ls -l your-bucket-uri/your-experiments/your-experiment-name ./your-experiment-path </pre></div> </div> <div class="inner-block-content rich-content"> Use the <code>ExperimentAnalysis</code> method to retrieve the best checkpoint according to relevant metrics and mode. </div> <div class="inner-block-content code-block"> <div class="highlight"><pre class="python">import ray from ray.tune import ExperimentAnalysis experiment_analysis = ExperimentAnalysis(“./your-experiment-path”) log_path = experiment_analysis.get_best_trial(metric="eval_rougeLsum", mode="max") best_checkpoint = experiment_analysis.get_best_checkpoint( log_path, metric="eval_rougeLsum", mode="max" ) </pre></div> </div> <div class="inner-block-content rich-content"> Now you know which one is the best checkpoint. Below is an example output. </div> <div class="inner-block-content"> <div class="image-wrapper"> <img class="regular-image" src="https://storage.googleapis.com/gweb-developer-goog-blog-assets/images/Screenshot_2024-06-18_at_5.10.47PM.original.png" alt="example output - code snippet" /> </div> </div> <div class="inner-block-content rich-content"> And load the fine-tuned model as described in the Hugging Face <a href="https://huggingface.co/docs/trl/use_model#use-adapters-peft">documentation</a>.Below is an example Python code to load the base model and merge the adapters into the base model so you can use the model like a normal transformers model. You can find the saved tuned model at <code>tuned_model_path</code>. For example, “<code>tutorial/models/xsum-tuned-gemma-it</code>” </div> <div class="inner-block-content code-block"> <div class="highlight"><pre class="python">import torch from etils import epath from transformers import AutoModelForCausalLM, AutoTokenizer from peft import PeftModel base_model_path = "google/gemma-2b-it" peft_model_path = epath.Path(best_checkpoint.path) / "checkpoint" tuned_model_path = models_path / "xsum-tuned-gemma-it" tokenizer = AutoTokenizer.from_pretrained(base_model_path) tokenizer.padding_side = "right" base_model = AutoModelForCausalLM.from_pretrained( base_model_path, device_map="auto", torch_dtype=torch.float16 ) peft_model = PeftModel.from_pretrained( base_model, peft_model_path, device_map="auto", torch_dtype=torch.bfloat16, is_trainable=False, ) tuned_model = peft_model.merge_and_unload() tuned_model.save_pretrained(tuned_model_path) </pre></div> </div> <div class="inner-block-content rich-content"> Tidbit: Since you fine tuned a model, you can also publish it to the Hugging Face Hub by using this single line of code. </div> <div class="inner-block-content code-block"> <div class="highlight"><pre class="python">tuned_model.push_to_hub("my-awesome-model") </pre></div> </div> <div class="inner-block-content rich-content"> To generate summaries with the tuned model, let’s use the validation set of the tutorial dataset.The following Python code example demonstrates how to sample one article from a dataset to summarize. It then generates the associated summary and prints both the reference summary from the dataset and the generated summary side by side. </div> <div class="inner-block-content code-block"> <div class="highlight"><pre class="python">import random, datasets from transformers import pipeline dataset = datasets.load_dataset( "xsum", split="validation", cache_dir=”./data”, trust_remote_code=True ) sample = dataset.select([random.randint(0, len(dataset) - 1)]) document = sample["document"][0] reference_summary = sample["summary"][0] messages = [ { "role": "user", "content": f"Summarize the following ARTICLE in one sentence.\n###ARTICLE: {document}", }, ] prompt = tokenizer.apply_chat_template( messages, tokenize=False, add_generation_prompt=True ) tuned_gemma_pipeline = pipeline( "text-generation", model=tuned_model, tokenizer=tokenizer, max_new_tokens=50 ) generated_tuned_gemma_summary = tuned_gemma_pipeline( prompt, do_sample=True, temperature=0.1, add_special_tokens=True )[0]["generated_text"][len(prompt) :] print(f"Reference summary: {reference_summary}") print("-" * 100) print(f"Tuned generated summary: {generated_tuned_gemma_summary}") </pre></div> </div> <div class="inner-block-content rich-content"> Below is an example output from the tuned model. Note that the tuned result might require further refinement. To achieve optimal quality, it's necessary to iterate through the process several times, adjusting factors such as the learning rate and the number of training steps. </div> <div class="inner-block-content"> <div class="image-wrapper"> <img class="regular-image" src="https://storage.googleapis.com/gweb-developer-goog-blog-assets/images/Screenshot_2024-06-18_at_5.14.57PM.original.png" alt="Example of output from the tuned model" /> </div> </div> <div class="inner-block-content rich-content"> <h2 data-block-key="589pj">Evaluate the tuned model</h2>As an additional step, you can evaluate the tuned model. To evaluate the model you compare models qualitatively and quantitatively.In one case, you compare responses generated by the base Gemma model with the ones generated by the tuned Gemma model. In the other case, you calculate ROUGE metrics and its improvements which gives you an idea of how well the tuned model is able to reproduce the reference summaries correctly with respect to the base model.Below is a Python code to evaluate models by comparing generated summaries. </div> <div class="inner-block-content code-block"> <div class="highlight"><pre class="python">gemma_pipeline = pipeline( "text-generation", model=base_model, tokenizer=tokenizer, max_new_tokens=50 ) generated_gemma_summary = gemma_pipeline( prompt, do_sample=True, temperature=0.1, add_special_tokens=True )[0]["generated_text"][len(prompt) :] print(f"Reference summary: {reference_summary}") print("-" * 100) print(f"Base generated summary: {generated_gemma_summary}") print("-" * 100) print(f"Tuned generated summary: {generated_tuned_gemma_summary}") </pre></div> </div> <div class="inner-block-content rich-content"> Below is an example output from the base model and tuned model. </div> <div class="inner-block-content"> <div class="image-wrapper"> <img class="regular-image" src="https://storage.googleapis.com/gweb-developer-goog-blog-assets/images/Screenshot_2024-06-18_at_5.16.55PM.original.png" alt="Example output from the base model and tuned model." /> </div> </div> <div class="inner-block-content rich-content"> And below is a code to evaluate models by computing <a href="https://en.wikipedia.org/wiki/ROUGE_(metric)">ROUGE metrics</a> and its improvements. </div> <div class="inner-block-content code-block"> <div class="highlight"><pre class="python">import evaluate rouge = evaluate.load("rouge") gemma_results = rouge.compute( predictions=[generated_gemma_summary], references=[reference_summary], rouge_types=["rouge1", "rouge2", "rougeL", "rougeLsum"], use_aggregator=True, use_stemmer=True, ) tuned_gemma_results = rouge.compute( predictions=[generated_tuned_gemma_summary], references=[reference_summary], rouge_types=["rouge1", "rouge2", "rougeL", "rougeLsum"], use_aggregator=True, use_stemmer=True, ) improvements = {} for rouge_metric, gemma_rouge in gemma_results.items(): tuned_gemma_rouge = tuned_gemma_results[rouge_metric] if gemma_rouge != 0: improvement = ((tuned_gemma_rouge - gemma_rouge) / gemma_rouge) * 100 else: improvement = None improvements[rouge_metric] = improvement print("Base Gemma vs Tuned Gemma - ROUGE improvements") for rouge_metric, improvement in improvements.items(): print(f"{rouge_metric}: {improvement:.3f}%") </pre></div> </div> <div class="inner-block-content rich-content"> And the example output for the evaluation. </div> <div class="inner-block-content"> <div class="image-wrapper"> <img class="regular-image" src="https://storage.googleapis.com/gweb-developer-goog-blog-assets/images/Screenshot_2024-06-18_at_5.18.58PM.original.png" alt="Base Gemma v Tuned Gemma - Rouge improvements" /> </div> </div> <div class="inner-block-content rich-content"> <h2 data-block-key="vn49f">Serving tuned Gemma model with Ray Data for offline predictions</h2>To generate offline predictions at scale with the tuned Gemma on Ray on Vertex AI, you can use Ray Data, a scalable data processing library for ML workloads.Using Ray Data for generating offline predictions with Gemma, you need to define a Python class to load the tuned model in Hugging Face Pipeline. Then, depending on your data source and its format, you use Ray Data to perform distributed data reading and you use a Ray dataset method to apply the Python class for performing predictions in parallel to multiple batches of data.<h3 data-block-key="bpn6r"> Batch prediction with Ray Data</h3>To generate batch prediction with the tuned model using Ray Data on Vertex AI, you need a dataset to generate predictions and the tuned model stored in the Cloud bucket.Then, you can leverage Ray Data which provides an easy-to-use API for offline batch inference.First, upload the tuned model on the Cloud storage with Google Cloud CLI </div> <div class="inner-block-content code-block"> <div class="highlight"><pre class="python">gsutil -q cp -r “./models” “your-bucket-uri/models” </pre></div> </div> <div class="inner-block-content rich-content"> Prepare the batch prediction training script file for executing the Ray batch prediction job.<ul><li data-block-key="vo87"><a href="https://github.com/bebechien/gemma/blob/main/ray/tutorial/src/batch_predictor.py"><code>tutorial/src/batch_predictor.py</code></a></li></ul>Again, you can initiate the client to submit the job like below with the Ray Jobs API via the Ray dashboard address. </div> <div class="inner-block-content code-block"> <div class="highlight"><pre class="python">import ray from ray.job_submission import JobSubmissionClient client = JobSubmissionClient( address="vertex_ray://{}".format(ray_cluster.dashboard_address) ) </pre></div> </div> <div class="inner-block-content rich-content"> Let’s set some job configuration including model path, job id, prediction entrypoint and more. </div> <div class="inner-block-content code-block"> <div class="highlight"><pre class="python">import random, string batch_predict_submission_id = "your-batch-prediction-job" tuned_model_uri_path = "/gcs/your-bucket-uri/models" batch_predict_entrypoint = f"python3 batch_predictor.py --tuned_model_path={tuned_model_uri_path} --num_gpus=1 --output_dir=”your-bucket-uri/predictions”" batch_predict_runtime_env = { "working_dir": "tutorial/src", "env_vars": {"HF_TOKEN": “your-hf-token”}, } </pre></div> </div> <div class="inner-block-content rich-content"> You can specify the number of GPUs to use with the "--num_gpus" argument. This should be a value that is equal to or less than the number of GPUs available in your Ray cluster.And submit the job. </div> <div class="inner-block-content code-block"> <div class="highlight"><pre class="python">batch_predict_job_id = client.submit_job( submission_id=batch_predict_submission_id, entrypoint=batch_predict_entrypoint, runtime_env=batch_predict_runtime_env, ) </pre></div> </div> <div class="inner-block-content rich-content"> Let’s have a quick view of generated summaries using a Pandas DataFrame. </div> <div class="inner-block-content code-block"> <div class="highlight"><pre class="python">import io import pandas as pd from google.cloud import storage def read_json_files(bucket_name, prefix=None): """Reads JSON files from a cloud storage bucket and returns a Pandas DataFrame""" # Set up storage client storage_client = storage.Client() bucket = storage_client.bucket(bucket_name) blobs = bucket.list_blobs(prefix=prefix) dfs = [] for blob in blobs: if blob.name.endswith(".json"): file_bytes = blob.download_as_bytes() file_string = file_bytes.decode("utf-8") with io.StringIO(file_string) as json_file: df = pd.read_json(json_file, lines=True) dfs.append(df) return pd.concat(dfs, ignore_index=True) predictions_df = read_json_files(prefix="predictions/", bucket_name=”your-bucket-uri”) predictions_df = predictions_df[ ["id", "document", "prompt", "summary", "generated_summary"] ] predictions_df.head() </pre></div> </div> <div class="inner-block-content rich-content"> And below is an example output. The default number of articles to summarize is 20. You can specify the number with the “--sample_size” argument. </div> <div class="inner-block-content"> <div class="image-wrapper"> <img class="regular-image" src="https://storage.googleapis.com/gweb-developer-goog-blog-assets/images/Screenshot_2024-06-18_at_5.25.12PM.original.png" alt="Example output of summaries" /> </div> </div> <div class="inner-block-content rich-content"> <h2 data-block-key="vn49f">Summary</h2>Now you have learned many things including:<ul><li data-block-key="5p456">How to create a Ray cluster on Vertex AI</li></ul><ul><li data-block-key="qu2s">How to tune Gemma with Ray Train on Vertex AI</li></ul><ul><li data-block-key="cjbmj">How to validate Gemma training on Vertex AI</li></ul><ul><li data-block-key="4g9oj">How to evaluate tuned Gemma model</li></ul><ul><li data-block-key="borh2">How to serve Gemma with Ray Data for offline predictions</li></ul>We hope that this tutorial has been enlightening and provided you with valuable insights.Consider joining the <a href="https://discord.gg/google-dev-community">Google Developer Community</a> Discord server. It offers an opportunity to share your projects, connect with other developers, and engage in collaborative discussions.And don’t forget to clean up all Google Cloud resources used in this project. You can simply <a href="https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects">delete the Google Cloud project</a> that you used for the tutorial. Otherwise, you can delete the individual resources that you created. </div> <div class="inner-block-content code-block"> <div class="highlight"><pre class="python"># Delete tensorboard tensorboard_list = vertex_ai.Tensorboard.list() for tensorboard in tensorboard_list: tensorboard.delete() # Delete experiments experiment_list = vertex_ai.Experiment.list() for experiment in experiment_list: experiment.delete() # Delete ray on vertex cluster ray_cluster_list = vertex_ray.list_ray_clusters() for ray_cluster in ray_cluster_list: vertex_ray.delete_ray_cluster(ray_cluster.cluster_resource_name) </pre></div> </div> <div class="inner-block-content code-block"> <div class="highlight"><pre class="python"># Delete artifacts repo gcloud artifacts repositories delete “your-repo” -q # Delete Cloud Storage objects that were created gsutil -q -m rm -r “your-bucker-uri” </pre></div> </div> <div class="inner-block-content rich-content"> Thanks for reading!<h2 data-block-key="6gs0p"> References</h2><ul><li data-block-key="8oohv"><a href="https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_gemma_fine_tuning_batch_deployment_on_rov.ipynb">https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_gemma_fine_tuning_batch_deployment_on_rov.ipynb</a></li></ul><ul><li data-block-key="d8flq"><a href="https://cloud.google.com/vertex-ai/docs/open-source/ray-on-vertex-ai/overview">https://cloud.google.com/vertex-ai/docs/open-source/ray-on-vertex-ai/overview</a></li></ul><ul><li data-block-key="5pp6i"><a href="https://github.com/googleapis/python-aiplatform/tree/main/google/cloud/aiplatform/preview/vertex_ray">https://github.com/googleapis/python-aiplatform/tree/main/google/cloud/aiplatform/preview/vertex_ray</a></li></ul><ul><li data-block-key="ced0k"><a href="https://github.com/bebechien/gemma/tree/main/ray/tutorial">https://github.com/bebechien/gemma/tree/main/ray/tutorial</a></li></ul> </div> </div> </section> <section class="navigation-container glue-page glue-spacer-6-top"> <div class="posted-in-section"> <div class="posted-in-section__heading"> posted in: </div> <div class="posted-in-section__tags"> <ul> <li> <a href="/en/search/?product_categories=Gemma" class="glue-caption">Gemma</a> </li> <li> <a href="/en/search/?technology_categories=AI" class="glue-caption">AI</a> </li> <li> <a href="/en/search/?technology_categories=Cloud" class="glue-caption">Cloud</a> </li> <li> <a href="/en/search/?content_type_categories=Tutorials" class="glue-caption">Tutorials</a> </li> <li> <a href="/en/search/?content_type_categories=How-To+Guides" class="glue-caption">How-To Guides</a> </li> <li> <a href="/en/search/?tag=Learn" class="glue-caption">Learn</a> </li> </ul> </div> </div> <div class="buttons-section"> <div class="buttons-section__left"> <a href="/en/model-explorer-simplifying-ml-models-for-edge-devices/" class="glue-button--icon glue-elevation-level-1 " aria-label="Previous"> <svg role="presentation" aria-hidden="true" class="glue-icon"> <use href="/glue-icon/#chevron-left"></use> </svg> </a> Previous </div> <div class="buttons-section__right"> Next <a href="/en/how-its-made-ai-roadtrip-a-pixel-campaign-powered-by-generative-ai-and-fans/" class="glue-button--icon glue-elevation-level-1 " aria-label="Next"> <svg role="presentation" aria-hidden="true" class="glue-icon"> <use href="/glue-icon/#chevron-right"></use> </svg> </a> </div> </div> </section> <section class="related-posts-container glue-page glue-spacer-6-top glue-spacer-3-bottom"> Related Posts <div class="related-posts-container__carousel glue-page glue-spacer-5-top"> <div class="glue-carousel glue-carousel--cards glue-carousel-related-posts" aria-label="Related Posts">  <button class="glue-carousel__button glue-carousel__button--prev" aria-label="Go to the previous slide"> <svg role="presentation" aria-hidden="true" class="glue-icon glue-icon--32px"> <use href="/glue-icon/#chevron-left"></use> </svg> </button>  <button class="glue-carousel__button glue-carousel__button--next" aria-label="Go to the next slide"> <svg role="presentation" aria-hidden="true" class="glue-icon glue-icon--32px"> <use href="/glue-icon/#chevron-right"></use> </svg> </button>  <div class="glue-carousel__viewport"> <div class="glue-carousel__list"> <a class="glue-card glue-carousel__item" href="/en/firebase-demo-day-24/"> <div aria-label="Learn to build and run AI powered apps at Firebase Demo Day ‘24" class="glue-card__inner"> <picture class="glue-card__asset"> <img alt="Learn to build and run AI powered apps at Firebase Demo Day ‘24" src="https://storage.googleapis.com/gweb-developer-goog-blog-assets/images/Newletter_DayOf.2e16d0ba.fill-800x400.jpg"> </picture> <div class="glue-card__content"> <div class="glue-card__tags glue-spacer-2-top"> Firebase Mobile Web Tutorials How-To Guides </div> Learn to build and run AI powered apps at Firebase Demo Day ‘24 <div class="glue-card__cta-custom glue-spacer-3-top"> Nov. 19, 2024 <svg aria-hidden="true" class="glue-icon glue-icon--24px" role="presentation"> <use href="/glue-icon/#arrow-forward"></use> </svg> </div> </div> </div> </a> <a class="glue-card glue-carousel__item" href="/en/inference-with-gemma-using-dataflow-and-vllm/"> <div aria-label="Inference with Gemma using Dataflow and vLLM" class="glue-card__inner"> <picture class="glue-card__asset"> <img alt="Inference with Gemma using Dataflow and vLLM" src="https://storage.googleapis.com/gweb-developer-goog-blog-assets/images/Gemma-Dataflow-ML-vLLM.2e16d0ba.fill-800x400.png"> </picture> <div class="glue-card__content"> <div class="glue-card__tags glue-spacer-2-top"> Gemma AI Cloud How-To Guides Problem-Solving </div> Inference with Gemma using Dataflow and vLLM <div class="glue-card__cta-custom glue-spacer-3-top"> Nov. 13, 2024 <svg aria-hidden="true" class="glue-icon glue-icon--24px" role="presentation"> <use href="/glue-icon/#arrow-forward"></use> </svg> </div> </div> </div> </a> <a class="glue-card glue-carousel__item" href="/en/gemini-is-now-accessible-from-the-openai-library/"> <div aria-label="Gemini is now accessible from the OpenAI Library" class="glue-card__inner"> <picture class="glue-card__asset"> <img alt="Gemini is now accessible from the OpenAI Library" src="https://storage.googleapis.com/gweb-developer-goog-blog-assets/images/Gemini-Social-2_1.2e16d0ba.fill-800x400.png"> </picture> <div class="glue-card__content"> <div class="glue-card__tags glue-spacer-2-top"> Gemini AI Tutorials How-To Guides </div> Gemini is now accessible from the OpenAI Library <div class="glue-card__cta-custom glue-spacer-3-top"> Nov. 8, 2024 <svg aria-hidden="true" class="glue-icon glue-icon--24px" role="presentation"> <use href="/glue-icon/#arrow-forward"></use> </svg> </div> </div> </div> </a> <a class="glue-card glue-carousel__item" href="/en/announcing-the-winners-of-the-gemini-api-developer-competition/"> <div aria-label="Announcing the Winners of the Gemini API Developer Competition!" class="glue-card__inner"> <picture class="glue-card__asset"> <img alt="Announcing the Winners of the Gemini API Developer Competition!" src="https://storage.googleapis.com/gweb-developer-goog-blog-assets/images/Gemini_Competition_G4D_-_meta.2e16d0ba.fill-800x400.png"> </picture> <div class="glue-card__content"> <div class="glue-card__tags glue-spacer-2-top"> Mobile Web Case Studies Community </div> Announcing the Winners of the Gemini API Developer Competition! <div class="glue-card__cta-custom glue-spacer-3-top"> Nov. 21, 2024 <svg aria-hidden="true" class="glue-icon glue-icon--24px" role="presentation"> <use href="/glue-icon/#arrow-forward"></use> </svg> </div> </div> </div> </a> <a class="glue-card glue-carousel__item" href="/en/introducing-keras-hub-for-pretrained-models/"> <div aria-label="Introducing Keras Hub: Your one-stop shop for pretrained models" class="glue-card__inner"> <picture class="glue-card__asset"> <img alt="Introducing Keras Hub: Your one-stop shop for pretrained models" src="https://storage.googleapis.com/gweb-developer-goog-blog-assets/images/Keras-GfD.2e16d0ba.fill-800x400.png"> </picture> <div class="glue-card__content"> <div class="glue-card__tags glue-spacer-2-top"> Gemma AI Announcements Best Practices </div> Introducing Keras Hub: Your one-stop shop for pretrained models <div class="glue-card__cta-custom glue-spacer-3-top"> Oct. 22, 2024 <svg aria-hidden="true" class="glue-icon glue-icon--24px" role="presentation"> <use href="/glue-icon/#arrow-forward"></use> </svg> </div> </div> </div> </a> <a class="glue-card glue-carousel__item" href="/en/join-us-at-the-women-in-ai-summit-2024/"> <div aria-label="Join us at the Women in AI Summit 2024!" class="glue-card__inner"> <picture class="glue-card__asset"> <img alt="Join us at the Women in AI Summit 2024!" src="https://storage.googleapis.com/gweb-developer-goog-blog-assets/images/WAI-2024-Feature.2e16d0ba.fill-800x400.png"> </picture> <div class="glue-card__content"> <div class="glue-card__tags glue-spacer-2-top"> AI Community Events </div> Join us at the Women in AI Summit 2024! <div class="glue-card__cta-custom glue-spacer-3-top"> Nov. 22, 2024 <svg aria-hidden="true" class="glue-icon glue-icon--24px" role="presentation"> <use href="/glue-icon/#arrow-forward"></use> </svg> </div> </div> </div> </a> </div> </div>  <div class="glue-carousel__navigation" aria-label="Choose a page" data-glue-carousel-navigation-label="Selected tab $glue_carousel_page_number$ of $glue_carousel_page_total$"> </div> </div> </div> </section> </div> <div class="footer-linkboxes__wrapper"> <nav class="footer-linkboxes" aria-label="Footer links"> <ul class="footer-linkboxes__list"> <li class="footer-linkbox"> Connect <ul class="footer-linkbox-list"> <li class="footer-linkbox-list__item"> <a href="//googledevelopers.blogspot.com" class="footer-linkbox-list__link"> Blog </a> </li> <li class="footer-linkbox-list__item"> <a href="https://goo.gle/googlefordevs" class="footer-linkbox-list__link"> Instagram </a> </li> <li class="footer-linkbox-list__item"> <a href="https://goo.gle/gdevs-li" class="footer-linkbox-list__link"> LinkedIn </a> </li> <li class="footer-linkbox-list__item"> <a href="https://goo.gle/gdevs-tw" class="footer-linkbox-list__link"> Twitter </a> </li> <li class="footer-linkbox-list__item"> <a href="https://goo.gle/developers" class="footer-linkbox-list__link"> YouTube </a> </li> </ul> </li> <li class="footer-linkbox"> Programs <ul class="footer-linkbox-list"> <li class="footer-linkbox-list__item"> <a href="//www.womentechmakers.com" class="footer-linkbox-list__link"> Women Techmakers </a> </li> <li class="footer-linkbox-list__item"> <a href="//developers.google.com/community/gdg" class="footer-linkbox-list__link"> Google Developer Groups </a> </li> <li class="footer-linkbox-list__item"> <a href="//developers.google.com/community/experts" class="footer-linkbox-list__link"> Google Developer Experts </a> </li> <li class="footer-linkbox-list__item"> <a href="//developers.google.com/community/accelerators" class="footer-linkbox-list__link"> Accelerators </a> </li> <li class="footer-linkbox-list__item"> <a href="//developers.google.com/community/gdsc" class="footer-linkbox-list__link"> Google Developer Student Clubs </a> </li> </ul> </li> <li class="footer-linkbox"> Developer consoles <ul class="footer-linkbox-list"> <li class="footer-linkbox-list__item"> <a href="//console.developers.google.com" class="footer-linkbox-list__link"> Google API Console </a> </li> <li class="footer-linkbox-list__item"> <a href="//console.cloud.google.com" class="footer-linkbox-list__link"> Google Cloud Platform Console </a> </li> <li class="footer-linkbox-list__item"> <a href="//play.google.com/apps/publish" class="footer-linkbox-list__link"> Google Play Console </a> </li> <li class="footer-linkbox-list__item"> <a href="//console.firebase.google.com" class="footer-linkbox-list__link"> Firebase Console </a> </li> <li class="footer-linkbox-list__item"> <a href="//console.actions.google.com" class="footer-linkbox-list__link"> Actions on Google Console </a> </li> <li class="footer-linkbox-list__item"> <a href="//cast.google.com/publish" class="footer-linkbox-list__link"> Cast SDK Developer Console </a> </li> <li class="footer-linkbox-list__item"> <a href="//chrome.google.com/webstore/developer/dashboard" class="footer-linkbox-list__link"> Chrome Web Store Dashboard </a> </li> </ul> </li> </ul> </nav> </div> <div class="footer-utility__wrapper"> <div> <nav class="footer-sites" aria-label="Other Google Developers websites"> <a href="https://developers.google.com/" class="site-logo-link" data-label="Site logo"> <img src="https://storage.googleapis.com/gweb-developer-goog-blog-cms-assets/f677f1b8-8552-4a23-83cc-57c36dbaf782/images/g-dev.svg" class="site-logo" alt="Google for Developers"> </a> <ul class="footer-sites-list"> <li class="footer-sites-item"> <a href="//developer.android.com" class="footer-sites-link"> Android </a> </li> <li class="footer-sites-item"> <a href="//developer.chrome.com/home" class="footer-sites-link"> Chrome </a> </li> <li class="footer-sites-item"> <a href="//firebase.google.com" class="footer-sites-link"> Firebase </a> </li> <li class="footer-sites-item"> <a href="//cloud.google.com" class="footer-sites-link"> Google Cloud Platform </a> </li> <li class="footer-sites-item"> <a href="//developers.google.com/products" class="footer-sites-link"> All products </a> </li> <li class="footer-sites-item"> <button aria-hidden="true" class="glue-cookie-notification-bar-control footer-sites-link"> Manage cookies </button> </li> </ul> </nav> <nav class="footer-utility-links"> <ul class="footer-utility-list"> <li class="footer-utility-item"> <a href="//developers.google.com/terms/site-terms" class="footer-utility-link"> Terms </a> </li> <li class="footer-utility-item"> <a href="//policies.google.com/privacy" class="footer-utility-link"> Privacy </a> </li> </ul> <div class="language-selector footer" aria-label="Select your language preference"> <button type="button" aria-controls="lang-selector-footer" aria-haspopup="true"> <svg role="presentation" aria-hidden="true" class="glue-icon lang-icon"> <use href="/glue-icon/#language"></use> </svg> English <svg role="presentation" aria-hidden="true" class="glue-icon lang-icon"> <use href="/glue-icon/#arrow-drop-down"></use> </svg> </button> <div id="lang-selector-footer" class="lang-menu hidden" role="menu"> <ul role="presentation"> <li role="presentation"> <a role="menuitem" lang="en" aria-current="true" href="/en/get-started-with-gemma-on-ray-on-vertex-ai/">English</a> </li> <li role="presentation"> <a role="menuitem" lang="es" href="/es/get-started-with-gemma-on-ray-on-vertex-ai/">Español (Latam)</a> </li> <li role="presentation"> <a role="menuitem" lang="id" href="/id/get-started-with-gemma-on-ray-on-vertex-ai/">Bahasa Indonesia</a> </li> <li role="presentation"> <a role="menuitem" lang="ja" href="/ja/get-started-with-gemma-on-ray-on-vertex-ai/">日本語</a> </li> <li role="presentation"> <a role="menuitem" lang="ko" href="/ko/get-started-with-gemma-on-ray-on-vertex-ai/">한국어</a> </li> <li role="presentation"> <a role="menuitem" lang="pt-br" href="/pt-br/get-started-with-gemma-on-ray-on-vertex-ai/">Português (Brasil)</a> </li> <li role="presentation"> <a role="menuitem" lang="zh-hans" href="/zh-hans/get-started-with-gemma-on-ray-on-vertex-ai/">简体中文</a> </li> </ul> </div> </div> </nav> </div> </div> <script src="https://www.youtube.com/player_api"></script> <script src="//www.gstatic.com/glue/v27_1/glue.min.js"></script> <script type="text/javascript" src="https://storage.googleapis.com/gweb-developer-goog-blog-cms-assets/f677f1b8-8552-4a23-83cc-57c36dbaf782/js/dgc_blog.js"></script> <script src="https://www.gstatic.com/glue/cookienotificationbar/cookienotificationbar.min.js" data-glue-cookie-notification-bar-category="2A" data-glue-cookie-notification-bar-site-id="developers.googleblog.com"> </script> <script src="https://storage.googleapis.com/gweb-developer-goog-blog-cms-assets/f677f1b8-8552-4a23-83cc-57c36dbaf782/js/blog_detail.js"></script> </body> </html>

CINXE.COM

Get started with Gemma on Ray on Vertex AI - Google Developers Blog