CINXE.COM

<!DOCTYPE html> <html class='v2' dir='ltr' lang='en' xmlns='http://www.w3.org/1999/xhtml' xmlns:b='http://www.google.com/2005/gml/b' xmlns:data='http://www.google.com/2005/gml/data' xmlns:expr='http://www.google.com/2005/gml/expr'> <head> <link href='https://www.blogger.com/static/v1/widgets/3566091532-css_bundle_v2.css' rel='stylesheet' type='text/css'/> <meta content='text/html; charset=UTF-8' http-equiv='Content-Type'/> <meta content='blogger' name='generator'/> <link href='https://blog.tensorflow.org/favicon.ico' rel='icon' type='image/x-icon'/> <link href='https://blog.tensorflow.org/2023/08/distributed-fast-fourier-transform-in-tensorflow.html' rel='canonical'/> <link rel="alternate" type="application/atom+xml" title="The TensorFlow Blog - Atom" href="https://blog.tensorflow.org/feeds/posts/default" /> <link rel="alternate" type="application/rss+xml" title="The TensorFlow Blog - RSS" href="https://blog.tensorflow.org/feeds/posts/default?alt=rss" /> <link rel="service.post" type="application/atom+xml" title="The TensorFlow Blog - Atom" href="https://www.blogger.com/feeds/7864883956188652345/posts/default" /> <link rel="alternate" type="application/atom+xml" title="The TensorFlow Blog - Atom" href="https://blog.tensorflow.org/feeds/5844569120834369979/comments/default" />  <link href='https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgYIMKKNGUkhWyF3gL1qLpn7cQ3iQu5GleP-36OHbTJwn90YdRUk8vGTefc9ctjwPcJhyBbFlprd581nDsWEKfMAeAo9xuX8zKfxti8Fvl2f2v69Qmvt695cCJY1dfVPbMIlfWqMFKMEyBCgIaRLXypYCrHlob-OiAb0mvVbhmBEt65-agfmRWDMuaI/s1600/image1.png' rel='image_src'/> <meta content='TensorFlow gains experimental support for Distributed FFT via DTensor.' name='description'/> <meta content='https://blog.tensorflow.org/2023/08/distributed-fast-fourier-transform-in-tensorflow.html' property='og:url'/> <meta content='Distributed Fast Fourier Transform in TensorFlow' property='og:title'/> <meta content='TensorFlow gains experimental support for Distributed FFT via DTensor.' property='og:description'/> <meta content='https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgYIMKKNGUkhWyF3gL1qLpn7cQ3iQu5GleP-36OHbTJwn90YdRUk8vGTefc9ctjwPcJhyBbFlprd581nDsWEKfMAeAo9xuX8zKfxti8Fvl2f2v69Qmvt695cCJY1dfVPbMIlfWqMFKMEyBCgIaRLXypYCrHlob-OiAb0mvVbhmBEt65-agfmRWDMuaI/w1200-h630-p-k-no-nu/image1.png' property='og:image'/> <meta charset='UTF-8'/> <meta content='IE=edge' http-equiv='X-UA-Compatible'/> <meta content='width=device-width, initial-scale=1' name='viewport'/> <meta content='https://www.gstatic.com/tf_blog/images/image_blank.png' property='og:image'/> <meta content='https://www.gstatic.com/tf_blog/images/image_blank.png' property='twitter:image'/> <meta content='summary_large_image' name='twitter:card'/> <meta content='Distributed Fast Fourier Transform in TensorFlow' property='twitter:title'/> <title>Distributed Fast Fourier Transform in TensorFlow — The TensorFlow Blog</title> <style id='page-skin-1' type='text/css'></style>  <link href='https://fonts.googleapis.com/css?family=Google+Sans:400,500,700|Roboto:400,400italic,500,500italic,700,700italic|Roboto+Mono:400,500,700|Material+Icons' rel='stylesheet'/>   <link href="//prismjs.com/themes/prism.css" rel="stylesheet"> <script src="//prismjs.com/prism.js" type="text/javascript"></script> <script src='https://cdnjs.cloudflare.com/ajax/libs/prism/1.17.1/plugins/autoloader/prism-autoloader.min.js'></script>   <link href='https://cdn.jsdelivr.net/npm/zoom-vanilla.js/dist/zoom.css' rel='stylesheet'/> <script defer='defer' src='https://cdn.jsdelivr.net/npm/zoom-vanilla.js/dist/zoom-vanilla.min.js' type='text/javascript'></script>  <link href='https://www.gstatic.com/tf_blog/images/favicon.png' rel='shortcut icon' type='image/png'/> <script type='text/javascript'> //<![CDATA[ const qs = (string, el = document) => el.querySelector(string); const qsa = (string, el = document) => el.querySelectorAll(string); class App { constructor() { this.body = qs('body'); this.detailBody = qs('.tensorsite-detail__body'); this.overlay = qs('.header__overlay'); this.hamburger = qs('.header__hamburger'); this.sideMenu = qs('.header__side-menu'); this.detailBodies = qsa('.tensorsite-detail__body'); this.searchForms = qsa('.searchbox'); this.searchInputs = qsa('.search-input'); this.homeHref = qs('#home-href'); this.featuredCard = qs('.tensorsite-card.featured'); this.featuredPostHref = this.featuredCard && this.featuredCard .querySelector('.tensorsite-card__href') .getAttribute('href'); this.cards = qsa('.tensorsite-card'); this.images = qsa('img[border]'); this.cardDescriptions = qsa('.tensorsite-content__description'); this.hiddenDescription = qsa('.tensorsite-detail__description'); this.iconLinks = qs('.social-icons__links').children this.iconTooltips = qsa('[class^="icon-tooltip"]') this._toggleMobileMenu = this._toggleMobileMenu.bind(this); this._closeMenu = this._closeMenu.bind(this); this._onResize = this._onResize.bind(this); this._getScreen = this._getScreen.bind(this); this._searchGoogle = this._searchGoogle.bind(this); this._handleSearchKeypress = this._handleSearchKeypress.bind(this); this._removeDividerAboveImage(); this._setAllTagActive(); this._showFeaturedPost(); this._redirectWithMaxResults(); this._makeImagesZoomable(); this._removeCardLineBreaks(); this._getNextPost().then(()=>{ this._removeCardLineBreaks(); }) this.addEventListeners(); } addEventListeners() { window.addEventListener('resize', this._onResize); this.hamburger.addEventListener('click', this._toggleMobileMenu); this.searchForms.forEach(el => el.addEventListener('submit', this._searchGoogle)); this.searchInputs.forEach(el => el.addEventListener('keypress', this._handleSearchKeypress)); Array.from(this.iconLinks).forEach((icon, i) => { icon.addEventListener("mouseover", () => icon.querySelectorAll('[class^="icon-tooltip"]')[0].style.display = 'block'); icon.addEventListener("mouseout", () => icon.querySelectorAll('[class^="icon-tooltip"]')[0].style.display = 'none'); }) } _getNextPost() { return new Promise((resolve) => { const nextHref = qs('.tensorsite-detail__next-url'); if (this.detailBody && nextHref) { let request = new XMLHttpRequest(); request.open('GET', nextHref.getAttribute('href'), true); request.onload = function() { if (this.status >= 200 && this.status < 400) { // Success! Should be an HTML response // Save html in variable so you're able to query select const parser = new DOMParser(); const html = parser.parseFromString(this.response, "text/html"); const nextTitle = html.querySelector('.tensorsite-detail__title'); const nextDesc = html.querySelector('.tensorsite-detail__description'); const nextTags = html.querySelector('.tensorsite-detail__tags'); const nextHref = qs('.tensorsite-detail__next-url').getAttribute('href'); const nextImgUrl = html.querySelector('.tensorsite-detail__main-image'); const nextTitleEl = document.querySelector('.tensorsite-content__title.next'); const nextDescEl = document.querySelector('.tensorsite-content__description.next'); let nextTagsEl = document.querySelector('.tensorsite-content__subtitle.next'); const nextHrefEl = document.querySelector('.tensorsite-card__href.next'); const nextImgEl = document.querySelector('.tensorsite-content__image-wrapper'); const nextContainer = qs('.tensorsite-next'); const footer = qs('.tensorsite-footer'); if (nextTitleEl && nextTitle) { nextTitleEl.innerHTML = nextTitle.innerHTML; } if (nextDescEl && nextDesc) { nextDescEl.innerHTML = nextDesc.innerHTML; } if (nextTagsEl && nextTags) { nextTagsEl.innerHTML = nextTags.innerHTML; } if (nextHref && nextHrefEl) { nextHrefEl.setAttribute('href', nextHref); } if (nextImgEl && nextImgUrl) { // If Blogger can't find a firstImageUrl, it returns a // message informing us of that, so this checks // if the string is a URL if(!/http/.test(nextImgUrl.innerHTML)){ nextImgEl.classList.add('hidden'); } else { nextImgEl.querySelector('img').src = nextImgUrl.innerHTML; } } if (nextHref) { nextContainer.classList.add('active'); footer.classList.add('grey'); } resolve(); } else { // We reached our target server, but it returned an error console.error('Error: Could not get the next title'); } }; request.send(); } }) } get isMenuOpen() { return this.sideMenu.classList.contains('is-open'); } _handleSearchKeypress(e) { if (e.which == 13) { this._searchGoogle(); } } _searchGoogle(e) { e.preventDefault(); const {value} = e.target.querySelector('.search-input'); window.location.href = 'https://www.google.com/search?q=site%3A' + window.location.hostname + '%20' + value; } _toggleMobileMenu() { this.body.classList.toggle('no-scroll'); this.overlay.classList.toggle('show'); this.sideMenu.classList.toggle('is-open'); if (this.isMenuOpen) { this.overlay.addEventListener('click', this._closeMenu); } else { this.overlay.removeEventListener('click', this._closeMenu); } } _closeMenu(e) { if (this.isMenuOpen) { this._toggleMobileMenu(); } } _onResize() { if (this._getScreen().width > 839 && this.isMenuOpen) { this._closeMenu(); } } _getScreen() { return { scrollY: window.scrollY, width: window.innerWidth, height: window.innerHeight, } }; _removeDividerAboveImage() { if (this.detailBody && this.detailBody.firstElementChild && this.detailBody.firstElementChild.querySelector('img')) { const firstDivider = qs('.divider'); firstDivider.style.display = 'none'; } } _isCurrentPathAllPosts(){ const {pathname, search} = window.location; return pathname === '/' || (pathname === '/search' && !/label/.test(search)) } _setAllTagActive() { if(this._isCurrentPathAllPosts()){ const allTag = qs('.header__nav-item.all'); allTag.parentElement.classList.add('active'); } } // Shows featured post only if not on the home page _showFeaturedPost() { if (window.location.pathname === '/' && this.featuredCard) { // If any posts in the list have the same href, hide them. // Using a boolean in order to skip the first match // since that is the actual featured card. let skippedFirst = false; this.cards.forEach(card => { const link = card .querySelector('.tensorsite-card__href') .getAttribute('href'); if (link === this.featuredPostHref) { if (skippedFirst) { card.classList.add('hidden'); } skippedFirst = true; } }) } } _makeImagesZoomable(){ this.images.forEach(image => { image.setAttribute('data-action', 'zoom'); if(/a/i.test(image.parentNode.tagName)){ image.parentNode.replaceWith(image) } }) } // Adds max-results query param if URL contains a label filter but // doesn't contain max-results _redirectWithMaxResults(){ const {search} = window.location; const isLabelMatch = /(tensorflow|tfx|community)/gi.test(search) if(isLabelMatch && !/max-results/.test(search)){ window.location.href = window.location.href + '&max-results=20' } } _removeCardLineBreaks(){ const descriptions = this.cardDescriptions || [this.hiddenDescription]; if(descriptions){ descriptions.forEach(node=> { let stringArray = node.innerText.split(/(\r\n|\n|\r)/g); for(let i = stringArray.length; i > 0; i--){ if(/(\r\n|\n|\r)/.test(stringArray[i]) || stringArray[i] === ''){ let j = i - 1; while(j > 0 && (/(\r\n|\n|\r)/.test(stringArray[j]) || stringArray[j] === '')){ stringArray.splice(j, 1); j-- } } } return node.innerText = stringArray.join('') }) } } } window.addEventListener('DOMContentLoaded', (event) => { new App(); }); //]]> </script> <script type='text/javascript'> (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) })(window,document,'script','https://www.google-analytics.com/analytics.js','ga'); ga('create', 'UA-142343919-1', 'auto', 'blogger'); ga('blogger.send', 'pageview'); </script> <link href='https://www.blogger.com/dyn-css/authorization.css?targetBlogID=7864883956188652345&zx=fec37b9b-5beb-4355-a145-f93f4e783e5d' media='none' onload='if(media!='all')media='all'' rel='stylesheet'/><noscript><link href='https://www.blogger.com/dyn-css/authorization.css?targetBlogID=7864883956188652345&zx=fec37b9b-5beb-4355-a145-f93f4e783e5d' rel='stylesheet'/></noscript> <meta name='google-adsense-platform-account' content='ca-host-pub-1556223355139109'/> <meta name='google-adsense-platform-domain' content='blogspot.com'/> </head> <body> <div class='header__overlay'></div> <div class='section' id='nav'><div class='widget HTML' data-version='1' id='HTML1'> <header class='header'> <div aria-hidden='true' data-href='https://blog.tensorflow.org/' hidden='true' id='home-href'></div> <div class='top-row'> <div class='top-row__left'> <button aria-label='Toggle menu' class='header__hamburger' type='button'> <i class='material-icons'>menu</i> </button> <a class='tensorsite-blog-logo' href='https://blog.tensorflow.org/'> <img alt='TensorFlow Blog Logo' class='tensorsite-blog-logo__image' src='https://www.gstatic.com/tf_blog/images/tfblog_logo.svg'/> </a> </div> <div class='top-row__right'> <div class='header__search-container'> <form action='' class='searchbox'> <input aria-label='Search box' class='search-input' name='q' onblur='if (this.value=="") {this.value="Search the Blog";}' onfocus='if (this.value=="Search the Blog") {this.value=""}' placeholder='Search the Blog' type='text' value=''/> <i class='material-icons'>search</i> <input style='visibility:hidden;position:absolute' type='submit'/> </form> </div> <a class='header__cta' href='https://www.tensorflow.org/'> <svg class='cta-icon' height='12' viewBox='0 0 18 18' width='12' xmlns='http://www.w3.org/2000/svg'> <g fill='none' fill-rule='evenodd' transform='translate(-3 -3)'> <rect height='24' width='24'></rect> <path d='M20.55,10.95 L13.05,3.45 C12.45,2.85 11.55,2.85 10.95,3.45 C10.35,4.05 10.35,4.95 10.95,5.55 L15.9,10.5 L4.5,10.5 C3.6,10.5 3,11.1 3,12 C3,12.9 3.6,13.5 4.5,13.5 L15.9,13.5 L10.95,18.45 C10.35,19.05 10.35,19.95 10.95,20.55 C10.95,20.55 10.95,20.55 10.95,20.55 C11.55,21.15 12.45,21.15 13.05,20.55 C13.05,20.55 13.05,20.55 13.05,20.55 L20.55,13.05 C21.15,12.45 21.15,11.55 20.55,10.95 C20.55,10.95 20.55,10.95 20.55,10.95 Z' fill='#000'></path> </g> </svg> Return to TensorFlow Home </a> </div> </div> <div class='nav-row'> <div class='nav-items'> <tab> <a class='header__nav-item all' dir='ltr' href='https://blog.tensorflow.org/'> All </a> </tab> <tab> <a class='header__nav-item' dir='ltr' href='https://blog.tensorflow.org/search?label=TensorFlow+Core&max-results=20'> TensorFlow Core </a> </tab> <tab> <a class='header__nav-item' dir='ltr' href='https://blog.tensorflow.org/search?label=TensorFlow.js&max-results=20'> TensorFlow.js </a> </tab> <tab> <a class='header__nav-item' dir='ltr' href='https://blog.tensorflow.org/search?label=TensorFlow+Lite&max-results=20'> TensorFlow Lite </a> </tab> <tab> <a class='header__nav-item' dir='ltr' href='https://blog.tensorflow.org/search?label=TFX&max-results=20'> TFX </a> </tab> <tab> <a class='header__nav-item' dir='ltr' href='https://blog.tensorflow.org/search?label=Community&max-results=20'> Community </a> </tab> </div> <section class='social-icons'> <div class='social-icons__container-header'> <div class='social-icons__links'> <a class='icon-link' href='https://discuss.tensorflow.org' rel='noopener noreferrer' target='_blank'> <img alt='TensorFlow Forum' src='https://www.gstatic.com/tf_blog/images/ic_forum_2.svg'/> <div class='icon-tooltip'>TensorFlow Forum</div> </a> <a class='icon-link' href='https://www.youtube.com/tensorflow' rel='noopener noreferrer' target='_blank'> <img alt='TensorFlow YouTube' src='https://www.gstatic.com/tf_blog/images/ic_youtube.svg'/> <div class='icon-tooltip'>TensorFlow YouTube</div> </a> <a class='icon-link' href='https://twitter.com/TensorFlow' rel='noopener noreferrer' target='_blank'> <img alt='TensorFlow Twitter' src='https://www.gstatic.com/tf_blog/images/ic_twitter.svg'/> <div class='icon-tooltip'>TensorFlow Twitter</div> </a> <a class='icon-link' href='https://github.com/tensorflow' rel='noopener noreferrer' target='_blank'> <img alt='TensorFlow GitHub' src='https://www.gstatic.com/tf_blog/images/ic_github.svg'/> <div class='icon-tooltip-github'>TensorFlow GitHub</div> </a> </div> </div> </section> </div> </header> <div class='header__side-menu'> <div class='header__side-menu__logo-container'> <a class='tensorsite-blog-logo' href='https://blog.tensorflow.org/'> <img alt='TensorFlow Blog Logo' class='tensorsite-blog-logo__image' src='https://www.gstatic.com/tf_blog/images/tfblog_logo.svg'/> </a> </div> <div class='header__side-menu__content'> <div class='header__side-menu__items'> <div class='header__search-container mobile'> <form action='' class='searchbox'> <input aria-label='Search box' class='search-input' name='q' onblur='if (this.value=="") {this.value="Search the Blog";}' onfocus='if (this.value=="Search the Blog") {this.value=""}' placeholder='Search the Blog' type='text' value=''/> <i class='material-icons'>search</i> <input style='visibility:hidden;position:absolute' type='submit'/> </form> </div> <div class='header__side-menu__title'>Tags</div> <tab> <li> <a class='header__nav-item mobile' dir='ltr' href='https://blog.tensorflow.org/'> All </a> </li> </tab> <tab> <li> <a class='header__nav-item mobile' dir='ltr' href='https://blog.tensorflow.org/search?label=TensorFlow+Core&max-results=20'> TensorFlow Core </a> </li> </tab> <tab> <li> <a class='header__nav-item mobile' dir='ltr' href='https://blog.tensorflow.org/search?label=TensorFlow.js&max-results=20'> TensorFlow.js </a> </li> </tab> <tab> <li> <a class='header__nav-item mobile' dir='ltr' href='https://blog.tensorflow.org/search?label=TensorFlow+Lite&max-results=20'> TensorFlow Lite </a> </li> </tab> <tab> <li> <a class='header__nav-item mobile' dir='ltr' href='https://blog.tensorflow.org/search?label=TFX&max-results=20'> TFX </a> </li> </tab> <tab> <li> <a class='header__nav-item mobile' dir='ltr' href='https://blog.tensorflow.org/search?label=Community&max-results=20'> Community </a> </li> </tab> </div> <div class='spacer'></div> <div class='header__side-menu__bottom'> <a class='header__cta mobile' href='https://www.tensorflow.org/'> Return to TensorFlow Home </a> </div> </div> </div> </div></div> <div class='content-wrap'> <div class='section' id='blog'><div class='widget FeaturedPost' data-version='1' id='FeaturedPost1'> </div><div class='widget Blog' data-version='1' id='Blog1'> <div class='tensorsite-container--narrow'> <div class='tensorsite-detail'> <a aria-hidden='true' class='tensorsite-detail__next-url' hidden='true' href='https://blog.tensorflow.org/2023/08/the-tensorflow-lite-plugin-for-flutter-officially-available.html'></a> <div aria-hidden='true' class='tensorsite-detail__current-url' hidden='true'>https://blog.tensorflow.org/2023/08/distributed-fast-fourier-transform-in-tensorflow.html</div> <div aria-hidden='true' class='tensorsite-detail__tags' hidden='true'> <span>TensorFlow Core</span> </div> <div aria-hidden='true' class='tensorsite-detail__main-image' hidden='true'> https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgYIMKKNGUkhWyF3gL1qLpn7cQ3iQu5GleP-36OHbTJwn90YdRUk8vGTefc9ctjwPcJhyBbFlprd581nDsWEKfMAeAo9xuX8zKfxti8Fvl2f2v69Qmvt695cCJY1dfVPbMIlfWqMFKMEyBCgIaRLXypYCrHlob-OiAb0mvVbhmBEt65-agfmRWDMuaI/s1600/image1.png </div> <p aria-hidden='true' class='tensorsite-detail__description' hidden='true'> <span class='tensorsite-content__info'> August 24, 2023 — </span> <em>Posted by <a href="https://www.linkedin.com/in/ruijiao-sun" target="_blank">Ruijiao Sun</a>, Google Intern - DTensor team</em><a href="https://en.wikipedia.org/wiki/Fast_Fourier_transform" target="_blank">Fast Fourier Transform</a> is an important method of signal processing, which is commonly used in a number of ways, including speeding up convolutions, extracting features, and regularizing models. Distributed Fast Fourier Transform (Distributed FFT) offers a way to compute Fourier Transforms in models that work with image-like datasets that are too … </p> <div class='tensorsite-content__subtitle'> <a href='https://blog.tensorflow.org/search?label=TensorFlow+Core&max-results=20'> <span>TensorFlow Core</span> </a> <b class='label-divider-dot'>·</b> <img alt='Google Article' class='community-icon' src='https://www.gstatic.com/tf_blog/images/ic_google.svg'/> </div> <div class='tensorsite-detail__title'> Distributed Fast Fourier Transform in TensorFlow </div> <div class='tensorsite-detail__contact'> <div class='tensorsite-detail__info'> <span class='tensorsite-detail__timestamp'>August 24, 2023</span> </div> <a class='icon-link' href='https://twitter.com/intent/tweet?text=%22Distributed Fast Fourier Transform in TensorFlow%22 from the TensorFlow Blog%0A%0Ahttps://blog.tensorflow.org/2023/08/distributed-fast-fourier-transform-in-tensorflow.html' rel='noopener noreferrer' target='_blank' title='Share this post on Twitter'> <svg alt='Twitter Social Icon' class='twitter-icon social-icon' height='19' viewBox='0 0 23 19' width='23' xmlns='http://www.w3.org/2000/svg'> <g fill='none' fill-rule='evenodd' transform='translate(-7 -9)'> <rect height='36' width='36'></rect> <path d='M14.076,27.2827953 C22.566,27.2827953 27.21,20.2477953 27.21,14.1477953 C27.21,13.9477953 27.21,13.7487953 27.197,13.5507953 C28.1,12.8977953 28.88,12.0887953 29.5,11.1617953 C28.657,11.5347953 27.764,11.7797953 26.848,11.8877953 C27.812,11.3107953 28.533,10.4037953 28.878,9.33479527 C27.972,9.87179527 26.98,10.2507953 25.947,10.4547953 C24.198,8.59579527 21.274,8.50679527 19.415,10.2547953 C18.217,11.3817953 17.708,13.0617953 18.08,14.6647953 C14.368,14.4787953 10.91,12.7257953 8.566,9.84279527 C7.341,11.9507953 7.967,14.6497953 9.995,16.0047953 C9.261,15.9827953 8.542,15.7837953 7.9,15.4267953 L7.9,15.4847953 C7.9,17.6827953 9.449,19.5747953 11.603,20.0107953 C10.924,20.1957953 10.211,20.2227953 9.519,20.0897953 C10.124,21.9707953 11.856,23.2587953 13.832,23.2957953 C12.197,24.5797953 10.178,25.2777953 8.098,25.2747953 C7.731,25.2747953 7.364,25.2527953 7,25.2087953 C9.111,26.5627953 11.567,27.2817953 14.076,27.2787953' fill='#545454'></path> </g> </svg> </a> </div> <div class='divider divider--article-top'></div> <div class='tensorsite-detail__body'> <meta content="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgYIMKKNGUkhWyF3gL1qLpn7cQ3iQu5GleP-36OHbTJwn90YdRUk8vGTefc9ctjwPcJhyBbFlprd581nDsWEKfMAeAo9xuX8zKfxti8Fvl2f2v69Qmvt695cCJY1dfVPbMIlfWqMFKMEyBCgIaRLXypYCrHlob-OiAb0mvVbhmBEt65-agfmRWDMuaI/s1600/image1.png" name="twitter:image"></meta> <img src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgYIMKKNGUkhWyF3gL1qLpn7cQ3iQu5GleP-36OHbTJwn90YdRUk8vGTefc9ctjwPcJhyBbFlprd581nDsWEKfMAeAo9xuX8zKfxti8Fvl2f2v69Qmvt695cCJY1dfVPbMIlfWqMFKMEyBCgIaRLXypYCrHlob-OiAb0mvVbhmBEt65-agfmRWDMuaI/s1600/image1.png" style="display: none;" /> <em>Posted by <a href="https://www.linkedin.com/in/ruijiao-sun" target="_blank">Ruijiao Sun</a>, Google Intern - DTensor team</em> <a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiRgwmbW2Tp7Q_Y3GTKSmkfboFNHuUizDaYh1uLrQSCZarfkvs1mK7OodevH9l318Ls8ddEmKNPewlpVhMJzKvtpktP6TeKniEEMAzgRrHq-D-kIEsoQnZyvc7n4pUVsn1RkFF066dnujQZ1htprWST0uSJftVZxQyc2Qm8aijQTMhrtJlj7rrEc6s7/s1600/Tensorflow-septmber-update-header%20%282%29.png"><img border="0" data-original-height="800" data-original-width="1058" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiRgwmbW2Tp7Q_Y3GTKSmkfboFNHuUizDaYh1uLrQSCZarfkvs1mK7OodevH9l318Ls8ddEmKNPewlpVhMJzKvtpktP6TeKniEEMAzgRrHq-D-kIEsoQnZyvc7n4pUVsn1RkFF066dnujQZ1htprWST0uSJftVZxQyc2Qm8aijQTMhrtJlj7rrEc6s7/s1600/Tensorflow-septmber-update-header%20%282%29.png" /></a> <a name='more'></a><p></p> <p><a href="https://en.wikipedia.org/wiki/Fast_Fourier_transform" target="_blank">Fast Fourier Transform</a> is an important method of signal processing, which is commonly used in a number of ways, including speeding up convolutions, extracting features, and regularizing models. Distributed Fast Fourier Transform (Distributed FFT) offers a way to compute Fourier Transforms in models that work with image-like datasets that are too large to fit into the memory of a single accelerator device. In a previous Google Research Paper, “<a href="https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=9465154" target="_blank"><i>Large-Scale Discrete Fourier Transform on TPUs</i></a>” by Tianjian Lu, a Distributed FFT algorithm was implemented for TensorFlow v1 as a library. This work presents the newly added native support in TensorFlow v2 for Distributed FFT, through the new TensorFlow distribution API, DTensor.</p> <h2>About DTensor</h2> <p><a href="https://www.tensorflow.org/guide/dtensor_overview" target="_blank">DTensor</a> is an extension to TensorFlow for synchronous distributed computing. It distributes the program and tensors through a procedure called <a href="https://en.wikipedia.org/wiki/SPMD" target="_blank">Single program, multiple data (SPMD)</a> extension. DTensor offers an uniform API for traditional data and model parallelism patterns used widely in Machine Learning. </p> <h2>Example Usage</h2> <p>The API interface for distributed FFT is the same as the original FFT in TensorFlow. Users just need to pass a sharded tensor as an input to the existing FFT ops in TensorFlow, such as <a href="https://www.tensorflow.org/api_docs/python/tf/signal/fft2d" target="_blank">tf.signal.fft2d</a>. The output of a distributed FFT becomes sharded too.</p><table class="leading-snug" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; background-color: #f8f8f8; border-collapse: collapse; border: none; font-family: ui-sans-serif, system-ui, -apple-system, "system-ui", "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji"; line-height: 1.375; max-width: min(90ch, 100%); table-layout: fixed; text-indent: 0px; width: 880px;"><colgroup style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box;"></colgroup><tbody style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; display: block; max-width: 100%; overflow-x: auto;"><tr style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; background-color: transparent; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; height: 0px;"><td style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; padding: 4pt; vertical-align: top;"><code class="m-0 p-0 whitespace-pre-wrap font-monospace" id="code-output" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; background-color: transparent; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; display: block; font-size: 12px; margin: 0px; padding: 0px; white-space-collapse: preserve; width: 80ch;"><span style="font-family: courier;"><span class="hljs-keyword" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: #7928a1;">import</span> TensorFlow <span class="hljs-keyword" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: #7928a1;">as</span> tf <span class="hljs-keyword" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: #7928a1;">from</span> TensorFlow.experimental <span class="hljs-keyword" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: #7928a1;">import</span> dtensor <span style="color: #b80672;"><span class="hljs-comment" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box;"># Set up devices</span> </span>device_type = dtensor.preferred_device_type() <span class="hljs-keyword" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: #7928a1;">if</span> device_type == <span class="hljs-string" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: green;">'CPU'</span>: cpu = tf.config.list_physical_devices(device_type) tf.config.set_logical_device_configuration(cpu[<span class="hljs-number" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: #aa5d00;">0</span>], [tf.config.LogicalDeviceConfiguration()] * <span class="hljs-number" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: #aa5d00;">8</span>) <span class="hljs-keyword" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: #7928a1;">if</span> device_type == <span class="hljs-string" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: green;">'GPU'</span>: gpu = tf.config.list_physical_devices(device_type) tf.config.set_logical_device_configuration(gpu[<span class="hljs-number" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: #aa5d00;">0</span>], [tf.config.LogicalDeviceConfiguration(memory_limit=<span class="hljs-number" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: #aa5d00;">1000</span>)] * <span class="hljs-number" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: #aa5d00;">8</span>) dtensor.initialize_accelerator_system() <span style="color: #b80672;"><span class="hljs-comment" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box;"># Create a mesh</span> </span>mesh = dtensor.create_distributed_mesh(mesh_dims=[(<span class="hljs-string" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: green;">'x'</span>, <span class="hljs-number" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: #aa5d00;">1</span>), (<span class="hljs-string" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: green;">'y'</span>, <span class="hljs-number" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: #aa5d00;">2</span>), (<span class="hljs-string" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: green;">'z'</span>, <span class="hljs-number" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: #aa5d00;">4</span>)], device_type=device_type) <span style="color: #b80672;"><span class="hljs-comment" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box;"># Set up a distributed input Tensor</span> </span><span class="hljs-built_in" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: #aa5d00;">input</span> = tf.<span class="hljs-built_in" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: #aa5d00;">complex</span>( tf.random.stateless_normal(shape=(<span class="hljs-number" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: #aa5d00;">2</span>, <span class="hljs-number" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: #aa5d00;">2</span>, <span class="hljs-number" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: #aa5d00;">4</span>), seed=(<span class="hljs-number" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: #aa5d00;">1</span>, <span class="hljs-number" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: #aa5d00;">2</span>), dtype=tf.float32), tf.random.stateless_normal(shape=(<span class="hljs-number" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: #aa5d00;">2</span>, <span class="hljs-number" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: #aa5d00;">2</span>, <span class="hljs-number" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: #aa5d00;">4</span>), seed=(<span class="hljs-number" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: #aa5d00;">2</span>, <span class="hljs-number" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: #aa5d00;">4</span>), dtype=tf.float32)) init_layout = dtensor.Layout([<span class="hljs-string" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: green;">'x'</span>, <span class="hljs-string" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: green;">'y'</span>, <span class="hljs-string" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: green;">'z'</span>], mesh) d_input = dtensor.relayout(<span class="hljs-built_in" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: #aa5d00;">input</span>, layout=init_layout) <span style="color: #b80672;"><span class="hljs-comment" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box;"># Run distributed fft2d. DTensor determines the most efficient</span> <span class="hljs-comment" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box;">#</span></span><span class="hljs-comment" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box; color: dimgrey;"> </span><span class="hljs-comment" style="--tw-border-spacing-x: 0; --tw-border-spacing-y: 0; --tw-ring-color: rgb(59 130 246 / 0.5); --tw-ring-offset-color: #fff; --tw-ring-offset-shadow: 0 0 #0000; --tw-ring-offset-width: 0px; --tw-ring-shadow: 0 0 #0000; --tw-rotate: 0; --tw-scale-x: 1; --tw-scale-y: 1; --tw-scroll-snap-strictness: proximity; --tw-shadow-colored: 0 0 #0000; --tw-shadow: 0 0 #0000; --tw-skew-x: 0; --tw-skew-y: 0; --tw-translate-x: 0; --tw-translate-y: 0; border-color: rgb(229, 231, 235); border-style: solid; border-width: 0px; box-sizing: border-box;">layout of of d_output.</span> d_output = tf.signal.fft2d(d_input)</span></code></td></tr></tbody></table> <h2>Performance Analysis</h2> <p>The following experiment demonstrates that the distributed FFT can process more data than the non-distributed one by utilizing memory across multiple devices. The tradeoff is spending additional time on communication and data transposes that slow down the calculation speed. </p> <div><table align="center" cellpadding="0" cellspacing="0" class="tr-caption-container" style="margin-left: auto; margin-right: auto; text-align: center;"><tbody><tr><td style="text-align: center;"><center><img alt="Graph of performance on different machines, measuri8ng wall clock time in seconds by size per dimension across single GPU, Distributed FFT and Undistributed FFT" border="0" data-original-height="1504" data-original-width="720" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEigTOJgosBdpn5Wr7djYt3eYpmcxmRy3nn_UQbG1aW5nO96v5fMFFzsx9p-ahhpTuWJPNRQ5IMRvUjdon-CollI87lu6mAEnnW7KcaIHyrZVLK0tUMQJ2jDBVxtYz9zcXBH2FI5kou775n-Yt4iBGVfzO3rAwaCo4vgkXrWU3BWqv-_YPKf2mkU8kihSxc/s1600/image2.png" style="width: 100%;" td="" /></center></td></tr><tr><td class="tr-caption" style="text-align: center;"><i></i></td></tr></tbody></table></div> <p>This phenomenon is shown in detail from the profiling result of the 10K*10K distributed FFT experiment. The current implementation of distributed FFT in TensorFlow follows the simple shuffle+local FFT method, which is also used by other popular distributed FFT libraries such as <a href="https://ieeexplore.ieee.org/document/681704" target="_blank">FFTW</a> and <a href="https://epubs.siam.org/doi/abs/10.1137/120885887" target="_blank">PFFT</a>. Notably, the two local FFT ops only take 3.6% of the total time (15ms). This is around 1/3 of the time for non-distributed fft2d. Most of the computing time is spent on data shuffling, represented by the ncclAllToAll Operation. Note that these experiments were conducted on an 8xV100 GPU system.</p> <div><table align="center" cellpadding="0" cellspacing="0" class="tr-caption-container" style="margin-left: auto; margin-right: auto; text-align: center;"><tbody><tr><td style="text-align: center;"><center><img alt="Table of Top 10 TensorFlow operations on GPU highlighting two local FFT ops in the top 3" border="0" data-original-height="1504" data-original-width="720" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEhNIgERau2VMVV1Uz5jqsjNow9OVkfhd_WI0Irr424u7sUAEo-iILbMOxhrp_dbjMsKhssjCw6iYfe8m5OCBGsldiqM-6_SjAjY0ZamWkBSJUGuW7Xwubw5fYfrNVY9WSr9B2SXX2m0fQHRS7Gqr-jjKMNxIzTax2xz5ntAGZhIMMb4biWvK98tqd4M7H4/s1600/image1.png" style="width: 100%;" td="" /></center></td></tr><tr><td class="tr-caption" style="text-align: center;"><i></i></td></tr></tbody></table></div> <h2>Next steps</h2> <p>The feature is new and we have adopted a simplest distributed FFT algorithm. A few ideas to fine tune or improve the performance are:</p> <ul><blockquote> <li>Switch to a different DFT/FFT algorithm.</li> <li>Tweaks on the NCCL communication settings for the particular FFT sizes may improve utilization of the network bandwidth and increase the speed.</li> <li>Reducing the number of collectives to minimize bandwidth requirements.</li> <li>Use N-d local FFTs, rather than multiple 1-d local FFTs.</li> </blockquote></ul> <p>Try the new distributed FFT! We welcome your feedback on the <a href="https://discuss.tensorflow.org/" target="_blank">TensorFlow Forum</a> and look forward to working with you on improving the performance. Your input would be invaluable!</p> </div> </div> <div class='tensorsite-detail-footer'> <div class='article-divider'> <img alt='Diamond Article Divider' src='https://www.gstatic.com/tf_blog/images/ic_article_end.svg'/> </div> <a class='tensorsite-chip' href='https://blog.tensorflow.org/search?label=TensorFlow+Core&max-results=20'> TensorFlow Core </a> </div> </div> <div class='tensorsite-next'> <div class='tensorsite-container--large'> <div class='tensorsite-next__title'>Next post</div> <div class='tensorsite-card'> <a aria-label='Next Card' class='tensorsite-card__href next' href='https://blog.tensorflow.org/2023/08/distributed-fast-fourier-transform-in-tensorflow.html'></a> <div class='tensorsite-content__image-wrapper'> <img alt='Distributed Fast Fourier Transform in TensorFlow' class='tensorsite-content__image' src='https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgYIMKKNGUkhWyF3gL1qLpn7cQ3iQu5GleP-36OHbTJwn90YdRUk8vGTefc9ctjwPcJhyBbFlprd581nDsWEKfMAeAo9xuX8zKfxti8Fvl2f2v69Qmvt695cCJY1dfVPbMIlfWqMFKMEyBCgIaRLXypYCrHlob-OiAb0mvVbhmBEt65-agfmRWDMuaI/s1600/image1.png'/> </div> <div class='tensorsite-content'> <div class='tensorsite-content__subtitle next'> <span>TensorFlow Core</span> <b class='label-divider-dot'>·</b> <img alt='Google Article' class='community-icon' src='https://www.gstatic.com/tf_blog/images/ic_google.svg'/> </div> <div class='tensorsite-content__title next'> Distributed Fast Fourier Transform in TensorFlow </div> <p class='tensorsite-content__description next'> <span class='tensorsite-content__info'> August 24, 2023 </span> — <span> <em>Posted by <a href="https://www.linkedin.com/in/ruijiao-sun" target="_blank">Ruijiao Sun</a>, Google Intern - DTensor team</em><a href="https://en.wikipedia.org/wiki/Fast_Fourier_transform" target="_blank">Fast Fourier Transform</a> is an important method of signal processing, which is commonly used in a number of ways, including speeding up convolutions, extracting features, and regularizing models. Distributed Fast Fourier Transform (Distributed FFT) offers a way to compute Fourier Transforms in models that work with image-like datasets that are too … </span> </p> </div> </div> </div> </div>  </div></div> </div>  <div class='tensorsite-full-footer'> <div class='section' id='footer'><div class='widget HTML' data-version='1' id='HTML2'> <section class='tensorsite-footer'> <div class='tensorsite-container tensorsite-footer__container'> <div class='tensorsite-footer__side tensorsite-footer__side--left'></div> <div class='tensorsite-footer__side tensorsite-footer__side--right'></div> <div class='tensorsite-footer__content'> <div class='tensorsite-content'> <div class='tensorsite-content__title tensorsite-content__title--grow'> Build, deploy, and experiment easily with TensorFlow </div> <div class='tensorsite-content__cta-wrapper'> <a class='tensorsite-content__button tensorsite-button tensorsite-button--white' href='https://www.tensorflow.org/'> Get started </a> </div> </div> </div> </div> <div class='tensorsite-footer__lines'></div> </section> <section class='social-icons'> <div class='social-icons__container-footer'> <a alt='TensorFlow Home' class='tensorsite-logo' href='https://www.tensorflow.org/'> <img alt='TensorFlow Logo' class='tensorsite-logo__image' src='https://www.gstatic.com/tf_blog/images/tf_lockup.svg'/> </a> <div class='footer__side--right'> <div class='footer__links'> <a alt='Youtube Social Link' class='footer-link' href='https://www.google.com/' rel='noopener noreferrer' target='_blank'> Google </a> <a alt='Twitter Social Link' class='footer-link' href='https://policies.google.com/privacy' rel='noopener noreferrer' target='_blank'> Privacy </a> <a alt='Github Link' class='footer-link' href='https://policies.google.com/terms' rel='noopener noreferrer' target='_blank'> Terms </a> <a class='footer-link' href='https://blog.tensorflow.org/p/tensorflow-blog-contribution-notice.html'> Contributions notice </a> </div> <div class='social-icons__links'> <a alt='Youtube Social Link' class='icon-link' href='https://www.youtube.com/channel/UC0rqucBdTuFTjJiefW5t-IQ' rel='noopener noreferrer' target='_blank'> <img src='https://www.gstatic.com/tf_blog/images/ic_youtube.svg'/></a> <a alt='Twitter Social Link' class='icon-link' href='https://twitter.com/TensorFlow' rel='noopener noreferrer' target='_blank'> <img src='https://www.gstatic.com/tf_blog/images/ic_twitter.svg'/></a> <a alt='Github Link' class='icon-link' href='https://github.com/tensorflow' rel='noopener noreferrer' target='_blank'> <img src='https://www.gstatic.com/tf_blog/images/ic_github.svg'/></a> </div> </div> </div> </section> </div></div> </div> <script type="text/javascript" src="https://www.blogger.com/static/v1/widgets/984859869-widgets.js"></script> <script type='text/javascript'> window['__wavt'] = 'AOuZoY4F-WSoJObrCYaY8bxgWrr3H9LBpQ:1732365844069';_WidgetManager._Init('//www.blogger.com/rearrange?blogID\x3d7864883956188652345','//blog.tensorflow.org/2023/08/distributed-fast-fourier-transform-in-tensorflow.html','7864883956188652345'); _WidgetManager._SetDataContext([{'name': 'blog', 'data': {'blogId': '7864883956188652345', 'title': 'The TensorFlow Blog', 'url': 'https://blog.tensorflow.org/2023/08/distributed-fast-fourier-transform-in-tensorflow.html', 'canonicalUrl': 'https://blog.tensorflow.org/2023/08/distributed-fast-fourier-transform-in-tensorflow.html', 'homepageUrl': 'https://blog.tensorflow.org/', 'searchUrl': 'https://blog.tensorflow.org/search', 'canonicalHomepageUrl': 'https://blog.tensorflow.org/', 'blogspotFaviconUrl': 'https://blog.tensorflow.org/favicon.ico', 'bloggerUrl': 'https://www.blogger.com', 'hasCustomDomain': true, 'httpsEnabled': true, 'enabledCommentProfileImages': true, 'gPlusViewType': 'FILTERED_POSTMOD', 'adultContent': false, 'analyticsAccountNumber': 'UA-142343919-1', 'encoding': 'UTF-8', 'locale': 'en', 'localeUnderscoreDelimited': 'en', 'languageDirection': 'ltr', 'isPrivate': false, 'isMobile': false, 'isMobileRequest': false, 'mobileClass': '', 'isPrivateBlog': false, 'isDynamicViewsAvailable': true, 'feedLinks': '\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22The TensorFlow Blog - Atom\x22 href\x3d\x22https://blog.tensorflow.org/feeds/posts/default\x22 /\x3e\n\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/rss+xml\x22 title\x3d\x22The TensorFlow Blog - RSS\x22 href\x3d\x22https://blog.tensorflow.org/feeds/posts/default?alt\x3drss\x22 /\x3e\n\x3clink rel\x3d\x22service.post\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22The TensorFlow Blog - Atom\x22 href\x3d\x22https://www.blogger.com/feeds/7864883956188652345/posts/default\x22 /\x3e\n\n\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22The TensorFlow Blog - Atom\x22 href\x3d\x22https://blog.tensorflow.org/feeds/5844569120834369979/comments/default\x22 /\x3e\n', 'meTag': '', 'adsenseHostId': 'ca-host-pub-1556223355139109', 'adsenseHasAds': true, 'adsenseAutoAds': false, 'boqCommentIframeForm': true, 'loginRedirectParam': '', 'view': '', 'dynamicViewsCommentsSrc': '//www.blogblog.com/dynamicviews/4224c15c4e7c9321/js/comments.js', 'dynamicViewsScriptSrc': '//www.blogblog.com/dynamicviews/d78375fb222d99b3', 'plusOneApiSrc': 'https://apis.google.com/js/platform.js', 'disableGComments': true, 'interstitialAccepted': false, 'sharing': {'platforms': [{'name': 'Get link', 'key': 'link', 'shareMessage': 'Get link', 'target': ''}, {'name': 'Facebook', 'key': 'facebook', 'shareMessage': 'Share to Facebook', 'target': 'facebook'}, {'name': 'BlogThis!', 'key': 'blogThis', 'shareMessage': 'BlogThis!', 'target': 'blog'}, {'name': 'X', 'key': 'twitter', 'shareMessage': 'Share to X', 'target': 'twitter'}, {'name': 'Pinterest', 'key': 'pinterest', 'shareMessage': 'Share to Pinterest', 'target': 'pinterest'}, {'name': 'Email', 'key': 'email', 'shareMessage': 'Email', 'target': 'email'}], 'disableGooglePlus': true, 'googlePlusShareButtonWidth': 0, 'googlePlusBootstrap': '\x3cscript type\x3d\x22text/javascript\x22\x3ewindow.___gcfg \x3d {\x27lang\x27: \x27en\x27};\x3c/script\x3e'}, 'hasCustomJumpLinkMessage': false, 'jumpLinkMessage': 'Read more', 'pageType': 'item', 'postId': '5844569120834369979', 'postImageThumbnailUrl': 'https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgYIMKKNGUkhWyF3gL1qLpn7cQ3iQu5GleP-36OHbTJwn90YdRUk8vGTefc9ctjwPcJhyBbFlprd581nDsWEKfMAeAo9xuX8zKfxti8Fvl2f2v69Qmvt695cCJY1dfVPbMIlfWqMFKMEyBCgIaRLXypYCrHlob-OiAb0mvVbhmBEt65-agfmRWDMuaI/s72-c/image1.png', 'postImageUrl': 'https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgYIMKKNGUkhWyF3gL1qLpn7cQ3iQu5GleP-36OHbTJwn90YdRUk8vGTefc9ctjwPcJhyBbFlprd581nDsWEKfMAeAo9xuX8zKfxti8Fvl2f2v69Qmvt695cCJY1dfVPbMIlfWqMFKMEyBCgIaRLXypYCrHlob-OiAb0mvVbhmBEt65-agfmRWDMuaI/s1600/image1.png', 'pageName': 'Distributed Fast Fourier Transform in TensorFlow', 'pageTitle': 'The TensorFlow Blog: Distributed Fast Fourier Transform in TensorFlow', 'metaDescription': 'TensorFlow gains experimental support for Distributed FFT via DTensor.'}}, {'name': 'features', 'data': {}}, {'name': 'messages', 'data': {'edit': 'Edit', 'linkCopiedToClipboard': 'Link copied to clipboard!', 'ok': 'Ok', 'postLink': 'Post Link'}}, {'name': 'template', 'data': {'name': 'custom', 'localizedName': 'Custom', 'isResponsive': false, 'isAlternateRendering': false, 'isCustom': true}}, {'name': 'view', 'data': {'classic': {'name': 'classic', 'url': '?view\x3dclassic'}, 'flipcard': {'name': 'flipcard', 'url': '?view\x3dflipcard'}, 'magazine': {'name': 'magazine', 'url': '?view\x3dmagazine'}, 'mosaic': {'name': 'mosaic', 'url': '?view\x3dmosaic'}, 'sidebar': {'name': 'sidebar', 'url': '?view\x3dsidebar'}, 'snapshot': {'name': 'snapshot', 'url': '?view\x3dsnapshot'}, 'timeslide': {'name': 'timeslide', 'url': '?view\x3dtimeslide'}, 'isMobile': false, 'title': 'Distributed Fast Fourier Transform in TensorFlow', 'description': 'TensorFlow gains experimental support for Distributed FFT via DTensor.', 'featuredImage': 'https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgYIMKKNGUkhWyF3gL1qLpn7cQ3iQu5GleP-36OHbTJwn90YdRUk8vGTefc9ctjwPcJhyBbFlprd581nDsWEKfMAeAo9xuX8zKfxti8Fvl2f2v69Qmvt695cCJY1dfVPbMIlfWqMFKMEyBCgIaRLXypYCrHlob-OiAb0mvVbhmBEt65-agfmRWDMuaI/s1600/image1.png', 'url': 'https://blog.tensorflow.org/2023/08/distributed-fast-fourier-transform-in-tensorflow.html', 'type': 'item', 'isSingleItem': true, 'isMultipleItems': false, 'isError': false, 'isPage': false, 'isPost': true, 'isHomepage': false, 'isArchive': false, 'isLabelSearch': false, 'postId': 5844569120834369979}}]); _WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML1', 'nav', document.getElementById('HTML1'), {}, 'displayModeFull')); _WidgetManager._RegisterWidget('_FeaturedPostView', new _WidgetInfo('FeaturedPost1', 'blog', document.getElementById('FeaturedPost1'), {}, 'displayModeFull')); _WidgetManager._RegisterWidget('_BlogView', new _WidgetInfo('Blog1', 'blog', document.getElementById('Blog1'), {'cmtInteractionsEnabled': false, 'lightboxEnabled': true, 'lightboxModuleUrl': 'https://www.blogger.com/static/v1/jsbin/2646514562-lbx.js', 'lightboxCssUrl': 'https://www.blogger.com/static/v1/v-css/1964470060-lightbox_bundle.css'}, 'displayModeFull')); _WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML2', 'footer', document.getElementById('HTML2'), {}, 'displayModeFull')); </script> </body> </html>