CINXE.COM

<!DOCTYPE html> <html class='v2' dir='ltr' lang='en' xmlns='http://www.w3.org/1999/xhtml' xmlns:b='http://www.google.com/2005/gml/b' xmlns:data='http://www.google.com/2005/gml/data' xmlns:expr='http://www.google.com/2005/gml/expr'> <head> <link href='https://www.blogger.com/static/v1/widgets/3566091532-css_bundle_v2.css' rel='stylesheet' type='text/css'/> <meta content='text/html; charset=UTF-8' http-equiv='Content-Type'/> <meta content='blogger' name='generator'/> <link href='https://blog.tensorflow.org/favicon.ico' rel='icon' type='image/x-icon'/> <link href='https://blog.tensorflow.org/2022/11/building-tensorflow-lite-based-computer-vision-emoji-input-device-openmv.html' rel='canonical'/> <link rel="alternate" type="application/atom+xml" title="The TensorFlow Blog - Atom" href="https://blog.tensorflow.org/feeds/posts/default" /> <link rel="alternate" type="application/rss+xml" title="The TensorFlow Blog - RSS" href="https://blog.tensorflow.org/feeds/posts/default?alt=rss" /> <link rel="service.post" type="application/atom+xml" title="The TensorFlow Blog - Atom" href="https://www.blogger.com/feeds/7864883956188652345/posts/default" /> <link rel="alternate" type="application/atom+xml" title="The TensorFlow Blog - Atom" href="https://blog.tensorflow.org/feeds/5968747221758897766/comments/default" />  <link href='https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjc40D2_Rf0thFm_3_jvI0tCAqEclYeWViMpGH-3GU-uBOOHVEecs0Qxz2vEW9vn2gJOPqtsdM30yXHYXLdrFv-caVtMZOVq2hhfyvOQeSlsjJ9rpUMKxdCohIFtqACoHokDBIeFrZ2Er5qkDvbwgQt-xFukK8REUJANDDrIskgBKx8LUrDWQ7M9UkU/w665-h374/image5.gif' rel='image_src'/> <meta content='This is an in-depth open-source guide that uses tinyML on an Arm Cortex-M based device to create a dedicated input device.' name='description'/> <meta content='https://blog.tensorflow.org/2022/11/building-tensorflow-lite-based-computer-vision-emoji-input-device-openmv.html' property='og:url'/> <meta content='Building a TensorFlow Lite based computer vision emoji input device with OpenMV' property='og:title'/> <meta content='This is an in-depth open-source guide that uses tinyML on an Arm Cortex-M based device to create a dedicated input device.' property='og:description'/> <meta content='https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjc40D2_Rf0thFm_3_jvI0tCAqEclYeWViMpGH-3GU-uBOOHVEecs0Qxz2vEW9vn2gJOPqtsdM30yXHYXLdrFv-caVtMZOVq2hhfyvOQeSlsjJ9rpUMKxdCohIFtqACoHokDBIeFrZ2Er5qkDvbwgQt-xFukK8REUJANDDrIskgBKx8LUrDWQ7M9UkU/w1200-h630-p-k-no-nu/image5.gif' property='og:image'/> <meta charset='UTF-8'/> <meta content='IE=edge' http-equiv='X-UA-Compatible'/> <meta content='width=device-width, initial-scale=1' name='viewport'/> <meta content='https://www.gstatic.com/tf_blog/images/image_blank.png' property='og:image'/> <meta content='https://www.gstatic.com/tf_blog/images/image_blank.png' property='twitter:image'/> <meta content='summary_large_image' name='twitter:card'/> <meta content='Building a TensorFlow Lite based computer vision emoji input device with OpenMV' property='twitter:title'/> <title>Building a TensorFlow Lite based computer vision emoji input device with OpenMV — The TensorFlow Blog</title> <style id='page-skin-1' type='text/css'></style>  <link href='https://fonts.googleapis.com/css?family=Google+Sans:400,500,700|Roboto:400,400italic,500,500italic,700,700italic|Roboto+Mono:400,500,700|Material+Icons' rel='stylesheet'/>   <link href="//prismjs.com/themes/prism.css" rel="stylesheet"> <script src="//prismjs.com/prism.js" type="text/javascript"></script> <script src='https://cdnjs.cloudflare.com/ajax/libs/prism/1.17.1/plugins/autoloader/prism-autoloader.min.js'></script>   <link href='https://cdn.jsdelivr.net/npm/zoom-vanilla.js/dist/zoom.css' rel='stylesheet'/> <script defer='defer' src='https://cdn.jsdelivr.net/npm/zoom-vanilla.js/dist/zoom-vanilla.min.js' type='text/javascript'></script>  <link href='https://www.gstatic.com/tf_blog/images/favicon.png' rel='shortcut icon' type='image/png'/> <script type='text/javascript'> //<![CDATA[ const qs = (string, el = document) => el.querySelector(string); const qsa = (string, el = document) => el.querySelectorAll(string); class App { constructor() { this.body = qs('body'); this.detailBody = qs('.tensorsite-detail__body'); this.overlay = qs('.header__overlay'); this.hamburger = qs('.header__hamburger'); this.sideMenu = qs('.header__side-menu'); this.detailBodies = qsa('.tensorsite-detail__body'); this.searchForms = qsa('.searchbox'); this.searchInputs = qsa('.search-input'); this.homeHref = qs('#home-href'); this.featuredCard = qs('.tensorsite-card.featured'); this.featuredPostHref = this.featuredCard && this.featuredCard .querySelector('.tensorsite-card__href') .getAttribute('href'); this.cards = qsa('.tensorsite-card'); this.images = qsa('img[border]'); this.cardDescriptions = qsa('.tensorsite-content__description'); this.hiddenDescription = qsa('.tensorsite-detail__description'); this.iconLinks = qs('.social-icons__links').children this.iconTooltips = qsa('[class^="icon-tooltip"]') this._toggleMobileMenu = this._toggleMobileMenu.bind(this); this._closeMenu = this._closeMenu.bind(this); this._onResize = this._onResize.bind(this); this._getScreen = this._getScreen.bind(this); this._searchGoogle = this._searchGoogle.bind(this); this._handleSearchKeypress = this._handleSearchKeypress.bind(this); this._removeDividerAboveImage(); this._setAllTagActive(); this._showFeaturedPost(); this._redirectWithMaxResults(); this._makeImagesZoomable(); this._removeCardLineBreaks(); this._getNextPost().then(()=>{ this._removeCardLineBreaks(); }) this.addEventListeners(); } addEventListeners() { window.addEventListener('resize', this._onResize); this.hamburger.addEventListener('click', this._toggleMobileMenu); this.searchForms.forEach(el => el.addEventListener('submit', this._searchGoogle)); this.searchInputs.forEach(el => el.addEventListener('keypress', this._handleSearchKeypress)); Array.from(this.iconLinks).forEach((icon, i) => { icon.addEventListener("mouseover", () => icon.querySelectorAll('[class^="icon-tooltip"]')[0].style.display = 'block'); icon.addEventListener("mouseout", () => icon.querySelectorAll('[class^="icon-tooltip"]')[0].style.display = 'none'); }) } _getNextPost() { return new Promise((resolve) => { const nextHref = qs('.tensorsite-detail__next-url'); if (this.detailBody && nextHref) { let request = new XMLHttpRequest(); request.open('GET', nextHref.getAttribute('href'), true); request.onload = function() { if (this.status >= 200 && this.status < 400) { // Success! Should be an HTML response // Save html in variable so you're able to query select const parser = new DOMParser(); const html = parser.parseFromString(this.response, "text/html"); const nextTitle = html.querySelector('.tensorsite-detail__title'); const nextDesc = html.querySelector('.tensorsite-detail__description'); const nextTags = html.querySelector('.tensorsite-detail__tags'); const nextHref = qs('.tensorsite-detail__next-url').getAttribute('href'); const nextImgUrl = html.querySelector('.tensorsite-detail__main-image'); const nextTitleEl = document.querySelector('.tensorsite-content__title.next'); const nextDescEl = document.querySelector('.tensorsite-content__description.next'); let nextTagsEl = document.querySelector('.tensorsite-content__subtitle.next'); const nextHrefEl = document.querySelector('.tensorsite-card__href.next'); const nextImgEl = document.querySelector('.tensorsite-content__image-wrapper'); const nextContainer = qs('.tensorsite-next'); const footer = qs('.tensorsite-footer'); if (nextTitleEl && nextTitle) { nextTitleEl.innerHTML = nextTitle.innerHTML; } if (nextDescEl && nextDesc) { nextDescEl.innerHTML = nextDesc.innerHTML; } if (nextTagsEl && nextTags) { nextTagsEl.innerHTML = nextTags.innerHTML; } if (nextHref && nextHrefEl) { nextHrefEl.setAttribute('href', nextHref); } if (nextImgEl && nextImgUrl) { // If Blogger can't find a firstImageUrl, it returns a // message informing us of that, so this checks // if the string is a URL if(!/http/.test(nextImgUrl.innerHTML)){ nextImgEl.classList.add('hidden'); } else { nextImgEl.querySelector('img').src = nextImgUrl.innerHTML; } } if (nextHref) { nextContainer.classList.add('active'); footer.classList.add('grey'); } resolve(); } else { // We reached our target server, but it returned an error console.error('Error: Could not get the next title'); } }; request.send(); } }) } get isMenuOpen() { return this.sideMenu.classList.contains('is-open'); } _handleSearchKeypress(e) { if (e.which == 13) { this._searchGoogle(); } } _searchGoogle(e) { e.preventDefault(); const {value} = e.target.querySelector('.search-input'); window.location.href = 'https://www.google.com/search?q=site%3A' + window.location.hostname + '%20' + value; } _toggleMobileMenu() { this.body.classList.toggle('no-scroll'); this.overlay.classList.toggle('show'); this.sideMenu.classList.toggle('is-open'); if (this.isMenuOpen) { this.overlay.addEventListener('click', this._closeMenu); } else { this.overlay.removeEventListener('click', this._closeMenu); } } _closeMenu(e) { if (this.isMenuOpen) { this._toggleMobileMenu(); } } _onResize() { if (this._getScreen().width > 839 && this.isMenuOpen) { this._closeMenu(); } } _getScreen() { return { scrollY: window.scrollY, width: window.innerWidth, height: window.innerHeight, } }; _removeDividerAboveImage() { if (this.detailBody && this.detailBody.firstElementChild && this.detailBody.firstElementChild.querySelector('img')) { const firstDivider = qs('.divider'); firstDivider.style.display = 'none'; } } _isCurrentPathAllPosts(){ const {pathname, search} = window.location; return pathname === '/' || (pathname === '/search' && !/label/.test(search)) } _setAllTagActive() { if(this._isCurrentPathAllPosts()){ const allTag = qs('.header__nav-item.all'); allTag.parentElement.classList.add('active'); } } // Shows featured post only if not on the home page _showFeaturedPost() { if (window.location.pathname === '/' && this.featuredCard) { // If any posts in the list have the same href, hide them. // Using a boolean in order to skip the first match // since that is the actual featured card. let skippedFirst = false; this.cards.forEach(card => { const link = card .querySelector('.tensorsite-card__href') .getAttribute('href'); if (link === this.featuredPostHref) { if (skippedFirst) { card.classList.add('hidden'); } skippedFirst = true; } }) } } _makeImagesZoomable(){ this.images.forEach(image => { image.setAttribute('data-action', 'zoom'); if(/a/i.test(image.parentNode.tagName)){ image.parentNode.replaceWith(image) } }) } // Adds max-results query param if URL contains a label filter but // doesn't contain max-results _redirectWithMaxResults(){ const {search} = window.location; const isLabelMatch = /(tensorflow|tfx|community)/gi.test(search) if(isLabelMatch && !/max-results/.test(search)){ window.location.href = window.location.href + '&max-results=20' } } _removeCardLineBreaks(){ const descriptions = this.cardDescriptions || [this.hiddenDescription]; if(descriptions){ descriptions.forEach(node=> { let stringArray = node.innerText.split(/(\r\n|\n|\r)/g); for(let i = stringArray.length; i > 0; i--){ if(/(\r\n|\n|\r)/.test(stringArray[i]) || stringArray[i] === ''){ let j = i - 1; while(j > 0 && (/(\r\n|\n|\r)/.test(stringArray[j]) || stringArray[j] === '')){ stringArray.splice(j, 1); j-- } } } return node.innerText = stringArray.join('') }) } } } window.addEventListener('DOMContentLoaded', (event) => { new App(); }); //]]> </script> <script type='text/javascript'> (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) })(window,document,'script','https://www.google-analytics.com/analytics.js','ga'); ga('create', 'UA-142343919-1', 'auto', 'blogger'); ga('blogger.send', 'pageview'); </script> <link href='https://www.blogger.com/dyn-css/authorization.css?targetBlogID=7864883956188652345&zx=fec37b9b-5beb-4355-a145-f93f4e783e5d' media='none' onload='if(media!='all')media='all'' rel='stylesheet'/><noscript><link href='https://www.blogger.com/dyn-css/authorization.css?targetBlogID=7864883956188652345&zx=fec37b9b-5beb-4355-a145-f93f4e783e5d' rel='stylesheet'/></noscript> <meta name='google-adsense-platform-account' content='ca-host-pub-1556223355139109'/> <meta name='google-adsense-platform-domain' content='blogspot.com'/> </head> <body> <div class='header__overlay'></div> <div class='section' id='nav'><div class='widget HTML' data-version='1' id='HTML1'> <header class='header'> <div aria-hidden='true' data-href='https://blog.tensorflow.org/' hidden='true' id='home-href'></div> <div class='top-row'> <div class='top-row__left'> <button aria-label='Toggle menu' class='header__hamburger' type='button'> <i class='material-icons'>menu</i> </button> <a class='tensorsite-blog-logo' href='https://blog.tensorflow.org/'> <img alt='TensorFlow Blog Logo' class='tensorsite-blog-logo__image' src='https://www.gstatic.com/tf_blog/images/tfblog_logo.svg'/> </a> </div> <div class='top-row__right'> <div class='header__search-container'> <form action='' class='searchbox'> <input aria-label='Search box' class='search-input' name='q' onblur='if (this.value=="") {this.value="Search the Blog";}' onfocus='if (this.value=="Search the Blog") {this.value=""}' placeholder='Search the Blog' type='text' value=''/> <i class='material-icons'>search</i> <input style='visibility:hidden;position:absolute' type='submit'/> </form> </div> <a class='header__cta' href='https://www.tensorflow.org/'> <svg class='cta-icon' height='12' viewBox='0 0 18 18' width='12' xmlns='http://www.w3.org/2000/svg'> <g fill='none' fill-rule='evenodd' transform='translate(-3 -3)'> <rect height='24' width='24'></rect> <path d='M20.55,10.95 L13.05,3.45 C12.45,2.85 11.55,2.85 10.95,3.45 C10.35,4.05 10.35,4.95 10.95,5.55 L15.9,10.5 L4.5,10.5 C3.6,10.5 3,11.1 3,12 C3,12.9 3.6,13.5 4.5,13.5 L15.9,13.5 L10.95,18.45 C10.35,19.05 10.35,19.95 10.95,20.55 C10.95,20.55 10.95,20.55 10.95,20.55 C11.55,21.15 12.45,21.15 13.05,20.55 C13.05,20.55 13.05,20.55 13.05,20.55 L20.55,13.05 C21.15,12.45 21.15,11.55 20.55,10.95 C20.55,10.95 20.55,10.95 20.55,10.95 Z' fill='#000'></path> </g> </svg> Return to TensorFlow Home </a> </div> </div> <div class='nav-row'> <div class='nav-items'> <tab> <a class='header__nav-item all' dir='ltr' href='https://blog.tensorflow.org/'> All </a> </tab> <tab> <a class='header__nav-item' dir='ltr' href='https://blog.tensorflow.org/search?label=TensorFlow+Core&max-results=20'> TensorFlow Core </a> </tab> <tab> <a class='header__nav-item' dir='ltr' href='https://blog.tensorflow.org/search?label=TensorFlow.js&max-results=20'> TensorFlow.js </a> </tab> <tab> <a class='header__nav-item' dir='ltr' href='https://blog.tensorflow.org/search?label=TensorFlow+Lite&max-results=20'> TensorFlow Lite </a> </tab> <tab> <a class='header__nav-item' dir='ltr' href='https://blog.tensorflow.org/search?label=TFX&max-results=20'> TFX </a> </tab> <tab> <a class='header__nav-item' dir='ltr' href='https://blog.tensorflow.org/search?label=Community&max-results=20'> Community </a> </tab> </div> <section class='social-icons'> <div class='social-icons__container-header'> <div class='social-icons__links'> <a class='icon-link' href='https://discuss.tensorflow.org' rel='noopener noreferrer' target='_blank'> <img alt='TensorFlow Forum' src='https://www.gstatic.com/tf_blog/images/ic_forum_2.svg'/> <div class='icon-tooltip'>TensorFlow Forum</div> </a> <a class='icon-link' href='https://www.youtube.com/tensorflow' rel='noopener noreferrer' target='_blank'> <img alt='TensorFlow YouTube' src='https://www.gstatic.com/tf_blog/images/ic_youtube.svg'/> <div class='icon-tooltip'>TensorFlow YouTube</div> </a> <a class='icon-link' href='https://twitter.com/TensorFlow' rel='noopener noreferrer' target='_blank'> <img alt='TensorFlow Twitter' src='https://www.gstatic.com/tf_blog/images/ic_twitter.svg'/> <div class='icon-tooltip'>TensorFlow Twitter</div> </a> <a class='icon-link' href='https://github.com/tensorflow' rel='noopener noreferrer' target='_blank'> <img alt='TensorFlow GitHub' src='https://www.gstatic.com/tf_blog/images/ic_github.svg'/> <div class='icon-tooltip-github'>TensorFlow GitHub</div> </a> </div> </div> </section> </div> </header> <div class='header__side-menu'> <div class='header__side-menu__logo-container'> <a class='tensorsite-blog-logo' href='https://blog.tensorflow.org/'> <img alt='TensorFlow Blog Logo' class='tensorsite-blog-logo__image' src='https://www.gstatic.com/tf_blog/images/tfblog_logo.svg'/> </a> </div> <div class='header__side-menu__content'> <div class='header__side-menu__items'> <div class='header__search-container mobile'> <form action='' class='searchbox'> <input aria-label='Search box' class='search-input' name='q' onblur='if (this.value=="") {this.value="Search the Blog";}' onfocus='if (this.value=="Search the Blog") {this.value=""}' placeholder='Search the Blog' type='text' value=''/> <i class='material-icons'>search</i> <input style='visibility:hidden;position:absolute' type='submit'/> </form> </div> <div class='header__side-menu__title'>Tags</div> <tab> <li> <a class='header__nav-item mobile' dir='ltr' href='https://blog.tensorflow.org/'> All </a> </li> </tab> <tab> <li> <a class='header__nav-item mobile' dir='ltr' href='https://blog.tensorflow.org/search?label=TensorFlow+Core&max-results=20'> TensorFlow Core </a> </li> </tab> <tab> <li> <a class='header__nav-item mobile' dir='ltr' href='https://blog.tensorflow.org/search?label=TensorFlow.js&max-results=20'> TensorFlow.js </a> </li> </tab> <tab> <li> <a class='header__nav-item mobile' dir='ltr' href='https://blog.tensorflow.org/search?label=TensorFlow+Lite&max-results=20'> TensorFlow Lite </a> </li> </tab> <tab> <li> <a class='header__nav-item mobile' dir='ltr' href='https://blog.tensorflow.org/search?label=TFX&max-results=20'> TFX </a> </li> </tab> <tab> <li> <a class='header__nav-item mobile' dir='ltr' href='https://blog.tensorflow.org/search?label=Community&max-results=20'> Community </a> </li> </tab> </div> <div class='spacer'></div> <div class='header__side-menu__bottom'> <a class='header__cta mobile' href='https://www.tensorflow.org/'> Return to TensorFlow Home </a> </div> </div> </div> </div></div> <div class='content-wrap'> <div class='section' id='blog'><div class='widget FeaturedPost' data-version='1' id='FeaturedPost1'> </div><div class='widget Blog' data-version='1' id='Blog1'> <div class='tensorsite-container--narrow'> <div class='tensorsite-detail'> <a aria-hidden='true' class='tensorsite-detail__next-url' hidden='true' href='https://blog.tensorflow.org/2022/11/how-hugging-face-improved-text-generation-performance-with-xla.html'></a> <div aria-hidden='true' class='tensorsite-detail__current-url' hidden='true'>https://blog.tensorflow.org/2022/11/building-tensorflow-lite-based-computer-vision-emoji-input-device-openmv.html</div> <div aria-hidden='true' class='tensorsite-detail__tags' hidden='true'> <span>Community</span> <b class='label-divider-dot'>·</b> <span>TensorFlow Lite</span> </div> <div aria-hidden='true' class='tensorsite-detail__main-image' hidden='true'> https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjc40D2_Rf0thFm_3_jvI0tCAqEclYeWViMpGH-3GU-uBOOHVEecs0Qxz2vEW9vn2gJOPqtsdM30yXHYXLdrFv-caVtMZOVq2hhfyvOQeSlsjJ9rpUMKxdCohIFtqACoHokDBIeFrZ2Er5qkDvbwgQt-xFukK8REUJANDDrIskgBKx8LUrDWQ7M9UkU/w665-h374/image5.gif </div> <p aria-hidden='true' class='tensorsite-detail__description' hidden='true'> <span class='tensorsite-content__info'> November 29, 2022 — </span> <em>A guest post by Sandeep Mistry, <a href="https://www.arm.com/" target="_blank">Arm</a></em>Introduction Emojis allow us to express emotions in the digital world, they are relatively easy to input on smartphone and tablet devices equipped with touch screen based virtual keyboards, but they are not as easy to input on traditional computing devices that have physical keyboards. To input emojis on these devices, users typically use a keyboard shortcut or … </p> <div class='tensorsite-content__subtitle'> <a href='https://blog.tensorflow.org/search?label=Community&max-results=20'> <span>Community</span> </a> <b class='label-divider-dot'>·</b> <a href='https://blog.tensorflow.org/search?label=TensorFlow+Lite&max-results=20'> <span>TensorFlow Lite</span> </a> <b class='label-divider-dot'>·</b> <svg class='community-icon' fill='none' height='24' viewBox='0 0 24 24' width='24' xmlns='http://www.w3.org/2000/svg'> <path clip-rule='evenodd' d='M16.67 13.13C18.04 14.06 19 15.32 19 17V20H23V17C23 14.82 19.43 13.53 16.67 13.13Z' fill='#555655' fill-rule='evenodd'></path> <path d='M9 12C11.2091 12 13 10.2091 13 8C13 5.79086 11.2091 4 9 4C6.79086 4 5 5.79086 5 8C5 10.2091 6.79086 12 9 12Z' fill='#555655'></path> <path clip-rule='evenodd' d='M15 12C17.21 12 19 10.21 19 8C19 5.79 17.21 4 15 4C14.53 4 14.09 4.1 13.67 4.24C14.5 5.27 15 6.58 15 8C15 9.42 14.5 10.73 13.67 11.76C14.09 11.9 14.53 12 15 12ZM9 13C6.33 13 1 14.34 1 17V20H17V17C17 14.34 11.67 13 9 13Z' fill='#555655' fill-rule='evenodd'></path> </svg> </div> <div class='tensorsite-detail__title'> Building a TensorFlow Lite based computer vision emoji input device with OpenMV </div> <div class='tensorsite-detail__contact'> <div class='tensorsite-detail__info'> <span class='tensorsite-detail__timestamp'>November 29, 2022</span> </div> <a class='icon-link' href='https://twitter.com/intent/tweet?text=%22Building a TensorFlow Lite based computer vision emoji input device with OpenMV%22 from the TensorFlow Blog%0A%0Ahttps://blog.tensorflow.org/2022/11/building-tensorflow-lite-based-computer-vision-emoji-input-device-openmv.html' rel='noopener noreferrer' target='_blank' title='Share this post on Twitter'> <svg alt='Twitter Social Icon' class='twitter-icon social-icon' height='19' viewBox='0 0 23 19' width='23' xmlns='http://www.w3.org/2000/svg'> <g fill='none' fill-rule='evenodd' transform='translate(-7 -9)'> <rect height='36' width='36'></rect> <path d='M14.076,27.2827953 C22.566,27.2827953 27.21,20.2477953 27.21,14.1477953 C27.21,13.9477953 27.21,13.7487953 27.197,13.5507953 C28.1,12.8977953 28.88,12.0887953 29.5,11.1617953 C28.657,11.5347953 27.764,11.7797953 26.848,11.8877953 C27.812,11.3107953 28.533,10.4037953 28.878,9.33479527 C27.972,9.87179527 26.98,10.2507953 25.947,10.4547953 C24.198,8.59579527 21.274,8.50679527 19.415,10.2547953 C18.217,11.3817953 17.708,13.0617953 18.08,14.6647953 C14.368,14.4787953 10.91,12.7257953 8.566,9.84279527 C7.341,11.9507953 7.967,14.6497953 9.995,16.0047953 C9.261,15.9827953 8.542,15.7837953 7.9,15.4267953 L7.9,15.4847953 C7.9,17.6827953 9.449,19.5747953 11.603,20.0107953 C10.924,20.1957953 10.211,20.2227953 9.519,20.0897953 C10.124,21.9707953 11.856,23.2587953 13.832,23.2957953 C12.197,24.5797953 10.178,25.2777953 8.098,25.2747953 C7.731,25.2747953 7.364,25.2527953 7,25.2087953 C9.111,26.5627953 11.567,27.2817953 14.076,27.2787953' fill='#545454'></path> </g> </svg> </a> </div> <div class='divider divider--article-top'></div> <div class='tensorsite-detail__body'> <p style="text-align: left;"><em>A guest post by Sandeep Mistry, <a href="https://www.arm.com/" target="_blank">Arm</a></em><a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjc40D2_Rf0thFm_3_jvI0tCAqEclYeWViMpGH-3GU-uBOOHVEecs0Qxz2vEW9vn2gJOPqtsdM30yXHYXLdrFv-caVtMZOVq2hhfyvOQeSlsjJ9rpUMKxdCohIFtqACoHokDBIeFrZ2Er5qkDvbwgQt-xFukK8REUJANDDrIskgBKx8LUrDWQ7M9UkU/s1600/image5.gif"><img border="0" data-original-height="800" data-original-width="1058" height="374" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjc40D2_Rf0thFm_3_jvI0tCAqEclYeWViMpGH-3GU-uBOOHVEecs0Qxz2vEW9vn2gJOPqtsdM30yXHYXLdrFv-caVtMZOVq2hhfyvOQeSlsjJ9rpUMKxdCohIFtqACoHokDBIeFrZ2Er5qkDvbwgQt-xFukK8REUJANDDrIskgBKx8LUrDWQ7M9UkU/w665-h374/image5.gif" width="665" /></a></p><a name='more'></a><p style="text-align: left;"></p> <h2 style="text-align: left;">Introduction</h2> <p style="text-align: left;"> Emojis allow us to express emotions in the digital world, they are relatively easy to input on smartphone and tablet devices equipped with touch screen based virtual keyboards, but they are not as easy to input on traditional computing devices that have physical keyboards. To input emojis on these devices, users typically use a keyboard shortcut or mouse to bring up an on-screen emoji selector, and then use a mouse to select the desired emoji from a series of categories. </p> <p style="text-align: left;"> This blog will highlight an in-depth open-source guide that uses tinyML on an <a href="https://developer.arm.com/ip-products/processors/cortex-m/" target="_blank">Arm Cortex-M</a> based device to create a dedicated input device. This device will take real-time input from a camera and applies a machine learning (ML) image classification model to detect if the image from the camera contains a set of known hand gestures (✋, 馃憥, 馃憤, 馃憡). When the hand gesture is detected with high certainty, the device will then use the <a href="https://en.wikipedia.org/wiki/USB_human_interface_device_class" target="_blank">USB Human Interface Device (HID) protocol</a> to “type” the emoji on the PC. </p> <p style="text-align: left;"> The <a href="https://www.tensorflow.org/lite/microcontrollers" target="_blank">TensorFlow Lite for Microcontrollers</a> run-time with Arm <a href="https://arm-software.github.io/CMSIS_5/NN/html/index.html" target="_blank">CMSIS-NN</a> is used as the on-device ML inferencing framework on the dedicated input device. On-device inferencing will allow us to reduce the latency of the system, as the image data will be processed at the source (instead of being transmitted to a cloud service). The user’s privacy will also be preserved, as no image data will leave the device at inference time.</p><p style="text-align: left;"><span id="docs-internal-guid-1a468ad8-7fff-67a4-c444-e98c9abd8939"></span></p><div align="center" dir="ltr" style="margin-left: 0pt;"><table style="border-collapse: collapse; border: none;"><colgroup><col width="723"></col></colgroup><tbody><tr style="height: 0pt;"><td style="background-color: #fce5cd; border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: inherit;"><span style="background-color: transparent; color: black; font-size: 11pt; font-style: normal; font-variant: normal; font-weight: 700; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;">NOTE:</span><span style="background-color: transparent; color: black; font-size: 11pt; font-style: normal; font-variant: normal; font-weight: 400; text-decoration: none; vertical-align: baseline; white-space: pre-wrap;"> </span>The complete in-depth and interactive tutorial is available on <a href="https://colab.research.google.com/github/ArmDeveloperEcosystem/ml-image-classification-example-for-openmv/blob/main/notebook.ipynb" target="_blank">Google Colab</a> and all technical assets for the guide can be found on <a href="https://github.com/ArmDeveloperEcosystem/ml-image-classification-example-for-openmv" target="_blank">GitHub</a>.</span></p></td></tr></tbody></table></div> <h3 style="text-align: left;">Microcontrollers and Keyboards</h3> <p style="text-align: left;"> Microcontroller Units (MCUs) are self-contained computing systems embedded in the devices you use every day, including your keyboard! Like all computing systems, they have inputs and outputs. </p> <div style="text-align: left;"> The MCU inside a USB keyboard reacts to the digital events that occur when one or more of the key switches on the keyboard are pressed or released. The MCU determines which key(s) triggered the event and then translates the event into a <a href="https://en.wikipedia.org/wiki/USB_human_interface_device_class" target="_blank">USB HID</a> message to send to the PC using the USB standard. </div> <div style="text-align: left;"><table align="center" cellpadding="0" cellspacing="0" class="tr-caption-container" style="margin-left: auto; margin-right: auto; text-align: center;"><tbody><tr><td style="text-align: center;"><center><img alt="Block diagram of USB keyboard" border="0" data-original-height="1504" data-original-width="720" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjjTkCRxTT94Y1EhXhwvin6CNezdQxG94qgUFgBByB3ylblpr1ODLoHcdgE8k3p2G_sIG4xr6mViA6Hau6tXNbssA-ccdLS5ImVt-FnZB4ZRi84plDCqbQ5Up_bU157S3GQRzXIScGkh2IxZyZ-oGkg94gPaw4912C1VphwM2AmoW4GnxucTySX_kpP/s1600/image2.png" style="width: 100%;" td="" /></center></td></tr> <tr><td class="tr-caption" style="text-align: center;"><i>Block diagram of USB keyboard</i></td></tr></tbody></table></div> <div style="text-align: left;"> The emoji ‘keyboard’ will use an image sensor for input (instead of key switches) and then process the image data locally on a more powerful <a href="https://developer.arm.com/Processors/Cortex-M7" target="_blank">Arm Cortex-M7</a> based microcontroller. All operations, including ML inferencing, are performed on a <a href="https://www.st.com/en/microcontrollers-microprocessors/stm32h743vi.html" target="_blank">STM32H7 MCU</a>, which contains an Arm Cortex-M7 CPU along with a digital interface for the image sensor and USB communications. </div> <div style="text-align: left;"><table align="center" cellpadding="0" cellspacing="0" class="tr-caption-container" style="margin-left: auto; margin-right: auto; text-align: center;"><tbody><tr><td style="text-align: center;"><center><img alt="Block diagram of computer vision based emoji 'keyboard'" border="0" data-original-height="1504" data-original-width="720" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEi_x6kuwMtUyT88UPG-5CAIJCfE_68Yxza0cdV1Tsb6mKlOcdB-1JIkDziV_CAznEQfSnwA1BZ46XB0f3--f3hCDUCi4wsSsTHSwp1smBtRPOIdsPLgT4IO8w8FrmCYu9M2Fwd74UlROQoQIQ0W5bPXXOavuPvMVPores8esWPARsy0ZmgS5ch_XclW/s1600/image3.png" style="width: 100%;" td="" /></center></td></tr><tr><td class="tr-caption" style="text-align: center;"><i>Block diagram of computer vision based emoji “keyboard”</i></td></tr></tbody></table></div> <div style="text-align: left;"> Even though the STM32 H7 is a constrained computing platform that runs at 480 MHz with 1 MB of on-board RAM - we can still process a grayscale 96x96 pixel image input from the camera at just under 20 frames per second (fps)! </div> <h3 style="text-align: left;">The OpenMV development platform</h3> <p style="text-align: left;"> <a href="https://openmv.io" target="_blank">OpenMV</a> is an open source (Micro) Python powered Machine Vision platform. The <a href="https://openmv.io/collections/cams" target="_blank">OpenMV product line-up</a> consists of several Arm Cortex-M based development boards. Each board is equipped with an on-board camera and MCU. For this project, the <a href="https://openmv.io/products/openmv-cam-h7" target="_blank">OpenMV Cam H7</a> or <a href="https://openmv.io/collections/products/products/openmv-cam-h7-r2" target="_blank">OpenMV Cam H7 R2</a> board will suit our needs. </p> <h2 style="text-align: left;">What we will need</h2> <table cellpadding="0" cellspacing="0" class="tr-caption-container" style="margin-left: auto; margin-right: auto; text-align: center;"><tbody><tr><td style="text-align: center;"><a href="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgJapZTSyc8Tol-VLMH-FU8JJdVf2nySGCJchSSj8TwfLtROPoPJ3F_h0sjpyokLeDs0r_XRocfeXBI-YoYma2Uz5n82XaxbdVZcZXsXat6iRYEWZiGZMI_3bj_6q-4RMwdwFpdxpYbCZ4hw1ly5gL1OCeBZvsu17HfmEfzvnZunlMhmzokF849bgO0/s1999/image4.jpg" style="clear: right; margin-bottom: 1em; margin-left: auto; margin-right: auto;"><img border="0" data-original-height="1500" data-original-width="1999" height="300" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgJapZTSyc8Tol-VLMH-FU8JJdVf2nySGCJchSSj8TwfLtROPoPJ3F_h0sjpyokLeDs0r_XRocfeXBI-YoYma2Uz5n82XaxbdVZcZXsXat6iRYEWZiGZMI_3bj_6q-4RMwdwFpdxpYbCZ4hw1ly5gL1OCeBZvsu17HfmEfzvnZunlMhmzokF849bgO0/w400-h300/image4.jpg" width="400" /></a></td></tr><tr><td class="tr-caption" style="text-align: center;"><i>OpenMV Cam H7 Camera (left) and microSD card (right)</i></td></tr></tbody></table><ul><li>Hardware</li></ul><ul style="text-align: left;"><ul><li><a href="https://openmv.io/products/openmv-cam-h7" target="_blank">OpenMV Cam H7</a> or <a href="https://openmv.io/collections/products/products/openmv-cam-h7-r2" target="_blank">OpenMV Cam H7 R2</a> board</li><li>MicroSD card with at least 2 MB of storage space</li><li>USB micro cable</li></ul><li>Software</li><ul><li><a href="https://openmv.io/pages/download" target="_blank">OpenMV IDE</a></li></ul><li>Services</li><ul><li><a href="https://colab.research.google.com" target="_blank">Google Colab</a></li><li><a href="https://www.kaggle.com" target="_blank">Kaggle Account</a></li></ul></ul><h2 style="text-align: left;">Dataset</h2><div style="text-align: left;"><a href="https://www.kaggle.com" target="_blank">Kaggle</a> user <a href="https://www.kaggle.com/imsparsh" target="_blank">Sparsh Gupta (@imsparsh)</a> has previously curated and shared an excellent <a href="https://www.kaggle.com/datasets/imsparsh/gesture-recognition" target="_blank">Gesture Recognition dataset</a> and made it publicly available on Kaggle under a permissive <a href="https://creativecommons.org/publicdomain/zero/1.0/" target="_blank">CC0 1.0 Universal (CC0 1.0) Public Domain license</a>.</div><p style="text-align: left;"> The dataset contains ~23k image files of people performing various hand gestures over a 30 second period.</p><p style="text-align: left;"> Images from the dataset will need to be relabeled as follows: </p><table style="text-align: left;"><tbody><tr><td><span id="docs-internal-guid-ba956c26-7fff-7478-2553-c2e17f57f7cd"><div align="left" dir="ltr" style="margin-left: 0pt;"><table style="border-collapse: collapse; border: none;"><colgroup><col width="227"></col><col width="252"></col></colgroup><tbody><tr style="height: 0pt;"><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: inherit;">Original Labels</span></span></p></td><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.2; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: inherit;">New Labels</span></span></p></td></tr><tr style="height: 78.6302pt;"><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><ol style="margin-bottom: 0px; margin-top: 0px; padding-inline-start: 48px;"><li aria-level="1" dir="ltr" style="font-variant-east-asian: normal; font-variant-numeric: normal; list-style-type: decimal; vertical-align: baseline; white-space: pre;"><p dir="ltr" role="presentation" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: inherit;">Left hand swipe</span></span></p></li><li aria-level="1" dir="ltr" style="font-variant-east-asian: normal; font-variant-numeric: normal; list-style-type: decimal; vertical-align: baseline; white-space: pre;"><p dir="ltr" role="presentation" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: inherit;">Right hand swipe</span></span></p></li><li aria-level="1" dir="ltr" style="font-variant-east-asian: normal; font-variant-numeric: normal; list-style-type: decimal; vertical-align: baseline; white-space: pre;"><p dir="ltr" role="presentation" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: inherit;">Thumbs down</span></span></p></li><li aria-level="1" dir="ltr" style="font-variant-east-asian: normal; font-variant-numeric: normal; list-style-type: decimal; vertical-align: baseline; white-space: pre;"><p dir="ltr" role="presentation" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: inherit;">Thumbs up</span></span></p></li></ol></td><td style="border-bottom: solid #000000 1pt; border-color: rgb(0, 0, 0); border-left: solid #000000 1pt; border-right: solid #000000 1pt; border-style: solid; border-top: solid #000000 1pt; border-width: 1pt; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><ol style="margin-bottom: 0px; margin-top: 0px; padding-inline-start: 48px;"><li aria-level="1" dir="ltr" style="font-variant-east-asian: normal; font-variant-numeric: normal; list-style-type: decimal; vertical-align: baseline; white-space: pre;"><p dir="ltr" role="presentation" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: inherit;">馃毇 - No gesture</span></span></p></li><li aria-level="1" dir="ltr" style="font-variant-east-asian: normal; font-variant-numeric: normal; list-style-type: decimal; vertical-align: baseline; white-space: pre;"><p dir="ltr" role="presentation" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: inherit;">✋ - Hand up</span></span></p></li><li aria-level="1" dir="ltr" style="font-variant-east-asian: normal; font-variant-numeric: normal; list-style-type: decimal; vertical-align: baseline; white-space: pre;"><p dir="ltr" role="presentation" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: inherit;">馃憥 - Thumbs down</span></span></p></li><li aria-level="1" dir="ltr" style="font-variant-east-asian: normal; font-variant-numeric: normal; list-style-type: decimal; vertical-align: baseline; white-space: pre;"><p dir="ltr" role="presentation" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: inherit;">馃憤 - Thumbs up</span></span></p></li><li aria-level="1" dir="ltr" style="font-variant-east-asian: normal; font-variant-numeric: normal; list-style-type: decimal; vertical-align: baseline; white-space: pre;"><p dir="ltr" role="presentation" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><span style="font-family: inherit;">馃憡 - Fist</span></span></p></li></ol></td></tr></tbody></table></div></span></td><td><br /></td></tr></tbody></table><p style="text-align: left;"> Since the swipe right and swipe left gestures in the Kaggle dataset do not correspond to any of these classes, any images in these classes will need to be discarded for our model. </p><p style="text-align: left;"> Images in the Kaggle dataset are taken over a 30 second period, they might contain other gestures at the start or end of the series. For example, some of the people in the dataset started with their hands in a fist position before eventually going to the labeled gesture hand up, thumbs up and thumbs down. Other times the person in the dataset starts off with no hand gesture in frame. </p><p style="text-align: left;"> We have gone ahead and manually re-labeled the images into the classes, it can be found in CSV format in the <em><a href="https://github.com/ArmDeveloperEcosystem/ml-image-classification-example-for-openmv/tree/main/data" target="_blank">data folder on GitHub</a></em>, and contains labels for ~14k images. </p><h2 style="text-align: left;">TensorFlow model</h2><p style="text-align: left;"> You can find more details on the training pipeline used here in <a href="https://colab.research.google.com/github/ArmDeveloperEcosystem/ml-image-classification-example-for-openmv/blob/main/notebook.ipynb" target="_blank">this Colab Notebook</a>. </p><h3 style="text-align: left;">Loading and Augmenting Images</h3><p style="text-align: left;"> Images from the dataset can be loaded as a <a href="https://www.tensorflow.org/api_docs/python/tf/data/Dataset" target="_blank">TensorFlow Dataset</a> using the <a href="https://www.tensorflow.org/api_docs/python/tf/keras/utils/image_dataset_from_directory" target="_blank">tf.keras.utils.image_dataset_from_directory(...) API.</a> This API supports adjusting the image’s color mode (to grayscale) and size (96x96 pixels) to meet the model’s desired input format. Built-in Keras layers for data augmentation (random: <a href="https://www.tensorflow.org/api_docs/python/tf/keras/layers/RandomFlip" target="_blank">flipping</a>, <a href="https://www.tensorflow.org/api_docs/python/tf/keras/layers/RandomRotation" target="_blank">rotation</a>, <a href="https://www.tensorflow.org/api_docs/python/tf/keras/layers/RandomZoom" target="_blank">zooming</a>, and <a href="https://www.tensorflow.org/api_docs/python/tf/keras/layers/RandomContrast" target="_blank">contrast adjustments</a>) will also be used during training. </p><h3 style="text-align: left;">Model Architecture</h3><p style="text-align: left;"><a href="https://arxiv.org/abs/1704.04861" target="_blank">MobileNetV1</a> is a well-known model architecture used for image classification tasks, including the <a href="https://github.com/tensorflow/tflite-micro/tree/main/tensorflow/lite/micro/examples/person_detection" target="_blank">TensorLite for Microcontrollers Person detection example</a>. This model architecture is trained on our dataset, with the same alpha (0.25) and image sizes (96x96x1) used in the <a href="https://arxiv.org/abs/1906.05721" target="_blank">Visual Wake Words Dataset</a> paper. A MobileNetV1 model is composed of 28 layers, but a <strong>single</strong> call to the Keras <a href="https://www.tensorflow.org/api_docs/python/tf/keras/applications/mobilenet/MobileNet" target="_blank">tf.keras.applications.mobilenet.MobileNet(...)</a> API can be used to easily create a MobileNetV1 model for 5 output classes and the desired <em>alpha</em> and <em>input shape</em> values:</p><div style="text-align: left;"><span style="font-family: courier;">python</span></div><div style="text-align: left;"><span id="docs-internal-guid-40b909a0-7fff-46cc-1513-167205aa276d"><div align="left" dir="ltr" style="margin-left: 0.75pt;"><table style="border-collapse: collapse; border: none;"><colgroup><col width="718"></col></colgroup><tbody><tr style="height: 0pt;"><td style="background-color: #f0f0f0; overflow-wrap: break-word; overflow: hidden; padding: 5pt; vertical-align: top;"><p dir="ltr" style="line-height: 1.38; margin-bottom: 0pt; margin-top: 0pt;"><span style="font-family: courier;"><span style="color: #444444; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">mobilenet_025_96 = tf.keras.applications.mobilenet.MobileNet(</span><span style="color: #444444; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><br /></span><span style="color: #444444; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">    input_shape=(</span><span style="color: #880000; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">96</span><span style="color: #444444; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">, </span><span style="color: #880000; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">96</span><span style="color: #444444; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">, </span><span style="color: #880000; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">1</span><span style="color: #444444; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">),</span><span style="color: #444444; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><br /></span><span style="color: #444444; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">    alpha=</span><span style="color: #880000; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">0.25</span><span style="color: #444444; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">,</span><span style="color: #444444; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><br /></span><span style="color: #444444; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">    dropout=</span><span style="color: #880000; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">0.10</span><span style="color: #444444; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">,</span><span style="color: #444444; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><br /></span><span style="color: #444444; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">    weights=</span><span style="color: #444444; font-variant-east-asian: normal; font-variant-numeric: normal; font-weight: 700; vertical-align: baseline; white-space: pre-wrap;">None</span><span style="color: #444444; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">,</span><span style="color: #444444; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><br /></span><span style="color: #444444; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">    pooling=</span><span style="color: #880000; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">'avg'</span><span style="color: #444444; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">,</span><span style="color: #444444; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><br /></span><span style="color: #444444; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">    classes=</span><span style="color: #880000; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">5</span><span style="color: #444444; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">,</span><span style="color: #444444; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;"><br /></span><span style="color: #444444; font-variant-east-asian: normal; font-variant-numeric: normal; vertical-align: baseline; white-space: pre-wrap;">)</span></span></p></td></tr></tbody></table><br /></div> The MicroPython based firmware used on the OpenMV Cam H7 does not include support for all of the layer types in the MobileNetV1 model created using the Keras API, however it can be adapted to use supported layers using only ~30 lines of Python code. Once the model is adapted and trained it can then be converted to TensorFlow Lite format using the <a href="https://www.tensorflow.org/api_docs/python/tf/lite/TFLiteConverter" target="_blank">tf.lite.TFLiteConverter.from_keras_model(..)</a> API. The resulting .tflite file can then be used for on-device inference on the OpenMV development board.<br />OpenMV Application and inferencing<br /><br />The .tflite model can then be integrated into the OpenMV application. You can find more details on the inference application in the <a href="https://colab.research.google.com/github/ArmDeveloperEcosystem/ml-image-classification-example-for-openmv/blob/main/notebook.ipynb" target="_blank">Colab Notebook</a> and full source code in the <a href="https://github.com/ArmDeveloperEcosystem/ml-image-classification-example-for-openmv/tree/main/openmv" target="_blank">openmv folder on GitHub</a>.<br /><br />The application will loop continuously performing the following steps:</span></div> <div style="text-align: left;"><table align="center" cellpadding="0" cellspacing="0" class="tr-caption-container" style="margin-left: auto; margin-right: auto; text-align: center;"><tbody><tr><td style="text-align: center;"><center><img alt="Block Diagram of Application processing pipeline" border="0" data-original-height="1504" data-original-width="720" src="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiU1pfsY_PVH34u5PXKAIUfMn0O_0vhczRrZAyNerRxVGSA560v11yx2VKvqo0deZ4DScifUCThMb-sEGiszGoyzFSHUJX7XPLg42vZ7Fj6HxSdoRSAjisjQCEJ-IRDXW2oWLfehHX1PTiEaMLnaeDsD-H6l_-BC19xsr7-6-2s2Y2_-WtuOU5Cykz6/s1600/image1.png" style="width: 100%;" td="" /></center></td></tr><tr><td class="tr-caption" style="text-align: center;"><i>Block Diagram of Application processing pipeline</i></td></tr></tbody></table></div> <ol style="text-align: left;"> <li>Grab an image frame from the camera. </li><li>Get the ML model’s output for the captured image frame. </li><li>Filter the ML model’s output for high certainty predictions using “low activation” and “margin of confidence” techniques. </li><li>Use an <a href="https://en.wikipedia.org/wiki/Exponential_smoothing#Basic_(simple)_exponential_smoothing_(Holt_linear)" target="_blank">exponential smoothing function</a> to smooth the model’s noisy (Softmax) outputs. </li><li>Use the exponentially smoothed model outputs to determine if a <strong>new</strong> hand gesture is present. </li><li>Then “type” the associated emoji on a PC using the USB HID protocol. </li> </ol> <h2 style="text-align: left;">Conclusion</h2> <p style="text-align: left;"> Throughout this project we’ve covered an end-to-end flow of training a custom image classification model and how to deploy it locally to a Arm Cortex-M7 based OpenMV development board using TensorFlow Lite! TensorFlow was used in a Google Colab notebook to train the model on a re-labeled public dataset from Kaggle. After training, the model was converted into TensorFlow Lite format to run on the OpenMV board using the TensorFlow Lite for Microcontrollers run-time along with accelerated Arm CMSIS-NN kernels. </p> <p style="text-align: left;"> </p> <p style="text-align: left;"> At inference time the model’s outputs were processed using model certainty techniques, and then fed output from the (Softmax) activation output into an exponential smoothing function to determine when to send keystrokes over USB HID to type emojis on a PC. The dedicated input device we created was able to capture and process grayscale 96x96 image data at just under 20 fps on an Arm Cortex-M7 processor running at 480 MHz. On-device inferencing provided a low latency response and preserved the privacy of the user by keeping all image data at the source and processing it locally. </p> <p style="text-align: left;"> </p> <p style="text-align: left;"> Build one yourself by purchasing an OpenMV Cam H7 R2 board on <a href="https://openmv.io/collections/products/products/openmv-cam-h7-r2" target="_blank">openmv.io</a> or <a href="https://openmv.io/collections/products" target="_blank">a distributor</a>. The project can be extended by fine tuning the model on your own data or applying transfer learning techniques and using the model we developed as base to train other hand gestures. Maybe you can find another public dataset for facial gestures and use it to type 馃榾 emojis when you smile! </p><p style="text-align: left;"><br /></p> <p style="text-align: left;"> <em>A big thanks to Sparsh Gupta for sharing the Gesture Recognition dataset on Kaggle under a public domain license and my Arm colleagues Rod Crawford, Prathyusha Venkata</em>,<em> Elham Harirpoush, and Liliya Wu for their help in reviewing the material for this blog post and associated tutorial!</em> </p> </div> </div> <div class='tensorsite-detail-footer'> <div class='article-divider'> <img alt='Diamond Article Divider' src='https://www.gstatic.com/tf_blog/images/ic_article_end.svg'/> </div> <a class='tensorsite-chip' href='https://blog.tensorflow.org/search?label=Community&max-results=20'> Community </a> <a class='tensorsite-chip' href='https://blog.tensorflow.org/search?label=TensorFlow+Lite&max-results=20'> TensorFlow Lite </a> </div> </div> <div class='tensorsite-next'> <div class='tensorsite-container--large'> <div class='tensorsite-next__title'>Next post</div> <div class='tensorsite-card'> <a aria-label='Next Card' class='tensorsite-card__href next' href='https://blog.tensorflow.org/2022/11/building-tensorflow-lite-based-computer-vision-emoji-input-device-openmv.html'></a> <div class='tensorsite-content__image-wrapper'> <img alt='Building a TensorFlow Lite based computer vision emoji input device with OpenMV' class='tensorsite-content__image' src='https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjc40D2_Rf0thFm_3_jvI0tCAqEclYeWViMpGH-3GU-uBOOHVEecs0Qxz2vEW9vn2gJOPqtsdM30yXHYXLdrFv-caVtMZOVq2hhfyvOQeSlsjJ9rpUMKxdCohIFtqACoHokDBIeFrZ2Er5qkDvbwgQt-xFukK8REUJANDDrIskgBKx8LUrDWQ7M9UkU/w665-h374/image5.gif'/> </div> <div class='tensorsite-content'> <div class='tensorsite-content__subtitle next'> <span>Community</span> <b class='label-divider-dot'>·</b> <span>TensorFlow Lite</span> <b class='label-divider-dot'>·</b> <svg class='community-icon' fill='none' height='24' viewBox='0 0 24 24' width='24' xmlns='http://www.w3.org/2000/svg'> <path clip-rule='evenodd' d='M16.67 13.13C18.04 14.06 19 15.32 19 17V20H23V17C23 14.82 19.43 13.53 16.67 13.13Z' fill='#555655' fill-rule='evenodd'></path> <path d='M9 12C11.2091 12 13 10.2091 13 8C13 5.79086 11.2091 4 9 4C6.79086 4 5 5.79086 5 8C5 10.2091 6.79086 12 9 12Z' fill='#555655'></path> <path clip-rule='evenodd' d='M15 12C17.21 12 19 10.21 19 8C19 5.79 17.21 4 15 4C14.53 4 14.09 4.1 13.67 4.24C14.5 5.27 15 6.58 15 8C15 9.42 14.5 10.73 13.67 11.76C14.09 11.9 14.53 12 15 12ZM9 13C6.33 13 1 14.34 1 17V20H17V17C17 14.34 11.67 13 9 13Z' fill='#555655' fill-rule='evenodd'></path> </svg> </div> <div class='tensorsite-content__title next'> Building a TensorFlow Lite based computer vision emoji input device with OpenMV </div> <p class='tensorsite-content__description next'> <span class='tensorsite-content__info'> November 29, 2022 </span> — <span> <em>A guest post by Sandeep Mistry, <a href="https://www.arm.com/" target="_blank">Arm</a></em>Introduction Emojis allow us to express emotions in the digital world, they are relatively easy to input on smartphone and tablet devices equipped with touch screen based virtual keyboards, but they are not as easy to input on traditional computing devices that have physical keyboards. To input emojis on these devices, users typically use a keyboard shortcut or … </span> </p> </div> </div> </div> </div>  </div></div> </div>  <div class='tensorsite-full-footer'> <div class='section' id='footer'><div class='widget HTML' data-version='1' id='HTML2'> <section class='tensorsite-footer'> <div class='tensorsite-container tensorsite-footer__container'> <div class='tensorsite-footer__side tensorsite-footer__side--left'></div> <div class='tensorsite-footer__side tensorsite-footer__side--right'></div> <div class='tensorsite-footer__content'> <div class='tensorsite-content'> <div class='tensorsite-content__title tensorsite-content__title--grow'> Build, deploy, and experiment easily with TensorFlow </div> <div class='tensorsite-content__cta-wrapper'> <a class='tensorsite-content__button tensorsite-button tensorsite-button--white' href='https://www.tensorflow.org/'> Get started </a> </div> </div> </div> </div> <div class='tensorsite-footer__lines'></div> </section> <section class='social-icons'> <div class='social-icons__container-footer'> <a alt='TensorFlow Home' class='tensorsite-logo' href='https://www.tensorflow.org/'> <img alt='TensorFlow Logo' class='tensorsite-logo__image' src='https://www.gstatic.com/tf_blog/images/tf_lockup.svg'/> </a> <div class='footer__side--right'> <div class='footer__links'> <a alt='Youtube Social Link' class='footer-link' href='https://www.google.com/' rel='noopener noreferrer' target='_blank'> Google </a> <a alt='Twitter Social Link' class='footer-link' href='https://policies.google.com/privacy' rel='noopener noreferrer' target='_blank'> Privacy </a> <a alt='Github Link' class='footer-link' href='https://policies.google.com/terms' rel='noopener noreferrer' target='_blank'> Terms </a> <a class='footer-link' href='https://blog.tensorflow.org/p/tensorflow-blog-contribution-notice.html'> Contributions notice </a> </div> <div class='social-icons__links'> <a alt='Youtube Social Link' class='icon-link' href='https://www.youtube.com/channel/UC0rqucBdTuFTjJiefW5t-IQ' rel='noopener noreferrer' target='_blank'> <img src='https://www.gstatic.com/tf_blog/images/ic_youtube.svg'/></a> <a alt='Twitter Social Link' class='icon-link' href='https://twitter.com/TensorFlow' rel='noopener noreferrer' target='_blank'> <img src='https://www.gstatic.com/tf_blog/images/ic_twitter.svg'/></a> <a alt='Github Link' class='icon-link' href='https://github.com/tensorflow' rel='noopener noreferrer' target='_blank'> <img src='https://www.gstatic.com/tf_blog/images/ic_github.svg'/></a> </div> </div> </div> </section> </div></div> </div> <script type="text/javascript" src="https://www.blogger.com/static/v1/widgets/984859869-widgets.js"></script> <script type='text/javascript'> window['__wavt'] = 'AOuZoY4l_UPWDzcKJ26jnwnckfZkWUEA5A:1732691800022';_WidgetManager._Init('//www.blogger.com/rearrange?blogID\x3d7864883956188652345','//blog.tensorflow.org/2022/11/building-tensorflow-lite-based-computer-vision-emoji-input-device-openmv.html','7864883956188652345'); _WidgetManager._SetDataContext([{'name': 'blog', 'data': {'blogId': '7864883956188652345', 'title': 'The TensorFlow Blog', 'url': 'https://blog.tensorflow.org/2022/11/building-tensorflow-lite-based-computer-vision-emoji-input-device-openmv.html', 'canonicalUrl': 'https://blog.tensorflow.org/2022/11/building-tensorflow-lite-based-computer-vision-emoji-input-device-openmv.html', 'homepageUrl': 'https://blog.tensorflow.org/', 'searchUrl': 'https://blog.tensorflow.org/search', 'canonicalHomepageUrl': 'https://blog.tensorflow.org/', 'blogspotFaviconUrl': 'https://blog.tensorflow.org/favicon.ico', 'bloggerUrl': 'https://www.blogger.com', 'hasCustomDomain': true, 'httpsEnabled': true, 'enabledCommentProfileImages': true, 'gPlusViewType': 'FILTERED_POSTMOD', 'adultContent': false, 'analyticsAccountNumber': 'UA-142343919-1', 'encoding': 'UTF-8', 'locale': 'en', 'localeUnderscoreDelimited': 'en', 'languageDirection': 'ltr', 'isPrivate': false, 'isMobile': false, 'isMobileRequest': false, 'mobileClass': '', 'isPrivateBlog': false, 'isDynamicViewsAvailable': true, 'feedLinks': '\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22The TensorFlow Blog - Atom\x22 href\x3d\x22https://blog.tensorflow.org/feeds/posts/default\x22 /\x3e\n\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/rss+xml\x22 title\x3d\x22The TensorFlow Blog - RSS\x22 href\x3d\x22https://blog.tensorflow.org/feeds/posts/default?alt\x3drss\x22 /\x3e\n\x3clink rel\x3d\x22service.post\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22The TensorFlow Blog - Atom\x22 href\x3d\x22https://www.blogger.com/feeds/7864883956188652345/posts/default\x22 /\x3e\n\n\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22The TensorFlow Blog - Atom\x22 href\x3d\x22https://blog.tensorflow.org/feeds/5968747221758897766/comments/default\x22 /\x3e\n', 'meTag': '', 'adsenseHostId': 'ca-host-pub-1556223355139109', 'adsenseHasAds': true, 'adsenseAutoAds': false, 'boqCommentIframeForm': true, 'loginRedirectParam': '', 'view': '', 'dynamicViewsCommentsSrc': '//www.blogblog.com/dynamicviews/4224c15c4e7c9321/js/comments.js', 'dynamicViewsScriptSrc': '//www.blogblog.com/dynamicviews/02de2df73990045b', 'plusOneApiSrc': 'https://apis.google.com/js/platform.js', 'disableGComments': true, 'interstitialAccepted': false, 'sharing': {'platforms': [{'name': 'Get link', 'key': 'link', 'shareMessage': 'Get link', 'target': ''}, {'name': 'Facebook', 'key': 'facebook', 'shareMessage': 'Share to Facebook', 'target': 'facebook'}, {'name': 'BlogThis!', 'key': 'blogThis', 'shareMessage': 'BlogThis!', 'target': 'blog'}, {'name': 'X', 'key': 'twitter', 'shareMessage': 'Share to X', 'target': 'twitter'}, {'name': 'Pinterest', 'key': 'pinterest', 'shareMessage': 'Share to Pinterest', 'target': 'pinterest'}, {'name': 'Email', 'key': 'email', 'shareMessage': 'Email', 'target': 'email'}], 'disableGooglePlus': true, 'googlePlusShareButtonWidth': 0, 'googlePlusBootstrap': '\x3cscript type\x3d\x22text/javascript\x22\x3ewindow.___gcfg \x3d {\x27lang\x27: \x27en\x27};\x3c/script\x3e'}, 'hasCustomJumpLinkMessage': false, 'jumpLinkMessage': 'Read more', 'pageType': 'item', 'postId': '5968747221758897766', 'postImageThumbnailUrl': 'https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjc40D2_Rf0thFm_3_jvI0tCAqEclYeWViMpGH-3GU-uBOOHVEecs0Qxz2vEW9vn2gJOPqtsdM30yXHYXLdrFv-caVtMZOVq2hhfyvOQeSlsjJ9rpUMKxdCohIFtqACoHokDBIeFrZ2Er5qkDvbwgQt-xFukK8REUJANDDrIskgBKx8LUrDWQ7M9UkU/s72-w665-c-h374/image5.gif', 'postImageUrl': 'https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjc40D2_Rf0thFm_3_jvI0tCAqEclYeWViMpGH-3GU-uBOOHVEecs0Qxz2vEW9vn2gJOPqtsdM30yXHYXLdrFv-caVtMZOVq2hhfyvOQeSlsjJ9rpUMKxdCohIFtqACoHokDBIeFrZ2Er5qkDvbwgQt-xFukK8REUJANDDrIskgBKx8LUrDWQ7M9UkU/w665-h374/image5.gif', 'pageName': 'Building a TensorFlow Lite based computer vision emoji input device with OpenMV', 'pageTitle': 'The TensorFlow Blog: Building a TensorFlow Lite based computer vision emoji input device with OpenMV', 'metaDescription': 'This is an in-depth open-source guide that uses tinyML on an Arm Cortex-M based device to create a dedicated input device.'}}, {'name': 'features', 'data': {}}, {'name': 'messages', 'data': {'edit': 'Edit', 'linkCopiedToClipboard': 'Link copied to clipboard!', 'ok': 'Ok', 'postLink': 'Post Link'}}, {'name': 'template', 'data': {'name': 'custom', 'localizedName': 'Custom', 'isResponsive': false, 'isAlternateRendering': false, 'isCustom': true}}, {'name': 'view', 'data': {'classic': {'name': 'classic', 'url': '?view\x3dclassic'}, 'flipcard': {'name': 'flipcard', 'url': '?view\x3dflipcard'}, 'magazine': {'name': 'magazine', 'url': '?view\x3dmagazine'}, 'mosaic': {'name': 'mosaic', 'url': '?view\x3dmosaic'}, 'sidebar': {'name': 'sidebar', 'url': '?view\x3dsidebar'}, 'snapshot': {'name': 'snapshot', 'url': '?view\x3dsnapshot'}, 'timeslide': {'name': 'timeslide', 'url': '?view\x3dtimeslide'}, 'isMobile': false, 'title': 'Building a TensorFlow Lite based computer vision emoji input device with OpenMV', 'description': 'This is an in-depth open-source guide that uses tinyML on an Arm Cortex-M based device to create a dedicated input device.', 'featuredImage': 'https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjc40D2_Rf0thFm_3_jvI0tCAqEclYeWViMpGH-3GU-uBOOHVEecs0Qxz2vEW9vn2gJOPqtsdM30yXHYXLdrFv-caVtMZOVq2hhfyvOQeSlsjJ9rpUMKxdCohIFtqACoHokDBIeFrZ2Er5qkDvbwgQt-xFukK8REUJANDDrIskgBKx8LUrDWQ7M9UkU/w665-h374/image5.gif', 'url': 'https://blog.tensorflow.org/2022/11/building-tensorflow-lite-based-computer-vision-emoji-input-device-openmv.html', 'type': 'item', 'isSingleItem': true, 'isMultipleItems': false, 'isError': false, 'isPage': false, 'isPost': true, 'isHomepage': false, 'isArchive': false, 'isLabelSearch': false, 'postId': 5968747221758897766}}]); _WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML1', 'nav', document.getElementById('HTML1'), {}, 'displayModeFull')); _WidgetManager._RegisterWidget('_FeaturedPostView', new _WidgetInfo('FeaturedPost1', 'blog', document.getElementById('FeaturedPost1'), {}, 'displayModeFull')); _WidgetManager._RegisterWidget('_BlogView', new _WidgetInfo('Blog1', 'blog', document.getElementById('Blog1'), {'cmtInteractionsEnabled': false, 'lightboxEnabled': true, 'lightboxModuleUrl': 'https://www.blogger.com/static/v1/jsbin/2646514562-lbx.js', 'lightboxCssUrl': 'https://www.blogger.com/static/v1/v-css/1964470060-lightbox_bundle.css'}, 'displayModeFull')); _WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML2', 'footer', document.getElementById('HTML2'), {}, 'displayModeFull')); </script> </body> </html>