CINXE.COM
Can robots learn from machine dreams? | MIT CSAIL
<!DOCTYPE html> <html lang="en" dir="ltr"> <head> <meta charset="utf-8" /> <script async src="https://www.googletagmanager.com/gtag/js?id=UA-5382944-3"></script> <script>window.dataLayer = window.dataLayer || [];function gtag(){dataLayer.push(arguments)};gtag("js", new Date());gtag("set", "developer_id.dMDhkMT", true);gtag("config", "UA-5382944-3", {"groups":"default","anonymize_ip":true,"page_placeholder":"PLACEHOLDER_page_path","allow_ad_personalization_signals":false});</script> <meta name="Generator" content="Drupal 10 (https://www.drupal.org)" /> <meta name="MobileOptimized" content="width" /> <meta name="HandheldFriendly" content="true" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> <link rel="icon" href="/themes/custom/csail/images/favicon.ico" type="image/vnd.microsoft.icon" /> <link rel="canonical" href="https://www.csail.mit.edu/news/can-robots-learn-machine-dreams" /> <link rel="shortlink" href="https://www.csail.mit.edu/node/12435" /> <title>Can robots learn from machine dreams? | MIT CSAIL</title> <link rel="stylesheet" media="all" href="/sites/default/files/css/css_pdQIowyIDD7QZL7mGlOGKtoK53dK-L-yRRy7VomD_S4.css?delta=0&language=en&theme=csail&include=eJxdyW0KAjEMhOEL1c2RJNtmu4H0g0wr1NMrKIj76x2eOXQ8NIGOT8O3pFVHyK1lkztXtjU0gq4QsDCk0M6QEI2B9beLAJwFITYXqs0Lmz7fN1iNsrWd7YaxTGu-YHTtA6Gzc3buJyj57GzbT7ZZ-9xNcUp6AcjLTlQ" /> <link rel="stylesheet" media="all" href="/sites/default/files/css/css_SzKmXz72cAgqsmYjJBKVS6I7DbEd68Iv3Ng8XDGVDxk.css?delta=1&language=en&theme=csail&include=eJxdyW0KAjEMhOEL1c2RJNtmu4H0g0wr1NMrKIj76x2eOXQ8NIGOT8O3pFVHyK1lkztXtjU0gq4QsDCk0M6QEI2B9beLAJwFITYXqs0Lmz7fN1iNsrWd7YaxTGu-YHTtA6Gzc3buJyj57GzbT7ZZ-9xNcUp6AcjLTlQ" /> <link rel="stylesheet" media="print" href="/sites/default/files/css/css_hjVYnGRcHfE0t18BdJM8ZyUPpy_j4SmB-ooqszk-zus.css?delta=2&language=en&theme=csail&include=eJxdyW0KAjEMhOEL1c2RJNtmu4H0g0wr1NMrKIj76x2eOXQ8NIGOT8O3pFVHyK1lkztXtjU0gq4QsDCk0M6QEI2B9beLAJwFITYXqs0Lmz7fN1iNsrWd7YaxTGu-YHTtA6Gzc3buJyj57GzbT7ZZ-9xNcUp6AcjLTlQ" /> </head> <body class="no-sidebars path-node page-node-type-news"> <a href="#main-content" class="visually-hidden focusable skip-link"> Skip to main content </a> <div class="dialog-off-canvas-main-canvas" data-off-canvas-main-canvas> <div role="document" class="page"> <div id="mobile-placeholder" class="mobile-header-placeholder"></div> <header id="site-header" class="navigation" role="banner"> <div class="audience-menu nav"> <div class="outer-wrapper navigation-wrapper"> <nav aria-label="Info for" id="block-audiencemenu"> <ul class="menu"> <li class="menu-item"> <a href="/students" data-drupal-link-system-path="node/2884">For Students</a> </li> <li class="menu-item"> <a href="http://cap.csail.mit.edu/" title="CSAIL Alliances Program website">For Industry</a> </li> <li class="menu-item"> <a href="/internal-resources" data-drupal-link-system-path="node/4866">For Members</a> </li> <li class="menu-item"> <a href="https://accessibility.mit.edu/" title="Accessibility">Accessibility</a> </li> <li class="menu-item"> <a href="/csail_login" data-drupal-link-system-path="csail_login">Login</a> </li> </ul> </nav> </div> </div> <div class="outer-wrapper navigation-wrapper"> <div class="container header"> <div class="main-menu-wrapper"> <div class="header-logo"> <a href="/" class="home-link"> <svg width="70px" height="40px" viewBox="0 0 150 86" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"> <g id="Page-1" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd"> <g id="CSAIL_Primary_Regular_RGB" transform="translate(-2.000000, -4.000000)"> <g id="Group" transform="translate(2.813800, 4.812400)"> <path d="M117.1642,2.233 L117.1642,34.729 L128.2572,34.729 L128.2572,54.917 L139.6382,54.917 L149.1102,12.778 L117.1642,2.233 Z M117.1642,85.099 L133.9002,80.434 L137.7002,63.54 L117.1642,63.54 L117.1642,85.099 Z M27.8702,0 L19.0112,0 L16.3062,10.112 L15.7572,13.366 L15.4432,13.366 L14.8942,10.19 L11.9942,0 L3.1742,0 L3.1742,18.265 L8.6222,18.265 L8.6222,8.739 L8.4272,5.407 L8.7802,5.407 L12.3862,18.265 L18.6192,18.265 L22.1462,5.368 L22.4992,5.368 L22.3022,9.015 L22.3022,18.265 L27.8702,18.265 L27.8702,0 Z M33.7102,18.266 L40.0212,18.266 L40.0212,0 L33.7102,0 L33.7102,18.266 Z M44.1772,0 L60.5612,0 L60.5612,5.133 L55.5432,5.133 L55.5432,18.265 L49.2322,18.265 L49.2322,5.133 L44.1772,5.133 L44.1772,0 Z" id="Fill-1" fill="#898D8D"></path> <g id="Group-6" transform="translate(0.000000, 14.698700)"> <path d="M74.5275,30.379 L76.5265,37.787 L70.9615,37.787 L72.9995,30.379 L73.6255,27.321 L73.9005,27.321 L74.5275,30.379 Z M111.7655,27.085 L107.8455,27.085 L107.8455,41.785 L111.7655,41.785 L111.7655,48.842 L92.9115,48.842 L92.9115,41.785 L96.8315,41.785 L96.8315,27.085 L92.9115,27.085 L92.9115,20.03 L111.7655,20.03 L111.7655,27.085 Z M93.5655,0 C84.6295,0 76.9855,1.843 73.6145,4.39 L67.8245,20.03 L80.2125,20.03 L90.7165,48.842 L79.1135,48.842 L77.8595,44.529 L69.7455,44.529 L68.4895,48.842 L57.1575,48.842 L52.6435,61.032 C55.7395,64.246 65.7755,66.755 78.0445,66.755 C89.5685,66.755 108.5015,64.167 117.1645,60.954 L117.1645,4.822 C113.9105,2.743 104.1105,0 93.5655,0 L93.5655,0 Z" id="Fill-2" fill="#F0B323"></path> <path d="M31.671,48.3302 L31.671,40.1782 C33.828,41.0002 37.121,41.5502 40.373,41.5502 C43.156,41.5502 44.254,41.1972 44.254,39.8252 C44.254,38.6862 43.354,38.4522 40.373,37.9432 C36.219,37.2382 31.515,36.1792 31.515,29.4762 C31.515,23.2052 35.71,19.2842 44.921,19.2842 C49.155,19.2842 53.348,20.0702 54.838,20.5392 L54.838,28.0262 C51.78,27.3592 49.075,26.9692 46.293,26.9692 C44.921,26.9692 43.156,27.0862 43.156,28.4172 C43.156,29.5942 44.177,29.8302 47.861,30.5742 C52.448,31.5152 55.975,33.0042 55.975,38.7672 C55.975,43.8222 52.879,49.5462 42.06,49.5462 C38.023,49.5462 33.318,48.8802 31.671,48.3302 L31.671,48.3302 Z M70.496,12.8122 L-7.10542736e-15,8.1912 L2.893,28.6612 C4.74,23.0992 9.731,19.2842 19.088,19.2842 C22.186,19.2842 25.401,19.8322 27.007,20.2652 L27.007,28.6142 C25.988,28.3002 23.205,27.9092 20.814,27.9092 C16.66,27.9092 13.562,29.3982 13.562,34.2972 C13.562,39.7862 17.482,41.0002 21.323,41.0002 C23.363,41.0002 24.577,40.8832 27.007,40.2972 L27.007,48.4502 C24.224,49.2702 22.147,49.5462 18.815,49.5462 C12.001,49.5462 7.678,47.5812 5.123,44.4512 L8.192,66.1672 L54.252,56.6872 L70.496,12.8122 Z" id="Fill-4" fill="#B94700"></path> </g> </g> </g> </g> </svg> </a> </div> <span class="header-mobile-wordmark">MIT CSAIL</span> <div class="header-navigation nav"> <div class="menus-wrapper"> <nav aria-label="Main " id="block-csail-main-menu"> <ul class="menu navigation-menu"> <li class="menu-item menu-item--collapsed nav-link"> <a href="/research" title="Research Areas, Impact Areas " data-drupal-link-system-path="node/9140">Research</a> </li> <li class="menu-item nav-link"> <a href="/people?roleFacets=Principal%20Investigators%2CCore/Dual%2CAssociates%2CEmeritus" data-drupal-link-query="{"roleFacets":"Principal Investigators,Core\/Dual,Associates,Emeritus"}" data-drupal-link-system-path="node/9844">People</a> </li> <li class="menu-item nav-link"> <a href="/news" title="CSAIL Articles Videos and Talks" data-drupal-link-system-path="node/9142">News</a> </li> <li class="menu-item nav-link"> <a href="/events" title="CSAIL Event calendar" data-drupal-link-system-path="events">Events</a> </li> <li class="menu-item nav-link"> <a href="/symposia">Symposia</a> </li> <li class="menu-item nav-link"> <a href="/about" title="Learn more about CSAIL" data-drupal-link-system-path="node/2851">About</a> </li> </ul> </nav> </div> </div> <div class="header-icons"> <a href="javascript:void(0)" class="navigation-menu-button" id="js-mobile-menu"> <i class="fal fa-bars mobile-menu-button"></i> </a> <a href="javascript:void(0)" id="js-search-trigger"><i class="fal fa-search"></i></a> <a target="_blank" href="http://web.mit.edu/"> <?xml version="1.0" encoding="UTF-8"?> <svg class="mit-logo" xmlns="http://www.w3.org/2000/svg" height="35px" widht="59px" viewBox="0 0 1480 840"><path fill="#FFFFFF" d="M880,880h160V400H880V880z M1120,320h400V160h-400V320z M880,160.00003h160v160H880V160.00003z M640,880h160 V160H640V880z M400,720h160V160H400V720z M160,880h160V160H160V880z M1120,880h160V400h-160V880z"></path></svg> </a> </div> </div> <div class="header-mobile-navigation"> <div class="mobile-nav"> <div class="mobile-menu"> <nav aria-label="Main " id="block-csail-main-menu"> <ul class="menu navigation-menu"> <li class="menu-item menu-item--collapsed nav-link"> <a href="/research" title="Research Areas, Impact Areas " data-drupal-link-system-path="node/9140">Research</a> </li> <li class="menu-item nav-link"> <a href="/people?roleFacets=Principal%20Investigators%2CCore/Dual%2CAssociates%2CEmeritus" data-drupal-link-query="{"roleFacets":"Principal Investigators,Core\/Dual,Associates,Emeritus"}" data-drupal-link-system-path="node/9844">People</a> </li> <li class="menu-item nav-link"> <a href="/news" title="CSAIL Articles Videos and Talks" data-drupal-link-system-path="node/9142">News</a> </li> <li class="menu-item nav-link"> <a href="/events" title="CSAIL Event calendar" data-drupal-link-system-path="events">Events</a> </li> <li class="menu-item nav-link"> <a href="/symposia">Symposia</a> </li> <li class="menu-item nav-link"> <a href="/about" title="Learn more about CSAIL" data-drupal-link-system-path="node/2851">About</a> </li> </ul> </nav> </div> <div class="mobile-audience-menu"> <nav aria-label="Info for" id="block-audiencemenu"> <ul class="menu"> <li class="menu-item"> <a href="/students" data-drupal-link-system-path="node/2884">For Students</a> </li> <li class="menu-item"> <a href="http://cap.csail.mit.edu/" title="CSAIL Alliances Program website">For Industry</a> </li> <li class="menu-item"> <a href="/internal-resources" data-drupal-link-system-path="node/4866">For Members</a> </li> <li class="menu-item"> <a href="https://accessibility.mit.edu/" title="Accessibility">Accessibility</a> </li> <li class="menu-item"> <a href="/csail_login" data-drupal-link-system-path="csail_login">Login</a> </li> </ul> </nav> </div> <div class="mobile-info-menu"> <nav aria-label="Info Menu" id="block-infomenu"> <ul class="menu"> <li class="menu-item"> <a href="http://live-csail.pantheonsite.io/contact-us">Contact</a> </li> <li class="menu-item"> <a href="mailto:news@csail.mit.edu?subject=CSAIL%20Media%20Inquiry">Press Requests</a> </li> <li class="menu-item"> <a href="https://accessibility.mit.edu/">Accessibility</a> </li> </ul> </nav> </div> </div> </div> </div> </div> <div class="search-section"> <div class="outer-wrapper"> <form action="/search" method="get" id="search-block-form" accept-charset="UTF-8"> <label for="edit-keys" class="visually-hidden">Search</label> <input title="Enter the terms you wish to search for." placeholder="Search CSAIL" data-drupal-selector="edit-keys" type="search" id="edit-keys" name="keywords" value="" size="15" maxlength="128" class="form-search"><i id="js-search-close" class="fal fa-times"></i> <div data-drupal-selector="edit-actions" class="form-actions js-form-wrapper form-wrapper" id="edit-actions"><input data-drupal-selector="edit-submit" type="submit" id="edit-submit" value="Search" class="button js-form-submit form-submit"> </div> </form> </div> </div> </header> <section id="featured"> <div class="columns"> <div class=""> <div data-drupal-messages-fallback class="hidden"></div> </div> </div> </section> <main role="main" class="main-wrapper"> <a id="main-content" tabindex="-1"></a> <section id="content"> <div class=""> <div id="block-csail-content" class="block block-system block-system-main-block"> <article id="node-12435" class=""> <div class="content-container news"> <div class="detail-top news-top clearfix green-background"> <div class="back-link hidden-mobile"><a href="/news"><i class="fa fa-angle-left"></i>Back to News</a></div> <div class="field field--name-field-publication-date field--type-datetime field--label-hidden field__item"><time datetime="2024-11-13T12:00:00Z" class="datetime">November 13 '24</time> </div> <section class="content-column left"> <h1 class="page-title detail-title news-title"><span class="field field--name-title field--type-string field--label-hidden">Can robots learn from machine dreams?</span> </h1> </section> <section class="sidebar-right content-page"> <h4 class="monobold-header author">Written By</h4> <div class="field field--name-field-author field--type-link field--label-hidden field__items"> <div class="field__item"><a href="/person/rachel-gordon">Rachel Gordon</a></div> </div> </section> </div> </div> <div class="detail-bottom news-bottom clearfix"> <div class="content-container news"> <div class="content-container-inner"> <div class="content-column left"> <div class="field field--name-field-image field--type-entity-reference field--label-hidden field__item"><article class="media media--type-image-media-bundle media--view-mode-default"> <div class="field field--name-field-image field--type-image field--label-hidden field__item"> <img loading="lazy" src="/sites/default/files/2024-11/LucidSim-6%20%282%29.jpg" width="3809" height="2425" alt="The MIT researchers developed an AI-powered simulator that generates unlimited, diverse, and realistic training data for robots. The team found that robots trained in this virtual environment called “LucidSim” can seamlessly transfer their skills to the real world, performing at expert levels without additional fine-tuning (Credit: Mike Grimmett/MIT CSAIL)." /> </div> </article> </div> </div> <div class="field--item body-text first-body"> <div class="paragraph paragraph--type--body-text paragraph--view-mode--default"> <div class="field field--name-field-text-rich field--type-text-long field--label-hidden content-column left content-page field__item"><p>For roboticists, one challenge towers above all others: generalization – the ability to create machines that can adapt to any environment or condition. Since the 1970s, the field has evolved from writing sophisticated programs to using deep learning, teaching robots to learn directly from human behavior. But a critical bottleneck remains: data quality. To improve, robots need to encounter scenarios that push the boundaries of their capabilities, operating at the edge of their mastery. This process traditionally requires human oversight, with operators carefully challenging robots to expand their abilities. As robots become more sophisticated, this hands-on approach hits a scaling problem: the demand for high-quality training data far outpaces humans' ability to provide it.</p><p>Now, a team of MIT CSAIL researchers have developed a novel approach to robot training that could significantly accelerate the deployment of adaptable, intelligent machines in real-world environments. The new system, called "LucidSim," uses recent advances in generative AI and physics simulators to create diverse and realistic virtual training environments, helping robots achieve expert-level performance in difficult tasks without any real-world data.</p><p><iframe width="560" height="315" src="https://www.youtube.com/embed/fgnJQrvTj70?si=Nj8VGfegI4ETDOeA" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" referrerpolicy="strict-origin-when-cross-origin" allowfullscreen></iframe>LucidSim combines physics simulation with generative AI models, addressing one of the most persistent challenges in robotics: transferring skills learned in simulation to the real world. “A fundamental challenge in robot learning has long been the ‘sim-to-real gap’ – the disparity between simulated training environments and the complex, unpredictable real world,” says MIT CSAIL postdoctoral associate Ge Yang, a lead researcher on LucidSim. “Previous approaches often relied on depth sensors, which simplified the problem but missed crucial real-world complexities.”</p><p>The multipronged system is a blend of different technologies. At its core, LucidSim uses large language models to generate various structured descriptions of environments. These descriptions are then transformed into images using generative models. To ensure that these images reflect real-world physics, an underlying physics simulator is used to guide the generation process. <br><br><strong>The Birth of an Idea: From Burritos to Breakthroughs</strong><br><br><span>The inspiration for LucidSim came from an unexpected place: a conversation outside Beantown Taqueria in Cambridge. ”We wanted to teach vision-equipped robots how to improve using human feedback. But then, we realized we didn't have a pure vision-based policy to begin with,” says Alan Yu, an undergraduate student at MIT and co-lead on LucidSim. "We kept talking about it as we walked down the street, and then we stopped outside the taqueria for about half an hour. That's where we had our moment."</span></p><p dir="ltr"><span>To cook up their data, the team generated realistic images by extracting depth maps, which provide geometric information, and semantic masks, which label different parts of an image, from the simulated scene. They quickly realized, however, that with tight control on the composition of the image content, the model would produce similar images that weren’t different from each other using the same prompt. So, they devised a way to source diverse text prompts from ChatGPT. </span></p><p dir="ltr"><span>This approach, however, only resulted in a single image. To make short, coherent videos which serve as little “experiences” for the robot, the scientists hacked together some image magic into another novel technique the team created, called “Dreams In Motion (DIM).” The system computes the movements of each pixel between frames, to warp a single generated image into a short, multi-frame video. Dreams In Motion does this by considering the 3D geometry of the scene and the relative changes in the robot’s perspective. </span></p><p dir="ltr"><span>"We outperform domain randomization, a method developed in 2017 that applies random colors and patterns to objects in the environment, which is still considered the go-to method these days," says Yu. "While this technique generates diverse data, it lacks realism. LucidSim addresses both diversity and realism problems. It’s exciting that even without seeing the real world during training, the robot can recognize and navigate obstacles in real environments." </span></p><p dir="ltr"><span>The team is particularly excited about the potential of applying LucidSim to domains outside quadruped locomotion and parkour, their main testbed. One example is mobile manipulation, where a mobile robot is tasked to handle objects in an open area, and also, color perception is critical. “Today, these robots still learn from real-world demonstrations,” says Yang. “Although collecting demonstrations is easy, scaling a real-world robot teleoperation setup to thousands of skills is challenging because a human has to physically set up each scene. We hope to make this easier, thus qualitatively more scalable, by moving data collection into a virtual environment.” </span></p><p dir="ltr"><span>The team put LucidSim to the test against an alternative, where an expert teacher demonstrates the skill for the robot to learn from. The results were surprising: robots trained by the expert struggled, succeeding only 15 percent of the time – and even quadrupling the amount of expert training data barely moved the needle. But when robots collected their own training data through LucidSim, the story changed dramatically. Just doubling the dataset size catapulted success rates to 88 percent. "And giving our robot more data monotonically improves its performance – eventually, the student becomes the expert," says Yang.</span></p><p dir="ltr"><span>“One of the main challenges in sim-to-real transfer for robotics is achieving visual realism in simulated environments,” says Stanford University Assistant Professor of Electrical Engineering Shuran Song, who wasn’t involved in the research. “The LucidSim framework provides an elegant solution by using generative models to create diverse, highly realistic visual data for any simulation. This work could significantly accelerate the deployment of robots trained in virtual environments to real-world tasks.”</span></p><p dir="ltr"><span>From the streets of Cambridge to the cutting edge of robotics research, LucidSim is paving the way toward a new generation of intelligent, adaptable machines – ones that learn to navigate our complex world without ever setting foot in it.</span></p><p dir="ltr"><span>Yu and Yang wrote the paper with four fellow CSAIL affiliates: mechanical engineering postdoc Ran Choi; undergraduate researcher Yajvan Ravan; John Leonard, Samuel C. Collins Professor of Mechanical and Ocean Engineering in the MIT Department of Mechanical Engineering; and MIT Associate Professor Phillip Isola. Their work was supported, in part, by a Packard Fellowship, a Sloan Research Fellowship, the Office of Naval Research, Singapore’s Defence Science and Technology Agency, Amazon, MIT Lincoln Laboratory, and the National Science Foundation Institute for Artificial Intelligence and Fundamental Interactions. The researchers will present their work at the Conference on Robot Learning (CoRL) in early November.</span></p></div> </div> </div> <div class="sidebar-right content-page"> <h4 class="monobold-header">People</h4> <div class="field field--name-field-featured-people field--type-entity-reference field--label-hidden field__items"> <div class="field__item"><a href="/person/alan-yu" rel="bookmark"> <article id="node-8831" class=""> <div class="member"> <div class="circle-crop"> <img src="/themes/custom/csail/images/core-images/default-headshot.png" alt="default headshot" /> </div> <h2 class="member-name"><span class="field field--name-title field--type-string field--label-hidden">Alan Yu</span> </h2> </div> </article> </a> </div> <div class="field__item"><a href="/person/ge-yang" rel="bookmark"> <article id="node-9544" class=""> <div class="member"> <div class="circle-crop"> <img src="/themes/custom/csail/images/core-images/default-headshot.png" alt="default headshot" /> </div> <h2 class="member-name"><span class="field field--name-title field--type-string field--label-hidden">Ge Yang</span> </h2> </div> </article> </a> </div> <div class="field__item"><a href="/person/phillip-john-isola" rel="bookmark"> <article id="node-4882" class=""> <div class="member"> <div class="circle-crop"> <div class="field field--name-field-card-image-file field--type-image field--label-hidden field__item"> <img loading="lazy" src="/sites/default/files/styles/headshot/public/images/people/card/Phillip%20Isola.jpg?h=86975c96&itok=sUfCjfVd" width="200" height="200" alt="Phillip Isola" class="image-style-headshot" /> </div> </div> <h2 class="member-name"><span class="field field--name-title field--type-string field--label-hidden">Phillip John Isola</span> </h2> </div> </article> </a> </div> <div class="field__item"><a href="/person/john-leonard" rel="bookmark"> <article id="node-3438" class=""> <div class="member"> <div class="circle-crop"> <div class="field field--name-field-card-image-file field--type-image field--label-hidden field__item"> <img loading="lazy" src="/sites/default/files/styles/headshot/public/images/people/card/jleonard_headshot3_nov2014.jpg?h=1adc8c62&itok=CrYhYPMZ" width="200" height="200" alt="Leonard" class="image-style-headshot" /> </div> </div> <h2 class="member-name"><span class="field field--name-title field--type-string field--label-hidden">John Leonard</span> </h2> </div> </article> </a> </div> </div> <div class="research-area-tags"> <h4 class="monobold-header">Research Areas</h4> <div class="field field--name-field-research-area field--type-entity-reference field--label-hidden field__items"> <div class="field__item"><div id="taxonomy-term-9" class="taxonomy-term vocabulary-research-area"> <a href="/taxonomy/term/9"> <div class="field field--name-name field--type-string field--label-hidden field__item">AI & ML</div> </a> <div class="content"> </div> </div> </div> <div class="field__item"><div id="taxonomy-term-17" class="taxonomy-term vocabulary-research-area"> <a href="/taxonomy/term/17"> <div class="field field--name-name field--type-string field--label-hidden field__item">Graphics & Vision</div> </a> <div class="content"> </div> </div> </div> <div class="field__item"><div id="taxonomy-term-15" class="taxonomy-term vocabulary-research-area"> <a href="/taxonomy/term/15"> <div class="field field--name-name field--type-string field--label-hidden field__item">Robotics</div> </a> <div class="content"> </div> </div> </div> </div> </div> <div class="impact-area-tags"> <h4 class="monobold-header">Impact Areas</h4> <div class="field field--name-field-impact-area field--type-entity-reference field--label-hidden field__items"> <div class="field__item"><div id="taxonomy-term-3" class="taxonomy-term vocabulary-impact-area"> <a href="/taxonomy/term/3"> <div class="field field--name-name field--type-string field--label-hidden field__item">Big Data</div> </a> <div class="content"> </div> </div> </div> <div class="field__item"><div id="taxonomy-term-21" class="taxonomy-term vocabulary-impact-area"> <a href="/taxonomy/term/21"> <div class="field field--name-name field--type-string field--label-hidden field__item">Transportation</div> </a> <div class="content"> </div> </div> </div> </div> </div> <div class="contact"> <h4 class="monobold-header">Press Contact</h4> <div class="field field--name-field-press-contact field--type-entity-reference field--label-hidden field__item"><article id="node-3239" class=""> <h2><span class="field field--name-title field--type-string field--label-hidden">Rachel Gordon</span> </h2> <div class="event-contact-person"> <div class="email"><span class="meta-label">E </span><a href="mailto:rachelg@csail.mit.edu">rachelg@csail.mit.edu</a></div> <div class="phone"><span class="meta-label">T </span>258-0675</div> </div> </article></div> </div> </div> <div class="field__item sidebar-item"> <div class="paragraph paragraph--type--sidebar-list-of-links paragraph--view-mode--default sidebar-right content-page"> <div class="field field--name-field-title field--type-string field--label-hidden field__item">RELATED</div> <div class="field field--name-field-list-of-links field--type-link field--label-hidden field__items"> <div class="field__item"><a href="https://arxiv.org/abs/2411.00083">Paper: “Learning Visual Parkour from Generated Images”</a></div> <div class="field__item"><a href="https://lucidsim.github.io/">Project website</a></div> <div class="field__item"><a href="https://lis.csail.mit.edu/">Learning and Intelligent Systems Group</a></div> <div class="field__item"><a href="https://www.eecs.mit.edu/">Department of Electrical Engineering and Computer Science</a></div> <div class="field__item"><a href="https://engineering.mit.edu/">School of Engineering</a></div> </div> </div> </div> <div class="field__item sidebar-item"> <div class="paragraph paragraph--type--sidebar-list-of-links paragraph--view-mode--default sidebar-right content-page"> <div class="field field--name-field-title field--type-string field--label-hidden field__item">PRESS MENTIONS</div> <div class="field field--name-field-list-of-links field--type-link field--label-hidden field__items"> <div class="field__item"><a href="https://www.newscientist.com/article/2455418-ai-helps-robot-dogs-navigate-the-real-world/">New Scientist</a></div> <div class="field__item"><a href="https://www.technologyreview.com/2024/11/12/1106811/generative-ai-taught-a-robot-dog-to-scramble-around-a-new-environment/">MIT Technology Review</a></div> <div class="field__item"><a href="https://spectrum.ieee.org/video-friday-extreme-off-road">IEEE Spectrum</a></div> <div class="field__item"><a href="https://www.yahoo.com/news/mit-lucidsim-turns-robot-dog-111308698.html">Yahoo! News</a></div> <div class="field__item"><a href="https://www.msn.com/en-us/news/technology/mit-s-lucidsim-turns-robot-dog-into-parkour-pro-without-real-world-practice/ar-AA1uqAQk?ocid=BingNewsVerp&apiversion=v2&noservercache=1&domshim=1&renderwebcomponents=1&wcseo=1&batchservertelemetry=1&noservertelemetry=1">MSN</a></div> <div class="field__item"><a href="https://interestingengineering.com/innovation/mit-turns-robot-dog-into-parkour-pro">Interesting Engineering</a></div> <div class="field__item"><a href="https://singularityhub.com/2024/11/15/mits-new-robot-dog-learned-to-walk-and-climb-in-a-simulation-whipped-up-by-generative-ai/">Singularity Hub</a></div> <div class="field__item"><a href="https://t.cj.sina.com.cn/articles/view/3996876140/ee3b7d6c001013o12?finpagefr=p_103">Sina (China)</a></div> <div class="field__item"><a href="https://www.163.com/dy/article/JH952FV20511AQHO.html">NetEase (China)</a></div> <div class="field__item"><a href="https://www.bilibili.com/opus/1001354432339771413">bilibili (China)</a></div> <div class="field__item"><a href="https://www.emergingtechbrew.com/stories/2024/11/07/ai-visuals-robot-training-mit-csail-research">Tech Brew</a></div> <div class="field__item"><a href="https://www.therobotreport.com/mit-lucidsim-training-system-helps-robots-close-sim2real-gap/">The Robot Report</a></div> <div class="field__item"><a href="https://www.hdblog.it/tecnologia/articoli/n599891/robot-ai-addestramento-virtuale-studio/">HDblog (Italy)</a></div> <div class="field__item"><a href="https://techxplore.com/news/2024-11-virtual-generative-ai-robots-traverse.html">Tech Xplore</a></div> </div> </div> </div> </div> <div class="clearfix content-divider"></div> <h2 class="monobold-header related-title">Related News</h2> <div class="tri-column gray-background related-view related-news"> <div class="clearfix"> <div class="third-column"> <article id="node-11222" class=""> <div class="card news-card js-preview-item"> <div class="card-top"> <div class="image-media"> <a href="/news/using-language-give-robots-better-grasp-open-ended-world"> <div class="field field--name-field-image field--type-entity-reference field--label-hidden field__item"><article class="media media--type-image-media-bundle media--view-mode-news-card-image"> <div class="field field--name-field-image field--type-image field--label-hidden field__item"> <img loading="lazy" src="/sites/default/files/styles/large/public/2023-11/MIT%20-%20F3RM%2015.png?itok=Pz85Fw7n" width="480" height="320" alt="Feature Fields for Robotic Manipulation (F3RM) enables robots to interpret open-ended text prompts using natural language, helping the machines manipulate unfamiliar objects. The system’s 3D feature fields could be helpful in environments that contain thousands of objects, such as warehouses (Credits: Courtesy of the researchers)." class="image-style-large" /> </div> </article> </div> </a> <div class="areas-list hidden-card inline-content"> <a href="/taxonomy/term/9" hreflang="en">AI & ML</a> <a href="/taxonomy/term/17" hreflang="en">Graphics & Vision</a> <a href="/taxonomy/term/15" hreflang="en">Robotics</a> <a href="/taxonomy/term/12" hreflang="en">Manufacturing</a> </div> </div> <div class="hidden-card"> <div class="field field--name-field-publication-date field--type-datetime field--label-hidden field__item"><time datetime="2023-11-02T12:00:00Z" class="datetime">November 02 '23</time> </div> <a href="/news/using-language-give-robots-better-grasp-open-ended-world" rel="bookmark"> <h2 class="news-media-heading"><span class="field field--name-title field--type-string field--label-hidden">Using language to give robots a better grasp of an open-ended world</span> </h2> </a> <div class="field field--name-field-teaser-text field--type-string-long field--label-hidden field__item">By blending 2D images with foundation models to build 3D feature fields, a new MIT method helps robots understand and manipulate nearby objects with open-ended language prompts.</div> </div> </div> <div class="card-bottom hidden-list"> <div class="hidden-list"> <a href="/news/using-language-give-robots-better-grasp-open-ended-world" rel="bookmark"> <h2 class="news-media-heading"><span class="field field--name-title field--type-string field--label-hidden">Using language to give robots a better grasp of an open-ended world</span> </h2> </a> </div> </div> </div> </article> </div> <div class="third-column"> <article id="node-10597" class=""> <div class="card news-card js-preview-item"> <div class="card-top"> <div class="image-media"> <a href="/news/four-legged-robotic-system-playing-soccer-various-terrains"> <div class="field field--name-field-image field--type-entity-reference field--label-hidden field__item"><article class="media media--type-image-media-bundle media--view-mode-news-card-image"> <div class="field field--name-field-image field--type-image field--label-hidden field__item"> <img loading="lazy" src="/sites/default/files/styles/large/public/2023-04/DribbleBot-MIT%20%281%29.jpeg?itok=bobIWTgt" width="480" height="319" alt="Researchers created DribbleBot, a system for in-the-wild dribbling on diverse natural terrains including sand, gravel, mud, and snow using onboard sensing and computing. In addition to these football feats, such robots may someday aid humans in search-and-rescue missions." class="image-style-large" /> </div> </article> </div> </a> <div class="areas-list hidden-card inline-content"> <a href="/taxonomy/term/14" hreflang="en">Algorithms & Theory</a> <a href="/taxonomy/term/9" hreflang="en">AI & ML</a> <a href="/taxonomy/term/15" hreflang="en">Robotics</a> <a href="/taxonomy/term/16" hreflang="en">Entertainment</a> </div> </div> <div class="hidden-card"> <div class="field field--name-field-publication-date field--type-datetime field--label-hidden field__item"><time datetime="2023-04-03T12:00:00Z" class="datetime">April 03 '23</time> </div> <a href="/news/four-legged-robotic-system-playing-soccer-various-terrains" rel="bookmark"> <h2 class="news-media-heading"><span class="field field--name-title field--type-string field--label-hidden">A four-legged robotic system for playing soccer on various terrains</span> </h2> </a> <div class="field field--name-field-teaser-text field--type-string-long field--label-hidden field__item">"DribbleBot" can maneuver a soccer ball on landscapes such as sand, gravel, mud, and snow, using reinforcement learning to adapt to varying ball dynamics.</div> </div> </div> <div class="card-bottom hidden-list"> <div class="hidden-list"> <a href="/news/four-legged-robotic-system-playing-soccer-various-terrains" rel="bookmark"> <h2 class="news-media-heading"><span class="field field--name-title field--type-string field--label-hidden">A four-legged robotic system for playing soccer on various terrains</span> </h2> </a> </div> </div> </div> </article> </div> </div> </div> </div> </div> </article> </div> </div> </section> </main> <footer id="site-footer" role="contentinfo"> <section class="footer-bottom"> <h2>MIT CSAIL</h2> <div class="footer-sections"> <div id="block-footerinfotextblock" class="block block-block-content block-block-content65e2ef2c-038f-4dc6-b6ed-2ff88073b056"> <div class="clearfix text-formatted field field--name-body field--type-text-with-summary field--label-hidden field__item"><p>Massachusetts Institute of Technology</p> <p>Computer Science & Artificial Intelligence Laboratory</p> <p>32 Vassar St, Cambridge MA 02139</p></div> </div> <nav aria-label="Info Menu" id="block-infomenu"> <ul class="menu"> <li class="menu-item"> <a href="http://live-csail.pantheonsite.io/contact-us">Contact</a> </li> <li class="menu-item"> <a href="mailto:news@csail.mit.edu?subject=CSAIL%20Media%20Inquiry">Press Requests</a> </li> <li class="menu-item"> <a href="https://accessibility.mit.edu/">Accessibility</a> </li> </ul> </nav> <div> <a href="https://computing.mit.edu" style="border-bottom: none !important;"><img style="max-width: 250px; margin-right: 100px;" src="/themes/custom/csail/images/SCC-black.png" alt="MIT Schwarzman College of Computing" /></a> </div> <div id="block-footersocialinfoblock" class="block block-block-content block-block-content6b8d3d82-2fda-46aa-8055-09f052768ffd"> <div class="footer-social-icons"> <a aria-label="twitter" target="_blank" href="https://twitter.com/MIT_CSAIL"><i class="fab fa-twitter-square"></i></a> <a aria-label="facebook" target="_blank" href="https://www.facebook.com/MITCSAIL/"><i class="fab fa-facebook-square"></i></a> <a aria-label="youtube" target="_blank" href="https://www.youtube.com/user/MITCSAIL"><i class="fab fa-youtube-square"></i></a> <a aria-label="instagram" target="_blank" href="https://www.instagram.com/mit_csail/"><i class="fab fa-instagram"></i></a> <a aria-label="linkedin" target="_blank" href="https://www.linkedin.com/company/mit-csail"><i class="fab fa-linkedin"></i></a> </div> </div> </div> </section> </footer> </div> </div> <script type="application/json" data-drupal-selector="drupal-settings-json">{"path":{"baseUrl":"\/","pathPrefix":"","currentPath":"node\/12435","currentPathIsAdmin":false,"isFront":false,"currentLanguage":"en"},"pluralDelimiter":"\u0003","suppressDeprecationErrors":true,"fitvids":{"selectors":".node\r,.main-wrapper","custom_vendors":"","ignore_selectors":""},"google_analytics":{"account":"UA-5382944-3","trackOutbound":true,"trackMailto":true,"trackTel":true,"trackDownload":true,"trackDownloadExtensions":"7z|aac|arc|arj|asf|asx|avi|bin|csv|doc(x|m)?|dot(x|m)?|exe|flv|gif|gz|gzip|hqx|jar|jpe?g|js|mp(2|3|4|e?g)|mov(ie)?|msi|msp|pdf|phps|png|ppt(x|m)?|pot(x|m)?|pps(x|m)?|ppam|sld(x|m)?|thmx|qtm?|ra(m|r)?|sea|sit|tar|tgz|torrent|txt|wav|wma|wmv|wpd|xls(x|m|b)?|xlt(x|m)|xlam|xml|z|zip","trackColorbox":true},"user":{"uid":0,"permissionsHash":"93e78a3f36d5585de87f6d42e888f6eb167bcf31de54b95e8aec1cf3380914cc"}}</script> <script src="/core/assets/vendor/jquery/jquery.min.js?v=3.7.1"></script> <script src="/core/assets/vendor/underscore/underscore-min.js?v=1.13.6"></script> <script src="/core/assets/vendor/once/once.min.js?v=1.0.1"></script> <script src="/core/misc/drupalSettingsLoader.js?v=10.2.5"></script> <script src="/core/misc/drupal.js?v=10.2.5"></script> <script src="/core/misc/drupal.init.js?v=10.2.5"></script> <script src="/modules/contrib/jquery_ui/assets/vendor/jquery.ui/ui/version-min.js?v=1.13.2"></script> <script src="/modules/contrib/jquery_ui/assets/vendor/jquery.ui/ui/jquery-var-for-color-min.js?v=1.13.2"></script> <script src="/modules/contrib/jquery_ui/assets/vendor/jquery.ui/ui/vendor/jquery-color/jquery.color-min.js?v=1.13.2"></script> <script src="/modules/contrib/jquery_ui/assets/vendor/jquery.ui/ui/effect-min.js?v=1.13.2"></script> <script src="/libraries/fitvids/jquery.fitvids.js?v=10.2.5"></script> <script src="/modules/contrib/fitvids/js/init-fitvids.js?v=10.2.5"></script> <script src="/modules/contrib/google_analytics/js/google_analytics.js?v=10.2.5"></script> <script src="/modules/contrib/jquery_ui/assets/vendor/jquery.ui/ui/effects/effect-slide-min.js?v=1.13.2"></script> <script src="/themes/custom/csail/js/jquery.colorbox-min.js?v=10.2.5"></script> <script src="/themes/custom/csail/js/scripts.js?v=10.2.5"></script> <script src="/themes/custom/csail/js/research-landing.js?v=10.2.5"></script> <script src="/themes/custom/csail/js/jquery.cycle2.min.js?v=10.2.5"></script> <script src="/themes/custom/csail/js/jquery.cycle2.swipe.min.js?v=10.2.5"></script> <script src="/themes/custom/csail/js/owl.carousel.js?v=10.2.5"></script> <script src="/themes/custom/csail/js/moment.js?v=10.2.5"></script> <script src="/themes/custom/csail/js/clndr.js?v=10.2.5"></script> </body> </html>