JoyVASA: Portrait and Animal Image Animation with Diffusion-Based Audio-Driven Facial Dynamics and Head Motion Generation

<!doctype html> <html lang="en"> <head> <meta charset="utf-8"> <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"> <script> const GTAG_ENABLED = true ; const GTAG_TRACKING_ID = "UA-121182717-1"; const SENTRY_DSN_FRONTEND = "".trim(); const GLOBAL_CSRF_TOKEN = 'Eo88EfbtFiRnMe4t80t1y4SHtGuQVdEsQULrrYIl7gfaullg59J8JGie74OOfmJT'; const MEDIA_URL = "https://production-media.paperswithcode.com/"; const ASSETS_URL = "https://production-assets.paperswithcode.com"; run_after_frontend_loaded = window.run_after_frontend_loaded || []; </script> <link rel="preconnect" href="https://production-assets.paperswithcode.com"><link rel="dns-prefetch" href="https://production-assets.paperswithcode.com"><link rel="preload" as="font" type="font/woff2" href="https://production-assets.paperswithcode.com/perf/fonts/65e877e527022735c1a1.woff2" crossorigin><link rel="preload" as="font" type="font/woff2" href="https://production-assets.paperswithcode.com/perf/fonts/917632e36982ca7933c8.woff2" crossorigin><link rel="preload" as="font" type="font/woff2" href="https://production-assets.paperswithcode.com/perf/fonts/f1405bd8a987c2ea8a67.woff2" crossorigin><script>(()=>{if(GTAG_ENABLED){const t=document.createElement("script");function n(){window.dataLayer.push(arguments)}t.src=`https://www.googletagmanager.com/gtag/js?id=${GTAG_TRACKING_ID}`,document.head.appendChild(t),window.dataLayer=window.dataLayer||[],window.gtag=n,n("js",new Date),n("config",GTAG_TRACKING_ID),window.captureOutboundLink=function(t){n("event","click",{event_category:"outbound",event_label:t})}}else window.captureOutboundLink=function(n){document.location=n}})();</script><link rel="preload" as="script" href="https://production-assets.paperswithcode.com/perf/766.4af6b88b.js"><link rel="preload" as="script" href="https://production-assets.paperswithcode.com/perf/351.a22a9607.js"><link rel="preload" as="style" href="https://production-assets.paperswithcode.com/perf/918.c41196c3.css"><link rel="preload" as="style" href="https://production-assets.paperswithcode.com/perf/view_paper.05773d2b.css"><link rel="stylesheet" href="https://production-assets.paperswithcode.com/perf/918.c41196c3.css"><link rel="stylesheet" href="https://production-assets.paperswithcode.com/perf/view_paper.05773d2b.css">  <title>JoyVASA: Portrait and Animal Image Animation with Diffusion-Based Audio-Driven Facial Dynamics and Head Motion Generation | Papers With Code</title> <meta name="description" content="Implemented in one code library." />  <meta property="og:title" content="Papers with Code - JoyVASA: Portrait and Animal Image Animation with Diffusion-Based Audio-Driven Facial Dynamics and Head Motion Generation"> <meta property="og:description" content="Implemented in one code library."> <meta property="og:image" content="https://raw.githubusercontent.com/jdh-algo/JoyVASA/master/assets/imgs/pipeline_inference.png"> <meta property="og:url" content="https://paperswithcode.com/paper/joyvasa-portrait-and-animal-image-animation">  <meta name="twitter:card" content="summary_large_image"> <meta name="twitter:site" content="@paperswithcode"> <meta name="twitter:title" content="Papers with Code - JoyVASA: Portrait and Animal Image Animation with Diffusion-Based Audio-Driven Facial Dynamics and Head Motion Generation"> <meta name="twitter:description" content="Implemented in one code library."> <meta name="twitter:creator" content="@paperswithcode"> <meta name="twitter:url" content="https://paperswithcode.com/paper/joyvasa-portrait-and-animal-image-animation"> <meta name="twitter:domain" content="paperswithcode.com">  <script type="application/ld+json">{ "@context": "http://schema.org", "@graph": { "@type": "ScholarlyArticle", "@id": "2411.09209", "name": "JoyVASA: Portrait and Animal Image Animation with Diffusion-Based Audio-Driven Facial Dynamics and Head Motion Generation", "description": "Implemented in one code library.", "url": "https://paperswithcode.com/paper/joyvasa-portrait-and-animal-image-animation", "image": "https://raw.githubusercontent.com/jdh-algo/JoyVASA/master/assets/imgs/pipeline_inference.png", "headline": "JoyVASA: Portrait and Animal Image Animation with Diffusion-Based Audio-Driven Facial Dynamics and Head Motion Generation", "abstract": "Implemented in one code library.", "author": [ { "@type": "Person", "@id": "#Xuyang_Cao", "name": "Xuyang Cao", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Guoxin_Wang", "name": "Guoxin Wang", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Sheng_Shi", "name": "Sheng Shi", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Jun_Zhao", "name": "Jun Zhao", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Yang_Yao", "name": "Yang Yao", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Jintao_Fei", "name": "Jintao Fei", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Minyu_Gao", "name": "Minyu Gao", "image": "https://paperswithcode.com/static/" } ], "workExample": [ { "@type": "SoftwareSourceCode", "@id": "https://github.com/jdh-algo/JoyVASA", "name": "JoyVASA", "description": "", "url": "https://github.com/jdh-algo/JoyVASA", "image": "https://paperswithcode.com/static/", "headline": "JoyVASA", "codeRepository": "https://github.com/jdh-algo/JoyVASA", "contentRating": "224" } ], "datePublished": "2024-11-14" } }</script> <meta name="theme-color" content="#fff"/> <link rel="manifest" href="https://production-assets.paperswithcode.com/static/manifest.web.json"> </head> <body> <nav class="navbar navbar-expand-lg navbar-light header"> <a class="navbar-brand" href="/"> <span class=" icon-wrapper" data-name="pwc"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><path d="M88 128h48v256H88zm144 0h48v256h-48zm-72 16h48v224h-48zm144 0h48v224h-48zm72-16h48v256h-48z"/><path d="M104 104V56H16v400h88v-48H64V104zm304-48v48h40v304h-40v48h88V56z"/></svg></span> </a> <div class="navbar-mobile-twitter d-lg-none"> <a rel="noreferrer" href="https://twitter.com/paperswithcode"> <span class=" icon-wrapper icon-fa icon-fa-brands" data-name="twitter"><svg viewBox="0 0 512.001 515.25" xmlns="http://www.w3.org/2000/svg"><path d="M459.37 152.016c.326 4.548.326 9.097.326 13.645 0 138.72-105.583 298.558-298.559 298.558C101.685 464.22 46.457 447 0 417.114c8.447.973 16.568 1.298 25.34 1.298 49.054 0 94.213-16.568 130.274-44.832-46.132-.975-84.792-31.188-98.113-72.772 6.499.975 12.996 1.624 19.819 1.624 9.42 0 18.843-1.3 27.613-3.573-48.08-9.747-84.142-51.98-84.142-102.984v-1.3c13.968 7.798 30.213 12.67 47.43 13.32-28.263-18.843-46.78-51.006-46.78-87.391 0-19.492 5.196-37.36 14.294-52.954 51.654 63.674 129.3 105.258 216.364 109.807-1.624-7.797-2.599-15.918-2.599-24.04 0-57.827 46.782-104.934 104.934-104.934 30.214 0 57.502 12.67 76.671 33.136 23.715-4.548 46.455-13.319 66.599-25.34-7.798 24.367-24.366 44.834-46.132 57.828 21.117-2.274 41.584-8.122 60.426-16.244-14.292 20.791-32.161 39.309-52.628 54.253z"/></svg></span> </a> </div> <button class="navbar-toggler" type="button" data-toggle="collapse" data-bs-toggle="collapse" data-target="#top-menu" data-bs-target="#top-menu" aria-controls="top-menu" aria-expanded="false" aria-label="Toggle navigation" > <span class="navbar-toggler-icon"></span> </button> <div class="collapse navbar-collapse" id="top-menu"> <ul class="navbar-nav mr-auto navbar-nav__left light-header"> <li class="nav-item header-search"> <form action="/search" method="get" id="id_global_search_form" autocomplete="off"> <input type="text" name="q_meta" style="display:none" id="q_meta" /> <input type="hidden" name="q_type" id="q_type" /> <input id="id_global_search_input" autocomplete="off" value="" name='q' class="global-search" type="search" placeholder='Search'/> <button type="submit" class="icon"><span class=" icon-wrapper icon-fa icon-fa-light" data-name="search"><svg viewBox="0 0 512.025 520.146" xmlns="http://www.w3.org/2000/svg"><path d="M508.5 482.6c4.7 4.7 4.7 12.3 0 17l-9.9 9.9c-4.7 4.7-12.3 4.7-17 0l-129-129c-2.2-2.3-3.5-5.3-3.5-8.5v-10.2C312 396 262.5 417 208 417 93.1 417 0 323.9 0 209S93.1 1 208 1s208 93.1 208 208c0 54.5-21 104-55.3 141.1H371c3.2 0 6.2 1.2 8.5 3.5zM208 385c97.3 0 176-78.7 176-176S305.3 33 208 33 32 111.7 32 209s78.7 176 176 176z"/></svg></span></button> </form> </li> <li class="nav-item"> <a class="nav-link" href="/sota"> Browse State-of-the-Art </a> </li> <li class="nav-item"> <a class="nav-link" href="/datasets"> Datasets </a> </li> <li class="nav-item"> <a class="nav-link" href="/methods">Methods</a> </li> <li class="nav-item dropdown"> <a class="nav-link dropdown-toggle" role="button" id="navbarDropdownRepro" data-toggle="dropdown" data-bs-toggle="dropdown" aria-haspopup="true" aria-expanded="false" > More </a> <div class="dropdown-menu" aria-labelledby="navbarDropdownRepro"> <a class="dropdown-item" href="/newsletter">Newsletter</a> <a class="dropdown-item" href="/rc2022">RC2022</a> <div class="dropdown-divider"></div> <a class="dropdown-item" href="/about">About</a> <a class="dropdown-item" href="/trends">Trends</a> <a class="dropdown-item" href="https://portal.paperswithcode.com/"> Portals </a> <a class="dropdown-item" href="/libraries"> Libraries </a> </div> </li> </ul> <ul class="navbar-nav ml-auto navbar-nav__right navbar-subscribe justify-content-center align-items-center"> <li class="nav-item"> <a class="nav-link" rel="noreferrer" href="https://twitter.com/paperswithcode"> <span class="nav-link-social-icon icon-wrapper icon-fa icon-fa-brands" data-name="twitter"><svg viewBox="0 0 512.001 515.25" xmlns="http://www.w3.org/2000/svg"><path d="M459.37 152.016c.326 4.548.326 9.097.326 13.645 0 138.72-105.583 298.558-298.559 298.558C101.685 464.22 46.457 447 0 417.114c8.447.973 16.568 1.298 25.34 1.298 49.054 0 94.213-16.568 130.274-44.832-46.132-.975-84.792-31.188-98.113-72.772 6.499.975 12.996 1.624 19.819 1.624 9.42 0 18.843-1.3 27.613-3.573-48.08-9.747-84.142-51.98-84.142-102.984v-1.3c13.968 7.798 30.213 12.67 47.43 13.32-28.263-18.843-46.78-51.006-46.78-87.391 0-19.492 5.196-37.36 14.294-52.954 51.654 63.674 129.3 105.258 216.364 109.807-1.624-7.797-2.599-15.918-2.599-24.04 0-57.827 46.782-104.934 104.934-104.934 30.214 0 57.502 12.67 76.671 33.136 23.715-4.548 46.455-13.319 66.599-25.34-7.798 24.367-24.366 44.834-46.132 57.828 21.117-2.274 41.584-8.122 60.426-16.244-14.292 20.791-32.161 39.309-52.628 54.253z"/></svg></span> </a> </li> <li class="nav-item"> <a id="signin-link" class="nav-link" href="/accounts/login?next=/paper/joyvasa-portrait-and-animal-image-animation">Sign In</a> </li> </ul> </div> </nav>  <div class="modal fade" id="emailModal" tabindex="-1" role="dialog" aria-labelledby="emailModalLabel" aria-hidden="true"> <div class="modal-dialog" role="document"> <div class="modal-content"> <div class="modal-header"> <h3 class="modal-title" id="emailModalLabel">Subscribe to the PwC Newsletter</h3> <button type="button" class="close" data-dismiss="modal" data-bs-dismiss="modal" aria-label="Close"> <span aria-hidden="true">×</span> </button> </div> <form action="" method="post"> <div class="modal-body"> <div class="modal-body-info-text"> Stay informed on the latest trending ML papers with code, research developments, libraries, methods, and datasets.<br/><br/> <a href="/newsletter">Read previous issues</a> </div> <input type="hidden" name="csrfmiddlewaretoken" value="Eo88EfbtFiRnMe4t80t1y4SHtGuQVdEsQULrrYIl7gfaullg59J8JGie74OOfmJT"> <input placeholder="Enter your email" type="email" class="form-control pwc-email" name="address" id="id_address" max_length="100" required> </div> <div class="modal-footer"> <button type="submit" class="btn btn-primary">Subscribe</button> </div> </form> </div> </div> </div>  <div class="modal fade" id="loginModal" tabindex="-1" role="dialog" aria-labelledby="loginModalLabel" aria-hidden="true"> <div class="modal-dialog" role="document"> <div class="modal-content"> <div class="modal-header"> <h5 class="modal-title" id="loginModalLabel">Join the community</h5> <button type="button" class="close btn-close" data-dismiss="modal" data-bs-dismiss="modal" aria-label="Close"> <span aria-hidden="true">×</span> </button> </div> <div class="login-modal-message"> You need to <a href="/accounts/login?next=/paper/joyvasa-portrait-and-animal-image-animation">log in</a> to edit.<br/> You can <a href="/accounts/register?next=/paper/joyvasa-portrait-and-animal-image-animation">create a new account</a> if you don't have one.<br/><br/> </div> </div> </div> </div>  <template id="modals-template"> <div class="modal fade" id="page-meta-modal"> <div class="modal-dialog"> <div class="modal-content"> <div class="modal-header"> <h5 class="modal-title">Edit Social Preview</h5> <button type="button" class="close btn-close" data-dismiss="modal" data-bs-dismiss="modal" aria-label="Close" > <span aria-hidden="true">×</span> </button> </div> <div id="page-meta-modal-body" class="modal-body"> <input type="hidden" name="csrfmiddlewaretoken" value="Rax6Q36PJzsaFvRJ1eVDHGp6yufr4doKO4h5D9CHX3BM2KIR1cDLhSK867A2D1LH"> <input type="hidden" id="page-meta-model-id-input" value="1566058" /> <input type="hidden" id="page-meta-model-name-input" value="Paper" /> <div class="form-group"> <label>Description</label><br /> <div class="form-check form-check-inline"> <input id="description-mode-default" class="form-check-input display-toggle-switch" checked type="radio" name="description-mode" value="default" data-target="display-description-default" /> <label class="form-check-label" for="description-mode-default">Default</label> </div> <div class="form-check form-check-inline"> <input id="description-mode-custom" class="form-check-input display-toggle-switch" type="radio" name="description-mode" value="custom" data-target="display-description-custom" /> <label class="form-check-label" for="description-mode-custom">Custom</label> </div> </div> <div class="form-group"> <div id="display-description-default" data-name="description-mode"> <textarea class="form-control" rows="3" readonly >Implemented in one code library.</textarea> </div> <div id="display-description-custom" data-name="description-mode"> <textarea class="form-control" id="description-input" rows="3" ></textarea> </div> </div> <div class="form-group"> <label>Image</label><br /> <div class="form-group"> <div class="form-check form-check-inline"> <input id="image-mode-default" class="form-check-input display-toggle-switch" checked type="radio" name="image-mode" value="default" data-target="display-image-default" /> <label class="form-check-label" for="image-mode-default">Default</label> </div> <div class="form-check form-check-inline"> <input id="image-mode-custom" class="form-check-input display-toggle-switch" type="radio" name="image-mode" value="custom" data-target="display-image-custom" /> <label class="form-check-label" for="image-mode-custom">Custom</label> </div> <div class="form-check form-check-inline"> <input id="image-mode-none" class="form-check-input display-toggle-switch" type="radio" name="image-mode" value="none" data-target="display-image-none" /> <label class="form-check-label" for="image-mode-none">None</label> </div> </div> </div> <div class="form-group"> <div id="display-image-default" data-name="image-mode"> <img class="page-meta-media" src="https://raw.githubusercontent.com/jdh-algo/JoyVASA/master/assets/imgs/pipeline_inference.png" /> </div> <div id="display-image-custom" data-name="image-mode"> <div id="file-too-large" style="display: none" class="alert alert-danger" role="alert"> File is too large </div> <p> Upload an image to customize your repository’s social media preview.<br /> Images should be at least 640×320px (1280×640px for best display). </P> <input type="file" class="form-control-file" id="image-input" /> </div> <div id="display-image-none" data-name="image-mode"> </div> </div> </div> <div class="modal-footer"> <button type="button" class="btn btn-secondary" data-dismiss="modal" data-bs-dismiss="modal"> Close </button> <button type="button" id="page-meta-submit" class="btn btn-primary"> Save </button> </div> </div> </div> </div>  <div class="modal fade" id="addCode" tabindex="-1" role="dialog" aria-labelledby="addCodeLabel" aria-hidden="true"> <div class="modal-dialog" role="document"> <div class="modal-content"> <div class="modal-header"> <h5 class="modal-title" id="addCodeLabel">Add a new code entry for this paper</h5> <button type="button" class="close btn-close" data-bs-dismiss="modal" aria-label="Close"> <span aria-hidden="true">×</span> </button> </div> <form action="" method="post"> <div class="modal-body"> <input type="hidden" name="csrfmiddlewaretoken" value="Rax6Q36PJzsaFvRJ1eVDHGp6yufr4doKO4h5D9CHX3BM2KIR1cDLhSK867A2D1LH"> <div id="div_id_url" class="form-group"> <label for="id_url" class=" requiredField"> GitHub, GitLab or BitBucket URL:<span class="asteriskField">*</span> </label> <div class=""> <input type="url" name="url" class="urlinput form-control" required id="id_url"> </div> </div> <div class="form-group"> <div id="div_id_is_official" class="form-check"> <input type="checkbox" name="is_official" class="checkboxinput form-check-input" id="id_is_official"> <label for="id_is_official" class="form-check-label"> Official code from paper authors </label> </div> </div> </div> <div class="modal-footer"> <button type="submit" class="btn btn-primary">Submit </button> </div> </form> </div> </div> </div>  <div class="modal fade" id="removeCode" tabindex="-1" role="dialog" aria-labelledby="removeCodeLabel" aria-hidden="true"> <div class="modal-dialog modal-lg" role="document"> <div class="modal-content"> <div class="modal-header"> <h5 class="modal-title" id="removeCodeLabel">Remove a code repository from this paper</h5> <button type="button" class="close btn-close" data-bs-dismiss="modal" aria-label="Close"> <span aria-hidden="true">×</span> </button> </div> <form action="" method="post"> <div class="modal-body"> <div class="paper-implementations"> <div class="row"> <div class="col-md-6"> <div class="paper-impl-cell"> <a href="https://github.com/jdh-algo/JoyVASA" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> jdh-algo/JoyVASA <span class="badge badge-info is-official-code"><span class=" icon-wrapper icon-ion" data-name="checkmark-circle-outline"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M448 256c0-106-86-192-192-192S64 150 64 256s86 192 192 192 192-86 192-192z" fill="none" stroke="#000" stroke-miterlimit="10" stroke-width="32"/><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M352 176L217.6 336 160 272"/></svg></span> official</span> </a> </div> </div> <div class="col-md-3"> <div class="paper-impl-cell"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 224 </div> </div> <div class="col-md-2"> <div class="paper-impl-cell"> <img class="" src="https://production-assets.paperswithcode.com/perf/images/frameworks/pytorch-2fbf2cb9.png" /> </div> </div> <div class="col-md-1"> <form action="" method="post"> <input type="hidden" name="csrfmiddlewaretoken" value="Rax6Q36PJzsaFvRJ1eVDHGp6yufr4doKO4h5D9CHX3BM2KIR1cDLhSK867A2D1LH"> <input type="hidden" name="remove_code_pk" value="1462003"> <button type="submit" class="btn btn-danger">- </button> </form> </div> </div> </div> </div> </form> </div> </div> </div>  <div class="modal fade" id="changeOfficialCode" tabindex="-1" role="dialog" aria-labelledby="changeOfficialCodeLabel" aria-hidden="true"> <div class="modal-dialog modal-lg" role="document"> <div class="modal-content"> <div class="modal-header"> <h5 class="modal-title" id="changeOfficialCodeLabel"> Mark the official implementation from paper authors </h5> <button type="button" class="close btn-close" data-bs-dismiss="modal" aria-label="Close"> <span aria-hidden="true">×</span> </button> </div> <form action="" method="post" id="official-pgr-form"> <input type="hidden" name="csrfmiddlewaretoken" value="Rax6Q36PJzsaFvRJ1eVDHGp6yufr4doKO4h5D9CHX3BM2KIR1cDLhSK867A2D1LH"> <input type="hidden" name="official_pgr_ids" id="official-pgr-ids" /> <div class="modal-body"> <div class="paper-implementations"> <div class="row align-items-center"> <div class="col-md-5"> <div class="paper-impl-cell"> <a href="https://github.com/jdh-algo/JoyVASA" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> jdh-algo/JoyVASA <span class="badge badge-info is-official-code"><span class=" icon-wrapper icon-ion" data-name="checkmark-circle-outline"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M448 256c0-106-86-192-192-192S64 150 64 256s86 192 192 192 192-86 192-192z" fill="none" stroke="#000" stroke-miterlimit="10" stroke-width="32"/><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M352 176L217.6 336 160 272"/></svg></span> official</span> </a> </div> </div> <div class="col-md-3"> <div class="paper-impl-cell"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 224 </div> </div> <div class="col-md-2"> <div class="paper-impl-cell"> <img class="" src="https://production-assets.paperswithcode.com/perf/images/frameworks/pytorch-2fbf2cb9.png" /> </div> </div> <div class="col-md-2 text-center"> <input type="radio" name="official-pgr-radio" class="official-pgr-input official-pgr-radio radios-version-element" value="2462751" checked > <input type="checkbox" class="official-pgr-input official-pgr-checkbox checkboxes-version-element" value="2462751" checked > </div> </div> <div class="radios-version-element"> <hr/> <div class="row align-items-center justify-content-between"> <div class="col-md-5"> <div class="paper-impl-cell"> There is no official implementation </div> </div> <div class="col-md-2 text-center"> <input type="radio" name="official-pgr-radio" value="" class="official-pgr-input official-pgr-radio" > </div> </div> </div> <hr/> <div class="row align-items-center justify-content-between"> <div class="col-md-5"> <div class="paper-impl-cell"> Multiple official implementations </div> </div> <div class="col-md-2 text-center"> <div class="custom-control custom-switch"> <input type="checkbox" class="custom-control-input" id="official-pgr-multiple-switch"> <label class="custom-control-label" for="official-pgr-multiple-switch" id="official-pgr-multiple-switch-label"></label> </div> </div> </div> </div> </div> <div class="modal-footer"> <button type="submit" class="btn btn-primary">Submit </button> </div> </form> </div> </div> </div>  <div class="modal fade" id="addRow" role="dialog" aria-labelledby="addRowLabel" aria-hidden="true"> <div class="modal-dialog" role="document"> <div class="modal-content"> <div class="modal-header"> <h5 class="modal-title" id="addRowLabel">Add a new evaluation result row</h5> <button type="button" class="close btn-close" data-bs-dismiss="modal" aria-label="Close"> <span aria-hidden="true">×</span> </button> </div> <form action="" method="post"> <div class="modal-body"> <input type="hidden" name="csrfmiddlewaretoken" value="Rax6Q36PJzsaFvRJ1eVDHGp6yufr4doKO4h5D9CHX3BM2KIR1cDLhSK867A2D1LH"> <div id="div_id_task" class="form-group"> <label for="id_task" class=" requiredField"> Task:<span class="asteriskField">*</span> </label> <div class=""> <select name="task" class="select form-control" required id="id_task"> <option value="" selected>---------</option> <option value="5901">Motion Generation</option> <option value="5710">Portrait Animation</option> <option value="707">Image Animation</option> </select> </div> </div> <div class="add-task-hint"> Not in the list? <a href="#addTask" data-bs-toggle="modal" data-bs-dismiss="modal">Add a task.</a> </div> <div id="div_id_dataset" class="form-group"> <label for="id_dataset" class=" requiredField"> Dataset:<span class="asteriskField">*</span> </label> <div class=""> <select name="dataset" class="modelselect2 form-control" required id="id_dataset" data-autocomplete-light-language="en" data-autocomplete-light-url="/dataset-autocomplete/" data-autocomplete-light-function="select2"> <option value="" selected>---------</option> </select> </div> </div> <div id="div_id_model_name" class="form-group"> <label for="id_model_name" class=" requiredField"> Model name:<span class="asteriskField">*</span> </label> <div class=""> <input type="text" name="model_name" class="textinput textInput form-control" required id="id_model_name"> </div> </div> <div id="div_id_metric" class="form-group"> <label for="id_metric" class=" requiredField"> Metric name:<span class="asteriskField">*</span> </label> <div class=""> <select name="metric" class="modelselect2 form-control" required id="id_metric" data-autocomplete-light-language="en" data-autocomplete-light-url="/metric-autocomplete/" data-autocomplete-light-function="select2"> <option value="" selected>---------</option> </select> </div> </div> <div id="sota-metric-names"> </div> <div class="form-group"> <div id="div_id_metric_higher_is_better" class="form-check"> <input type="checkbox" name="metric_higher_is_better" class="checkboxinput form-check-input" id="id_metric_higher_is_better"> <label for="id_metric_higher_is_better" class="form-check-label"> Higher is better (for the metric) </label> </div> </div> <div id="div_id_metric_value" class="form-group"> <label for="id_metric_value" class=" requiredField"> Metric value:<span class="asteriskField">*</span> </label> <div class=""> <input type="text" name="metric_value" class="textinput textInput form-control" required id="id_metric_value"> </div> </div> <div id="sota-metric-values"> </div> <div class="form-group"> <div id="div_id_uses_additional_data" class="form-check"> <input type="checkbox" name="uses_additional_data" class="checkboxinput form-check-input" id="id_uses_additional_data"> <label for="id_uses_additional_data" class="form-check-label"> Uses extra training data </label> </div> </div> <div id="div_id_evaluated_on" class="form-group"> <label for="id_evaluated_on" class=""> Data evaluated on </label> <div class=""> <input type="text" name="evaluated_on" value="2024-11-14" autocomplete="off" class="dateinput form-control" id="id_evaluated_on"> </div> </div> </div> <div class="modal-footer"> <button type="submit" class="btn btn-primary"> Submit </button> </div> </form> </div> </div> </div>  <div class="modal fade" id="removeRow" role="dialog" aria-labelledby="removeRowLabel" aria-hidden="true"> <div class="modal-dialog modal-lg" role="document"> <div class="modal-content"> <div class="modal-header"> <h5 class="modal-title" id="removeRowLabel">Add a new evaluation result row</h5> <button type="button" class="close btn-close" data-bs-dismiss="modal" aria-label="Close"> <span aria-hidden="true">×</span> </button> </div> <form action="" method="post"> <div class="modal-body"> <div class="sota-table"> <table class="table-striped"> <tr> <th>TASK</th> <th>DATASET</th> <th>MODEL</th> <th>METRIC NAME</th> <th>METRIC VALUE</th> <th>GLOBAL RANK</th> <th>REMOVE</th> </tr> </table> </div> </div> </form> </div> </div> </div>  <div class="modal fade" id="addTask" role="dialog" aria-labelledby="addTaskLabel" aria-hidden="true"> <div class="modal-dialog" role="document"> <div class="modal-content"> <div class="modal-header"> <h5 class="modal-title" id="addTaskLabel">Add a task</h5> <button type="button" class="close btn-close" data-bs-dismiss="modal" aria-label="Close"> <span aria-hidden="true">×</span> </button> </div> <div class="modal-body"> <div class="current-tasks-title">Attached tasks:</div> <ul class="list-unstyled"> <li> <a href="/task/image-animation"> <span class="badge badge-primary">IMAGE ANIMATION</span> </a> </li> </ul> <ul class="list-unstyled"> <li> <a href="/task/motion-generation"> <span class="badge badge-primary">MOTION GENERATION</span> </a> </li> </ul> <ul class="list-unstyled"> <li> <a href="/task/portrait-animation"> <span class="badge badge-primary">PORTRAIT ANIMATION</span> </a> </li> </ul> <form action="" method="post"> <input type="hidden" name="csrfmiddlewaretoken" value="Rax6Q36PJzsaFvRJ1eVDHGp6yufr4doKO4h5D9CHX3BM2KIR1cDLhSK867A2D1LH"> <div id="div_id_task" class="form-group"> <label for="id_task" class=""> Add: </label> <div class=""> <select name="task" class="modelselect2 form-control" id="id_task" data-autocomplete-light-language="en" data-autocomplete-light-url="/task-autocomplete/" data-autocomplete-light-function="select2"> <option value="" selected>---------</option> </select> </div> </div> <div class="modal-help-text"> Not in the list?<br/> <a href="#" id="new-task-form-toggle"> <span class=" icon-wrapper icon-ion" data-name="add"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M256 112v288m144-144H112"/></svg></span> Create a new task</a>. </div> <div id="new-task-form" style="display: none"> <div id="div_id_new_task_name" class="form-group"> <label for="id_new_task_name" class=""> New task name: </label> <div class=""> <input type="text" name="new_task_name" maxlength="200" class="textinput textInput form-control" id="id_new_task_name"> </div> </div> <div id="div_id_new_task_area" class="form-group"> <label for="id_new_task_area" class=""> Top-level area: </label> <div class=""> <select name="new_task_area" class="select form-control" id="id_new_task_area"> <option value="" selected>---------</option> <option value="17">Adversarial</option> <option value="18">Audio</option> <option value="11">Computer Code</option> <option value="3">Computer Vision</option> <option value="9">Graphs</option> <option value="15">Knowledge Base</option> <option value="7">Medical</option> <option value="6">Methodology</option> <option value="5">Miscellaneous</option> <option value="12">Music</option> <option value="4">Natural Language Processing</option> <option value="13">Playing Games</option> <option value="14">Reasoning</option> <option value="16">Robots</option> <option value="10">Speech</option> <option value="8">Time Series</option> </select> </div> </div> <div id="div_id_new_task_parent" class="form-group"> <label for="id_new_task_parent" class=""> Parent task (if any): </label> <div class=""> <select name="new_task_parent" class="modelselect2 form-control" id="id_new_task_parent" data-autocomplete-light-language="en" data-autocomplete-light-url="/task-and-tag-autocomplete/" data-autocomplete-light-function="select2"> <option value="" selected>---------</option> </select> </div> </div> <div id="div_id_new_task_desc" class="form-group"> <label for="id_new_task_desc" class=""> Description: </label> <div class=""> <textarea name="new_task_desc" cols="40" rows="3" class="textarea form-control" id="id_new_task_desc"> </textarea> </div> </div> </div> <div class="modal-footer"> <button type="submit" class="btn btn-primary"> Submit </button> </div> </form> </div> </div> </div> </div>  <div class="modal fade" id="removeTask" tabindex="-1" role="dialog" aria-labelledby="removeTaskLabel" aria-hidden="true"> <div class="modal-dialog" role="document"> <div class="modal-content"> <div class="modal-header"> <h5 class="modal-title" id="removeTaskLabel">Remove a task</h5> <button type="button" class="close btn-close" data-bs-dismiss="modal" aria-label="Close"> <span aria-hidden="true">×</span> </button> </div> <form action="" method="post"> <div class="modal-body"> <ul class="list-unstyled paper-tasks"> <form action="" method="post"> <li> <a href="/task/image-animation"> <span class="badge badge-primary"> <img src="https://production-media.paperswithcode.com/tasks/default.gif"> <span>Image Animation</span> </span> </a> <input type="hidden" name="csrfmiddlewaretoken" value="Rax6Q36PJzsaFvRJ1eVDHGp6yufr4doKO4h5D9CHX3BM2KIR1cDLhSK867A2D1LH"> <input type="hidden" name="remove_task_pk" value="707"> <button type="submit" class="btn btn-danger" style="width:2.5em">- </button> </li> </form> </ul> <ul class="list-unstyled paper-tasks"> <form action="" method="post"> <li> <a href="/task/motion-generation"> <span class="badge badge-primary"> <img src="https://production-media.paperswithcode.com/tasks/default.gif"> <span>Motion Generation</span> </span> </a> <input type="hidden" name="csrfmiddlewaretoken" value="Rax6Q36PJzsaFvRJ1eVDHGp6yufr4doKO4h5D9CHX3BM2KIR1cDLhSK867A2D1LH"> <input type="hidden" name="remove_task_pk" value="5901"> <button type="submit" class="btn btn-danger" style="width:2.5em">- </button> </li> </form> </ul> <ul class="list-unstyled paper-tasks"> <form action="" method="post"> <li> <a href="/task/portrait-animation"> <span class="badge badge-primary"> <img src="https://production-media.paperswithcode.com/tasks/default.gif"> <span>Portrait Animation</span> </span> </a> <input type="hidden" name="csrfmiddlewaretoken" value="Rax6Q36PJzsaFvRJ1eVDHGp6yufr4doKO4h5D9CHX3BM2KIR1cDLhSK867A2D1LH"> <input type="hidden" name="remove_task_pk" value="5710"> <button type="submit" class="btn btn-danger" style="width:2.5em">- </button> </li> </form> </ul> </div> </form> </div> </div> </div>  <div class="modal fade" id="addMethod" role="dialog" aria-labelledby="addMethodLabel" aria-hidden="true"> <div class="modal-dialog" role="document"> <div class="modal-content"> <div class="modal-header"> <h5 class="modal-title" id="addMethodLabel">Add a method</h5> <button type="button" class="close btn-close" data-bs-dismiss="modal" aria-label="Close"> <span aria-hidden="true">×</span> </button> </div> <div class="modal-body"> <div class="current-methods-title">Attached methods:</div> <ul class="list-unstyled"> <li> <a href="/method/diffusion"> <span class="badge badge-primary">DIFFUSION</span> </a> </li> </ul> <ul class="list-unstyled"> <li> <a href="/method/focus"> <span class="badge badge-primary">FOCUS</span> </a> </li> </ul> <form action="" method="post"> <input type="hidden" name="csrfmiddlewaretoken" value="Rax6Q36PJzsaFvRJ1eVDHGp6yufr4doKO4h5D9CHX3BM2KIR1cDLhSK867A2D1LH"> <div id="div_id_method" class="form-group"> <label for="id_method" class=""> Add: </label> <div class=""> <select name="method" class="modelselect2 form-control" id="id_method" data-autocomplete-light-language="en" data-autocomplete-light-url="/method-autocomplete/" data-autocomplete-light-function="select2"> <option value="" selected>---------</option> </select> </div> </div> <div class="modal-help-text"> Not in the list?<br/> <a href="#" id="new-method-form-toggle"> <span class=" icon-wrapper icon-ion" data-name="add"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M256 112v288m144-144H112"/></svg></span> Create a new method</a>. </div> <div id="new-method-form" style="display: none"> <div id="div_id_new_method_name" class="form-group"> <label for="id_new_method_name" class=""> <b>New method name</b> (e.g. ReLU): </label> <div class=""> <input type="text" name="new_method_name" maxlength="200" minlength="2" class="textinput textInput form-control" id="id_new_method_name"> </div> </div> <div id="div_id_new_method_full_name" class="form-group"> <label for="id_new_method_full_name" class=""> <b>New method full name</b> (e.g. Rectified Linear Unit): </label> <div class=""> <input type="text" name="new_method_full_name" maxlength="200" minlength="2" class="textinput textInput form-control" id="id_new_method_full_name"> </div> </div> <div id="div_id_new_method_paper" class="form-group"> <label for="id_new_method_paper" class=""> <b>Paper where method was first introduced</b>: </label> <div class=""> <select name="new_method_paper" class="modelselect2 form-control" id="id_new_method_paper" data-autocomplete-light-language="en" data-autocomplete-light-url="/paper-autocomplete/" data-autocomplete-light-function="select2"> <option value="" selected>---------</option> </select> </div> </div> <div id="div_id_new_method_collection" class="form-group"> <label for="id_new_method_collection" class=""> <b>Method category</b> (e.g. Activation Functions): <i>If no match, add something for now then you can add a new category afterwards.</i> </label> <div class=""> <select name="new_method_collection" class="modelselect2 form-control" id="id_new_method_collection" data-autocomplete-light-language="en" data-autocomplete-light-url="/method-collection-autocomplete/" data-autocomplete-light-function="select2"> <option value="" selected>---------</option> </select> </div> </div> <div id="div_id_new_method_desc" class="form-group"> <label for="id_new_method_desc" class=""> <b>Markdown description</b> (optional; $\LaTeX$ enabled): <i>You can edit this later, so feel free to start with something succinct.</i> </label> <div class=""> <textarea name="new_method_desc" cols="40" rows="10" class="textarea form-control" id="id_new_method_desc"> </textarea> </div> </div> </div> <div class="modal-footer"> <button type="submit" class="btn btn-primary"> Submit </button> </div> </form> </div> </div> </div> </div>  <div class="modal fade" id="removeMethod" tabindex="-1" role="dialog" aria-labelledby="removeMethodLabel" aria-hidden="true"> <div class="modal-dialog" role="document"> <div class="modal-content"> <div class="modal-header"> <h5 class="modal-title" id="removeMethodLabel">Remove a method</h5> <button type="button" class="close btn-close" data-bs-dismiss="modal" aria-label="Close"> <span aria-hidden="true">×</span> </button> </div> <form action="" method="post"> <div class="modal-body"> <ul class="list-unstyled"> <form action="" method="post"> <li> <a href="/method/diffusion"> <span class="badge badge-primary">DIFFUSION</span> </a> <input type="hidden" name="csrfmiddlewaretoken" value="Rax6Q36PJzsaFvRJ1eVDHGp6yufr4doKO4h5D9CHX3BM2KIR1cDLhSK867A2D1LH"> <input type="hidden" name="remove_method_pk" value="2008"> <button type="submit" class="btn btn-danger" style="width:2.5em">- </button> </li> </form> </ul> <ul class="list-unstyled"> <form action="" method="post"> <li> <a href="/method/focus"> <span class="badge badge-primary">FOCUS</span> </a> <input type="hidden" name="csrfmiddlewaretoken" value="Rax6Q36PJzsaFvRJ1eVDHGp6yufr4doKO4h5D9CHX3BM2KIR1cDLhSK867A2D1LH"> <input type="hidden" name="remove_method_pk" value="2168"> <button type="submit" class="btn btn-danger" style="width:2.5em">- </button> </li> </form> </ul> </div> </form> </div> </div> </div>  <div class="modal fade" id="badgeModal" tabindex="-1" role="dialog" aria-labelledby="badgeModalLabel" aria-hidden="true" > <div class="modal-dialog modal-lg" role="document"> <div class="modal-content modal-badge"> </div> </div> </div>  <div class="modal fade" id="editDatasets" role="dialog" aria-labelledby="editDatasetsLabel" aria-hidden="true"> <div class="modal-dialog" role="document"> <div class="modal-content"> <div class="modal-header"> <h5 class="modal-title" id="editDatasetsLabel">Edit Datasets</h5> <button type="button" class="close btn-close" data-bs-dismiss="modal" aria-label="Close"> <span aria-hidden="true">×</span> </button> </div> <form action="" method="post"> <div class="modal-body paper-page-edit-dataset-modal"> <input type="hidden" name="csrfmiddlewaretoken" value="Rax6Q36PJzsaFvRJ1eVDHGp6yufr4doKO4h5D9CHX3BM2KIR1cDLhSK867A2D1LH"> <div id="div_id_introduced" class="form-group"> <label for="id_introduced" class=""> Add or remove datasets <b>introduced</b> in this paper: </label> <div class=""> <select name="introduced" data-container-css-class="" data-allow-clear="false" style="width: 100%" class="modelselect2multiple form-control" id="id_introduced" data-autocomplete-light-language="en" data-autocomplete-light-url="/dataset-autocomplete/" data-autocomplete-light-function="select2" multiple> </select><div style="display:none" class="dal-forward-conf" id="dal-forward-conf-for_id_introduced"><script type="text/dal-forward-conf">[{"type": "const", "val": true, "dst": "canonical_only"}, {"type": "const", "val": true, "dst": "disable_create_option"}]</script></div> </div> </div> <div id="div_id_used" class="form-group"> <label for="id_used" class=""> Add or remove other datasets <b>used</b> in this paper: </label> <div class=""> <select name="used" data-container-css-class="" data-allow-clear="false" style="width: 100%" class="modelselect2multiple form-control" id="id_used" data-autocomplete-light-language="en" data-autocomplete-light-url="/dataset-autocomplete/" data-autocomplete-light-function="select2" multiple> <option value="12070" selected>CelebV-HQ</option> </select><div style="display:none" class="dal-forward-conf" id="dal-forward-conf-for_id_used"><script type="text/dal-forward-conf">[{"type": "const", "val": true, "dst": "canonical_only"}, {"type": "const", "val": true, "dst": "disable_create_option"}]</script></div> </div> </div> <div style="display: inline-block; padding-bottom: 15px;font-size:14px;"> Paper introduces a new dataset? <div style="padding-top:4px"> <a href="/contribute/dataset/new"> <span class=" icon-wrapper icon-fa icon-fa-solid" data-name="plus-circle"><svg viewBox="0 0 512 514.999" xmlns="http://www.w3.org/2000/svg"><path d="M256 9.998c137 0 248 111 248 248s-111 248-248 248-248-111-248-248 111-248 248-248zm144 276v-56c0-6.6-5.4-12-12-12h-92v-92c0-6.6-5.4-12-12-12h-56c-6.6 0-12 5.4-12 12v92h-92c-6.6 0-12 5.4-12 12v56c0 6.6 5.4 12 12 12h92v92c0 6.6 5.4 12 12 12h56c6.6 0 12-5.4 12-12v-92h92c6.6 0 12-5.4 12-12z"/></svg></span> Add a new dataset here </a> </div> </div> </div> <div class="modal-footer"> <button type="submit" class="btn btn-primary" name="edit-datasets"> Save </button> </div> </form> </div> </div> </div> </template> <div class="container content content-buffer "> <main> <div class="paper-title"> <div class="row"> <div class="col-md-12"> <h1> JoyVASA: Portrait and Animal Image Animation with Diffusion-Based Audio-Driven Facial Dynamics and Head Motion Generation </h1> <div class="authors"> <p> <span class="author-span">14 Nov 2024</span>  ·  <span class="author-span"> <a href="/author/xuyang-cao">Xuyang Cao</a></span>, <span class="author-span"> <a href="/author/guoxin-wang">Guoxin Wang</a></span>, <span class="author-span"> <a href="/author/sheng-shi">Sheng Shi</a></span>, <span class="author-span"> <a href="/author/jun-zhao">Jun Zhao</a></span>, <span class="author-span"> <a href="/author/yang-yao">Yang Yao</a></span>, <span class="author-span"> <a href="/author/jintao-fei">Jintao Fei</a></span>, <span class="author-span"> <a href="/author/minyu-gao">Minyu Gao</a></span> <span class="hidden-element">· </span><button type="button" class="badge-edit" data-bs-toggle="modal" data-bs-toggle="modal" data-bs-target="#loginModal"> <span class=" icon-wrapper icon-fa icon-fa-solid" data-name="edit"><svg viewBox="0 0 576 514.999" xmlns="http://www.w3.org/2000/svg"><path d="M402.6 85.198l90.2 90.2c3.8 3.8 3.8 10 0 13.8l-218.399 218.4-92.8 10.3c-12.4 1.4-22.9-9.1-21.5-21.5l10.3-92.8 218.4-218.4c3.799-3.8 10-3.8 13.799 0zm162-22.9c15.2 15.2 15.2 39.9 0 55.2l-35.4 35.4c-3.8 3.8-10 3.8-13.8 0l-90.2-90.2c-3.8-3.8-3.8-10 0-13.8l35.4-35.4c15.3-15.2 40-15.2 55.2 0zM384 348.198c0-3.2 1.3-6.2 3.5-8.5l40-40c7.6-7.5 20.5-2.2 20.5 8.5v157.8c0 26.5-21.5 48-48 48H48c-26.5 0-48-21.5-48-48v-352c0-26.5 21.5-48 48-48h285.8c10.7 0 16.1 12.9 8.5 20.5l-40 40c-2.3 2.2-5.3 3.5-8.5 3.5H64v320h320v-101.8z"/></svg></span> <span>Edit social preview</span> </button> </p> </div> </div> </div> </div> <div class="paper-abstract"> <div class="row"> <div class="col-md-12"> <p> Audio-driven portrait animation has made significant advances with diffusion-based models, improving video quality and lipsync accuracy. However, the increasing complexity of these models has led to inefficiencies in training and inference, as well as constraints on video length and inter-frame continuity. In this paper, we propose JoyVASA, a diffusion-based method for generating facial dynamics and head motion in audio-driven facial animation. Specifically, in the first stage, we introduce a decoupled facial representation framework that separates dynamic facial expressions from static 3D facial representations. This decoupling allows the system to generate longer videos by combining any static 3D facial representation with dynamic motion sequences. Then, in the second stage, a diffusion transformer is trained to generate motion sequences directly from audio cues, independent of character identity. Finally, a generator trained in the first stage uses the 3D facial representation and the generated motion sequences as inputs to render high-quality animations. With the decoupled facial representation and the identity-independent motion generation process, JoyVASA extends beyond human portraits to animate animal faces seamlessly. The model is trained on a hybrid dataset of private Chinese and public English data, enabling multilingual support. Experimental results validate the effectiveness of our approach. Future work will focus on improving real-time performance and refining expression control, further expanding the applications in portrait animation. The code is available at: https://github.com/jdh-algo/JoyVASA. </p> <a href="https://arxiv.org/pdf/2411.09209v3.pdf" onclick="captureOutboundLink('https://arxiv.org/pdf/2411.09209v3.pdf'); return true;" class="badge badge-light "> <span class=" icon-wrapper icon-fa icon-fa-regular" data-name="file-pdf"><svg viewBox="0 0 384 513.795" xmlns="http://www.w3.org/2000/svg"><path d="M369.9 98.88c9 9 14.1 21.3 14.1 34v332.1c0 26.5-21.5 48-48 48H48c-26.5 0-48-21.5-48-48v-416c0-26.5 21.5-48 48-48.1h204.1c12.7 0 24.9 5.1 33.9 14.1zm-37.8 30.1L256 52.88v76.1h76.1zM48 464.98h288v-288H232c-13.3 0-24-10.7-24-24v-104H48v416zm250.2-143.7c10.5 10.5 8 38.7-17.5 38.7-14.8 0-36.9-6.8-55.8-17-21.6 3.6-46 12.7-68.4 20.1-50.1 86.4-79.4 47-76.1 31.2 4-20 31-35.9 51-46.2 10.5-18.4 25.4-50.5 35.4-74.4-7.4-28.6-11.4-51-7-67.1 4.8-17.7 38.4-20.3 42.6 5.9 4.7 15.4-1.5 39.9-5.4 56 8.1 21.3 19.6 35.8 36.8 46.3 17.4-2.2 52.2-5.5 64.4 6.5zm-198.1 77.8c0 .7 11.4-4.7 30.4-35-5.9 5.5-25.299 21.3-30.4 35zm81.6-190.6c-2.5 0-2.6 26.9 1.8 40.8 4.9-8.7 5.6-40.8-1.8-40.8zm-24.4 136.6c15.9-6.1 34-14.9 54.8-19.2-11.199-8.3-21.8-20.4-30.1-35.5-6.7 17.7-15 37.8-24.7 54.7zm131.6-5c3.6-2.4-2.2-10.4-37.3-7.8 32.3 13.8 37.3 7.8 37.3 7.8z"/></svg></span> <span>PDF</span> </a> <a href="https://arxiv.org/abs/2411.09209v3" onclick="captureOutboundLink('https://arxiv.org/abs/2411.09209v3'); return true;" class="badge badge-light "> <span class=" icon-wrapper icon-fa icon-fa-regular" data-name="file"><svg viewBox="0 0 384 513.795" xmlns="http://www.w3.org/2000/svg"><path d="M369.9 98.88c9 9 14.1 21.3 14.1 34v332.1c0 26.5-21.5 48-48 48H48c-26.5 0-48-21.5-48-48v-416c0-26.5 21.5-48 48-48.1h204.1c12.7 0 24.9 5.1 33.9 14.1zm-37.8 30.1L256 52.88v76.1h76.1zM48 464.98h288v-288H232c-13.3 0-24-10.7-24-24v-104H48v416z"/></svg></span> <span>Abstract</span> </a> </div> </div> </div> <div class="row"> <div class="col-md-7 paper-section-first" id="code"> <div class="paper-section-title"> <div class="row"> <div class="col-md-12"> <h2>Code <div class="float-right"> <div class="dropdown edit-button"> <button class="dropdown-toggle badge badge-edit" type="button" id="codeEditMenu" data-bs-toggle="dropdown" aria-haspopup="true" aria-expanded="false"> <span class=" icon-wrapper icon-fa icon-fa-solid" data-name="edit"><svg viewBox="0 0 576 514.999" xmlns="http://www.w3.org/2000/svg"><path d="M402.6 85.198l90.2 90.2c3.8 3.8 3.8 10 0 13.8l-218.399 218.4-92.8 10.3c-12.4 1.4-22.9-9.1-21.5-21.5l10.3-92.8 218.4-218.4c3.799-3.8 10-3.8 13.799 0zm162-22.9c15.2 15.2 15.2 39.9 0 55.2l-35.4 35.4c-3.8 3.8-10 3.8-13.8 0l-90.2-90.2c-3.8-3.8-3.8-10 0-13.8l35.4-35.4c15.3-15.2 40-15.2 55.2 0zM384 348.198c0-3.2 1.3-6.2 3.5-8.5l40-40c7.6-7.5 20.5-2.2 20.5 8.5v157.8c0 26.5-21.5 48-48 48H48c-26.5 0-48-21.5-48-48v-352c0-26.5 21.5-48 48-48h285.8c10.7 0 16.1 12.9 8.5 20.5l-40 40c-2.3 2.2-5.3 3.5-8.5 3.5H64v320h320v-101.8z"/></svg></span> Edit </button> <div class="dropdown-menu dropdown-menu-end" aria-labelledby="codeEditMenu"> <a class="dropdown-item" href="#loginModal" data-bs-toggle="modal"> <span class=" icon-wrapper icon-ion" data-name="add"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M256 112v288m144-144H112"/></svg></span> Add</a> <a class="dropdown-item" href="#loginModal" data-bs-toggle="modal"> <span class=" icon-wrapper icon-ion" data-name="remove"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M400 256H112"/></svg></span> Remove</a> <a class="dropdown-item" href="#loginModal" data-bs-toggle="modal"> <span class=" icon-wrapper icon-ion" data-name="checkmark-outline"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M416 128L192 384l-96-96"/></svg></span> Mark official</a> </div> </div> </div> </h2> <hr/> </div> </div> </div> <div class="paper-implementations code-table"> <div id="implementations-short-list"> <div class="row"> <div class="col-sm-7"> <div class="paper-impl-cell"> <a href="https://github.com/jdh-algo/JoyVASA" onclick="captureOutboundLink('https://github.com/jdh-algo/JoyVASA'); return true;" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> jdh-algo/JoyVASA <span class="badge badge-info is-official-code"><span class=" icon-wrapper icon-ion" data-name="checkmark-circle-outline"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M448 256c0-106-86-192-192-192S64 150 64 256s86 192 192 192 192-86 192-192z" fill="none" stroke="#000" stroke-miterlimit="10" stroke-width="32"/><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M352 176L217.6 336 160 272"/></svg></span> official</span> </a> </div> </div> <div class="col-3"> <div class="paper-impl-cell text-nowrap"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 224 </div> </div> <div class="col-2"> <div class="paper-impl-cell text-center"> <img class="" src="https://production-assets.paperswithcode.com/perf/images/frameworks/pytorch-2fbf2cb9.png" /> </div> </div> </div> </div> <div id="implementations-full-list" style="display:none"> <div class="row"> <div class="col-sm-7"> <div class="paper-impl-cell"> <a href="https://github.com/jdh-algo/JoyVASA" onclick="captureOutboundLink('https://github.com/jdh-algo/JoyVASA'); return true;" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> jdh-algo/JoyVASA <span class="badge badge-info is-official-code"><span class=" icon-wrapper icon-ion" data-name="checkmark-circle-outline"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M448 256c0-106-86-192-192-192S64 150 64 256s86 192 192 192 192-86 192-192z" fill="none" stroke="#000" stroke-miterlimit="10" stroke-width="32"/><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M352 176L217.6 336 160 272"/></svg></span> official</span> </a> </div> </div> <div class="col-3"> <div class="paper-impl-cell text-nowrap"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 224 </div> </div> <div class="col-2"> <div class="paper-impl-cell text-center"> <img class="" src="https://production-assets.paperswithcode.com/perf/images/frameworks/pytorch-2fbf2cb9.png" /> </div> </div> </div> </div> </div> </div> <div class="col-md-5 paper-section" id="tasks"> <div class="paper-section-title"> <div class="row"> <div class="col-md-12"> <h2>Tasks <div class="float-right"> <div class="dropdown edit-button"> <button class="dropdown-toggle badge badge-edit" type="button" id="taskEditMenu" data-bs-toggle="dropdown" aria-haspopup="true" aria-expanded="false"> <span class=" icon-wrapper icon-fa icon-fa-solid" data-name="edit"><svg viewBox="0 0 576 514.999" xmlns="http://www.w3.org/2000/svg"><path d="M402.6 85.198l90.2 90.2c3.8 3.8 3.8 10 0 13.8l-218.399 218.4-92.8 10.3c-12.4 1.4-22.9-9.1-21.5-21.5l10.3-92.8 218.4-218.4c3.799-3.8 10-3.8 13.799 0zm162-22.9c15.2 15.2 15.2 39.9 0 55.2l-35.4 35.4c-3.8 3.8-10 3.8-13.8 0l-90.2-90.2c-3.8-3.8-3.8-10 0-13.8l35.4-35.4c15.3-15.2 40-15.2 55.2 0zM384 348.198c0-3.2 1.3-6.2 3.5-8.5l40-40c7.6-7.5 20.5-2.2 20.5 8.5v157.8c0 26.5-21.5 48-48 48H48c-26.5 0-48-21.5-48-48v-352c0-26.5 21.5-48 48-48h285.8c10.7 0 16.1 12.9 8.5 20.5l-40 40c-2.3 2.2-5.3 3.5-8.5 3.5H64v320h320v-101.8z"/></svg></span> Edit </button> <div class="dropdown-menu dropdown-menu-end" aria-labelledby="taskEditMenu"> <a class="dropdown-item" href="#loginModal" data-bs-toggle="modal"> <span class=" icon-wrapper icon-ion" data-name="add"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M256 112v288m144-144H112"/></svg></span> Add</a> <a class="dropdown-item" href="#loginModal" data-bs-toggle="modal"> <span class=" icon-wrapper icon-ion" data-name="remove"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M400 256H112"/></svg></span> Remove</a> </div> </div> </div> </h2> <hr/> </div> </div> <div class="paper-tasks"> <div class="row"> <div class="col-md-12"> <a href="/task/image-animation"> <span class="badge badge-primary"> <img src="https://production-media.paperswithcode.com/tasks/default.gif"> <span>Image Animation</span> </span> </a> <a href="/task/motion-generation"> <span class="badge badge-primary"> <img src="https://production-media.paperswithcode.com/tasks/default.gif"> <span>Motion Generation</span> </span> </a> <a href="/task/portrait-animation"> <span class="badge badge-primary"> <img src="https://production-media.paperswithcode.com/tasks/default.gif"> <span>Portrait Animation</span> </span> </a> </div> </div> </div> </div> </div> </div> <div class="row"> <div class="col-md-12 paper-section paper-evaluation-section-title" id="datasets"> <div class="paper-section-title"> <div class="row"> <div class="col-md-12 zero-padding-datasets"> <h2>Datasets <div class="float-right"> <div class="dropdown edit-button"> <button class="dropdown-toggle badge badge-edit" type="button" id="datasetEditMenu" data-bs-toggle="modal" data-bs-target="#loginModal" aria-haspopup="true" aria-expanded="false"> <span class=" icon-wrapper icon-fa icon-fa-solid" data-name="edit"><svg viewBox="0 0 576 514.999" xmlns="http://www.w3.org/2000/svg"><path d="M402.6 85.198l90.2 90.2c3.8 3.8 3.8 10 0 13.8l-218.399 218.4-92.8 10.3c-12.4 1.4-22.9-9.1-21.5-21.5l10.3-92.8 218.4-218.4c3.799-3.8 10-3.8 13.799 0zm162-22.9c15.2 15.2 15.2 39.9 0 55.2l-35.4 35.4c-3.8 3.8-10 3.8-13.8 0l-90.2-90.2c-3.8-3.8-3.8-10 0-13.8l35.4-35.4c15.3-15.2 40-15.2 55.2 0zM384 348.198c0-3.2 1.3-6.2 3.5-8.5l40-40c7.6-7.5 20.5-2.2 20.5 8.5v157.8c0 26.5-21.5 48-48 48H48c-26.5 0-48-21.5-48-48v-352c0-26.5 21.5-48 48-48h285.8c10.7 0 16.1 12.9 8.5 20.5l-40 40c-2.3 2.2-5.3 3.5-8.5 3.5H64v320h320v-101.8z"/></svg></span> Edit </button> </div> </div> </h2> <hr/> </div> </div> <div class="paper-datasets"> <div class="row"> <div class="col-md-12"> <span class="badge badge-primary"> <a href="/dataset/celebv-hq"> <img class="dataset-list-image" src="https://production-media.paperswithcode.com/thumbnails/dataset-small/e4118080-f260-42f1-9e21-49af66d881c6.jpg"> CelebV-HQ </a> </span> </div> </div> </div> </div> </div> </div>  <div class="row"> <div id="results" class="col-md-12 paper-evaluation-section-title"> <div class="paper-section-title"> <div class="row"> <div class="col-md-12 zero-padding"> <h2>Results from the Paper <div class="float-right"> <div class="edit-button"> <a class="dropdown-toggle badge badge-edit" id="evalEditMenu" href="/paper/joyvasa-portrait-and-animal-image-animation/review/"> <span class=" icon-wrapper icon-fa icon-fa-solid" data-name="edit"><svg viewBox="0 0 576 514.999" xmlns="http://www.w3.org/2000/svg"><path d="M402.6 85.198l90.2 90.2c3.8 3.8 3.8 10 0 13.8l-218.399 218.4-92.8 10.3c-12.4 1.4-22.9-9.1-21.5-21.5l10.3-92.8 218.4-218.4c3.799-3.8 10-3.8 13.799 0zm162-22.9c15.2 15.2 15.2 39.9 0 55.2l-35.4 35.4c-3.8 3.8-10 3.8-13.8 0l-90.2-90.2c-3.8-3.8-3.8-10 0-13.8l35.4-35.4c15.3-15.2 40-15.2 55.2 0zM384 348.198c0-3.2 1.3-6.2 3.5-8.5l40-40c7.6-7.5 20.5-2.2 20.5 8.5v157.8c0 26.5-21.5 48-48 48H48c-26.5 0-48-21.5-48-48v-352c0-26.5 21.5-48 48-48h285.8c10.7 0 16.1 12.9 8.5 20.5l-40 40c-2.3 2.2-5.3 3.5-8.5 3.5H64v320h320v-101.8z"/></svg></span> Edit </a> </div> </div> </h2> <hr/> <div class="paper-evaluation-badge" style="font-size:15px;margin-bottom: 30px;"> <span style="font-size:20px"><span class=" icon-wrapper icon-fa icon-fa-light" data-name="chart-bar"><svg viewBox="0 0 512 520.146" xmlns="http://www.w3.org/2000/svg"><path d="M424 353c-4.4 0-8-3.6-8-8V105c0-4.4 3.6-8 8-8h16c4.4 0 8 3.6 8 8v240c0 4.4-3.6 8-8 8h-16zm-96 0c-4.4 0-8-3.6-8-8V201c0-4.4 3.6-8 8-8h16c4.4 0 8 3.6 8 8v144c0 4.4-3.6 8-8 8h-16zm-192 0c-4.4 0-8-3.6-8-8v-80c0-4.4 3.6-8 8-8h16c4.4 0 8 3.6 8 8v80c0 4.4-3.6 8-8 8h-16zm96 0c-4.4 0-8-3.6-8-8V137c0-4.4 3.6-8 8-8h16c4.4 0 8 3.6 8 8v208c0 4.4-3.6 8-8 8h-16zm272 64c4.42 0 8 3.58 8 8v16c0 4.42-3.58 8-8 8H16c-8.84 0-16-7.16-16-16V73c0-4.42 3.58-8 8-8h16c4.42 0 8 3.58 8 8v344h472z"/></svg></span></span>   Submit <a href="/paper/joyvasa-portrait-and-animal-image-animation/review/"> results from this paper </a> to get state-of-the-art GitHub badges and help the community compare results to other papers. </div> </div> </div> </div> </div> </div> <div class="paper-evaluation-section" id="evaluation"> <div class="row"> <div class="col-md-12"> </div> </div> </div> <div class="row"> <div id="methods" class="col-md-12 paper-evaluation-section-title"> <div class="paper-section-title"> <div class="row"> <div class="col-md-12 zero-padding"> <h2> Methods <div class="float-right"> <div class="dropdown edit-button"> <button class="dropdown-toggle badge badge-edit" type="button" id="methodEditMenu" data-bs-toggle="dropdown" aria-haspopup="true" aria-expanded="false"> <span class=" icon-wrapper icon-fa icon-fa-solid" data-name="edit"><svg viewBox="0 0 576 514.999" xmlns="http://www.w3.org/2000/svg"><path d="M402.6 85.198l90.2 90.2c3.8 3.8 3.8 10 0 13.8l-218.399 218.4-92.8 10.3c-12.4 1.4-22.9-9.1-21.5-21.5l10.3-92.8 218.4-218.4c3.799-3.8 10-3.8 13.799 0zm162-22.9c15.2 15.2 15.2 39.9 0 55.2l-35.4 35.4c-3.8 3.8-10 3.8-13.8 0l-90.2-90.2c-3.8-3.8-3.8-10 0-13.8l35.4-35.4c15.3-15.2 40-15.2 55.2 0zM384 348.198c0-3.2 1.3-6.2 3.5-8.5l40-40c7.6-7.5 20.5-2.2 20.5 8.5v157.8c0 26.5-21.5 48-48 48H48c-26.5 0-48-21.5-48-48v-352c0-26.5 21.5-48 48-48h285.8c10.7 0 16.1 12.9 8.5 20.5l-40 40c-2.3 2.2-5.3 3.5-8.5 3.5H64v320h320v-101.8z"/></svg></span> Edit </button> <div class="dropdown-menu dropdown-menu-end" aria-labelledby="methodEditMenu"> <a class="dropdown-item" href="#loginModal" data-bs-toggle="modal"> <span class=" icon-wrapper icon-ion" data-name="add"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M256 112v288m144-144H112"/></svg></span> Add</a> <a class="dropdown-item" href="#loginModal" data-bs-toggle="modal"> <span class=" icon-wrapper icon-ion" data-name="remove"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M400 256H112"/></svg></span> Remove</a> </div> </div> </div> </h2> <hr/> </div> </div> </div> </div> </div> <div class="method-section" id="methods"> <a href="/method/diffusion"> Diffusion</a> • <a href="/method/focus"> Focus</a> </div>  </div> </div> <div class="footer"> <div class="footer-contact"> <span class="footer-contact-item">Contact us on:</span> <a class="footer-contact-item" href="mailto:hello@paperswithcode.com"> <span class=" icon-wrapper icon-ion" data-name="mail"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M424 80H88a56.06 56.06 0 0 0-56 56v240a56.06 56.06 0 0 0 56 56h336a56.06 56.06 0 0 0 56-56V136a56.06 56.06 0 0 0-56-56zm-14.18 92.63l-144 112a16 16 0 0 1-19.64 0l-144-112a16 16 0 1 1 19.64-25.26L256 251.73l134.18-104.36a16 16 0 0 1 19.64 25.26z"/></svg></span> hello@paperswithcode.com </a>. <span class="footer-contact-item"> Papers With Code is a free resource with all data licensed under <a rel="noreferrer" href="https://creativecommons.org/licenses/by-sa/4.0/">CC-BY-SA</a>. </span> </div> <div class="footer-links"> <a href="/site/terms">Terms</a> <a href="/site/data-policy">Data policy</a> <a href="/site/cookies-policy">Cookies policy</a> <a href="/about#team" class="fair-logo"> from <img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAANAAAAAgCAMAAABU6AZfAAAABGdBTUEAALGPC/xhBQAAAAFzUkdCAK7OHOkAAAAJcEhZcwAAFiUAABYlAUlSJPAAAABFUExURUdwTBwqMhwqMxsqMhkqMxsqMhwqMgCA+hwrMxJIgBsrMxsqMgJ28AF58wF38BsqMwB58hsqMwF17wF07hwrMwRm4QJz7Wj6SIIAAAAUdFJOUwDP87wcPIT+4A1tVti1Ta0smZVzG3JP8wAABR9JREFUWMO1memWpCoMgF0QxX1//0e9kCAkAadq5tzKjzndQmM+szNFEWQ9puu6xn02BXm4j23bTsdapKJAMguFgRVT/Ejyx4uH5hgvL1PUfm69jEd6bN05GTJvXF5X/hfRcPyWe2kTLDFdRA4ENVMbZZJGMt3ppEttNMDC2X/Qa7MK1OrveZoKz2/445I+U4znuvaExxKZLFCqtym/A6rzn+OjbHj8ubwDmfESslvtgWea13WeckQPUKJTf/4USHkDnVXzCrT74DnmeX+8rjgcxA4QBmPpyAKdOm+5XwFpgHH/bG9AMzLMqM9DxxCQaM0qLr7U4xE/AgIDVRBHlcoDeYd7lFee6GZOBvaaskD8S6nut0Dg0ItZEt+IQAfjseIzRDvS/WCxWQJ17phqEGqepQBS/VaXZa0H/4XUYMVt6nr309DEjYvduPT2gWELQTr0iQbC1+SADOg/kjVvspGqX6zSRAgEKbqOf6zgd82AVB+8s0YNm5NL6Y8MGzttwKt0krP9+9A/+hzQTALoUX5MnxW7iCIEUmD7IVZb8G0G1HRE9UqbWKkEUFPSR0MWqH5eB65XmgzQdN3WGjxReROxPD2LROeBIEiD7UGLraBAjMcS9W9AquTPckBgoMqEWG1SIGN57otn5KO9Y30N4rq6MQFC5TX1cEWBfJLY+mbQ5ZMUm8UK7F1A9GNc90T3enkpCZhCdUzfdQq0Wp774gnZao55YU3SgkmAVBez1eDfR4BABd/XqY36ichyaLUnyJZ8jatimUBjqQTouK2M3OGs4miiiduN5bkHCL15C9Zw7heBRMHYSMRxIGyYFsPqpwTqactT8w0P0OSA9iRY9jQvrDyIAhCoAjrrR90I1PNCpcivHEh+cATUmS5xoCaNB3ggMzqgRO/RYPIb1WviDkB4sv22kB8ghQcgUIFWzyUmaQ6kpf5DCoTFh5fwQQCt493e9ypD5Xjq7S5cMQeEubpBf2oKCoSMohPzduBAi2yimhRIc3NvrOd+gCxPexvhcGPM3SRoJpbmIhAGSudTNgNCR+qIRL05UCebsxTIiAYOX6sEkONphRkw9A9ZjADIZIDg857we5MBSiQHVMlWJgXyeTBIyVpGD4RttHC4yVtENHn7K5ASdeM3QGX2sKcKBCBmITYmrGii9TOQT7JYwxOgrhbyby4XJrvs54kuR8vlCg4XEgEOEs8Q8R5DYZboCwEESpTmi/Hhc1Lo8zxPlghZjpbLqWVGUGxSes1y4W2lkkC+Wf0C6GPaxtZo0VQW4nOhsJLqAg01HXqgGN0+083MegKoYLdisbDqzHVG1iZJYe0EUDoB+dj149gDRCCgt2lZ1zA5nhvCyEwvrc/b3N/HiZlMgINmZaR/aX3MJluf7Kepo8+F5tRfUh1wR0odzg8Srnm9w7L5SyB/p6H9Ptt0Vj310ngAlDHbnLo3mGc00sJiQ+4KEM+I8xC7fWv5VGcz3Y0C2ZCa70sgf0tXbnbY1jXpln3W6jYXDG4jNthdrfVWn8n4gAVAZe+0GgaEaeGFx4XRQyTM9yWQnNuIAy5/HPAWPuDJ8Yc66sYvSeY/8dhlYqH0kuQzkFQ03nnHCyI/gtc0GfM7BVPmL5J0yHPkXm6d3u6v/TLw3GL5ayDr6WW47awHYmS1VC+XJOVQcCCZBPk13SCvgmcb8uI/UqjqdvlOlk3j5OU20C0putdO1ZWNo0a8oumXslx0vMYaNrfPURt2hnp5G2rhtsEP5j/3Wqt0fQd1YgAAAABJRU5ErkJggg=="> </a> </div> </div> <script> // MathJax window.MathJax = { tex: { inlineMath: [ ["$", "$"], ["\$", "\$"], ], }, }; const mathjaxScript = document.createElement("script"); mathjaxScript.src = "https://production-assets.paperswithcode.com/static/js/mathjax/tex-chtml.js"; document.head.appendChild(mathjaxScript); </script> <script src="https://production-assets.paperswithcode.com/perf/766.4af6b88b.js" defer></script><script src="https://production-assets.paperswithcode.com/perf/351.a22a9607.js" defer></script><script>(()=>{"use strict";var e,t,n,r,a={73487:(e,t,n)=>{n(26029),n(96869),n(22696),n(89527),n(7233),n(80591);var r=n(23279),a=n.n(r);const o=["tasks","leaderboards","papers","datasets","methods"],l=document.getElementById("id_global_search_form"),s=document.getElementById("id_global_search_input"),i=document.getElementById("q_meta"),d=document.getElementById("q_type"),c=document.createElement("ul");c.id="result-box",l.appendChild(c);let u=0,m=!1,p=[],f=null,g="";const h=a()((function(e){const t=e.target.value;if(t.length<=1)return c.classList.remove("show"),m=!1,p=[],void(f=null);(async e=>{const n=await fetch(`/api/search-autocomplete/?q=${encodeURIComponent(t)}`),r=await n.json();e===u&&function(e){if(e=function(e){let t=o.reduce(((e,t)=>(e[t]=[],e)),{}),n=12;for(let r=0;r<5;r++){for(const a of o)if(e[a].length>r&&(t[a].push(e[a][r]),n--,n<=0))break;if(n<=0)break}return n<12?t:null}(e),f=null,!e)return c.classList.remove("show"),m=!1,void(p=[]);let t="";for(const n of o)if(e[n].length){t+=`<li class='category-name'>${b(n[0].toUpperCase()+n.substring(1))}</li>`;for(const r of e[n]){let e="";["leaderboards","datasets"].includes(n)&&(e=r.slug),t+=`<li class='search-item' data-category="${b(n)}" data-meta="${b(e)}" data-label="${b(r.name||r.title)}"><div class='search-item-inner'>`,r.image?(r.image.startsWith("media")&&(r.image="/"+r.image),t+=`<img src="${b(r.image)}">`):"papers"!==n&&(t+=`<img src='${MEDIA_URL}tasks/default.gif'>`),t+=`<span>${b(r.name||r.title)}</span></div></li>`}}c.innerHTML=t,c.classList.add("show"),m=!0,p=[...document.getElementsByClassName("search-item")]}(r)})(++u)}),250,{maxWait:1e3});function y(e){if(!e)return void l.submit();const t=e.dataset.meta,n=e.dataset.category,r=e.dataset.label;GTAG_ENABLED&&window.gtag("event","SiteActions",{event_category:"Search",event_label:n}),s.value=r,t?i.value=t:(i.value="",i.removeAttribute("name")),d.value=n,l.submit()}function v(e){if(null!==e&&e>=p.length)throw Error("idx out of bound");f=e;for(const e of p)e.classList.remove("selected");null!==e?(p[e].classList.add("selected"),s.value=p[e].dataset.label):s.value=g}function b(e){return e.replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">").replace(/"/g,""").replace(/'/g,"'")}s.addEventListener("input",h),document.body.addEventListener("click",(()=>{c.classList.remove("show"),m=!1})),s.addEventListener("click",(e=>{e.stopPropagation()})),s.addEventListener("input",(()=>{g=s.value})),s.addEventListener("keydown",(e=>{if("Escape"===e.key&&m&&(e.preventDefault(),c.classList.remove("show"),m=!1,v(null)),"ArrowDown"===e.key){if(e.preventDefault(),!p.length)return;p.length&&(c.classList.add("show"),m=!0),null===f?v(0):f>=p.length-1?v(null):v(f+1)}if("ArrowUp"===e.key){if(e.preventDefault(),!p.length)return;p.length&&(c.classList.add("show"),m=!0),v(null===f?p.length-1:f<=0?null:f-1)}})),c.addEventListener("click",(e=>{e.stopPropagation(),y(e.target.closest(".search-item"))})),l.addEventListener("submit",(e=>{y(p[f])}));var E=n(179);""!==SENTRY_DSN_FRONTEND&&E.S1({dsn:SENTRY_DSN_FRONTEND});var w=n(45852);let k=!1;const L=document.getElementsByClassName("read-more-toggle")[0],_=document.getElementsByClassName("read-more-dots")[0],x=document.getElementsByClassName("read-more-rest")[0];L&&L.addEventListener("click",(e=>{e.preventDefault(),k?(_.style.display="",x.style.display="",L.text="read more"):(_.style.display="none",x.style.display="inline",L.text="(read less)"),k=!k}));const C=document.getElementById("implementations-see-more-trigger"),B=document.getElementById("implementations-see-less-trigger"),N=document.getElementById("implementations-short-list"),S=document.getElementById("implementations-full-list");C&&C.addEventListener("click",(e=>{e.preventDefault(),N.style.display="none",S.style.display=""})),B&&B.addEventListener("click",(e=>{e.preventDefault(),N.style.display="",S.style.display="none"})),(()=>{const e=[...document.querySelectorAll(".modal-body")];let t=!1;for(const t of e)t.style.opacity=0;window.addEventListener("click",(r=>{if(r.target.closest('[data-bs-toggle="modal"]')){const r=document.getElementById("modals-template");document.body.appendChild(r.content);let a=(0,w.Z)("csrftoken");for(const e of[...document.querySelectorAll("input[name='csrfmiddlewaretoken']")])e.value=a;!async function(){t||(t=!0,Promise.all([n.e(2),n.e(109),n.e(702),n.e(90)]).then(n.bind(n,56090)).then((()=>{n.e(43).then(n.bind(n,36043));for(const t of e)t.style.opacity=""})))}(),(()=>{const e=document.getElementById("new-method-form"),t=document.getElementById("new-method-form-toggle");let n=!1;t.addEventListener("click",(t=>{t.preventDefault(),e.style.display=n?"none":"",n=!n}))})(),(()=>{const e=document.getElementById("new-task-form"),t=document.getElementById("new-task-form-toggle");let n=!1;t.addEventListener("click",(t=>{t.preventDefault(),e.style.display=n?"none":"",n=!n}))})()}}),!0)})()},45852:(e,t,n)=>{n.d(t,{Z:()=>r});const r=e=>{var t=null;if(document.cookie&&""!==document.cookie)for(var n=document.cookie.split(";"),r=0;r<n.length;r++){var a=n[r].trim();if(a.substring(0,e.length+1)===e+"="){t=decodeURIComponent(a.substring(e.length+1));break}}return t}}},o={};function l(e){if(o[e])return o[e].exports;var t=o[e]={id:e,loaded:!1,exports:{}};return a[e](t,t.exports,l),t.loaded=!0,t.exports}l.m=a,l.x=e=>{},l.n=e=>{var t=e&&e.__esModule?()=>e.default:()=>e;return l.d(t,{a:t}),t},l.d=(e,t)=>{for(var n in t)l.o(t,n)&&!l.o(e,n)&&Object.defineProperty(e,n,{enumerable:!0,get:t[n]})},l.f={},l.e=e=>Promise.all(Object.keys(l.f).reduce(((t,n)=>(l.f[n](e,t),t)),[])),l.u=e=>e+"."+{2:"6da00df7",43:"b3f6a007",90:"ead00655",109:"5aa180f0",702:"c05a3709"}[e]+".js",l.miniCssF=e=>e+"."+{43:"b2664180",90:"d7a7e4c6",109:"6ee1c62e",918:"c41196c3"}[e]+".css",l.g=function(){if("object"==typeof globalThis)return globalThis;try{return this||new Function("return this")()}catch(e){if("object"==typeof window)return window}}(),l.hmd=e=>((e=Object.create(e)).children||(e.children=[]),Object.defineProperty(e,"exports",{enumerable:!0,set:()=>{throw new Error("ES Modules may not assign module.exports or exports.*, Use ESM export syntax, instead: "+e.id)}}),e),l.o=(e,t)=>Object.prototype.hasOwnProperty.call(e,t),e={},t="perf_frontend:",l.l=(n,r,a,o)=>{if(e[n])e[n].push(r);else{var s,i;if(void 0!==a)for(var d=document.getElementsByTagName("script"),c=0;c<d.length;c++){var u=d[c];if(u.getAttribute("src")==n||u.getAttribute("data-webpack")==t+a){s=u;break}}s||(i=!0,(s=document.createElement("script")).charset="utf-8",s.timeout=120,l.nc&&s.setAttribute("nonce",l.nc),s.setAttribute("data-webpack",t+a),s.src=n),e[n]=[r];var m=(t,r)=>{s.onerror=s.onload=null,clearTimeout(p);var a=e[n];if(delete e[n],s.parentNode&&s.parentNode.removeChild(s),a&&a.forEach((e=>e(r))),t)return t(r)},p=setTimeout(m.bind(null,void 0,{type:"timeout",target:s}),12e4);s.onerror=m.bind(null,s.onerror),s.onload=m.bind(null,s.onload),i&&document.head.appendChild(s)}},l.r=e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},l.p="https://production-assets.paperswithcode.com/perf/",n=e=>new Promise(((t,n)=>{var r=l.miniCssF(e),a=l.p+r;if(((e,t)=>{for(var n=document.getElementsByTagName("link"),r=0;r<n.length;r++){var a=(l=n[r]).getAttribute("data-href")||l.getAttribute("href");if("stylesheet"===l.rel&&(a===e||a===t))return l}var o=document.getElementsByTagName("style");for(r=0;r<o.length;r++){var l;if((a=(l=o[r]).getAttribute("data-href"))===e||a===t)return l}})(r,a))return t();((e,t,n,r)=>{var a=document.createElement("link");a.rel="stylesheet",a.type="text/css",a.onerror=a.onload=o=>{if(a.onerror=a.onload=null,"load"===o.type)n();else{var l=o&&("load"===o.type?"missing":o.type),s=o&&o.target&&o.target.href||t,i=new Error("Loading CSS chunk "+e+" failed.\n("+s+")");i.code="CSS_CHUNK_LOAD_FAILED",i.type=l,i.request=s,a.parentNode.removeChild(a),r(i)}},a.href=t,document.head.appendChild(a)})(e,a,t,n)})),r={645:0},l.f.miniCss=(e,t)=>{r[e]?t.push(r[e]):0!==r[e]&&{43:1,90:1,109:1}[e]&&t.push(r[e]=n(e).then((()=>{r[e]=0}),(t=>{throw delete r[e],t})))},(()=>{var e={645:0},t=[[73487,766,351]];l.f.j=(t,n)=>{var r=l.o(e,t)?e[t]:void 0;if(0!==r)if(r)n.push(r[2]);else if(918!=t){var a=new Promise(((n,a)=>{r=e[t]=[n,a]}));n.push(r[2]=a);var o=l.p+l.u(t),s=new Error;l.l(o,(n=>{if(l.o(e,t)&&(0!==(r=e[t])&&(e[t]=void 0),r)){var a=n&&("load"===n.type?"missing":n.type),o=n&&n.target&&n.target.src;s.message="Loading chunk "+t+" failed.\n("+a+": "+o+")",s.name="ChunkLoadError",s.type=a,s.request=o,r[1](s)}}),"chunk-"+t,t)}else e[t]=0};var n=e=>{},r=(r,a)=>{for(var o,s,[i,d,c,u]=a,m=0,p=[];m<i.length;m++)s=i[m],l.o(e,s)&&e[s]&&p.push(e[s][0]),e[s]=0;for(o in d)l.o(d,o)&&(l.m[o]=d[o]);for(c&&c(l),r&&r(a);p.length;)p.shift()();return u&&t.push.apply(t,u),n()},a=self.webpackChunkperf_frontend=self.webpackChunkperf_frontend||[];function o(){for(var n,r=0;r<t.length;r++){for(var a=t[r],o=!0,s=1;s<a.length;s++){var i=a[s];0!==e[i]&&(o=!1)}o&&(t.splice(r--,1),n=l(l.s=a[0]))}return 0===t.length&&(l.x(),l.x=e=>{}),n}a.forEach(r.bind(null,0)),a.push=r.bind(null,a.push.bind(a));var s=l.x;l.x=()=>(l.x=s||(e=>{}),(n=o)())})(),l.x()})();</script> </body> </html>

CINXE.COM

JoyVASA: Portrait and Animal Image Animation with Diffusion-Based Audio-Driven Facial Dynamics and Head Motion Generation | Papers With Code