CINXE.COM

<!DOCTYPE html> <html lang="en"> <head> <meta charset="utf-8"> <meta http-equiv="X-UA-Compatible" content="IE=edge"> <meta name="viewport" content="width=device-width, initial-scale=1"> <title>The Chamber Ensemble Generator and CocoChorales Dataset</title> <meta name="description" content=" In this post, we’re excited to introduce the Chamber Ensemble Generator, a system for generating realistic chamber ensemble performances, and the correspon...">  <meta property="og:image" content="https://magenta.tensorflow.org/assets/cocochorales/ceg_logo.png"> <meta property="og:title" content="The Chamber Ensemble Generator and CocoChorales Dataset"> <meta property="og:description" content=" In this post, we’re excited to introduce the Chamber Ensemble Generator, a system for generating realistic chamber ensemble performances, and the correspon..."> <meta property="og:url" content="https://magenta.tensorflow.org/ceg-and-cocochorales"> <meta property="og:site_name" content="Magenta">  <meta name="twitter:card" content="summary"> <meta name="twitter:title" content="The Chamber Ensemble Generator and CocoChorales Dataset"> <meta name="twitter:description" content=" In this post, we’re excited to introduce the Chamber Ensemble Generator, a system for generating realistic chamber ensemble performances, and the correspon..."> <meta name="twitter:image" content="https://magenta.tensorflow.org/assets/cocochorales/ceg_logo.png"> <link rel="stylesheet" href="/css/main.css"> <link rel="canonical" href="https://magenta.tensorflow.org/ceg-and-cocochorales"> <link rel="alternate" type="application/rss+xml" title="Magenta" href="https://magenta.tensorflow.org/feed.xml"> <link href="https://fonts.googleapis.com/css?family=Google+Sans:+400,500,700" media="all" rel="stylesheet"> <script src="//www.google.com/js/gweb/analytics/autotrack.js"></script> <script> new gweb.analytics.AutoTrack({ profile: 'UA-80107903-1' }); </script> </head> <body> <div class="scrim" onclick="document.body.classList.toggle('drawer-opened', false)"></div> <header> <div class="top-bar background"> <div class="top-bar-content"> <div class="logo"> <a href="/"><img src="/assets/magenta-logo.png" height="70" alt="magenta logo"></a> </div> <nav> <button class="menu-button" onclick="document.body.classList.toggle('drawer-opened', true)" aria-label="open nav menu"> <svg viewBox="0 0 18 15"> <path fill="#424242" d="M18,1.484c0,0.82-0.665,1.484-1.484,1.484H1.484C0.665,2.969,0,2.304,0,1.484l0,0C0,0.665,0.665,0,1.484,0 h15.031C17.335,0,18,0.665,18,1.484L18,1.484z"/> <path fill="#424242" d="M18,7.516C18,8.335,17.335,9,16.516,9H1.484C0.665,9,0,8.335,0,7.516l0,0c0-0.82,0.665-1.484,1.484-1.484 h15.031C17.335,6.031,18,6.696,18,7.516L18,7.516z"/> <path fill="#424242" d="M18,13.516C18,14.335,17.335,15,16.516,15H1.484C0.665,15,0,14.335,0,13.516l0,0 c0-0.82,0.665-1.484,1.484-1.484h15.031C17.335,12.031,18,12.696,18,13.516L18,13.516z"/> </svg> </button> <div class="links"> <a href="/get-started">Get Started</a> <a href="/studio">Studio</a> <a href="/ddsp-vst">DDSP-VST</a> <a href="/demos">Demos</a> <a href="/blog">Blog</a> <a href="/research">Research</a> <a href="/talks">Talks</a> <a href="/community">Community</a> </div> </nav> </div> </div> </header> <div class="drawer"> <div class="drawer-content"> <a href="/get-started">Get Started</a> <a href="/studio">Studio</a> <a href="/ddsp-vst">DDSP-VST</a> <a href="/demos">Demos</a> <a href="/blog">Blog</a> <a href="/research">Research</a> <a href="/talks">Talks</a> <a href="/community">Community</a> </div> </div> <div class="main"> <section class="white"> <article class="content single" itemscope itemtype="http://schema.org/BlogPosting"> <h1 class="post-title" itemprop="name headline">The Chamber Ensemble Generator and CocoChorales Dataset</h1> <p class="post-meta"> <time datetime="2022-09-30T09:00:00-07:00" itemprop="datePublished">Sep 30, 2022</time> <br> <a class="inverted" href=https://lukewys.github.io/>Yusong Wu</a> <a class="inverted" href="https://github.com/lukewys" alt="github logo"><span class="icon icon--github"><svg viewBox="0 0 16 16"><path d="M7.999,0.431c-4.285,0-7.76,3.474-7.76,7.761 c0,3.428,2.223,6.337,5.307,7.363c0.388,0.071,0.53-0.168,0.53-0.374c0-0.184-0.007-0.672-0.01-1.32 c-2.159,0.469-2.614-1.04-2.614-1.04c-0.353-0.896-0.862-1.135-0.862-1.135c-0.705-0.481,0.053-0.472,0.053-0.472 c0.779,0.055,1.189,0.8,1.189,0.8c0.692,1.186,1.816,0.843,2.258,0.645c0.071-0.502,0.271-0.843,0.493-1.037 C4.86,11.425,3.049,10.76,3.049,7.786c0-0.847,0.302-1.54,0.799-2.082C3.768,5.507,3.501,4.718,3.924,3.65 c0,0,0.652-0.209,2.134,0.796C6.677,4.273,7.34,4.187,8,4.184c0.659,0.003,1.323,0.089,1.943,0.261 c1.482-1.004,2.132-0.796,2.132-0.796c0.423,1.068,0.157,1.857,0.077,2.054c0.497,0.542,0.798,1.235,0.798,2.082 c0,2.981-1.814,3.637-3.543,3.829c0.279,0.24,0.527,0.713,0.527,1.437c0,1.037-0.01,1.874-0.01,2.129 c0,0.208,0.14,0.449,0.534,0.373c3.081-1.028,5.302-3.935,5.302-7.362C15.76,3.906,12.285,0.431,7.999,0.431z"/></svg> </span><span class="username">lukewys</span></a> <a class="inverted" href="https://twitter.com/wuyusongwys" alt="twitter logo"><span class="icon icon--twitter"><svg viewBox="0 0 16 16"><path d="M15.969,3.058c-0.586,0.26-1.217,0.436-1.878,0.515c0.675-0.405,1.194-1.045,1.438-1.809c-0.632,0.375-1.332,0.647-2.076,0.793c-0.596-0.636-1.446-1.033-2.387-1.033c-1.806,0-3.27,1.464-3.27,3.27 c0,0.256,0.029,0.506,0.085,0.745C5.163,5.404,2.753,4.102,1.14,2.124C0.859,2.607,0.698,3.168,0.698,3.767 c0,1.134,0.577,2.135,1.455,2.722C1.616,6.472,1.112,6.325,0.671,6.08c0,0.014,0,0.027,0,0.041c0,1.584,1.127,2.906,2.623,3.206 C3.02,9.402,2.731,9.442,2.433,9.442c-0.211,0-0.416-0.021-0.615-0.059c0.416,1.299,1.624,2.245,3.055,2.271 c-1.119,0.877-2.529,1.4-4.061,1.4c-0.264,0-0.524-0.015-0.78-0.046c1.447,0.928,3.166,1.469,5.013,1.469 c6.015,0,9.304-4.983,9.304-9.304c0-0.142-0.003-0.283-0.009-0.423C14.976,4.29,15.531,3.714,15.969,3.058z"/></svg> </span><span class="username">wuyusongwys</span></a> <br/> <a class="inverted" href=https://homes.cs.washington.edu/~jpgard/>Josh Gardner</a> <a class="inverted" href="https://github.com/jpgard" alt="github logo"><span class="icon icon--github"><svg viewBox="0 0 16 16"><path d="M7.999,0.431c-4.285,0-7.76,3.474-7.76,7.761 c0,3.428,2.223,6.337,5.307,7.363c0.388,0.071,0.53-0.168,0.53-0.374c0-0.184-0.007-0.672-0.01-1.32 c-2.159,0.469-2.614-1.04-2.614-1.04c-0.353-0.896-0.862-1.135-0.862-1.135c-0.705-0.481,0.053-0.472,0.053-0.472 c0.779,0.055,1.189,0.8,1.189,0.8c0.692,1.186,1.816,0.843,2.258,0.645c0.071-0.502,0.271-0.843,0.493-1.037 C4.86,11.425,3.049,10.76,3.049,7.786c0-0.847,0.302-1.54,0.799-2.082C3.768,5.507,3.501,4.718,3.924,3.65 c0,0,0.652-0.209,2.134,0.796C6.677,4.273,7.34,4.187,8,4.184c0.659,0.003,1.323,0.089,1.943,0.261 c1.482-1.004,2.132-0.796,2.132-0.796c0.423,1.068,0.157,1.857,0.077,2.054c0.497,0.542,0.798,1.235,0.798,2.082 c0,2.981-1.814,3.637-3.543,3.829c0.279,0.24,0.527,0.713,0.527,1.437c0,1.037-0.01,1.874-0.01,2.129 c0,0.208,0.14,0.449,0.534,0.373c3.081-1.028,5.302-3.935,5.302-7.362C15.76,3.906,12.285,0.431,7.999,0.431z"/></svg> </span><span class="username">jpgard</span></a> <a class="inverted" href="https://twitter.com/jpgard" alt="twitter logo"><span class="icon icon--twitter"><svg viewBox="0 0 16 16"><path d="M15.969,3.058c-0.586,0.26-1.217,0.436-1.878,0.515c0.675-0.405,1.194-1.045,1.438-1.809c-0.632,0.375-1.332,0.647-2.076,0.793c-0.596-0.636-1.446-1.033-2.387-1.033c-1.806,0-3.27,1.464-3.27,3.27 c0,0.256,0.029,0.506,0.085,0.745C5.163,5.404,2.753,4.102,1.14,2.124C0.859,2.607,0.698,3.168,0.698,3.767 c0,1.134,0.577,2.135,1.455,2.722C1.616,6.472,1.112,6.325,0.671,6.08c0,0.014,0,0.027,0,0.041c0,1.584,1.127,2.906,2.623,3.206 C3.02,9.402,2.731,9.442,2.433,9.442c-0.211,0-0.416-0.021-0.615-0.059c0.416,1.299,1.624,2.245,3.055,2.271 c-1.119,0.877-2.529,1.4-4.061,1.4c-0.264,0-0.524-0.015-0.78-0.046c1.447,0.928,3.166,1.469,5.013,1.469 c6.015,0,9.304-4.983,9.304-9.304c0-0.142-0.003-0.283-0.009-0.423C14.976,4.29,15.531,3.714,15.969,3.058z"/></svg> </span><span class="username">jpgard</span></a> <br/> <a class="inverted" href=https://ethman.github.io/>Ethan Manilow</a> <a class="inverted" href="https://github.com/ethman" alt="github logo"><span class="icon icon--github"><svg viewBox="0 0 16 16"><path d="M7.999,0.431c-4.285,0-7.76,3.474-7.76,7.761 c0,3.428,2.223,6.337,5.307,7.363c0.388,0.071,0.53-0.168,0.53-0.374c0-0.184-0.007-0.672-0.01-1.32 c-2.159,0.469-2.614-1.04-2.614-1.04c-0.353-0.896-0.862-1.135-0.862-1.135c-0.705-0.481,0.053-0.472,0.053-0.472 c0.779,0.055,1.189,0.8,1.189,0.8c0.692,1.186,1.816,0.843,2.258,0.645c0.071-0.502,0.271-0.843,0.493-1.037 C4.86,11.425,3.049,10.76,3.049,7.786c0-0.847,0.302-1.54,0.799-2.082C3.768,5.507,3.501,4.718,3.924,3.65 c0,0,0.652-0.209,2.134,0.796C6.677,4.273,7.34,4.187,8,4.184c0.659,0.003,1.323,0.089,1.943,0.261 c1.482-1.004,2.132-0.796,2.132-0.796c0.423,1.068,0.157,1.857,0.077,2.054c0.497,0.542,0.798,1.235,0.798,2.082 c0,2.981-1.814,3.637-3.543,3.829c0.279,0.24,0.527,0.713,0.527,1.437c0,1.037-0.01,1.874-0.01,2.129 c0,0.208,0.14,0.449,0.534,0.373c3.081-1.028,5.302-3.935,5.302-7.362C15.76,3.906,12.285,0.431,7.999,0.431z"/></svg> </span><span class="username">ethman</span></a> <a class="inverted" href="https://twitter.com/ethanmanilow" alt="twitter logo"><span class="icon icon--twitter"><svg viewBox="0 0 16 16"><path d="M15.969,3.058c-0.586,0.26-1.217,0.436-1.878,0.515c0.675-0.405,1.194-1.045,1.438-1.809c-0.632,0.375-1.332,0.647-2.076,0.793c-0.596-0.636-1.446-1.033-2.387-1.033c-1.806,0-3.27,1.464-3.27,3.27 c0,0.256,0.029,0.506,0.085,0.745C5.163,5.404,2.753,4.102,1.14,2.124C0.859,2.607,0.698,3.168,0.698,3.767 c0,1.134,0.577,2.135,1.455,2.722C1.616,6.472,1.112,6.325,0.671,6.08c0,0.014,0,0.027,0,0.041c0,1.584,1.127,2.906,2.623,3.206 C3.02,9.402,2.731,9.442,2.433,9.442c-0.211,0-0.416-0.021-0.615-0.059c0.416,1.299,1.624,2.245,3.055,2.271 c-1.119,0.877-2.529,1.4-4.061,1.4c-0.264,0-0.524-0.015-0.78-0.046c1.447,0.928,3.166,1.469,5.013,1.469 c6.015,0,9.304-4.983,9.304-9.304c0-0.142-0.003-0.283-0.009-0.423C14.976,4.29,15.531,3.714,15.969,3.058z"/></svg> </span><span class="username">ethanmanilow</span></a> <br/> <a class="inverted" href=https://g.co/magenta/ian_simon>Ian Simon</a> <a class="inverted" href="https://github.com/iansimon" alt="github logo"><span class="icon icon--github"><svg viewBox="0 0 16 16"><path d="M7.999,0.431c-4.285,0-7.76,3.474-7.76,7.761 c0,3.428,2.223,6.337,5.307,7.363c0.388,0.071,0.53-0.168,0.53-0.374c0-0.184-0.007-0.672-0.01-1.32 c-2.159,0.469-2.614-1.04-2.614-1.04c-0.353-0.896-0.862-1.135-0.862-1.135c-0.705-0.481,0.053-0.472,0.053-0.472 c0.779,0.055,1.189,0.8,1.189,0.8c0.692,1.186,1.816,0.843,2.258,0.645c0.071-0.502,0.271-0.843,0.493-1.037 C4.86,11.425,3.049,10.76,3.049,7.786c0-0.847,0.302-1.54,0.799-2.082C3.768,5.507,3.501,4.718,3.924,3.65 c0,0,0.652-0.209,2.134,0.796C6.677,4.273,7.34,4.187,8,4.184c0.659,0.003,1.323,0.089,1.943,0.261 c1.482-1.004,2.132-0.796,2.132-0.796c0.423,1.068,0.157,1.857,0.077,2.054c0.497,0.542,0.798,1.235,0.798,2.082 c0,2.981-1.814,3.637-3.543,3.829c0.279,0.24,0.527,0.713,0.527,1.437c0,1.037-0.01,1.874-0.01,2.129 c0,0.208,0.14,0.449,0.534,0.373c3.081-1.028,5.302-3.935,5.302-7.362C15.76,3.906,12.285,0.431,7.999,0.431z"/></svg> </span><span class="username">iansimon</span></a> <a class="inverted" href="https://twitter.com/iansimon" alt="twitter logo"><span class="icon icon--twitter"><svg viewBox="0 0 16 16"><path d="M15.969,3.058c-0.586,0.26-1.217,0.436-1.878,0.515c0.675-0.405,1.194-1.045,1.438-1.809c-0.632,0.375-1.332,0.647-2.076,0.793c-0.596-0.636-1.446-1.033-2.387-1.033c-1.806,0-3.27,1.464-3.27,3.27 c0,0.256,0.029,0.506,0.085,0.745C5.163,5.404,2.753,4.102,1.14,2.124C0.859,2.607,0.698,3.168,0.698,3.767 c0,1.134,0.577,2.135,1.455,2.722C1.616,6.472,1.112,6.325,0.671,6.08c0,0.014,0,0.027,0,0.041c0,1.584,1.127,2.906,2.623,3.206 C3.02,9.402,2.731,9.442,2.433,9.442c-0.211,0-0.416-0.021-0.615-0.059c0.416,1.299,1.624,2.245,3.055,2.271 c-1.119,0.877-2.529,1.4-4.061,1.4c-0.264,0-0.524-0.015-0.78-0.046c1.447,0.928,3.166,1.469,5.013,1.469 c6.015,0,9.304-4.983,9.304-9.304c0-0.142-0.003-0.283-0.009-0.423C14.976,4.29,15.531,3.714,15.969,3.058z"/></svg> </span><span class="username">iansimon</span></a> <br/> <a class="inverted" href=https://g.co/magenta/curtis_hawthorne>Curtis Hawthorne</a> <a class="inverted" href="https://github.com/cghawthorne" alt="github logo"><span class="icon icon--github"><svg viewBox="0 0 16 16"><path d="M7.999,0.431c-4.285,0-7.76,3.474-7.76,7.761 c0,3.428,2.223,6.337,5.307,7.363c0.388,0.071,0.53-0.168,0.53-0.374c0-0.184-0.007-0.672-0.01-1.32 c-2.159,0.469-2.614-1.04-2.614-1.04c-0.353-0.896-0.862-1.135-0.862-1.135c-0.705-0.481,0.053-0.472,0.053-0.472 c0.779,0.055,1.189,0.8,1.189,0.8c0.692,1.186,1.816,0.843,2.258,0.645c0.071-0.502,0.271-0.843,0.493-1.037 C4.86,11.425,3.049,10.76,3.049,7.786c0-0.847,0.302-1.54,0.799-2.082C3.768,5.507,3.501,4.718,3.924,3.65 c0,0,0.652-0.209,2.134,0.796C6.677,4.273,7.34,4.187,8,4.184c0.659,0.003,1.323,0.089,1.943,0.261 c1.482-1.004,2.132-0.796,2.132-0.796c0.423,1.068,0.157,1.857,0.077,2.054c0.497,0.542,0.798,1.235,0.798,2.082 c0,2.981-1.814,3.637-3.543,3.829c0.279,0.24,0.527,0.713,0.527,1.437c0,1.037-0.01,1.874-0.01,2.129 c0,0.208,0.14,0.449,0.534,0.373c3.081-1.028,5.302-3.935,5.302-7.362C15.76,3.906,12.285,0.431,7.999,0.431z"/></svg> </span><span class="username">cghawthorne</span></a> <a class="inverted" href="https://twitter.com/fjord41" alt="twitter logo"><span class="icon icon--twitter"><svg viewBox="0 0 16 16"><path d="M15.969,3.058c-0.586,0.26-1.217,0.436-1.878,0.515c0.675-0.405,1.194-1.045,1.438-1.809c-0.632,0.375-1.332,0.647-2.076,0.793c-0.596-0.636-1.446-1.033-2.387-1.033c-1.806,0-3.27,1.464-3.27,3.27 c0,0.256,0.029,0.506,0.085,0.745C5.163,5.404,2.753,4.102,1.14,2.124C0.859,2.607,0.698,3.168,0.698,3.767 c0,1.134,0.577,2.135,1.455,2.722C1.616,6.472,1.112,6.325,0.671,6.08c0,0.014,0,0.027,0,0.041c0,1.584,1.127,2.906,2.623,3.206 C3.02,9.402,2.731,9.442,2.433,9.442c-0.211,0-0.416-0.021-0.615-0.059c0.416,1.299,1.624,2.245,3.055,2.271 c-1.119,0.877-2.529,1.4-4.061,1.4c-0.264,0-0.524-0.015-0.78-0.046c1.447,0.928,3.166,1.469,5.013,1.469 c6.015,0,9.304-4.983,9.304-9.304c0-0.142-0.003-0.283-0.009-0.423C14.976,4.29,15.531,3.714,15.969,3.058z"/></svg> </span><span class="username">fjord41</span></a> <br/> <a class="inverted" href=https://g.co/magenta/jesse_engel>Jesse Engel</a> <a class="inverted" href="https://github.com/jesseengel" alt="github logo"><span class="icon icon--github"><svg viewBox="0 0 16 16"><path d="M7.999,0.431c-4.285,0-7.76,3.474-7.76,7.761 c0,3.428,2.223,6.337,5.307,7.363c0.388,0.071,0.53-0.168,0.53-0.374c0-0.184-0.007-0.672-0.01-1.32 c-2.159,0.469-2.614-1.04-2.614-1.04c-0.353-0.896-0.862-1.135-0.862-1.135c-0.705-0.481,0.053-0.472,0.053-0.472 c0.779,0.055,1.189,0.8,1.189,0.8c0.692,1.186,1.816,0.843,2.258,0.645c0.071-0.502,0.271-0.843,0.493-1.037 C4.86,11.425,3.049,10.76,3.049,7.786c0-0.847,0.302-1.54,0.799-2.082C3.768,5.507,3.501,4.718,3.924,3.65 c0,0,0.652-0.209,2.134,0.796C6.677,4.273,7.34,4.187,8,4.184c0.659,0.003,1.323,0.089,1.943,0.261 c1.482-1.004,2.132-0.796,2.132-0.796c0.423,1.068,0.157,1.857,0.077,2.054c0.497,0.542,0.798,1.235,0.798,2.082 c0,2.981-1.814,3.637-3.543,3.829c0.279,0.24,0.527,0.713,0.527,1.437c0,1.037-0.01,1.874-0.01,2.129 c0,0.208,0.14,0.449,0.534,0.373c3.081-1.028,5.302-3.935,5.302-7.362C15.76,3.906,12.285,0.431,7.999,0.431z"/></svg> </span><span class="username">jesseengel</span></a> <a class="inverted" href="https://twitter.com/jesseengel" alt="twitter logo"><span class="icon icon--twitter"><svg viewBox="0 0 16 16"><path d="M15.969,3.058c-0.586,0.26-1.217,0.436-1.878,0.515c0.675-0.405,1.194-1.045,1.438-1.809c-0.632,0.375-1.332,0.647-2.076,0.793c-0.596-0.636-1.446-1.033-2.387-1.033c-1.806,0-3.27,1.464-3.27,3.27 c0,0.256,0.029,0.506,0.085,0.745C5.163,5.404,2.753,4.102,1.14,2.124C0.859,2.607,0.698,3.168,0.698,3.767 c0,1.134,0.577,2.135,1.455,2.722C1.616,6.472,1.112,6.325,0.671,6.08c0,0.014,0,0.027,0,0.041c0,1.584,1.127,2.906,2.623,3.206 C3.02,9.402,2.731,9.442,2.433,9.442c-0.211,0-0.416-0.021-0.615-0.059c0.416,1.299,1.624,2.245,3.055,2.271 c-1.119,0.877-2.529,1.4-4.061,1.4c-0.264,0-0.524-0.015-0.78-0.046c1.447,0.928,3.166,1.469,5.013,1.469 c6.015,0,9.304-4.983,9.304-9.304c0-0.142-0.003-0.283-0.009-0.423C14.976,4.29,15.531,3.714,15.969,3.058z"/></svg> </span><span class="username">jesseengel</span></a> <br/> </p> <div class="article-body" itemprop="articleBody"> <style> table tr.wrap { display: flex; flex-direction: row; flex-wrap: wrap; } table tr.wrap > td { display: block; flex: 1; } td {text-align: center !important} .from {background-color: #d3d3d3;} img.inline { vertical-align: middle; display: inline-block; max-height: 16px; width: auto !important; margin-right: 6px; } img.centered { max-width: 90%; margin: auto; } </style> <figure style="text-align: center;"> <img src="assets/cocochorales/logos.png" style="width: 90%; height: auto; margin: auto" alt="Logos for the Chamber Ensemble Generator and CocoChorales Dataset." /> </figure> <p>In this post, we’re excited to introduce the <strong>Chamber Ensemble Generator</strong>, a system for generating realistic chamber ensemble performances, and the corresponding <strong>CocoChorales Dataset</strong>, which contains over 1,400 hours of audio mixes with corresponding source data and MIDI, multi-f<sub>0</sub>, and per-note performance annotations.</p> <table align="center" class="overview"> <tbody><tr> <td>🎵<a href="https://lukewys.github.io/cocochorales/">Audio Examples</a></td> <td>📝<a href="https://arxiv.org/abs/2209.14458">arXiv Paper</a></td> <td>📂<a href="https://magenta.tensorflow.org/datasets/cocochorales">Dataset Download Instructions</a></td> <td><img alt="" src="/assets/ddsp/github.png" class="inline" /><a href="https://github.com/lukewys/chamber-ensemble-generator">Github Code</a></td> </tr> </tbody></table> <p>Data is the bedrock that all machine learning systems are built upon. Historically, researchers applying machine learning to music have not had access to the same scale of data that other fields have. Whereas image and language machine learning researchers measure their datasets by the millions or billions of examples, music researchers feel extremely lucky if they can scrape together a few thousand examples for a given task.</p> <p>Modern machine learning systems require large quantities of <em>annotated</em> data. With music systems, getting annotations for some tasks–like transcription or f<sub>0</sub> estimation–requires tedious work by expert musicians. When annotating a single example correctly is difficult, how can we annotate hundreds of thousands of examples to make enough data to train a machine learning system?</p> <p>In this post, we introduce a new approach to solving these problems by using generative models to create large amounts of realistic-sounding, finely annotated, freely available music data. We combined two structured generative models–a note generation model, <a href="https://magenta.tensorflow.org/coconet">Coconet</a>, and a notes-to-audio generative synthesis model, <a href="https://magenta.tensorflow.org/midi-ddsp">MIDI-DDSP</a>–into a system we call the <strong>Chamber Ensemble Generator</strong>. As its name suggests, the Chamber Ensemble Generator (or CEG) can generate performances of chamber ensembles playing in the style of four-part Bach chorales. Listen to the following examples performed by the CEG:</p> <table align="center"> <tbody> <tr class="wrap"> <td colspan="2">String Ensemble Mixture:</td> </tr> <tr class="wrap"> <td colspan="2"><audio controls=""> <source src="assets/cocochorales/audio/strings/mix.wav?raw=true" /> </audio></td> </tr> <tr class="wrap"> <td colspan="1">Soprano: Violin 1</td> <td colspan="1">Alto: Violin 2</td> </tr> <tr class="wrap"> <td colspan="1"><audio controls=""> <source src="assets/cocochorales/audio/strings/1_violin.wav?raw=true" /> </audio></td> <td colspan="1"><audio controls=""> <source src="assets/cocochorales/audio/strings/2_violin.wav?raw=true" /> </audio></td> </tr> <tr class="wrap"> <td colspan="1">Tenor: Viola</td> <td colspan="1">Bass: Cello</td> </tr> <tr class="wrap"> <td colspan="1"><audio controls=""> <source src="assets/cocochorales/audio/strings/3_viola.wav?raw=true" /> </audio></td> <td colspan="1"><audio controls=""> <source src="assets/cocochorales/audio/strings/4_cello.wav?raw=true" /> </audio></td> </tr> </tbody> </table> <table align="center"> <tbody> <tr class="wrap"> <td colspan="2">Woodwind Ensemble Mixture:</td> </tr> <tr class="wrap"> <td colspan="2"><audio controls=""> <source src="assets/cocochorales/audio/woodwind/mix.wav?raw=true" /> </audio></td> </tr> <tr class="wrap"> <td colspan="1">Soprano: Flute</td> <td colspan="1">Alto: Oboe</td> </tr> <tr class="wrap"> <td colspan="1"><audio controls=""> <source src="assets/cocochorales/audio/woodwind/1_flute.wav?raw=true" /> </audio></td> <td colspan="1"><audio controls=""> <source src="assets/cocochorales/audio/woodwind/2_oboe.wav?raw=true" /> </audio></td> </tr> <tr class="wrap"> <td colspan="1">Tenor: Clarinet</td> <td colspan="1">Bass: Bassoon</td> </tr> <tr class="wrap"> <td colspan="1"><audio controls=""> <source src="assets/cocochorales/audio/woodwind/3_clarinet.wav?raw=true" /> </audio></td> <td colspan="1"><audio controls=""> <source src="assets/cocochorales/audio/woodwind/4_bassoon.wav?raw=true" /> </audio></td> </tr> </tbody> </table> <p>We then used the CEG to create a massive music dataset for machine learning systems. We call this dataset <strong>CocoChorales</strong>. What’s exciting about the CEG is that it uses a set of structured generative models which provide annotations for many music machine learning applications like automatic music transcription, multi-f<sub>0</sub> estimation, source separation, performance analysis, and more.</p> <p>Below, we dig deeper into each of these projects.</p>  <h1 id="the-chamber-ensemble-generator">The Chamber Ensemble Generator</h1> <figure style="text-align: center;"> <img src="assets/cocochorales/hero_diagram.png" style="width: 100%; height: auto; margin: auto" alt="Overview image of the Chamber Ensemble Generator." /> </figure> <p>As we mentioned, the Chamber Ensemble Generator (CEG) is a set of two structured generative models that work together to create new chamber ensemble performances of four-part <a href="https://en.wikipedia.org/wiki/Chorale">chorales</a> in the style of <a href="https://en.wikipedia.org/wiki/List_of_chorale_harmonisations_by_Johann_Sebastian_Bach">J.S. Bach</a>.</p> <p>As seen in the figure above, the constituent models in the CEG are two previous Magenta models: <a href="https://magenta.tensorflow.org/coconet">Coconet</a> and <a href="https://magenta.tensorflow.org/midi-ddsp">MIDI-DDSP</a>. Coconet is a generative model of notes, creating a set of four-instrument music pieces (“note sequences”), harmonized in the style of a Bach Chorale. Each of these four note sequences is then individually synthesized by MIDI-DDSP. MIDI-DDSP is a generative synthesis model that uses <a href="https://magenta.tensorflow.org/ddsp">Differentiable Digital Signal Processing (DDSP)</a> that turns note sequences into realistic audio that can sound like a number of different instruments (e.g., violin, bassoon, or french horn).</p> <p>It’s important to note that the CEG is built on <em>structured</em> generative models, i.e., models that have interpretable intermediate representations. On the one hand, this structure leads to a very opinionated view of music. The CEG is limited in ways that other generative music models are not; it cannot generate all styles of music, like a rock and roll ensemble for example. It can only generate chorales. However, many generative music models are notoriously “black boxes,” whose internal structures are difficult to interpret. By being built on a modular set of structured models, the internals of the CEG are easy to understand and modify. This also allows us to create a dataset with many types of annotations that would be tedious or impossible to acquire with other types of generative models (such as annotations of the velocity and vibrato applied to each individual note in a performance). In the next section, we will showcase how these interpretable structures can be used to mitigate biases of these generative models.</p>  <h1 id="the-cocochorales-dataset">The CocoChorales Dataset</h1> <p>CocoChorales is a dataset of 240,000 examples totalling over 1,400 hours of mixture data. The Chamber Ensemble Generator (CEG) was used to create CocoChorales by sampling from the CEG’s two constituent generative models, Coconet and MIDI-DDSP. Using the CEG in this way is an example of dataset “amplification,” whereby a generative model trained on a small dataset is used to produce a much larger dataset. In this case, we are amplifying two very small datasets: Coconet is trained on the <a href="https://github.com/czhuang/JSB-Chorales-dataset">J.S. Bach Chorales Dataset</a>, which contains 382 examples, and MIDI-DDSP is trained on <a href="https://labsites.rochester.edu/air/projects/URMP.html">URMP</a>, which contains only 44 examples. But, using the CEG, we were able to generate 240,000 examples!</p> <p>CocoChorales has examples performed by 13 different instruments (violin, viola, cello, double bass, flute, oboe, clarinet, bassoon, saxophone, trumpet, french horn, trombone, and tuba) organized into 4 different types of ensembles: a string ensemble, a brass ensemble, a woodwind ensemble, and a random ensemble (see the <a href="https://magenta.tensorflow.org/datasets/cocochorales">CocoChorales dataset page</a> for more info). Each example contains an audio mixture, audio for each source, aligned MIDI, instrument labels, fundamental frequency (f<sub>0</sub>) for each instrument, notewise performance characteristics (e.g., vibrato, loudness, brightness etc of each note), and raw synthesis parameters.</p> <figure style="text-align: center;"> <img src="assets/cocochorales/f0_distributions.png" style="width: 50%; height: auto; margin: auto" alt="Fundamental frequencies (f0's) histograms showing that we are able to correct for a bias in the model." /> </figure> <p>What’s cool about using the structured models in the CEG, is that because the system is modular, it is easy to interpret the output of the intermediate steps of the internal CEG models. For example, the MIDI-DDSP model we used tended to produce performances that were oftentimes out of tune and skewed sharp (i.e., frequency of a note being played was often slightly higher than the “proper” tuned frequency of the note in the piece, according to a <a href="https://en.wikipedia.org/wiki/12_equal_temperament">12-TET scale</a>). This is visualized by the orange histogram in the above image (labeled “w/o pitch aug”), which shows how in or out of tune each note is once every 4ms (here, 0.0 means perfectly “in tune”). We were able to correct for this systematic bias by directly adjusting the f<sub>0</sub> curves output by the synthesis generation module of the MIDI-DDSP model, as shown by the blue histogram (labeled “w/ pitch aug”), which shows a distribution that is more centered on 0.0 in the figure above. This level of control is hard to achieve with black box generative models, and a big reason why we’re very excited about using the structured models in the CEG.</p> <h1 id="downloading-the-dataset">Downloading the Dataset</h1> <p>We’re really excited to see what the research community can do with the CocoChorales dataset. Further details on the dataset can be found <a href="https://magenta.tensorflow.org/datasets/cocochorales">here</a>. Instructions on how to download the dataset can be found at <a href="https://github.com/lukewys/chamber-ensemble-generator#dataset-download">this Github link</a>.</p> <p>If you want to learn more about either project, please see our <a href="https://arxiv.org/abs/2209.14458">arXiv paper</a>. The code for the Chamber Ensemble Generator is available <a href="https://github.com/lukewys/chamber-ensemble-generator">here</a> and usage instructions are <a href="https://github.com/lukewys/chamber-ensemble-generator/blob/master/data_pipeline.md">here</a>. If you use the Chamber Ensemble Generator or CocoChorales in a research publication, we kindly ask that you use the following bibtex entry to cite it:</p> <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>@article{wu2022chamber, title = {The Chamber Ensemble Generator: Limitless High-Quality MIR Data via Generative Modeling}, author = {Wu, Yusong and Gardner, Josh and Manilow, Ethan and Simon, Ian and Hawthorne, Curtis and Engel, Jesse}, journal={arXiv preprint arXiv:2209.14458}, year = {2022}, } </code></pre></div></div> </div> </article> </section> </div> <footer> <div class="footer-content"> <div class="logo"> <a href="https://ai.google/" target="_blank" rel="noopener" title="Google AI"> <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 157.2 40.88"><defs><style>.cls-1{fill:none;}.cls-2{fill:#80868b;}.cls-3{fill:#80868b;}.cls-4{fill:#80868b;}.cls-5{fill:#80868b;}.cls-6{fill:#80868b;}</style></defs><g id="Running_copy" data-name="Running copy"><path class="cls-1" d="M82.91,16.35A4.8,4.8,0,0,0,79.29,18a5.66,5.66,0,0,0-1.49,4,5.53,5.53,0,0,0,1.49,3.94,4.78,4.78,0,0,0,3.62,1.58,4.47,4.47,0,0,0,3.49-1.58A5.7,5.7,0,0,0,87.81,22a5.84,5.84,0,0,0-1.41-4A4.48,4.48,0,0,0,82.91,16.35Z"></path><path class="cls-1" d="M42.8,16.35a4.92,4.92,0,0,0-3.66,1.57,5.49,5.49,0,0,0-1.51,4,5.52,5.52,0,0,0,1.52,4,5,5,0,0,0,7.3,0,5.48,5.48,0,0,0,1.53-4,5.49,5.49,0,0,0-1.51-4A4.93,4.93,0,0,0,42.8,16.35Z"></path><path class="cls-1" d="M62.89,16.35a4.93,4.93,0,0,0-3.67,1.57,5.53,5.53,0,0,0-1.51,4,5.48,5.48,0,0,0,1.53,4,5,5,0,0,0,7.3,0,5.48,5.48,0,0,0,1.53-4,5.49,5.49,0,0,0-1.51-4A4.93,4.93,0,0,0,62.89,16.35Z"></path><path class="cls-1" d="M111,16.82a4.15,4.15,0,0,0-2.12-.54,4.79,4.79,0,0,0-3.32,1.46,4.9,4.9,0,0,0-1.47,3.9l8.2-3.41A2.82,2.82,0,0,0,111,16.82Z"></path><rect class="cls-2" x="94.13" y="3.56" width="4.03" height="26.97"></rect><path class="cls-3" d="M42.8,12.74a9,9,0,0,0-6.53,2.62,8.83,8.83,0,0,0-2.68,6.55,8.84,8.84,0,0,0,2.68,6.56,9.46,9.46,0,0,0,13.07,0A8.83,8.83,0,0,0,52,21.91a8.82,8.82,0,0,0-2.67-6.55A9,9,0,0,0,42.8,12.74Zm3.65,13.15a5,5,0,0,1-7.3,0,5.52,5.52,0,0,1-1.52-4,5.49,5.49,0,0,1,1.51-4,5.06,5.06,0,0,1,7.33,0,5.49,5.49,0,0,1,1.51,4A5.48,5.48,0,0,1,46.45,25.89Z"></path><path class="cls-4" d="M18.89,15.55v3.9h9.32a8.27,8.27,0,0,1-2.12,4.9,9.76,9.76,0,0,1-7.2,2.85,9.75,9.75,0,0,1-7.24-3,10.07,10.07,0,0,1-3-7.33,10.07,10.07,0,0,1,3-7.33,9.75,9.75,0,0,1,7.24-3,9.89,9.89,0,0,1,7,2.78l2.75-2.74a13.63,13.63,0,0,0-9.77-3.93A14.07,14.07,0,0,0,8.71,6.78,13.58,13.58,0,0,0,4.44,16.84,13.56,13.56,0,0,0,8.71,26.9a14.07,14.07,0,0,0,10.18,4.19,13.12,13.12,0,0,0,9.94-4q3.38-3.36,3.37-9.1a12.59,12.59,0,0,0-.2-2.44Z"></path><path class="cls-4" d="M87.53,14.79h-.14a5.64,5.64,0,0,0-2-1.46,6.66,6.66,0,0,0-2.83-.59,8.37,8.37,0,0,0-6.15,2.69A9,9,0,0,0,73.77,22a8.86,8.86,0,0,0,2.64,6.46,8.36,8.36,0,0,0,6.15,2.68A5.87,5.87,0,0,0,87.39,29h.14v1.32a5.63,5.63,0,0,1-1.3,4,4.69,4.69,0,0,1-3.6,1.39,4.34,4.34,0,0,1-2.88-1A5.94,5.94,0,0,1,78,32.44L74.5,33.9a9.43,9.43,0,0,0,3,3.79,8.07,8.07,0,0,0,5.14,1.64,8.61,8.61,0,0,0,6.27-2.39c1.64-1.58,2.45-4,2.45-7.17V13.3H87.53ZM86.4,25.89a4.47,4.47,0,0,1-3.49,1.58,4.78,4.78,0,0,1-3.62-1.58A5.53,5.53,0,0,1,77.8,22a5.66,5.66,0,0,1,1.49-4,4.8,4.8,0,0,1,3.62-1.6A4.48,4.48,0,0,1,86.4,18a5.84,5.84,0,0,1,1.41,4A5.7,5.7,0,0,1,86.4,25.89Z"></path><path class="cls-5" d="M62.89,12.74a9,9,0,0,0-6.53,2.62,8.79,8.79,0,0,0-2.68,6.55,8.8,8.8,0,0,0,2.68,6.56,9.45,9.45,0,0,0,13.06,0,8.8,8.8,0,0,0,2.68-6.56,8.79,8.79,0,0,0-2.68-6.55A9,9,0,0,0,62.89,12.74Zm3.65,13.15a5,5,0,0,1-7.3,0,5.48,5.48,0,0,1-1.53-4,5.53,5.53,0,0,1,1.51-4,5.07,5.07,0,0,1,7.34,0,5.49,5.49,0,0,1,1.51,4A5.48,5.48,0,0,1,66.54,25.89Z"></path><path class="cls-3" d="M109.22,27.47a4.68,4.68,0,0,1-4.45-2.78L117,19.62l-.42-1a11,11,0,0,0-.91-1.81,10.64,10.64,0,0,0-1.49-1.86,7.14,7.14,0,0,0-2.36-1.56,7.73,7.73,0,0,0-3.1-.61,8.27,8.27,0,0,0-6.13,2.57,9.05,9.05,0,0,0-2.52,6.6,8.93,8.93,0,0,0,2.61,6.54,8.74,8.74,0,0,0,6.5,2.64,8.43,8.43,0,0,0,4.69-1.25,10.13,10.13,0,0,0,3-2.82l-3.13-2.08A5.26,5.26,0,0,1,109.22,27.47Zm-3.64-9.73a4.79,4.79,0,0,1,3.32-1.46,4.15,4.15,0,0,1,2.12.54,2.82,2.82,0,0,1,1.29,1.41l-8.2,3.41A4.9,4.9,0,0,1,105.58,17.74Z"></path><path class="cls-6" d="M127.47,30.54h-3.55l9.39-24.9h3.62l9.39,24.9h-3.55l-2.4-6.75H129.9Zm7.58-21L131,20.8h8.28L135.19,9.57Z"></path><path class="cls-6" d="M152.44,30.54h-3.2V5.64h3.2Z"></path></g></svg> </a> </div> <ul> <li> <a href="https://twitter.com/search?q=%23madewithmagenta" target="_blank" rel="noopener"> Twitter </a> </li> <li> <a href="/blog" target="_blank" rel="noopener"> Blog </a> </li> <li> <a href="https://github.com/tensorflow/magenta" target="_blank" rel="noopener"> GitHub </a> </li> <li> <a href="https://www.google.com/policies/privacy/" target="_blank" rel="noopener"> Privacy </a> </li> <li> <a href="https://www.google.com/policies/terms/" target="_blank" rel="noopener"> Terms </a> </li> </ul> </div> </footer> </body> <script src="/js/main.js"></script> </html>