CINXE.COM

<!doctype html> <html class=""> <head> <meta charset="utf-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=no" /> <meta name="description" content="We’re on a journey to advance and democratize artificial intelligence through open source and open science." /> <meta property="fb:app_id" content="1321688464574422" /> <meta name="twitter:card" content="summary_large_image" /> <meta name="twitter:site" content="@huggingface" /> <meta name="twitter:image" content="https://huggingface.co/front/thumbnails/docs/datasets.png" /> <meta property="og:title" content="Create an audio dataset" /> <meta property="og:type" content="website" /> <meta property="og:url" content="https://huggingface.co/docs/datasets/audio_dataset" /> <meta property="og:image" content="https://huggingface.co/front/thumbnails/docs/datasets.png" /> <link rel="stylesheet" href="/front/build/kube-b666922/style.css" /> <link rel="preconnect" href="https://fonts.gstatic.com" /> <link href="https://fonts.googleapis.com/css2?family=Source+Sans+Pro:ital,wght@0,200;0,300;0,400;0,600;0,700;0,900;1,200;1,300;1,400;1,600;1,700;1,900&display=swap" rel="stylesheet" /> <link href="https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:wght@400;600;700&display=swap" rel="stylesheet" /> <link rel="preload" href="https://cdnjs.cloudflare.com/ajax/libs/KaTeX/0.12.0/katex.min.css" as="style" onload="this.onload=null;this.rel='stylesheet'" /> <noscript> <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/KaTeX/0.12.0/katex.min.css" /> </noscript> <script>const guestTheme = document.cookie.match(/theme=(\w+)/)?.[1]; document.documentElement.classList.toggle('dark', guestTheme === 'dark' || ( (!guestTheme || guestTheme === 'system') && window.matchMedia('(prefers-color-scheme: dark)').matches));</script> <link rel="canonical" href="https://huggingface.co/docs/datasets/audio_dataset"> <link rel="alternate" hreflang="en" href="https://huggingface.co/docs/datasets/en/audio_dataset"> <link rel="alternate" hreflang="x-default" href="https://huggingface.co/docs/datasets/audio_dataset"> <title>Create an audio dataset</title> <script defer data-domain="huggingface.co" event-loggedIn="false" src="/js/script.pageview-props.js" ></script> <script> window.plausible = window.plausible || function () { (window.plausible.q = window.plausible.q || []).push(arguments); }; </script> <script> window.hubConfig = {"features":{"signupDisabled":false},"sshGitUrl":"git@hf.co","moonHttpUrl":"https:\/\/huggingface.co","captchaApiKey":"bd5f2066-93dc-4bdd-a64b-a24646ca3859","captchaDisabledOnSignup":true,"datasetViewerPublicUrl":"https:\/\/datasets-server.huggingface.co","stripePublicKey":"pk_live_x2tdjFXBCvXo2FFmMybezpeM00J6gPCAAc","environment":"production","userAgent":"HuggingFace (production)","spacesIframeDomain":"hf.space","spacesApiUrl":"https:\/\/api.hf.space","docSearchKey":"ece5e02e57300e17d152c08056145326e90c4bff3dd07d7d1ae40cf1c8d39cb6","logoDev":{"apiUrl":"https:\/\/img.logo.dev\/","apiKey":"pk_UHS2HZOeRnaSOdDp7jbd5w"}}; </script> <script type="text/javascript" src="https://de5282c3ca0c.edge.sdk.awswaf.com/de5282c3ca0c/526cf06acb0d/challenge.js" defer></script> </head> <body class="flex flex-col min-h-dvh bg-white dark:bg-gray-950 text-black DocBuilderPage"> <div class="flex min-h-dvh flex-col"><div class="SVELTE_HYDRATER contents" data-target="SystemThemeMonitor" data-props="{"isLoggedIn":false}"></div> <div class="SVELTE_HYDRATER contents" data-target="MainHeader" data-props="{"classNames":"","isWide":true,"isZh":false,"isPro":false}"><header class="border-b border-gray-100 "><div class="w-full px-4 flex h-16 items-center"><div class="flex flex-1 items-center"><a class="mr-5 flex flex-none items-center lg:mr-6" href="/"><img alt="Hugging Face's logo" class="w-7 md:mr-2" src="/front/assets/huggingface_logo-noborder.svg"> <span class="hidden whitespace-nowrap text-lg font-bold md:block">Hugging Face</span></a> <div class="relative flex-1 lg:max-w-sm mr-2 sm:mr-4 md:mr-3 xl:mr-6"><input autocomplete="off" class="w-full dark:bg-gray-950 pl-8 form-input-alt h-9 pr-3 focus:shadow-xl " name="" placeholder="Search models, datasets, users..." spellcheck="false" type="text" value=""> <svg class="absolute left-2.5 text-gray-400 top-1/2 transform -translate-y-1/2" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M30 28.59L22.45 21A11 11 0 1 0 21 22.45L28.59 30zM5 14a9 9 0 1 1 9 9a9 9 0 0 1-9-9z" fill="currentColor"></path></svg> </div> <div class="flex flex-none items-center justify-center p-0.5 place-self-stretch lg:hidden"><button class="relative z-40 flex h-6 w-8 items-center justify-center" type="button"><svg width="1em" height="1em" viewBox="0 0 10 10" class="text-xl" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" preserveAspectRatio="xMidYMid meet" fill="currentColor"><path fill-rule="evenodd" clip-rule="evenodd" d="M1.65039 2.9999C1.65039 2.8066 1.80709 2.6499 2.00039 2.6499H8.00039C8.19369 2.6499 8.35039 2.8066 8.35039 2.9999C8.35039 3.1932 8.19369 3.3499 8.00039 3.3499H2.00039C1.80709 3.3499 1.65039 3.1932 1.65039 2.9999ZM1.65039 4.9999C1.65039 4.8066 1.80709 4.6499 2.00039 4.6499H8.00039C8.19369 4.6499 8.35039 4.8066 8.35039 4.9999C8.35039 5.1932 8.19369 5.3499 8.00039 5.3499H2.00039C1.80709 5.3499 1.65039 5.1932 1.65039 4.9999ZM2.00039 6.6499C1.80709 6.6499 1.65039 6.8066 1.65039 6.9999C1.65039 7.1932 1.80709 7.3499 2.00039 7.3499H8.00039C8.19369 7.3499 8.35039 7.1932 8.35039 6.9999C8.35039 6.8066 8.19369 6.6499 8.00039 6.6499H2.00039Z"></path></svg> </button> </div></div> <nav aria-label="Main" class="ml-auto hidden lg:block"><ul class="flex items-center space-x-1.5 2xl:space-x-2"><li class="hover:text-indigo-700"><a class="group flex items-center px-2 py-0.5 dark:text-gray-300 dark:hover:text-gray-100" href="/models"><svg class="mr-1.5 text-gray-400 group-hover:text-indigo-500" style="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path class="uim-quaternary" d="M20.23 7.24L12 12L3.77 7.24a1.98 1.98 0 0 1 .7-.71L11 2.76c.62-.35 1.38-.35 2 0l6.53 3.77c.29.173.531.418.7.71z" opacity=".25" fill="currentColor"></path><path class="uim-tertiary" d="M12 12v9.5a2.09 2.09 0 0 1-.91-.21L4.5 17.48a2.003 2.003 0 0 1-1-1.73v-7.5a2.06 2.06 0 0 1 .27-1.01L12 12z" opacity=".5" fill="currentColor"></path><path class="uim-primary" d="M20.5 8.25v7.5a2.003 2.003 0 0 1-1 1.73l-6.62 3.82c-.275.13-.576.198-.88.2V12l8.23-4.76c.175.308.268.656.27 1.01z" fill="currentColor"></path></svg> Models</a> </li><li class="hover:text-red-700"><a class="group flex items-center px-2 py-0.5 dark:text-gray-300 dark:hover:text-gray-100" href="/datasets"><svg class="mr-1.5 text-gray-400 group-hover:text-red-500" style="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 25 25"><ellipse cx="12.5" cy="5" fill="currentColor" fill-opacity="0.25" rx="7.5" ry="2"></ellipse><path d="M12.5 15C16.6421 15 20 14.1046 20 13V20C20 21.1046 16.6421 22 12.5 22C8.35786 22 5 21.1046 5 20V13C5 14.1046 8.35786 15 12.5 15Z" fill="currentColor" opacity="0.5"></path><path d="M12.5 7C16.6421 7 20 6.10457 20 5V11.5C20 12.6046 16.6421 13.5 12.5 13.5C8.35786 13.5 5 12.6046 5 11.5V5C5 6.10457 8.35786 7 12.5 7Z" fill="currentColor" opacity="0.5"></path><path d="M5.23628 12C5.08204 12.1598 5 12.8273 5 13C5 14.1046 8.35786 15 12.5 15C16.6421 15 20 14.1046 20 13C20 12.8273 19.918 12.1598 19.7637 12C18.9311 12.8626 15.9947 13.5 12.5 13.5C9.0053 13.5 6.06886 12.8626 5.23628 12Z" fill="currentColor"></path></svg> Datasets</a> </li><li class="hover:text-blue-700"><a class="group flex items-center px-2 py-0.5 dark:text-gray-300 dark:hover:text-gray-100" href="/spaces"><svg class="mr-1.5 text-gray-400 group-hover:text-blue-500" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" viewBox="0 0 25 25"><path opacity=".5" d="M6.016 14.674v4.31h4.31v-4.31h-4.31ZM14.674 14.674v4.31h4.31v-4.31h-4.31ZM6.016 6.016v4.31h4.31v-4.31h-4.31Z" fill="currentColor"></path><path opacity=".75" fill-rule="evenodd" clip-rule="evenodd" d="M3 4.914C3 3.857 3.857 3 4.914 3h6.514c.884 0 1.628.6 1.848 1.414a5.171 5.171 0 0 1 7.31 7.31c.815.22 1.414.964 1.414 1.848v6.514A1.914 1.914 0 0 1 20.086 22H4.914A1.914 1.914 0 0 1 3 20.086V4.914Zm3.016 1.102v4.31h4.31v-4.31h-4.31Zm0 12.968v-4.31h4.31v4.31h-4.31Zm8.658 0v-4.31h4.31v4.31h-4.31Zm0-10.813a2.155 2.155 0 1 1 4.31 0 2.155 2.155 0 0 1-4.31 0Z" fill="currentColor"></path><path opacity=".25" d="M16.829 6.016a2.155 2.155 0 1 0 0 4.31 2.155 2.155 0 0 0 0-4.31Z" fill="currentColor"></path></svg> Spaces</a> </li><li class="hover:text-yellow-700 max-xl:hidden"><a class="group flex items-center px-2 py-0.5 dark:text-gray-300 dark:hover:text-gray-100" href="/posts"><svg class="mr-1.5 text-gray-400 group-hover:text-yellow-500 !text-yellow-500" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" viewBox="0 0 12 12" preserveAspectRatio="xMidYMid meet"><path fill="currentColor" fill-rule="evenodd" d="M3.73 2.4A4.25 4.25 0 1 1 6 10.26H2.17l-.13-.02a.43.43 0 0 1-.3-.43l.01-.06a.43.43 0 0 1 .12-.22l.84-.84A4.26 4.26 0 0 1 3.73 2.4Z" clip-rule="evenodd"></path></svg> Posts</a> </li><li class="hover:text-yellow-700"><a class="group flex items-center px-2 py-0.5 dark:text-gray-300 dark:hover:text-gray-100" href="/docs"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="mr-1.5 text-gray-400 group-hover:text-yellow-500" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 16 16"><path d="m2.28 3.7-.3.16a.67.67 0 0 0-.34.58v8.73l.01.04.02.07.01.04.03.06.02.04.02.03.04.06.05.05.04.04.06.04.06.04.08.04.08.02h.05l.07.02h.11l.04-.01.07-.02.03-.01.07-.03.22-.12a5.33 5.33 0 0 1 5.15.1.67.67 0 0 0 .66 0 5.33 5.33 0 0 1 5.33 0 .67.67 0 0 0 1-.58V4.36a.67.67 0 0 0-.34-.5l-.3-.17v7.78a.63.63 0 0 1-.87.59 4.9 4.9 0 0 0-4.35.35l-.65.39a.29.29 0 0 1-.15.04.29.29 0 0 1-.16-.04l-.65-.4a4.9 4.9 0 0 0-4.34-.34.63.63 0 0 1-.87-.59V3.7Z" fill="currentColor" class="dark:opacity-40"></path><path fill-rule="evenodd" clip-rule="evenodd" d="M8 3.1a5.99 5.99 0 0 0-5.3-.43.66.66 0 0 0-.42.62v8.18c0 .45.46.76.87.59a4.9 4.9 0 0 1 4.34.35l.65.39c.05.03.1.04.16.04.05 0 .1-.01.15-.04l.65-.4a4.9 4.9 0 0 1 4.35-.34.63.63 0 0 0 .86-.59V3.3a.67.67 0 0 0-.41-.62 5.99 5.99 0 0 0-5.3.43l-.3.17L8 3.1Zm.73 1.87a.43.43 0 1 0-.86 0v5.48a.43.43 0 0 0 .86 0V4.97Z" fill="currentColor" class="opacity-40 dark:opacity-100"></path><path d="M8.73 4.97a.43.43 0 1 0-.86 0v5.48a.43.43 0 1 0 .86 0V4.96Z" fill="currentColor" class="dark:opacity-40"></path></svg> Docs</a> </li><li class="hover:text-green-700"><a class="group flex items-center px-2 py-0.5 dark:text-gray-300 dark:hover:text-gray-100" href="/enterprise"><svg class="mr-1.5 text-gray-400 group-hover:text-green-500" xmlns="http://www.w3.org/2000/svg" fill="none" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 33 27"><path fill="currentColor" fill-rule="evenodd" d="M13.5.7a8.7 8.7 0 0 0-7.7 5.7L1 20.6c-1 3.1.9 5.7 4.1 5.7h15c3.3 0 6.8-2.6 7.8-5.7l4.6-14.2c1-3.1-.8-5.7-4-5.7h-15Zm1.1 5.7L9.8 20.3h9.8l1-3.1h-5.8l.8-2.5h4.8l1.1-3h-4.8l.8-2.3H23l1-3h-9.5Z" clip-rule="evenodd"></path></svg> Enterprise</a> </li> <li><a class="group flex items-center px-2 py-0.5 dark:text-gray-300 dark:hover:text-gray-100" href="/pricing">Pricing </a></li> <li><div class="relative group"> <button class="px-2 py-0.5 hover:text-gray-500 dark:hover:text-gray-600 flex items-center " type="button"> <svg class=" text-gray-500 w-5 group-hover:text-gray-400 dark:text-gray-300 dark:group-hover:text-gray-100" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" viewBox="0 0 32 18" preserveAspectRatio="xMidYMid meet"><path fill-rule="evenodd" clip-rule="evenodd" d="M14.4504 3.30221C14.4504 2.836 14.8284 2.45807 15.2946 2.45807H28.4933C28.9595 2.45807 29.3374 2.836 29.3374 3.30221C29.3374 3.76842 28.9595 4.14635 28.4933 4.14635H15.2946C14.8284 4.14635 14.4504 3.76842 14.4504 3.30221Z" fill="currentColor"></path><path fill-rule="evenodd" clip-rule="evenodd" d="M14.4504 9.00002C14.4504 8.53382 14.8284 8.15588 15.2946 8.15588H28.4933C28.9595 8.15588 29.3374 8.53382 29.3374 9.00002C29.3374 9.46623 28.9595 9.84417 28.4933 9.84417H15.2946C14.8284 9.84417 14.4504 9.46623 14.4504 9.00002Z" fill="currentColor"></path><path fill-rule="evenodd" clip-rule="evenodd" d="M14.4504 14.6978C14.4504 14.2316 14.8284 13.8537 15.2946 13.8537H28.4933C28.9595 13.8537 29.3374 14.2316 29.3374 14.6978C29.3374 15.164 28.9595 15.542 28.4933 15.542H15.2946C14.8284 15.542 14.4504 15.164 14.4504 14.6978Z" fill="currentColor"></path><path fill-rule="evenodd" clip-rule="evenodd" d="M1.94549 6.87377C2.27514 6.54411 2.80962 6.54411 3.13928 6.87377L6.23458 9.96907L9.32988 6.87377C9.65954 6.54411 10.194 6.54411 10.5237 6.87377C10.8533 7.20343 10.8533 7.73791 10.5237 8.06756L6.23458 12.3567L1.94549 8.06756C1.61583 7.73791 1.61583 7.20343 1.94549 6.87377Z" fill="currentColor"></path></svg> </button> </div></li> <li><hr class="h-5 w-0.5 border-none bg-gray-100 dark:bg-gray-800"></li> <li><a class="block cursor-pointer whitespace-nowrap px-2 py-0.5 hover:text-gray-500 dark:text-gray-300 dark:hover:text-gray-100" href="/login">Log In </a></li> <li><a class="whitespace-nowrap rounded-full border border-transparent bg-gray-900 px-3 py-1 leading-none text-white hover:border-black hover:bg-white hover:text-black" href="/join">Sign Up </a></li></ul></nav></div></header></div> <div class="SVELTE_HYDRATER contents" data-target="SSOBanner" data-props="{}"></div> <main class="flex flex-1 flex-col"><div class="relative lg:flex" id="hf-doc-container"><div class="sticky top-0 z-20 self-start"><div class="SVELTE_HYDRATER contents" data-target="SideMenu" data-props="{"chapters":[{"title":"Get started","isExpanded":true,"sections":[{"title":"🤗 Datasets","isExpanded":true,"id":"index","url":"/docs/datasets/index"},{"title":"Quickstart","isExpanded":true,"id":"quickstart","url":"/docs/datasets/quickstart"},{"title":"Installation","isExpanded":true,"id":"installation","url":"/docs/datasets/installation"}]},{"title":"Tutorials","isExpanded":true,"sections":[{"title":"Overview","isExpanded":true,"id":"tutorial","url":"/docs/datasets/tutorial"},{"title":"Load a dataset from the Hub","isExpanded":true,"id":"load_hub","url":"/docs/datasets/load_hub"},{"title":"Know your dataset","isExpanded":true,"id":"access","url":"/docs/datasets/access"},{"title":"Preprocess","isExpanded":true,"id":"use_dataset","url":"/docs/datasets/use_dataset"},{"title":"Create a dataset","isExpanded":true,"id":"create_dataset","url":"/docs/datasets/create_dataset"},{"title":"Share a dataset to the Hub","isExpanded":true,"id":"upload_dataset","url":"/docs/datasets/upload_dataset"}]},{"title":"How-to guides","isExpanded":true,"sections":[{"title":"Overview","isExpanded":true,"id":"how_to","url":"/docs/datasets/how_to"},{"title":"General usage","isExpanded":true,"sections":[{"title":"Load","isExpanded":true,"id":"loading","url":"/docs/datasets/loading"},{"title":"Process","isExpanded":true,"id":"process","url":"/docs/datasets/process"},{"title":"Stream","isExpanded":true,"id":"stream","url":"/docs/datasets/stream"},{"title":"Use with PyTorch","isExpanded":true,"id":"use_with_pytorch","url":"/docs/datasets/use_with_pytorch"},{"title":"Use with TensorFlow","isExpanded":true,"id":"use_with_tensorflow","url":"/docs/datasets/use_with_tensorflow"},{"title":"Use with NumPy","isExpanded":true,"id":"use_with_numpy","url":"/docs/datasets/use_with_numpy"},{"title":"Use with JAX","isExpanded":true,"id":"use_with_jax","url":"/docs/datasets/use_with_jax"},{"title":"Use with Pandas","isExpanded":true,"id":"use_with_pandas","url":"/docs/datasets/use_with_pandas"},{"title":"Use with Polars","isExpanded":true,"id":"use_with_polars","url":"/docs/datasets/use_with_polars"},{"title":"Use with PyArrow","isExpanded":true,"id":"use_with_pyarrow","url":"/docs/datasets/use_with_pyarrow"},{"title":"Use with Spark","isExpanded":true,"id":"use_with_spark","url":"/docs/datasets/use_with_spark"},{"title":"Cache management","isExpanded":true,"id":"cache","url":"/docs/datasets/cache"},{"title":"Cloud storage","isExpanded":true,"id":"filesystems","url":"/docs/datasets/filesystems"},{"title":"Search index","isExpanded":true,"id":"faiss_es","url":"/docs/datasets/faiss_es"},{"title":"CLI","isExpanded":true,"id":"cli","url":"/docs/datasets/cli"},{"title":"Troubleshooting","isExpanded":true,"id":"troubleshoot","url":"/docs/datasets/troubleshoot"}]},{"title":"Audio","isExpanded":true,"sections":[{"title":"Load audio data","isExpanded":true,"id":"audio_load","url":"/docs/datasets/audio_load"},{"title":"Process audio data","isExpanded":true,"id":"audio_process","url":"/docs/datasets/audio_process"},{"title":"Create an audio dataset","isExpanded":true,"id":"audio_dataset","url":"/docs/datasets/audio_dataset"}]},{"title":"Vision","isExpanded":true,"sections":[{"title":"Load image data","isExpanded":true,"id":"image_load","url":"/docs/datasets/image_load"},{"title":"Process image data","isExpanded":true,"id":"image_process","url":"/docs/datasets/image_process"},{"title":"Create an image dataset","isExpanded":true,"id":"image_dataset","url":"/docs/datasets/image_dataset"},{"title":"Depth estimation","isExpanded":true,"id":"depth_estimation","url":"/docs/datasets/depth_estimation"},{"title":"Image classification","isExpanded":true,"id":"image_classification","url":"/docs/datasets/image_classification"},{"title":"Semantic segmentation","isExpanded":true,"id":"semantic_segmentation","url":"/docs/datasets/semantic_segmentation"},{"title":"Object detection","isExpanded":true,"id":"object_detection","url":"/docs/datasets/object_detection"},{"title":"Load video data","isExpanded":true,"id":"video_load","url":"/docs/datasets/video_load"},{"title":"Create a video dataset","isExpanded":true,"id":"video_dataset","url":"/docs/datasets/video_dataset"}]},{"title":"Text","isExpanded":true,"sections":[{"title":"Load text data","isExpanded":true,"id":"nlp_load","url":"/docs/datasets/nlp_load"},{"title":"Process text data","isExpanded":true,"id":"nlp_process","url":"/docs/datasets/nlp_process"}]},{"title":"Tabular","isExpanded":true,"sections":[{"title":"Load tabular data","isExpanded":true,"id":"tabular_load","url":"/docs/datasets/tabular_load"}]},{"title":"Dataset repository","isExpanded":true,"sections":[{"title":"Share","isExpanded":true,"id":"share","url":"/docs/datasets/share"},{"title":"Create a dataset card","isExpanded":true,"id":"dataset_card","url":"/docs/datasets/dataset_card"},{"title":"Structure your repository","isExpanded":true,"id":"repository_structure","url":"/docs/datasets/repository_structure"},{"title":"Create a dataset loading script","isExpanded":true,"id":"dataset_script","url":"/docs/datasets/dataset_script"}]}]},{"title":"Conceptual guides","isExpanded":true,"sections":[{"title":"Datasets 🤝 Arrow","isExpanded":true,"id":"about_arrow","url":"/docs/datasets/about_arrow"},{"title":"The cache","isExpanded":true,"id":"about_cache","url":"/docs/datasets/about_cache"},{"title":"Dataset or IterableDataset","isExpanded":true,"id":"about_mapstyle_vs_iterable","url":"/docs/datasets/about_mapstyle_vs_iterable"},{"title":"Dataset features","isExpanded":true,"id":"about_dataset_features","url":"/docs/datasets/about_dataset_features"},{"title":"Build and load","isExpanded":true,"id":"about_dataset_load","url":"/docs/datasets/about_dataset_load"},{"title":"Batch mapping","isExpanded":true,"id":"about_map_batch","url":"/docs/datasets/about_map_batch"}]},{"title":"Reference","isExpanded":true,"sections":[{"title":"Main classes","isExpanded":true,"id":"package_reference/main_classes","url":"/docs/datasets/package_reference/main_classes"},{"title":"Builder classes","isExpanded":true,"id":"package_reference/builder_classes","url":"/docs/datasets/package_reference/builder_classes"},{"title":"Loading methods","isExpanded":true,"id":"package_reference/loading_methods","url":"/docs/datasets/package_reference/loading_methods"},{"title":"Table Classes","isExpanded":true,"id":"package_reference/table_classes","url":"/docs/datasets/package_reference/table_classes"},{"title":"Utilities","isExpanded":true,"id":"package_reference/utilities","url":"/docs/datasets/package_reference/utilities"}]}],"chapterId":"audio_dataset","docType":"docs","isLoggedIn":false,"lang":"en","langs":["en"],"library":"datasets","theme":"light","version":"v3.3.2","versions":[{"version":"main"},{"version":"v3.3.2"},{"version":"v3.3.1"},{"version":"v3.3.0"},{"version":"v3.2.0"},{"version":"v3.1.0"},{"version":"v3.0.2"},{"version":"v3.0.1"},{"version":"v3.0.0"},{"version":"v2.21.0"},{"version":"v2.20.0"},{"version":"v2.19.0"},{"version":"v2.18.0"},{"version":"v2.17.1"},{"version":"v2.17.0"},{"version":"v2.16.1"},{"version":"v2.16.0"},{"version":"v2.15.0"},{"version":"v2.14.7"},{"version":"v2.14.6"},{"version":"v2.14.5"},{"version":"v2.14.4"},{"version":"v2.14.3"},{"version":"v2.14.2"},{"version":"v2.14.1"},{"version":"v2.14.0"},{"version":"v2.13.2"},{"version":"v2.13.1"},{"version":"v2.13.0"},{"version":"v2.12.0"},{"version":"v2.11.0"},{"version":"v2.10.0"},{"version":"v2.9.0"},{"version":"v2.8.0"},{"version":"v2.7.1"},{"version":"v2.7.0"},{"version":"v2.6.2"},{"version":"v2.6.1"},{"version":"v2.6.0"},{"version":"v2.5.2"},{"version":"v2.5.1"},{"version":"v2.5.0"},{"version":"v2.4.0"},{"version":"v2.3.2"},{"version":"v2.3.1"},{"version":"v2.3.0"},{"version":"v2.2.1"},{"version":"v2.2.0"},{"version":"v2.1.0"},{"version":"v2.0.0"},{"sphinx":true,"version":"v1.18.3"},{"sphinx":true,"version":"v1.18.2"},{"sphinx":true,"version":"v1.18.1"},{"sphinx":true,"version":"v1.18.0"},{"sphinx":true,"version":"v1.17.0"},{"sphinx":true,"version":"v1.16.1"},{"sphinx":true,"version":"v1.16.0"},{"sphinx":true,"version":"v1.15.1"},{"sphinx":true,"version":"v1.15.0"},{"sphinx":true,"version":"v1.14.0"},{"sphinx":true,"version":"v1.13.3"},{"sphinx":true,"version":"v1.13.2"},{"sphinx":true,"version":"v1.13.1"},{"sphinx":true,"version":"v1.13.0"},{"sphinx":true,"version":"v1.12.1"},{"sphinx":true,"version":"v1.12.0"},{"sphinx":true,"version":"v1.11.0"},{"sphinx":true,"version":"v1.10.2"},{"sphinx":true,"version":"v1.10.1"},{"sphinx":true,"version":"v1.10.0"},{"sphinx":true,"version":"v1.9.0"},{"sphinx":true,"version":"v1.8.0"},{"sphinx":true,"version":"v1.7.0"},{"sphinx":true,"version":"v1.6.2"},{"sphinx":true,"version":"v1.6.1"},{"sphinx":true,"version":"v1.6.0"},{"sphinx":true,"version":"v1.5.0"},{"sphinx":true,"version":"v1.4.1"},{"sphinx":true,"version":"v1.4.0"},{"sphinx":true,"version":"v1.3.0"},{"sphinx":true,"version":"v1.2.1"},{"sphinx":true,"version":"v1.2.0"},{"sphinx":true,"version":"v1.1.3"},{"sphinx":true,"version":"v1.1.2"},{"sphinx":true,"version":"v1.1.1"},{"sphinx":true,"version":"v1.1.0"},{"sphinx":true,"version":"v1.0.2"},{"sphinx":true,"version":"v1.0.1"},{"sphinx":true,"version":"v1.0.0"},{"sphinx":true,"version":"v0.4.0"},{"sphinx":true,"version":"v0.3.0"}],"title":"Create an audio dataset"}"> <div class="z-2 w-full flex-none lg:flex lg:h-dvh lg:w-[270px] lg:flex-col 2xl:w-[300px] false"><div class="shadow-alternate flex h-auto w-full items-center rounded-b-xl border-b bg-white py-2 text-lg leading-tight lg:hidden"> <div class="flex flex-1 cursor-pointer flex-col justify-center self-stretch pl-6"><p class="text-sm text-gray-400 first-letter:capitalize">Datasets documentation </p> <div class="mr-2 flex items-center"><p class="font-semibold">Create an audio dataset</p> <svg class="text-xl false" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path d="M16.293 9.293L12 13.586L7.707 9.293l-1.414 1.414L12 16.414l5.707-5.707z" fill="currentColor"></path></svg></div></div> <button class="hover:shadow-alternate group ml-auto mr-6 inline-flex flex-none cursor-pointer rounded-xl border p-2"><svg class="text-gray-500 group-hover:text-gray-700" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M30 28.59L22.45 21A11 11 0 1 0 21 22.45L28.59 30zM5 14a9 9 0 1 1 9 9a9 9 0 0 1-9-9z" fill="currentColor"></path></svg></button></div> <div class="hidden flex-col justify-between border-b border-r bg-white bg-gradient-to-r p-4 lg:flex from-red-50 to-white dark:from-gray-900 dark:to-gray-950"><div class="group relative mb-2 flex min-w-[50%] items-center self-start text-lg font-bold leading-tight first-letter:capitalize"><div class="mr-1.5 h-1.5 w-1.5 rounded-full bg-red-500 flex-none"></div> <h1>Datasets</h1> <svg class="opacity-50 ml-0.5 flex-none group-hover:opacity-100" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path d="M16.293 9.293L12 13.586L7.707 9.293l-1.414 1.414L12 16.414l5.707-5.707z" fill="currentColor"></path></svg> <select class="absolute inset-0 border-none bg-white text-base opacity-0 outline-none"><option value="/docs">🏡 View all docs</option><option value="/docs/optimum-neuron" >AWS Trainium & Inferentia</option><option value="/docs/accelerate" >Accelerate</option><option value="/docs/sagemaker" >Amazon SageMaker</option><option value="https://argilla-io.github.io/argilla/" >Argilla</option><option value="/docs/autotrain" >AutoTrain</option><option value="/docs/bitsandbytes" >Bitsandbytes</option><option value="/docs/chat-ui" >Chat UI</option><option value="/docs/competitions" >Competitions</option><option value="/docs/dataset-viewer" >Dataset viewer</option><option value="/docs/datasets" selected>Datasets</option><option value="/docs/diffusers" >Diffusers</option><option value="https://distilabel.argilla.io/" >Distilabel</option><option value="/docs/evaluate" >Evaluate</option><option value="https://www.gradio.app/docs/" >Gradio</option><option value="/docs/hub" >Hub</option><option value="/docs/huggingface_hub" >Hub Python Library</option><option value="/docs/hugs" >Hugging Face Generative AI Services (HUGS)</option><option value="/docs/huggingface.js" >Huggingface.js</option><option value="/docs/api-inference" >Inference API (serverless)</option><option value="/docs/inference-endpoints" >Inference Endpoints (dedicated)</option><option value="/docs/leaderboards" >Leaderboards</option><option value="/docs/lighteval" >Lighteval</option><option value="/docs/optimum" >Optimum</option><option value="/docs/peft" >PEFT</option><option value="/docs/safetensors" >Safetensors</option><option value="https://sbert.net/" >Sentence Transformers</option><option value="/docs/trl" >TRL</option><option value="/tasks" >Tasks</option><option value="/docs/text-embeddings-inference" >Text Embeddings Inference</option><option value="/docs/text-generation-inference" >Text Generation Inference</option><option value="/docs/tokenizers" >Tokenizers</option><option value="/docs/transformers" >Transformers</option><option value="/docs/transformers.js" >Transformers.js</option><option value="/docs/smolagents" >smolagents</option><option value="/docs/timm" >timm</option></select></div> <button class="shadow-alternate mb-2 flex w-full items-center rounded-full border bg-white px-2 py-1 text-left text-sm text-gray-400 ring-indigo-200 hover:bg-indigo-50 hover:ring-2 dark:border-gray-700 dark:ring-yellow-600 dark:hover:bg-gray-900 dark:hover:text-yellow-500"><svg class="flex-none mr-1.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M30 28.59L22.45 21A11 11 0 1 0 21 22.45L28.59 30zM5 14a9 9 0 1 1 9 9a9 9 0 0 1-9-9z" fill="currentColor"></path></svg> <div>Search documentation</div> </button> <div class="flex items-center"> <select class="form-input !mt-0 mr-1 !w-20 rounded !border border-gray-200 p-1 text-xs uppercase dark:!text-gray-400"><option value="0" >main</option><option value="1" selected>v3.3.2</option><option value="2" >v3.2.0</option><option value="3" >v3.1.0</option><option value="4" >v3.0.2</option><option value="5" >v2.21.0</option><option value="6" >v2.20.0</option><option value="7" >v2.19.0</option><option value="8" >v2.18.0</option><option value="9" >v2.17.1</option><option value="10" >v2.16.1</option><option value="11" >v2.15.0</option><option value="12" >v2.14.7</option><option value="13" >v2.13.2</option><option value="14" >v2.12.0</option><option value="15" >v2.11.0</option><option value="16" >v2.10.0</option><option value="17" >v2.9.0</option><option value="18" >v2.8.0</option><option value="19" >v2.7.1</option><option value="20" >v2.6.2</option><option value="21" >v2.5.2</option><option value="22" >v2.4.0</option><option value="23" >v2.3.2</option><option value="24" >v2.2.1</option><option value="25" >v2.1.0</option><option value="26" >v2.0.0</option><option value="27" >v1.18.3</option><option value="28" >v1.17.0</option><option value="29" >v1.16.1</option><option value="30" >v1.15.1</option><option value="31" >v1.14.0</option><option value="32" >v1.13.3</option><option value="33" >v1.12.1</option><option value="34" >v1.11.0</option><option value="35" >v1.10.2</option><option value="36" >v1.9.0</option><option value="37" >v1.8.0</option><option value="38" >v1.7.0</option><option value="39" >v1.6.2</option><option value="40" >v1.5.0</option><option value="41" >v1.4.1</option><option value="42" >v1.3.0</option><option value="43" >v1.2.1</option><option value="44" >v1.1.3</option><option value="45" >v1.0.2</option><option value="46" >v0.4.0</option><option value="47" >v0.3.0</option></select> <select class="form-input mr-1 rounded border-gray-200 p-1 text-xs dark:!text-gray-400 !w-12 !mt-0 !border"><option value="en" selected>EN</option></select> <div class="relative inline-block "> <button class="rounded-full border border-gray-100 pl-2 py-1 pr-2.5 flex items-center text-sm text-gray-500 bg-white hover:bg-yellow-50 hover:border-yellow-200 dark:hover:bg-gray-800 dark:hover:border-gray-950 dark:border-gray-800 " type="button"> <svg class=" text-yellow-500" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24" fill="currentColor"><path d="M6.05 4.14l-.39-.39a.993.993 0 0 0-1.4 0l-.01.01a.984.984 0 0 0 0 1.4l.39.39c.39.39 1.01.39 1.4 0l.01-.01a.984.984 0 0 0 0-1.4zM3.01 10.5H1.99c-.55 0-.99.44-.99.99v.01c0 .55.44.99.99.99H3c.56.01 1-.43 1-.98v-.01c0-.56-.44-1-.99-1zm9-9.95H12c-.56 0-1 .44-1 .99v.96c0 .55.44.99.99.99H12c.56.01 1-.43 1-.98v-.97c0-.55-.44-.99-.99-.99zm7.74 3.21c-.39-.39-1.02-.39-1.41-.01l-.39.39a.984.984 0 0 0 0 1.4l.01.01c.39.39 1.02.39 1.4 0l.39-.39a.984.984 0 0 0 0-1.4zm-1.81 15.1l.39.39a.996.996 0 1 0 1.41-1.41l-.39-.39a.993.993 0 0 0-1.4 0c-.4.4-.4 1.02-.01 1.41zM20 11.49v.01c0 .55.44.99.99.99H22c.55 0 .99-.44.99-.99v-.01c0-.55-.44-.99-.99-.99h-1.01c-.55 0-.99.44-.99.99zM12 5.5c-3.31 0-6 2.69-6 6s2.69 6 6 6s6-2.69 6-6s-2.69-6-6-6zm-.01 16.95H12c.55 0 .99-.44.99-.99v-.96c0-.55-.44-.99-.99-.99h-.01c-.55 0-.99.44-.99.99v.96c0 .55.44.99.99.99zm-7.74-3.21c.39.39 1.02.39 1.41 0l.39-.39a.993.993 0 0 0 0-1.4l-.01-.01a.996.996 0 0 0-1.41 0l-.39.39c-.38.4-.38 1.02.01 1.41z"></path></svg> </button> </div> <a href="https://github.com/huggingface/datasets" class="group ml-auto text-xs text-gray-500 hover:text-gray-700 hover:underline dark:hover:text-gray-300"><svg class="inline-block text-gray-500 group-hover:text-gray-700 dark:group-hover:text-gray-300 mr-1.5 -mt-1 w-4 h-4" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1.03em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 250"><path d="M128.001 0C57.317 0 0 57.307 0 128.001c0 56.554 36.676 104.535 87.535 121.46c6.397 1.185 8.746-2.777 8.746-6.158c0-3.052-.12-13.135-.174-23.83c-35.61 7.742-43.124-15.103-43.124-15.103c-5.823-14.795-14.213-18.73-14.213-18.73c-11.613-7.944.876-7.78.876-7.78c12.853.902 19.621 13.19 19.621 13.19c11.417 19.568 29.945 13.911 37.249 10.64c1.149-8.272 4.466-13.92 8.127-17.116c-28.431-3.236-58.318-14.212-58.318-63.258c0-13.975 5-25.394 13.188-34.358c-1.329-3.224-5.71-16.242 1.24-33.874c0 0 10.749-3.44 35.21 13.121c10.21-2.836 21.16-4.258 32.038-4.307c10.878.049 21.837 1.47 32.066 4.307c24.431-16.56 35.165-13.12 35.165-13.12c6.967 17.63 2.584 30.65 1.255 33.873c8.207 8.964 13.173 20.383 13.173 34.358c0 49.163-29.944 59.988-58.447 63.157c4.591 3.972 8.682 11.762 8.682 23.704c0 17.126-.148 30.91-.148 35.126c0 3.407 2.304 7.398 8.792 6.14C219.37 232.5 256 184.537 256 128.002C256 57.307 198.691 0 128.001 0zm-80.06 182.34c-.282.636-1.283.827-2.194.39c-.929-.417-1.45-1.284-1.15-1.922c.276-.655 1.279-.838 2.205-.399c.93.418 1.46 1.293 1.139 1.931zm6.296 5.618c-.61.566-1.804.303-2.614-.591c-.837-.892-.994-2.086-.375-2.66c.63-.566 1.787-.301 2.626.591c.838.903 1 2.088.363 2.66zm4.32 7.188c-.785.545-2.067.034-2.86-1.104c-.784-1.138-.784-2.503.017-3.05c.795-.547 2.058-.055 2.861 1.075c.782 1.157.782 2.522-.019 3.08zm7.304 8.325c-.701.774-2.196.566-3.29-.49c-1.119-1.032-1.43-2.496-.726-3.27c.71-.776 2.213-.558 3.315.49c1.11 1.03 1.45 2.505.701 3.27zm9.442 2.81c-.31 1.003-1.75 1.459-3.199 1.033c-1.448-.439-2.395-1.613-2.103-2.626c.301-1.01 1.747-1.484 3.207-1.028c1.446.436 2.396 1.602 2.095 2.622zm10.744 1.193c.036 1.055-1.193 1.93-2.715 1.95c-1.53.034-2.769-.82-2.786-1.86c0-1.065 1.202-1.932 2.733-1.958c1.522-.03 2.768.818 2.768 1.868zm10.555-.405c.182 1.03-.875 2.088-2.387 2.37c-1.485.271-2.861-.365-3.05-1.386c-.184-1.056.893-2.114 2.376-2.387c1.514-.263 2.868.356 3.061 1.403z" fill="currentColor"></path></svg> </a></div></div> <nav class="hidden flex-auto lg:flex bottom-0 left-0 w-full flex-col overflow-y-auto border-r px-4 pb-16 pt-3 text-[0.95rem] lg:w-[270px] 2xl:w-[300px]"> <div class="group flex cursor-pointer items-center pl-2 text-[0.8rem] font-semibold uppercase leading-9 hover:text-gray-700 dark:hover:text-gray-300 ml-0"><div class="flex after:absolute after:right-4 after:text-gray-500 group-hover:after:content-['▶'] after:rotate-90 after:transform"><span><span class="inline-block space-x-1 leading-5"><span>Get started</span> </span></span> </div></div> <div class="flex flex-col"><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/datasets/index" id="index">🤗 Datasets </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/datasets/quickstart" id="quickstart">Quickstart </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/datasets/installation" id="installation">Installation </a> </div> <div class="group flex cursor-pointer items-center pl-2 text-[0.8rem] font-semibold uppercase leading-9 hover:text-gray-700 dark:hover:text-gray-300 ml-0"><div class="flex after:absolute after:right-4 after:text-gray-500 group-hover:after:content-['▶'] after:rotate-90 after:transform"><span><span class="inline-block space-x-1 leading-5"><span>Tutorials</span> </span></span> </div></div> <div class="flex flex-col"><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/datasets/tutorial" id="tutorial">Overview </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/datasets/load_hub" id="load_hub">Load a dataset from the Hub </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/datasets/access" id="access">Know your dataset </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/datasets/use_dataset" id="use_dataset">Preprocess </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/datasets/create_dataset" id="create_dataset">Create a dataset </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/datasets/upload_dataset" id="upload_dataset">Share a dataset to the Hub </a> </div> <div class="group flex cursor-pointer items-center pl-2 text-[0.8rem] font-semibold uppercase leading-9 hover:text-gray-700 dark:hover:text-gray-300 ml-0"><div class="flex after:absolute after:right-4 after:text-gray-500 group-hover:after:content-['▶'] after:rotate-90 after:transform"><span><span class="inline-block space-x-1 leading-5"><span>How-to guides</span> </span></span> </div></div> <div class="flex flex-col"><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/datasets/how_to" id="how_to">Overview </a> <div class="group flex cursor-pointer items-center pl-2 text-[0.8rem] font-semibold uppercase leading-9 hover:text-gray-700 dark:hover:text-gray-300 ml-2"><div class="flex after:absolute after:right-4 after:text-gray-500 group-hover:after:content-['▶'] after:rotate-90 after:transform"><span><span class="inline-block space-x-1 leading-5"><span>General usage</span> </span></span> </div></div> <div class="flex flex-col"><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/loading" id="loading">Load </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/process" id="process">Process </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/stream" id="stream">Stream </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/use_with_pytorch" id="use_with_pytorch">Use with PyTorch </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/use_with_tensorflow" id="use_with_tensorflow">Use with TensorFlow </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/use_with_numpy" id="use_with_numpy">Use with NumPy </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/use_with_jax" id="use_with_jax">Use with JAX </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/use_with_pandas" id="use_with_pandas">Use with Pandas </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/use_with_polars" id="use_with_polars">Use with Polars </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/use_with_pyarrow" id="use_with_pyarrow">Use with PyArrow </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/use_with_spark" id="use_with_spark">Use with Spark </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/cache" id="cache">Cache management </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/filesystems" id="filesystems">Cloud storage </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/faiss_es" id="faiss_es">Search index </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/cli" id="cli">CLI </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/troubleshoot" id="troubleshoot">Troubleshooting </a> </div> <div class="group flex cursor-pointer items-center pl-2 text-[0.8rem] font-semibold uppercase leading-9 hover:text-gray-700 dark:hover:text-gray-300 ml-2"><div class="flex after:absolute after:right-4 after:text-gray-500 group-hover:after:content-['▶'] after:rotate-90 after:transform"><span><span class="inline-block space-x-1 leading-5"><span>Audio</span> </span></span> </div></div> <div class="flex flex-col"><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/audio_load" id="audio_load">Load audio data </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/audio_process" id="audio_process">Process audio data </a><a class="rounded-xl bg-gradient-to-br from-black to-gray-900 py-1 pl-2 pr-2 text-white first:mt-1 last:mb-4 dark:from-gray-800 dark:to-gray-900 ml-4" href="/docs/datasets/audio_dataset" id="audio_dataset">Create an audio dataset </a> </div> <div class="group flex cursor-pointer items-center pl-2 text-[0.8rem] font-semibold uppercase leading-9 hover:text-gray-700 dark:hover:text-gray-300 ml-2"><div class="flex after:absolute after:right-4 after:text-gray-500 group-hover:after:content-['▶'] after:rotate-90 after:transform"><span><span class="inline-block space-x-1 leading-5"><span>Vision</span> </span></span> </div></div> <div class="flex flex-col"><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/image_load" id="image_load">Load image data </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/image_process" id="image_process">Process image data </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/image_dataset" id="image_dataset">Create an image dataset </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/depth_estimation" id="depth_estimation">Depth estimation </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/image_classification" id="image_classification">Image classification </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/semantic_segmentation" id="semantic_segmentation">Semantic segmentation </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/object_detection" id="object_detection">Object detection </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/video_load" id="video_load">Load video data </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/video_dataset" id="video_dataset">Create a video dataset </a> </div> <div class="group flex cursor-pointer items-center pl-2 text-[0.8rem] font-semibold uppercase leading-9 hover:text-gray-700 dark:hover:text-gray-300 ml-2"><div class="flex after:absolute after:right-4 after:text-gray-500 group-hover:after:content-['▶'] after:rotate-90 after:transform"><span><span class="inline-block space-x-1 leading-5"><span>Text</span> </span></span> </div></div> <div class="flex flex-col"><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/nlp_load" id="nlp_load">Load text data </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/nlp_process" id="nlp_process">Process text data </a> </div> <div class="group flex cursor-pointer items-center pl-2 text-[0.8rem] font-semibold uppercase leading-9 hover:text-gray-700 dark:hover:text-gray-300 ml-2"><div class="flex after:absolute after:right-4 after:text-gray-500 group-hover:after:content-['▶'] after:rotate-90 after:transform"><span><span class="inline-block space-x-1 leading-5"><span>Tabular</span> </span></span> </div></div> <div class="flex flex-col"><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/tabular_load" id="tabular_load">Load tabular data </a> </div> <div class="group flex cursor-pointer items-center pl-2 text-[0.8rem] font-semibold uppercase leading-9 hover:text-gray-700 dark:hover:text-gray-300 ml-2"><div class="flex after:absolute after:right-4 after:text-gray-500 group-hover:after:content-['▶'] after:rotate-90 after:transform"><span><span class="inline-block space-x-1 leading-5"><span>Dataset repository</span> </span></span> </div></div> <div class="flex flex-col"><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/share" id="share">Share </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/dataset_card" id="dataset_card">Create a dataset card </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/repository_structure" id="repository_structure">Structure your repository </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/datasets/dataset_script" id="dataset_script">Create a dataset loading script </a> </div> </div> <div class="group flex cursor-pointer items-center pl-2 text-[0.8rem] font-semibold uppercase leading-9 hover:text-gray-700 dark:hover:text-gray-300 ml-0"><div class="flex after:absolute after:right-4 after:text-gray-500 group-hover:after:content-['▶'] after:rotate-90 after:transform"><span><span class="inline-block space-x-1 leading-5"><span>Conceptual guides</span> </span></span> </div></div> <div class="flex flex-col"><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/datasets/about_arrow" id="about_arrow">Datasets 🤝 Arrow </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/datasets/about_cache" id="about_cache">The cache </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/datasets/about_mapstyle_vs_iterable" id="about_mapstyle_vs_iterable">Dataset or IterableDataset </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/datasets/about_dataset_features" id="about_dataset_features">Dataset features </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/datasets/about_dataset_load" id="about_dataset_load">Build and load </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/datasets/about_map_batch" id="about_map_batch">Batch mapping </a> </div> <div class="group flex cursor-pointer items-center pl-2 text-[0.8rem] font-semibold uppercase leading-9 hover:text-gray-700 dark:hover:text-gray-300 ml-0"><div class="flex after:absolute after:right-4 after:text-gray-500 group-hover:after:content-['▶'] after:rotate-90 after:transform"><span><span class="inline-block space-x-1 leading-5"><span>Reference</span> </span></span> </div></div> <div class="flex flex-col"><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/datasets/package_reference/main_classes" id="package_reference/main_classes">Main classes </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/datasets/package_reference/builder_classes" id="package_reference/builder_classes">Builder classes </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/datasets/package_reference/loading_methods" id="package_reference/loading_methods">Loading methods </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/datasets/package_reference/table_classes" id="package_reference/table_classes">Table Classes </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/datasets/package_reference/utilities" id="package_reference/utilities">Utilities </a> </div></nav></div></div></div> <div class="z-1 min-w-0 flex-1"> <div class="px-6 pt-6 md:px-12 md:pb-16 md:pt-16"><div class="max-w-4xl mx-auto mb-10"><div class="relative overflow-hidden rounded-xl bg-gradient-to-br from-orange-300/10 px-4 py-5 ring-1 ring-orange-100/70 md:px-6 md:py-8"><img alt="Hugging Face's logo" class="absolute -bottom-6 -right-6 w-28 -rotate-45 md:hidden" src="/front/assets/huggingface_logo-noborder.svg"> <div class="mb-2 text-2xl font-bold dark:text-gray-200 md:mb-0">Join the Hugging Face community</div> <p class="mb-4 text-lg text-gray-400 dark:text-gray-300 md:mb-8">and get access to the augmented documentation experience </p> <div class="mb-8 hidden space-y-4 md:block xl:flex xl:space-x-6 xl:space-y-0"><div class="flex items-center"><div class="mr-3 flex h-9 w-9 flex-none items-center justify-center rounded-lg bg-gradient-to-br from-indigo-100 to-indigo-100/20 dark:to-indigo-100"><svg class="text-indigo-400 group-hover:text-indigo-500" style="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path class="uim-quaternary" d="M20.23 7.24L12 12L3.77 7.24a1.98 1.98 0 0 1 .7-.71L11 2.76c.62-.35 1.38-.35 2 0l6.53 3.77c.29.173.531.418.7.71z" opacity=".25" fill="currentColor"></path><path class="uim-tertiary" d="M12 12v9.5a2.09 2.09 0 0 1-.91-.21L4.5 17.48a2.003 2.003 0 0 1-1-1.73v-7.5a2.06 2.06 0 0 1 .27-1.01L12 12z" opacity=".5" fill="currentColor"></path><path class="uim-primary" d="M20.5 8.25v7.5a2.003 2.003 0 0 1-1 1.73l-6.62 3.82c-.275.13-.576.198-.88.2V12l8.23-4.76c.175.308.268.656.27 1.01z" fill="currentColor"></path></svg></div> <div class="text-smd leading-tight text-gray-500 dark:text-gray-300 xl:max-w-[200px] 2xl:text-base">Collaborate on models, datasets and Spaces </div></div> <div class="flex items-center"><div class="mr-3 flex h-9 w-9 flex-none items-center justify-center rounded-lg bg-gradient-to-br from-orange-100 to-orange-100/20 dark:to-orange-50"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" class="text-xl text-yellow-400" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path d="M11 15H6l7-14v8h5l-7 14v-8z" fill="currentColor"></path></svg></div> <div class="text-smd leading-tight text-gray-500 dark:text-gray-300 xl:max-w-[200px] 2xl:text-base">Faster examples with accelerated inference </div></div> <div class="flex items-center"><div class="mr-3 flex h-9 w-9 flex-none items-center justify-center rounded-lg bg-gradient-to-br from-gray-500/10 to-gray-500/5"><svg class="text-gray-400" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M14.9804 3C14.9217 3.0002 14.8631 3.00555 14.8054 3.016C11.622 3.58252 8.76073 5.30669 6.77248 7.85653C4.78422 10.4064 3.80955 13.6016 4.03612 16.8271C4.26268 20.0525 5.67447 23.0801 7.99967 25.327C10.3249 27.5738 13.3991 28.8811 16.6304 28.997C16.7944 29.003 16.9584 28.997 17.1204 28.997C19.2193 28.9984 21.2877 28.4943 23.1507 27.5274C25.0137 26.5605 26.6164 25.1592 27.8234 23.442C27.9212 23.294 27.9783 23.1229 27.9889 22.9458C27.9995 22.7687 27.9633 22.592 27.884 22.4333C27.8046 22.2747 27.6848 22.1397 27.5367 22.0421C27.3887 21.9444 27.2175 21.8875 27.0404 21.877C25.0426 21.7017 23.112 21.0693 21.3976 20.0288C19.6832 18.9884 18.231 17.5676 17.1533 15.8764C16.0756 14.1852 15.4011 12.2688 15.1822 10.2754C14.9632 8.28193 15.2055 6.26484 15.8904 4.38C15.9486 4.22913 15.97 4.06652 15.9527 3.90572C15.9354 3.74492 15.8799 3.59059 15.7909 3.45557C15.7019 3.32055 15.5819 3.20877 15.4409 3.12952C15.2999 3.05028 15.142 3.00587 14.9804 3Z" fill="currentColor"></path></svg></div> <div class="text-smd leading-tight text-gray-500 dark:text-gray-300 xl:max-w-[200px] 2xl:text-base">Switch between documentation themes </div></div></div> <div class="flex items-center space-x-2.5"><a href="/join"><button class="rounded-lg bg-white bg-gradient-to-br from-gray-100/20 to-gray-200/60 px-5 py-1.5 font-semibold text-gray-700 shadow-sm ring-1 ring-gray-300/60 hover:to-gray-100/70 hover:ring-gray-300/30 active:shadow-inner">Sign Up</button></a> <p class="text-gray-500 dark:text-gray-300">to get started</p></div></div></div> <div class="prose-doc prose relative mx-auto max-w-4xl break-words"> <link href="/docs/datasets/v3.3.2/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload"> <link rel="modulepreload" href="/docs/datasets/v3.3.2/en/_app/immutable/entry/start.bbd56df5.js"> <link rel="modulepreload" href="/docs/datasets/v3.3.2/en/_app/immutable/chunks/scheduler.bdbef820.js"> <link rel="modulepreload" href="/docs/datasets/v3.3.2/en/_app/immutable/chunks/singletons.f24e6ffc.js"> <link rel="modulepreload" href="/docs/datasets/v3.3.2/en/_app/immutable/chunks/index.8a885b74.js"> <link rel="modulepreload" href="/docs/datasets/v3.3.2/en/_app/immutable/chunks/paths.2f1c449c.js"> <link rel="modulepreload" href="/docs/datasets/v3.3.2/en/_app/immutable/entry/app.b1a61c37.js"> <link rel="modulepreload" href="/docs/datasets/v3.3.2/en/_app/immutable/chunks/index.c0aea24a.js"> <link rel="modulepreload" href="/docs/datasets/v3.3.2/en/_app/immutable/nodes/0.10770c08.js"> <link rel="modulepreload" href="/docs/datasets/v3.3.2/en/_app/immutable/chunks/each.e59479a4.js"> <link rel="modulepreload" href="/docs/datasets/v3.3.2/en/_app/immutable/nodes/9.40a1afd6.js"> <link rel="modulepreload" href="/docs/datasets/v3.3.2/en/_app/immutable/chunks/Tip.31005f7d.js"> <link rel="modulepreload" href="/docs/datasets/v3.3.2/en/_app/immutable/chunks/CodeBlock.6ccca92e.js"> <link rel="modulepreload" href="/docs/datasets/v3.3.2/en/_app/immutable/chunks/EditOnGithub.725ee0c1.js"><meta name="hf:doc:metadata" content="{"title":"Create an audio dataset","local":"create-an-audio-dataset","sections":[{"title":"Local files","local":"local-files","sections":[],"depth":2},{"title":"AudioFolder","local":"audiofolder","sections":[],"depth":2},{"title":"(Legacy) Loading script","local":"legacy-loading-script","sections":[{"title":"Create a dataset builder class","local":"create-a-dataset-builder-class","sections":[{"title":"Multiple configurations","local":"multiple-configurations","sections":[],"depth":4}],"depth":3},{"title":"Add dataset metadata","local":"add-dataset-metadata","sections":[],"depth":3},{"title":"Download and define the dataset splits","local":"download-and-define-the-dataset-splits","sections":[],"depth":3},{"title":"Generate the dataset","local":"generate-the-dataset","sections":[],"depth":3},{"title":"Upload the dataset to the Hub","local":"upload-the-dataset-to-the-hub","sections":[],"depth":3},{"title":"(Advanced) Extract TAR archives locally","local":"advanced-extract-tar-archives-locally","sections":[{"title":"Download and define the dataset splits","local":"download-and-define-the-dataset-splits","sections":[],"depth":4},{"title":"Generate the dataset","local":"generate-the-dataset","sections":[],"depth":4}],"depth":3}],"depth":2}],"depth":1}"> <p></p> <h1 class="relative group"><a id="create-an-audio-dataset" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#create-an-audio-dataset"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Create an audio dataset</span></h1> <p data-svelte-h="svelte-12uzys7">You can share a dataset with your team or with anyone in the community by creating a dataset repository on the Hugging Face Hub:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset dataset = load_dataset(<span class="hljs-string">"<username>/my_dataset"</span>)</pre></div> <p data-svelte-h="svelte-37xnq4">There are several methods for creating and sharing an audio dataset:</p> <ul data-svelte-h="svelte-14edvfa"><li><p>Create an audio dataset from local files in python with <a href="/docs/datasets/v3.3.2/en/package_reference/main_classes#datasets.Dataset.push_to_hub">Dataset.push_to_hub()</a>. This is an easy way that requires only a few steps in python.</p></li> <li><p>Create an audio dataset repository with the <code>AudioFolder</code> builder. This is a no-code solution for quickly creating an audio dataset with several thousand audio files.</p></li></ul> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-ztusze">You can control access to your dataset by requiring users to share their contact information first. Check out the <a href="https://huggingface.co/docs/hub/datasets-gated" rel="nofollow">Gated datasets</a> guide for more information about how to enable this feature on the Hub.</p></div> <h2 class="relative group"><a id="local-files" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#local-files"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Local files</span></h2> <p data-svelte-h="svelte-tdidsm">You can load your own dataset using the paths to your audio files. Use the <a href="/docs/datasets/v3.3.2/en/package_reference/main_classes#datasets.Dataset.cast_column">cast_column()</a> function to take a column of audio file paths, and cast it to the <a href="/docs/datasets/v3.3.2/en/package_reference/main_classes#datasets.Audio">Audio</a> feature:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><span class="hljs-meta">>>> </span>audio_dataset = Dataset.from_dict({<span class="hljs-string">"audio"</span>: [<span class="hljs-string">"path/to/audio_1"</span>, <span class="hljs-string">"path/to/audio_2"</span>, ..., <span class="hljs-string">"path/to/audio_n"</span>]}).cast_column(<span class="hljs-string">"audio"</span>, Audio()) <span class="hljs-meta">>>> </span>audio_dataset[<span class="hljs-number">0</span>][<span class="hljs-string">"audio"</span>] {<span class="hljs-string">'array'</span>: array([ <span class="hljs-number">0.</span> , <span class="hljs-number">0.00024414</span>, -<span class="hljs-number">0.00024414</span>, ..., -<span class="hljs-number">0.00024414</span>, <span class="hljs-number">0.</span> , <span class="hljs-number">0.</span> ], dtype=float32), <span class="hljs-string">'path'</span>: <span class="hljs-string">'path/to/audio_1'</span>, <span class="hljs-string">'sampling_rate'</span>: <span class="hljs-number">16000</span>}</pre></div> <p data-svelte-h="svelte-15r76mj">Then upload the dataset to the Hugging Face Hub using <a href="/docs/datasets/v3.3.2/en/package_reference/main_classes#datasets.Dataset.push_to_hub">Dataset.push_to_hub()</a>:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="">audio_dataset.push_to_hub(<span class="hljs-string">"<username>/my_dataset"</span>)</pre></div> <p data-svelte-h="svelte-132x6mi">This will create a dataset repository containing your audio dataset:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><span class="hljs-title">my_dataset</span>/ ├── <span class="hljs-type">README</span>.md └── <span class="hljs-class"><span class="hljs-keyword">data</span>/</span> └── train-<span class="hljs-number">00000</span>-<span class="hljs-keyword">of</span>-<span class="hljs-number">00001</span>.parquet</pre></div> <h2 class="relative group"><a id="audiofolder" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#audiofolder"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>AudioFolder</span></h2> <p data-svelte-h="svelte-56y3x5">The <code>AudioFolder</code> is a dataset builder designed to quickly load an audio dataset with several thousand audio files without requiring you to write any code. Any additional information about your dataset - such as transcription, speaker accent, or speaker intent - is automatically loaded by <code>AudioFolder</code> as long as you include this information in a metadata file (<code>metadata.csv</code>/<code>metadata.jsonl</code>).</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-176117l">💡 Take a look at the <a href="repository_structure#split-pattern-hierarchy">Split pattern hierarchy</a> to learn more about how <code>AudioFolder</code> creates dataset splits based on your dataset repository structure.</p></div> <p data-svelte-h="svelte-1ifhvjg">Create a dataset repository on the Hugging Face Hub and upload your dataset directory following the <code>AudioFolder</code> structure:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><span class="hljs-title">my_dataset</span>/ ├── <span class="hljs-type">README</span>.md ├── metadata.csv └── <span class="hljs-class"><span class="hljs-keyword">data</span>/</span></pre></div> <p data-svelte-h="svelte-1ftf9en">The <code>data</code> folder can be any name you want.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-p5lfwp">It can be helpful to store your metadata as a <code>jsonl</code> file if the data columns contain a more complex format (like a list of floats) to avoid parsing errors or reading complex values as strings.</p></div> <p data-svelte-h="svelte-1pccrhs">The metadata file should include a <code>file_name</code> column to link an audio file to it’s metadata:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="">file_name,transcription data/first_audio_file<span class="hljs-selector-class">.mp3</span>,znowu się duch z ciał<span class="hljs-selector-tag">em</span> zrośnie w młodocianej wstaniesz wiosnie <span class="hljs-selector-tag">i</span> możesz skutkiem tych leków umierać wstawać wiek wieków dalej tam były przestrogi jak siekać głowę jak nogi data/second_audio_file<span class="hljs-selector-class">.mp3</span>,już u źwierzyńca podwojów król zasiada przy nim książęta <span class="hljs-selector-tag">i</span> panowie rada <span class="hljs-selector-tag">a</span> gdzie wzniosły krążył ganek rycerze obok kochanek król skinął palcem zaczę<span class="hljs-selector-tag">to</span> igrzysko data/third_audio_file<span class="hljs-selector-class">.mp3</span>,pewnie kędyś w obłędzie ubite minęły szlaki zaczekajmy dzień jaki poślemy szukać wszędzie dziś jutro pewnie <span class="hljs-selector-tag">b</span>ędzie posłali wszędzie sługi czekali dzień <span class="hljs-selector-tag">i</span> drugi gdy nic nie doczekali z <span class="hljs-selector-tag">p</span>łaczem chcą jechać dali</pre></div> <p data-svelte-h="svelte-1jikwvx">Then you can store your dataset in a directory structure like this:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><span class="hljs-title">metadata</span>.csv <span class="hljs-class"><span class="hljs-keyword">data</span>/first_audio_file.mp3</span> <span class="hljs-class"><span class="hljs-keyword">data</span>/second_audio_file.mp3</span> <span class="hljs-class"><span class="hljs-keyword">data</span>/third_audio_file.mp3</span> </pre></div> <p data-svelte-h="svelte-1hh95j5">Users can now load your dataset and the associated metadata by specifying <code>audiofolder</code> in <a href="/docs/datasets/v3.3.2/en/package_reference/loading_methods#datasets.load_dataset">load_dataset()</a> and the dataset directory in <code>data_dir</code>:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset <span class="hljs-meta">>>> </span>dataset = load_dataset(<span class="hljs-string">"audiofolder"</span>, data_dir=<span class="hljs-string">"/path/to/data"</span>) <span class="hljs-meta">>>> </span>dataset[<span class="hljs-string">"train"</span>][<span class="hljs-number">0</span>] {<span class="hljs-string">'audio'</span>: {<span class="hljs-string">'path'</span>: <span class="hljs-string">'/path/to/extracted/audio/first_audio_file.mp3'</span>, <span class="hljs-string">'array'</span>: array([ <span class="hljs-number">0.00088501</span>, <span class="hljs-number">0.0012207</span> , <span class="hljs-number">0.00131226</span>, ..., -<span class="hljs-number">0.00045776</span>, -<span class="hljs-number">0.00054932</span>, -<span class="hljs-number">0.00054932</span>], dtype=float32), <span class="hljs-string">'sampling_rate'</span>: <span class="hljs-number">16000</span>}, <span class="hljs-string">'transcription'</span>: <span class="hljs-string">'znowu się duch z ciałem zrośnie w młodocianej wstaniesz wiosnie i możesz skutkiem tych leków umierać wstawać wiek wieków dalej tam były przestrogi jak siekać głowę jak nogi'</span> }</pre></div> <p data-svelte-h="svelte-gvbmuf">You can also use <code>audiofolder</code> to load datasets involving multiple splits. To do so, your dataset directory might have the following structure:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="">data<span class="hljs-regexp">/train/</span>first_train_audio_file.mp3 data<span class="hljs-regexp">/train/</span>second_train_audio_file.mp3 data<span class="hljs-regexp">/test/</span>first_test_audio_file.mp3 data<span class="hljs-regexp">/test/</span>second_test_audio_file.mp3 </pre></div> <div class="course-tip course-tip-orange bg-gradient-to-br dark:bg-gradient-to-r before:border-orange-500 dark:before:border-orange-800 from-orange-50 dark:from-gray-900 to-white dark:to-gray-950 border border-orange-50 text-orange-700 dark:text-gray-400"><p data-svelte-h="svelte-1x0iz2q">Note that if audio files are located not right next to a metadata file, <code>file_name</code> column should be a full relative path to an audio file, not just its filename.</p></div> <p data-svelte-h="svelte-1h25o8p">For audio datasets that don’t have any associated metadata, <code>AudioFolder</code> automatically infers the class labels of the dataset based on the directory name. It might be useful for audio classification tasks. Your dataset directory might look like:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="">data<span class="hljs-regexp">/train/</span>electronic/<span class="hljs-number">01</span>.mp3 data<span class="hljs-regexp">/train/</span>punk/<span class="hljs-number">01</span>.mp3 data<span class="hljs-regexp">/test/</span>electronic/<span class="hljs-number">09</span>.mp3 data<span class="hljs-regexp">/test/</span>punk/<span class="hljs-number">09</span>.mp3</pre></div> <p data-svelte-h="svelte-5cy7rr">Load the dataset with <code>AudioFolder</code>, and it will create a <code>label</code> column from the directory name (language id):</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset <span class="hljs-meta">>>> </span>dataset = load_dataset(<span class="hljs-string">"audiofolder"</span>, data_dir=<span class="hljs-string">"/path/to/data"</span>) <span class="hljs-meta">>>> </span>dataset[<span class="hljs-string">"train"</span>][<span class="hljs-number">0</span>] {<span class="hljs-string">'audio'</span>: {<span class="hljs-string">'path'</span>: <span class="hljs-string">'/path/to/electronic/01.mp3'</span>, <span class="hljs-string">'array'</span>: array([ <span class="hljs-number">3.9714024e-07</span>, <span class="hljs-number">7.3031038e-07</span>, <span class="hljs-number">7.5640685e-07</span>, ..., -<span class="hljs-number">1.1963668e-01</span>, -<span class="hljs-number">1.1681189e-01</span>, -<span class="hljs-number">1.1244172e-01</span>], dtype=float32), <span class="hljs-string">'sampling_rate'</span>: <span class="hljs-number">44100</span>}, <span class="hljs-string">'label'</span>: <span class="hljs-number">0</span> <span class="hljs-comment"># "electronic"</span> } <span class="hljs-meta">>>> </span>dataset[<span class="hljs-string">"train"</span>][-<span class="hljs-number">1</span>] {<span class="hljs-string">'audio'</span>: {<span class="hljs-string">'path'</span>: <span class="hljs-string">'/path/to/punk/01.mp3'</span>, <span class="hljs-string">'array'</span>: array([<span class="hljs-number">0.15237972</span>, <span class="hljs-number">0.13222949</span>, <span class="hljs-number">0.10627693</span>, ..., <span class="hljs-number">0.41940814</span>, <span class="hljs-number">0.37578005</span>, <span class="hljs-number">0.33717662</span>], dtype=float32), <span class="hljs-string">'sampling_rate'</span>: <span class="hljs-number">44100</span>}, <span class="hljs-string">'label'</span>: <span class="hljs-number">1</span> <span class="hljs-comment"># "punk"</span> }</pre></div> <div class="course-tip course-tip-orange bg-gradient-to-br dark:bg-gradient-to-r before:border-orange-500 dark:before:border-orange-800 from-orange-50 dark:from-gray-900 to-white dark:to-gray-950 border border-orange-50 text-orange-700 dark:text-gray-400"><p data-svelte-h="svelte-qmtrul">If all audio files are contained in a single directory or if they are not on the same level of directory structure, <code>label</code> column won’t be added automatically. If you need it, set <code>drop_labels=False</code> explicitly.</p></div> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-1v982nl">Some audio datasets, like those found in <a href="https://www.kaggle.com/competitions/kaggle-pog-series-s01e02/overview" rel="nofollow">Kaggle competitions</a>, have separate metadata files for each split. Provided the metadata features are the same for each split, <code>audiofolder</code> can be used to load all splits at once. If the metadata features differ across each split, you should load them with separate <code>load_dataset()</code> calls.</p></div> <h2 class="relative group"><a id="legacy-loading-script" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#legacy-loading-script"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>(Legacy) Loading script</span></h2> <p data-svelte-h="svelte-bpk8cf">Write a dataset loading script to manually create a dataset. It defines a dataset’s splits and configurations, and handles downloading and generating the dataset examples. The script should have the same name as your dataset folder or repository:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><span class="hljs-title">my_dataset</span>/ ├── <span class="hljs-type">README</span>.md ├── my_dataset.py └── <span class="hljs-class"><span class="hljs-keyword">data</span>/</span></pre></div> <p data-svelte-h="svelte-11idm66">The <code>data</code> folder can be any name you want, it doesn’t have to be <code>data</code>. This folder is optional, unless you’re hosting your dataset on the Hub.</p> <p data-svelte-h="svelte-kjjd8v">This directory structure allows your dataset to be loaded in one line:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset <span class="hljs-meta">>>> </span>dataset = load_dataset(<span class="hljs-string">"path/to/my_dataset"</span>)</pre></div> <p data-svelte-h="svelte-byoqih">This guide will show you how to create a dataset loading script for audio datasets, which is a bit different from <a class="underline decoration-green-400 decoration-2 font-semibold" href="./dataset_script">creating a loading script for text datasets</a>. Audio datasets are commonly stored in <code>tar.gz</code> archives which requires a particular approach to support streaming mode. While streaming is not required, we highly encourage implementing streaming support in your audio dataset because users without a lot of disk space can use your dataset without downloading it. Learn more about streaming in the <a href="./stream">Stream</a> guide!</p> <p data-svelte-h="svelte-heufux">Here is an example using TAR archives:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="">my_dataset/ ├── README<span class="hljs-selector-class">.md</span> ├── my_dataset<span class="hljs-selector-class">.py</span> └── data/ ├── train<span class="hljs-selector-class">.tar</span><span class="hljs-selector-class">.gz</span> ├── test<span class="hljs-selector-class">.tar</span><span class="hljs-selector-class">.gz</span> └── metadata.csv</pre></div> <p data-svelte-h="svelte-24jgoi">In addition to learning how to create a streamable dataset, you’ll also learn how to:</p> <ul data-svelte-h="svelte-1oiags4"><li>Create a dataset builder class.</li> <li>Create dataset configurations.</li> <li>Add dataset metadata.</li> <li>Download and define the dataset splits.</li> <li>Generate the dataset.</li> <li>Upload the dataset to the Hub.</li></ul> <p data-svelte-h="svelte-1igujnr">The best way to learn is to open up an existing audio dataset loading script, like <a href="https://huggingface.co/datasets/vivos/blob/main/vivos.py" rel="nofollow">Vivos</a>, and follow along!</p> <div class="course-tip course-tip-orange bg-gradient-to-br dark:bg-gradient-to-r before:border-orange-500 dark:before:border-orange-800 from-orange-50 dark:from-gray-900 to-white dark:to-gray-950 border border-orange-50 text-orange-700 dark:text-gray-400"><p data-svelte-h="svelte-redz3b">This guide shows how to process audio data stored in TAR archives - the most frequent case for audio datasets. Check out <a href="https://huggingface.co/datasets/PolyAI/minds14/blob/main/minds14.py" rel="nofollow">minds14</a> dataset for an example of an audio script which uses ZIP archives.</p></div> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-vvzcy8">To help you get started, we created a loading script <a href="https://github.com/huggingface/datasets/blob/main/templates/new_dataset_script.py" rel="nofollow">template</a> you can copy and use as a starting point!</p></div> <h3 class="relative group"><a id="create-a-dataset-builder-class" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#create-a-dataset-builder-class"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Create a dataset builder class</span></h3> <p data-svelte-h="svelte-fptr0t"><a href="/docs/datasets/v3.3.2/en/package_reference/builder_classes#datasets.GeneratorBasedBuilder">GeneratorBasedBuilder</a> is the base class for datasets generated from a dictionary generator. Within this class, there are three methods to help create your dataset:</p> <ul data-svelte-h="svelte-1pfe1y4"><li><code>_info</code> stores information about your dataset like its description, license, and features.</li> <li><code>_split_generators</code> downloads the dataset and defines its splits.</li> <li><code>_generate_examples</code> generates the dataset’s samples containing the audio data and other features specified in <code>info</code> for each split.</li></ul> <p data-svelte-h="svelte-104wz1h">Start by creating your dataset class as a subclass of <a href="/docs/datasets/v3.3.2/en/package_reference/builder_classes#datasets.GeneratorBasedBuilder">GeneratorBasedBuilder</a> and add the three methods. Don’t worry about filling in each of these methods yet, you’ll develop those over the next few sections:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><span class="hljs-keyword">class</span> <span class="hljs-title class_">VivosDataset</span>(datasets.GeneratorBasedBuilder): <span class="hljs-string">"""VIVOS is a free Vietnamese speech corpus consisting of 15 hours of recording speech prepared for Vietnamese Automatic Speech Recognition task."""</span> <span class="hljs-keyword">def</span> <span class="hljs-title function_">_info</span>(<span class="hljs-params">self</span>): <span class="hljs-keyword">def</span> <span class="hljs-title function_">_split_generators</span>(<span class="hljs-params">self, dl_manager</span>): <span class="hljs-keyword">def</span> <span class="hljs-title function_">_generate_examples</span>(<span class="hljs-params">self, prompts_path, path_to_clips, audio_files</span>): </pre></div> <h4 class="relative group"><a id="multiple-configurations" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#multiple-configurations"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Multiple configurations</span></h4> <p data-svelte-h="svelte-b0v130">In some cases, a dataset may have more than one configuration. For example, <a href="https://huggingface.co/datasets/indonesian-nlp/librivox-indonesia" rel="nofollow">LibriVox Indonesia</a> dataset has several configurations corresponding to different languages.</p> <p data-svelte-h="svelte-k9u9mf">To create different configurations, use the <a href="/docs/datasets/v3.3.2/en/package_reference/builder_classes#datasets.BuilderConfig">BuilderConfig</a> class to create a subclass of your dataset. The only required parameter is the <code>name</code> of the configuration, which must be passed to the configuration’s superclass <code>__init__()</code>. Otherwise, you can specify any custom parameters you want in your configuration class.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><span class="hljs-keyword">class</span> <span class="hljs-title class_">LibriVoxIndonesiaConfig</span>(datasets.BuilderConfig): <span class="hljs-string">"""BuilderConfig for LibriVoxIndonesia."""</span> <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, name, version, **kwargs</span>): self.language = kwargs.pop(<span class="hljs-string">"language"</span>, <span class="hljs-literal">None</span>) self.release_date = kwargs.pop(<span class="hljs-string">"release_date"</span>, <span class="hljs-literal">None</span>) self.num_clips = kwargs.pop(<span class="hljs-string">"num_clips"</span>, <span class="hljs-literal">None</span>) self.num_speakers = kwargs.pop(<span class="hljs-string">"num_speakers"</span>, <span class="hljs-literal">None</span>) self.validated_hr = kwargs.pop(<span class="hljs-string">"validated_hr"</span>, <span class="hljs-literal">None</span>) self.total_hr = kwargs.pop(<span class="hljs-string">"total_hr"</span>, <span class="hljs-literal">None</span>) self.size_bytes = kwargs.pop(<span class="hljs-string">"size_bytes"</span>, <span class="hljs-literal">None</span>) self.size_human = size_str(self.size_bytes) description = ( <span class="hljs-string">f"LibriVox-Indonesia speech to text dataset in <span class="hljs-subst">{self.language}</span> released on <span class="hljs-subst">{self.release_date}</span>. "</span> <span class="hljs-string">f"The dataset comprises <span class="hljs-subst">{self.validated_hr}</span> hours of transcribed speech data"</span> ) <span class="hljs-built_in">super</span>(LibriVoxIndonesiaConfig, self).__init__( name=name, version=datasets.Version(version), description=description, **kwargs, )</pre></div> <p data-svelte-h="svelte-1etkb90">Define your configurations in the <code>BUILDER_CONFIGS</code> class variable inside <a href="/docs/datasets/v3.3.2/en/package_reference/builder_classes#datasets.GeneratorBasedBuilder">GeneratorBasedBuilder</a>. In this example, the author imports the languages from a separate <code>release_stats.py</code> <a href="https://huggingface.co/datasets/indonesian-nlp/librivox-indonesia/blob/main/release_stats.py" rel="nofollow">file</a> from their repository, and then loops through each language to create a configuration:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><span class="hljs-keyword">class</span> <span class="hljs-title class_">LibriVoxIndonesia</span>(datasets.GeneratorBasedBuilder): DEFAULT_CONFIG_NAME = <span class="hljs-string">"all"</span> BUILDER_CONFIGS = [ LibriVoxIndonesiaConfig( name=lang, version=STATS[<span class="hljs-string">"version"</span>], language=LANGUAGES[lang], release_date=STATS[<span class="hljs-string">"date"</span>], num_clips=lang_stats[<span class="hljs-string">"clips"</span>], num_speakers=lang_stats[<span class="hljs-string">"users"</span>], total_hr=<span class="hljs-built_in">float</span>(lang_stats[<span class="hljs-string">"totalHrs"</span>]) <span class="hljs-keyword">if</span> lang_stats[<span class="hljs-string">"totalHrs"</span>] <span class="hljs-keyword">else</span> <span class="hljs-literal">None</span>, size_bytes=<span class="hljs-built_in">int</span>(lang_stats[<span class="hljs-string">"size"</span>]) <span class="hljs-keyword">if</span> lang_stats[<span class="hljs-string">"size"</span>] <span class="hljs-keyword">else</span> <span class="hljs-literal">None</span>, ) <span class="hljs-keyword">for</span> lang, lang_stats <span class="hljs-keyword">in</span> STATS[<span class="hljs-string">"locales"</span>].items() ]</pre></div> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-10n7x1">Typically, users need to specify a configuration to load in <a href="/docs/datasets/v3.3.2/en/package_reference/loading_methods#datasets.load_dataset">load_dataset()</a>, otherwise a <code>ValueError</code> is raised. You can avoid this by setting a default dataset configuration to load in <code>DEFAULT_CONFIG_NAME</code>.</p></div> <p data-svelte-h="svelte-1jvzcsc">Now if users want to load the Balinese (<code>bal</code>) configuration, they can use the configuration name:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset <span class="hljs-meta">>>> </span>dataset = load_dataset(<span class="hljs-string">"indonesian-nlp/librivox-indonesia"</span>, <span class="hljs-string">"bal"</span>, split=<span class="hljs-string">"train"</span>)</pre></div> <h3 class="relative group"><a id="add-dataset-metadata" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#add-dataset-metadata"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Add dataset metadata</span></h3> <p data-svelte-h="svelte-13e3ls7">Adding information about your dataset helps users to learn more about it. This information is stored in the <a href="/docs/datasets/v3.3.2/en/package_reference/main_classes#datasets.DatasetInfo">DatasetInfo</a> class which is returned by the <code>info</code> method. Users can access this information by:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset_builder <span class="hljs-meta">>>> </span>ds_builder = load_dataset_builder(<span class="hljs-string">"vivos"</span>) <span class="hljs-meta">>>> </span>ds_builder.info</pre></div> <p data-svelte-h="svelte-7pkskw">There is a lot of information you can include about your dataset, but some important ones are:</p> <ol data-svelte-h="svelte-n4ned9"><li><code>description</code> provides a concise description of the dataset.</li> <li><code>features</code> specify the dataset column types. Since you’re creating an audio loading script, you’ll need to include the <a href="/docs/datasets/v3.3.2/en/package_reference/main_classes#datasets.Audio">Audio</a> feature and the <code>sampling_rate</code> of the dataset.</li> <li><code>homepage</code> provides a link to the dataset homepage.</li> <li><code>license</code> specify the permissions for using a dataset as defined by the license type.</li> <li><code>citation</code> is a BibTeX citation of the dataset.</li></ol> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-od2b21">You’ll notice a lot of the dataset information is defined earlier in the loading script which can make it easier to read. There are also other <code>~Dataset.Features</code> you can input, so be sure to check out the full list and <a href="./about_dataset_features">features guide</a> for more details.</p></div> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><span class="hljs-keyword">def</span> <span class="hljs-title function_">_info</span>(<span class="hljs-params">self</span>): <span class="hljs-keyword">return</span> datasets.DatasetInfo( description=_DESCRIPTION, features=datasets.Features( { <span class="hljs-string">"speaker_id"</span>: datasets.Value(<span class="hljs-string">"string"</span>), <span class="hljs-string">"path"</span>: datasets.Value(<span class="hljs-string">"string"</span>), <span class="hljs-string">"audio"</span>: datasets.Audio(sampling_rate=<span class="hljs-number">16_000</span>), <span class="hljs-string">"sentence"</span>: datasets.Value(<span class="hljs-string">"string"</span>), } ), supervised_keys=<span class="hljs-literal">None</span>, homepage=_HOMEPAGE, license=_LICENSE, citation=_CITATION, )</pre></div> <h3 class="relative group"><a id="download-and-define-the-dataset-splits" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#download-and-define-the-dataset-splits"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Download and define the dataset splits</span></h3> <p data-svelte-h="svelte-jffxhg">Now that you’ve added some information about your dataset, the next step is to download the dataset and define the splits.</p> <ol data-svelte-h="svelte-gpdiuj"><li><p>Use the <a href="/docs/datasets/v3.3.2/en/package_reference/builder_classes#datasets.DownloadManager.download">download()</a> method to download metadata file at <code>_PROMPTS_URLS</code> and audio TAR archive at <code>_DATA_URL</code>. This method returns the path to the local file/archive. In streaming mode, it doesn’t download the file(s) and just returns a URL to stream the data from. This method accepts:</p> <ul><li>a relative path to a file inside a Hub dataset repository (for example, in the <code>data/</code> folder)</li> <li>a URL to a file hosted somewhere else</li> <li>a (nested) list or dictionary of file names or URLs</li></ul></li> <li><p>After you’ve downloaded the dataset, use the <a href="/docs/datasets/v3.3.2/en/package_reference/builder_classes#datasets.SplitGenerator">SplitGenerator</a> to organize the audio files and sentence prompts in each split. Name each split with a standard name like: <code>Split.TRAIN</code>, <code>Split.TEST</code>, and <code>SPLIT.Validation</code>.</p> <p>In the <code>gen_kwargs</code> parameter, specify the file path to the <code>prompts_path</code> and <code>path_to_clips</code>. For <code>audio_files</code>, you’ll need to use <a href="/docs/datasets/v3.3.2/en/package_reference/builder_classes#datasets.DownloadManager.iter_archive">iter_archive()</a> to iterate over the audio files in the TAR archive. This enables streaming for your dataset. All of these file paths are passed onto the next step where you’ll actually generate the dataset.</p></li></ol> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><span class="hljs-keyword">def</span> <span class="hljs-title function_">_split_generators</span>(<span class="hljs-params">self, dl_manager</span>): <span class="hljs-string">"""Returns SplitGenerators."""</span> prompts_paths = dl_manager.download(_PROMPTS_URLS) archive = dl_manager.download(_DATA_URL) train_dir = <span class="hljs-string">"vivos/train"</span> test_dir = <span class="hljs-string">"vivos/test"</span> <span class="hljs-keyword">return</span> [ datasets.SplitGenerator( name=datasets.Split.TRAIN, gen_kwargs={ <span class="hljs-string">"prompts_path"</span>: prompts_paths[<span class="hljs-string">"train"</span>], <span class="hljs-string">"path_to_clips"</span>: train_dir + <span class="hljs-string">"/waves"</span>, <span class="hljs-string">"audio_files"</span>: dl_manager.iter_archive(archive), }, ), datasets.SplitGenerator( name=datasets.Split.TEST, gen_kwargs={ <span class="hljs-string">"prompts_path"</span>: prompts_paths[<span class="hljs-string">"test"</span>], <span class="hljs-string">"path_to_clips"</span>: test_dir + <span class="hljs-string">"/waves"</span>, <span class="hljs-string">"audio_files"</span>: dl_manager.iter_archive(archive), }, ), ]</pre></div> <div class="course-tip course-tip-orange bg-gradient-to-br dark:bg-gradient-to-r before:border-orange-500 dark:before:border-orange-800 from-orange-50 dark:from-gray-900 to-white dark:to-gray-950 border border-orange-50 text-orange-700 dark:text-gray-400"><p data-svelte-h="svelte-1c9m46t">This implementation does not extract downloaded archives. If you want to extract files after download, you need to additionally use <a href="/docs/datasets/v3.3.2/en/package_reference/builder_classes#datasets.DownloadManager.extract">extract()</a>, see the <a href="#advanced-extract-tar-archives-locally">(Advanced) Extract TAR archives</a> section.</p></div> <h3 class="relative group"><a id="generate-the-dataset" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#generate-the-dataset"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Generate the dataset</span></h3> <p data-svelte-h="svelte-86keqr">The last method in the <a href="/docs/datasets/v3.3.2/en/package_reference/builder_classes#datasets.GeneratorBasedBuilder">GeneratorBasedBuilder</a> class actually generates the samples in the dataset. It yields a dataset according to the structure specified in <code>features</code> from the <code>info</code> method. As you can see, <code>generate_examples</code> accepts the <code>prompts_path</code>, <code>path_to_clips</code>, and <code>audio_files</code> from the previous method as arguments.</p> <p data-svelte-h="svelte-1xx8z2i">Files inside TAR archives are accessed and yielded sequentially. This means you need to have the metadata associated with the audio files in the TAR file in hand first so you can yield it with its corresponding audio file.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="">examples = {} <span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(prompts_path, encoding=<span class="hljs-string">"utf-8"</span>) <span class="hljs-keyword">as</span> f: <span class="hljs-keyword">for</span> row <span class="hljs-keyword">in</span> f: data = row.strip().split(<span class="hljs-string">" "</span>, <span class="hljs-number">1</span>) speaker_id = data[<span class="hljs-number">0</span>].split(<span class="hljs-string">"_"</span>)[<span class="hljs-number">0</span>] audio_path = <span class="hljs-string">"/"</span>.join([path_to_clips, speaker_id, data[<span class="hljs-number">0</span>] + <span class="hljs-string">".wav"</span>]) examples[audio_path] = { <span class="hljs-string">"speaker_id"</span>: speaker_id, <span class="hljs-string">"path"</span>: audio_path, <span class="hljs-string">"sentence"</span>: data[<span class="hljs-number">1</span>], }</pre></div> <p data-svelte-h="svelte-whbg1p">Finally, iterate over files in <code>audio_files</code> and yield them along with their corresponding metadata. <a href="/docs/datasets/v3.3.2/en/package_reference/builder_classes#datasets.DownloadManager.iter_archive">iter_archive()</a> yields a tuple of (<code>path</code>, <code>f</code>) where <code>path</code> is a <strong>relative</strong> path to a file inside TAR archive and <code>f</code> is a file object itself.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="">inside_clips_dir = <span class="hljs-literal">False</span> id_ = <span class="hljs-number">0</span> <span class="hljs-keyword">for</span> path, f <span class="hljs-keyword">in</span> audio_files: <span class="hljs-keyword">if</span> path.startswith(path_to_clips): inside_clips_dir = <span class="hljs-literal">True</span> <span class="hljs-keyword">if</span> path <span class="hljs-keyword">in</span> examples: audio = {<span class="hljs-string">"path"</span>: path, <span class="hljs-string">"bytes"</span>: f.read()} <span class="hljs-keyword">yield</span> id_, {**examples[path], <span class="hljs-string">"audio"</span>: audio} id_ += <span class="hljs-number">1</span> <span class="hljs-keyword">elif</span> inside_clips_dir: <span class="hljs-keyword">break</span></pre></div> <p data-svelte-h="svelte-1cd1abn">Put these two steps together, and the whole <code>_generate_examples</code> method looks like:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><span class="hljs-keyword">def</span> <span class="hljs-title function_">_generate_examples</span>(<span class="hljs-params">self, prompts_path, path_to_clips, audio_files</span>): <span class="hljs-string">"""Yields examples as (key, example) tuples."""</span> examples = {} <span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(prompts_path, encoding=<span class="hljs-string">"utf-8"</span>) <span class="hljs-keyword">as</span> f: <span class="hljs-keyword">for</span> row <span class="hljs-keyword">in</span> f: data = row.strip().split(<span class="hljs-string">" "</span>, <span class="hljs-number">1</span>) speaker_id = data[<span class="hljs-number">0</span>].split(<span class="hljs-string">"_"</span>)[<span class="hljs-number">0</span>] audio_path = <span class="hljs-string">"/"</span>.join([path_to_clips, speaker_id, data[<span class="hljs-number">0</span>] + <span class="hljs-string">".wav"</span>]) examples[audio_path] = { <span class="hljs-string">"speaker_id"</span>: speaker_id, <span class="hljs-string">"path"</span>: audio_path, <span class="hljs-string">"sentence"</span>: data[<span class="hljs-number">1</span>], } inside_clips_dir = <span class="hljs-literal">False</span> id_ = <span class="hljs-number">0</span> <span class="hljs-keyword">for</span> path, f <span class="hljs-keyword">in</span> audio_files: <span class="hljs-keyword">if</span> path.startswith(path_to_clips): inside_clips_dir = <span class="hljs-literal">True</span> <span class="hljs-keyword">if</span> path <span class="hljs-keyword">in</span> examples: audio = {<span class="hljs-string">"path"</span>: path, <span class="hljs-string">"bytes"</span>: f.read()} <span class="hljs-keyword">yield</span> id_, {**examples[path], <span class="hljs-string">"audio"</span>: audio} id_ += <span class="hljs-number">1</span> <span class="hljs-keyword">elif</span> inside_clips_dir: <span class="hljs-keyword">break</span></pre></div> <h3 class="relative group"><a id="upload-the-dataset-to-the-hub" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#upload-the-dataset-to-the-hub"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Upload the dataset to the Hub</span></h3> <p data-svelte-h="svelte-1rbksh3">Once your script is ready, <a href="./dataset_card">create a dataset card</a> and <a href="./share">upload it to the Hub</a>.</p> <p data-svelte-h="svelte-1539djf">Congratulations, you can now load your dataset from the Hub! 🥳</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset <span class="hljs-meta">>>> </span>load_dataset(<span class="hljs-string">"<username>/my_dataset"</span>)</pre></div> <h3 class="relative group"><a id="advanced-extract-tar-archives-locally" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#advanced-extract-tar-archives-locally"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>(Advanced) Extract TAR archives locally</span></h3> <p data-svelte-h="svelte-yog3v">In the example above downloaded archives are not extracted and therefore examples do not contain information about where they are stored locally. To explain how to do the extraction in a way that it also supports streaming, we will briefly go through the <a href="https://huggingface.co/datasets/indonesian-nlp/librivox-indonesia/blob/main/librivox-indonesia.py" rel="nofollow">LibriVox Indonesia</a> loading script.</p> <h4 class="relative group"><a id="download-and-define-the-dataset-splits" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#download-and-define-the-dataset-splits"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Download and define the dataset splits</span></h4> <ol><li data-svelte-h="svelte-vz667v"><p>Use the <a href="/docs/datasets/v3.3.2/en/package_reference/builder_classes#datasets.DownloadManager.download">download()</a> method to download the audio data at <code>_AUDIO_URL</code>.</p></li> <li><p data-svelte-h="svelte-1pmu06u">To extract audio TAR archive locally, use the <a href="/docs/datasets/v3.3.2/en/package_reference/builder_classes#datasets.DownloadManager.extract">extract()</a>. You can use this method only in non-streaming mode (when <code>dl_manager.is_streaming=False</code>). This returns a local path to the extracted archive directory:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="">local_extracted_archive = dl_manager.extract(audio_path) <span class="hljs-keyword">if</span> <span class="hljs-keyword">not</span> dl_manager.is_streaming <span class="hljs-keyword">else</span> <span class="hljs-literal">None</span></pre></div></li> <li data-svelte-h="svelte-1whgndn"><p>Use the <a href="/docs/datasets/v3.3.2/en/package_reference/builder_classes#datasets.DownloadManager.iter_archive">iter_archive()</a> method to iterate over the archive at <code>audio_path</code>, just like in the Vivos example above. <a href="/docs/datasets/v3.3.2/en/package_reference/builder_classes#datasets.DownloadManager.iter_archive">iter_archive()</a> doesn’t provide any information about the full paths of files from the archive, even if it has been extracted. As a result, you need to pass the <code>local_extracted_archive</code> path to the next step in <code>gen_kwargs</code>, in order to preserve information about where the archive was extracted to. This is required to construct the correct paths to the local files when you generate the examples.</p></li></ol> <div class="course-tip course-tip-orange bg-gradient-to-br dark:bg-gradient-to-r before:border-orange-500 dark:before:border-orange-800 from-orange-50 dark:from-gray-900 to-white dark:to-gray-950 border border-orange-50 text-orange-700 dark:text-gray-400"><p data-svelte-h="svelte-1w9o6k0">The reason you need to use a combination of <a href="/docs/datasets/v3.3.2/en/package_reference/builder_classes#datasets.DownloadManager.download">download()</a> and <a href="/docs/datasets/v3.3.2/en/package_reference/builder_classes#datasets.DownloadManager.iter_archive">iter_archive()</a> is because files in TAR archives can’t be accessed directly by their paths. Instead, you’ll need to iterate over the files within the archive! You can use <a href="/docs/datasets/v3.3.2/en/package_reference/builder_classes#datasets.DownloadManager.download_and_extract">download_and_extract()</a> and <a href="/docs/datasets/v3.3.2/en/package_reference/builder_classes#datasets.DownloadManager.extract">extract()</a> with TAR archives only in non-streaming mode, otherwise it would throw an error.</p></div> <ol start="4" data-svelte-h="svelte-1vz4x07"><li><p>Use the <a href="/docs/datasets/v3.3.2/en/package_reference/builder_classes#datasets.DownloadManager.download_and_extract">download_and_extract()</a> method to download the metadata file specified in <code>_METADATA_URL</code>. This method returns a path to a local file in non-streaming mode. In streaming mode, it doesn’t download file locally and returns the same URL.</p></li> <li><p>Now use the <a href="/docs/datasets/v3.3.2/en/package_reference/builder_classes#datasets.SplitGenerator">SplitGenerator</a> to organize the audio files and metadata in each split. Name each split with a standard name like: <code>Split.TRAIN</code>, <code>Split.TEST</code>, and <code>SPLIT.Validation</code>.</p> <p>In the <code>gen_kwargs</code> parameter, specify the file paths to <code>local_extracted_archive</code>, <code>audio_files</code>, <code>metadata_path</code>, and <code>path_to_clips</code>. Remember, for <code>audio_files</code>, you need to use <a href="/docs/datasets/v3.3.2/en/package_reference/builder_classes#datasets.DownloadManager.iter_archive">iter_archive()</a> to iterate over the audio files in the TAR archives. This enables streaming for your dataset! All of these file paths are passed onto the next step where the dataset samples are generated.</p></li></ol> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><span class="hljs-keyword">def</span> <span class="hljs-title function_">_split_generators</span>(<span class="hljs-params">self, dl_manager</span>): <span class="hljs-string">"""Returns SplitGenerators."""</span> audio_path = dl_manager.download(_AUDIO_URL) local_extracted_archive = dl_manager.extract(audio_path) <span class="hljs-keyword">if</span> <span class="hljs-keyword">not</span> dl_manager.is_streaming <span class="hljs-keyword">else</span> <span class="hljs-literal">None</span> path_to_clips = <span class="hljs-string">"librivox-indonesia"</span> <span class="hljs-keyword">return</span> [ datasets.SplitGenerator( name=datasets.Split.TRAIN, gen_kwargs={ <span class="hljs-string">"local_extracted_archive"</span>: local_extracted_archive, <span class="hljs-string">"audio_files"</span>: dl_manager.iter_archive(audio_path), <span class="hljs-string">"metadata_path"</span>: dl_manager.download_and_extract(_METADATA_URL + <span class="hljs-string">"/metadata_train.csv.gz"</span>), <span class="hljs-string">"path_to_clips"</span>: path_to_clips, }, ), datasets.SplitGenerator( name=datasets.Split.TEST, gen_kwargs={ <span class="hljs-string">"local_extracted_archive"</span>: local_extracted_archive, <span class="hljs-string">"audio_files"</span>: dl_manager.iter_archive(audio_path), <span class="hljs-string">"metadata_path"</span>: dl_manager.download_and_extract(_METADATA_URL + <span class="hljs-string">"/metadata_test.csv.gz"</span>), <span class="hljs-string">"path_to_clips"</span>: path_to_clips, }, ), ]</pre></div> <h4 class="relative group"><a id="generate-the-dataset" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#generate-the-dataset"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Generate the dataset</span></h4> <p data-svelte-h="svelte-j2l6r9">Here <code>_generate_examples</code> accepts <code>local_extracted_archive</code>, <code>audio_files</code>, <code>metadata_path</code>, and <code>path_to_clips</code> from the previous method as arguments.</p> <ol><li><p data-svelte-h="svelte-2fcq95">TAR files are accessed and yielded sequentially. This means you need to have the metadata in <code>metadata_path</code> associated with the audio files in the TAR file in hand first so that you can yield it with its corresponding audio file further:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(metadata_path, <span class="hljs-string">"r"</span>, encoding=<span class="hljs-string">"utf-8"</span>) <span class="hljs-keyword">as</span> f: reader = csv.DictReader(f) <span class="hljs-keyword">for</span> row <span class="hljs-keyword">in</span> reader: <span class="hljs-keyword">if</span> self.config.name == <span class="hljs-string">"all"</span> <span class="hljs-keyword">or</span> self.config.name == row[<span class="hljs-string">"language"</span>]: row[<span class="hljs-string">"path"</span>] = os.path.join(path_to_clips, row[<span class="hljs-string">"path"</span>]) <span class="hljs-comment"># if data is incomplete, fill with empty values</span> <span class="hljs-keyword">for</span> field <span class="hljs-keyword">in</span> data_fields: <span class="hljs-keyword">if</span> field <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> row: row[field] = <span class="hljs-string">""</span> metadata[row[<span class="hljs-string">"path"</span>]] = row</pre></div></li> <li><p data-svelte-h="svelte-1h7nhnk">Now you can yield the files in <code>audio_files</code> archive. When you use <a href="/docs/datasets/v3.3.2/en/package_reference/builder_classes#datasets.DownloadManager.iter_archive">iter_archive()</a>, it yielded a tuple of (<code>path</code>, <code>f</code>) where <code>path</code> is a <strong>relative path</strong> to a file inside the archive, and <code>f</code> is the file object itself. To get the <strong>full path</strong> to the locally extracted file, join the path of the directory (<code>local_extracted_path</code>) where the archive is extracted to and the relative audio file path (<code>path</code>):</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><span class="hljs-keyword">for</span> path, f <span class="hljs-keyword">in</span> audio_files: <span class="hljs-keyword">if</span> path <span class="hljs-keyword">in</span> metadata: result = <span class="hljs-built_in">dict</span>(metadata[path]) <span class="hljs-comment"># set the audio feature and the path to the extracted file</span> path = os.path.join(local_extracted_archive, path) <span class="hljs-keyword">if</span> local_extracted_archive <span class="hljs-keyword">else</span> path result[<span class="hljs-string">"audio"</span>] = {<span class="hljs-string">"path"</span>: path, <span class="hljs-string">"bytes"</span>: f.read()} result[<span class="hljs-string">"path"</span>] = path <span class="hljs-keyword">yield</span> id_, result id_ += <span class="hljs-number">1</span></pre></div></li></ol> <p data-svelte-h="svelte-6r7nb">Put both of these steps together, and the whole <code>_generate_examples</code> method should look like:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><span class="hljs-keyword">def</span> <span class="hljs-title function_">_generate_examples</span>(<span class="hljs-params"> self, local_extracted_archive, audio_files, metadata_path, path_to_clips, </span>): <span class="hljs-string">"""Yields examples."""</span> data_fields = <span class="hljs-built_in">list</span>(self._info().features.keys()) metadata = {} <span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(metadata_path, <span class="hljs-string">"r"</span>, encoding=<span class="hljs-string">"utf-8"</span>) <span class="hljs-keyword">as</span> f: reader = csv.DictReader(f) <span class="hljs-keyword">for</span> row <span class="hljs-keyword">in</span> reader: <span class="hljs-keyword">if</span> self.config.name == <span class="hljs-string">"all"</span> <span class="hljs-keyword">or</span> self.config.name == row[<span class="hljs-string">"language"</span>]: row[<span class="hljs-string">"path"</span>] = os.path.join(path_to_clips, row[<span class="hljs-string">"path"</span>]) <span class="hljs-comment"># if data is incomplete, fill with empty values</span> <span class="hljs-keyword">for</span> field <span class="hljs-keyword">in</span> data_fields: <span class="hljs-keyword">if</span> field <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> row: row[field] = <span class="hljs-string">""</span> metadata[row[<span class="hljs-string">"path"</span>]] = row id_ = <span class="hljs-number">0</span> <span class="hljs-keyword">for</span> path, f <span class="hljs-keyword">in</span> audio_files: <span class="hljs-keyword">if</span> path <span class="hljs-keyword">in</span> metadata: result = <span class="hljs-built_in">dict</span>(metadata[path]) <span class="hljs-comment"># set the audio feature and the path to the extracted file</span> path = os.path.join(local_extracted_archive, path) <span class="hljs-keyword">if</span> local_extracted_archive <span class="hljs-keyword">else</span> path result[<span class="hljs-string">"audio"</span>] = {<span class="hljs-string">"path"</span>: path, <span class="hljs-string">"bytes"</span>: f.read()} result[<span class="hljs-string">"path"</span>] = path <span class="hljs-keyword">yield</span> id_, result id_ += <span class="hljs-number">1</span></pre></div> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/datasets/blob/main/docs/source/audio_dataset.mdx" target="_blank"><span data-svelte-h="svelte-1kd6by1"><</span> <span data-svelte-h="svelte-x0xyl0">></span> <span data-svelte-h="svelte-1dajgef"><span class="underline ml-1.5">Update</span> on GitHub</span></a> <p></p> <script> { __sveltekit_kewd99 = { assets: "/docs/datasets/v3.3.2/en", base: "/docs/datasets/v3.3.2/en", env: {} }; const element = document.currentScript.parentElement; const data = [null,null]; Promise.all([ import("/docs/datasets/v3.3.2/en/_app/immutable/entry/start.bbd56df5.js"), import("/docs/datasets/v3.3.2/en/_app/immutable/entry/app.b1a61c37.js") ]).then(([kit, app]) => { kit.start(app, element, { node_ids: [0, 9], data, form: null, error: null }); }); } </script> </div> <div class="SVELTE_HYDRATER contents" data-target="DocFooterNav" data-props="{"classNames":"mx-auto mt-16 flex max-w-4xl items-center pb-8 font-sans font-medium leading-6 xl:mt-32","chapterPrev":{"title":"Process audio data","isExpanded":true,"id":"audio_process","url":"/docs/datasets/audio_process"},"chapterNext":{"title":"Load image data","isExpanded":true,"id":"image_load","url":"/docs/datasets/image_load"},"isCourse":false,"isLoggedIn":false}"><div class="mx-auto mt-16 flex max-w-4xl items-center pb-8 font-sans font-medium leading-6 xl:mt-32"><a href="/docs/datasets/audio_process" class="mr-8 flex transform items-center text-gray-600 transition-all hover:-translate-x-px hover:text-gray-900 dark:hover:text-gray-300"><span class="mr-2 translate-y-px">←</span>Process audio data</a> <a href="/docs/datasets/image_load" class="ml-auto flex transform items-center text-right text-gray-600 transition-all hover:translate-x-px hover:text-gray-900 dark:hover:text-gray-300">Load image data<span class="ml-2 translate-y-px">→</span></a></div></div></div></div> <div class="sticky top-0 self-start"><div class="SVELTE_HYDRATER contents" data-target="SubSideMenu" data-props="{"chapter":{"title":"Create an audio dataset","isExpanded":true,"id":"create-an-audio-dataset","url":"#create-an-audio-dataset","sections":[{"title":"Local files","isExpanded":true,"id":"local-files","url":"#local-files","sections":[]},{"title":"AudioFolder","isExpanded":true,"id":"audiofolder","url":"#audiofolder","sections":[]},{"title":"(Legacy) Loading script","isExpanded":true,"id":"legacy-loading-script","url":"#legacy-loading-script","sections":[{"title":"Create a dataset builder class","isExpanded":true,"id":"create-a-dataset-builder-class","url":"#create-a-dataset-builder-class","sections":[{"title":"Multiple configurations","isExpanded":true,"id":"multiple-configurations","url":"#multiple-configurations","sections":[]}]},{"title":"Add dataset metadata","isExpanded":true,"id":"add-dataset-metadata","url":"#add-dataset-metadata","sections":[]},{"title":"Download and define the dataset splits","isExpanded":true,"id":"download-and-define-the-dataset-splits","url":"#download-and-define-the-dataset-splits","sections":[]},{"title":"Generate the dataset","isExpanded":true,"id":"generate-the-dataset","url":"#generate-the-dataset","sections":[]},{"title":"Upload the dataset to the Hub","isExpanded":true,"id":"upload-the-dataset-to-the-hub","url":"#upload-the-dataset-to-the-hub","sections":[]},{"title":"(Advanced) Extract TAR archives locally","isExpanded":true,"id":"advanced-extract-tar-archives-locally","url":"#advanced-extract-tar-archives-locally","sections":[{"title":"Download and define the dataset splits","isExpanded":true,"id":"download-and-define-the-dataset-splits","url":"#download-and-define-the-dataset-splits","sections":[]},{"title":"Generate the dataset","isExpanded":true,"id":"generate-the-dataset","url":"#generate-the-dataset","sections":[]}]}]}]}}"> <nav class="hidden h-dvh w-[270px] flex-none flex-col space-y-3 overflow-y-auto break-words border-l pb-16 pl-6 pr-10 pt-24 text-sm lg:flex 2xl:w-[305px]"> <a href="#create-an-audio-dataset" class=" text-gray-400 transform hover:translate-x-px hover:text-gray-700 dark:hover:text-gray-300" id="nav-create-an-audio-dataset"><wbr>Create an audio dataset</a> <a href="#local-files" class="pl-4 text-gray-400 transform hover:translate-x-px hover:text-gray-700 dark:hover:text-gray-300" id="nav-local-files"><wbr>Local files</a> <a href="#audiofolder" class="pl-4 text-gray-400 transform hover:translate-x-px hover:text-gray-700 dark:hover:text-gray-300" id="nav-audiofolder"><wbr>Audio<wbr>Folder</a> <a href="#legacy-loading-script" class="pl-4 text-gray-400 transform hover:translate-x-px hover:text-gray-700 dark:hover:text-gray-300" id="nav-legacy-loading-script">(<wbr>Legacy) <wbr>Loading script</a> <a href="#create-a-dataset-builder-class" class="pl-8 text-gray-400 transform hover:translate-x-px hover:text-gray-700 dark:hover:text-gray-300" id="nav-create-a-dataset-builder-class"><wbr>Create a dataset builder class</a> <a href="#multiple-configurations" class="pl-12 text-xs text-gray-400 transform hover:translate-x-px hover:text-gray-700 dark:hover:text-gray-300" id="nav-multiple-configurations"><wbr>Multiple configurations</a> <a href="#add-dataset-metadata" class="pl-8 text-gray-400 transform hover:translate-x-px hover:text-gray-700 dark:hover:text-gray-300" id="nav-add-dataset-metadata"><wbr>Add dataset metadata</a> <a href="#download-and-define-the-dataset-splits" class="pl-8 text-gray-400 transform hover:translate-x-px hover:text-gray-700 dark:hover:text-gray-300" id="nav-download-and-define-the-dataset-splits"><wbr>Download and define the dataset splits</a> <a href="#generate-the-dataset" class="pl-8 text-gray-400 transform hover:translate-x-px hover:text-gray-700 dark:hover:text-gray-300" id="nav-generate-the-dataset"><wbr>Generate the dataset</a> <a href="#upload-the-dataset-to-the-hub" class="pl-8 text-gray-400 transform hover:translate-x-px hover:text-gray-700 dark:hover:text-gray-300" id="nav-upload-the-dataset-to-the-hub"><wbr>Upload the dataset to the <wbr>Hub</a> <a href="#advanced-extract-tar-archives-locally" class="pl-8 text-gray-400 transform hover:translate-x-px hover:text-gray-700 dark:hover:text-gray-300" id="nav-advanced-extract-tar-archives-locally">(<wbr>Advanced) <wbr>Extract TA<wbr>R archives locally</a> <a href="#download-and-define-the-dataset-splits" class="pl-12 text-xs text-gray-400 transform hover:translate-x-px hover:text-gray-700 dark:hover:text-gray-300" id="nav-download-and-define-the-dataset-splits"><wbr>Download and define the dataset splits</a> <a href="#generate-the-dataset" class="pl-12 text-xs text-gray-400 transform hover:translate-x-px hover:text-gray-700 dark:hover:text-gray-300" id="nav-generate-the-dataset"><wbr>Generate the dataset</a></nav></div></div></div> <div id="doc-footer"></div></main> </div> <script> import("\/front\/build\/kube-b666922\/index.js"); window.moonSha = "kube-b666922\/"; window.__hf_deferred = {}; </script>  <script> if (["hf.co", "huggingface.co"].includes(window.location.hostname)) { const script = document.createElement("script"); script.src = "https://js.stripe.com/v3/"; script.async = true; document.head.appendChild(script); } </script> </body> </html>