CINXE.COM

<!doctype html> <html class=""> <head> <meta charset="utf-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=no" /> <meta name="description" content="We’re on a journey to advance and democratize artificial intelligence through open source and open science." /> <meta property="fb:app_id" content="1321688464574422" /> <meta name="twitter:card" content="summary_large_image" /> <meta name="twitter:site" content="@huggingface" /> <meta name="twitter:image" content="https://huggingface.co/front/thumbnails/docs/peft.png" /> <meta property="og:title" content="Model merging" /> <meta property="og:type" content="website" /> <meta property="og:url" content="https://huggingface.co/docs/peft/developer_guides/model_merging" /> <meta property="og:image" content="https://huggingface.co/front/thumbnails/docs/peft.png" /> <link rel="stylesheet" href="/front/build/kube-0a779c5/style.css" /> <link rel="preconnect" href="https://fonts.gstatic.com" /> <link href="https://fonts.googleapis.com/css2?family=Source+Sans+Pro:ital,wght@0,200;0,300;0,400;0,600;0,700;0,900;1,200;1,300;1,400;1,600;1,700;1,900&display=swap" rel="stylesheet" /> <link href="https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:wght@400;600;700&display=swap" rel="stylesheet" /> <link rel="preload" href="https://cdnjs.cloudflare.com/ajax/libs/KaTeX/0.12.0/katex.min.css" as="style" onload="this.onload=null;this.rel='stylesheet'" /> <noscript> <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/KaTeX/0.12.0/katex.min.css" /> </noscript> <script>const guestTheme = document.cookie.match(/theme=(\w+)/)?.[1]; document.documentElement.classList.toggle('dark', guestTheme === 'dark' || ( (!guestTheme || guestTheme === 'system') && window.matchMedia('(prefers-color-scheme: dark)').matches));</script> <link rel="canonical" href="https://huggingface.co/docs/peft/developer_guides/model_merging"> <link rel="alternate" hreflang="en" href="https://huggingface.co/docs/peft/en/developer_guides/model_merging"> <link rel="alternate" hreflang="x-default" href="https://huggingface.co/docs/peft/developer_guides/model_merging"> <title>Model merging</title> <script defer data-domain="huggingface.co" event-loggedIn="false" src="/js/script.pageview-props.js" ></script> <script> window.plausible = window.plausible || function () { (window.plausible.q = window.plausible.q || []).push(arguments); }; </script> <script> window.hubConfig = {"features":{"signupDisabled":false},"sshGitUrl":"git@hf.co","moonHttpUrl":"https:\/\/huggingface.co","captchaApiKey":"bd5f2066-93dc-4bdd-a64b-a24646ca3859","captchaDisabledOnSignup":true,"datasetViewerPublicUrl":"https:\/\/datasets-server.huggingface.co","stripePublicKey":"pk_live_x2tdjFXBCvXo2FFmMybezpeM00J6gPCAAc","environment":"production","userAgent":"HuggingFace (production)","spacesIframeDomain":"hf.space","spacesApiUrl":"https:\/\/api.hf.space","docSearchKey":"ece5e02e57300e17d152c08056145326e90c4bff3dd07d7d1ae40cf1c8d39cb6","logoDev":{"apiUrl":"https:\/\/img.logo.dev\/","apiKey":"pk_UHS2HZOeRnaSOdDp7jbd5w"}}; </script> <script type="text/javascript" src="https://de5282c3ca0c.edge.sdk.awswaf.com/de5282c3ca0c/526cf06acb0d/challenge.js" defer></script> </head> <body class="flex flex-col min-h-dvh bg-white dark:bg-gray-950 text-black DocBuilderPage"> <div class="flex min-h-dvh flex-col"><div class="SVELTE_HYDRATER contents" data-target="SystemThemeMonitor" data-props="{"isLoggedIn":false}"></div> <div class="SVELTE_HYDRATER contents" data-target="MainHeader" data-props="{"classNames":"","isWide":true,"isZh":false,"isPro":false}"><header class="border-b border-gray-100 "><div class="w-full px-4 flex h-16 items-center"><div class="flex flex-1 items-center"><a class="mr-5 flex flex-none items-center lg:mr-6" href="/"><img alt="Hugging Face's logo" class="w-7 md:mr-2" src="/front/assets/huggingface_logo-noborder.svg"> <span class="hidden whitespace-nowrap text-lg font-bold md:block">Hugging Face</span></a> <div class="relative flex-1 lg:max-w-sm mr-2 sm:mr-4 md:mr-3 xl:mr-6"><input autocomplete="off" class="w-full dark:bg-gray-950 pl-8 form-input-alt h-9 pr-3 focus:shadow-xl " name="" placeholder="Search models, datasets, users..." spellcheck="false" type="text" value=""> <svg class="absolute left-2.5 text-gray-400 top-1/2 transform -translate-y-1/2" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M30 28.59L22.45 21A11 11 0 1 0 21 22.45L28.59 30zM5 14a9 9 0 1 1 9 9a9 9 0 0 1-9-9z" fill="currentColor"></path></svg> </div> <div class="flex flex-none items-center justify-center p-0.5 place-self-stretch lg:hidden"><button class="relative z-40 flex h-6 w-8 items-center justify-center" type="button"><svg width="1em" height="1em" viewBox="0 0 10 10" class="text-xl" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" preserveAspectRatio="xMidYMid meet" fill="currentColor"><path fill-rule="evenodd" clip-rule="evenodd" d="M1.65039 2.9999C1.65039 2.8066 1.80709 2.6499 2.00039 2.6499H8.00039C8.19369 2.6499 8.35039 2.8066 8.35039 2.9999C8.35039 3.1932 8.19369 3.3499 8.00039 3.3499H2.00039C1.80709 3.3499 1.65039 3.1932 1.65039 2.9999ZM1.65039 4.9999C1.65039 4.8066 1.80709 4.6499 2.00039 4.6499H8.00039C8.19369 4.6499 8.35039 4.8066 8.35039 4.9999C8.35039 5.1932 8.19369 5.3499 8.00039 5.3499H2.00039C1.80709 5.3499 1.65039 5.1932 1.65039 4.9999ZM2.00039 6.6499C1.80709 6.6499 1.65039 6.8066 1.65039 6.9999C1.65039 7.1932 1.80709 7.3499 2.00039 7.3499H8.00039C8.19369 7.3499 8.35039 7.1932 8.35039 6.9999C8.35039 6.8066 8.19369 6.6499 8.00039 6.6499H2.00039Z"></path></svg> </button> </div></div> <nav aria-label="Main" class="ml-auto hidden lg:block"><ul class="flex items-center space-x-1.5 2xl:space-x-2"><li class="hover:text-indigo-700"><a class="group flex items-center px-2 py-0.5 dark:text-gray-300 dark:hover:text-gray-100" href="/models"><svg class="mr-1.5 text-gray-400 group-hover:text-indigo-500" style="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path class="uim-quaternary" d="M20.23 7.24L12 12L3.77 7.24a1.98 1.98 0 0 1 .7-.71L11 2.76c.62-.35 1.38-.35 2 0l6.53 3.77c.29.173.531.418.7.71z" opacity=".25" fill="currentColor"></path><path class="uim-tertiary" d="M12 12v9.5a2.09 2.09 0 0 1-.91-.21L4.5 17.48a2.003 2.003 0 0 1-1-1.73v-7.5a2.06 2.06 0 0 1 .27-1.01L12 12z" opacity=".5" fill="currentColor"></path><path class="uim-primary" d="M20.5 8.25v7.5a2.003 2.003 0 0 1-1 1.73l-6.62 3.82c-.275.13-.576.198-.88.2V12l8.23-4.76c.175.308.268.656.27 1.01z" fill="currentColor"></path></svg> Models</a> </li><li class="hover:text-red-700"><a class="group flex items-center px-2 py-0.5 dark:text-gray-300 dark:hover:text-gray-100" href="/datasets"><svg class="mr-1.5 text-gray-400 group-hover:text-red-500" style="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 25 25"><ellipse cx="12.5" cy="5" fill="currentColor" fill-opacity="0.25" rx="7.5" ry="2"></ellipse><path d="M12.5 15C16.6421 15 20 14.1046 20 13V20C20 21.1046 16.6421 22 12.5 22C8.35786 22 5 21.1046 5 20V13C5 14.1046 8.35786 15 12.5 15Z" fill="currentColor" opacity="0.5"></path><path d="M12.5 7C16.6421 7 20 6.10457 20 5V11.5C20 12.6046 16.6421 13.5 12.5 13.5C8.35786 13.5 5 12.6046 5 11.5V5C5 6.10457 8.35786 7 12.5 7Z" fill="currentColor" opacity="0.5"></path><path d="M5.23628 12C5.08204 12.1598 5 12.8273 5 13C5 14.1046 8.35786 15 12.5 15C16.6421 15 20 14.1046 20 13C20 12.8273 19.918 12.1598 19.7637 12C18.9311 12.8626 15.9947 13.5 12.5 13.5C9.0053 13.5 6.06886 12.8626 5.23628 12Z" fill="currentColor"></path></svg> Datasets</a> </li><li class="hover:text-blue-700"><a class="group flex items-center px-2 py-0.5 dark:text-gray-300 dark:hover:text-gray-100" href="/spaces"><svg class="mr-1.5 text-gray-400 group-hover:text-blue-500" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" viewBox="0 0 25 25"><path opacity=".5" d="M6.016 14.674v4.31h4.31v-4.31h-4.31ZM14.674 14.674v4.31h4.31v-4.31h-4.31ZM6.016 6.016v4.31h4.31v-4.31h-4.31Z" fill="currentColor"></path><path opacity=".75" fill-rule="evenodd" clip-rule="evenodd" d="M3 4.914C3 3.857 3.857 3 4.914 3h6.514c.884 0 1.628.6 1.848 1.414a5.171 5.171 0 0 1 7.31 7.31c.815.22 1.414.964 1.414 1.848v6.514A1.914 1.914 0 0 1 20.086 22H4.914A1.914 1.914 0 0 1 3 20.086V4.914Zm3.016 1.102v4.31h4.31v-4.31h-4.31Zm0 12.968v-4.31h4.31v4.31h-4.31Zm8.658 0v-4.31h4.31v4.31h-4.31Zm0-10.813a2.155 2.155 0 1 1 4.31 0 2.155 2.155 0 0 1-4.31 0Z" fill="currentColor"></path><path opacity=".25" d="M16.829 6.016a2.155 2.155 0 1 0 0 4.31 2.155 2.155 0 0 0 0-4.31Z" fill="currentColor"></path></svg> Spaces</a> </li><li class="hover:text-yellow-700 max-xl:hidden"><a class="group flex items-center px-2 py-0.5 dark:text-gray-300 dark:hover:text-gray-100" href="/posts"><svg class="mr-1.5 text-gray-400 group-hover:text-yellow-500 text-yellow-500!" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" viewBox="0 0 12 12" preserveAspectRatio="xMidYMid meet"><path fill="currentColor" fill-rule="evenodd" d="M3.73 2.4A4.25 4.25 0 1 1 6 10.26H2.17l-.13-.02a.43.43 0 0 1-.3-.43l.01-.06a.43.43 0 0 1 .12-.22l.84-.84A4.26 4.26 0 0 1 3.73 2.4Z" clip-rule="evenodd"></path></svg> Posts</a> </li><li class="hover:text-yellow-700"><a class="group flex items-center px-2 py-0.5 dark:text-gray-300 dark:hover:text-gray-100" href="/docs"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="mr-1.5 text-gray-400 group-hover:text-yellow-500" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 16 16"><path d="m2.28 3.7-.3.16a.67.67 0 0 0-.34.58v8.73l.01.04.02.07.01.04.03.06.02.04.02.03.04.06.05.05.04.04.06.04.06.04.08.04.08.02h.05l.07.02h.11l.04-.01.07-.02.03-.01.07-.03.22-.12a5.33 5.33 0 0 1 5.15.1.67.67 0 0 0 .66 0 5.33 5.33 0 0 1 5.33 0 .67.67 0 0 0 1-.58V4.36a.67.67 0 0 0-.34-.5l-.3-.17v7.78a.63.63 0 0 1-.87.59 4.9 4.9 0 0 0-4.35.35l-.65.39a.29.29 0 0 1-.15.04.29.29 0 0 1-.16-.04l-.65-.4a4.9 4.9 0 0 0-4.34-.34.63.63 0 0 1-.87-.59V3.7Z" fill="currentColor" class="dark:opacity-40"></path><path fill-rule="evenodd" clip-rule="evenodd" d="M8 3.1a5.99 5.99 0 0 0-5.3-.43.66.66 0 0 0-.42.62v8.18c0 .45.46.76.87.59a4.9 4.9 0 0 1 4.34.35l.65.39c.05.03.1.04.16.04.05 0 .1-.01.15-.04l.65-.4a4.9 4.9 0 0 1 4.35-.34.63.63 0 0 0 .86-.59V3.3a.67.67 0 0 0-.41-.62 5.99 5.99 0 0 0-5.3.43l-.3.17L8 3.1Zm.73 1.87a.43.43 0 1 0-.86 0v5.48a.43.43 0 0 0 .86 0V4.97Z" fill="currentColor" class="opacity-40 dark:opacity-100"></path><path d="M8.73 4.97a.43.43 0 1 0-.86 0v5.48a.43.43 0 1 0 .86 0V4.96Z" fill="currentColor" class="dark:opacity-40"></path></svg> Docs</a> </li><li class="hover:text-green-700"><a class="group flex items-center px-2 py-0.5 dark:text-gray-300 dark:hover:text-gray-100" href="/enterprise"><svg class="mr-1.5 text-gray-400 group-hover:text-green-500" xmlns="http://www.w3.org/2000/svg" fill="none" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 33 27"><path fill="currentColor" fill-rule="evenodd" d="M13.5.7a8.7 8.7 0 0 0-7.7 5.7L1 20.6c-1 3.1.9 5.7 4.1 5.7h15c3.3 0 6.8-2.6 7.8-5.7l4.6-14.2c1-3.1-.8-5.7-4-5.7h-15Zm1.1 5.7L9.8 20.3h9.8l1-3.1h-5.8l.8-2.5h4.8l1.1-3h-4.8l.8-2.3H23l1-3h-9.5Z" clip-rule="evenodd"></path></svg> Enterprise</a> </li> <li><a class="group flex items-center px-2 py-0.5 dark:text-gray-300 dark:hover:text-gray-100" href="/pricing">Pricing </a></li> <li><div class="relative group"> <button class="px-2 py-0.5 hover:text-gray-500 dark:hover:text-gray-600 flex items-center " type="button"> <svg class=" text-gray-500 w-5 group-hover:text-gray-400 dark:text-gray-300 dark:group-hover:text-gray-100" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" viewBox="0 0 32 18" preserveAspectRatio="xMidYMid meet"><path fill-rule="evenodd" clip-rule="evenodd" d="M14.4504 3.30221C14.4504 2.836 14.8284 2.45807 15.2946 2.45807H28.4933C28.9595 2.45807 29.3374 2.836 29.3374 3.30221C29.3374 3.76842 28.9595 4.14635 28.4933 4.14635H15.2946C14.8284 4.14635 14.4504 3.76842 14.4504 3.30221Z" fill="currentColor"></path><path fill-rule="evenodd" clip-rule="evenodd" d="M14.4504 9.00002C14.4504 8.53382 14.8284 8.15588 15.2946 8.15588H28.4933C28.9595 8.15588 29.3374 8.53382 29.3374 9.00002C29.3374 9.46623 28.9595 9.84417 28.4933 9.84417H15.2946C14.8284 9.84417 14.4504 9.46623 14.4504 9.00002Z" fill="currentColor"></path><path fill-rule="evenodd" clip-rule="evenodd" d="M14.4504 14.6978C14.4504 14.2316 14.8284 13.8537 15.2946 13.8537H28.4933C28.9595 13.8537 29.3374 14.2316 29.3374 14.6978C29.3374 15.164 28.9595 15.542 28.4933 15.542H15.2946C14.8284 15.542 14.4504 15.164 14.4504 14.6978Z" fill="currentColor"></path><path fill-rule="evenodd" clip-rule="evenodd" d="M1.94549 6.87377C2.27514 6.54411 2.80962 6.54411 3.13928 6.87377L6.23458 9.96907L9.32988 6.87377C9.65954 6.54411 10.194 6.54411 10.5237 6.87377C10.8533 7.20343 10.8533 7.73791 10.5237 8.06756L6.23458 12.3567L1.94549 8.06756C1.61583 7.73791 1.61583 7.20343 1.94549 6.87377Z" fill="currentColor"></path></svg> </button> </div></li> <li><hr class="h-5 w-0.5 border-none bg-gray-100 dark:bg-gray-800"></li> <li><a class="block cursor-pointer whitespace-nowrap px-2 py-0.5 hover:text-gray-500 dark:text-gray-300 dark:hover:text-gray-100" href="/login">Log In </a></li> <li><a class="whitespace-nowrap rounded-full border border-transparent bg-gray-900 px-3 py-1 leading-none text-white hover:border-black hover:bg-white hover:text-black" href="/join">Sign Up </a></li></ul></nav></div></header></div> <div class="SVELTE_HYDRATER contents" data-target="SSOBanner" data-props="{}"></div> <main class="flex flex-1 flex-col"><div class="relative lg:flex" id="hf-doc-container"><div class="sticky top-0 z-20 self-start"><div class="SVELTE_HYDRATER contents" data-target="SideMenu" data-props="{"chapters":[{"title":"Get started","isExpanded":true,"sections":[{"title":"🤗 PEFT","isExpanded":true,"id":"index","url":"/docs/peft/index"},{"title":"Quicktour","isExpanded":true,"id":"quicktour","url":"/docs/peft/quicktour"},{"title":"Installation","isExpanded":true,"id":"install","url":"/docs/peft/install"}]},{"title":"Tutorial","isExpanded":true,"sections":[{"title":"Configurations and models","isExpanded":true,"id":"tutorial/peft_model_config","url":"/docs/peft/tutorial/peft_model_config"},{"title":"Integrations","isExpanded":true,"id":"tutorial/peft_integrations","url":"/docs/peft/tutorial/peft_integrations"}]},{"title":"PEFT method guides","isExpanded":true,"sections":[{"title":"Prompt-based methods","isExpanded":true,"id":"task_guides/prompt_based_methods","url":"/docs/peft/task_guides/prompt_based_methods"},{"title":"LoRA methods","isExpanded":true,"id":"task_guides/lora_based_methods","url":"/docs/peft/task_guides/lora_based_methods"},{"title":"IA3","isExpanded":true,"id":"task_guides/ia3","url":"/docs/peft/task_guides/ia3"}]},{"title":"Developer guides","isExpanded":true,"sections":[{"title":"Model merging","isExpanded":true,"id":"developer_guides/model_merging","url":"/docs/peft/developer_guides/model_merging"},{"title":"Quantization","isExpanded":true,"id":"developer_guides/quantization","url":"/docs/peft/developer_guides/quantization"},{"title":"LoRA","isExpanded":true,"id":"developer_guides/lora","url":"/docs/peft/developer_guides/lora"},{"title":"Custom models","isExpanded":true,"id":"developer_guides/custom_models","url":"/docs/peft/developer_guides/custom_models"},{"title":"Adapter injection","isExpanded":true,"id":"developer_guides/low_level_api","url":"/docs/peft/developer_guides/low_level_api"},{"title":"Mixed adapter types","isExpanded":true,"id":"developer_guides/mixed_models","url":"/docs/peft/developer_guides/mixed_models"},{"title":"torch.compile","isExpanded":true,"id":"developer_guides/torch_compile","url":"/docs/peft/developer_guides/torch_compile"},{"title":"Contribute to PEFT","isExpanded":true,"id":"developer_guides/contributing","url":"/docs/peft/developer_guides/contributing"},{"title":"Troubleshooting","isExpanded":true,"id":"developer_guides/troubleshooting","url":"/docs/peft/developer_guides/troubleshooting"},{"title":"PEFT checkpoint format","isExpanded":true,"id":"developer_guides/checkpoint","url":"/docs/peft/developer_guides/checkpoint"}]},{"title":"🤗 Accelerate integrations","isExpanded":true,"sections":[{"title":"DeepSpeed","isExpanded":true,"id":"accelerate/deepspeed","url":"/docs/peft/accelerate/deepspeed"},{"title":"Fully Sharded Data Parallel","isExpanded":true,"id":"accelerate/fsdp","url":"/docs/peft/accelerate/fsdp"}]},{"title":"Conceptual guides","isExpanded":true,"sections":[{"title":"Adapters","isExpanded":true,"id":"conceptual_guides/adapter","url":"/docs/peft/conceptual_guides/adapter"},{"title":"Soft prompts","isExpanded":true,"id":"conceptual_guides/prompting","url":"/docs/peft/conceptual_guides/prompting"},{"title":"IA3","isExpanded":true,"id":"conceptual_guides/ia3","url":"/docs/peft/conceptual_guides/ia3"},{"title":"OFT/BOFT","isExpanded":true,"id":"conceptual_guides/oft","url":"/docs/peft/conceptual_guides/oft"}]},{"title":"API reference","isExpanded":true,"sections":[{"title":"Main classes","isExpanded":true,"sections":[{"title":"AutoPeftModel","isExpanded":true,"id":"package_reference/auto_class","url":"/docs/peft/package_reference/auto_class"},{"title":"PEFT model","isExpanded":true,"id":"package_reference/peft_model","url":"/docs/peft/package_reference/peft_model"},{"title":"PEFT types","isExpanded":true,"id":"package_reference/peft_types","url":"/docs/peft/package_reference/peft_types"},{"title":"Configuration","isExpanded":true,"id":"package_reference/config","url":"/docs/peft/package_reference/config"},{"title":"Tuner","isExpanded":true,"id":"package_reference/tuners","url":"/docs/peft/package_reference/tuners"}]},{"title":"Adapters","isExpanded":true,"sections":[{"title":"AdaLoRA","isExpanded":true,"id":"package_reference/adalora","url":"/docs/peft/package_reference/adalora"},{"title":"IA3","isExpanded":true,"id":"package_reference/ia3","url":"/docs/peft/package_reference/ia3"},{"title":"Llama-Adapter","isExpanded":true,"id":"package_reference/llama_adapter","url":"/docs/peft/package_reference/llama_adapter"},{"title":"LoHa","isExpanded":true,"id":"package_reference/loha","url":"/docs/peft/package_reference/loha"},{"title":"LoKr","isExpanded":true,"id":"package_reference/lokr","url":"/docs/peft/package_reference/lokr"},{"title":"LoRA","isExpanded":true,"id":"package_reference/lora","url":"/docs/peft/package_reference/lora"},{"title":"X-LoRA","isExpanded":true,"id":"package_reference/xlora","url":"/docs/peft/package_reference/xlora"},{"title":"LyCORIS","isExpanded":true,"id":"package_reference/adapter_utils","url":"/docs/peft/package_reference/adapter_utils"},{"title":"Multitask Prompt Tuning","isExpanded":true,"id":"package_reference/multitask_prompt_tuning","url":"/docs/peft/package_reference/multitask_prompt_tuning"},{"title":"OFT","isExpanded":true,"id":"package_reference/oft","url":"/docs/peft/package_reference/oft"},{"title":"BOFT","isExpanded":true,"id":"package_reference/boft","url":"/docs/peft/package_reference/boft"},{"title":"Polytropon","isExpanded":true,"id":"package_reference/poly","url":"/docs/peft/package_reference/poly"},{"title":"P-tuning","isExpanded":true,"id":"package_reference/p_tuning","url":"/docs/peft/package_reference/p_tuning"},{"title":"Prefix tuning","isExpanded":true,"id":"package_reference/prefix_tuning","url":"/docs/peft/package_reference/prefix_tuning"},{"title":"Prompt tuning","isExpanded":true,"id":"package_reference/prompt_tuning","url":"/docs/peft/package_reference/prompt_tuning"},{"title":"Layernorm tuning","isExpanded":true,"id":"package_reference/layernorm_tuning","url":"/docs/peft/package_reference/layernorm_tuning"},{"title":"VeRA","isExpanded":true,"id":"package_reference/vera","url":"/docs/peft/package_reference/vera"},{"title":"FourierFT","isExpanded":true,"id":"package_reference/fourierft","url":"/docs/peft/package_reference/fourierft"},{"title":"VB-LoRA","isExpanded":true,"id":"package_reference/vblora","url":"/docs/peft/package_reference/vblora"},{"title":"HRA","isExpanded":true,"id":"package_reference/hra","url":"/docs/peft/package_reference/hra"},{"title":"CPT","isExpanded":true,"id":"package_reference/cpt","url":"/docs/peft/package_reference/cpt"},{"title":"Bone","isExpanded":true,"id":"package_reference/bone","url":"/docs/peft/package_reference/bone"}]},{"title":"Utilities","isExpanded":true,"sections":[{"title":"Model merge","isExpanded":true,"id":"package_reference/merge_utils","url":"/docs/peft/package_reference/merge_utils"},{"title":"Helpers","isExpanded":true,"id":"package_reference/helpers","url":"/docs/peft/package_reference/helpers"},{"title":"Hotswapping adapters","isExpanded":true,"id":"package_reference/hotswap","url":"/docs/peft/package_reference/hotswap"}]}]}],"chapterId":"developer_guides/model_merging","docType":"docs","isLoggedIn":false,"lang":"en","langs":["en"],"library":"peft","theme":"light","version":"v0.14.0","versions":[{"version":"main"},{"version":"v0.14.0"},{"version":"v0.13.0"},{"version":"v0.12.0"},{"version":"v0.11.0"},{"version":"v0.10.0"},{"version":"v0.9.0"},{"version":"v0.8.2"},{"version":"v0.8.1"},{"version":"v0.8.0"},{"version":"v0.7.1"},{"version":"v0.7.0"},{"version":"v0.6.2"},{"version":"v0.6.1"},{"version":"v0.6.0"}],"title":"Model merging"}"> <div class="z-2 w-full flex-none lg:flex lg:h-dvh lg:w-[270px] lg:flex-col 2xl:w-[300px] false"><div class="shadow-alternate flex h-auto w-full items-center rounded-b-xl border-b bg-white py-2 text-lg leading-tight lg:hidden"> <div class="flex flex-1 cursor-pointer flex-col justify-center self-stretch pl-6"><p class="text-sm text-gray-400 first-letter:capitalize">PEFT documentation </p> <div class="mr-2 flex items-center"><p class="font-semibold">Model merging</p> <svg class="text-xl false" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path d="M16.293 9.293L12 13.586L7.707 9.293l-1.414 1.414L12 16.414l5.707-5.707z" fill="currentColor"></path></svg></div></div> <button class="hover:shadow-alternate group ml-auto mr-6 inline-flex flex-none cursor-pointer rounded-xl border p-2"><svg class="text-gray-500 group-hover:text-gray-700" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M30 28.59L22.45 21A11 11 0 1 0 21 22.45L28.59 30zM5 14a9 9 0 1 1 9 9a9 9 0 0 1-9-9z" fill="currentColor"></path></svg></button></div> <div class="bg-linear-to-r hidden flex-col justify-between border-b border-r bg-white p-4 lg:flex from-red-50 to-white dark:from-gray-900 dark:to-gray-950"><div class="group relative mb-2 flex min-w-[50%] items-center self-start text-lg font-bold leading-tight first-letter:capitalize"><div class="mr-1.5 h-1.5 w-1.5 rounded-full bg-red-500 flex-none"></div> <h1>PEFT</h1> <svg class="opacity-50 ml-0.5 flex-none group-hover:opacity-100" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path d="M16.293 9.293L12 13.586L7.707 9.293l-1.414 1.414L12 16.414l5.707-5.707z" fill="currentColor"></path></svg> <select class="outline-hidden absolute inset-0 border-none bg-white text-base opacity-0"><option value="/docs">🏡 View all docs</option><option value="/docs/optimum-neuron" >AWS Trainium & Inferentia</option><option value="/docs/accelerate" >Accelerate</option><option value="/docs/sagemaker" >Amazon SageMaker</option><option value="https://argilla-io.github.io/argilla/" >Argilla</option><option value="/docs/autotrain" >AutoTrain</option><option value="/docs/bitsandbytes" >Bitsandbytes</option><option value="/docs/chat-ui" >Chat UI</option><option value="/docs/competitions" >Competitions</option><option value="/docs/dataset-viewer" >Dataset viewer</option><option value="/docs/datasets" >Datasets</option><option value="/docs/diffusers" >Diffusers</option><option value="https://distilabel.argilla.io/" >Distilabel</option><option value="/docs/evaluate" >Evaluate</option><option value="https://www.gradio.app/docs/" >Gradio</option><option value="/docs/hub" >Hub</option><option value="/docs/huggingface_hub" >Hub Python Library</option><option value="/docs/hugs" >Hugging Face Generative AI Services (HUGS)</option><option value="/docs/huggingface.js" >Huggingface.js</option><option value="/docs/api-inference" >Inference API (serverless)</option><option value="/docs/inference-endpoints" >Inference Endpoints (dedicated)</option><option value="/docs/leaderboards" >Leaderboards</option><option value="/docs/lighteval" >Lighteval</option><option value="/docs/optimum" >Optimum</option><option value="/docs/peft" selected>PEFT</option><option value="/docs/safetensors" >Safetensors</option><option value="https://sbert.net/" >Sentence Transformers</option><option value="/docs/trl" >TRL</option><option value="/tasks" >Tasks</option><option value="/docs/text-embeddings-inference" >Text Embeddings Inference</option><option value="/docs/text-generation-inference" >Text Generation Inference</option><option value="/docs/tokenizers" >Tokenizers</option><option value="/docs/transformers" >Transformers</option><option value="/docs/transformers.js" >Transformers.js</option><option value="/docs/smolagents" >smolagents</option><option value="/docs/timm" >timm</option></select></div> <button class="shadow-alternate mb-2 flex w-full items-center rounded-full border bg-white px-2 py-1 text-left text-sm text-gray-400 ring-indigo-200 hover:bg-indigo-50 hover:ring-2 dark:border-gray-700 dark:ring-yellow-600 dark:hover:bg-gray-900 dark:hover:text-yellow-500"><svg class="flex-none mr-1.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M30 28.59L22.45 21A11 11 0 1 0 21 22.45L28.59 30zM5 14a9 9 0 1 1 9 9a9 9 0 0 1-9-9z" fill="currentColor"></path></svg> <div>Search documentation</div> </button> <div class="flex items-center"> <select class="form-input mt-0! w-20! border! dark:text-gray-400! mr-1 rounded-sm border-gray-200 p-1 text-xs uppercase"><option value="0" >main</option><option value="1" selected>v0.14.0</option><option value="2" >v0.13.0</option><option value="3" >v0.12.0</option><option value="4" >v0.11.0</option><option value="5" >v0.10.0</option><option value="6" >v0.9.0</option><option value="7" >v0.8.2</option><option value="8" >v0.7.1</option><option value="9" >v0.6.2</option></select> <select class="form-input dark:text-gray-400! mr-1 rounded-sm border-gray-200 p-1 text-xs w-12! mt-0! border!"><option value="en" selected>EN</option></select> <div class="relative inline-block "> <button class="rounded-full border border-gray-100 pl-2 py-1 pr-2.5 flex items-center text-sm text-gray-500 bg-white hover:bg-yellow-50 hover:border-yellow-200 dark:hover:bg-gray-800 dark:hover:border-gray-950 dark:border-gray-800 " type="button"> <svg class=" text-yellow-500" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24" fill="currentColor"><path d="M6.05 4.14l-.39-.39a.993.993 0 0 0-1.4 0l-.01.01a.984.984 0 0 0 0 1.4l.39.39c.39.39 1.01.39 1.4 0l.01-.01a.984.984 0 0 0 0-1.4zM3.01 10.5H1.99c-.55 0-.99.44-.99.99v.01c0 .55.44.99.99.99H3c.56.01 1-.43 1-.98v-.01c0-.56-.44-1-.99-1zm9-9.95H12c-.56 0-1 .44-1 .99v.96c0 .55.44.99.99.99H12c.56.01 1-.43 1-.98v-.97c0-.55-.44-.99-.99-.99zm7.74 3.21c-.39-.39-1.02-.39-1.41-.01l-.39.39a.984.984 0 0 0 0 1.4l.01.01c.39.39 1.02.39 1.4 0l.39-.39a.984.984 0 0 0 0-1.4zm-1.81 15.1l.39.39a.996.996 0 1 0 1.41-1.41l-.39-.39a.993.993 0 0 0-1.4 0c-.4.4-.4 1.02-.01 1.41zM20 11.49v.01c0 .55.44.99.99.99H22c.55 0 .99-.44.99-.99v-.01c0-.55-.44-.99-.99-.99h-1.01c-.55 0-.99.44-.99.99zM12 5.5c-3.31 0-6 2.69-6 6s2.69 6 6 6s6-2.69 6-6s-2.69-6-6-6zm-.01 16.95H12c.55 0 .99-.44.99-.99v-.96c0-.55-.44-.99-.99-.99h-.01c-.55 0-.99.44-.99.99v.96c0 .55.44.99.99.99zm-7.74-3.21c.39.39 1.02.39 1.41 0l.39-.39a.993.993 0 0 0 0-1.4l-.01-.01a.996.996 0 0 0-1.41 0l-.39.39c-.38.4-.38 1.02.01 1.41z"></path></svg> </button> </div> <a href="https://github.com/huggingface/peft" class="group ml-auto text-xs text-gray-500 hover:text-gray-700 hover:underline dark:hover:text-gray-300"><svg class="inline-block text-gray-500 group-hover:text-gray-700 dark:group-hover:text-gray-300 mr-1.5 -mt-1 w-4 h-4" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1.03em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 250"><path d="M128.001 0C57.317 0 0 57.307 0 128.001c0 56.554 36.676 104.535 87.535 121.46c6.397 1.185 8.746-2.777 8.746-6.158c0-3.052-.12-13.135-.174-23.83c-35.61 7.742-43.124-15.103-43.124-15.103c-5.823-14.795-14.213-18.73-14.213-18.73c-11.613-7.944.876-7.78.876-7.78c12.853.902 19.621 13.19 19.621 13.19c11.417 19.568 29.945 13.911 37.249 10.64c1.149-8.272 4.466-13.92 8.127-17.116c-28.431-3.236-58.318-14.212-58.318-63.258c0-13.975 5-25.394 13.188-34.358c-1.329-3.224-5.71-16.242 1.24-33.874c0 0 10.749-3.44 35.21 13.121c10.21-2.836 21.16-4.258 32.038-4.307c10.878.049 21.837 1.47 32.066 4.307c24.431-16.56 35.165-13.12 35.165-13.12c6.967 17.63 2.584 30.65 1.255 33.873c8.207 8.964 13.173 20.383 13.173 34.358c0 49.163-29.944 59.988-58.447 63.157c4.591 3.972 8.682 11.762 8.682 23.704c0 17.126-.148 30.91-.148 35.126c0 3.407 2.304 7.398 8.792 6.14C219.37 232.5 256 184.537 256 128.002C256 57.307 198.691 0 128.001 0zm-80.06 182.34c-.282.636-1.283.827-2.194.39c-.929-.417-1.45-1.284-1.15-1.922c.276-.655 1.279-.838 2.205-.399c.93.418 1.46 1.293 1.139 1.931zm6.296 5.618c-.61.566-1.804.303-2.614-.591c-.837-.892-.994-2.086-.375-2.66c.63-.566 1.787-.301 2.626.591c.838.903 1 2.088.363 2.66zm4.32 7.188c-.785.545-2.067.034-2.86-1.104c-.784-1.138-.784-2.503.017-3.05c.795-.547 2.058-.055 2.861 1.075c.782 1.157.782 2.522-.019 3.08zm7.304 8.325c-.701.774-2.196.566-3.29-.49c-1.119-1.032-1.43-2.496-.726-3.27c.71-.776 2.213-.558 3.315.49c1.11 1.03 1.45 2.505.701 3.27zm9.442 2.81c-.31 1.003-1.75 1.459-3.199 1.033c-1.448-.439-2.395-1.613-2.103-2.626c.301-1.01 1.747-1.484 3.207-1.028c1.446.436 2.396 1.602 2.095 2.622zm10.744 1.193c.036 1.055-1.193 1.93-2.715 1.95c-1.53.034-2.769-.82-2.786-1.86c0-1.065 1.202-1.932 2.733-1.958c1.522-.03 2.768.818 2.768 1.868zm10.555-.405c.182 1.03-.875 2.088-2.387 2.37c-1.485.271-2.861-.365-3.05-1.386c-.184-1.056.893-2.114 2.376-2.387c1.514-.263 2.868.356 3.061 1.403z" fill="currentColor"></path></svg> </a></div></div> <nav class="hidden flex-auto lg:flex bottom-0 left-0 w-full flex-col overflow-y-auto border-r px-4 pb-16 pt-3 text-[0.95rem] lg:w-[270px] 2xl:w-[300px]"> <div class="group flex cursor-pointer items-center pl-2 text-[0.8rem] font-semibold uppercase leading-9 hover:text-gray-700 dark:hover:text-gray-300 ml-0"><div class="flex after:absolute after:right-4 after:text-gray-500 group-hover:after:content-['▶'] after:rotate-90 after:transform"><span><span class="inline-block space-x-1 leading-5"><span>Get started</span> </span></span> </div></div> <div class="flex flex-col"><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/peft/index" id="index">🤗 PEFT </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/peft/quicktour" id="quicktour">Quicktour </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/peft/install" id="install">Installation </a> </div> <div class="group flex cursor-pointer items-center pl-2 text-[0.8rem] font-semibold uppercase leading-9 hover:text-gray-700 dark:hover:text-gray-300 ml-0"><div class="flex after:absolute after:right-4 after:text-gray-500 group-hover:after:content-['▶'] after:rotate-90 after:transform"><span><span class="inline-block space-x-1 leading-5"><span>Tutorial</span> </span></span> </div></div> <div class="flex flex-col"><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/peft/tutorial/peft_model_config" id="tutorial/peft_model_config">Configurations and models </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/peft/tutorial/peft_integrations" id="tutorial/peft_integrations">Integrations </a> </div> <div class="group flex cursor-pointer items-center pl-2 text-[0.8rem] font-semibold uppercase leading-9 hover:text-gray-700 dark:hover:text-gray-300 ml-0"><div class="flex after:absolute after:right-4 after:text-gray-500 group-hover:after:content-['▶'] after:rotate-90 after:transform"><span><span class="inline-block space-x-1 leading-5"><span>PEFT method guides</span> </span></span> </div></div> <div class="flex flex-col"><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/peft/task_guides/prompt_based_methods" id="task_guides/prompt_based_methods">Prompt-based methods </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/peft/task_guides/lora_based_methods" id="task_guides/lora_based_methods">LoRA methods </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/peft/task_guides/ia3" id="task_guides/ia3">IA3 </a> </div> <div class="group flex cursor-pointer items-center pl-2 text-[0.8rem] font-semibold uppercase leading-9 hover:text-gray-700 dark:hover:text-gray-300 ml-0"><div class="flex after:absolute after:right-4 after:text-gray-500 group-hover:after:content-['▶'] after:rotate-90 after:transform"><span><span class="inline-block space-x-1 leading-5"><span>Developer guides</span> </span></span> </div></div> <div class="flex flex-col"><a class="bg-linear-to-br rounded-xl from-black to-gray-900 py-1 pl-2 pr-2 text-white first:mt-1 last:mb-4 dark:from-gray-800 dark:to-gray-900 ml-2" href="/docs/peft/developer_guides/model_merging" id="developer_guides/model_merging">Model merging </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/peft/developer_guides/quantization" id="developer_guides/quantization">Quantization </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/peft/developer_guides/lora" id="developer_guides/lora">LoRA </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/peft/developer_guides/custom_models" id="developer_guides/custom_models">Custom models </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/peft/developer_guides/low_level_api" id="developer_guides/low_level_api">Adapter injection </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/peft/developer_guides/mixed_models" id="developer_guides/mixed_models">Mixed adapter types </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/peft/developer_guides/torch_compile" id="developer_guides/torch_compile">torch.compile </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/peft/developer_guides/contributing" id="developer_guides/contributing">Contribute to PEFT </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/peft/developer_guides/troubleshooting" id="developer_guides/troubleshooting">Troubleshooting </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/peft/developer_guides/checkpoint" id="developer_guides/checkpoint">PEFT checkpoint format </a> </div> <div class="group flex cursor-pointer items-center pl-2 text-[0.8rem] font-semibold uppercase leading-9 hover:text-gray-700 dark:hover:text-gray-300 ml-0"><div class="flex after:absolute after:right-4 after:text-gray-500 group-hover:after:content-['▶'] after:rotate-90 after:transform"><span><span class="inline-block space-x-1 leading-5"><span>🤗 Accelerate integrations</span> </span></span> </div></div> <div class="flex flex-col"><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/peft/accelerate/deepspeed" id="accelerate/deepspeed">DeepSpeed </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/peft/accelerate/fsdp" id="accelerate/fsdp">Fully Sharded Data Parallel </a> </div> <div class="group flex cursor-pointer items-center pl-2 text-[0.8rem] font-semibold uppercase leading-9 hover:text-gray-700 dark:hover:text-gray-300 ml-0"><div class="flex after:absolute after:right-4 after:text-gray-500 group-hover:after:content-['▶'] after:rotate-90 after:transform"><span><span class="inline-block space-x-1 leading-5"><span>Conceptual guides</span> </span></span> </div></div> <div class="flex flex-col"><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/peft/conceptual_guides/adapter" id="conceptual_guides/adapter">Adapters </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/peft/conceptual_guides/prompting" id="conceptual_guides/prompting">Soft prompts </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/peft/conceptual_guides/ia3" id="conceptual_guides/ia3">IA3 </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-2" href="/docs/peft/conceptual_guides/oft" id="conceptual_guides/oft">OFT/BOFT </a> </div> <div class="group flex cursor-pointer items-center pl-2 text-[0.8rem] font-semibold uppercase leading-9 hover:text-gray-700 dark:hover:text-gray-300 ml-0"><div class="flex after:absolute after:right-4 after:text-gray-500 group-hover:after:content-['▶'] after:rotate-90 after:transform"><span><span class="inline-block space-x-1 leading-5"><span>API reference</span> </span></span> </div></div> <div class="flex flex-col"> <div class="group flex cursor-pointer items-center pl-2 text-[0.8rem] font-semibold uppercase leading-9 hover:text-gray-700 dark:hover:text-gray-300 ml-2"><div class="flex after:absolute after:right-4 after:text-gray-500 group-hover:after:content-['▶'] after:rotate-90 after:transform"><span><span class="inline-block space-x-1 leading-5"><span>Main classes</span> </span></span> </div></div> <div class="flex flex-col"><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/peft/package_reference/auto_class" id="package_reference/auto_class">AutoPeftModel </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/peft/package_reference/peft_model" id="package_reference/peft_model">PEFT model </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/peft/package_reference/peft_types" id="package_reference/peft_types">PEFT types </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/peft/package_reference/config" id="package_reference/config">Configuration </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/peft/package_reference/tuners" id="package_reference/tuners">Tuner </a> </div> <div class="group flex cursor-pointer items-center pl-2 text-[0.8rem] font-semibold uppercase leading-9 hover:text-gray-700 dark:hover:text-gray-300 ml-2"><div class="flex after:absolute after:right-4 after:text-gray-500 group-hover:after:content-['▶'] after:rotate-90 after:transform"><span><span class="inline-block space-x-1 leading-5"><span>Adapters</span> </span></span> </div></div> <div class="flex flex-col"><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/peft/package_reference/adalora" id="package_reference/adalora">AdaLoRA </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/peft/package_reference/ia3" id="package_reference/ia3">IA3 </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/peft/package_reference/llama_adapter" id="package_reference/llama_adapter">Llama-Adapter </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/peft/package_reference/loha" id="package_reference/loha">LoHa </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/peft/package_reference/lokr" id="package_reference/lokr">LoKr </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/peft/package_reference/lora" id="package_reference/lora">LoRA </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/peft/package_reference/xlora" id="package_reference/xlora">X-LoRA </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/peft/package_reference/adapter_utils" id="package_reference/adapter_utils">LyCORIS </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/peft/package_reference/multitask_prompt_tuning" id="package_reference/multitask_prompt_tuning">Multitask Prompt Tuning </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/peft/package_reference/oft" id="package_reference/oft">OFT </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/peft/package_reference/boft" id="package_reference/boft">BOFT </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/peft/package_reference/poly" id="package_reference/poly">Polytropon </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/peft/package_reference/p_tuning" id="package_reference/p_tuning">P-tuning </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/peft/package_reference/prefix_tuning" id="package_reference/prefix_tuning">Prefix tuning </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/peft/package_reference/prompt_tuning" id="package_reference/prompt_tuning">Prompt tuning </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/peft/package_reference/layernorm_tuning" id="package_reference/layernorm_tuning">Layernorm tuning </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/peft/package_reference/vera" id="package_reference/vera">VeRA </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/peft/package_reference/fourierft" id="package_reference/fourierft">FourierFT </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/peft/package_reference/vblora" id="package_reference/vblora">VB-LoRA </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/peft/package_reference/hra" id="package_reference/hra">HRA </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/peft/package_reference/cpt" id="package_reference/cpt">CPT </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/peft/package_reference/bone" id="package_reference/bone">Bone </a> </div> <div class="group flex cursor-pointer items-center pl-2 text-[0.8rem] font-semibold uppercase leading-9 hover:text-gray-700 dark:hover:text-gray-300 ml-2"><div class="flex after:absolute after:right-4 after:text-gray-500 group-hover:after:content-['▶'] after:rotate-90 after:transform"><span><span class="inline-block space-x-1 leading-5"><span>Utilities</span> </span></span> </div></div> <div class="flex flex-col"><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/peft/package_reference/merge_utils" id="package_reference/merge_utils">Model merge </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/peft/package_reference/helpers" id="package_reference/helpers">Helpers </a><a class="transform py-1 pl-2 pr-2 text-gray-500 first:mt-1 last:mb-4 hover:translate-x-px hover:text-black dark:hover:text-gray-300 ml-4" href="/docs/peft/package_reference/hotswap" id="package_reference/hotswap">Hotswapping adapters </a> </div> </div></nav></div></div></div> <div class="z-1 min-w-0 flex-1"> <div class="px-6 pt-6 md:px-12 md:pb-16 md:pt-16"><div class="max-w-4xl mx-auto mb-10"><div class="bg-linear-to-br relative overflow-hidden rounded-xl from-orange-300/10 px-4 py-5 ring-1 ring-orange-100/70 md:px-6 md:py-8"><img alt="Hugging Face's logo" class="absolute -bottom-6 -right-6 w-28 -rotate-45 md:hidden" src="/front/assets/huggingface_logo-noborder.svg"> <div class="mb-2 text-2xl font-bold dark:text-gray-200 md:mb-0">Join the Hugging Face community</div> <p class="mb-4 text-lg text-gray-400 dark:text-gray-300 md:mb-8">and get access to the augmented documentation experience </p> <div class="mb-8 hidden space-y-4 md:block xl:flex xl:space-x-6 xl:space-y-0"><div class="flex items-center"><div class="bg-linear-to-br mr-3 flex h-9 w-9 flex-none items-center justify-center rounded-lg from-indigo-100 to-indigo-100/20 dark:to-indigo-100"><svg class="text-indigo-400 group-hover:text-indigo-500" style="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path class="uim-quaternary" d="M20.23 7.24L12 12L3.77 7.24a1.98 1.98 0 0 1 .7-.71L11 2.76c.62-.35 1.38-.35 2 0l6.53 3.77c.29.173.531.418.7.71z" opacity=".25" fill="currentColor"></path><path class="uim-tertiary" d="M12 12v9.5a2.09 2.09 0 0 1-.91-.21L4.5 17.48a2.003 2.003 0 0 1-1-1.73v-7.5a2.06 2.06 0 0 1 .27-1.01L12 12z" opacity=".5" fill="currentColor"></path><path class="uim-primary" d="M20.5 8.25v7.5a2.003 2.003 0 0 1-1 1.73l-6.62 3.82c-.275.13-.576.198-.88.2V12l8.23-4.76c.175.308.268.656.27 1.01z" fill="currentColor"></path></svg></div> <div class="text-smd leading-tight text-gray-500 dark:text-gray-300 xl:max-w-[200px] 2xl:text-base">Collaborate on models, datasets and Spaces </div></div> <div class="flex items-center"><div class="bg-linear-to-br mr-3 flex h-9 w-9 flex-none items-center justify-center rounded-lg from-orange-100 to-orange-100/20 dark:to-orange-50"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" class="text-xl text-yellow-400" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path d="M11 15H6l7-14v8h5l-7 14v-8z" fill="currentColor"></path></svg></div> <div class="text-smd leading-tight text-gray-500 dark:text-gray-300 xl:max-w-[200px] 2xl:text-base">Faster examples with accelerated inference </div></div> <div class="flex items-center"><div class="bg-linear-to-br mr-3 flex h-9 w-9 flex-none items-center justify-center rounded-lg from-gray-500/10 to-gray-500/5"><svg class="text-gray-400" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M14.9804 3C14.9217 3.0002 14.8631 3.00555 14.8054 3.016C11.622 3.58252 8.76073 5.30669 6.77248 7.85653C4.78422 10.4064 3.80955 13.6016 4.03612 16.8271C4.26268 20.0525 5.67447 23.0801 7.99967 25.327C10.3249 27.5738 13.3991 28.8811 16.6304 28.997C16.7944 29.003 16.9584 28.997 17.1204 28.997C19.2193 28.9984 21.2877 28.4943 23.1507 27.5274C25.0137 26.5605 26.6164 25.1592 27.8234 23.442C27.9212 23.294 27.9783 23.1229 27.9889 22.9458C27.9995 22.7687 27.9633 22.592 27.884 22.4333C27.8046 22.2747 27.6848 22.1397 27.5367 22.0421C27.3887 21.9444 27.2175 21.8875 27.0404 21.877C25.0426 21.7017 23.112 21.0693 21.3976 20.0288C19.6832 18.9884 18.231 17.5676 17.1533 15.8764C16.0756 14.1852 15.4011 12.2688 15.1822 10.2754C14.9632 8.28193 15.2055 6.26484 15.8904 4.38C15.9486 4.22913 15.97 4.06652 15.9527 3.90572C15.9354 3.74492 15.8799 3.59059 15.7909 3.45557C15.7019 3.32055 15.5819 3.20877 15.4409 3.12952C15.2999 3.05028 15.142 3.00587 14.9804 3Z" fill="currentColor"></path></svg></div> <div class="text-smd leading-tight text-gray-500 dark:text-gray-300 xl:max-w-[200px] 2xl:text-base">Switch between documentation themes </div></div></div> <div class="flex items-center space-x-2.5"><a href="/join"><button class="bg-linear-to-br shadow-xs rounded-lg bg-white from-gray-100/20 to-gray-200/60 px-5 py-1.5 font-semibold text-gray-700 ring-1 ring-gray-300/60 hover:to-gray-100/70 hover:ring-gray-300/30 active:shadow-inner">Sign Up</button></a> <p class="text-gray-500 dark:text-gray-300">to get started</p></div></div></div> <div class="prose-doc prose relative mx-auto max-w-4xl break-words"> <link href="/docs/peft/v0.14.0/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload"> <link rel="modulepreload" href="/docs/peft/v0.14.0/en/_app/immutable/entry/start.e2f9cd26.js"> <link rel="modulepreload" href="/docs/peft/v0.14.0/en/_app/immutable/chunks/scheduler.c57aa7ef.js"> <link rel="modulepreload" href="/docs/peft/v0.14.0/en/_app/immutable/chunks/singletons.868508d1.js"> <link rel="modulepreload" href="/docs/peft/v0.14.0/en/_app/immutable/chunks/index.c4ac7298.js"> <link rel="modulepreload" href="/docs/peft/v0.14.0/en/_app/immutable/chunks/paths.7ce3a05b.js"> <link rel="modulepreload" href="/docs/peft/v0.14.0/en/_app/immutable/entry/app.5dc42805.js"> <link rel="modulepreload" href="/docs/peft/v0.14.0/en/_app/immutable/chunks/index.394f8801.js"> <link rel="modulepreload" href="/docs/peft/v0.14.0/en/_app/immutable/nodes/0.0975f0a0.js"> <link rel="modulepreload" href="/docs/peft/v0.14.0/en/_app/immutable/chunks/each.e59479a4.js"> <link rel="modulepreload" href="/docs/peft/v0.14.0/en/_app/immutable/nodes/14.8e3e3c3d.js"> <link rel="modulepreload" href="/docs/peft/v0.14.0/en/_app/immutable/chunks/Tip.23dfb1a9.js"> <link rel="modulepreload" href="/docs/peft/v0.14.0/en/_app/immutable/chunks/CodeBlock.5228ed01.js"> <link rel="modulepreload" href="/docs/peft/v0.14.0/en/_app/immutable/chunks/EditOnGithub.157fddbf.js"> <link rel="modulepreload" href="/docs/peft/v0.14.0/en/_app/immutable/chunks/HfOption.41e9cb41.js"><meta name="hf:doc:metadata" content="{"title":"Model merging","local":"model-merging","sections":[{"title":"Merge method","local":"merge-method","sections":[],"depth":2},{"title":"Merging (IA)³ Models","local":"merging-ia³-models","sections":[],"depth":2}],"depth":1}"> <p></p> <h1 class="relative group"><a id="model-merging" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#model-merging"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Model merging</span></h1> <p data-svelte-h="svelte-1aet04z">Training a model for each task can be costly, take up storage space, and the models aren’t able to learn new information to improve their performance. Multitask learning can overcome some of these limitations by training a model to learn several tasks, but it is expensive to train and designing a dataset for it is challenging. <em>Model merging</em> offers a solution to these challenges by combining multiple pretrained models into one model, giving it the combined abilities of each individual model without any additional training.</p> <p data-svelte-h="svelte-1c8162a">PEFT provides several methods for merging models like a linear or SVD combination. This guide focuses on two methods that are more efficient for merging LoRA adapters by eliminating redundant parameters:</p> <ul data-svelte-h="svelte-wlbyrl"><li><a href="https://hf.co/papers/2306.01708" rel="nofollow">TIES</a> - TrIm, Elect, and Merge (TIES) is a three-step method for merging models. First, redundant parameters are trimmed, then conflicting signs are resolved into an aggregated vector, and finally the parameters whose signs are the same as the aggregate sign are averaged. This method takes into account that some values (redundant and sign disagreement) can degrade performance in the merged model.</li> <li><a href="https://hf.co/papers/2311.03099" rel="nofollow">DARE</a> - Drop And REscale is a method that can be used to prepare for other model merging methods like TIES. It works by randomly dropping parameters according to a drop rate and rescaling the remaining parameters. This helps to reduce the number of redundant and potentially interfering parameters among multiple models.</li></ul> <p data-svelte-h="svelte-1wh0rno">Models are merged with the <a href="/docs/peft/v0.14.0/en/package_reference/lora#peft.LoraModel.add_weighted_adapter">add_weighted_adapter()</a> method, and the specific model merging method is specified in the <code>combination_type</code> parameter.</p> <h2 class="relative group"><a id="merge-method" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#merge-method"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Merge method</span></h2> <p data-svelte-h="svelte-gz5qim">With TIES and DARE, merging is enabled by setting <code>combination_type</code> and <code>density</code> to a value of the weights to keep from the individual models. For example, let’s merge three finetuned <a href="https://huggingface.co/TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T" rel="nofollow">TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T</a> models: <a href="https://huggingface.co/smangrul/tinyllama_lora_norobots" rel="nofollow">tinyllama_lora_nobots</a>, <a href="https://huggingface.co/smangrul/tinyllama_lora_sql" rel="nofollow">tinyllama_lora_sql</a>, and <a href="https://huggingface.co/smangrul/tinyllama_lora_adcopy" rel="nofollow">tinyllama_lora_adcopy</a>.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-lxcpzr">When you’re attempting to merge fully trained models with TIES, you should be aware of any special tokens each model may have added to the embedding layer which are not a part of the original checkpoint’s vocabulary. This may cause an issue because each model may have added a special token to the same embedding position. If this is the case, you should use the <a href="https://huggingface.co/docs/transformers/v4.47.0/en/main_classes/model#transformers.PreTrainedModel.resize_token_embeddings" rel="nofollow">resize_token_embeddings</a> method to avoid merging the special tokens at the same embedding index.</p> <br> <p data-svelte-h="svelte-1bgcfcq">This shouldn’t be an issue if you’re only merging LoRA adapters trained from the same base model.</p></div> <p data-svelte-h="svelte-r0kc03">Load a base model and can use the <a href="/docs/peft/v0.14.0/en/package_reference/peft_model#peft.PeftModel.load_adapter">load_adapter()</a> method to load and assign each adapter a name:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> PeftConfig, PeftModel <span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer <span class="hljs-keyword">import</span> torch config = PeftConfig.from_pretrained(<span class="hljs-string">"smangrul/tinyllama_lora_norobots"</span>) model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path, load_in_4bit=<span class="hljs-literal">True</span>, device_map=<span class="hljs-string">"auto"</span>).<span class="hljs-built_in">eval</span>() tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">"smangrul/tinyllama_lora_norobots"</span>) model = PeftModel.from_pretrained(model, <span class="hljs-string">"smangrul/tinyllama_lora_norobots"</span>, adapter_name=<span class="hljs-string">"norobots"</span>) _ = model.load_adapter(<span class="hljs-string">"smangrul/tinyllama_lora_sql"</span>, adapter_name=<span class="hljs-string">"sql"</span>) _ = model.load_adapter(<span class="hljs-string">"smangrul/tinyllama_lora_adcopy"</span>, adapter_name=<span class="hljs-string">"adcopy"</span>)</pre></div> <p data-svelte-h="svelte-1ghl6si">Set the adapters, weights, <code>adapter_name</code>, <code>combination_type</code>, and <code>density</code> with the <a href="/docs/peft/v0.14.0/en/package_reference/lora#peft.LoraModel.add_weighted_adapter">add_weighted_adapter()</a> method.</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">TIES </div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">DARE </div></div> <div class="language-select"><p data-svelte-h="svelte-11ncbcn">Weight values greater than <code>1.0</code> typically produce better results because they preserve the correct scale. A good default starting value for the weights is to set all values to <code>1.0</code>.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="">adapters = [<span class="hljs-string">"norobots"</span>, <span class="hljs-string">"adcopy"</span>, <span class="hljs-string">"sql"</span>] weights = [<span class="hljs-number">2.0</span>, <span class="hljs-number">1.0</span>, <span class="hljs-number">1.0</span>] adapter_name = <span class="hljs-string">"merge"</span> density = <span class="hljs-number">0.2</span> model.add_weighted_adapter(adapters, weights, adapter_name, combination_type=<span class="hljs-string">"ties"</span>, density=density)</pre></div> </div> <p data-svelte-h="svelte-7aulzq">Set the newly merged model as the active model with the <a href="/docs/peft/v0.14.0/en/package_reference/lora#peft.LoraModel.set_adapter">set_adapter()</a> method.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="">model.set_adapter(<span class="hljs-string">"merge"</span>)</pre></div> <p data-svelte-h="svelte-12c7cn9">Now you can use the merged model as an instruction-tuned model to write ad copy or SQL queries!</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">instruct </div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">ad copy </div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">SQL </div></div> <div class="language-select"><div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="">messages = [ {<span class="hljs-string">"role"</span>: <span class="hljs-string">"user"</span>, <span class="hljs-string">"content"</span>: <span class="hljs-string">"Write an essay about Generative AI."</span>}, ] text = tokenizer.apply_chat_template(messages, add_generation_prompt=<span class="hljs-literal">True</span>, tokenize=<span class="hljs-literal">False</span>) inputs = tokenizer(text, return_tensors=<span class="hljs-string">"pt"</span>) inputs = {k: v.to(<span class="hljs-string">"cuda"</span>) <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> inputs.items()} outputs = model.generate(**inputs, max_new_tokens=<span class="hljs-number">256</span>, do_sample=<span class="hljs-literal">True</span>, top_p=<span class="hljs-number">0.95</span>, temperature=<span class="hljs-number">0.2</span>, repetition_penalty=<span class="hljs-number">1.2</span>, eos_token_id=tokenizer.eos_token_id) <span class="hljs-built_in">print</span>(tokenizer.decode(outputs[<span class="hljs-number">0</span>]))</pre></div> </div> <h2 class="relative group"><a id="merging-ia³-models" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#merging-ia³-models"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Merging (IA)³ Models</span></h2> <p data-svelte-h="svelte-r9tv7c">The (IA)³ models facilitate linear merging of adapters. To merge adapters in an (IA)³ model, utilize the <code>add_weighted_adapter</code> method from the <code>IA3Model</code> class. This method is analogous to the <code>add_weighted_adapter</code> method used in <code>LoraModel</code>, with the key difference being the absence of the <code>combination_type</code> parameter. For example, to merge three (IA)³ adapters into a PEFT model, you would proceed as follows:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="">adapters = [<span class="hljs-string">"adapter1"</span>, <span class="hljs-string">"adapter2"</span>, <span class="hljs-string">"adapter3"</span>] weights = [<span class="hljs-number">0.4</span>, <span class="hljs-number">0.3</span>, <span class="hljs-number">0.3</span>] adapter_name = <span class="hljs-string">"merge"</span> model.add_weighted_adapter(adapters, weights, adapter_name)</pre></div> <p data-svelte-h="svelte-dmzjk6">It is recommended that the weights sum to 1.0 to preserve the scale of the model. The merged model can then be set as the active model using the <code>set_adapter</code> method:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="">model.set_adapter(<span class="hljs-string">"merge"</span>)</pre></div> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/peft/blob/main/docs/source/developer_guides/model_merging.md" target="_blank"><span data-svelte-h="svelte-1kd6by1"><</span> <span data-svelte-h="svelte-x0xyl0">></span> <span data-svelte-h="svelte-1dajgef"><span class="underline ml-1.5">Update</span> on GitHub</span></a> <p></p> <script> { __sveltekit_18aamxh = { assets: "/docs/peft/v0.14.0/en", base: "/docs/peft/v0.14.0/en", env: {} }; const element = document.currentScript.parentElement; const data = [null,null]; Promise.all([ import("/docs/peft/v0.14.0/en/_app/immutable/entry/start.e2f9cd26.js"), import("/docs/peft/v0.14.0/en/_app/immutable/entry/app.5dc42805.js") ]).then(([kit, app]) => { kit.start(app, element, { node_ids: [0, 14], data, form: null, error: null }); }); } </script> </div> <div class="SVELTE_HYDRATER contents" data-target="DocFooterNav" data-props="{"classNames":"mx-auto mt-16 flex max-w-4xl items-center pb-8 font-sans font-medium leading-6 xl:mt-32","chapterPrev":{"title":"IA3","isExpanded":true,"id":"task_guides/ia3","url":"/docs/peft/task_guides/ia3"},"chapterNext":{"title":"Quantization","isExpanded":true,"id":"developer_guides/quantization","url":"/docs/peft/developer_guides/quantization"},"isCourse":false,"isLoggedIn":false}"><div class="mx-auto mt-16 flex max-w-4xl items-center pb-8 font-sans font-medium leading-6 xl:mt-32"><a href="/docs/peft/task_guides/ia3" class="mr-8 flex transform items-center text-gray-600 transition-all hover:-translate-x-px hover:text-gray-900 dark:hover:text-gray-300"><span class="mr-2 translate-y-px">←</span>IA3</a> <a href="/docs/peft/developer_guides/quantization" class="ml-auto flex transform items-center text-right text-gray-600 transition-all hover:translate-x-px hover:text-gray-900 dark:hover:text-gray-300">Quantization<span class="ml-2 translate-y-px">→</span></a></div></div></div></div> <div class="sticky top-0 self-start"><div class="SVELTE_HYDRATER contents" data-target="SubSideMenu" data-props="{"chapter":{"title":"Model merging","isExpanded":true,"id":"model-merging","url":"#model-merging","sections":[{"title":"Merge method","isExpanded":true,"id":"merge-method","url":"#merge-method","sections":[]},{"title":"Merging (IA)³ Models","isExpanded":true,"id":"merging-ia³-models","url":"#merging-ia³-models","sections":[]}]}}"> <nav class="hidden h-dvh w-[270px] flex-none flex-col space-y-3 overflow-y-auto break-words border-l pb-16 pl-6 pr-10 pt-24 text-sm lg:flex 2xl:w-[305px]"> <a href="#model-merging" class=" text-gray-400 transform hover:translate-x-px hover:text-gray-700 dark:hover:text-gray-300" id="nav-model-merging"><wbr>Model merging</a> <a href="#merge-method" class="pl-4 text-gray-400 transform hover:translate-x-px hover:text-gray-700 dark:hover:text-gray-300" id="nav-merge-method"><wbr>Merge method</a> <a href="#merging-ia³-models" class="pl-4 text-gray-400 transform hover:translate-x-px hover:text-gray-700 dark:hover:text-gray-300" id="nav-merging-ia³-models"><wbr>Merging (I<wbr>A)³ <wbr>Models</a> </nav></div></div></div> <div id="doc-footer"></div></main> </div> <script> import("\/front\/build\/kube-0a779c5\/index.js"); window.moonSha = "kube-0a779c5\/"; window.__hf_deferred = {}; </script>  <script> if (["hf.co", "huggingface.co"].includes(window.location.hostname)) { const script = document.createElement("script"); script.src = "https://js.stripe.com/v3/"; script.async = true; document.head.appendChild(script); } </script> </body> </html>