model-alignment

<!DOCTYPE html> <html lang="en" dir="ltr"> <head> <meta charset="utf-8"> <meta http-equiv="X-UA-Compatible" content="IE=edge"> <meta name="viewport" content="width=device-width, initial-scale=1"> <meta name="defaultLanguage" content="en"> <meta name="availableLanguages" content="en, es, fr, ja, pt_BR, uk, el, de, zh_Hans, zh_Hant, ru, he, eo"> <title>model-alignment · PyPI</title> <meta name="description" content="Model Alignment: Aligning prompts to human preferences through natural language feedback"> <link rel="stylesheet" href="/static/css/warehouse-ltr.4c38f301.css"> <link rel="stylesheet" href="/static/css/fontawesome.da0464c1.css"> <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Source+Sans+3:400,400italic,600,600italic,700,700italic%7CSource+Code+Pro:500"> <noscript> <link rel="stylesheet" href="/static/css/noscript.0673c9ea.css"> </noscript> <link rel="icon" href="/static/images/favicon.35549fe8.ico" type="image/x-icon"> <link rel="alternate" type="application/rss+xml" title="RSS: 40 latest updates" href="/rss/updates.xml"> <link rel="alternate" type="application/rss+xml" title="RSS: 40 newest packages" href="/rss/packages.xml"> <link rel="alternate" type="application/rss+xml" title="RSS: latest releases for model-alignment" href="/rss/project/model-alignment/releases.xml"> <link rel="canonical" href="https://pypi.org/project/model-alignment/"> <meta property="og:url" content="https://pypi.org/project/model-alignment/"> <meta property="og:site_name" content="PyPI"> <meta property="og:type" content="website"> <meta property="og:image" content="https://pypi.org/static/images/twitter.abaf4b19.webp"> <meta property="og:title" content="model-alignment"> <meta property="og:description" content="Model Alignment: Aligning prompts to human preferences through natural language feedback"> <link rel="search" type="application/opensearchdescription+xml" title="PyPI" href="/opensearch.xml"> <script async data-ga-id="UA-55961911-1" data-ga4-id="G-RW7D75DF8V" src="/static/js/warehouse.f780b8ef.js"> </script> <script> MathJax = { tex: { inlineMath: [['$', '$'], ['\$', '\$']] }, }; </script> <script async src="https://cdn.jsdelivr.net/npm/mathjax@3.2.2/es5/tex-svg.js" integrity="sha256-1CldwzdEg2k1wTmf7s5RWVd7NMXI/7nxxjJM2C4DqII=" crossorigin="anonymous" ></script> <script async src="https://www.googletagmanager.com/gtag/js?id=UA-55961911-1"></script> <script async src="https://www.googletagmanager.com/gtag/js?id=G-RW7D75DF8V"></script> <script defer src="https://www.fastly-insights.com/insights.js?k=6a52360a-f306-421e-8ed5-7417d0d4a4e9&dnt=true"></script> <script async src="https://media.ethicalads.io/media/client/v1.4.0/ethicalads.min.js" integrity="sha256-U3hKDidudIaxBDEzwGJApJgPEf2mWk6cfMWghrAa6i0= sha384-UcmsCqcNRSLW/dV3Lo1oCi2/VaurXbib6p4HyUEOeIa/4OpsrnucrugAefzVZJfI sha512-q4t1L4xEjGV2R4hzqCa41P8jrgFUS8xTb8rdNv4FGvw7FpydVj/kkxBJHOiaoxHa8olCcx1Slk9K+3sNbsM4ug==" crossorigin="anonymous" ></script> </head> <body data-controller="viewport-toggle">  <a href="#content" class="skip-to-content">Skip to main content</a> <button type="button" class="button button--primary button--switch-to-mobile hidden" data-viewport-toggle-target="switchToMobile" data-action="viewport-toggle#switchToMobile"> Switch to mobile version </button> <div id="sticky-notifications" class="stick-to-top js-stick-to-top">   <noscript> <div class="notification-bar notification-bar--warning" role="status"> <span class="notification-bar__icon"> <i class="fa fa-exclamation-triangle" aria-hidden="true"></i> <span class="sr-only">Warning</span> </span> <span class="notification-bar__message">Some features may not work without JavaScript. Please try enabling it if you encounter problems.</span> </div> </noscript> <div data-html-include="/_includes/notification-banners/"></div> </div> <div data-html-include="/_includes/flash-messages/"></div> <div data-html-include="/_includes/session-notifications/"></div> <header class="site-header "> <div class="site-container"> <div class="split-layout"> <div class="split-layout"> <div> <a class="site-header__logo" href="/"> <img alt="PyPI" src="/static/images/logo-small.8998e9d1.svg"> </a> </div> <form class="search-form search-form--primary" action="/search/" role="search"> <label for="search" class="sr-only">Search PyPI</label> <input id="search" class="search-form__search" type="text" name="q" placeholder="Search projects" value="" autocomplete="off" autocapitalize="off" spellcheck="false" data-controller="search-focus" data-action="keydown@window->search-focus#focusSearchField" data-search-focus-target="searchField"> <button type="submit" class="search-form__button"> <i class="fa fa-search" aria-hidden="true"></i> <span class="sr-only">Search</span> </button> </form> </div> <div data-html-include="/_includes/current-user-indicator/"> <div id="user-indicator" class="horizontal-menu horizontal-menu--light horizontal-menu--tall"> <nav class="horizontal-menu horizontal-menu--light horizontal-menu--tall hide-on-tablet" aria-label="Main navigation"> <ul> <li class="horizontal-menu__item"><a href="/help/" class="horizontal-menu__link">Help</a></li> <li class="horizontal-menu__item"><a href="/sponsors/" class="horizontal-menu__link">Sponsors</a></li> <li class="horizontal-menu__item"><a href="/account/login/" class="horizontal-menu__link">Log in</a></li> <li class="horizontal-menu__item"><a href="/account/register/" class="horizontal-menu__link">Register</a></li> </ul> </nav> <nav class="dropdown dropdown--on-menu hidden show-on-tablet" aria-label="Main navigation"> <button type="button" class="horizontal-menu__link dropdown__trigger" aria-haspopup="true" aria-expanded="false" aria-label="View menu"> Menu <span class="dropdown__trigger-caret"> <i class="fa fa-caret-down" aria-hidden="true"></i> </span> </button> <ul class="dropdown__content" aria-hidden="true" aria-label="Main menu"> <li><a class="dropdown__link" href="/help/">Help</a></li> <li><a class="dropdown__link" href="/sponsors/">Sponsors</a></li> <li><a class="dropdown__link" href="/account/login/">Log in</a></li> <li><a class="dropdown__link" href="/account/register/">Register</a></li> </ul> </nav> </div> </div> </div> </div> </header> <div class="mobile-search"> <form class="search-form search-form--fullwidth" action="/search/" role="search"> <label for="mobile-search" class="sr-only">Search PyPI</label> <input id="mobile-search" class="search-form__search" type="text" name="q" placeholder="Search projects" value="" autocomplete="off" autocapitalize="off" spellcheck="false"> <button type="submit" class="search-form__button"> <i class="fa fa-search" aria-hidden="true"></i> <span class="sr-only">Search</span> </button> </form> </div> <main id="content"> <div class="banner"> <div class="package-header"> <div class="package-header__left"> <h1 class="package-header__name"> model-alignment 0.2 </h1> <div data-controller="clipboard"> <p class="package-header__pip-instructions"> <span id="pip-command" data-clipboard-target="source">pip install model-alignment</span> <button type="button" class="copy-tooltip copy-tooltip-s" data-action="clipboard#copy" data-clipboard-target="tooltip" data-clipboard-tooltip-value="Copy to clipboard"> <i class="fa fa-copy" aria-hidden="true"></i> <span class="sr-only">Copy PIP instructions</span> </button> </p> </div> </div> <div class="package-header__right"> <a class="status-badge status-badge--good" href="/project/model-alignment/"> <span>Latest version</span> </a> <p class="package-header__date"> Released: <time datetime="2024-10-07T17:26:47+0000" data-controller="localized-time" data-localized-time-relative="true" data-localized-time-show-time="false"> Oct 7, 2024 </time> </p> </div> </div> </div> <div class="horizontal-section horizontal-section--grey horizontal-section--thin"> <div class="site-container"> <div data-html-include="/_includes/administer-project-include/model-alignment"></div> <div class="split-layout split-layout--middle package-description"> <p class="package-description__summary">Model Alignment: Aligning prompts to human preferences through natural language feedback</p> <div data-html-include="/_includes/edit-project-button/model-alignment"></div> </div> </div> </div> <div data-controller="project-tabs"> <div class="tabs-container"> <div class="vertical-tabs"> <div class="vertical-tabs__tabs"> <div class="sidebar-section"> <h3 class="sidebar-section__title">Navigation</h3> <nav aria-label="Navigation for model-alignment"> <ul class="vertical-tabs__list" role="tablist"> <li role="tab"> <a id="description-tab" href="#description" data-project-tabs-target="tab" data-action="project-tabs#onTabClick" class="vertical-tabs__tab vertical-tabs__tab--with-icon vertical-tabs__tab--is-active" aria-selected="true" aria-label="Project description. Focus will be moved to the description."> <i class="fa fa-align-left" aria-hidden="true"></i> Project description </a> </li> <li role="tab"> <a id="history-tab" href="#history" data-project-tabs-target="tab" data-action="project-tabs#onTabClick" class="vertical-tabs__tab vertical-tabs__tab--with-icon" aria-label="Release history. Focus will be moved to the history panel."> <i class="fa fa-history" aria-hidden="true"></i> Release history </a> </li> <li role="tab"> <a id="files-tab" href="#files" data-project-tabs-target="tab" data-action="project-tabs#onTabClick" class="vertical-tabs__tab vertical-tabs__tab--with-icon" aria-label="Download files. Focus will be moved to the project files."> <i class="fa fa-download" aria-hidden="true"></i> Download files </a> </li> </ul> </nav> </div> <div class="sidebar-section verified"> <h3 class="sidebar-section__title"> Verified details <i class="fa fa-circle-check check" title="Verified by PyPI on 2024-10-07"></i> </h3> <small><i>These details have been <a href="https://docs.pypi.org/project_metadata/#verified-details">verified by PyPI</a></i></small> <h6>Maintainers</h6> <span class="sidebar-section__maintainer"> <a href="/user/jwexler/" aria-label=""> <span class="sidebar-section__user-gravatar"> <img src="https://pypi-camo.freetls.fastly.net/b2f88d821f503d8b7464803b17bf8cb734b5749f/68747470733a2f2f7365637572652e67726176617461722e636f6d2f6176617461722f34393034343934633864316139316564356234356339363835383061376530353f73697a653d3530" height="50" width="50" alt="Avatar for jwexler from gravatar.com" title="Avatar for jwexler from gravatar.com"> </span> <span class="sidebar-section__user-gravatar-text"> jwexler </span> </a> </span> </div> <div class="sidebar-section unverified"> <h3 class="sidebar-section__title">Unverified details</h3> <small><i>These details have <b>not</b> been verified by PyPI</i></small> <h6>Project links</h6> <ul class="vertical-tabs__list"> <li> <a class="vertical-tabs__tab vertical-tabs__tab--with-icon vertical-tabs__tab--condensed" href="https://github.com/PAIR-code/model-alignment/issues" rel="nofollow"> <i class="fas fa-bug" aria-hidden="true"></i>Bug Tracker </a> </li> <li> <a class="vertical-tabs__tab vertical-tabs__tab--with-icon vertical-tabs__tab--condensed" href="https://github.com/pair-code/model-alignment" rel="nofollow"> <i class="fas fa-home" aria-hidden="true"></i>Homepage </a> </li> <li> <a class="vertical-tabs__tab vertical-tabs__tab--with-icon vertical-tabs__tab--condensed" href="https://github.com/PAIR-code/model-alignment" rel="nofollow"> <i class="fab fa-github" aria-hidden="true"></i>Repository </a> </li> </ul> <div class="sidebar-section unverified"> <h6>Meta</h6> <ul> <li> <span> <strong>License:</strong> Apache Software License ( Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR ...) </span> </li> <li> <span> <strong>Author:</strong> Google, LLC </span> </li> <li class="tags"> <span> <i class="fa fa-tags" aria-hidden="true"></i> <span class="sr-only">Tags</span> <span class="package-keyword"> explainable, </span> <span class="package-keyword"> explainable artifical intelligence, </span> <span class="package-keyword"> XAI, </span> <span class="package-keyword"> natural language processing, </span> <span class="package-keyword"> artifical intelligence, </span> <span class="package-keyword"> machine learning, </span> <span class="package-keyword"> deep learning </span> </span> </li> <li> <span> <strong>Requires:</strong> Python >=3.9 </span> </li> </ul> </div> <div class="sidebar-section unverified"> <h6 class="sidebar-section__title">Classifiers</h6> <ul class="sidebar-section__classifiers"> <li> <strong>Intended Audience</strong> <ul> <li> <a href="/search/?c=Intended+Audience+%3A%3A+Developers"> Developers </a> </li> <li> <a href="/search/?c=Intended+Audience+%3A%3A+Science%2FResearch"> Science/Research </a> </li> </ul> </li> <li> <strong>License</strong> <ul> <li> <a href="/search/?c=License+%3A%3A+OSI+Approved+%3A%3A+Apache+Software+License"> OSI Approved :: Apache Software License </a> </li> </ul> </li> <li> <strong>Operating System</strong> <ul> <li> <a href="/search/?c=Operating+System+%3A%3A+OS+Independent"> OS Independent </a> </li> </ul> </li> <li> <strong>Programming Language</strong> <ul> <li> <a href="/search/?c=Programming+Language+%3A%3A+Python+%3A%3A+3+%3A%3A+Only"> Python :: 3 :: Only </a> </li> <li> <a href="/search/?c=Programming+Language+%3A%3A+Python+%3A%3A+3.9"> Python :: 3.9 </a> </li> <li> <a href="/search/?c=Programming+Language+%3A%3A+Python+%3A%3A+3.10"> Python :: 3.10 </a> </li> </ul> </li> <li> <strong>Topic</strong> <ul> <li> <a href="/search/?c=Topic+%3A%3A+Scientific%2FEngineering+%3A%3A+Artificial+Intelligence"> Scientific/Engineering :: Artificial Intelligence </a> </li> </ul> </li> </ul> </div> </div><div class="sidebar-section" data-ea-publisher="psf" data-ea-type="psf" data-ea-keywords="pypi-sidebar"></div> <div data-html-include="https://pypi.org/_includes/submit-malware-report/model-alignment"></div> </div> <div class="vertical-tabs__panel">  <nav aria-label="Navigation for model-alignment"> <ul class="vertical-tabs__list" role="tablist"> <li role="tab"> <a id="mobile-description-tab" href="#description" data-project-tabs-target="mobileTab" data-action="project-tabs#onTabClick" class="vertical-tabs__tab vertical-tabs__tab--with-icon vertical-tabs__tab--mobile vertical-tabs__tab--no-top-border vertical-tabs__tab--is-active" aria-selected="true" aria-label="Project description. Focus will be moved to the description."> <i class="fa fa-align-left" aria-hidden="true"></i> Project description </a> </li> <li role="tab"> <a id="mobile-data-tab" href="#data" data-project-tabs-target="mobileTab" data-action="project-tabs#onTabClick" class="vertical-tabs__tab vertical-tabs__tab--with-icon vertical-tabs__tab--mobile" aria-label="Project details. Focus will be moved to the project details."> <i class="fa fa-info-circle" aria-hidden="true"></i> Project details </a> </li> <li role="tab"> <a id="mobile-history-tab" href="#history" data-project-tabs-target="mobileTab" data-action="project-tabs#onTabClick" class="vertical-tabs__tab vertical-tabs__tab--with-icon vertical-tabs__tab--mobile" aria-label="Release history. Focus will be moved to the history panel."> <i class="fa fa-history" aria-hidden="true"></i> Release history </a> </li> <li role="tab"> <a id="mobile-files-tab" href="#files" data-project-tabs-target="mobileTab" data-action="project-tabs#onTabClick" class="vertical-tabs__tab vertical-tabs__tab--with-icon vertical-tabs__tab--mobile" aria-label="Download files. Focus will be moved to the project files."> <i class="fa fa-download" aria-hidden="true"></i> Download files </a> </li> </ul> </nav> <div id="description" data-project-tabs-target="content" class="vertical-tabs__content" role="tabpanel" aria-labelledby="description-tab mobile-description-tab" tabindex="-1"> <h2 class="page-title">Project description</h2> <div class="project-description"> <h1>Model Alignment</h1> <p>Model Alignment is a python library from the <a href="https://pair.withgoogle.com/" rel=nofollow>PAIR</a> team that enable users to create model prompts through user feedback instead of manual prompt writing and editing. The technique makes use of constitutional principles to align prompts to users' desired values.</p> <p>The library ships with two different APIs:</p> <ul> <li> <p>Single-run Prompts: Interactively critique/kudos/rewrite responses to a prompt in order to create principles that are used to update a prompt.</p> </li> <li> <p>Constitutional Experts: Use labeled training data to automatically create a prompt based on principles derived from the data.</p> </li> </ul> <h2>Installation</h2> <p>Install the model-alignment library through pip:</p> <p><code>pip install model-alignment</code></p> <h2>Use-Cases</h2> <h3>Single-Run Prompts</h3> <p>Check out our research paper <a href="https://arxiv.org/abs/2310.15428" rel=nofollow>ConstitutionMaker: Interactively Critiquing Large Language Models by Converting Feedback into Principles</a> from <a href="https://iui.acm.org/2024/" rel=nofollow>IUI 2024</a> for details on principle-based prompt alignment through interactive feedback.</p> <p>A simplified workflow of aligning single-run prompts can be seen in the image below:</p> <p><img src="https://pypi-camo.freetls.fastly.net/72bd59d28904a2da11ff94bd2b3f2d82cf606fa5/696d672f6d6f64656c5f616c69676e6d656e742e737667" alt="Simplified workflow for alignment for single-run prompts"></p> <h4>Demo Notebook</h4> <p>A demo Colab notebook for single-run interactive prompt alignment can be found at <a href="https://colab.research.google.com/github/pair-code/model-alignment/blob/main/notebooks/Single_Run_Prompt_API_for_Model_Alignment.ipynb" rel=nofollow>https://colab.research.google.com/github/pair-code/model-alignment/blob/main/notebooks/Single_Run_Prompt_API_for_Model_Alignment.ipynb</a>.</p> <p>A separate demo Colab notebook that makes use of the open-weights Gemma model can be found at <a href="https://colab.research.google.com/github/pair-code/model-alignment/blob/main/notebooks/Gemma_for_Model_Alignment.ipynb" rel=nofollow>https://colab.research.google.com/github/pair-code/model-alignment/blob/main/notebooks/Gemma_for_Model_Alignment.ipynb</a>.</p> <h4>API Details</h4> <p>The main class to support single-run prompt alignment can be found in the <code>AlignableSingleRun</code> class. The data for a given single-run prompt, including its principles, is stored by this python class in the <code>ConstitutionalPrompt</code> object.</p> <p><code>AlignableSingleRun</code> must be provided an instance of a <code>ModelHelper</code> in its constructor. The <code>ModelHelper</code> wraps the LLM that will be used to both provide a response given the prompt and also to create principles from user feedback. We provide an implementation of <code>ModelHelper</code> for using <a href="https://ai.google.dev/api/python/google/generativeai" rel=nofollow>Gemini models</a> through Google's Developer API and one for using the open-weights <a href="https://www.kaggle.com/models/google/gemma-2" rel=nofollow>Gemma models</a> running locally. If desired, separate <code>ModelHelper</code> instances can be provided in the constructor with the first instance used for the running of the prompt and the second used for the model calls to perform alignment of the prompt.</p> <h5>Prompt Construction and Model Response.</h5> <p>Single-run prompts consist of three parts:</p> <ol> <li>A prompt preamble provided by the client, referred to as a <code>model_description</code>. Prompts can include variables that can be set when the prompt is run by wrapping the variables in curly brackets.</li> <li>A set of constitutional principles that are automatically added after the preamble.</li> <li>A user input for the specific request to the model.</li> </ol> <p>For example, you could create a single-run prompt for providing creative uses for an item by setting the model description to "Give three uses for the following item: {item}". Note that any variables in a prompt must be wrapped in curly brackets.</p> <pre lang=python3><span class=n>single_run_prompt</span> <span class=o>=</span> <span class=n>single_run</span><span class=o>.</span><span class=n>AlignableSingleRun</span><span class=p>(</span><span class=n>model_helper</span><span class=o>.</span><span class=n>PalmModelHelper</span><span class=p>(</span><span class=n>palm_api_key</span><span class=p>))</span> <span class=n>single_run_prompt</span><span class=o>.</span><span class=n>set_model_description</span><span class=p>(</span><span class=s2>"Give three uses for the following item: </span><span class=si>{item}</span><span class=s2>"</span><span class=p>)</span> </pre> <p>Then you can call the model through the <code>send_input</code> method which takes in a dictionary with the mapping of any variables in the prompt to the values they should use. If the prompt does not contain any variables, then no inputs are necessary to the <code>send_input</code> method.</p> <pre lang=python3><span class=n>response</span> <span class=o>=</span> <span class=n>single_run_prompt</span><span class=o>.</span><span class=n>send_input</span><span class=p>({</span><span class=s2>"item"</span><span class=p>:</span> <span class=s2>"scissors"</span><span class=p>})</span> <span class=nb>print</span><span class=p>(</span><span class=n>response</span><span class=p>)</span> </pre> <h5>Generating Principles From Feedback and Applying Them.</h5> <p>After getting a model response through the <code>send_input</code> method, you can provide feedback for the prompt with that input through one of three methods. The result of any of these methods is that the principles list stored by the single-run prompt object will be updated based on the feedback, and the updated principle list will be returned:</p> <ol> <li><code>critique_response</code> where you provide a reason why the response could use improvement.</li> <li><code>kudos_response</code> where you provide a reason why the response was a good response.</li> <li><code>rewrite_response</code> where you provide your own version of the model response that better aligns with what you wanted the model to return.</li> </ol> <p>In all of these cases, the principles stored by the prompt will be updated in one of three ways:</p> <ol> <li>A principle may be added to capture the feedback.</li> <li>An existing principle may be deleted if the feedback contradicts an existing principle.</li> <li>A principle may be updated if the feedback is related to an existing principle.</li> </ol> <p>Once you have a set of principles you wish to apply to the prompt, the method <code>update_model_description_from_principles</code> will update the stored prompt based on the principles and return the newly-updated prompt.</p> <p>Future calls to <code>send_input</code> will make use of the updated prompt.</p> <pre lang=python3><span class=n>response</span> <span class=o>=</span> <span class=n>single_run_prompt</span><span class=o>.</span><span class=n>send_input</span><span class=p>(</span><span class=s2>"scissors"</span><span class=p>)</span> <span class=c1># Response contains "1. Cut paper.\n2. Cut cloth.\n3. Cut hair."</span> <span class=n>single_run_prompt</span><span class=o>.</span><span class=n>critique_response</span><span class=p>(</span><span class=s2>"the list doesn't include any creative uses"</span><span class=p>)</span> <span class=c1># The returned principles list now includes the principle "Include at least one creative use for the item"</span> <span class=n>single_run_prompt</span><span class=o>.</span><span class=n>update_model_description_from_principles</span><span class=p>()</span> <span class=c1># The prompt is now updated based on the new principle and the updated prompt is returned.</span> <span class=n>new_response</span> <span class=o>=</span> <span class=n>single_run_prompt</span><span class=o>.</span><span class=n>send_input</span><span class=p>(</span><span class=s2>"scissors"</span><span class=p>)</span> <span class=c1># The new response contains "1. Cut paper.\n2. Cut cloth.\n3. Make paper snowflakes."</span> </pre> <h5>Automatically Creating Feedback</h5> <p>After calling <code>send_input</code>, there are two methods you can call to get auto-generated feedback that can be provided to the <code>critique_response</code> and <code>kudos_response</code> methods described above. This can help you ideate the types of feedback you may wish to provide for generating principles.</p> <p>These methods are <code>generate_critiques</code> and <code>generate_kudos</code> and each will return a list of proposed feedback statements.</p> <pre lang=python3><span class=n>response</span> <span class=o>=</span> <span class=n>single_run_prompt</span><span class=o>.</span><span class=n>send_input</span><span class=p>(</span><span class=s2>"scissors"</span><span class=p>)</span> <span class=c1># Response contains "1. Cut paper.\n2. Cut cloth.\n3. Cut hair."</span> <span class=n>single_run_prompt</span><span class=o>.</span><span class=n>generate_critiques</span><span class=p>()</span> <span class=c1># The returned critiques list is ["The first item is too general.", "The items are missing supporting details.", "The first and second items are too similar."]</span> </pre> <h5>Direct Access to the Underlying Prompt Info</h5> <p>The <code>data</code> property of the <code>AlignableSingleRun</code> object contains the <code>ConstitutionalPrompt</code> object that stores the model description, list of principles, and the list of inputs and their corresponding outputs from using this prompt.</p> <h5>Updating Prompts Directly From Feedback</h5> <p>This library also provides an alternative approach to updating prompts from user-provided feedback. Instead of turning feedback into constitutional principles, and then updating prompts from a list of principles, prompts can be updated directly based on feedback without going through the intermediate step of constitutional principle creation.</p> <p>This approach is more efficient in that it only requires a single model call for each piece of feedback, as opposed to two calls (one to turn feedback into a principle and then another to update the prompt based on the principles). But, this approach doesn't provide an explicit list of principles. Such a list can be helpful for understanding the goals of a prompt and also for sharing principles between prompts.</p> <p>For this approach, call <code>update_model_description_from_feedback</code> with feedback based on a use of the prompt. The prompt will be automatically updated and returned by this method call.</p> <pre lang=python3><span class=n>response</span> <span class=o>=</span> <span class=n>single_run_prompt</span><span class=o>.</span><span class=n>send_input</span><span class=p>(</span><span class=s2>"scissors"</span><span class=p>)</span> <span class=c1># Response contains "1. Cut paper.\n2. Cut cloth.\n3. Cut hair."</span> <span class=n>single_run_prompt</span><span class=o>.</span><span class=n>update_model_description_from_feedback</span><span class=p>(</span><span class=s2>"the list doesn't include any creative uses"</span><span class=p>)</span> <span class=c1># The returned updated prompt may be something like ""Give three uses for the following item, including at least one creative use: {item}"</span> </pre> <h3>Constitutional Experts</h3> <p>We provide a library for training / evolving natural language classification prompts. The evolution protocol is modeled off standard neural network training concepts:</p> <ul> <li>Weights / parameters: the natural language prompt (the final artifact from a training run)</li> <li>Gradient descent: diagnose the issue with the current version of the prompt (e.g. the prompt uses overly formal language), and edit the prompt in the opposite direction (e.g. edit the prompt to use less formal language)</li> <li>Optimization objective: to find a natural language prompt that generates accurate labels given a labeled dataset</li> </ul> <p>Check out our research paper <a href="https://arxiv.org/abs/2403.04894" rel=nofollow>ConstitutionalExperts: Training a Mixture of Principle-based Prompts</a> from <a href="https://2024.aclweb.org/" rel=nofollow>ACL 2024</a> for more information including detailed evaluations.</p> <h4>Demo Notebook</h4> <p>A demo Colab notebook for single-run interactive prompt alignment can be found at <a href="https://colab.research.google.com/github/pair-code/model-alignment/blob/main/notebooks/Labeler_API_for_Model_Alignment.ipynb" rel=nofollow>https://colab.research.google.com/github/pair-code/model-alignment/blob/main/notebooks/Labeler_API_for_Model_Alignment.ipynb</a>.</p> <h4>API Details</h4> <p>The main class for building a labeler can be found in the <code>Labeler</code> class. To initialize a labeler, the user provides a few parameters that describe the labeling task:</p> <pre lang=python3><span class=n>labeler_maker</span> <span class=o>=</span> <span class=n>labeler</span><span class=o>.</span><span class=n>Labeler</span><span class=p>(</span><span class=n>input_names</span><span class=o>=</span><span class=p>[</span><span class=s1>'comment'</span><span class=p>],</span> <span class=c1># The input features on which classification is based.</span> <span class=n>label_name</span><span class=o>=</span><span class=s1>'isHate'</span><span class=p>,</span> <span class=c1># The feature to be predicted.</span> <span class=n>label_values</span> <span class=o>=</span> <span class=p>[</span><span class=kc>True</span><span class=p>,</span> <span class=kc>False</span><span class=p>],</span> <span class=n>task_description</span><span class=o>=</span><span class=s1>'Does the example contain offensive text?'</span><span class=p>,</span> <span class=n>train_model_helper</span><span class=o>=</span><span class=n>model_helper</span><span class=o>.</span><span class=n>GeminiModelHelper</span><span class=p>(</span><span class=n>api_key</span><span class=p>,</span> <span class=n>model_name</span><span class=o>=</span><span class=s1>'gemini-pro'</span><span class=p>),</span> <span class=n>eval_model_helper</span><span class=o>=</span><span class=n>model_helper</span><span class=o>.</span><span class=n>GeminiModelHelper</span><span class=p>(</span><span class=n>api_key</span><span class=p>,</span> <span class=n>model_name</span><span class=o>==</span><span class=s1>'gemini-pro'</span><span class=p>))</span> </pre> <p>As part of initialization the user also defines a <code>train_model_helper</code> (the model to be used in training) and <code>eval_model_helper</code> (the model to be used for evaluation). We recommend using a larger model for training.</p> <p>We provide a high-level API that lets users train classification prompts in a hands-off fashion:</p> <pre lang=python3><span class=n>toxicity_labeler</span> <span class=o>=</span> <span class=n>labeler_maker</span><span class=o>.</span><span class=n>initialize_checkpoint</span><span class=p>(</span><span class=n>train_df</span><span class=p>)</span> <span class=k>for</span> <span class=n>i</span> <span class=ow>in</span> <span class=nb>range</span><span class=p>(</span><span class=n>NUM_TRAIN_STEPS</span><span class=p>):</span> <span class=n>toxicity_labeler</span> <span class=o>=</span> <span class=n>labeler_maker</span><span class=o>.</span><span class=n>train_step</span><span class=p>(</span><span class=n>toxicity_labeler</span><span class=p>,</span> <span class=n>train_df</span><span class=p>)</span> <span class=n>predictions</span> <span class=o>=</span> <span class=n>labeler_maker</span><span class=o>.</span><span class=n>infer_checkpoint</span><span class=p>(</span><span class=n>toxicity_labeler</span><span class=p>,</span> <span class=n>test_df</span><span class=p>)</span> <span class=n>scorecard</span> <span class=o>=</span> <span class=n>labeler_maker</span><span class=o>.</span><span class=n>get_scorecard</span><span class=p>(</span><span class=n>test_df</span><span class=p>,</span> <span class=n>predictions</span><span class=p>)</span> <span class=nb>print</span><span class=p>(</span><span class=sa>f</span><span class=s2>"Accuracy: </span><span class=si>{</span><span class=n>scorecard</span><span class=p>[</span><span class=s1>'accuracy'</span><span class=p>]</span><span class=si>}</span><span class=s2>"</span><span class=p>)</span> </pre> <p>The user only needs to provide a training (<code>train_df</code>) and evaluation (<code>test_df</code>) set.</p> <p>The final artifact produced at the end of training is a natural language prompt, formulated in a "constitutional" style (the classification objective is formulated in terms of principles). For example, the following is a sample constitutional prompt trained on a toxicity classification objective:</p> <pre lang=python3><span class=n>labeler</span><span class=o>.</span><span class=n>print_checkpoint</span><span class=p>(</span><span class=n>toxicity_labeler</span><span class=p>)</span> </pre> <p>==================== Attributes for: False ====================</p> <p>93931: Comment does not contain negative or hateful language towards a person or group.</p> <p>==================== Attributes for: True ====================</p> <p>62194: Comment contains content that expresses hate or discrimination towards a marginalized group.</p> <p>This prompt can be used on unlabeled examples like so:</p> <pre lang=python3><span class=n>new_examples</span> <span class=o>=</span> <span class=n>pd</span><span class=o>.</span><span class=n>DataFrame</span><span class=p>({</span><span class=s1>'comment'</span><span class=p>:</span> <span class=p>[</span><span class=s1>'I hate all people'</span><span class=p>,</span> <span class=s1>'I love my sister'</span><span class=p>]})</span> <span class=n>predictions</span> <span class=o>=</span> <span class=n>labeler_maker</span><span class=o>.</span><span class=n>infer_checkpoint</span><span class=p>(</span><span class=n>toxicity_labeler</span><span class=p>,</span> <span class=n>new_examples</span><span class=p>)</span> <span class=c1># Review predictions</span> <span class=k>for</span> <span class=n>index</span><span class=p>,</span> <span class=n>item</span> <span class=ow>in</span> <span class=nb>enumerate</span><span class=p>(</span><span class=nb>zip</span><span class=p>(</span><span class=n>new_examples</span><span class=o>.</span><span class=n>iterrows</span><span class=p>(),</span> <span class=n>predictions</span><span class=p>)):</span> <span class=n>i</span><span class=p>,</span> <span class=n>example</span> <span class=o>=</span> <span class=n>item</span><span class=p>[</span><span class=mi>0</span><span class=p>]</span> <span class=nb>print</span><span class=p>(</span><span class=sa>f</span><span class=s2>"Example </span><span class=si>{</span><span class=n>index</span><span class=si>}</span><span class=s2>"</span><span class=p>)</span> <span class=n>features</span> <span class=o>=</span> <span class=n>example</span><span class=p>[</span><span class=n>INPUTS</span><span class=p>]</span><span class=o>.</span><span class=n>to_dict</span><span class=p>()</span> <span class=k>for</span> <span class=n>feature</span> <span class=ow>in</span> <span class=n>INPUTS</span><span class=p>:</span> <span class=nb>print</span><span class=p>(</span><span class=sa>f</span><span class=s1>'</span><span class=si>{</span><span class=n>feature</span><span class=si>}</span><span class=s1>: </span><span class=si>{</span><span class=n>features</span><span class=p>[</span><span class=n>feature</span><span class=p>]</span><span class=si>}</span><span class=s1>'</span><span class=p>)</span> <span class=nb>print</span><span class=p>(</span><span class=s2>"Prediction:"</span><span class=p>,</span> <span class=n>item</span><span class=p>[</span><span class=mi>1</span><span class=p>][</span><span class=s1>'prediction'</span><span class=p>])</span> </pre> <h4>Training hyperparameters</h4> <p>Labeler performance can be greatly impacted by the choice of hyperparameters. By default, we initialize with the following hyperparameters:</p> <pre lang=python3><span class=n>DEFAULT_HPARAMS</span><span class=p>:</span> <span class=n>TrainingHyperparameters</span> <span class=o>=</span> <span class=p>{</span> <span class=s1>'n_incorrect_predictions_to_sample'</span><span class=p>:</span> <span class=mi>3</span><span class=p>,</span> <span class=s1>'n_mutations_to_try'</span><span class=p>:</span> <span class=mi>2</span><span class=p>,</span> <span class=s1>'n_best'</span><span class=p>:</span> <span class=mi>3</span><span class=p>,</span> <span class=s1>'score_fn'</span><span class=p>:</span> <span class=s1>'accuracy'</span><span class=p>,</span> <span class=s1>'n_validation_examples'</span><span class=p>:</span> <span class=mi>100</span><span class=p>,</span> <span class=s1>'truncation_length'</span><span class=p>:</span> <span class=mi>100</span><span class=p>,</span> <span class=p>}</span> </pre> <p>Hyperparameters are documented in <code>labeler_types.py</code>. In general, increasing the hyperparameter values will improve the accuracy of the prompt, but will increase training time.</p> </div> </div> <div id="data" data-project-tabs-target="content" class="vertical-tabs__content" role="tabpanel" aria-labelledby="mobile-data-tab" tabindex="-1"> <h2 class="page-title">Project details</h2> <div class="sidebar-section verified"> <h3 class="sidebar-section__title"> Verified details <i class="fa fa-circle-check check" title="Verified by PyPI on 2024-10-07"></i> </h3> <small><i>These details have been <a href="https://docs.pypi.org/project_metadata/#verified-details">verified by PyPI</a></i></small> <h6>Maintainers</h6> <span class="sidebar-section__maintainer"> <a href="/user/jwexler/" aria-label=""> <span class="sidebar-section__user-gravatar"> <img src="https://pypi-camo.freetls.fastly.net/b2f88d821f503d8b7464803b17bf8cb734b5749f/68747470733a2f2f7365637572652e67726176617461722e636f6d2f6176617461722f34393034343934633864316139316564356234356339363835383061376530353f73697a653d3530" height="50" width="50" alt="Avatar for jwexler from gravatar.com" title="Avatar for jwexler from gravatar.com"> </span> <span class="sidebar-section__user-gravatar-text"> jwexler </span> </a> </span> </div> <div class="sidebar-section unverified"> <h3 class="sidebar-section__title">Unverified details</h3> <small><i>These details have <b>not</b> been verified by PyPI</i></small> <h6>Project links</h6> <ul class="vertical-tabs__list"> <li> <a class="vertical-tabs__tab vertical-tabs__tab--with-icon vertical-tabs__tab--condensed" href="https://github.com/PAIR-code/model-alignment/issues" rel="nofollow"> <i class="fas fa-bug" aria-hidden="true"></i>Bug Tracker </a> </li> <li> <a class="vertical-tabs__tab vertical-tabs__tab--with-icon vertical-tabs__tab--condensed" href="https://github.com/pair-code/model-alignment" rel="nofollow"> <i class="fas fa-home" aria-hidden="true"></i>Homepage </a> </li> <li> <a class="vertical-tabs__tab vertical-tabs__tab--with-icon vertical-tabs__tab--condensed" href="https://github.com/PAIR-code/model-alignment" rel="nofollow"> <i class="fab fa-github" aria-hidden="true"></i>Repository </a> </li> </ul> <div class="sidebar-section unverified"> <h6>Meta</h6> <ul> <li> <span> <strong>License:</strong> Apache Software License ( Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR ...) </span> </li> <li> <span> <strong>Author:</strong> Google, LLC </span> </li> <li class="tags"> <span> <i class="fa fa-tags" aria-hidden="true"></i> <span class="sr-only">Tags</span> <span class="package-keyword"> explainable, </span> <span class="package-keyword"> explainable artifical intelligence, </span> <span class="package-keyword"> XAI, </span> <span class="package-keyword"> natural language processing, </span> <span class="package-keyword"> artifical intelligence, </span> <span class="package-keyword"> machine learning, </span> <span class="package-keyword"> deep learning </span> </span> </li> <li> <span> <strong>Requires:</strong> Python >=3.9 </span> </li> </ul> </div> <div class="sidebar-section unverified"> <h6 class="sidebar-section__title">Classifiers</h6> <ul class="sidebar-section__classifiers"> <li> <strong>Intended Audience</strong> <ul> <li> <a href="/search/?c=Intended+Audience+%3A%3A+Developers"> Developers </a> </li> <li> <a href="/search/?c=Intended+Audience+%3A%3A+Science%2FResearch"> Science/Research </a> </li> </ul> </li> <li> <strong>License</strong> <ul> <li> <a href="/search/?c=License+%3A%3A+OSI+Approved+%3A%3A+Apache+Software+License"> OSI Approved :: Apache Software License </a> </li> </ul> </li> <li> <strong>Operating System</strong> <ul> <li> <a href="/search/?c=Operating+System+%3A%3A+OS+Independent"> OS Independent </a> </li> </ul> </li> <li> <strong>Programming Language</strong> <ul> <li> <a href="/search/?c=Programming+Language+%3A%3A+Python+%3A%3A+3+%3A%3A+Only"> Python :: 3 :: Only </a> </li> <li> <a href="/search/?c=Programming+Language+%3A%3A+Python+%3A%3A+3.9"> Python :: 3.9 </a> </li> <li> <a href="/search/?c=Programming+Language+%3A%3A+Python+%3A%3A+3.10"> Python :: 3.10 </a> </li> </ul> </li> <li> <strong>Topic</strong> <ul> <li> <a href="/search/?c=Topic+%3A%3A+Scientific%2FEngineering+%3A%3A+Artificial+Intelligence"> Scientific/Engineering :: Artificial Intelligence </a> </li> </ul> </li> </ul> </div> </div> <br> </div> <div id="history" data-project-tabs-target="content" class="vertical-tabs__content" role="tabpanel" aria-labelledby="history-tab mobile-history-tab" tabindex="-1"> <h2 class="page-title split-layout"> <span>Release history</span> <span class="reset-text margin-top"> <a href="/help/#project-release-notifications">Release notifications</a> | <a href="/rss/project/model-alignment/releases.xml">RSS feed <i class="fa fa-rss" aria-hidden="true"></i></a> </span> </h2> <div class="release-timeline"> <div class="release release--latest release--current"> <div class="release__meta"> <span class="badge">This version</span> </div> <div class="release__graphic"> <div class="release__line"></div> <img class="release__node" alt="" src="https://pypi.org/static/images/blue-cube.572a5bfb.svg"> </div> <a class="card release__card" href="/project/model-alignment/0.2/"> <p class="release__version"> 0.2 </p> <p class="release__version-date"> <time datetime="2024-10-07T17:26:47+0000" data-controller="localized-time" data-localized-time-relative="true" data-localized-time-show-time="false"> Oct 7, 2024 </time> </p> </a> </div> <div class="release release--oldest"> <div class="release__meta"> </div> <div class="release__graphic"> <div class="release__line"></div> <img class="release__node" alt="" src="https://pypi.org/static/images/white-cube.2351a86c.svg"> </div> <a class="card release__card" href="/project/model-alignment/0.1/"> <p class="release__version"> 0.1 </p> <p class="release__version-date"> <time datetime="2024-05-13T14:23:12+0000" data-controller="localized-time" data-localized-time-relative="true" data-localized-time-show-time="false"> May 13, 2024 </time> </p> </a> </div> </div> </div> <div id="files" data-project-tabs-target="content" class="vertical-tabs__content" role="tabpanel" aria-labelledby="files-tab mobile-files-tab" tabindex="-1"> <h2 class="page-title">Download files</h2> <p>Download the file for your platform. If you're not sure which to choose, learn more about <a href="https://packaging.python.org/tutorials/installing-packages/" title="External link" target="_blank" rel="noopener">installing packages</a>.</p> <h3> Source Distributions </h3> <div class="file"> <div class="file__graphic"> <i class="fas fa-exclamation-circle" aria-hidden="true"></i> </div> <div class="card"> No source distribution files available for this release.See tutorial on <a href="https://packaging.python.org/tutorials/packaging-projects/#generating-distribution-archives" title="External link" target="_blank" rel="noopener">generating distribution archives</a>. </div> </div> <h3> Built Distribution </h3> <div class="file"> <div class="file__graphic"> <i class="far fa-file" aria-hidden="true"></i> </div> <div class="card file__card"> <a href="https://files.pythonhosted.org/packages/8f/13/84513d2c3c6605056c73c9e8f9c4d7999c98616ff780c28ff31e1ee5bb4a/model_alignment-0.2-py3-none-any.whl"> model_alignment-0.2-py3-none-any.whl </a> (38.9 kB <a href="#model_alignment-0.2-py3-none-any.whl" data-project-tabs-target="tab" data-action="project-tabs#onTabClick">view details</a>) <p class="file__meta"> Uploaded <time datetime="2024-10-07T17:26:47+0000" data-controller="localized-time" data-localized-time-relative="true" data-localized-time-show-time="false"> Oct 7, 2024 </time> <code>Python 3</code> </p> </div> </div> </div> <div id="model_alignment-0.2-py3-none-any.whl" data-project-tabs-target="content" class="vertical-tabs__content" role="tabpanel" aria-labelledby="file-tab mobile-file-tab" tabindex="-1"> <h2 class="page-title">File details</h2> <p>Details for the file <code>model_alignment-0.2-py3-none-any.whl</code>.</p> <h3>File metadata</h3> <div> <ul> <li> Download URL: <a href="https://files.pythonhosted.org/packages/8f/13/84513d2c3c6605056c73c9e8f9c4d7999c98616ff780c28ff31e1ee5bb4a/model_alignment-0.2-py3-none-any.whl"> model_alignment-0.2-py3-none-any.whl </a> </li> <li>Upload date: <time datetime="2024-10-07T17:26:47+0000" data-controller="localized-time" data-localized-time-relative="true" data-localized-time-show-time="false"> Oct 7, 2024 </time></li> <li>Size: 38.9 kB</li> <li>Tags: Python 3</li> <li> Uploaded using Trusted Publishing? No </li> <li>Uploaded via: twine/4.0.2 CPython/3.9.11</li> </ul> </div> <h3>File hashes</h3> <div> <table class="table table--hashes"> <caption class="sr-only">Hashes for model_alignment-0.2-py3-none-any.whl</caption> <thead> <tr> <th scope="col">Algorithm</th> <th scope="col">Hash digest</th> <th></th> </tr> </thead> <tbody> <tr data-controller="clipboard"> <th scope="row">SHA256</th> <td><code data-clipboard-target="source">29499e50ec775797c57554aea9a888de8e8a696c812cc8e0c2a0eba8126cdab3</code></td> <td class="table__align-right"> <button type="button" class="button button--small copy-tooltip copy-tooltip-w" data-action="clipboard#copy" data-clipboard-target="tooltip" data-clipboard-tooltip-value="Copy to clipboard"> Copy </button> </td> </tr> <tr data-controller="clipboard"> <th scope="row">MD5</th> <td><code data-clipboard-target="source">2de4494af006289a590d434c54de4abd</code></td> <td class="table__align-right"> <button type="button" class="button button--small copy-tooltip copy-tooltip-w" data-action="clipboard#copy" data-clipboard-target="tooltip" data-clipboard-tooltip-value="Copy to clipboard"> Copy </button> </td> </tr> <tr data-controller="clipboard"> <th scope="row">BLAKE2b-256</th> <td><code data-clipboard-target="source">8f1384513d2c3c6605056c73c9e8f9c4d7999c98616ff780c28ff31e1ee5bb4a</code></td> <td class="table__align-right"> <button type="button" class="button button--small copy-tooltip copy-tooltip-w" data-action="clipboard#copy" data-clipboard-target="tooltip" data-clipboard-tooltip-value="Copy to clipboard"> Copy </button> </td> </tr> </tbody> </table> <p> <a href="https://pip.pypa.io/en/stable/topics/secure-installs/#hash-checking-mode" title="External link" target="_blank" rel="noopener">See more details on using hashes here.</a> </p> </div> </div> </div> </div> </div> </div> </main> <footer class="footer"> <div class="footer__logo"> <img src="/static/images/white-cube.2351a86c.svg" alt="" class="-js-white-cube"> </div> <div class="footer__menus"> <div class="footer__menu"> <h2>Help</h2> <nav aria-label="Help navigation"> <ul> <li><a href="https://packaging.python.org/tutorials/installing-packages/" title="External link" target="_blank" rel="noopener">Installing packages</a></li> <li><a href="https://packaging.python.org/tutorials/packaging-projects/" title="External link" target="_blank" rel="noopener">Uploading packages</a></li> <li><a href="https://packaging.python.org/" title="External link" target="_blank" rel="noopener">User guide</a></li> <li><a href="https://www.python.org/dev/peps/pep-0541/" title="External link" target="_blank" rel="noopener">Project name retention</a></li> <li><a href="/help/">FAQs</a></li> </ul> </nav> </div> <div class="footer__menu"> <h2>About PyPI</h2> <nav aria-label="About PyPI navigation"> <ul> <li><a href="https://blog.pypi.org" title="External link" target="_blank" rel="noopener">PyPI Blog</a></li> <li><a href="https://dtdg.co/pypi" title="External link" target="_blank" rel="noopener">Infrastructure dashboard</a></li> <li><a href="/stats/">Statistics</a></li> <li><a href="/trademarks/">Logos & trademarks</a></li> <li><a href="/sponsors/">Our sponsors</a></li> </ul> </nav> </div> <div class="footer__menu"> <h2>Contributing to PyPI</h2> <nav aria-label="How to contribute navigation"> <ul> <li><a href="/help/#feedback">Bugs and feedback</a></li> <li><a href="https://github.com/pypi/warehouse" title="External link" target="_blank" rel="noopener">Contribute on GitHub</a></li> <li><a href="https://hosted.weblate.org/projects/pypa/warehouse/" title="External link" target="_blank" rel="noopener">Translate PyPI</a></li> <li><a href="/sponsors/">Sponsor PyPI</a></li> <li><a href="https://github.com/pypi/warehouse/graphs/contributors" title="External link" target="_blank" rel="noopener">Development credits</a></li> </ul> </nav> </div> <div class="footer__menu"> <h2>Using PyPI</h2> <nav aria-label="Using PyPI navigation"> <ul> <li><a href="https://policies.python.org/python.org/code-of-conduct/" title="External link" target="_blank" rel="noopener">Code of conduct</a></li> <li><a href="/security/">Report security issue</a></li> <li><a href="https://policies.python.org/pypi.org/Privacy-Notice/" title="External link" target="_blank" rel="noopener">Privacy Notice</a></li> <li><a href="https://policies.python.org/pypi.org/Terms-of-Use/" title="External link" target="_blank" rel="noopener">Terms of Use</a></li> <li><a href="https://policies.python.org/pypi.org/Acceptable-Use-Policy/" title="External link" target="_blank" rel="noopener">Acceptable Use Policy</a></li> </ul> </nav> </div> </div> <hr class="footer__divider"> <div class="footer__text"> <p>Status:<a href="https://status.python.org/" title="External link" target="_blank" rel="noopener"> <span data-statuspage-domain="https://2p66nmmycsj3.statuspage.io">all systems operational</span></a> </p> <p> Developed and maintained by the Python community, for the Python community. <br> <a href="https://donate.pypi.org">Donate today!</a> </p> <p> "PyPI", "Python Package Index", and the blocks logos are registered <a href="/trademarks/">trademarks</a> of the <a href="https://www.python.org/psf-landing" target="_blank" rel="noopener">Python Software Foundation</a>.<br> </p> <p> © 2024 <a href="https://www.python.org/psf-landing/" title="External link" target="_blank" rel="noopener">Python Software Foundation</a><br> <a href="/sitemap/">Site map</a> </p> </div> <div class="centered hide-on-desktop"> <button type="button" class="button button--switch-to-desktop hidden" data-viewport-toggle-target="switchToDesktop" data-action="viewport-toggle#switchToDesktop"> Switch to desktop version </button> </div> </footer> <div class="language-switcher"> <form action="/locale/"> <ul> <li> <button class="language-switcher__selected" name="locale_id" value="en" type="submit" > English </button> </li> <li> <button name="locale_id" value="es" type="submit" > español </button> </li> <li> <button name="locale_id" value="fr" type="submit" > français </button> </li> <li> <button name="locale_id" value="ja" type="submit" > 日本語 </button> </li> <li> <button name="locale_id" value="pt_BR" type="submit" > português (Brasil) </button> </li> <li> <button name="locale_id" value="uk" type="submit" > українська </button> </li> <li> <button name="locale_id" value="el" type="submit" > Ελληνικά </button> </li> <li> <button name="locale_id" value="de" type="submit" > Deutsch </button> </li> <li> <button name="locale_id" value="zh_Hans" type="submit" > 中文 (简体) </button> </li> <li> <button name="locale_id" value="zh_Hant" type="submit" > 中文 (繁體) </button> </li> <li> <button name="locale_id" value="ru" type="submit" > русский </button> </li> <li> <button name="locale_id" value="he" type="submit" > עברית </button> </li> <li> <button name="locale_id" value="eo" type="submit" > Esperanto </button> </li> </ul> </form> </div> <div class="sponsors"> <p class="sponsors__title">Supported by</p> <div class="sponsors__divider"></div> <a class="sponsors__sponsor" target="_blank" rel="noopener" href="https://aws.amazon.com/"> <img class=sponsors__image src="https://pypi-camo.freetls.fastly.net/ed7074cadad1a06f56bc520ad9bd3e00d0704c5b/68747470733a2f2f73746f726167652e676f6f676c65617069732e636f6d2f707970692d6173736574732f73706f6e736f726c6f676f732f6177732d77686974652d6c6f676f2d7443615473387a432e706e67" alt=AWS loading=lazy> <span class="sponsors__name">AWS</span> <span class="sponsors__service"> Cloud computing and Security Sponsor </span> </a> <a class="sponsors__sponsor" target="_blank" rel="noopener" href="https://www.datadoghq.com/"> <img class=sponsors__image src="https://pypi-camo.freetls.fastly.net/8855f7c063a3bdb5b0ce8d91bfc50cf851cc5c51/68747470733a2f2f73746f726167652e676f6f676c65617069732e636f6d2f707970692d6173736574732f73706f6e736f726c6f676f732f64617461646f672d77686974652d6c6f676f2d6668644c4e666c6f2e706e67" alt=Datadog loading=lazy> <span class="sponsors__name">Datadog</span> <span class="sponsors__service"> Monitoring </span> </a> <a class="sponsors__sponsor" target="_blank" rel="noopener" href="https://www.fastly.com/"> <img class=sponsors__image src="https://pypi-camo.freetls.fastly.net/df6fe8829cbff2d7f668d98571df1fd011f36192/68747470733a2f2f73746f726167652e676f6f676c65617069732e636f6d2f707970692d6173736574732f73706f6e736f726c6f676f732f666173746c792d77686974652d6c6f676f2d65684d3077735f6f2e706e67" alt=Fastly loading=lazy> <span class="sponsors__name">Fastly</span> <span class="sponsors__service"> CDN </span> </a> <a class="sponsors__sponsor" target="_blank" rel="noopener" href="https://careers.google.com/"> <img class=sponsors__image src="https://pypi-camo.freetls.fastly.net/420cc8cf360bac879e24c923b2f50ba7d1314fb0/68747470733a2f2f73746f726167652e676f6f676c65617069732e636f6d2f707970692d6173736574732f73706f6e736f726c6f676f732f676f6f676c652d77686974652d6c6f676f2d616734424e3774332e706e67" alt=Google loading=lazy> <span class="sponsors__name">Google</span> <span class="sponsors__service"> Download Analytics </span> </a> <a class="sponsors__sponsor" target="_blank" rel="noopener" href="https://www.python.org/psf/sponsors/#microsoft"> <img class=sponsors__image src="https://pypi-camo.freetls.fastly.net/524d1ce72f7772294ca4c1fe05d21dec8fa3f8ea/68747470733a2f2f73746f726167652e676f6f676c65617069732e636f6d2f707970692d6173736574732f73706f6e736f726c6f676f732f6d6963726f736f66742d77686974652d6c6f676f2d5a443172685444462e706e67" alt=Microsoft loading=lazy> <span class="sponsors__name">Microsoft</span> <span class="sponsors__service"> PSF Sponsor </span> </a> <a class="sponsors__sponsor" target="_blank" rel="noopener" href="https://www.pingdom.com/"> <img class=sponsors__image src="https://pypi-camo.freetls.fastly.net/d01053c02f3a626b73ffcb06b96367fdbbf9e230/68747470733a2f2f73746f726167652e676f6f676c65617069732e636f6d2f707970692d6173736574732f73706f6e736f726c6f676f732f70696e67646f6d2d77686974652d6c6f676f2d67355831547546362e706e67" alt=Pingdom loading=lazy> <span class="sponsors__name">Pingdom</span> <span class="sponsors__service"> Monitoring </span> </a> <a class="sponsors__sponsor" target="_blank" rel="noopener" href="https://getsentry.com/for/python"> <img class=sponsors__image src="https://pypi-camo.freetls.fastly.net/67af7117035e2345bacb5a82e9aa8b5b3e70701d/68747470733a2f2f73746f726167652e676f6f676c65617069732e636f6d2f707970692d6173736574732f73706f6e736f726c6f676f732f73656e7472792d77686974652d6c6f676f2d4a2d6b64742d706e2e706e67" alt=Sentry loading=lazy> <span class="sponsors__name">Sentry</span> <span class="sponsors__service"> Error logging </span> </a> <a class="sponsors__sponsor" target="_blank" rel="noopener" href="https://statuspage.io"> <img class=sponsors__image src="https://pypi-camo.freetls.fastly.net/b611884ff90435a0575dbab7d9b0d3e60f136466/68747470733a2f2f73746f726167652e676f6f676c65617069732e636f6d2f707970692d6173736574732f73706f6e736f726c6f676f732f737461747573706167652d77686974652d6c6f676f2d5467476c6a4a2d502e706e67" alt=StatusPage loading=lazy> <span class="sponsors__name">StatusPage</span> <span class="sponsors__service"> Status page </span> </a> </div> </body> </html>

CINXE.COM

model-alignment · PyPI