Evaluating Large Language Models Trained on Code

<!doctype html> <html lang="en"> <head> <meta charset="utf-8"> <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"> <script> const GTAG_ENABLED = true ; const GTAG_TRACKING_ID = "UA-121182717-1"; const SENTRY_DSN_FRONTEND = "".trim(); const GLOBAL_CSRF_TOKEN = 'hQurCrJoTt16xIeNCdbU3aMf0MKpk0kHPBajWIZ9GHxZkzqbcE6oAmj65NeVKQwM'; const MEDIA_URL = "https://production-media.paperswithcode.com/"; const ASSETS_URL = "https://production-assets.paperswithcode.com"; run_after_frontend_loaded = window.run_after_frontend_loaded || []; </script> <link rel="preconnect" href="https://production-assets.paperswithcode.com"><link rel="dns-prefetch" href="https://production-assets.paperswithcode.com"><link rel="preload" as="font" type="font/woff2" href="https://production-assets.paperswithcode.com/perf/fonts/65e877e527022735c1a1.woff2" crossorigin><link rel="preload" as="font" type="font/woff2" href="https://production-assets.paperswithcode.com/perf/fonts/917632e36982ca7933c8.woff2" crossorigin><link rel="preload" as="font" type="font/woff2" href="https://production-assets.paperswithcode.com/perf/fonts/f1405bd8a987c2ea8a67.woff2" crossorigin><script>(()=>{if(GTAG_ENABLED){const t=document.createElement("script");function n(){window.dataLayer.push(arguments)}t.src=`https://www.googletagmanager.com/gtag/js?id=${GTAG_TRACKING_ID}`,document.head.appendChild(t),window.dataLayer=window.dataLayer||[],window.gtag=n,n("js",new Date),n("config",GTAG_TRACKING_ID),window.captureOutboundLink=function(t){n("event","click",{event_category:"outbound",event_label:t})}}else window.captureOutboundLink=function(n){document.location=n}})();</script><link rel="preload" as="script" href="https://production-assets.paperswithcode.com/perf/766.4af6b88b.js"><link rel="preload" as="script" href="https://production-assets.paperswithcode.com/perf/351.a22a9607.js"><link rel="preload" as="style" href="https://production-assets.paperswithcode.com/perf/918.c41196c3.css"><link rel="preload" as="style" href="https://production-assets.paperswithcode.com/perf/view_paper.05773d2b.css"><link rel="stylesheet" href="https://production-assets.paperswithcode.com/perf/918.c41196c3.css"><link rel="stylesheet" href="https://production-assets.paperswithcode.com/perf/view_paper.05773d2b.css">  <title>Evaluating Large Language Models Trained on Code | Papers With Code</title> <meta name="description" content="🏆 SOTA for Multi-task Language Understanding on BBH-alg (Average (%) metric)" />  <meta property="og:title" content="Papers with Code - Evaluating Large Language Models Trained on Code"> <meta property="og:description" content="🏆 SOTA for Multi-task Language Understanding on BBH-alg (Average (%) metric)"> <meta property="og:image" content="https://raw.githubusercontent.com/THUDM/CodeGeeX/master/resources/logo/codegeex_logo.png"> <meta property="og:url" content="https://paperswithcode.com/paper/evaluating-large-language-models-trained-on">  <meta name="twitter:card" content="summary_large_image"> <meta name="twitter:site" content="@paperswithcode"> <meta name="twitter:title" content="Papers with Code - Evaluating Large Language Models Trained on Code"> <meta name="twitter:description" content="🏆 SOTA for Multi-task Language Understanding on BBH-alg (Average (%) metric)"> <meta name="twitter:creator" content="@paperswithcode"> <meta name="twitter:url" content="https://paperswithcode.com/paper/evaluating-large-language-models-trained-on"> <meta name="twitter:domain" content="paperswithcode.com">  <script type="application/ld+json">{ "@context": "http://schema.org", "@graph": { "@type": "ScholarlyArticle", "@id": "2107.03374", "name": "Evaluating Large Language Models Trained on Code", "description": "\ud83c\udfc6 SOTA for Multi-task Language Understanding on BBH-alg (Average (%) metric)", "url": "https://paperswithcode.com/paper/evaluating-large-language-models-trained-on", "image": "https://raw.githubusercontent.com/THUDM/CodeGeeX/master/resources/logo/codegeex_logo.png", "headline": "Evaluating Large Language Models Trained on Code", "abstract": "\ud83c\udfc6 SOTA for Multi-task Language Understanding on BBH-alg (Average (%) metric)", "author": [ { "@type": "Person", "@id": "#Mark_Chen", "name": "Mark Chen", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Jerry_Tworek", "name": "Jerry Tworek", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Heewoo_Jun", "name": "Heewoo Jun", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Qiming_Yuan", "name": "Qiming Yuan", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Henrique_Ponde_de_Oliveira_Pinto", "name": "Henrique Ponde de Oliveira Pinto", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Jared_Kaplan", "name": "Jared Kaplan", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Harri_Edwards", "name": "Harri Edwards", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Yuri_Burda", "name": "Yuri Burda", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Nicholas_Joseph", "name": "Nicholas Joseph", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Greg_Brockman", "name": "Greg Brockman", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Alex_Ray", "name": "Alex Ray", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Raul_Puri", "name": "Raul Puri", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Gretchen_Krueger", "name": "Gretchen Krueger", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Michael_Petrov", "name": "Michael Petrov", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Heidy_Khlaaf", "name": "Heidy Khlaaf", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Girish_Sastry", "name": "Girish Sastry", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Pamela_Mishkin", "name": "Pamela Mishkin", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Brooke_Chan", "name": "Brooke Chan", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Scott_Gray", "name": "Scott Gray", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Nick_Ryder", "name": "Nick Ryder", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Mikhail_Pavlov", "name": "Mikhail Pavlov", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Alethea_Power", "name": "Alethea Power", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Lukasz_Kaiser", "name": "Lukasz Kaiser", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Mohammad_Bavarian", "name": "Mohammad Bavarian", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Clemens_Winter", "name": "Clemens Winter", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Philippe_Tillet", "name": "Philippe Tillet", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Felipe_Petroski_Such", "name": "Felipe Petroski Such", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Dave_Cummings", "name": "Dave Cummings", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Matthias_Plappert", "name": "Matthias Plappert", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Fotios_Chantzis", "name": "Fotios Chantzis", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Elizabeth_Barnes", "name": "Elizabeth Barnes", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Ariel_Herbert-Voss", "name": "Ariel Herbert-Voss", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#William_Hebgen_Guss", "name": "William Hebgen Guss", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Alex_Nichol", "name": "Alex Nichol", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Alex_Paino", "name": "Alex Paino", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Nikolas_Tezak", "name": "Nikolas Tezak", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Jie_Tang", "name": "Jie Tang", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Igor_Babuschkin", "name": "Igor Babuschkin", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Suchir_Balaji", "name": "Suchir Balaji", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Shantanu_Jain", "name": "Shantanu Jain", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#William_Saunders", "name": "William Saunders", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Christopher_Hesse", "name": "Christopher Hesse", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Andrew_N._Carr", "name": "Andrew N. Carr", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Jan_Leike", "name": "Jan Leike", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Josh_Achiam", "name": "Josh Achiam", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Vedant_Misra", "name": "Vedant Misra", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Evan_Morikawa", "name": "Evan Morikawa", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Alec_Radford", "name": "Alec Radford", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Matthew_Knight", "name": "Matthew Knight", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Miles_Brundage", "name": "Miles Brundage", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Mira_Murati", "name": "Mira Murati", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Katie_Mayer", "name": "Katie Mayer", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Peter_Welinder", "name": "Peter Welinder", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Bob_McGrew", "name": "Bob McGrew", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Dario_Amodei", "name": "Dario Amodei", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Sam_McCandlish", "name": "Sam McCandlish", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Ilya_Sutskever", "name": "Ilya Sutskever", "image": "https://paperswithcode.com/static/" }, { "@type": "Person", "@id": "#Wojciech_Zaremba", "name": "Wojciech Zaremba", "image": "https://paperswithcode.com/static/" } ], "workExample": [ { "@type": "SoftwareSourceCode", "@id": "https://github.com/THUDM/CodeGeeX", "name": "CodeGeeX", "description": "CodeGeeX: An Open Multilingual Code Generation Model (KDD 2023)", "url": "https://github.com/THUDM/CodeGeeX", "image": "https://paperswithcode.com/static/", "headline": "CodeGeeX", "codeRepository": "https://github.com/THUDM/CodeGeeX", "contentRating": "8258" }, { "@type": "SoftwareSourceCode", "@id": "https://github.com/ncoop57/gpt-code-clippy", "name": "gpt-code-clippy", "description": "Full description can be found here: https://discuss.huggingface.co/t/pretrain-gpt-neo-for-open-source-github-copilot-model/7678?u=ncoop57", "url": "https://github.com/ncoop57/gpt-code-clippy", "image": "https://paperswithcode.com/static/", "headline": "gpt-code-clippy", "codeRepository": "https://github.com/ncoop57/gpt-code-clippy", "contentRating": "3298" }, { "@type": "SoftwareSourceCode", "@id": "https://github.com/codedotal/gpt-code-clippy", "name": "gpt-code-clippy", "description": "Full description can be found here: https://discuss.huggingface.co/t/pretrain-gpt-neo-for-open-source-github-copilot-model/7678?u=ncoop57", "url": "https://github.com/codedotal/gpt-code-clippy", "image": "https://paperswithcode.com/static/", "headline": "gpt-code-clippy", "codeRepository": "https://github.com/codedotal/gpt-code-clippy", "contentRating": "3298" }, { "@type": "SoftwareSourceCode", "@id": "https://github.com/openai/human-eval", "name": "human-eval", "description": "Code for the paper \"Evaluating Large Language Models Trained on Code\"", "url": "https://github.com/openai/human-eval", "image": "https://paperswithcode.com/static/", "headline": "human-eval", "codeRepository": "https://github.com/openai/human-eval", "contentRating": "2424" }, { "@type": "SoftwareSourceCode", "@id": "https://github.com/vhellendoorn/code-lms", "name": "code-lms", "description": "Guide to using pre-trained large language models of source code", "url": "https://github.com/vhellendoorn/code-lms", "image": "https://paperswithcode.com/static/", "headline": "code-lms", "codeRepository": "https://github.com/vhellendoorn/code-lms", "contentRating": "1784" }, { "@type": "SoftwareSourceCode", "@id": "https://github.com/glouppe/info8010-deep-learning", "name": "info8010-deep-learning", "description": "Lectures for INFO8010 Deep Learning, ULi\u00e8ge", "url": "https://github.com/glouppe/info8010-deep-learning", "image": "https://paperswithcode.com/static/", "headline": "info8010-deep-learning", "codeRepository": "https://github.com/glouppe/info8010-deep-learning", "contentRating": "1222" }, { "@type": "SoftwareSourceCode", "@id": "https://github.com/microsoft/PythonProgrammingPuzzles", "name": "PythonProgrammingPuzzles", "description": "A Dataset of Python Challenges for AI Research", "url": "https://github.com/microsoft/PythonProgrammingPuzzles", "image": "https://paperswithcode.com/static/", "headline": "PythonProgrammingPuzzles", "codeRepository": "https://github.com/microsoft/PythonProgrammingPuzzles", "contentRating": "968" }, { "@type": "SoftwareSourceCode", "@id": "https://github.com/fsoft-ai4code/codecapybara", "name": "codecapybara", "description": "Open-source Self-Instruction Tuning Code LLM", "url": "https://github.com/fsoft-ai4code/codecapybara", "image": "https://paperswithcode.com/static/", "headline": "codecapybara", "codeRepository": "https://github.com/fsoft-ai4code/codecapybara", "contentRating": "168" }, { "@type": "SoftwareSourceCode", "@id": "https://github.com/my-other-github-account/llm-humaneval-benchmarks", "name": "llm-humaneval-benchmarks", "description": "", "url": "https://github.com/my-other-github-account/llm-humaneval-benchmarks", "image": "https://paperswithcode.com/static/", "headline": "llm-humaneval-benchmarks", "codeRepository": "https://github.com/my-other-github-account/llm-humaneval-benchmarks", "contentRating": "86" }, { "@type": "SoftwareSourceCode", "@id": "https://github.com/codefuse-ai/codefuse-evaluation", "name": "codefuse-evaluation", "description": "Industrial-level evaluation benchmarks for Coding LLMs in the full life-cycle of AI native software developing.\u4f01\u4e1a\u7ea7\u4ee3\u7801\u5927\u6a21\u578b\u8bc4\u6d4b\u4f53\u7cfb,\u6301\u7eed\u5f00\u653e\u4e2d", "url": "https://github.com/codefuse-ai/codefuse-evaluation", "image": "https://paperswithcode.com/static/", "headline": "codefuse-evaluation", "codeRepository": "https://github.com/codefuse-ai/codefuse-evaluation", "contentRating": "80" }, { "@type": "SoftwareSourceCode", "@id": "https://github.com/superli3/codenavi", "name": "codenavi", "description": "A framework to help developers identify bugs in the software development process by applying neural machine translation models to structured representations of code", "url": "https://github.com/superli3/codenavi", "image": "https://paperswithcode.com/static/", "headline": "codenavi", "codeRepository": "https://github.com/superli3/codenavi", "contentRating": "1" }, { "@type": "SoftwareSourceCode", "@id": "https://github.com/superli3/CYRMPR", "name": "CYRMPR", "description": "A framework to help developers identify bugs in the software development process by applying neural machine translation models to structured representations of code", "url": "https://github.com/superli3/CYRMPR", "image": "https://paperswithcode.com/static/", "headline": "CYRMPR", "codeRepository": "https://github.com/superli3/CYRMPR", "contentRating": "1" }, { "@type": "SoftwareSourceCode", "@id": "https://github.com/2796gaurav/human-eval", "name": "human-eval", "description": "Code for the paper \"Evaluating Large Language Models Trained on Code\"", "url": "https://github.com/2796gaurav/human-eval", "image": "https://paperswithcode.com/static/", "headline": "human-eval", "codeRepository": "https://github.com/2796gaurav/human-eval", "contentRating": "1" } ], "datePublished": "2021-07-07" } }</script> <meta name="theme-color" content="#fff"/> <link rel="manifest" href="https://production-assets.paperswithcode.com/static/manifest.web.json"> </head> <body> <nav class="navbar navbar-expand-lg navbar-light header"> <a class="navbar-brand" href="/"> <span class=" icon-wrapper" data-name="pwc"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><path d="M88 128h48v256H88zm144 0h48v256h-48zm-72 16h48v224h-48zm144 0h48v224h-48zm72-16h48v256h-48z"/><path d="M104 104V56H16v400h88v-48H64V104zm304-48v48h40v304h-40v48h88V56z"/></svg></span> </a> <div class="navbar-mobile-twitter d-lg-none"> <a rel="noreferrer" href="https://twitter.com/paperswithcode"> <span class=" icon-wrapper icon-fa icon-fa-brands" data-name="twitter"><svg viewBox="0 0 512.001 515.25" xmlns="http://www.w3.org/2000/svg"><path d="M459.37 152.016c.326 4.548.326 9.097.326 13.645 0 138.72-105.583 298.558-298.559 298.558C101.685 464.22 46.457 447 0 417.114c8.447.973 16.568 1.298 25.34 1.298 49.054 0 94.213-16.568 130.274-44.832-46.132-.975-84.792-31.188-98.113-72.772 6.499.975 12.996 1.624 19.819 1.624 9.42 0 18.843-1.3 27.613-3.573-48.08-9.747-84.142-51.98-84.142-102.984v-1.3c13.968 7.798 30.213 12.67 47.43 13.32-28.263-18.843-46.78-51.006-46.78-87.391 0-19.492 5.196-37.36 14.294-52.954 51.654 63.674 129.3 105.258 216.364 109.807-1.624-7.797-2.599-15.918-2.599-24.04 0-57.827 46.782-104.934 104.934-104.934 30.214 0 57.502 12.67 76.671 33.136 23.715-4.548 46.455-13.319 66.599-25.34-7.798 24.367-24.366 44.834-46.132 57.828 21.117-2.274 41.584-8.122 60.426-16.244-14.292 20.791-32.161 39.309-52.628 54.253z"/></svg></span> </a> </div> <button class="navbar-toggler" type="button" data-toggle="collapse" data-bs-toggle="collapse" data-target="#top-menu" data-bs-target="#top-menu" aria-controls="top-menu" aria-expanded="false" aria-label="Toggle navigation" > <span class="navbar-toggler-icon"></span> </button> <div class="collapse navbar-collapse" id="top-menu"> <ul class="navbar-nav mr-auto navbar-nav__left light-header"> <li class="nav-item header-search"> <form action="/search" method="get" id="id_global_search_form" autocomplete="off"> <input type="text" name="q_meta" style="display:none" id="q_meta" /> <input type="hidden" name="q_type" id="q_type" /> <input id="id_global_search_input" autocomplete="off" value="" name='q' class="global-search" type="search" placeholder='Search'/> <button type="submit" class="icon"><span class=" icon-wrapper icon-fa icon-fa-light" data-name="search"><svg viewBox="0 0 512.025 520.146" xmlns="http://www.w3.org/2000/svg"><path d="M508.5 482.6c4.7 4.7 4.7 12.3 0 17l-9.9 9.9c-4.7 4.7-12.3 4.7-17 0l-129-129c-2.2-2.3-3.5-5.3-3.5-8.5v-10.2C312 396 262.5 417 208 417 93.1 417 0 323.9 0 209S93.1 1 208 1s208 93.1 208 208c0 54.5-21 104-55.3 141.1H371c3.2 0 6.2 1.2 8.5 3.5zM208 385c97.3 0 176-78.7 176-176S305.3 33 208 33 32 111.7 32 209s78.7 176 176 176z"/></svg></span></button> </form> </li> <li class="nav-item"> <a class="nav-link" href="/sota"> Browse State-of-the-Art </a> </li> <li class="nav-item"> <a class="nav-link" href="/datasets"> Datasets </a> </li> <li class="nav-item"> <a class="nav-link" href="/methods">Methods</a> </li> <li class="nav-item dropdown"> <a class="nav-link dropdown-toggle" role="button" id="navbarDropdownRepro" data-toggle="dropdown" data-bs-toggle="dropdown" aria-haspopup="true" aria-expanded="false" > More </a> <div class="dropdown-menu" aria-labelledby="navbarDropdownRepro"> <a class="dropdown-item" href="/newsletter">Newsletter</a> <a class="dropdown-item" href="/rc2022">RC2022</a> <div class="dropdown-divider"></div> <a class="dropdown-item" href="/about">About</a> <a class="dropdown-item" href="/trends">Trends</a> <a class="dropdown-item" href="https://portal.paperswithcode.com/"> Portals </a> <a class="dropdown-item" href="/libraries"> Libraries </a> </div> </li> </ul> <ul class="navbar-nav ml-auto navbar-nav__right navbar-subscribe justify-content-center align-items-center"> <li class="nav-item"> <a class="nav-link" rel="noreferrer" href="https://twitter.com/paperswithcode"> <span class="nav-link-social-icon icon-wrapper icon-fa icon-fa-brands" data-name="twitter"><svg viewBox="0 0 512.001 515.25" xmlns="http://www.w3.org/2000/svg"><path d="M459.37 152.016c.326 4.548.326 9.097.326 13.645 0 138.72-105.583 298.558-298.559 298.558C101.685 464.22 46.457 447 0 417.114c8.447.973 16.568 1.298 25.34 1.298 49.054 0 94.213-16.568 130.274-44.832-46.132-.975-84.792-31.188-98.113-72.772 6.499.975 12.996 1.624 19.819 1.624 9.42 0 18.843-1.3 27.613-3.573-48.08-9.747-84.142-51.98-84.142-102.984v-1.3c13.968 7.798 30.213 12.67 47.43 13.32-28.263-18.843-46.78-51.006-46.78-87.391 0-19.492 5.196-37.36 14.294-52.954 51.654 63.674 129.3 105.258 216.364 109.807-1.624-7.797-2.599-15.918-2.599-24.04 0-57.827 46.782-104.934 104.934-104.934 30.214 0 57.502 12.67 76.671 33.136 23.715-4.548 46.455-13.319 66.599-25.34-7.798 24.367-24.366 44.834-46.132 57.828 21.117-2.274 41.584-8.122 60.426-16.244-14.292 20.791-32.161 39.309-52.628 54.253z"/></svg></span> </a> </li> <li class="nav-item"> <a id="signin-link" class="nav-link" href="/accounts/login?next=/paper/evaluating-large-language-models-trained-on">Sign In</a> </li> </ul> </div> </nav>  <div class="modal fade" id="emailModal" tabindex="-1" role="dialog" aria-labelledby="emailModalLabel" aria-hidden="true"> <div class="modal-dialog" role="document"> <div class="modal-content"> <div class="modal-header"> <h3 class="modal-title" id="emailModalLabel">Subscribe to the PwC Newsletter</h3> <button type="button" class="close" data-dismiss="modal" data-bs-dismiss="modal" aria-label="Close"> <span aria-hidden="true">×</span> </button> </div> <form action="" method="post"> <div class="modal-body"> <div class="modal-body-info-text"> Stay informed on the latest trending ML papers with code, research developments, libraries, methods, and datasets.<br/><br/> <a href="/newsletter">Read previous issues</a> </div> <input type="hidden" name="csrfmiddlewaretoken" value="hQurCrJoTt16xIeNCdbU3aMf0MKpk0kHPBajWIZ9GHxZkzqbcE6oAmj65NeVKQwM"> <input placeholder="Enter your email" type="email" class="form-control pwc-email" name="address" id="id_address" max_length="100" required> </div> <div class="modal-footer"> <button type="submit" class="btn btn-primary">Subscribe</button> </div> </form> </div> </div> </div>  <div class="modal fade" id="loginModal" tabindex="-1" role="dialog" aria-labelledby="loginModalLabel" aria-hidden="true"> <div class="modal-dialog" role="document"> <div class="modal-content"> <div class="modal-header"> <h5 class="modal-title" id="loginModalLabel">Join the community</h5> <button type="button" class="close btn-close" data-dismiss="modal" data-bs-dismiss="modal" aria-label="Close"> <span aria-hidden="true">×</span> </button> </div> <div class="login-modal-message"> You need to <a href="/accounts/login?next=/paper/evaluating-large-language-models-trained-on">log in</a> to edit.<br/> You can <a href="/accounts/register?next=/paper/evaluating-large-language-models-trained-on">create a new account</a> if you don't have one.<br/><br/> </div> </div> </div> </div>  <template id="modals-template"> <div class="modal fade" id="page-meta-modal"> <div class="modal-dialog"> <div class="modal-content"> <div class="modal-header"> <h5 class="modal-title">Edit Social Preview</h5> <button type="button" class="close btn-close" data-dismiss="modal" data-bs-dismiss="modal" aria-label="Close" > <span aria-hidden="true">×</span> </button> </div> <div id="page-meta-modal-body" class="modal-body"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" id="page-meta-model-id-input" value="831822" /> <input type="hidden" id="page-meta-model-name-input" value="Paper" /> <div class="form-group"> <label>Description</label><br /> <div class="form-check form-check-inline"> <input id="description-mode-default" class="form-check-input display-toggle-switch" checked type="radio" name="description-mode" value="default" data-target="display-description-default" /> <label class="form-check-label" for="description-mode-default">Default</label> </div> <div class="form-check form-check-inline"> <input id="description-mode-custom" class="form-check-input display-toggle-switch" type="radio" name="description-mode" value="custom" data-target="display-description-custom" /> <label class="form-check-label" for="description-mode-custom">Custom</label> </div> </div> <div class="form-group"> <div id="display-description-default" data-name="description-mode"> <textarea class="form-control" rows="3" readonly >🏆 SOTA for Multi-task Language Understanding on BBH-alg (Average (%) metric)</textarea> </div> <div id="display-description-custom" data-name="description-mode"> <textarea class="form-control" id="description-input" rows="3" ></textarea> </div> </div> <div class="form-group"> <label>Image</label><br /> <div class="form-group"> <div class="form-check form-check-inline"> <input id="image-mode-default" class="form-check-input display-toggle-switch" checked type="radio" name="image-mode" value="default" data-target="display-image-default" /> <label class="form-check-label" for="image-mode-default">Default</label> </div> <div class="form-check form-check-inline"> <input id="image-mode-custom" class="form-check-input display-toggle-switch" type="radio" name="image-mode" value="custom" data-target="display-image-custom" /> <label class="form-check-label" for="image-mode-custom">Custom</label> </div> <div class="form-check form-check-inline"> <input id="image-mode-none" class="form-check-input display-toggle-switch" type="radio" name="image-mode" value="none" data-target="display-image-none" /> <label class="form-check-label" for="image-mode-none">None</label> </div> </div> </div> <div class="form-group"> <div id="display-image-default" data-name="image-mode"> <img class="page-meta-media" src="https://raw.githubusercontent.com/THUDM/CodeGeeX/master/resources/logo/codegeex_logo.png" /> </div> <div id="display-image-custom" data-name="image-mode"> <div id="file-too-large" style="display: none" class="alert alert-danger" role="alert"> File is too large </div> <p> Upload an image to customize your repository’s social media preview.<br /> Images should be at least 640×320px (1280×640px for best display). </P> <input type="file" class="form-control-file" id="image-input" /> </div> <div id="display-image-none" data-name="image-mode"> </div> </div> </div> <div class="modal-footer"> <button type="button" class="btn btn-secondary" data-dismiss="modal" data-bs-dismiss="modal"> Close </button> <button type="button" id="page-meta-submit" class="btn btn-primary"> Save </button> </div> </div> </div> </div>  <div class="modal fade" id="addCode" tabindex="-1" role="dialog" aria-labelledby="addCodeLabel" aria-hidden="true"> <div class="modal-dialog" role="document"> <div class="modal-content"> <div class="modal-header"> <h5 class="modal-title" id="addCodeLabel">Add a new code entry for this paper</h5> <button type="button" class="close btn-close" data-bs-dismiss="modal" aria-label="Close"> <span aria-hidden="true">×</span> </button> </div> <form action="" method="post"> <div class="modal-body"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <div id="div_id_url" class="form-group"> <label for="id_url" class=" requiredField"> GitHub, GitLab or BitBucket URL:<span class="asteriskField">*</span> </label> <div class=""> <input type="url" name="url" class="urlinput form-control" required id="id_url"> </div> </div> <div class="form-group"> <div id="div_id_is_official" class="form-check"> <input type="checkbox" name="is_official" class="checkboxinput form-check-input" id="id_is_official"> <label for="id_is_official" class="form-check-label"> Official code from paper authors </label> </div> </div> </div> <div class="modal-footer"> <button type="submit" class="btn btn-primary">Submit </button> </div> </form> </div> </div> </div>  <div class="modal fade" id="removeCode" tabindex="-1" role="dialog" aria-labelledby="removeCodeLabel" aria-hidden="true"> <div class="modal-dialog modal-lg" role="document"> <div class="modal-content"> <div class="modal-header"> <h5 class="modal-title" id="removeCodeLabel">Remove a code repository from this paper</h5> <button type="button" class="close btn-close" data-bs-dismiss="modal" aria-label="Close"> <span aria-hidden="true">×</span> </button> </div> <form action="" method="post"> <div class="modal-body"> <div class="paper-implementations"> <div class="row"> <div class="col-md-6"> <div class="paper-impl-cell"> <a href="https://github.com/openai/human-eval" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> openai/human-eval <span class="badge badge-info is-official-code"><span class=" icon-wrapper icon-ion" data-name="checkmark-circle-outline"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M448 256c0-106-86-192-192-192S64 150 64 256s86 192 192 192 192-86 192-192z" fill="none" stroke="#000" stroke-miterlimit="10" stroke-width="32"/><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M352 176L217.6 336 160 272"/></svg></span> official</span> </a> </div> </div> <div class="col-md-3"> <div class="paper-impl-cell"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 2,424 </div> </div> <div class="col-md-2"> <div class="paper-impl-cell"> </div> </div> <div class="col-md-1"> <form action="" method="post"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_code_pk" value="1234444"> <button type="submit" class="btn btn-danger">- </button> </form> </div> </div> <div class="row"> <div class="col-md-6"> <div class="paper-impl-cell"> <a href="https://github.com/THUDM/CodeGeeX" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> THUDM/CodeGeeX </a> </div> </div> <div class="col-md-3"> <div class="paper-impl-cell"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 8,258 </div> </div> <div class="col-md-2"> <div class="paper-impl-cell"> <img class="" src="https://production-assets.paperswithcode.com/perf/images/frameworks/mindspore-beada7dc.png" /> </div> </div> <div class="col-md-1"> <form action="" method="post"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_code_pk" value="1342823"> <button type="submit" class="btn btn-danger">- </button> </form> </div> </div> <div class="row"> <div class="col-md-6"> <div class="paper-impl-cell"> <a href="https://github.com/ncoop57/gpt-code-clippy" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> ncoop57/gpt-code-clippy </a> </div> </div> <div class="col-md-3"> <div class="paper-impl-cell"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 3,298 </div> </div> <div class="col-md-2"> <div class="paper-impl-cell"> <img class="" src="https://production-assets.paperswithcode.com/perf/images/frameworks/jax-6ee30fa5.png" /> </div> </div> <div class="col-md-1"> <form action="" method="post"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_code_pk" value="1357922"> <button type="submit" class="btn btn-danger">- </button> </form> </div> </div> <div class="row"> <div class="col-md-6"> <div class="paper-impl-cell"> <a href="https://github.com/codedotal/gpt-code-clippy" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> codedotal/gpt-code-clippy </a> </div> </div> <div class="col-md-3"> <div class="paper-impl-cell"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 3,298 </div> </div> <div class="col-md-2"> <div class="paper-impl-cell"> <img class="" src="https://production-assets.paperswithcode.com/perf/images/frameworks/jax-6ee30fa5.png" /> </div> </div> <div class="col-md-1"> <form action="" method="post"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_code_pk" value="1277336"> <button type="submit" class="btn btn-danger">- </button> </form> </div> </div> <div class="row"> <div class="col-md-6"> <div class="paper-impl-cell"> <a href="https://github.com/vhellendoorn/code-lms" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> vhellendoorn/code-lms </a> </div> </div> <div class="col-md-3"> <div class="paper-impl-cell"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 1,784 </div> </div> <div class="col-md-2"> <div class="paper-impl-cell"> </div> </div> <div class="col-md-1"> <form action="" method="post"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_code_pk" value="1279587"> <button type="submit" class="btn btn-danger">- </button> </form> </div> </div> <div class="row"> <div class="col-md-6"> <div class="paper-impl-cell"> <a href="https://github.com/glouppe/info8010-deep-learning" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> glouppe/info8010-deep-learning </a> </div> </div> <div class="col-md-3"> <div class="paper-impl-cell"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 1,222 </div> </div> <div class="col-md-2"> <div class="paper-impl-cell"> <img class="" src="https://production-assets.paperswithcode.com/perf/images/frameworks/pytorch-2fbf2cb9.png" /> </div> </div> <div class="col-md-1"> <form action="" method="post"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_code_pk" value="68582"> <button type="submit" class="btn btn-danger">- </button> </form> </div> </div> <div class="row"> <div class="col-md-6"> <div class="paper-impl-cell"> <a href="https://github.com/microsoft/PythonProgrammingPuzzles" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> microsoft/PythonProgrammingPuzzles </a> </div> </div> <div class="col-md-3"> <div class="paper-impl-cell"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 968 </div> </div> <div class="col-md-2"> <div class="paper-impl-cell"> </div> </div> <div class="col-md-1"> <form action="" method="post"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_code_pk" value="1228420"> <button type="submit" class="btn btn-danger">- </button> </form> </div> </div> <div class="row"> <div class="col-md-6"> <div class="paper-impl-cell"> <a href="https://github.com/fsoft-ai4code/codecapybara" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> fsoft-ai4code/codecapybara </a> </div> </div> <div class="col-md-3"> <div class="paper-impl-cell"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 168 </div> </div> <div class="col-md-2"> <div class="paper-impl-cell"> <img class="" src="https://production-assets.paperswithcode.com/perf/images/frameworks/pytorch-2fbf2cb9.png" /> </div> </div> <div class="col-md-1"> <form action="" method="post"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_code_pk" value="1390466"> <button type="submit" class="btn btn-danger">- </button> </form> </div> </div> <div class="row"> <div class="col-md-6"> <div class="paper-impl-cell"> <a href="https://github.com/my-other-github-account/llm-humaneval-benchmarks" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> my-other-github-account/llm-humaneval-benchmarks </a> </div> </div> <div class="col-md-3"> <div class="paper-impl-cell"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 86 </div> </div> <div class="col-md-2"> <div class="paper-impl-cell"> </div> </div> <div class="col-md-1"> <form action="" method="post"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_code_pk" value="1357968"> <button type="submit" class="btn btn-danger">- </button> </form> </div> </div> <div class="row"> <div class="col-md-6"> <div class="paper-impl-cell"> <a href="https://github.com/codefuse-ai/codefuse-evaluation" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> codefuse-ai/codefuse-evaluation </a> </div> </div> <div class="col-md-3"> <div class="paper-impl-cell"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 80 </div> </div> <div class="col-md-2"> <div class="paper-impl-cell"> <img class="" src="https://production-assets.paperswithcode.com/perf/images/frameworks/pytorch-2fbf2cb9.png" /> </div> </div> <div class="col-md-1"> <form action="" method="post"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_code_pk" value="1379382"> <button type="submit" class="btn btn-danger">- </button> </form> </div> </div> <div class="row"> <div class="col-md-6"> <div class="paper-impl-cell"> <a href="https://github.com/superli3/codenavi" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> superli3/codenavi </a> </div> </div> <div class="col-md-3"> <div class="paper-impl-cell"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 1 </div> </div> <div class="col-md-2"> <div class="paper-impl-cell"> <img class="" src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAbYAAABiBAMAAADO/+vOAAAACXBIWXMAAA7EAAAOxAGVKw4bAAAAMFBMVEVHcExBUGZCUGZKUmNCUGZCUGZCUGb/igD/jwD/hwD/iwBCUGb/kAD/hAD/mwD/eABjkbvFAAAAC3RSTlMAxZ4pVHvkPbaF3mTG43EAAAjMSURBVHja7Zy/b9tGFMdPImlD8CLXyaRFdRIvWhQrCVJosRMnixblh7NwcWNXixYDXRwvCZChQNGm3rq06RhkSGLJNhf9A2khc81iQWtaCJqDAILK+/2LEk+2AsQEr4BDU7w7fu579967d1YBSEpSkpKUpCSFlh/BdlzRbn4AP+/FEy3T+QB++SeWys2+gWwn74uxRDsJ2Nrtv2OHZv3a6RxD3U7iB/dnp0N0a7dfxQtttwML1A3C/RQvtEEwJZFuEK73LD5oO92BpFu7Fxs3lxkOoG4drpv/X0zcXOZdp9uFsh1z3fyPsYCbfTccIjRBt16v/zEOaL8Nu12sG19vvu/3/z3/bH8Nh4PBQLGTgW5+/1UM0LpDIpusW79/zt3c7jAoAdqA2cljplu/f67d3A5EGxDdTlTd+v1z7OYywyGBU/wbLJBNdnOLeslP/6WWlS4ASM/Pz184FdqAognrrY3IgiK6Oaull/3ps1WULgBIBT+PJmzlKZYN+gDZTgZTskfono1nO/ja2QIfIPk3n+sWxfa169btDvGcZHbS96kxiYNuEE6xk34sdBsMBgMpnvTbvX5MdOsOFDvZY1NSct9ruNwNuvLI9cqXYTta4+WMun1+ItvJ9ndhdpIW2FXjyzndijYdzqTbXqYj20mwE2InzyXbc5CR4xI/CDTjwfYKpsvleDL48GUc2F7D6Ksj7QN8aBVfGLLd+XZ+8Tb9pVQqIpN6f3FdNDPXlN+DJ+6sPywqDV9fvFKMZrPX19eKZmyfUGQp7996aEP+1oTNLiBDfZH8Cn8rguU6vHeJcSxJz8CyCm94l8lvaYRzNfhZjWKz7qGauG27Vqs9xvdnarUtfHU3uFlFbJ+2AdVNsJM4lDZgs3LEC20JbA65V1We4WP/SK6F2JyWAZtVIDUXqMf12ExuiMMbsH3Gcb5mJxlcBNtd5mLxbgdirFCWJn7GZc8skFpXlTuQ7dAlrYxl422hgasjDkqEH6mjq4CNbD4VO9kj83knis3h4UOTdXGP3VtRnvFwu7Z6B7I1WwZsaV6zQbtbYUQrVMsGZHtOd3KKnXxNUw4RbK4QG1VpZ7xsqM9saFHVgfzKEWxi81nSUp4NF7sKHn7KMlmqbiwN9HIsGxGgzt9SZjuiUah3ZT3HxJXiUiScKZsj1oRtzdHxctjIYbsEnvAduJLnalO9ZvfGsd2ALf5QBNfr9BZeahdLJWzOSF/eCrUCkGQGTanbpVts+E3ZkODN26VVOuNT9NkZNropNj1E3eT85Og0EGcr0Ks0XV0FZv0qBAX2usmeqZJJiuRCJuWQsXlX1qATHMOGhrBIB3UDywUbAGVmh+eYXRuhW7APGH3awdgsOu6YKUv+xfbBIbgV5gxy+H2UWlTblleNiksc2R4f4baabFE3aPUVTTc5P9n/WIxiS3PTVyYvVOAvRmRymZl28V7W4U4pRfgR22bo/u1AYJvhRuwG3tlBJYW1UARifyPsZBvCbUexzXG/kyaTI8dnBLnkfRH+GcETEwnZlB7PJihqkxo54ta4qc6pwW5Gz08asFV4UsEmk6PA2cg0dXmAAiSJAZc5LUZxo9kKbDLjpZdlM9DmbkGLq/X11h9tTVKi58xyBRoym8t73xSru8IbVrDcaR7EjGOrC8Pk4sVbxreCBrwCGmiLWpcxdtJEt5zQF1lEOtucFDYrtcisNmOzxJkrTPAs+ueogqAcLUGl+TeYL4nUDfb1oIQLWds5jQ3b92+46aoLb5jCUGZsdovFjHhUiOL7iPRgjjWVBRF20o/UTct7heoGcNTC6VqCISMLTWNrzNOSldk8xS/YGD/oayONPp1Rl7dmJ32/F62bxlYMZbtBP77EbSNlI/5AYwv1b47K1mSmIxi/vI3aLasuQMuXoLxyFJttxmbV6ecXGBsQ2FqTsDFrSqsgk2+haY4Uqwj8IHT/htKT26dg09ebGOFuAmXVnJ3NhSwOaiUH+ysILYXaSf+PKeoGcChNw9sp6UbnJJqDKXTtwno51QVodtJ64Ruut3mhjGID9hJhOwxh807NRuOw6hxquQx/6nl8NS4Bsy98MzYlaR7OBsA1TOeF2EljNslO0sgNVs1XUA2onq3ucMLyk7NvzfybwpYbwUYnZlX3bw1TNivEvyHgjQLqCerqaC4gLD+ZiWarq1ul0brR5blhFpeE73FC4hIEfFBHLcLLVX24w/KTN43iyX1jNhpam8ST4WxqPLlPBrhJZnlwuSRIG24n22NTtozN1XPZGttMrVbb4PUO5Hcno2PI5gqLKUdHqCBnvfR0fnh+MoqtzPemANwKX29z/E3TPLxt6vs3A7Yyj/Ftllmr8LRuRcnwhsclZmzplrjJyYfqlpL3r2SzRgxlWth3G7CllB07W7JiQkg7yg05xzFgs1piprIRyubIeQeywyI7OlfIlxiw2XKmpckpefpOcwGn1E3Iaq0KuSDZljASm5rVAsnqgWUxz2XAhmY8ynMtc4GclpR21Sx3qJ00YMMh/uP1O0stPV/i8pxC63LgvXPUhKOp4z1U8pMmbGWSn7xV5xAWz86DVojHPaWdVHY5+VDdUmLueUxe2YRNyis3hL2ux41nqzgVOynvkZvh/o1vcdgyE2sdggnYpJT8puBHhBMjD0xnveE1JB5Sjdubhp3jtIoTsYnnOEXZ/9M52wRTsZPSSdrWyHiyoJzRibgLYCI2QfK8mBPc5w8ejtHteBLdyOEuPPEYHXPJZ53S2dYWmJCNtXVB+jTLt0z74CzrDf6pJj+7vp4jhhCV+4uLi8RQrQaP5YUz6u8fCCf5+PyR1nLEFmG9Ba1DymI9Es67ceUgqiNdWsFlHoy3k/5Ef+5wbS3y7wastbUSiPw7BbNiT1hT3XeDGJWJ7OS5Y5PsZC/GusVqTqKvZPL1Fq8vZaKv0lLdXoN4ldk3TLf4fQk6g7/ffdx+vw3iB4d1i+f/dGAHsrX3QCzLbsD2DMS0/B78l5SkJCUpSUlKUpKSlGmW/wErOKTxUOS81gAAAABJRU5ErkJggg==" /> </div> </div> <div class="col-md-1"> <form action="" method="post"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_code_pk" value="1267746"> <button type="submit" class="btn btn-danger">- </button> </form> </div> </div> <div class="row"> <div class="col-md-6"> <div class="paper-impl-cell"> <a href="https://github.com/superli3/CYRMPR" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> superli3/CYRMPR </a> </div> </div> <div class="col-md-3"> <div class="paper-impl-cell"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 1 </div> </div> <div class="col-md-2"> <div class="paper-impl-cell"> <img class="" src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAbYAAABiBAMAAADO/+vOAAAACXBIWXMAAA7EAAAOxAGVKw4bAAAAMFBMVEVHcExBUGZCUGZKUmNCUGZCUGZCUGb/igD/jwD/hwD/iwBCUGb/kAD/hAD/mwD/eABjkbvFAAAAC3RSTlMAxZ4pVHvkPbaF3mTG43EAAAjMSURBVHja7Zy/b9tGFMdPImlD8CLXyaRFdRIvWhQrCVJosRMnixblh7NwcWNXixYDXRwvCZChQNGm3rq06RhkSGLJNhf9A2khc81iQWtaCJqDAILK+/2LEk+2AsQEr4BDU7w7fu579967d1YBSEpSkpKUpCSFlh/BdlzRbn4AP+/FEy3T+QB++SeWys2+gWwn74uxRDsJ2Nrtv2OHZv3a6RxD3U7iB/dnp0N0a7dfxQtttwML1A3C/RQvtEEwJZFuEK73LD5oO92BpFu7Fxs3lxkOoG4drpv/X0zcXOZdp9uFsh1z3fyPsYCbfTccIjRBt16v/zEOaL8Nu12sG19vvu/3/z3/bH8Nh4PBQLGTgW5+/1UM0LpDIpusW79/zt3c7jAoAdqA2cljplu/f67d3A5EGxDdTlTd+v1z7OYywyGBU/wbLJBNdnOLeslP/6WWlS4ASM/Pz184FdqAognrrY3IgiK6Oaull/3ps1WULgBIBT+PJmzlKZYN+gDZTgZTskfono1nO/ja2QIfIPk3n+sWxfa169btDvGcZHbS96kxiYNuEE6xk34sdBsMBgMpnvTbvX5MdOsOFDvZY1NSct9ruNwNuvLI9cqXYTta4+WMun1+ItvJ9ndhdpIW2FXjyzndijYdzqTbXqYj20mwE2InzyXbc5CR4xI/CDTjwfYKpsvleDL48GUc2F7D6Ksj7QN8aBVfGLLd+XZ+8Tb9pVQqIpN6f3FdNDPXlN+DJ+6sPywqDV9fvFKMZrPX19eKZmyfUGQp7996aEP+1oTNLiBDfZH8Cn8rguU6vHeJcSxJz8CyCm94l8lvaYRzNfhZjWKz7qGauG27Vqs9xvdnarUtfHU3uFlFbJ+2AdVNsJM4lDZgs3LEC20JbA65V1We4WP/SK6F2JyWAZtVIDUXqMf12ExuiMMbsH3Gcb5mJxlcBNtd5mLxbgdirFCWJn7GZc8skFpXlTuQ7dAlrYxl422hgasjDkqEH6mjq4CNbD4VO9kj83knis3h4UOTdXGP3VtRnvFwu7Z6B7I1WwZsaV6zQbtbYUQrVMsGZHtOd3KKnXxNUw4RbK4QG1VpZ7xsqM9saFHVgfzKEWxi81nSUp4NF7sKHn7KMlmqbiwN9HIsGxGgzt9SZjuiUah3ZT3HxJXiUiScKZsj1oRtzdHxctjIYbsEnvAduJLnalO9ZvfGsd2ALf5QBNfr9BZeahdLJWzOSF/eCrUCkGQGTanbpVts+E3ZkODN26VVOuNT9NkZNropNj1E3eT85Og0EGcr0Ks0XV0FZv0qBAX2usmeqZJJiuRCJuWQsXlX1qATHMOGhrBIB3UDywUbAGVmh+eYXRuhW7APGH3awdgsOu6YKUv+xfbBIbgV5gxy+H2UWlTblleNiksc2R4f4baabFE3aPUVTTc5P9n/WIxiS3PTVyYvVOAvRmRymZl28V7W4U4pRfgR22bo/u1AYJvhRuwG3tlBJYW1UARifyPsZBvCbUexzXG/kyaTI8dnBLnkfRH+GcETEwnZlB7PJihqkxo54ta4qc6pwW5Gz08asFV4UsEmk6PA2cg0dXmAAiSJAZc5LUZxo9kKbDLjpZdlM9DmbkGLq/X11h9tTVKi58xyBRoym8t73xSru8IbVrDcaR7EjGOrC8Pk4sVbxreCBrwCGmiLWpcxdtJEt5zQF1lEOtucFDYrtcisNmOzxJkrTPAs+ueogqAcLUGl+TeYL4nUDfb1oIQLWds5jQ3b92+46aoLb5jCUGZsdovFjHhUiOL7iPRgjjWVBRF20o/UTct7heoGcNTC6VqCISMLTWNrzNOSldk8xS/YGD/oayONPp1Rl7dmJ32/F62bxlYMZbtBP77EbSNlI/5AYwv1b47K1mSmIxi/vI3aLasuQMuXoLxyFJttxmbV6ecXGBsQ2FqTsDFrSqsgk2+haY4Uqwj8IHT/htKT26dg09ebGOFuAmXVnJ3NhSwOaiUH+ysILYXaSf+PKeoGcChNw9sp6UbnJJqDKXTtwno51QVodtJ64Ruut3mhjGID9hJhOwxh807NRuOw6hxquQx/6nl8NS4Bsy98MzYlaR7OBsA1TOeF2EljNslO0sgNVs1XUA2onq3ucMLyk7NvzfybwpYbwUYnZlX3bw1TNivEvyHgjQLqCerqaC4gLD+ZiWarq1ul0brR5blhFpeE73FC4hIEfFBHLcLLVX24w/KTN43iyX1jNhpam8ST4WxqPLlPBrhJZnlwuSRIG24n22NTtozN1XPZGttMrVbb4PUO5Hcno2PI5gqLKUdHqCBnvfR0fnh+MoqtzPemANwKX29z/E3TPLxt6vs3A7Yyj/Ftllmr8LRuRcnwhsclZmzplrjJyYfqlpL3r2SzRgxlWth3G7CllB07W7JiQkg7yg05xzFgs1piprIRyubIeQeywyI7OlfIlxiw2XKmpckpefpOcwGn1E3Iaq0KuSDZljASm5rVAsnqgWUxz2XAhmY8ynMtc4GclpR21Sx3qJ00YMMh/uP1O0stPV/i8pxC63LgvXPUhKOp4z1U8pMmbGWSn7xV5xAWz86DVojHPaWdVHY5+VDdUmLueUxe2YRNyis3hL2ux41nqzgVOynvkZvh/o1vcdgyE2sdggnYpJT8puBHhBMjD0xnveE1JB5Sjdubhp3jtIoTsYnnOEXZ/9M52wRTsZPSSdrWyHiyoJzRibgLYCI2QfK8mBPc5w8ejtHteBLdyOEuPPEYHXPJZ53S2dYWmJCNtXVB+jTLt0z74CzrDf6pJj+7vp4jhhCV+4uLi8RQrQaP5YUz6u8fCCf5+PyR1nLEFmG9Ba1DymI9Es67ceUgqiNdWsFlHoy3k/5Ef+5wbS3y7wastbUSiPw7BbNiT1hT3XeDGJWJ7OS5Y5PsZC/GusVqTqKvZPL1Fq8vZaKv0lLdXoN4ldk3TLf4fQk6g7/ffdx+vw3iB4d1i+f/dGAHsrX3QCzLbsD2DMS0/B78l5SkJCUpSUlKUpKSlGmW/wErOKTxUOS81gAAAABJRU5ErkJggg==" /> </div> </div> <div class="col-md-1"> <form action="" method="post"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_code_pk" value="1264601"> <button type="submit" class="btn btn-danger">- </button> </form> </div> </div> <div class="row"> <div class="col-md-6"> <div class="paper-impl-cell"> <a href="https://github.com/2796gaurav/human-eval" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> 2796gaurav/human-eval </a> </div> </div> <div class="col-md-3"> <div class="paper-impl-cell"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 1 </div> </div> <div class="col-md-2"> <div class="paper-impl-cell"> </div> </div> <div class="col-md-1"> <form action="" method="post"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_code_pk" value="1240821"> <button type="submit" class="btn btn-danger">- </button> </form> </div> </div> </div> </div> </form> </div> </div> </div>  <div class="modal fade" id="changeOfficialCode" tabindex="-1" role="dialog" aria-labelledby="changeOfficialCodeLabel" aria-hidden="true"> <div class="modal-dialog modal-lg" role="document"> <div class="modal-content"> <div class="modal-header"> <h5 class="modal-title" id="changeOfficialCodeLabel"> Mark the official implementation from paper authors </h5> <button type="button" class="close btn-close" data-bs-dismiss="modal" aria-label="Close"> <span aria-hidden="true">×</span> </button> </div> <form action="" method="post" id="official-pgr-form"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="official_pgr_ids" id="official-pgr-ids" /> <div class="modal-body"> <div class="paper-implementations"> <div class="row align-items-center"> <div class="col-md-5"> <div class="paper-impl-cell"> <a href="https://github.com/openai/human-eval" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> openai/human-eval <span class="badge badge-info is-official-code"><span class=" icon-wrapper icon-ion" data-name="checkmark-circle-outline"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M448 256c0-106-86-192-192-192S64 150 64 256s86 192 192 192 192-86 192-192z" fill="none" stroke="#000" stroke-miterlimit="10" stroke-width="32"/><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M352 176L217.6 336 160 272"/></svg></span> official</span> </a> </div> </div> <div class="col-md-3"> <div class="paper-impl-cell"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 2,424 </div> </div> <div class="col-md-2"> <div class="paper-impl-cell"> </div> </div> <div class="col-md-2 text-center"> <input type="radio" name="official-pgr-radio" class="official-pgr-input official-pgr-radio radios-version-element" value="1800369" checked > <input type="checkbox" class="official-pgr-input official-pgr-checkbox checkboxes-version-element" value="1800369" checked > </div> </div> <div class="row align-items-center"> <div class="col-md-5"> <div class="paper-impl-cell"> <a href="https://github.com/THUDM/CodeGeeX" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> THUDM/CodeGeeX </a> </div> </div> <div class="col-md-3"> <div class="paper-impl-cell"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 8,258 </div> </div> <div class="col-md-2"> <div class="paper-impl-cell"> <img class="" src="https://production-assets.paperswithcode.com/perf/images/frameworks/mindspore-beada7dc.png" /> </div> </div> <div class="col-md-2 text-center"> <input type="radio" name="official-pgr-radio" class="official-pgr-input official-pgr-radio radios-version-element" value="2113949" > <input type="checkbox" class="official-pgr-input official-pgr-checkbox checkboxes-version-element" value="2113949" > </div> </div> <div class="row align-items-center"> <div class="col-md-5"> <div class="paper-impl-cell"> <a href="https://github.com/ncoop57/gpt-code-clippy" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> ncoop57/gpt-code-clippy </a> </div> </div> <div class="col-md-3"> <div class="paper-impl-cell"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 3,298 </div> </div> <div class="col-md-2"> <div class="paper-impl-cell"> <img class="" src="https://production-assets.paperswithcode.com/perf/images/frameworks/jax-6ee30fa5.png" /> </div> </div> <div class="col-md-2 text-center"> <input type="radio" name="official-pgr-radio" class="official-pgr-input official-pgr-radio radios-version-element" value="2152729" > <input type="checkbox" class="official-pgr-input official-pgr-checkbox checkboxes-version-element" value="2152729" > </div> </div> <div class="row align-items-center"> <div class="col-md-5"> <div class="paper-impl-cell"> <a href="https://github.com/codedotal/gpt-code-clippy" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> codedotal/gpt-code-clippy </a> </div> </div> <div class="col-md-3"> <div class="paper-impl-cell"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 3,298 </div> </div> <div class="col-md-2"> <div class="paper-impl-cell"> <img class="" src="https://production-assets.paperswithcode.com/perf/images/frameworks/jax-6ee30fa5.png" /> </div> </div> <div class="col-md-2 text-center"> <input type="radio" name="official-pgr-radio" class="official-pgr-input official-pgr-radio radios-version-element" value="1911891" > <input type="checkbox" class="official-pgr-input official-pgr-checkbox checkboxes-version-element" value="1911891" > </div> </div> <div class="row align-items-center"> <div class="col-md-5"> <div class="paper-impl-cell"> <a href="https://github.com/vhellendoorn/code-lms" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> vhellendoorn/code-lms </a> </div> </div> <div class="col-md-3"> <div class="paper-impl-cell"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 1,784 </div> </div> <div class="col-md-2"> <div class="paper-impl-cell"> </div> </div> <div class="col-md-2 text-center"> <input type="radio" name="official-pgr-radio" class="official-pgr-input official-pgr-radio radios-version-element" value="1920259" > <input type="checkbox" class="official-pgr-input official-pgr-checkbox checkboxes-version-element" value="1920259" > </div> </div> <div class="row align-items-center"> <div class="col-md-5"> <div class="paper-impl-cell"> <a href="https://github.com/glouppe/info8010-deep-learning" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> glouppe/info8010-deep-learning </a> </div> </div> <div class="col-md-3"> <div class="paper-impl-cell"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 1,222 </div> </div> <div class="col-md-2"> <div class="paper-impl-cell"> <img class="" src="https://production-assets.paperswithcode.com/perf/images/frameworks/pytorch-2fbf2cb9.png" /> </div> </div> <div class="col-md-2 text-center"> <input type="radio" name="official-pgr-radio" class="official-pgr-input official-pgr-radio radios-version-element" value="2092911" > <input type="checkbox" class="official-pgr-input official-pgr-checkbox checkboxes-version-element" value="2092911" > </div> </div> <div class="row align-items-center"> <div class="col-md-5"> <div class="paper-impl-cell"> <a href="https://github.com/microsoft/PythonProgrammingPuzzles" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> microsoft/PythonProgrammingPuzzles </a> </div> </div> <div class="col-md-3"> <div class="paper-impl-cell"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 968 </div> </div> <div class="col-md-2"> <div class="paper-impl-cell"> </div> </div> <div class="col-md-2 text-center"> <input type="radio" name="official-pgr-radio" class="official-pgr-input official-pgr-radio radios-version-element" value="1816912" > <input type="checkbox" class="official-pgr-input official-pgr-checkbox checkboxes-version-element" value="1816912" > </div> </div> <div class="row align-items-center"> <div class="col-md-5"> <div class="paper-impl-cell"> <a href="https://github.com/fsoft-ai4code/codecapybara" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> fsoft-ai4code/codecapybara </a> </div> </div> <div class="col-md-3"> <div class="paper-impl-cell"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 168 </div> </div> <div class="col-md-2"> <div class="paper-impl-cell"> <img class="" src="https://production-assets.paperswithcode.com/perf/images/frameworks/pytorch-2fbf2cb9.png" /> </div> </div> <div class="col-md-2 text-center"> <input type="radio" name="official-pgr-radio" class="official-pgr-input official-pgr-radio radios-version-element" value="2240187" > <input type="checkbox" class="official-pgr-input official-pgr-checkbox checkboxes-version-element" value="2240187" > </div> </div> <div class="row align-items-center"> <div class="col-md-5"> <div class="paper-impl-cell"> <a href="https://github.com/my-other-github-account/llm-humaneval-benchmarks" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> my-other-github-account/llm-humaneval-benchmarks </a> </div> </div> <div class="col-md-3"> <div class="paper-impl-cell"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 86 </div> </div> <div class="col-md-2"> <div class="paper-impl-cell"> </div> </div> <div class="col-md-2 text-center"> <input type="radio" name="official-pgr-radio" class="official-pgr-input official-pgr-radio radios-version-element" value="2152858" > <input type="checkbox" class="official-pgr-input official-pgr-checkbox checkboxes-version-element" value="2152858" > </div> </div> <div class="row align-items-center"> <div class="col-md-5"> <div class="paper-impl-cell"> <a href="https://github.com/codefuse-ai/codefuse-evaluation" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> codefuse-ai/codefuse-evaluation </a> </div> </div> <div class="col-md-3"> <div class="paper-impl-cell"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 80 </div> </div> <div class="col-md-2"> <div class="paper-impl-cell"> <img class="" src="https://production-assets.paperswithcode.com/perf/images/frameworks/pytorch-2fbf2cb9.png" /> </div> </div> <div class="col-md-2 text-center"> <input type="radio" name="official-pgr-radio" class="official-pgr-input official-pgr-radio radios-version-element" value="2206778" > <input type="checkbox" class="official-pgr-input official-pgr-checkbox checkboxes-version-element" value="2206778" > </div> </div> <div class="row align-items-center"> <div class="col-md-5"> <div class="paper-impl-cell"> <a href="https://github.com/superli3/codenavi" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> superli3/codenavi </a> </div> </div> <div class="col-md-3"> <div class="paper-impl-cell"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 1 </div> </div> <div class="col-md-2"> <div class="paper-impl-cell"> <img class="" src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAbYAAABiBAMAAADO/+vOAAAACXBIWXMAAA7EAAAOxAGVKw4bAAAAMFBMVEVHcExBUGZCUGZKUmNCUGZCUGZCUGb/igD/jwD/hwD/iwBCUGb/kAD/hAD/mwD/eABjkbvFAAAAC3RSTlMAxZ4pVHvkPbaF3mTG43EAAAjMSURBVHja7Zy/b9tGFMdPImlD8CLXyaRFdRIvWhQrCVJosRMnixblh7NwcWNXixYDXRwvCZChQNGm3rq06RhkSGLJNhf9A2khc81iQWtaCJqDAILK+/2LEk+2AsQEr4BDU7w7fu579967d1YBSEpSkpKUpCSFlh/BdlzRbn4AP+/FEy3T+QB++SeWys2+gWwn74uxRDsJ2Nrtv2OHZv3a6RxD3U7iB/dnp0N0a7dfxQtttwML1A3C/RQvtEEwJZFuEK73LD5oO92BpFu7Fxs3lxkOoG4drpv/X0zcXOZdp9uFsh1z3fyPsYCbfTccIjRBt16v/zEOaL8Nu12sG19vvu/3/z3/bH8Nh4PBQLGTgW5+/1UM0LpDIpusW79/zt3c7jAoAdqA2cljplu/f67d3A5EGxDdTlTd+v1z7OYywyGBU/wbLJBNdnOLeslP/6WWlS4ASM/Pz184FdqAognrrY3IgiK6Oaull/3ps1WULgBIBT+PJmzlKZYN+gDZTgZTskfono1nO/ja2QIfIPk3n+sWxfa169btDvGcZHbS96kxiYNuEE6xk34sdBsMBgMpnvTbvX5MdOsOFDvZY1NSct9ruNwNuvLI9cqXYTta4+WMun1+ItvJ9ndhdpIW2FXjyzndijYdzqTbXqYj20mwE2InzyXbc5CR4xI/CDTjwfYKpsvleDL48GUc2F7D6Ksj7QN8aBVfGLLd+XZ+8Tb9pVQqIpN6f3FdNDPXlN+DJ+6sPywqDV9fvFKMZrPX19eKZmyfUGQp7996aEP+1oTNLiBDfZH8Cn8rguU6vHeJcSxJz8CyCm94l8lvaYRzNfhZjWKz7qGauG27Vqs9xvdnarUtfHU3uFlFbJ+2AdVNsJM4lDZgs3LEC20JbA65V1We4WP/SK6F2JyWAZtVIDUXqMf12ExuiMMbsH3Gcb5mJxlcBNtd5mLxbgdirFCWJn7GZc8skFpXlTuQ7dAlrYxl422hgasjDkqEH6mjq4CNbD4VO9kj83knis3h4UOTdXGP3VtRnvFwu7Z6B7I1WwZsaV6zQbtbYUQrVMsGZHtOd3KKnXxNUw4RbK4QG1VpZ7xsqM9saFHVgfzKEWxi81nSUp4NF7sKHn7KMlmqbiwN9HIsGxGgzt9SZjuiUah3ZT3HxJXiUiScKZsj1oRtzdHxctjIYbsEnvAduJLnalO9ZvfGsd2ALf5QBNfr9BZeahdLJWzOSF/eCrUCkGQGTanbpVts+E3ZkODN26VVOuNT9NkZNropNj1E3eT85Og0EGcr0Ks0XV0FZv0qBAX2usmeqZJJiuRCJuWQsXlX1qATHMOGhrBIB3UDywUbAGVmh+eYXRuhW7APGH3awdgsOu6YKUv+xfbBIbgV5gxy+H2UWlTblleNiksc2R4f4baabFE3aPUVTTc5P9n/WIxiS3PTVyYvVOAvRmRymZl28V7W4U4pRfgR22bo/u1AYJvhRuwG3tlBJYW1UARifyPsZBvCbUexzXG/kyaTI8dnBLnkfRH+GcETEwnZlB7PJihqkxo54ta4qc6pwW5Gz08asFV4UsEmk6PA2cg0dXmAAiSJAZc5LUZxo9kKbDLjpZdlM9DmbkGLq/X11h9tTVKi58xyBRoym8t73xSru8IbVrDcaR7EjGOrC8Pk4sVbxreCBrwCGmiLWpcxdtJEt5zQF1lEOtucFDYrtcisNmOzxJkrTPAs+ueogqAcLUGl+TeYL4nUDfb1oIQLWds5jQ3b92+46aoLb5jCUGZsdovFjHhUiOL7iPRgjjWVBRF20o/UTct7heoGcNTC6VqCISMLTWNrzNOSldk8xS/YGD/oayONPp1Rl7dmJ32/F62bxlYMZbtBP77EbSNlI/5AYwv1b47K1mSmIxi/vI3aLasuQMuXoLxyFJttxmbV6ecXGBsQ2FqTsDFrSqsgk2+haY4Uqwj8IHT/htKT26dg09ebGOFuAmXVnJ3NhSwOaiUH+ysILYXaSf+PKeoGcChNw9sp6UbnJJqDKXTtwno51QVodtJ64Ruut3mhjGID9hJhOwxh807NRuOw6hxquQx/6nl8NS4Bsy98MzYlaR7OBsA1TOeF2EljNslO0sgNVs1XUA2onq3ucMLyk7NvzfybwpYbwUYnZlX3bw1TNivEvyHgjQLqCerqaC4gLD+ZiWarq1ul0brR5blhFpeE73FC4hIEfFBHLcLLVX24w/KTN43iyX1jNhpam8ST4WxqPLlPBrhJZnlwuSRIG24n22NTtozN1XPZGttMrVbb4PUO5Hcno2PI5gqLKUdHqCBnvfR0fnh+MoqtzPemANwKX29z/E3TPLxt6vs3A7Yyj/Ftllmr8LRuRcnwhsclZmzplrjJyYfqlpL3r2SzRgxlWth3G7CllB07W7JiQkg7yg05xzFgs1piprIRyubIeQeywyI7OlfIlxiw2XKmpckpefpOcwGn1E3Iaq0KuSDZljASm5rVAsnqgWUxz2XAhmY8ynMtc4GclpR21Sx3qJ00YMMh/uP1O0stPV/i8pxC63LgvXPUhKOp4z1U8pMmbGWSn7xV5xAWz86DVojHPaWdVHY5+VDdUmLueUxe2YRNyis3hL2ux41nqzgVOynvkZvh/o1vcdgyE2sdggnYpJT8puBHhBMjD0xnveE1JB5Sjdubhp3jtIoTsYnnOEXZ/9M52wRTsZPSSdrWyHiyoJzRibgLYCI2QfK8mBPc5w8ejtHteBLdyOEuPPEYHXPJZ53S2dYWmJCNtXVB+jTLt0z74CzrDf6pJj+7vp4jhhCV+4uLi8RQrQaP5YUz6u8fCCf5+PyR1nLEFmG9Ba1DymI9Es67ceUgqiNdWsFlHoy3k/5Ef+5wbS3y7wastbUSiPw7BbNiT1hT3XeDGJWJ7OS5Y5PsZC/GusVqTqKvZPL1Fq8vZaKv0lLdXoN4ldk3TLf4fQk6g7/ffdx+vw3iB4d1i+f/dGAHsrX3QCzLbsD2DMS0/B78l5SkJCUpSUlKUpKSlGmW/wErOKTxUOS81gAAAABJRU5ErkJggg==" /> </div> </div> <div class="col-md-2 text-center"> <input type="radio" name="official-pgr-radio" class="official-pgr-input official-pgr-radio radios-version-element" value="1880893" > <input type="checkbox" class="official-pgr-input official-pgr-checkbox checkboxes-version-element" value="1880893" > </div> </div> <div class="row align-items-center"> <div class="col-md-5"> <div class="paper-impl-cell"> <a href="https://github.com/superli3/CYRMPR" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> superli3/CYRMPR </a> </div> </div> <div class="col-md-3"> <div class="paper-impl-cell"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 1 </div> </div> <div class="col-md-2"> <div class="paper-impl-cell"> <img class="" src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAbYAAABiBAMAAADO/+vOAAAACXBIWXMAAA7EAAAOxAGVKw4bAAAAMFBMVEVHcExBUGZCUGZKUmNCUGZCUGZCUGb/igD/jwD/hwD/iwBCUGb/kAD/hAD/mwD/eABjkbvFAAAAC3RSTlMAxZ4pVHvkPbaF3mTG43EAAAjMSURBVHja7Zy/b9tGFMdPImlD8CLXyaRFdRIvWhQrCVJosRMnixblh7NwcWNXixYDXRwvCZChQNGm3rq06RhkSGLJNhf9A2khc81iQWtaCJqDAILK+/2LEk+2AsQEr4BDU7w7fu579967d1YBSEpSkpKUpCSFlh/BdlzRbn4AP+/FEy3T+QB++SeWys2+gWwn74uxRDsJ2Nrtv2OHZv3a6RxD3U7iB/dnp0N0a7dfxQtttwML1A3C/RQvtEEwJZFuEK73LD5oO92BpFu7Fxs3lxkOoG4drpv/X0zcXOZdp9uFsh1z3fyPsYCbfTccIjRBt16v/zEOaL8Nu12sG19vvu/3/z3/bH8Nh4PBQLGTgW5+/1UM0LpDIpusW79/zt3c7jAoAdqA2cljplu/f67d3A5EGxDdTlTd+v1z7OYywyGBU/wbLJBNdnOLeslP/6WWlS4ASM/Pz184FdqAognrrY3IgiK6Oaull/3ps1WULgBIBT+PJmzlKZYN+gDZTgZTskfono1nO/ja2QIfIPk3n+sWxfa169btDvGcZHbS96kxiYNuEE6xk34sdBsMBgMpnvTbvX5MdOsOFDvZY1NSct9ruNwNuvLI9cqXYTta4+WMun1+ItvJ9ndhdpIW2FXjyzndijYdzqTbXqYj20mwE2InzyXbc5CR4xI/CDTjwfYKpsvleDL48GUc2F7D6Ksj7QN8aBVfGLLd+XZ+8Tb9pVQqIpN6f3FdNDPXlN+DJ+6sPywqDV9fvFKMZrPX19eKZmyfUGQp7996aEP+1oTNLiBDfZH8Cn8rguU6vHeJcSxJz8CyCm94l8lvaYRzNfhZjWKz7qGauG27Vqs9xvdnarUtfHU3uFlFbJ+2AdVNsJM4lDZgs3LEC20JbA65V1We4WP/SK6F2JyWAZtVIDUXqMf12ExuiMMbsH3Gcb5mJxlcBNtd5mLxbgdirFCWJn7GZc8skFpXlTuQ7dAlrYxl422hgasjDkqEH6mjq4CNbD4VO9kj83knis3h4UOTdXGP3VtRnvFwu7Z6B7I1WwZsaV6zQbtbYUQrVMsGZHtOd3KKnXxNUw4RbK4QG1VpZ7xsqM9saFHVgfzKEWxi81nSUp4NF7sKHn7KMlmqbiwN9HIsGxGgzt9SZjuiUah3ZT3HxJXiUiScKZsj1oRtzdHxctjIYbsEnvAduJLnalO9ZvfGsd2ALf5QBNfr9BZeahdLJWzOSF/eCrUCkGQGTanbpVts+E3ZkODN26VVOuNT9NkZNropNj1E3eT85Og0EGcr0Ks0XV0FZv0qBAX2usmeqZJJiuRCJuWQsXlX1qATHMOGhrBIB3UDywUbAGVmh+eYXRuhW7APGH3awdgsOu6YKUv+xfbBIbgV5gxy+H2UWlTblleNiksc2R4f4baabFE3aPUVTTc5P9n/WIxiS3PTVyYvVOAvRmRymZl28V7W4U4pRfgR22bo/u1AYJvhRuwG3tlBJYW1UARifyPsZBvCbUexzXG/kyaTI8dnBLnkfRH+GcETEwnZlB7PJihqkxo54ta4qc6pwW5Gz08asFV4UsEmk6PA2cg0dXmAAiSJAZc5LUZxo9kKbDLjpZdlM9DmbkGLq/X11h9tTVKi58xyBRoym8t73xSru8IbVrDcaR7EjGOrC8Pk4sVbxreCBrwCGmiLWpcxdtJEt5zQF1lEOtucFDYrtcisNmOzxJkrTPAs+ueogqAcLUGl+TeYL4nUDfb1oIQLWds5jQ3b92+46aoLb5jCUGZsdovFjHhUiOL7iPRgjjWVBRF20o/UTct7heoGcNTC6VqCISMLTWNrzNOSldk8xS/YGD/oayONPp1Rl7dmJ32/F62bxlYMZbtBP77EbSNlI/5AYwv1b47K1mSmIxi/vI3aLasuQMuXoLxyFJttxmbV6ecXGBsQ2FqTsDFrSqsgk2+haY4Uqwj8IHT/htKT26dg09ebGOFuAmXVnJ3NhSwOaiUH+ysILYXaSf+PKeoGcChNw9sp6UbnJJqDKXTtwno51QVodtJ64Ruut3mhjGID9hJhOwxh807NRuOw6hxquQx/6nl8NS4Bsy98MzYlaR7OBsA1TOeF2EljNslO0sgNVs1XUA2onq3ucMLyk7NvzfybwpYbwUYnZlX3bw1TNivEvyHgjQLqCerqaC4gLD+ZiWarq1ul0brR5blhFpeE73FC4hIEfFBHLcLLVX24w/KTN43iyX1jNhpam8ST4WxqPLlPBrhJZnlwuSRIG24n22NTtozN1XPZGttMrVbb4PUO5Hcno2PI5gqLKUdHqCBnvfR0fnh+MoqtzPemANwKX29z/E3TPLxt6vs3A7Yyj/Ftllmr8LRuRcnwhsclZmzplrjJyYfqlpL3r2SzRgxlWth3G7CllB07W7JiQkg7yg05xzFgs1piprIRyubIeQeywyI7OlfIlxiw2XKmpckpefpOcwGn1E3Iaq0KuSDZljASm5rVAsnqgWUxz2XAhmY8ynMtc4GclpR21Sx3qJ00YMMh/uP1O0stPV/i8pxC63LgvXPUhKOp4z1U8pMmbGWSn7xV5xAWz86DVojHPaWdVHY5+VDdUmLueUxe2YRNyis3hL2ux41nqzgVOynvkZvh/o1vcdgyE2sdggnYpJT8puBHhBMjD0xnveE1JB5Sjdubhp3jtIoTsYnnOEXZ/9M52wRTsZPSSdrWyHiyoJzRibgLYCI2QfK8mBPc5w8ejtHteBLdyOEuPPEYHXPJZ53S2dYWmJCNtXVB+jTLt0z74CzrDf6pJj+7vp4jhhCV+4uLi8RQrQaP5YUz6u8fCCf5+PyR1nLEFmG9Ba1DymI9Es67ceUgqiNdWsFlHoy3k/5Ef+5wbS3y7wastbUSiPw7BbNiT1hT3XeDGJWJ7OS5Y5PsZC/GusVqTqKvZPL1Fq8vZaKv0lLdXoN4ldk3TLf4fQk6g7/ffdx+vw3iB4d1i+f/dGAHsrX3QCzLbsD2DMS0/B78l5SkJCUpSUlKUpKSlGmW/wErOKTxUOS81gAAAABJRU5ErkJggg==" /> </div> </div> <div class="col-md-2 text-center"> <input type="radio" name="official-pgr-radio" class="official-pgr-input official-pgr-radio radios-version-element" value="1876077" > <input type="checkbox" class="official-pgr-input official-pgr-checkbox checkboxes-version-element" value="1876077" > </div> </div> <div class="row align-items-center"> <div class="col-md-5"> <div class="paper-impl-cell"> <a href="https://github.com/2796gaurav/human-eval" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> 2796gaurav/human-eval </a> </div> </div> <div class="col-md-3"> <div class="paper-impl-cell"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 1 </div> </div> <div class="col-md-2"> <div class="paper-impl-cell"> </div> </div> <div class="col-md-2 text-center"> <input type="radio" name="official-pgr-radio" class="official-pgr-input official-pgr-radio radios-version-element" value="1814298" > <input type="checkbox" class="official-pgr-input official-pgr-checkbox checkboxes-version-element" value="1814298" > </div> </div> <div class="radios-version-element"> <hr/> <div class="row align-items-center justify-content-between"> <div class="col-md-5"> <div class="paper-impl-cell"> There is no official implementation </div> </div> <div class="col-md-2 text-center"> <input type="radio" name="official-pgr-radio" value="" class="official-pgr-input official-pgr-radio" > </div> </div> </div> <hr/> <div class="row align-items-center justify-content-between"> <div class="col-md-5"> <div class="paper-impl-cell"> Multiple official implementations </div> </div> <div class="col-md-2 text-center"> <div class="custom-control custom-switch"> <input type="checkbox" class="custom-control-input" id="official-pgr-multiple-switch"> <label class="custom-control-label" for="official-pgr-multiple-switch" id="official-pgr-multiple-switch-label"></label> </div> </div> </div> </div> </div> <div class="modal-footer"> <button type="submit" class="btn btn-primary">Submit </button> </div> </form> </div> </div> </div>  <div class="modal fade" id="addRow" role="dialog" aria-labelledby="addRowLabel" aria-hidden="true"> <div class="modal-dialog" role="document"> <div class="modal-content"> <div class="modal-header"> <h5 class="modal-title" id="addRowLabel">Add a new evaluation result row</h5> <button type="button" class="close btn-close" data-bs-dismiss="modal" aria-label="Close"> <span aria-hidden="true">×</span> </button> </div> <form action="" method="post"> <div class="modal-body"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <div id="div_id_task" class="form-group"> <label for="id_task" class=" requiredField"> Task:<span class="asteriskField">*</span> </label> <div class=""> <select name="task" class="select form-control" required id="id_task"> <option value="" selected>---------</option> <option value="5892">HumanEval</option> <option value="3294">Multi-task Language Understanding</option> <option value="267">Language Modelling</option> <option value="211">Code Generation</option> </select> </div> </div> <div class="add-task-hint"> Not in the list? <a href="#addTask" data-bs-toggle="modal" data-bs-dismiss="modal">Add a task.</a> </div> <div id="div_id_dataset" class="form-group"> <label for="id_dataset" class=" requiredField"> Dataset:<span class="asteriskField">*</span> </label> <div class=""> <select name="dataset" class="modelselect2 form-control" required id="id_dataset" data-autocomplete-light-language="en" data-autocomplete-light-url="/dataset-autocomplete/" data-autocomplete-light-function="select2"> <option value="" selected>---------</option> </select> </div> </div> <div id="div_id_model_name" class="form-group"> <label for="id_model_name" class=" requiredField"> Model name:<span class="asteriskField">*</span> </label> <div class=""> <input type="text" name="model_name" class="textinput textInput form-control" required id="id_model_name"> </div> </div> <div id="div_id_metric" class="form-group"> <label for="id_metric" class=" requiredField"> Metric name:<span class="asteriskField">*</span> </label> <div class=""> <select name="metric" class="modelselect2 form-control" required id="id_metric" data-autocomplete-light-language="en" data-autocomplete-light-url="/metric-autocomplete/" data-autocomplete-light-function="select2"> <option value="" selected>---------</option> </select> </div> </div> <div id="sota-metric-names"> </div> <div class="form-group"> <div id="div_id_metric_higher_is_better" class="form-check"> <input type="checkbox" name="metric_higher_is_better" class="checkboxinput form-check-input" id="id_metric_higher_is_better"> <label for="id_metric_higher_is_better" class="form-check-label"> Higher is better (for the metric) </label> </div> </div> <div id="div_id_metric_value" class="form-group"> <label for="id_metric_value" class=" requiredField"> Metric value:<span class="asteriskField">*</span> </label> <div class=""> <input type="text" name="metric_value" class="textinput textInput form-control" required id="id_metric_value"> </div> </div> <div id="sota-metric-values"> </div> <div class="form-group"> <div id="div_id_uses_additional_data" class="form-check"> <input type="checkbox" name="uses_additional_data" class="checkboxinput form-check-input" id="id_uses_additional_data"> <label for="id_uses_additional_data" class="form-check-label"> Uses extra training data </label> </div> </div> <div id="div_id_evaluated_on" class="form-group"> <label for="id_evaluated_on" class=""> Data evaluated on </label> <div class=""> <input type="text" name="evaluated_on" value="2021-07-07" autocomplete="off" class="dateinput form-control" id="id_evaluated_on"> </div> </div> </div> <div class="modal-footer"> <button type="submit" class="btn btn-primary"> Submit </button> </div> </form> </div> </div> </div>  <div class="modal fade" id="removeRow" role="dialog" aria-labelledby="removeRowLabel" aria-hidden="true"> <div class="modal-dialog modal-lg" role="document"> <div class="modal-content"> <div class="modal-header"> <h5 class="modal-title" id="removeRowLabel">Add a new evaluation result row</h5> <button type="button" class="close btn-close" data-bs-dismiss="modal" aria-label="Close"> <span aria-hidden="true">×</span> </button> </div> <form action="" method="post"> <div class="modal-body"> <div class="sota-table"> <table class="table-striped"> <tr> <th>TASK</th> <th>DATASET</th> <th>MODEL</th> <th>METRIC NAME</th> <th>METRIC VALUE</th> <th>GLOBAL RANK</th> <th>REMOVE</th> </tr> <tr> <td> Code Generation </td> <td> APPS </td> <td> Codex 12B (Raw) </td> <td> Introductory Pass@1 </td> <td> 4.14% </td> <td> # 9 </td> <td> <form action="" method="post"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_row_pk" value="36732"> <input type="hidden" name="remove_metric_pk" value="27634"> <button type="submit" class="btn btn-danger"> - </button> </form> </td> </tr> <tr> <td> Code Generation </td> <td> APPS </td> <td> Codex 12B (Raw) </td> <td> Interview Pass@1 </td> <td> 0.14% </td> <td> # 11 </td> <td> <form action="" method="post"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_row_pk" value="36732"> <input type="hidden" name="remove_metric_pk" value="27635"> <button type="submit" class="btn btn-danger"> - </button> </form> </td> </tr> <tr> <td> Code Generation </td> <td> APPS </td> <td> Codex 12B (Raw) </td> <td> Competition Pass@1 </td> <td> 0.02% </td> <td> # 8 </td> <td> <form action="" method="post"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_row_pk" value="36732"> <input type="hidden" name="remove_metric_pk" value="27636"> <button type="submit" class="btn btn-danger"> - </button> </form> </td> </tr> <tr> <td> Code Generation </td> <td> APPS </td> <td> Codex 12B (Raw) </td> <td> Introductory Pass@1000 </td> <td> 25.02% </td> <td> # 4 </td> <td> <form action="" method="post"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_row_pk" value="36732"> <input type="hidden" name="remove_metric_pk" value="40497"> <button type="submit" class="btn btn-danger"> - </button> </form> </td> </tr> <tr> <td> Code Generation </td> <td> APPS </td> <td> Codex 12B (Raw) </td> <td> Interview Pass@1000 </td> <td> 3.70% </td> <td> # 6 </td> <td> <form action="" method="post"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_row_pk" value="36732"> <input type="hidden" name="remove_metric_pk" value="40498"> <button type="submit" class="btn btn-danger"> - </button> </form> </td> </tr> <tr> <td> Code Generation </td> <td> APPS </td> <td> Codex 12B (Raw) </td> <td> Competition Pass@1000 </td> <td> 3.23% </td> <td> # 6 </td> <td> <form action="" method="post"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_row_pk" value="36732"> <input type="hidden" name="remove_metric_pk" value="40499"> <button type="submit" class="btn btn-danger"> - </button> </form> </td> </tr> <tr> <td> Code Generation </td> <td> APPS </td> <td> Codex 12B (Raw) </td> <td> Competition Pass@5 </td> <td> 0.09% </td> <td> # 4 </td> <td> <form action="" method="post"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_row_pk" value="36732"> <input type="hidden" name="remove_metric_pk" value="57517"> <button type="submit" class="btn btn-danger"> - </button> </form> </td> </tr> <tr> <td> Code Generation </td> <td> APPS </td> <td> Codex 12B (Raw) </td> <td> Interview Pass@5 </td> <td> 0.51% </td> <td> # 7 </td> <td> <form action="" method="post"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_row_pk" value="36732"> <input type="hidden" name="remove_metric_pk" value="57518"> <button type="submit" class="btn btn-danger"> - </button> </form> </td> </tr> <tr> <td> Code Generation </td> <td> APPS </td> <td> Codex 12B (Raw) </td> <td> Introductory Pass@5 </td> <td> 9.65% </td> <td> # 3 </td> <td> <form action="" method="post"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_row_pk" value="36732"> <input type="hidden" name="remove_metric_pk" value="57519"> <button type="submit" class="btn btn-danger"> - </button> </form> </td> </tr> <tr> <td> Code Generation </td> <td> APPS </td> <td> Codex 12B (Raw) </td> <td> Competition Pass@any </td> <td> 3.32% </td> <td> # 8 </td> <td> <form action="" method="post"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_row_pk" value="36732"> <input type="hidden" name="remove_metric_pk" value="58145"> <button type="submit" class="btn btn-danger"> - </button> </form> </td> </tr> <tr> <td> Code Generation </td> <td> APPS </td> <td> Codex 12B (Raw) </td> <td> Interview Pass@any </td> <td> 3.70% </td> <td> # 7 </td> <td> <form action="" method="post"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_row_pk" value="36732"> <input type="hidden" name="remove_metric_pk" value="58146"> <button type="submit" class="btn btn-danger"> - </button> </form> </td> </tr> <tr> <td> Code Generation </td> <td> APPS </td> <td> Codex 12B (Raw) </td> <td> Introductory Pass@any </td> <td> 25.02% </td> <td> # 5 </td> <td> <form action="" method="post"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_row_pk" value="36732"> <input type="hidden" name="remove_metric_pk" value="58147"> <button type="submit" class="btn btn-danger"> - </button> </form> </td> </tr> <tr> <td> Multi-task Language Understanding </td> <td> BBH-alg </td> <td> code-davinci-002 175B (CoT) </td> <td> Average (%) </td> <td> 73.9 </td> <td> # 1 </td> <td> <form action="" method="post"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_row_pk" value="117128"> <input type="hidden" name="remove_metric_pk" value="54479"> <button type="submit" class="btn btn-danger"> - </button> </form> </td> </tr> <tr> <td> Multi-task Language Understanding </td> <td> BBH-nlp </td> <td> code-davinci-002 175B (CoT) </td> <td> Average (%) </td> <td> 73.5 </td> <td> # 8 </td> <td> <form action="" method="post"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_row_pk" value="117129"> <input type="hidden" name="remove_metric_pk" value="54478"> <button type="submit" class="btn btn-danger"> - </button> </form> </td> </tr> </table> </div> </div> </form> </div> </div> </div>  <div class="modal fade" id="addTask" role="dialog" aria-labelledby="addTaskLabel" aria-hidden="true"> <div class="modal-dialog" role="document"> <div class="modal-content"> <div class="modal-header"> <h5 class="modal-title" id="addTaskLabel">Add a task</h5> <button type="button" class="close btn-close" data-bs-dismiss="modal" aria-label="Close"> <span aria-hidden="true">×</span> </button> </div> <div class="modal-body"> <div class="current-tasks-title">Attached tasks:</div> <ul class="list-unstyled"> <li> <a href="/task/code-generation"> <span class="badge badge-primary">CODE GENERATION</span> </a> </li> </ul> <ul class="list-unstyled"> <li> <a href="/task/humaneval"> <span class="badge badge-primary">HUMANEVAL</span> </a> </li> </ul> <ul class="list-unstyled"> <li> <a href="/task/language-modelling"> <span class="badge badge-primary">LANGUAGE MODELLING</span> </a> </li> </ul> <ul class="list-unstyled"> <li> <a href="/task/multi-task-language-understanding"> <span class="badge badge-primary">MULTI-TASK LANGUAGE UNDERSTANDING</span> </a> </li> </ul> <form action="" method="post"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <div id="div_id_task" class="form-group"> <label for="id_task" class=""> Add: </label> <div class=""> <select name="task" class="modelselect2 form-control" id="id_task" data-autocomplete-light-language="en" data-autocomplete-light-url="/task-autocomplete/" data-autocomplete-light-function="select2"> <option value="" selected>---------</option> </select> </div> </div> <div class="modal-help-text"> Not in the list?<br/> <a href="#" id="new-task-form-toggle"> <span class=" icon-wrapper icon-ion" data-name="add"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M256 112v288m144-144H112"/></svg></span> Create a new task</a>. </div> <div id="new-task-form" style="display: none"> <div id="div_id_new_task_name" class="form-group"> <label for="id_new_task_name" class=""> New task name: </label> <div class=""> <input type="text" name="new_task_name" maxlength="200" class="textinput textInput form-control" id="id_new_task_name"> </div> </div> <div id="div_id_new_task_area" class="form-group"> <label for="id_new_task_area" class=""> Top-level area: </label> <div class=""> <select name="new_task_area" class="select form-control" id="id_new_task_area"> <option value="" selected>---------</option> <option value="17">Adversarial</option> <option value="18">Audio</option> <option value="11">Computer Code</option> <option value="3">Computer Vision</option> <option value="9">Graphs</option> <option value="15">Knowledge Base</option> <option value="7">Medical</option> <option value="6">Methodology</option> <option value="5">Miscellaneous</option> <option value="12">Music</option> <option value="4">Natural Language Processing</option> <option value="13">Playing Games</option> <option value="14">Reasoning</option> <option value="16">Robots</option> <option value="10">Speech</option> <option value="8">Time Series</option> </select> </div> </div> <div id="div_id_new_task_parent" class="form-group"> <label for="id_new_task_parent" class=""> Parent task (if any): </label> <div class=""> <select name="new_task_parent" class="modelselect2 form-control" id="id_new_task_parent" data-autocomplete-light-language="en" data-autocomplete-light-url="/task-and-tag-autocomplete/" data-autocomplete-light-function="select2"> <option value="" selected>---------</option> </select> </div> </div> <div id="div_id_new_task_desc" class="form-group"> <label for="id_new_task_desc" class=""> Description: </label> <div class=""> <textarea name="new_task_desc" cols="40" rows="3" class="textarea form-control" id="id_new_task_desc"> </textarea> </div> </div> </div> <div class="modal-footer"> <button type="submit" class="btn btn-primary"> Submit </button> </div> </form> </div> </div> </div> </div>  <div class="modal fade" id="removeTask" tabindex="-1" role="dialog" aria-labelledby="removeTaskLabel" aria-hidden="true"> <div class="modal-dialog" role="document"> <div class="modal-content"> <div class="modal-header"> <h5 class="modal-title" id="removeTaskLabel">Remove a task</h5> <button type="button" class="close btn-close" data-bs-dismiss="modal" aria-label="Close"> <span aria-hidden="true">×</span> </button> </div> <form action="" method="post"> <div class="modal-body"> <ul class="list-unstyled paper-tasks"> <form action="" method="post"> <li> <a href="/task/code-generation"> <span class="badge badge-primary"> <img src="https://production-media.paperswithcode.com/thumbnails/task/658fcf6a-52ca-4865-b111-28bf61c346dc.jpg"> <span>Code Generation</span> </span> </a> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_task_pk" value="211"> <button type="submit" class="btn btn-danger" style="width:2.5em">- </button> </li> </form> </ul> <ul class="list-unstyled paper-tasks"> <form action="" method="post"> <li> <a href="/task/humaneval"> <span class="badge badge-primary"> <img src="https://production-media.paperswithcode.com/tasks/default.gif"> <span>HumanEval</span> </span> </a> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_task_pk" value="5892"> <button type="submit" class="btn btn-danger" style="width:2.5em">- </button> </li> </form> </ul> <ul class="list-unstyled paper-tasks"> <form action="" method="post"> <li> <a href="/task/language-modelling"> <span class="badge badge-primary"> <img src="https://production-media.paperswithcode.com/thumbnails/task/task-0000000267-8df06634.jpg"> <span>Language Modelling</span> </span> </a> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_task_pk" value="267"> <button type="submit" class="btn btn-danger" style="width:2.5em">- </button> </li> </form> </ul> <ul class="list-unstyled paper-tasks"> <form action="" method="post"> <li> <a href="/task/multi-task-language-understanding"> <span class="badge badge-primary"> <img src="https://production-media.paperswithcode.com/thumbnails/task/93c61892-0ee4-4896-8201-6f7497f6058e.jpg"> <span>Multi-task Language Understanding</span> </span> </a> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_task_pk" value="3294"> <button type="submit" class="btn btn-danger" style="width:2.5em">- </button> </li> </form> </ul> </div> </form> </div> </div> </div>  <div class="modal fade" id="addMethod" role="dialog" aria-labelledby="addMethodLabel" aria-hidden="true"> <div class="modal-dialog" role="document"> <div class="modal-content"> <div class="modal-header"> <h5 class="modal-title" id="addMethodLabel">Add a method</h5> <button type="button" class="close btn-close" data-bs-dismiss="modal" aria-label="Close"> <span aria-hidden="true">×</span> </button> </div> <div class="modal-body"> <div class="current-methods-title">Attached methods:</div> <ul class="list-unstyled"> <li> <a href="/method/adam"> <span class="badge badge-primary">ADAM</span> </a> </li> </ul> <ul class="list-unstyled"> <li> <a href="/method/attention-dropout"> <span class="badge badge-primary">ATTENTION DROPOUT</span> </a> </li> </ul> <ul class="list-unstyled"> <li> <a href="/method/bpe"> <span class="badge badge-primary">BPE</span> </a> </li> </ul> <ul class="list-unstyled"> <li> <a href="/method/cosine-annealing"> <span class="badge badge-primary">COSINE ANNEALING</span> </a> </li> </ul> <ul class="list-unstyled"> <li> <a href="/method/dense-connections"> <span class="badge badge-primary">DENSE CONNECTIONS</span> </a> </li> </ul> <ul class="list-unstyled"> <li> <a href="/method/discriminative-fine-tuning"> <span class="badge badge-primary">DISCRIMINATIVE FINE-TUNING</span> </a> </li> </ul> <ul class="list-unstyled"> <li> <a href="/method/dropout"> <span class="badge badge-primary">DROPOUT</span> </a> </li> </ul> <ul class="list-unstyled"> <li> <a href="/method/fixed-factorized-attention"> <span class="badge badge-primary">FIXED FACTORIZED ATTENTION</span> </a> </li> </ul> <ul class="list-unstyled"> <li> <a href="/method/gelu"> <span class="badge badge-primary">GELU</span> </a> </li> </ul> <ul class="list-unstyled"> <li> <a href="/method/gpt"> <span class="badge badge-primary">GPT</span> </a> </li> </ul> <ul class="list-unstyled"> <li> <a href="/method/gpt-3"> <span class="badge badge-primary">GPT-3</span> </a> </li> </ul> <ul class="list-unstyled"> <li> <a href="/method/layer-normalization"> <span class="badge badge-primary">LAYER NORMALIZATION</span> </a> </li> </ul> <ul class="list-unstyled"> <li> <a href="/method/linear-layer"> <span class="badge badge-primary">LINEAR LAYER</span> </a> </li> </ul> <ul class="list-unstyled"> <li> <a href="/method/linear-warmup-with-cosine-annealing"> <span class="badge badge-primary">LINEAR WARMUP WITH COSINE ANNEALING</span> </a> </li> </ul> <ul class="list-unstyled"> <li> <a href="/method/multi-head-attention"> <span class="badge badge-primary">MULTI-HEAD ATTENTION</span> </a> </li> </ul> <ul class="list-unstyled"> <li> <a href="/method/residual-connection"> <span class="badge badge-primary">RESIDUAL CONNECTION</span> </a> </li> </ul> <ul class="list-unstyled"> <li> <a href="/method/scaled"> <span class="badge badge-primary">SCALED DOT-PRODUCT ATTENTION</span> </a> </li> </ul> <ul class="list-unstyled"> <li> <a href="/method/softmax"> <span class="badge badge-primary">SOFTMAX</span> </a> </li> </ul> <ul class="list-unstyled"> <li> <a href="/method/strided-attention"> <span class="badge badge-primary">STRIDED ATTENTION</span> </a> </li> </ul> <ul class="list-unstyled"> <li> <a href="/method/weight-decay"> <span class="badge badge-primary">WEIGHT DECAY</span> </a> </li> </ul> <form action="" method="post"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <div id="div_id_method" class="form-group"> <label for="id_method" class=""> Add: </label> <div class=""> <select name="method" class="modelselect2 form-control" id="id_method" data-autocomplete-light-language="en" data-autocomplete-light-url="/method-autocomplete/" data-autocomplete-light-function="select2"> <option value="" selected>---------</option> </select> </div> </div> <div class="modal-help-text"> Not in the list?<br/> <a href="#" id="new-method-form-toggle"> <span class=" icon-wrapper icon-ion" data-name="add"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M256 112v288m144-144H112"/></svg></span> Create a new method</a>. </div> <div id="new-method-form" style="display: none"> <div id="div_id_new_method_name" class="form-group"> <label for="id_new_method_name" class=""> <b>New method name</b> (e.g. ReLU): </label> <div class=""> <input type="text" name="new_method_name" maxlength="200" minlength="2" class="textinput textInput form-control" id="id_new_method_name"> </div> </div> <div id="div_id_new_method_full_name" class="form-group"> <label for="id_new_method_full_name" class=""> <b>New method full name</b> (e.g. Rectified Linear Unit): </label> <div class=""> <input type="text" name="new_method_full_name" maxlength="200" minlength="2" class="textinput textInput form-control" id="id_new_method_full_name"> </div> </div> <div id="div_id_new_method_paper" class="form-group"> <label for="id_new_method_paper" class=""> <b>Paper where method was first introduced</b>: </label> <div class=""> <select name="new_method_paper" class="modelselect2 form-control" id="id_new_method_paper" data-autocomplete-light-language="en" data-autocomplete-light-url="/paper-autocomplete/" data-autocomplete-light-function="select2"> <option value="" selected>---------</option> </select> </div> </div> <div id="div_id_new_method_collection" class="form-group"> <label for="id_new_method_collection" class=""> <b>Method category</b> (e.g. Activation Functions): <i>If no match, add something for now then you can add a new category afterwards.</i> </label> <div class=""> <select name="new_method_collection" class="modelselect2 form-control" id="id_new_method_collection" data-autocomplete-light-language="en" data-autocomplete-light-url="/method-collection-autocomplete/" data-autocomplete-light-function="select2"> <option value="" selected>---------</option> </select> </div> </div> <div id="div_id_new_method_desc" class="form-group"> <label for="id_new_method_desc" class=""> <b>Markdown description</b> (optional; $\LaTeX$ enabled): <i>You can edit this later, so feel free to start with something succinct.</i> </label> <div class=""> <textarea name="new_method_desc" cols="40" rows="10" class="textarea form-control" id="id_new_method_desc"> </textarea> </div> </div> </div> <div class="modal-footer"> <button type="submit" class="btn btn-primary"> Submit </button> </div> </form> </div> </div> </div> </div>  <div class="modal fade" id="removeMethod" tabindex="-1" role="dialog" aria-labelledby="removeMethodLabel" aria-hidden="true"> <div class="modal-dialog" role="document"> <div class="modal-content"> <div class="modal-header"> <h5 class="modal-title" id="removeMethodLabel">Remove a method</h5> <button type="button" class="close btn-close" data-bs-dismiss="modal" aria-label="Close"> <span aria-hidden="true">×</span> </button> </div> <form action="" method="post"> <div class="modal-body"> <ul class="list-unstyled"> <form action="" method="post"> <li> <a href="/method/adam"> <span class="badge badge-primary">ADAM</span> </a> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_method_pk" value="496"> <button type="submit" class="btn btn-danger" style="width:2.5em">- </button> </li> </form> </ul> <ul class="list-unstyled"> <form action="" method="post"> <li> <a href="/method/attention-dropout"> <span class="badge badge-primary">ATTENTION DROPOUT</span> </a> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_method_pk" value="623"> <button type="submit" class="btn btn-danger" style="width:2.5em">- </button> </li> </form> </ul> <ul class="list-unstyled"> <form action="" method="post"> <li> <a href="/method/bpe"> <span class="badge badge-primary">BPE</span> </a> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_method_pk" value="58"> <button type="submit" class="btn btn-danger" style="width:2.5em">- </button> </li> </form> </ul> <ul class="list-unstyled"> <form action="" method="post"> <li> <a href="/method/cosine-annealing"> <span class="badge badge-primary">COSINE ANNEALING</span> </a> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_method_pk" value="267"> <button type="submit" class="btn btn-danger" style="width:2.5em">- </button> </li> </form> </ul> <ul class="list-unstyled"> <form action="" method="post"> <li> <a href="/method/dense-connections"> <span class="badge badge-primary">DENSE CONNECTIONS</span> </a> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_method_pk" value="374"> <button type="submit" class="btn btn-danger" style="width:2.5em">- </button> </li> </form> </ul> <ul class="list-unstyled"> <form action="" method="post"> <li> <a href="/method/discriminative-fine-tuning"> <span class="badge badge-primary">DISCRIMINATIVE FINE-TUNING</span> </a> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_method_pk" value="81"> <button type="submit" class="btn btn-danger" style="width:2.5em">- </button> </li> </form> </ul> <ul class="list-unstyled"> <form action="" method="post"> <li> <a href="/method/dropout"> <span class="badge badge-primary">DROPOUT</span> </a> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_method_pk" value="169"> <button type="submit" class="btn btn-danger" style="width:2.5em">- </button> </li> </form> </ul> <ul class="list-unstyled"> <form action="" method="post"> <li> <a href="/method/fixed-factorized-attention"> <span class="badge badge-primary">FIXED FACTORIZED ATTENTION</span> </a> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_method_pk" value="68"> <button type="submit" class="btn btn-danger" style="width:2.5em">- </button> </li> </form> </ul> <ul class="list-unstyled"> <form action="" method="post"> <li> <a href="/method/gelu"> <span class="badge badge-primary">GELU</span> </a> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_method_pk" value="347"> <button type="submit" class="btn btn-danger" style="width:2.5em">- </button> </li> </form> </ul> <ul class="list-unstyled"> <form action="" method="post"> <li> <a href="/method/gpt"> <span class="badge badge-primary">GPT</span> </a> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_method_pk" value="508"> <button type="submit" class="btn btn-danger" style="width:2.5em">- </button> </li> </form> </ul> <ul class="list-unstyled"> <form action="" method="post"> <li> <a href="/method/gpt-3"> <span class="badge badge-primary">GPT-3</span> </a> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_method_pk" value="722"> <button type="submit" class="btn btn-danger" style="width:2.5em">- </button> </li> </form> </ul> <ul class="list-unstyled"> <form action="" method="post"> <li> <a href="/method/layer-normalization"> <span class="badge badge-primary">LAYER NORMALIZATION</span> </a> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_method_pk" value="386"> <button type="submit" class="btn btn-danger" style="width:2.5em">- </button> </li> </form> </ul> <ul class="list-unstyled"> <form action="" method="post"> <li> <a href="/method/linear-layer"> <span class="badge badge-primary">LINEAR LAYER</span> </a> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_method_pk" value="57"> <button type="submit" class="btn btn-danger" style="width:2.5em">- </button> </li> </form> </ul> <ul class="list-unstyled"> <form action="" method="post"> <li> <a href="/method/linear-warmup-with-cosine-annealing"> <span class="badge badge-primary">LINEAR WARMUP WITH COSINE ANNEALING</span> </a> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_method_pk" value="592"> <button type="submit" class="btn btn-danger" style="width:2.5em">- </button> </li> </form> </ul> <ul class="list-unstyled"> <form action="" method="post"> <li> <a href="/method/multi-head-attention"> <span class="badge badge-primary">MULTI-HEAD ATTENTION</span> </a> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_method_pk" value="37"> <button type="submit" class="btn btn-danger" style="width:2.5em">- </button> </li> </form> </ul> <ul class="list-unstyled"> <form action="" method="post"> <li> <a href="/method/residual-connection"> <span class="badge badge-primary">RESIDUAL CONNECTION</span> </a> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_method_pk" value="9"> <button type="submit" class="btn btn-danger" style="width:2.5em">- </button> </li> </form> </ul> <ul class="list-unstyled"> <form action="" method="post"> <li> <a href="/method/scaled"> <span class="badge badge-primary">SCALED DOT-PRODUCT ATTENTION</span> </a> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_method_pk" value="6"> <button type="submit" class="btn btn-danger" style="width:2.5em">- </button> </li> </form> </ul> <ul class="list-unstyled"> <form action="" method="post"> <li> <a href="/method/softmax"> <span class="badge badge-primary">SOFTMAX</span> </a> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_method_pk" value="537"> <button type="submit" class="btn btn-danger" style="width:2.5em">- </button> </li> </form> </ul> <ul class="list-unstyled"> <form action="" method="post"> <li> <a href="/method/strided-attention"> <span class="badge badge-primary">STRIDED ATTENTION</span> </a> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_method_pk" value="60"> <button type="submit" class="btn btn-danger" style="width:2.5em">- </button> </li> </form> </ul> <ul class="list-unstyled"> <form action="" method="post"> <li> <a href="/method/weight-decay"> <span class="badge badge-primary">WEIGHT DECAY</span> </a> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <input type="hidden" name="remove_method_pk" value="564"> <button type="submit" class="btn btn-danger" style="width:2.5em">- </button> </li> </form> </ul> </div> </form> </div> </div> </div>  <div class="modal fade" id="badgeModal" tabindex="-1" role="dialog" aria-labelledby="badgeModalLabel" aria-hidden="true" > <div class="modal-dialog modal-lg" role="document"> <div class="modal-content modal-badge"> <div class="row"> <div class="col-md-12 paper-evaluation-section-title"> <div class="paper-section-title"> <div class="row"> <div class="col-md-12 zero-padding"> <h1>🦡 Badges</h1> <hr/> </div> </div> </div> </div> </div> <div class="paper-evaluation-section" id="badges"> <div class="row"> <div class="col-md-12"> <p> Include the markdown at the top of your GitHub <code>README.md</code> file to showcase the performance of the model. </p> <p> Badges are live and will be dynamically updated with the latest ranking of this paper. </p> <div class="sota-table badge-table"> <table class="table-striped"> <tr> <th> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> Badge </th> <th style="width: 50%">Markdown </th> </tr> <tr> <td> <img src="https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/evaluating-large-language-models-trained-on/multi-task-language-understanding-on-bbh-alg" /> </td> <td> <code>[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/evaluating-large-language-models-trained-on/multi-task-language-understanding-on-bbh-alg)](https://paperswithcode.com/sota/multi-task-language-understanding-on-bbh-alg?p=evaluating-large-language-models-trained-on)</code> </td> </tr> <tr> <td> <img src="https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/evaluating-large-language-models-trained-on/multi-task-language-understanding-on-bbh-nlp" /> </td> <td> <code>[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/evaluating-large-language-models-trained-on/multi-task-language-understanding-on-bbh-nlp)](https://paperswithcode.com/sota/multi-task-language-understanding-on-bbh-nlp?p=evaluating-large-language-models-trained-on)</code> </td> </tr> <tr> <td> <img src="https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/evaluating-large-language-models-trained-on/code-generation-on-apps" /> </td> <td> <code>[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/evaluating-large-language-models-trained-on/code-generation-on-apps)](https://paperswithcode.com/sota/code-generation-on-apps?p=evaluating-large-language-models-trained-on)</code> </td> </tr> </table> </div> </div> </div> </div> </div> </div> </div>  <div class="modal fade" id="editDatasets" role="dialog" aria-labelledby="editDatasetsLabel" aria-hidden="true"> <div class="modal-dialog" role="document"> <div class="modal-content"> <div class="modal-header"> <h5 class="modal-title" id="editDatasetsLabel">Edit Datasets</h5> <button type="button" class="close btn-close" data-bs-dismiss="modal" aria-label="Close"> <span aria-hidden="true">×</span> </button> </div> <form action="" method="post"> <div class="modal-body paper-page-edit-dataset-modal"> <input type="hidden" name="csrfmiddlewaretoken" value="PiwqMWpekGiQJxsJDlBFYEnOg1LKB1GmBnMlMG9rhxERnfh6k888hx1PRBXdj2zA"> <div id="div_id_introduced" class="form-group"> <label for="id_introduced" class=""> Add or remove datasets <b>introduced</b> in this paper: </label> <div class=""> <select name="introduced" data-container-css-class="" data-allow-clear="false" style="width: 100%" class="modelselect2multiple form-control" id="id_introduced" data-autocomplete-light-language="en" data-autocomplete-light-url="/dataset-autocomplete/" data-autocomplete-light-function="select2" multiple> <option value="8024" selected>HumanEval</option> </select><div style="display:none" class="dal-forward-conf" id="dal-forward-conf-for_id_introduced"><script type="text/dal-forward-conf">[{"type": "const", "val": true, "dst": "canonical_only"}, {"type": "const", "val": true, "dst": "disable_create_option"}]</script></div> </div> </div> <div id="div_id_used" class="form-group"> <label for="id_used" class=""> Add or remove other datasets <b>used</b> in this paper: </label> <div class=""> <select name="used" data-container-css-class="" data-allow-clear="false" style="width: 100%" class="modelselect2multiple form-control" id="id_used" data-autocomplete-light-language="en" data-autocomplete-light-url="/dataset-autocomplete/" data-autocomplete-light-function="select2" multiple> <option value="7444" selected>APPS</option> <option value="17909" selected>BBH</option> <option value="9378" selected>BIG-bench</option> </select><div style="display:none" class="dal-forward-conf" id="dal-forward-conf-for_id_used"><script type="text/dal-forward-conf">[{"type": "const", "val": true, "dst": "canonical_only"}, {"type": "const", "val": true, "dst": "disable_create_option"}]</script></div> </div> </div> <div style="display: inline-block; padding-bottom: 15px;font-size:14px;"> Paper introduces a new dataset? <div style="padding-top:4px"> <a href="/contribute/dataset/new"> <span class=" icon-wrapper icon-fa icon-fa-solid" data-name="plus-circle"><svg viewBox="0 0 512 514.999" xmlns="http://www.w3.org/2000/svg"><path d="M256 9.998c137 0 248 111 248 248s-111 248-248 248-248-111-248-248 111-248 248-248zm144 276v-56c0-6.6-5.4-12-12-12h-92v-92c0-6.6-5.4-12-12-12h-56c-6.6 0-12 5.4-12 12v92h-92c-6.6 0-12 5.4-12 12v56c0 6.6 5.4 12 12 12h92v92c0 6.6 5.4 12 12 12h56c6.6 0 12-5.4 12-12v-92h92c6.6 0 12-5.4 12-12z"/></svg></span> Add a new dataset here </a> </div> </div> </div> <div class="modal-footer"> <button type="submit" class="btn btn-primary" name="edit-datasets"> Save </button> </div> </form> </div> </div> </div> </template> <div class="container content content-buffer "> <main> <div class="paper-title"> <div class="row"> <div class="col-md-12"> <h1> Evaluating Large Language Models Trained on Code </h1> <div class="authors"> <p> <span class="author-span">7 Jul 2021</span>  ·  <span class="author-span"> <a href="/author/mark-chen">Mark Chen</a></span>, <span class="author-span"> <a href="/author/jerry-tworek">Jerry Tworek</a></span>, <span class="author-span"> <a href="/author/heewoo-jun">Heewoo Jun</a></span>, <span class="author-span"> <a href="/author/qiming-yuan">Qiming Yuan</a></span>, <span class="author-span"> <a href="/author/henrique-ponde-de-oliveira-pinto-1">Henrique Ponde de Oliveira Pinto</a></span>, <span class="author-span"> <a href="/author/jared-kaplan">Jared Kaplan</a></span>, <span class="author-span"> <a href="/author/harri-edwards">Harri Edwards</a></span>, <span class="author-span"> <a href="/author/yuri-burda">Yuri Burda</a></span>, <span class="author-span"> <a href="/author/nicholas-joseph">Nicholas Joseph</a></span>, <span class="author-span"> <a href="/author/greg-brockman">Greg Brockman</a></span>, <span class="author-span"> <a href="/author/alex-ray">Alex Ray</a></span>, <span class="author-span"> <a href="/author/raul-puri">Raul Puri</a></span>, <span class="author-span"> <a href="/author/gretchen-krueger">Gretchen Krueger</a></span>, <span class="author-span"> <a href="/author/michael-petrov">Michael Petrov</a></span>, <span class="author-span"> <a href="/author/heidy-khlaaf">Heidy Khlaaf</a></span>, <span class="author-span"> <a href="/author/girish-sastry">Girish Sastry</a></span>, <span class="author-span"> <a href="/author/pamela-mishkin">Pamela Mishkin</a></span>, <span class="author-span"> <a href="/author/brooke-chan">Brooke Chan</a></span>, <span class="author-span"> <a href="/author/scott-gray">Scott Gray</a></span>, <span class="author-span"> <a href="/author/nick-ryder">Nick Ryder</a></span>, <span class="author-span"> <a href="/author/mikhail-pavlov">Mikhail Pavlov</a></span>, <span class="author-span"> <a href="/author/alethea-power">Alethea Power</a></span>, <span class="author-span"> <a href="/author/lukasz-kaiser">Lukasz Kaiser</a></span>, <span class="author-span"> <a href="/author/mohammad-bavarian">Mohammad Bavarian</a></span>, <span class="author-span"> <a href="/author/clemens-winter">Clemens Winter</a></span>, <span class="author-span"> <a href="/author/philippe-tillet">Philippe Tillet</a></span>, <span class="author-span"> <a href="/author/felipe-petroski-such">Felipe Petroski Such</a></span>, <span class="author-span"> <a href="/author/dave-cummings">Dave Cummings</a></span>, <span class="author-span"> <a href="/author/matthias-plappert">Matthias Plappert</a></span>, <span class="author-span"> <a href="/author/fotios-chantzis">Fotios Chantzis</a></span>, <span class="author-span"> <a href="/author/elizabeth-barnes">Elizabeth Barnes</a></span>, <span class="author-span"> <a href="/author/ariel-herbert-voss">Ariel Herbert-Voss</a></span>, <span class="author-span"> <a href="/author/william-hebgen-guss">William Hebgen Guss</a></span>, <span class="author-span"> <a href="/author/alex-nichol">Alex Nichol</a></span>, <span class="author-span"> <a href="/author/alex-paino">Alex Paino</a></span>, <span class="author-span"> <a href="/author/nikolas-tezak">Nikolas Tezak</a></span>, <span class="author-span"> <a href="/author/jie-tang">Jie Tang</a></span>, <span class="author-span"> <a href="/author/igor-babuschkin">Igor Babuschkin</a></span>, <span class="author-span"> <a href="/author/suchir-balaji">Suchir Balaji</a></span>, <span class="author-span"> <a href="/author/shantanu-jain">Shantanu Jain</a></span>, <span class="author-span"> <a href="/author/william-saunders">William Saunders</a></span>, <span class="author-span"> <a href="/author/christopher-hesse">Christopher Hesse</a></span>, <span class="author-span"> <a href="/author/andrew-n-carr-1">Andrew N. Carr</a></span>, <span class="author-span"> <a href="/author/jan-leike">Jan Leike</a></span>, <span class="author-span"> <a href="/author/josh-achiam">Josh Achiam</a></span>, <span class="author-span"> <a href="/author/vedant-misra">Vedant Misra</a></span>, <span class="author-span"> <a href="/author/evan-morikawa">Evan Morikawa</a></span>, <span class="author-span"> <a href="/author/alec-radford">Alec Radford</a></span>, <span class="author-span"> <a href="/author/matthew-knight">Matthew Knight</a></span>, <span class="author-span"> <a href="/author/miles-brundage">Miles Brundage</a></span>, <span class="author-span"> <a href="/author/mira-murati">Mira Murati</a></span>, <span class="author-span"> <a href="/author/katie-mayer">Katie Mayer</a></span>, <span class="author-span"> <a href="/author/peter-welinder">Peter Welinder</a></span>, <span class="author-span"> <a href="/author/bob-mcgrew">Bob McGrew</a></span>, <span class="author-span"> <a href="/author/dario-amodei">Dario Amodei</a></span>, <span class="author-span"> <a href="/author/sam-mccandlish">Sam McCandlish</a></span>, <span class="author-span"> <a href="/author/ilya-sutskever">Ilya Sutskever</a></span>, <span class="author-span"> <a href="/author/wojciech-zaremba">Wojciech Zaremba</a></span> <span class="hidden-element">· </span><button type="button" class="badge-edit" data-bs-toggle="modal" data-bs-toggle="modal" data-bs-target="#loginModal"> <span class=" icon-wrapper icon-fa icon-fa-solid" data-name="edit"><svg viewBox="0 0 576 514.999" xmlns="http://www.w3.org/2000/svg"><path d="M402.6 85.198l90.2 90.2c3.8 3.8 3.8 10 0 13.8l-218.399 218.4-92.8 10.3c-12.4 1.4-22.9-9.1-21.5-21.5l10.3-92.8 218.4-218.4c3.799-3.8 10-3.8 13.799 0zm162-22.9c15.2 15.2 15.2 39.9 0 55.2l-35.4 35.4c-3.8 3.8-10 3.8-13.8 0l-90.2-90.2c-3.8-3.8-3.8-10 0-13.8l35.4-35.4c15.3-15.2 40-15.2 55.2 0zM384 348.198c0-3.2 1.3-6.2 3.5-8.5l40-40c7.6-7.5 20.5-2.2 20.5 8.5v157.8c0 26.5-21.5 48-48 48H48c-26.5 0-48-21.5-48-48v-352c0-26.5 21.5-48 48-48h285.8c10.7 0 16.1 12.9 8.5 20.5l-40 40c-2.3 2.2-5.3 3.5-8.5 3.5H64v320h320v-101.8z"/></svg></span> <span>Edit social preview</span> </button> </p> </div> </div> </div> </div> <div class="paper-abstract"> <div class="row"> <div class="col-md-12"> <p> We introduce Codex, a GPT language model fine-tuned on publicly available code from GitHub, and study its Python code-writing capabilities. A distinct production version of Codex powers GitHub Copilot. On HumanEval, a new evaluation set we release to measure functional correctness for synthesizing programs from docstrings, our model solves 28.8% of the problems, while GPT-3 solves 0% and GPT-J solves 11.4%. Furthermore, we find that repeated sampling from the model is a surprisingly effective strategy for producing working solutions to difficult prompts. Using this method, we solve 70.2% of our problems with 100 samples per problem. Careful investigation of our model reveals its limitations, including difficulty with docstrings describing long chains of operations and with binding operations to variables. Finally, we discuss the potential broader impacts of deploying powerful code generation technologies, covering safety, security, and economics. </p> <a href="https://arxiv.org/pdf/2107.03374v2.pdf" onclick="captureOutboundLink('https://arxiv.org/pdf/2107.03374v2.pdf'); return true;" class="badge badge-light "> <span class=" icon-wrapper icon-fa icon-fa-regular" data-name="file-pdf"><svg viewBox="0 0 384 513.795" xmlns="http://www.w3.org/2000/svg"><path d="M369.9 98.88c9 9 14.1 21.3 14.1 34v332.1c0 26.5-21.5 48-48 48H48c-26.5 0-48-21.5-48-48v-416c0-26.5 21.5-48 48-48.1h204.1c12.7 0 24.9 5.1 33.9 14.1zm-37.8 30.1L256 52.88v76.1h76.1zM48 464.98h288v-288H232c-13.3 0-24-10.7-24-24v-104H48v416zm250.2-143.7c10.5 10.5 8 38.7-17.5 38.7-14.8 0-36.9-6.8-55.8-17-21.6 3.6-46 12.7-68.4 20.1-50.1 86.4-79.4 47-76.1 31.2 4-20 31-35.9 51-46.2 10.5-18.4 25.4-50.5 35.4-74.4-7.4-28.6-11.4-51-7-67.1 4.8-17.7 38.4-20.3 42.6 5.9 4.7 15.4-1.5 39.9-5.4 56 8.1 21.3 19.6 35.8 36.8 46.3 17.4-2.2 52.2-5.5 64.4 6.5zm-198.1 77.8c0 .7 11.4-4.7 30.4-35-5.9 5.5-25.299 21.3-30.4 35zm81.6-190.6c-2.5 0-2.6 26.9 1.8 40.8 4.9-8.7 5.6-40.8-1.8-40.8zm-24.4 136.6c15.9-6.1 34-14.9 54.8-19.2-11.199-8.3-21.8-20.4-30.1-35.5-6.7 17.7-15 37.8-24.7 54.7zm131.6-5c3.6-2.4-2.2-10.4-37.3-7.8 32.3 13.8 37.3 7.8 37.3 7.8z"/></svg></span> <span>PDF</span> </a> <a href="https://arxiv.org/abs/2107.03374v2" onclick="captureOutboundLink('https://arxiv.org/abs/2107.03374v2'); return true;" class="badge badge-light "> <span class=" icon-wrapper icon-fa icon-fa-regular" data-name="file"><svg viewBox="0 0 384 513.795" xmlns="http://www.w3.org/2000/svg"><path d="M369.9 98.88c9 9 14.1 21.3 14.1 34v332.1c0 26.5-21.5 48-48 48H48c-26.5 0-48-21.5-48-48v-416c0-26.5 21.5-48 48-48.1h204.1c12.7 0 24.9 5.1 33.9 14.1zm-37.8 30.1L256 52.88v76.1h76.1zM48 464.98h288v-288H232c-13.3 0-24-10.7-24-24v-104H48v416z"/></svg></span> <span>Abstract</span> </a> </div> </div> </div> <div class="row"> <div class="col-md-7 paper-section-first" id="code"> <div class="paper-section-title"> <div class="row"> <div class="col-md-12"> <h2>Code <div class="float-right"> <div class="dropdown edit-button"> <button class="dropdown-toggle badge badge-edit" type="button" id="codeEditMenu" data-bs-toggle="dropdown" aria-haspopup="true" aria-expanded="false"> <span class=" icon-wrapper icon-fa icon-fa-solid" data-name="edit"><svg viewBox="0 0 576 514.999" xmlns="http://www.w3.org/2000/svg"><path d="M402.6 85.198l90.2 90.2c3.8 3.8 3.8 10 0 13.8l-218.399 218.4-92.8 10.3c-12.4 1.4-22.9-9.1-21.5-21.5l10.3-92.8 218.4-218.4c3.799-3.8 10-3.8 13.799 0zm162-22.9c15.2 15.2 15.2 39.9 0 55.2l-35.4 35.4c-3.8 3.8-10 3.8-13.8 0l-90.2-90.2c-3.8-3.8-3.8-10 0-13.8l35.4-35.4c15.3-15.2 40-15.2 55.2 0zM384 348.198c0-3.2 1.3-6.2 3.5-8.5l40-40c7.6-7.5 20.5-2.2 20.5 8.5v157.8c0 26.5-21.5 48-48 48H48c-26.5 0-48-21.5-48-48v-352c0-26.5 21.5-48 48-48h285.8c10.7 0 16.1 12.9 8.5 20.5l-40 40c-2.3 2.2-5.3 3.5-8.5 3.5H64v320h320v-101.8z"/></svg></span> Edit </button> <div class="dropdown-menu dropdown-menu-end" aria-labelledby="codeEditMenu"> <a class="dropdown-item" href="#loginModal" data-bs-toggle="modal"> <span class=" icon-wrapper icon-ion" data-name="add"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M256 112v288m144-144H112"/></svg></span> Add</a> <a class="dropdown-item" href="#loginModal" data-bs-toggle="modal"> <span class=" icon-wrapper icon-ion" data-name="remove"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M400 256H112"/></svg></span> Remove</a> <a class="dropdown-item" href="#loginModal" data-bs-toggle="modal"> <span class=" icon-wrapper icon-ion" data-name="checkmark-outline"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M416 128L192 384l-96-96"/></svg></span> Mark official</a> </div> </div> </div> </h2> <hr/> </div> </div> </div> <div class="paper-implementations code-table"> <div id="implementations-short-list"> <div class="row"> <div class="col-sm-7"> <div class="paper-impl-cell"> <a href="https://github.com/openai/human-eval" onclick="captureOutboundLink('https://github.com/openai/human-eval'); return true;" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> openai/human-eval <span class="badge badge-info is-official-code"><span class=" icon-wrapper icon-ion" data-name="checkmark-circle-outline"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M448 256c0-106-86-192-192-192S64 150 64 256s86 192 192 192 192-86 192-192z" fill="none" stroke="#000" stroke-miterlimit="10" stroke-width="32"/><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M352 176L217.6 336 160 272"/></svg></span> official</span> </a> </div> </div> <div class="col-3"> <div class="paper-impl-cell text-nowrap"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 2,424 </div> </div> <div class="col-2"> <div class="paper-impl-cell text-center"> </div> </div> </div> <div class="row"> <div class="col-sm-7"> <div class="paper-impl-cell"> <a href="https://github.com/THUDM/CodeGeeX" onclick="captureOutboundLink('https://github.com/THUDM/CodeGeeX'); return true;" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> THUDM/CodeGeeX </a> </div> </div> <div class="col-3"> <div class="paper-impl-cell text-nowrap"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 8,258 </div> </div> <div class="col-2"> <div class="paper-impl-cell text-center"> <img class="" src="https://production-assets.paperswithcode.com/perf/images/frameworks/mindspore-beada7dc.png" /> </div> </div> </div> <div class="row"> <div class="col-sm-7"> <div class="paper-impl-cell"> <a href="https://github.com/ncoop57/gpt-code-clippy" onclick="captureOutboundLink('https://github.com/ncoop57/gpt-code-clippy'); return true;" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> ncoop57/gpt-code-clippy </a> </div> </div> <div class="col-3"> <div class="paper-impl-cell text-nowrap"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 3,298 </div> </div> <div class="col-2"> <div class="paper-impl-cell text-center"> <img class="" src="https://production-assets.paperswithcode.com/perf/images/frameworks/jax-6ee30fa5.png" /> </div> </div> </div> <div class="row"> <div class="col-sm-7"> <div class="paper-impl-cell"> <a href="https://github.com/codedotal/gpt-code-clippy" onclick="captureOutboundLink('https://github.com/codedotal/gpt-code-clippy'); return true;" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> codedotal/gpt-code-clippy </a> </div> </div> <div class="col-3"> <div class="paper-impl-cell text-nowrap"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 3,298 </div> </div> <div class="col-2"> <div class="paper-impl-cell text-center"> <img class="" src="https://production-assets.paperswithcode.com/perf/images/frameworks/jax-6ee30fa5.png" /> </div> </div> </div> <div class="row"> <div class="col-sm-7"> <div class="paper-impl-cell"> <a href="https://github.com/vhellendoorn/code-lms" onclick="captureOutboundLink('https://github.com/vhellendoorn/code-lms'); return true;" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> vhellendoorn/code-lms </a> </div> </div> <div class="col-3"> <div class="paper-impl-cell text-nowrap"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 1,784 </div> </div> <div class="col-2"> <div class="paper-impl-cell text-center"> </div> </div> </div> <div class="table-options"> <a href="#" id="implementations-see-more-trigger">See all 13</a> implementations </div> </div> <div id="implementations-full-list" style="display:none"> <div class="row"> <div class="col-sm-7"> <div class="paper-impl-cell"> <a href="https://github.com/openai/human-eval" onclick="captureOutboundLink('https://github.com/openai/human-eval'); return true;" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> openai/human-eval <span class="badge badge-info is-official-code"><span class=" icon-wrapper icon-ion" data-name="checkmark-circle-outline"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M448 256c0-106-86-192-192-192S64 150 64 256s86 192 192 192 192-86 192-192z" fill="none" stroke="#000" stroke-miterlimit="10" stroke-width="32"/><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M352 176L217.6 336 160 272"/></svg></span> official</span> </a> </div> </div> <div class="col-3"> <div class="paper-impl-cell text-nowrap"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 2,424 </div> </div> <div class="col-2"> <div class="paper-impl-cell text-center"> </div> </div> </div> <div class="row"> <div class="col-sm-7"> <div class="paper-impl-cell"> <a href="https://github.com/THUDM/CodeGeeX" onclick="captureOutboundLink('https://github.com/THUDM/CodeGeeX'); return true;" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> THUDM/CodeGeeX </a> </div> </div> <div class="col-3"> <div class="paper-impl-cell text-nowrap"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 8,258 </div> </div> <div class="col-2"> <div class="paper-impl-cell text-center"> <img class="" src="https://production-assets.paperswithcode.com/perf/images/frameworks/mindspore-beada7dc.png" /> </div> </div> </div> <div class="row"> <div class="col-sm-7"> <div class="paper-impl-cell"> <a href="https://github.com/ncoop57/gpt-code-clippy" onclick="captureOutboundLink('https://github.com/ncoop57/gpt-code-clippy'); return true;" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> ncoop57/gpt-code-clippy </a> </div> </div> <div class="col-3"> <div class="paper-impl-cell text-nowrap"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 3,298 </div> </div> <div class="col-2"> <div class="paper-impl-cell text-center"> <img class="" src="https://production-assets.paperswithcode.com/perf/images/frameworks/jax-6ee30fa5.png" /> </div> </div> </div> <div class="row"> <div class="col-sm-7"> <div class="paper-impl-cell"> <a href="https://github.com/codedotal/gpt-code-clippy" onclick="captureOutboundLink('https://github.com/codedotal/gpt-code-clippy'); return true;" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> codedotal/gpt-code-clippy </a> </div> </div> <div class="col-3"> <div class="paper-impl-cell text-nowrap"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 3,298 </div> </div> <div class="col-2"> <div class="paper-impl-cell text-center"> <img class="" src="https://production-assets.paperswithcode.com/perf/images/frameworks/jax-6ee30fa5.png" /> </div> </div> </div> <div class="row"> <div class="col-sm-7"> <div class="paper-impl-cell"> <a href="https://github.com/vhellendoorn/code-lms" onclick="captureOutboundLink('https://github.com/vhellendoorn/code-lms'); return true;" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> vhellendoorn/code-lms </a> </div> </div> <div class="col-3"> <div class="paper-impl-cell text-nowrap"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 1,784 </div> </div> <div class="col-2"> <div class="paper-impl-cell text-center"> </div> </div> </div> <div class="row"> <div class="col-sm-7"> <div class="paper-impl-cell"> <a href="https://github.com/glouppe/info8010-deep-learning" onclick="captureOutboundLink('https://github.com/glouppe/info8010-deep-learning'); return true;" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> glouppe/info8010-deep-learning </a> </div> </div> <div class="col-3"> <div class="paper-impl-cell text-nowrap"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 1,222 </div> </div> <div class="col-2"> <div class="paper-impl-cell text-center"> <img class="" src="https://production-assets.paperswithcode.com/perf/images/frameworks/pytorch-2fbf2cb9.png" /> </div> </div> </div> <div class="row"> <div class="col-sm-7"> <div class="paper-impl-cell"> <a href="https://github.com/microsoft/PythonProgrammingPuzzles" onclick="captureOutboundLink('https://github.com/microsoft/PythonProgrammingPuzzles'); return true;" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> microsoft/PythonProgrammingPuzzles </a> </div> </div> <div class="col-3"> <div class="paper-impl-cell text-nowrap"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 968 </div> </div> <div class="col-2"> <div class="paper-impl-cell text-center"> </div> </div> </div> <div class="row"> <div class="col-sm-7"> <div class="paper-impl-cell"> <a href="https://github.com/fsoft-ai4code/codecapybara" onclick="captureOutboundLink('https://github.com/fsoft-ai4code/codecapybara'); return true;" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> fsoft-ai4code/codecapybara </a> </div> </div> <div class="col-3"> <div class="paper-impl-cell text-nowrap"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 168 </div> </div> <div class="col-2"> <div class="paper-impl-cell text-center"> <img class="" src="https://production-assets.paperswithcode.com/perf/images/frameworks/pytorch-2fbf2cb9.png" /> </div> </div> </div> <div class="row"> <div class="col-sm-7"> <div class="paper-impl-cell"> <a href="https://github.com/my-other-github-account/llm-humaneval-benchmarks" onclick="captureOutboundLink('https://github.com/my-other-github-account/llm-humaneval-benchmarks'); return true;" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> my-other-github-account/llm-humanev… </a> </div> </div> <div class="col-3"> <div class="paper-impl-cell text-nowrap"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 86 </div> </div> <div class="col-2"> <div class="paper-impl-cell text-center"> </div> </div> </div> <div class="row"> <div class="col-sm-7"> <div class="paper-impl-cell"> <a href="https://github.com/codefuse-ai/codefuse-evaluation" onclick="captureOutboundLink('https://github.com/codefuse-ai/codefuse-evaluation'); return true;" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> codefuse-ai/codefuse-evaluation </a> </div> </div> <div class="col-3"> <div class="paper-impl-cell text-nowrap"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 80 </div> </div> <div class="col-2"> <div class="paper-impl-cell text-center"> <img class="" src="https://production-assets.paperswithcode.com/perf/images/frameworks/pytorch-2fbf2cb9.png" /> </div> </div> </div> <div class="row"> <div class="col-sm-7"> <div class="paper-impl-cell"> <a href="https://github.com/superli3/codenavi" onclick="captureOutboundLink('https://github.com/superli3/codenavi'); return true;" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> superli3/codenavi </a> </div> </div> <div class="col-3"> <div class="paper-impl-cell text-nowrap"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 1 </div> </div> <div class="col-2"> <div class="paper-impl-cell text-center"> <img class="" src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAbYAAABiBAMAAADO/+vOAAAACXBIWXMAAA7EAAAOxAGVKw4bAAAAMFBMVEVHcExBUGZCUGZKUmNCUGZCUGZCUGb/igD/jwD/hwD/iwBCUGb/kAD/hAD/mwD/eABjkbvFAAAAC3RSTlMAxZ4pVHvkPbaF3mTG43EAAAjMSURBVHja7Zy/b9tGFMdPImlD8CLXyaRFdRIvWhQrCVJosRMnixblh7NwcWNXixYDXRwvCZChQNGm3rq06RhkSGLJNhf9A2khc81iQWtaCJqDAILK+/2LEk+2AsQEr4BDU7w7fu579967d1YBSEpSkpKUpCSFlh/BdlzRbn4AP+/FEy3T+QB++SeWys2+gWwn74uxRDsJ2Nrtv2OHZv3a6RxD3U7iB/dnp0N0a7dfxQtttwML1A3C/RQvtEEwJZFuEK73LD5oO92BpFu7Fxs3lxkOoG4drpv/X0zcXOZdp9uFsh1z3fyPsYCbfTccIjRBt16v/zEOaL8Nu12sG19vvu/3/z3/bH8Nh4PBQLGTgW5+/1UM0LpDIpusW79/zt3c7jAoAdqA2cljplu/f67d3A5EGxDdTlTd+v1z7OYywyGBU/wbLJBNdnOLeslP/6WWlS4ASM/Pz184FdqAognrrY3IgiK6Oaull/3ps1WULgBIBT+PJmzlKZYN+gDZTgZTskfono1nO/ja2QIfIPk3n+sWxfa169btDvGcZHbS96kxiYNuEE6xk34sdBsMBgMpnvTbvX5MdOsOFDvZY1NSct9ruNwNuvLI9cqXYTta4+WMun1+ItvJ9ndhdpIW2FXjyzndijYdzqTbXqYj20mwE2InzyXbc5CR4xI/CDTjwfYKpsvleDL48GUc2F7D6Ksj7QN8aBVfGLLd+XZ+8Tb9pVQqIpN6f3FdNDPXlN+DJ+6sPywqDV9fvFKMZrPX19eKZmyfUGQp7996aEP+1oTNLiBDfZH8Cn8rguU6vHeJcSxJz8CyCm94l8lvaYRzNfhZjWKz7qGauG27Vqs9xvdnarUtfHU3uFlFbJ+2AdVNsJM4lDZgs3LEC20JbA65V1We4WP/SK6F2JyWAZtVIDUXqMf12ExuiMMbsH3Gcb5mJxlcBNtd5mLxbgdirFCWJn7GZc8skFpXlTuQ7dAlrYxl422hgasjDkqEH6mjq4CNbD4VO9kj83knis3h4UOTdXGP3VtRnvFwu7Z6B7I1WwZsaV6zQbtbYUQrVMsGZHtOd3KKnXxNUw4RbK4QG1VpZ7xsqM9saFHVgfzKEWxi81nSUp4NF7sKHn7KMlmqbiwN9HIsGxGgzt9SZjuiUah3ZT3HxJXiUiScKZsj1oRtzdHxctjIYbsEnvAduJLnalO9ZvfGsd2ALf5QBNfr9BZeahdLJWzOSF/eCrUCkGQGTanbpVts+E3ZkODN26VVOuNT9NkZNropNj1E3eT85Og0EGcr0Ks0XV0FZv0qBAX2usmeqZJJiuRCJuWQsXlX1qATHMOGhrBIB3UDywUbAGVmh+eYXRuhW7APGH3awdgsOu6YKUv+xfbBIbgV5gxy+H2UWlTblleNiksc2R4f4baabFE3aPUVTTc5P9n/WIxiS3PTVyYvVOAvRmRymZl28V7W4U4pRfgR22bo/u1AYJvhRuwG3tlBJYW1UARifyPsZBvCbUexzXG/kyaTI8dnBLnkfRH+GcETEwnZlB7PJihqkxo54ta4qc6pwW5Gz08asFV4UsEmk6PA2cg0dXmAAiSJAZc5LUZxo9kKbDLjpZdlM9DmbkGLq/X11h9tTVKi58xyBRoym8t73xSru8IbVrDcaR7EjGOrC8Pk4sVbxreCBrwCGmiLWpcxdtJEt5zQF1lEOtucFDYrtcisNmOzxJkrTPAs+ueogqAcLUGl+TeYL4nUDfb1oIQLWds5jQ3b92+46aoLb5jCUGZsdovFjHhUiOL7iPRgjjWVBRF20o/UTct7heoGcNTC6VqCISMLTWNrzNOSldk8xS/YGD/oayONPp1Rl7dmJ32/F62bxlYMZbtBP77EbSNlI/5AYwv1b47K1mSmIxi/vI3aLasuQMuXoLxyFJttxmbV6ecXGBsQ2FqTsDFrSqsgk2+haY4Uqwj8IHT/htKT26dg09ebGOFuAmXVnJ3NhSwOaiUH+ysILYXaSf+PKeoGcChNw9sp6UbnJJqDKXTtwno51QVodtJ64Ruut3mhjGID9hJhOwxh807NRuOw6hxquQx/6nl8NS4Bsy98MzYlaR7OBsA1TOeF2EljNslO0sgNVs1XUA2onq3ucMLyk7NvzfybwpYbwUYnZlX3bw1TNivEvyHgjQLqCerqaC4gLD+ZiWarq1ul0brR5blhFpeE73FC4hIEfFBHLcLLVX24w/KTN43iyX1jNhpam8ST4WxqPLlPBrhJZnlwuSRIG24n22NTtozN1XPZGttMrVbb4PUO5Hcno2PI5gqLKUdHqCBnvfR0fnh+MoqtzPemANwKX29z/E3TPLxt6vs3A7Yyj/Ftllmr8LRuRcnwhsclZmzplrjJyYfqlpL3r2SzRgxlWth3G7CllB07W7JiQkg7yg05xzFgs1piprIRyubIeQeywyI7OlfIlxiw2XKmpckpefpOcwGn1E3Iaq0KuSDZljASm5rVAsnqgWUxz2XAhmY8ynMtc4GclpR21Sx3qJ00YMMh/uP1O0stPV/i8pxC63LgvXPUhKOp4z1U8pMmbGWSn7xV5xAWz86DVojHPaWdVHY5+VDdUmLueUxe2YRNyis3hL2ux41nqzgVOynvkZvh/o1vcdgyE2sdggnYpJT8puBHhBMjD0xnveE1JB5Sjdubhp3jtIoTsYnnOEXZ/9M52wRTsZPSSdrWyHiyoJzRibgLYCI2QfK8mBPc5w8ejtHteBLdyOEuPPEYHXPJZ53S2dYWmJCNtXVB+jTLt0z74CzrDf6pJj+7vp4jhhCV+4uLi8RQrQaP5YUz6u8fCCf5+PyR1nLEFmG9Ba1DymI9Es67ceUgqiNdWsFlHoy3k/5Ef+5wbS3y7wastbUSiPw7BbNiT1hT3XeDGJWJ7OS5Y5PsZC/GusVqTqKvZPL1Fq8vZaKv0lLdXoN4ldk3TLf4fQk6g7/ffdx+vw3iB4d1i+f/dGAHsrX3QCzLbsD2DMS0/B78l5SkJCUpSUlKUpKSlGmW/wErOKTxUOS81gAAAABJRU5ErkJggg==" /> </div> </div> </div> <div class="row"> <div class="col-sm-7"> <div class="paper-impl-cell"> <a href="https://github.com/superli3/CYRMPR" onclick="captureOutboundLink('https://github.com/superli3/CYRMPR'); return true;" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> superli3/CYRMPR </a> </div> </div> <div class="col-3"> <div class="paper-impl-cell text-nowrap"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 1 </div> </div> <div class="col-2"> <div class="paper-impl-cell text-center"> <img class="" src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAbYAAABiBAMAAADO/+vOAAAACXBIWXMAAA7EAAAOxAGVKw4bAAAAMFBMVEVHcExBUGZCUGZKUmNCUGZCUGZCUGb/igD/jwD/hwD/iwBCUGb/kAD/hAD/mwD/eABjkbvFAAAAC3RSTlMAxZ4pVHvkPbaF3mTG43EAAAjMSURBVHja7Zy/b9tGFMdPImlD8CLXyaRFdRIvWhQrCVJosRMnixblh7NwcWNXixYDXRwvCZChQNGm3rq06RhkSGLJNhf9A2khc81iQWtaCJqDAILK+/2LEk+2AsQEr4BDU7w7fu579967d1YBSEpSkpKUpCSFlh/BdlzRbn4AP+/FEy3T+QB++SeWys2+gWwn74uxRDsJ2Nrtv2OHZv3a6RxD3U7iB/dnp0N0a7dfxQtttwML1A3C/RQvtEEwJZFuEK73LD5oO92BpFu7Fxs3lxkOoG4drpv/X0zcXOZdp9uFsh1z3fyPsYCbfTccIjRBt16v/zEOaL8Nu12sG19vvu/3/z3/bH8Nh4PBQLGTgW5+/1UM0LpDIpusW79/zt3c7jAoAdqA2cljplu/f67d3A5EGxDdTlTd+v1z7OYywyGBU/wbLJBNdnOLeslP/6WWlS4ASM/Pz184FdqAognrrY3IgiK6Oaull/3ps1WULgBIBT+PJmzlKZYN+gDZTgZTskfono1nO/ja2QIfIPk3n+sWxfa169btDvGcZHbS96kxiYNuEE6xk34sdBsMBgMpnvTbvX5MdOsOFDvZY1NSct9ruNwNuvLI9cqXYTta4+WMun1+ItvJ9ndhdpIW2FXjyzndijYdzqTbXqYj20mwE2InzyXbc5CR4xI/CDTjwfYKpsvleDL48GUc2F7D6Ksj7QN8aBVfGLLd+XZ+8Tb9pVQqIpN6f3FdNDPXlN+DJ+6sPywqDV9fvFKMZrPX19eKZmyfUGQp7996aEP+1oTNLiBDfZH8Cn8rguU6vHeJcSxJz8CyCm94l8lvaYRzNfhZjWKz7qGauG27Vqs9xvdnarUtfHU3uFlFbJ+2AdVNsJM4lDZgs3LEC20JbA65V1We4WP/SK6F2JyWAZtVIDUXqMf12ExuiMMbsH3Gcb5mJxlcBNtd5mLxbgdirFCWJn7GZc8skFpXlTuQ7dAlrYxl422hgasjDkqEH6mjq4CNbD4VO9kj83knis3h4UOTdXGP3VtRnvFwu7Z6B7I1WwZsaV6zQbtbYUQrVMsGZHtOd3KKnXxNUw4RbK4QG1VpZ7xsqM9saFHVgfzKEWxi81nSUp4NF7sKHn7KMlmqbiwN9HIsGxGgzt9SZjuiUah3ZT3HxJXiUiScKZsj1oRtzdHxctjIYbsEnvAduJLnalO9ZvfGsd2ALf5QBNfr9BZeahdLJWzOSF/eCrUCkGQGTanbpVts+E3ZkODN26VVOuNT9NkZNropNj1E3eT85Og0EGcr0Ks0XV0FZv0qBAX2usmeqZJJiuRCJuWQsXlX1qATHMOGhrBIB3UDywUbAGVmh+eYXRuhW7APGH3awdgsOu6YKUv+xfbBIbgV5gxy+H2UWlTblleNiksc2R4f4baabFE3aPUVTTc5P9n/WIxiS3PTVyYvVOAvRmRymZl28V7W4U4pRfgR22bo/u1AYJvhRuwG3tlBJYW1UARifyPsZBvCbUexzXG/kyaTI8dnBLnkfRH+GcETEwnZlB7PJihqkxo54ta4qc6pwW5Gz08asFV4UsEmk6PA2cg0dXmAAiSJAZc5LUZxo9kKbDLjpZdlM9DmbkGLq/X11h9tTVKi58xyBRoym8t73xSru8IbVrDcaR7EjGOrC8Pk4sVbxreCBrwCGmiLWpcxdtJEt5zQF1lEOtucFDYrtcisNmOzxJkrTPAs+ueogqAcLUGl+TeYL4nUDfb1oIQLWds5jQ3b92+46aoLb5jCUGZsdovFjHhUiOL7iPRgjjWVBRF20o/UTct7heoGcNTC6VqCISMLTWNrzNOSldk8xS/YGD/oayONPp1Rl7dmJ32/F62bxlYMZbtBP77EbSNlI/5AYwv1b47K1mSmIxi/vI3aLasuQMuXoLxyFJttxmbV6ecXGBsQ2FqTsDFrSqsgk2+haY4Uqwj8IHT/htKT26dg09ebGOFuAmXVnJ3NhSwOaiUH+ysILYXaSf+PKeoGcChNw9sp6UbnJJqDKXTtwno51QVodtJ64Ruut3mhjGID9hJhOwxh807NRuOw6hxquQx/6nl8NS4Bsy98MzYlaR7OBsA1TOeF2EljNslO0sgNVs1XUA2onq3ucMLyk7NvzfybwpYbwUYnZlX3bw1TNivEvyHgjQLqCerqaC4gLD+ZiWarq1ul0brR5blhFpeE73FC4hIEfFBHLcLLVX24w/KTN43iyX1jNhpam8ST4WxqPLlPBrhJZnlwuSRIG24n22NTtozN1XPZGttMrVbb4PUO5Hcno2PI5gqLKUdHqCBnvfR0fnh+MoqtzPemANwKX29z/E3TPLxt6vs3A7Yyj/Ftllmr8LRuRcnwhsclZmzplrjJyYfqlpL3r2SzRgxlWth3G7CllB07W7JiQkg7yg05xzFgs1piprIRyubIeQeywyI7OlfIlxiw2XKmpckpefpOcwGn1E3Iaq0KuSDZljASm5rVAsnqgWUxz2XAhmY8ynMtc4GclpR21Sx3qJ00YMMh/uP1O0stPV/i8pxC63LgvXPUhKOp4z1U8pMmbGWSn7xV5xAWz86DVojHPaWdVHY5+VDdUmLueUxe2YRNyis3hL2ux41nqzgVOynvkZvh/o1vcdgyE2sdggnYpJT8puBHhBMjD0xnveE1JB5Sjdubhp3jtIoTsYnnOEXZ/9M52wRTsZPSSdrWyHiyoJzRibgLYCI2QfK8mBPc5w8ejtHteBLdyOEuPPEYHXPJZ53S2dYWmJCNtXVB+jTLt0z74CzrDf6pJj+7vp4jhhCV+4uLi8RQrQaP5YUz6u8fCCf5+PyR1nLEFmG9Ba1DymI9Es67ceUgqiNdWsFlHoy3k/5Ef+5wbS3y7wastbUSiPw7BbNiT1hT3XeDGJWJ7OS5Y5PsZC/GusVqTqKvZPL1Fq8vZaKv0lLdXoN4ldk3TLf4fQk6g7/ffdx+vw3iB4d1i+f/dGAHsrX3QCzLbsD2DMS0/B78l5SkJCUpSUlKUpKSlGmW/wErOKTxUOS81gAAAABJRU5ErkJggg==" /> </div> </div> </div> <div class="row"> <div class="col-sm-7"> <div class="paper-impl-cell"> <a href="https://github.com/2796gaurav/human-eval" onclick="captureOutboundLink('https://github.com/2796gaurav/human-eval'); return true;" class="code-table-link"> <span class=" icon-wrapper icon-ion" data-name="logo-github"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M256 32C132.3 32 32 134.9 32 261.7c0 101.5 64.2 187.5 153.2 217.9a17.56 17.56 0 0 0 3.8.4c8.3 0 11.5-6.1 11.5-11.4 0-5.5-.2-19.9-.3-39.1a102.4 102.4 0 0 1-22.6 2.7c-43.1 0-52.9-33.5-52.9-33.5-10.2-26.5-24.9-33.6-24.9-33.6-19.5-13.7-.1-14.1 1.4-14.1h.1c22.5 2 34.3 23.8 34.3 23.8 11.2 19.6 26.2 25.1 39.6 25.1a63 63 0 0 0 25.6-6c2-14.8 7.8-24.9 14.2-30.7-49.7-5.8-102-25.5-102-113.5 0-25.1 8.7-45.6 23-61.6-2.3-5.8-10-29.2 2.2-60.8a18.64 18.64 0 0 1 5-.5c8.1 0 26.4 3.1 56.6 24.1a208.21 208.21 0 0 1 112.2 0c30.2-21 48.5-24.1 56.6-24.1a18.64 18.64 0 0 1 5 .5c12.2 31.6 4.5 55 2.2 60.8 14.3 16.1 23 36.6 23 61.6 0 88.2-52.4 107.6-102.3 113.3 8 7.1 15.2 21.1 15.2 42.5 0 30.7-.3 55.5-.3 63 0 5.4 3.1 11.5 11.4 11.5a19.35 19.35 0 0 0 4-.4C415.9 449.2 480 363.1 480 261.7 480 134.9 379.7 32 256 32z"/></svg></span> 2796gaurav/human-eval </a> </div> </div> <div class="col-3"> <div class="paper-impl-cell text-nowrap"> <span class=" icon-wrapper icon-ion" data-name="star"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M394 480a16 16 0 0 1-9.39-3L256 383.76 127.39 477a16 16 0 0 1-24.55-18.08L153 310.35 23 221.2a16 16 0 0 1 9-29.2h160.38l48.4-148.95a16 16 0 0 1 30.44 0l48.4 149H480a16 16 0 0 1 9.05 29.2L359 310.35l50.13 148.53A16 16 0 0 1 394 480z"/></svg></span> 1 </div> </div> <div class="col-2"> <div class="paper-impl-cell text-center"> </div> </div> </div> <div class="table-options"> <a href="#" id="implementations-see-less-trigger">Collapse 13</a> implementations </div> </div> </div> </div> <div class="col-md-5 paper-section" id="tasks"> <div class="paper-section-title"> <div class="row"> <div class="col-md-12"> <h2>Tasks <div class="float-right"> <div class="dropdown edit-button"> <button class="dropdown-toggle badge badge-edit" type="button" id="taskEditMenu" data-bs-toggle="dropdown" aria-haspopup="true" aria-expanded="false"> <span class=" icon-wrapper icon-fa icon-fa-solid" data-name="edit"><svg viewBox="0 0 576 514.999" xmlns="http://www.w3.org/2000/svg"><path d="M402.6 85.198l90.2 90.2c3.8 3.8 3.8 10 0 13.8l-218.399 218.4-92.8 10.3c-12.4 1.4-22.9-9.1-21.5-21.5l10.3-92.8 218.4-218.4c3.799-3.8 10-3.8 13.799 0zm162-22.9c15.2 15.2 15.2 39.9 0 55.2l-35.4 35.4c-3.8 3.8-10 3.8-13.8 0l-90.2-90.2c-3.8-3.8-3.8-10 0-13.8l35.4-35.4c15.3-15.2 40-15.2 55.2 0zM384 348.198c0-3.2 1.3-6.2 3.5-8.5l40-40c7.6-7.5 20.5-2.2 20.5 8.5v157.8c0 26.5-21.5 48-48 48H48c-26.5 0-48-21.5-48-48v-352c0-26.5 21.5-48 48-48h285.8c10.7 0 16.1 12.9 8.5 20.5l-40 40c-2.3 2.2-5.3 3.5-8.5 3.5H64v320h320v-101.8z"/></svg></span> Edit </button> <div class="dropdown-menu dropdown-menu-end" aria-labelledby="taskEditMenu"> <a class="dropdown-item" href="#loginModal" data-bs-toggle="modal"> <span class=" icon-wrapper icon-ion" data-name="add"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M256 112v288m144-144H112"/></svg></span> Add</a> <a class="dropdown-item" href="#loginModal" data-bs-toggle="modal"> <span class=" icon-wrapper icon-ion" data-name="remove"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M400 256H112"/></svg></span> Remove</a> </div> </div> </div> </h2> <hr/> </div> </div> <div class="paper-tasks"> <div class="row"> <div class="col-md-12"> <a href="/task/code-generation"> <span class="badge badge-primary"> <img src="https://production-media.paperswithcode.com/thumbnails/task/658fcf6a-52ca-4865-b111-28bf61c346dc.jpg"> <span>Code Generation</span> </span> </a> <a href="/task/humaneval"> <span class="badge badge-primary"> <img src="https://production-media.paperswithcode.com/tasks/default.gif"> <span>HumanEval</span> </span> </a> <a href="/task/language-modelling"> <span class="badge badge-primary"> <img src="https://production-media.paperswithcode.com/thumbnails/task/task-0000000267-8df06634.jpg"> <span>Language Modelling</span> </span> </a> <a href="/task/multi-task-language-understanding"> <span class="badge badge-primary"> <img src="https://production-media.paperswithcode.com/thumbnails/task/93c61892-0ee4-4896-8201-6f7497f6058e.jpg"> <span>Multi-task Language Understanding</span> </span> </a> </div> </div> </div> </div> </div> </div> <div class="row"> <div class="col-md-12 paper-section paper-evaluation-section-title" id="datasets"> <div class="paper-section-title"> <div class="row"> <div class="col-md-12 zero-padding-datasets"> <h2>Datasets <div class="float-right"> <div class="dropdown edit-button"> <button class="dropdown-toggle badge badge-edit" type="button" id="datasetEditMenu" data-bs-toggle="modal" data-bs-target="#loginModal" aria-haspopup="true" aria-expanded="false"> <span class=" icon-wrapper icon-fa icon-fa-solid" data-name="edit"><svg viewBox="0 0 576 514.999" xmlns="http://www.w3.org/2000/svg"><path d="M402.6 85.198l90.2 90.2c3.8 3.8 3.8 10 0 13.8l-218.399 218.4-92.8 10.3c-12.4 1.4-22.9-9.1-21.5-21.5l10.3-92.8 218.4-218.4c3.799-3.8 10-3.8 13.799 0zm162-22.9c15.2 15.2 15.2 39.9 0 55.2l-35.4 35.4c-3.8 3.8-10 3.8-13.8 0l-90.2-90.2c-3.8-3.8-3.8-10 0-13.8l35.4-35.4c15.3-15.2 40-15.2 55.2 0zM384 348.198c0-3.2 1.3-6.2 3.5-8.5l40-40c7.6-7.5 20.5-2.2 20.5 8.5v157.8c0 26.5-21.5 48-48 48H48c-26.5 0-48-21.5-48-48v-352c0-26.5 21.5-48 48-48h285.8c10.7 0 16.1 12.9 8.5 20.5l-40 40c-2.3 2.2-5.3 3.5-8.5 3.5H64v320h320v-101.8z"/></svg></span> Edit </button> </div> </div> </h2> <hr/> </div> </div> <div class="paper-datasets"> <div class="row"> <div class="col-md-12"> <p><b>Introduced in the Paper:</b></p> <span class="badge badge-primary"> <a href="/dataset/humaneval"> <img class="dataset-list-image" src="https://production-media.paperswithcode.com/thumbnails/dataset-small/fb954d4f-19de-4767-8b70-d9da28333bb5.jpg"> HumanEval </a> </span> <p><b>Used in the Paper:</b></p> <span class="badge badge-primary"> <a href="/dataset/big-bench"> <img class="dataset-list-image" src="https://production-media.paperswithcode.com/thumbnails/dataset-small/bb186a0b-c9de-4e96-bf6c-9beefc59f138.jpg"> BIG-bench </a> </span> <span class="badge badge-primary"> <a href="/dataset/bbh"> <span class="icon-list-image" style="opacity:0.4;color:#A59F78"><span class=" icon-wrapper icon-fa icon-fa-solid" data-name="file"><svg viewBox="0 0 384 514.999" xmlns="http://www.w3.org/2000/svg"><path d="M224 137.998c0 13.2 10.8 24 24 24h136v328c0 13.3-10.7 24-24 24H24c-13.3 0-24-10.7-24-24v-464c0-13.3 10.7-24 24-24h200v136zm160-14.1v6.1H256v-128h6.1c6.4 0 12.5 2.5 17 7l97.9 98c4.5 4.5 7 10.6 7 16.9z"/></svg></span></span> BBH </a> </span> <span class="badge badge-primary"> <a href="/dataset/apps"> <img class="dataset-list-image" src="https://production-media.paperswithcode.com/thumbnails/dataset-small/dataset-0000007444-2dbde32e.jpg"> APPS </a> </span> </div> </div> </div> </div> </div> </div>  <div class="row"> <div id="results" class="col-md-12 paper-evaluation-section-title"> <div class="paper-section-title"> <div class="row"> <div class="col-md-12 zero-padding"> <h2>Results from the Paper <div class="float-right"> <div class="edit-button"> <a class="dropdown-toggle badge badge-edit" id="evalEditMenu" href="/paper/evaluating-large-language-models-trained-on/review/"> <span class=" icon-wrapper icon-fa icon-fa-solid" data-name="edit"><svg viewBox="0 0 576 514.999" xmlns="http://www.w3.org/2000/svg"><path d="M402.6 85.198l90.2 90.2c3.8 3.8 3.8 10 0 13.8l-218.399 218.4-92.8 10.3c-12.4 1.4-22.9-9.1-21.5-21.5l10.3-92.8 218.4-218.4c3.799-3.8 10-3.8 13.799 0zm162-22.9c15.2 15.2 15.2 39.9 0 55.2l-35.4 35.4c-3.8 3.8-10 3.8-13.8 0l-90.2-90.2c-3.8-3.8-3.8-10 0-13.8l35.4-35.4c15.3-15.2 40-15.2 55.2 0zM384 348.198c0-3.2 1.3-6.2 3.5-8.5l40-40c7.6-7.5 20.5-2.2 20.5 8.5v157.8c0 26.5-21.5 48-48 48H48c-26.5 0-48-21.5-48-48v-352c0-26.5 21.5-48 48-48h285.8c10.7 0 16.1 12.9 8.5 20.5l-40 40c-2.3 2.2-5.3 3.5-8.5 3.5H64v320h320v-101.8z"/></svg></span> Edit </a> </div> </div> </h2> <hr/> <div class="paper-evaluation-badge"> <div class="sota"> <p> <a href="/sota/multi-task-language-understanding-on-bbh-alg"> <img style="height:20px;width:35px;position:relative;top:1px;" src="https://production-media.paperswithcode.com/sota-thumbs/multi-task-language-understanding-on-bbh-alg-small_1ab7fd59.png"/> </a>  Ranked #1 on <a class="sota-task" href="/sota/multi-task-language-understanding-on-bbh-alg"> Multi-task Language Understanding on BBH-alg </a> </p> </div>      <a href="#" class="get-badge-button float-right" data-bs-toggle="modal" data-bs-target="#badgeModal"> <span class=" icon-wrapper icon-fa icon-fa-light" data-name="arrow-to-right"><svg viewBox="0 0 448 520.146" xmlns="http://www.w3.org/2000/svg"><path d="M215 100.5c4.7-4.7 12.3-4.7 17 0l148.6 148c4.7 4.7 4.7 12.3 0 17l-148.5 148c-4.7 4.7-12.3 4.7-17 0l-7.1-7.1c-4.7-4.7-4.7-12.3 0-17L323.9 274H12c-6.6 0-12-5.4-12-12v-10c0-6.6 5.4-12 12-12h311.9l-116-115.4a12.01 12.01 0 0 1 0-17zM448 77v360c0 6.6-5.4 12-12 12h-8c-6.6 0-12-5.4-12-12V77c0-6.6 5.4-12 12-12h8c6.6 0 12 5.4 12 12z"/></svg></span> Get a GitHub badge </a> </div> </div> </div> </div> </div> </div> <div class="paper-evaluation-section" id="evaluation"> <div class="row"> <div class="col-md-12"> <div class="sota-table table-responsive"> <table class="table-striped"> <tr> <th>Task</th> <th>Dataset</th> <th>Model</th> <th>Metric Name</th> <th>Metric Value</th> <th>Global Rank</th> <th>Result</th> <th>Benchmark</th> </tr> <tr> <td rowspan="12" class="rowspan-td"> Code Generation </td> <td rowspan="12" class="rowspan-td"> APPS </td> <td rowspan="12" class="rowspan-td model-col"> Codex 12B (Raw) </td> <td style="vertical-align: top; padding-top: 18px;"> Introductory Pass@1 </td> <td> 4.14% </td> <td> # 9 </td> <td class="results-icon"> <a href="/paper/evaluating-large-language-models-trained-on/review/?hl=36732"> <span class=" icon-wrapper icon-ion" data-name="enter-outline"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M176 176v-40a40 40 0 0 1 40-40h208a40 40 0 0 1 40 40v240a40 40 0 0 1-40 40H216a40 40 0 0 1-40-40v-40" fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32"/><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M272 336l80-80-80-80M48 256h288"/></svg></span> </a> </td> <td> <div class="sota-table-link"> <a class="btn btn-primary" href="/sota/code-generation-on-apps" style="text-decoration: underline"> Compare</a> </div> </td> </tr> <tr> <td style="vertical-align: top; padding-top: 18px;"> Interview Pass@1 </td> <td> 0.14% </td> <td> # 11 </td> <td class="results-icon"> <a href="/paper/evaluating-large-language-models-trained-on/review/?hl=36732"> <span class=" icon-wrapper icon-ion" data-name="enter-outline"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M176 176v-40a40 40 0 0 1 40-40h208a40 40 0 0 1 40 40v240a40 40 0 0 1-40 40H216a40 40 0 0 1-40-40v-40" fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32"/><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M272 336l80-80-80-80M48 256h288"/></svg></span> </a> </td> <td> <div class="sota-table-link"> <a class="btn btn-primary" href="/sota/code-generation-on-apps" style="text-decoration: underline"> Compare</a> </div> </td> </tr> <tr> <td style="vertical-align: top; padding-top: 18px;"> Competition Pass@1 </td> <td> 0.02% </td> <td> # 8 </td> <td class="results-icon"> <a href="/paper/evaluating-large-language-models-trained-on/review/?hl=36732"> <span class=" icon-wrapper icon-ion" data-name="enter-outline"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M176 176v-40a40 40 0 0 1 40-40h208a40 40 0 0 1 40 40v240a40 40 0 0 1-40 40H216a40 40 0 0 1-40-40v-40" fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32"/><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M272 336l80-80-80-80M48 256h288"/></svg></span> </a> </td> <td> <div class="sota-table-link"> <a class="btn btn-primary" href="/sota/code-generation-on-apps" style="text-decoration: underline"> Compare</a> </div> </td> </tr> <tr> <td style="vertical-align: top; padding-top: 18px;"> Introductory Pass@1000 </td> <td> 25.02% </td> <td> # 4 </td> <td class="results-icon"> <a href="/paper/evaluating-large-language-models-trained-on/review/?hl=36732"> <span class=" icon-wrapper icon-ion" data-name="enter-outline"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M176 176v-40a40 40 0 0 1 40-40h208a40 40 0 0 1 40 40v240a40 40 0 0 1-40 40H216a40 40 0 0 1-40-40v-40" fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32"/><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M272 336l80-80-80-80M48 256h288"/></svg></span> </a> </td> <td> <div class="sota-table-link"> <a class="btn btn-primary" href="/sota/code-generation-on-apps" style="text-decoration: underline"> Compare</a> </div> </td> </tr> <tr> <td style="vertical-align: top; padding-top: 18px;"> Interview Pass@1000 </td> <td> 3.70% </td> <td> # 6 </td> <td class="results-icon"> <a href="/paper/evaluating-large-language-models-trained-on/review/?hl=36732"> <span class=" icon-wrapper icon-ion" data-name="enter-outline"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M176 176v-40a40 40 0 0 1 40-40h208a40 40 0 0 1 40 40v240a40 40 0 0 1-40 40H216a40 40 0 0 1-40-40v-40" fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32"/><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M272 336l80-80-80-80M48 256h288"/></svg></span> </a> </td> <td> <div class="sota-table-link"> <a class="btn btn-primary" href="/sota/code-generation-on-apps" style="text-decoration: underline"> Compare</a> </div> </td> </tr> <tr> <td style="vertical-align: top; padding-top: 18px;"> Competition Pass@1000 </td> <td> 3.23% </td> <td> # 6 </td> <td class="results-icon"> <a href="/paper/evaluating-large-language-models-trained-on/review/?hl=36732"> <span class=" icon-wrapper icon-ion" data-name="enter-outline"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M176 176v-40a40 40 0 0 1 40-40h208a40 40 0 0 1 40 40v240a40 40 0 0 1-40 40H216a40 40 0 0 1-40-40v-40" fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32"/><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M272 336l80-80-80-80M48 256h288"/></svg></span> </a> </td> <td> <div class="sota-table-link"> <a class="btn btn-primary" href="/sota/code-generation-on-apps" style="text-decoration: underline"> Compare</a> </div> </td> </tr> <tr> <td style="vertical-align: top; padding-top: 18px;"> Competition Pass@5 </td> <td> 0.09% </td> <td> # 4 </td> <td class="results-icon"> <a href="/paper/evaluating-large-language-models-trained-on/review/?hl=36732"> <span class=" icon-wrapper icon-ion" data-name="enter-outline"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M176 176v-40a40 40 0 0 1 40-40h208a40 40 0 0 1 40 40v240a40 40 0 0 1-40 40H216a40 40 0 0 1-40-40v-40" fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32"/><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M272 336l80-80-80-80M48 256h288"/></svg></span> </a> </td> <td> <div class="sota-table-link"> <a class="btn btn-primary" href="/sota/code-generation-on-apps" style="text-decoration: underline"> Compare</a> </div> </td> </tr> <tr> <td style="vertical-align: top; padding-top: 18px;"> Interview Pass@5 </td> <td> 0.51% </td> <td> # 7 </td> <td class="results-icon"> <a href="/paper/evaluating-large-language-models-trained-on/review/?hl=36732"> <span class=" icon-wrapper icon-ion" data-name="enter-outline"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M176 176v-40a40 40 0 0 1 40-40h208a40 40 0 0 1 40 40v240a40 40 0 0 1-40 40H216a40 40 0 0 1-40-40v-40" fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32"/><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M272 336l80-80-80-80M48 256h288"/></svg></span> </a> </td> <td> <div class="sota-table-link"> <a class="btn btn-primary" href="/sota/code-generation-on-apps" style="text-decoration: underline"> Compare</a> </div> </td> </tr> <tr> <td style="vertical-align: top; padding-top: 18px;"> Introductory Pass@5 </td> <td> 9.65% </td> <td> # 3 </td> <td class="results-icon"> <a href="/paper/evaluating-large-language-models-trained-on/review/?hl=36732"> <span class=" icon-wrapper icon-ion" data-name="enter-outline"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M176 176v-40a40 40 0 0 1 40-40h208a40 40 0 0 1 40 40v240a40 40 0 0 1-40 40H216a40 40 0 0 1-40-40v-40" fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32"/><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M272 336l80-80-80-80M48 256h288"/></svg></span> </a> </td> <td> <div class="sota-table-link"> <a class="btn btn-primary" href="/sota/code-generation-on-apps" style="text-decoration: underline"> Compare</a> </div> </td> </tr> <tr> <td style="vertical-align: top; padding-top: 18px;"> Competition Pass@any </td> <td> 3.32% </td> <td> # 8 </td> <td class="results-icon"> <a href="/paper/evaluating-large-language-models-trained-on/review/?hl=36732"> <span class=" icon-wrapper icon-ion" data-name="enter-outline"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M176 176v-40a40 40 0 0 1 40-40h208a40 40 0 0 1 40 40v240a40 40 0 0 1-40 40H216a40 40 0 0 1-40-40v-40" fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32"/><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M272 336l80-80-80-80M48 256h288"/></svg></span> </a> </td> <td> <div class="sota-table-link"> <a class="btn btn-primary" href="/sota/code-generation-on-apps" style="text-decoration: underline"> Compare</a> </div> </td> </tr> <tr> <td style="vertical-align: top; padding-top: 18px;"> Interview Pass@any </td> <td> 3.70% </td> <td> # 7 </td> <td class="results-icon"> <a href="/paper/evaluating-large-language-models-trained-on/review/?hl=36732"> <span class=" icon-wrapper icon-ion" data-name="enter-outline"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M176 176v-40a40 40 0 0 1 40-40h208a40 40 0 0 1 40 40v240a40 40 0 0 1-40 40H216a40 40 0 0 1-40-40v-40" fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32"/><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M272 336l80-80-80-80M48 256h288"/></svg></span> </a> </td> <td> <div class="sota-table-link"> <a class="btn btn-primary" href="/sota/code-generation-on-apps" style="text-decoration: underline"> Compare</a> </div> </td> </tr> <tr> <td style="vertical-align: top; padding-top: 18px;"> Introductory Pass@any </td> <td> 25.02% </td> <td> # 5 </td> <td class="results-icon"> <a href="/paper/evaluating-large-language-models-trained-on/review/?hl=36732"> <span class=" icon-wrapper icon-ion" data-name="enter-outline"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M176 176v-40a40 40 0 0 1 40-40h208a40 40 0 0 1 40 40v240a40 40 0 0 1-40 40H216a40 40 0 0 1-40-40v-40" fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32"/><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M272 336l80-80-80-80M48 256h288"/></svg></span> </a> </td> <td> <div class="sota-table-link"> <a class="btn btn-primary" href="/sota/code-generation-on-apps" style="text-decoration: underline"> Compare</a> </div> </td> </tr> <tr> <td rowspan="1" class="rowspan-td"> Multi-task Language Understanding </td> <td rowspan="1" class="rowspan-td"> BBH-alg </td> <td rowspan="1" class="rowspan-td model-col"> code-davinci-002 175B (CoT) </td> <td style="vertical-align: top; padding-top: 18px;"> Average (%) </td> <td> 73.9 </td> <td> # 1 </td> <td class="results-icon"> <a href="/paper/evaluating-large-language-models-trained-on/review/?hl=117128"> <span class=" icon-wrapper icon-ion" data-name="enter-outline"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M176 176v-40a40 40 0 0 1 40-40h208a40 40 0 0 1 40 40v240a40 40 0 0 1-40 40H216a40 40 0 0 1-40-40v-40" fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32"/><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M272 336l80-80-80-80M48 256h288"/></svg></span> </a> </td> <td> <div class="sota-table-link"> <a class="btn btn-primary" href="/sota/multi-task-language-understanding-on-bbh-alg" style="text-decoration: underline"> Compare</a> </div> </td> </tr> <tr> <td rowspan="1" class="rowspan-td"> Multi-task Language Understanding </td> <td rowspan="1" class="rowspan-td"> BBH-nlp </td> <td rowspan="1" class="rowspan-td model-col"> code-davinci-002 175B (CoT) </td> <td style="vertical-align: top; padding-top: 18px;"> Average (%) </td> <td> 73.5 </td> <td> # 8 </td> <td class="results-icon"> <a href="/paper/evaluating-large-language-models-trained-on/review/?hl=117129"> <span class=" icon-wrapper icon-ion" data-name="enter-outline"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M176 176v-40a40 40 0 0 1 40-40h208a40 40 0 0 1 40 40v240a40 40 0 0 1-40 40H216a40 40 0 0 1-40-40v-40" fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32"/><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M272 336l80-80-80-80M48 256h288"/></svg></span> </a> </td> <td> <div class="sota-table-link"> <a class="btn btn-primary" href="/sota/multi-task-language-understanding-on-bbh-nlp" style="text-decoration: underline"> Compare</a> </div> </td> </tr> </table> </div> </div> </div> </div> <div class="row"> <div id="methods" class="col-md-12 paper-evaluation-section-title"> <div class="paper-section-title"> <div class="row"> <div class="col-md-12 zero-padding"> <h2> Methods <div class="float-right"> <div class="dropdown edit-button"> <button class="dropdown-toggle badge badge-edit" type="button" id="methodEditMenu" data-bs-toggle="dropdown" aria-haspopup="true" aria-expanded="false"> <span class=" icon-wrapper icon-fa icon-fa-solid" data-name="edit"><svg viewBox="0 0 576 514.999" xmlns="http://www.w3.org/2000/svg"><path d="M402.6 85.198l90.2 90.2c3.8 3.8 3.8 10 0 13.8l-218.399 218.4-92.8 10.3c-12.4 1.4-22.9-9.1-21.5-21.5l10.3-92.8 218.4-218.4c3.799-3.8 10-3.8 13.799 0zm162-22.9c15.2 15.2 15.2 39.9 0 55.2l-35.4 35.4c-3.8 3.8-10 3.8-13.8 0l-90.2-90.2c-3.8-3.8-3.8-10 0-13.8l35.4-35.4c15.3-15.2 40-15.2 55.2 0zM384 348.198c0-3.2 1.3-6.2 3.5-8.5l40-40c7.6-7.5 20.5-2.2 20.5 8.5v157.8c0 26.5-21.5 48-48 48H48c-26.5 0-48-21.5-48-48v-352c0-26.5 21.5-48 48-48h285.8c10.7 0 16.1 12.9 8.5 20.5l-40 40c-2.3 2.2-5.3 3.5-8.5 3.5H64v320h320v-101.8z"/></svg></span> Edit </button> <div class="dropdown-menu dropdown-menu-end" aria-labelledby="methodEditMenu"> <a class="dropdown-item" href="#loginModal" data-bs-toggle="modal"> <span class=" icon-wrapper icon-ion" data-name="add"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M256 112v288m144-144H112"/></svg></span> Add</a> <a class="dropdown-item" href="#loginModal" data-bs-toggle="modal"> <span class=" icon-wrapper icon-ion" data-name="remove"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M400 256H112"/></svg></span> Remove</a> </div> </div> </div> </h2> <hr/> </div> </div> </div> </div> </div> <div class="method-section" id="methods"> <a href="/method/adam"> Adam</a> • <a href="/method/attention-dropout"> Attention Dropout</a> • <a href="/method/bpe"> BPE</a> • <a href="/method/cosine-annealing"> Cosine Annealing</a> • <a href="/method/dense-connections"> Dense Connections</a> • <a href="/method/discriminative-fine-tuning"> Discriminative Fine-Tuning</a> • <a href="/method/dropout"> Dropout</a> • <a href="/method/fixed-factorized-attention"> Fixed Factorized Attention</a> • <a href="/method/gelu"> GELU</a> • <a href="/method/gpt"> GPT</a> • <a href="/method/gpt-3"> GPT-3</a> • <a href="/method/layer-normalization"> Layer Normalization</a> • <a href="/method/linear-layer"> Linear Layer</a> • <a href="/method/linear-warmup-with-cosine-annealing"> Linear Warmup With Cosine Annealing</a> • <a href="/method/multi-head-attention"> Multi-Head Attention</a> • <a href="/method/residual-connection"> Residual Connection</a> • <a href="/method/scaled"> Scaled Dot-Product Attention</a> • <a href="/method/softmax"> Softmax</a> • <a href="/method/strided-attention"> Strided Attention</a> • <a href="/method/weight-decay"> Weight Decay</a> </div>  </div> </div> <div class="footer"> <div class="footer-contact"> <span class="footer-contact-item">Contact us on:</span> <a class="footer-contact-item" href="mailto:hello@paperswithcode.com"> <span class=" icon-wrapper icon-ion" data-name="mail"><svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512"><path d="M424 80H88a56.06 56.06 0 0 0-56 56v240a56.06 56.06 0 0 0 56 56h336a56.06 56.06 0 0 0 56-56V136a56.06 56.06 0 0 0-56-56zm-14.18 92.63l-144 112a16 16 0 0 1-19.64 0l-144-112a16 16 0 1 1 19.64-25.26L256 251.73l134.18-104.36a16 16 0 0 1 19.64 25.26z"/></svg></span> hello@paperswithcode.com </a>. <span class="footer-contact-item"> Papers With Code is a free resource with all data licensed under <a rel="noreferrer" href="https://creativecommons.org/licenses/by-sa/4.0/">CC-BY-SA</a>. </span> </div> <div class="footer-links"> <a href="/site/terms">Terms</a> <a href="/site/data-policy">Data policy</a> <a href="/site/cookies-policy">Cookies policy</a> <a href="/about#team" class="fair-logo"> from <img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAANAAAAAgCAMAAABU6AZfAAAABGdBTUEAALGPC/xhBQAAAAFzUkdCAK7OHOkAAAAJcEhZcwAAFiUAABYlAUlSJPAAAABFUExURUdwTBwqMhwqMxsqMhkqMxsqMhwqMgCA+hwrMxJIgBsrMxsqMgJ28AF58wF38BsqMwB58hsqMwF17wF07hwrMwRm4QJz7Wj6SIIAAAAUdFJOUwDP87wcPIT+4A1tVti1Ta0smZVzG3JP8wAABR9JREFUWMO1memWpCoMgF0QxX1//0e9kCAkAadq5tzKjzndQmM+szNFEWQ9puu6xn02BXm4j23bTsdapKJAMguFgRVT/Ejyx4uH5hgvL1PUfm69jEd6bN05GTJvXF5X/hfRcPyWe2kTLDFdRA4ENVMbZZJGMt3ppEttNMDC2X/Qa7MK1OrveZoKz2/445I+U4znuvaExxKZLFCqtym/A6rzn+OjbHj8ubwDmfESslvtgWea13WeckQPUKJTf/4USHkDnVXzCrT74DnmeX+8rjgcxA4QBmPpyAKdOm+5XwFpgHH/bG9AMzLMqM9DxxCQaM0qLr7U4xE/AgIDVRBHlcoDeYd7lFee6GZOBvaaskD8S6nut0Dg0ItZEt+IQAfjseIzRDvS/WCxWQJ17phqEGqepQBS/VaXZa0H/4XUYMVt6nr309DEjYvduPT2gWELQTr0iQbC1+SADOg/kjVvspGqX6zSRAgEKbqOf6zgd82AVB+8s0YNm5NL6Y8MGzttwKt0krP9+9A/+hzQTALoUX5MnxW7iCIEUmD7IVZb8G0G1HRE9UqbWKkEUFPSR0MWqH5eB65XmgzQdN3WGjxReROxPD2LROeBIEiD7UGLraBAjMcS9W9AquTPckBgoMqEWG1SIGN57otn5KO9Y30N4rq6MQFC5TX1cEWBfJLY+mbQ5ZMUm8UK7F1A9GNc90T3enkpCZhCdUzfdQq0Wp774gnZao55YU3SgkmAVBez1eDfR4BABd/XqY36ichyaLUnyJZ8jatimUBjqQTouK2M3OGs4miiiduN5bkHCL15C9Zw7heBRMHYSMRxIGyYFsPqpwTqactT8w0P0OSA9iRY9jQvrDyIAhCoAjrrR90I1PNCpcivHEh+cATUmS5xoCaNB3ggMzqgRO/RYPIb1WviDkB4sv22kB8ghQcgUIFWzyUmaQ6kpf5DCoTFh5fwQQCt493e9ypD5Xjq7S5cMQeEubpBf2oKCoSMohPzduBAi2yimhRIc3NvrOd+gCxPexvhcGPM3SRoJpbmIhAGSudTNgNCR+qIRL05UCebsxTIiAYOX6sEkONphRkw9A9ZjADIZIDg857we5MBSiQHVMlWJgXyeTBIyVpGD4RttHC4yVtENHn7K5ASdeM3QGX2sKcKBCBmITYmrGii9TOQT7JYwxOgrhbyby4XJrvs54kuR8vlCg4XEgEOEs8Q8R5DYZboCwEESpTmi/Hhc1Lo8zxPlghZjpbLqWVGUGxSes1y4W2lkkC+Wf0C6GPaxtZo0VQW4nOhsJLqAg01HXqgGN0+083MegKoYLdisbDqzHVG1iZJYe0EUDoB+dj149gDRCCgt2lZ1zA5nhvCyEwvrc/b3N/HiZlMgINmZaR/aX3MJluf7Kepo8+F5tRfUh1wR0odzg8Srnm9w7L5SyB/p6H9Ptt0Vj310ngAlDHbnLo3mGc00sJiQ+4KEM+I8xC7fWv5VGcz3Y0C2ZCa70sgf0tXbnbY1jXpln3W6jYXDG4jNthdrfVWn8n4gAVAZe+0GgaEaeGFx4XRQyTM9yWQnNuIAy5/HPAWPuDJ8Yc66sYvSeY/8dhlYqH0kuQzkFQ03nnHCyI/gtc0GfM7BVPmL5J0yHPkXm6d3u6v/TLw3GL5ayDr6WW47awHYmS1VC+XJOVQcCCZBPk13SCvgmcb8uI/UqjqdvlOlk3j5OU20C0putdO1ZWNo0a8oumXslx0vMYaNrfPURt2hnp5G2rhtsEP5j/3Wqt0fQd1YgAAAABJRU5ErkJggg=="> </a> </div> </div> <script> // MathJax window.MathJax = { tex: { inlineMath: [ ["$", "$"], ["\$", "\$"], ], }, }; const mathjaxScript = document.createElement("script"); mathjaxScript.src = "https://production-assets.paperswithcode.com/static/js/mathjax/tex-chtml.js"; document.head.appendChild(mathjaxScript); </script> <script src="https://production-assets.paperswithcode.com/perf/766.4af6b88b.js" defer></script><script src="https://production-assets.paperswithcode.com/perf/351.a22a9607.js" defer></script><script>(()=>{"use strict";var e,t,n,r,a={73487:(e,t,n)=>{n(26029),n(96869),n(22696),n(89527),n(7233),n(80591);var r=n(23279),a=n.n(r);const o=["tasks","leaderboards","papers","datasets","methods"],l=document.getElementById("id_global_search_form"),s=document.getElementById("id_global_search_input"),i=document.getElementById("q_meta"),d=document.getElementById("q_type"),c=document.createElement("ul");c.id="result-box",l.appendChild(c);let u=0,m=!1,p=[],f=null,g="";const h=a()((function(e){const t=e.target.value;if(t.length<=1)return c.classList.remove("show"),m=!1,p=[],void(f=null);(async e=>{const n=await fetch(`/api/search-autocomplete/?q=${encodeURIComponent(t)}`),r=await n.json();e===u&&function(e){if(e=function(e){let t=o.reduce(((e,t)=>(e[t]=[],e)),{}),n=12;for(let r=0;r<5;r++){for(const a of o)if(e[a].length>r&&(t[a].push(e[a][r]),n--,n<=0))break;if(n<=0)break}return n<12?t:null}(e),f=null,!e)return c.classList.remove("show"),m=!1,void(p=[]);let t="";for(const n of o)if(e[n].length){t+=`<li class='category-name'>${b(n[0].toUpperCase()+n.substring(1))}</li>`;for(const r of e[n]){let e="";["leaderboards","datasets"].includes(n)&&(e=r.slug),t+=`<li class='search-item' data-category="${b(n)}" data-meta="${b(e)}" data-label="${b(r.name||r.title)}"><div class='search-item-inner'>`,r.image?(r.image.startsWith("media")&&(r.image="/"+r.image),t+=`<img src="${b(r.image)}">`):"papers"!==n&&(t+=`<img src='${MEDIA_URL}tasks/default.gif'>`),t+=`<span>${b(r.name||r.title)}</span></div></li>`}}c.innerHTML=t,c.classList.add("show"),m=!0,p=[...document.getElementsByClassName("search-item")]}(r)})(++u)}),250,{maxWait:1e3});function y(e){if(!e)return void l.submit();const t=e.dataset.meta,n=e.dataset.category,r=e.dataset.label;GTAG_ENABLED&&window.gtag("event","SiteActions",{event_category:"Search",event_label:n}),s.value=r,t?i.value=t:(i.value="",i.removeAttribute("name")),d.value=n,l.submit()}function v(e){if(null!==e&&e>=p.length)throw Error("idx out of bound");f=e;for(const e of p)e.classList.remove("selected");null!==e?(p[e].classList.add("selected"),s.value=p[e].dataset.label):s.value=g}function b(e){return e.replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">").replace(/"/g,""").replace(/'/g,"'")}s.addEventListener("input",h),document.body.addEventListener("click",(()=>{c.classList.remove("show"),m=!1})),s.addEventListener("click",(e=>{e.stopPropagation()})),s.addEventListener("input",(()=>{g=s.value})),s.addEventListener("keydown",(e=>{if("Escape"===e.key&&m&&(e.preventDefault(),c.classList.remove("show"),m=!1,v(null)),"ArrowDown"===e.key){if(e.preventDefault(),!p.length)return;p.length&&(c.classList.add("show"),m=!0),null===f?v(0):f>=p.length-1?v(null):v(f+1)}if("ArrowUp"===e.key){if(e.preventDefault(),!p.length)return;p.length&&(c.classList.add("show"),m=!0),v(null===f?p.length-1:f<=0?null:f-1)}})),c.addEventListener("click",(e=>{e.stopPropagation(),y(e.target.closest(".search-item"))})),l.addEventListener("submit",(e=>{y(p[f])}));var E=n(179);""!==SENTRY_DSN_FRONTEND&&E.S1({dsn:SENTRY_DSN_FRONTEND});var w=n(45852);let k=!1;const L=document.getElementsByClassName("read-more-toggle")[0],_=document.getElementsByClassName("read-more-dots")[0],x=document.getElementsByClassName("read-more-rest")[0];L&&L.addEventListener("click",(e=>{e.preventDefault(),k?(_.style.display="",x.style.display="",L.text="read more"):(_.style.display="none",x.style.display="inline",L.text="(read less)"),k=!k}));const C=document.getElementById("implementations-see-more-trigger"),B=document.getElementById("implementations-see-less-trigger"),N=document.getElementById("implementations-short-list"),S=document.getElementById("implementations-full-list");C&&C.addEventListener("click",(e=>{e.preventDefault(),N.style.display="none",S.style.display=""})),B&&B.addEventListener("click",(e=>{e.preventDefault(),N.style.display="",S.style.display="none"})),(()=>{const e=[...document.querySelectorAll(".modal-body")];let t=!1;for(const t of e)t.style.opacity=0;window.addEventListener("click",(r=>{if(r.target.closest('[data-bs-toggle="modal"]')){const r=document.getElementById("modals-template");document.body.appendChild(r.content);let a=(0,w.Z)("csrftoken");for(const e of[...document.querySelectorAll("input[name='csrfmiddlewaretoken']")])e.value=a;!async function(){t||(t=!0,Promise.all([n.e(2),n.e(109),n.e(702),n.e(90)]).then(n.bind(n,56090)).then((()=>{n.e(43).then(n.bind(n,36043));for(const t of e)t.style.opacity=""})))}(),(()=>{const e=document.getElementById("new-method-form"),t=document.getElementById("new-method-form-toggle");let n=!1;t.addEventListener("click",(t=>{t.preventDefault(),e.style.display=n?"none":"",n=!n}))})(),(()=>{const e=document.getElementById("new-task-form"),t=document.getElementById("new-task-form-toggle");let n=!1;t.addEventListener("click",(t=>{t.preventDefault(),e.style.display=n?"none":"",n=!n}))})()}}),!0)})()},45852:(e,t,n)=>{n.d(t,{Z:()=>r});const r=e=>{var t=null;if(document.cookie&&""!==document.cookie)for(var n=document.cookie.split(";"),r=0;r<n.length;r++){var a=n[r].trim();if(a.substring(0,e.length+1)===e+"="){t=decodeURIComponent(a.substring(e.length+1));break}}return t}}},o={};function l(e){if(o[e])return o[e].exports;var t=o[e]={id:e,loaded:!1,exports:{}};return a[e](t,t.exports,l),t.loaded=!0,t.exports}l.m=a,l.x=e=>{},l.n=e=>{var t=e&&e.__esModule?()=>e.default:()=>e;return l.d(t,{a:t}),t},l.d=(e,t)=>{for(var n in t)l.o(t,n)&&!l.o(e,n)&&Object.defineProperty(e,n,{enumerable:!0,get:t[n]})},l.f={},l.e=e=>Promise.all(Object.keys(l.f).reduce(((t,n)=>(l.f[n](e,t),t)),[])),l.u=e=>e+"."+{2:"6da00df7",43:"b3f6a007",90:"ead00655",109:"5aa180f0",702:"c05a3709"}[e]+".js",l.miniCssF=e=>e+"."+{43:"b2664180",90:"d7a7e4c6",109:"6ee1c62e",918:"c41196c3"}[e]+".css",l.g=function(){if("object"==typeof globalThis)return globalThis;try{return this||new Function("return this")()}catch(e){if("object"==typeof window)return window}}(),l.hmd=e=>((e=Object.create(e)).children||(e.children=[]),Object.defineProperty(e,"exports",{enumerable:!0,set:()=>{throw new Error("ES Modules may not assign module.exports or exports.*, Use ESM export syntax, instead: "+e.id)}}),e),l.o=(e,t)=>Object.prototype.hasOwnProperty.call(e,t),e={},t="perf_frontend:",l.l=(n,r,a,o)=>{if(e[n])e[n].push(r);else{var s,i;if(void 0!==a)for(var d=document.getElementsByTagName("script"),c=0;c<d.length;c++){var u=d[c];if(u.getAttribute("src")==n||u.getAttribute("data-webpack")==t+a){s=u;break}}s||(i=!0,(s=document.createElement("script")).charset="utf-8",s.timeout=120,l.nc&&s.setAttribute("nonce",l.nc),s.setAttribute("data-webpack",t+a),s.src=n),e[n]=[r];var m=(t,r)=>{s.onerror=s.onload=null,clearTimeout(p);var a=e[n];if(delete e[n],s.parentNode&&s.parentNode.removeChild(s),a&&a.forEach((e=>e(r))),t)return t(r)},p=setTimeout(m.bind(null,void 0,{type:"timeout",target:s}),12e4);s.onerror=m.bind(null,s.onerror),s.onload=m.bind(null,s.onload),i&&document.head.appendChild(s)}},l.r=e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},l.p="https://production-assets.paperswithcode.com/perf/",n=e=>new Promise(((t,n)=>{var r=l.miniCssF(e),a=l.p+r;if(((e,t)=>{for(var n=document.getElementsByTagName("link"),r=0;r<n.length;r++){var a=(l=n[r]).getAttribute("data-href")||l.getAttribute("href");if("stylesheet"===l.rel&&(a===e||a===t))return l}var o=document.getElementsByTagName("style");for(r=0;r<o.length;r++){var l;if((a=(l=o[r]).getAttribute("data-href"))===e||a===t)return l}})(r,a))return t();((e,t,n,r)=>{var a=document.createElement("link");a.rel="stylesheet",a.type="text/css",a.onerror=a.onload=o=>{if(a.onerror=a.onload=null,"load"===o.type)n();else{var l=o&&("load"===o.type?"missing":o.type),s=o&&o.target&&o.target.href||t,i=new Error("Loading CSS chunk "+e+" failed.\n("+s+")");i.code="CSS_CHUNK_LOAD_FAILED",i.type=l,i.request=s,a.parentNode.removeChild(a),r(i)}},a.href=t,document.head.appendChild(a)})(e,a,t,n)})),r={645:0},l.f.miniCss=(e,t)=>{r[e]?t.push(r[e]):0!==r[e]&&{43:1,90:1,109:1}[e]&&t.push(r[e]=n(e).then((()=>{r[e]=0}),(t=>{throw delete r[e],t})))},(()=>{var e={645:0},t=[[73487,766,351]];l.f.j=(t,n)=>{var r=l.o(e,t)?e[t]:void 0;if(0!==r)if(r)n.push(r[2]);else if(918!=t){var a=new Promise(((n,a)=>{r=e[t]=[n,a]}));n.push(r[2]=a);var o=l.p+l.u(t),s=new Error;l.l(o,(n=>{if(l.o(e,t)&&(0!==(r=e[t])&&(e[t]=void 0),r)){var a=n&&("load"===n.type?"missing":n.type),o=n&&n.target&&n.target.src;s.message="Loading chunk "+t+" failed.\n("+a+": "+o+")",s.name="ChunkLoadError",s.type=a,s.request=o,r[1](s)}}),"chunk-"+t,t)}else e[t]=0};var n=e=>{},r=(r,a)=>{for(var o,s,[i,d,c,u]=a,m=0,p=[];m<i.length;m++)s=i[m],l.o(e,s)&&e[s]&&p.push(e[s][0]),e[s]=0;for(o in d)l.o(d,o)&&(l.m[o]=d[o]);for(c&&c(l),r&&r(a);p.length;)p.shift()();return u&&t.push.apply(t,u),n()},a=self.webpackChunkperf_frontend=self.webpackChunkperf_frontend||[];function o(){for(var n,r=0;r<t.length;r++){for(var a=t[r],o=!0,s=1;s<a.length;s++){var i=a[s];0!==e[i]&&(o=!1)}o&&(t.splice(r--,1),n=l(l.s=a[0]))}return 0===t.length&&(l.x(),l.x=e=>{}),n}a.forEach(r.bind(null,0)),a.push=r.bind(null,a.push.bind(a));var s=l.x;l.x=()=>(l.x=s||(e=>{}),(n=o)())})(),l.x()})();</script> </body> </html>

CINXE.COM

Evaluating Large Language Models Trained on Code | Papers With Code