{"task":"Code Generation","dataset":"APPS","metric_names":["Introductory Pass@1","Interview Pass@1","Competition Pass@1","Competition Pass@any","Interview Pass@any","Introductory Pass@any","Competition Pass@5","Interview Pass@5"],"rows":[{"id":29382,"task":"Code Generation","parent_task":null,"dataset":"APPS","model_name":"LPW (GPT-4o)","metrics":{"Competition Pass@1":"34.8","Interview Pass@1":"65.2","Introductory Pass@1":"87.2","Pass@1":"62.6"},"paper_url":"https://arxiv.org/abs/2411.14503v3","paper_title":"Planning-Driven Programming: A Large Language Model Programming Workflow","paper_date":"2024-11-21","code_links":[{"title":"you68681/lpw","url":"https://github.com/you68681/lpw"}],"metrics_order":"[\"Introductory Pass@1\", \"Interview Pass@1\", \"Competition Pass@1\", \"Competition Pass@any\", \"Interview Pass@any\", \"Introductory Pass@any\", \"Competition Pass@5\", \"Interview Pass@5\", \"Introductory Pass@5\", \"Competition Pass@1000\", \"Interview Pass@1000\", \"Introductory Pass@1000\", \"Pass@1\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]},{"id":29383,"task":"Code Generation","parent_task":null,"dataset":"APPS","model_name":"MoTCoder-32B-V1.5","metrics":{"Competition Pass@1":"27.84","Interview Pass@1":"44.49","Introductory Pass@1":"68.44"},"paper_url":"https://arxiv.org/abs/2312.15960v5","paper_title":"MoTCoder: Elevating Large Language Models with Modular of Thought for Challenging Programming Tasks","paper_date":"2023-12-26","code_links":[{"title":"dvlab-research/motcoder","url":"https://github.com/dvlab-research/motcoder"}],"metrics_order":"[\"Introductory Pass@1\", \"Interview Pass@1\", \"Competition Pass@1\", \"Competition Pass@any\", \"Interview Pass@any\", \"Introductory Pass@any\", \"Competition Pass@5\", \"Interview Pass@5\", \"Introductory Pass@5\", \"Competition Pass@1000\", \"Interview Pass@1000\", \"Introductory Pass@1000\", \"Pass@1\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]},{"id":29384,"task":"Code Generation","parent_task":null,"dataset":"APPS","model_name":"MoTCoder-7B-V1.5","metrics":{"Competition Pass@1":"21.18","Interview Pass@1":"32.63","Introductory Pass@1":"54.26"},"paper_url":"https://arxiv.org/abs/2312.15960v5","paper_title":"MoTCoder: Elevating Large Language Models with Modular of Thought for Challenging Programming Tasks","paper_date":"2023-12-26","code_links":[{"title":"dvlab-research/motcoder","url":"https://github.com/dvlab-research/motcoder"}],"metrics_order":"[\"Introductory Pass@1\", \"Interview Pass@1\", \"Competition Pass@1\", \"Competition Pass@any\", \"Interview Pass@any\", \"Introductory Pass@any\", \"Competition Pass@5\", \"Interview Pass@5\", \"Introductory Pass@5\", \"Competition Pass@1000\", \"Interview Pass@1000\", \"Introductory Pass@1000\", \"Pass@1\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]},{"id":29385,"task":"Code Generation","parent_task":null,"dataset":"APPS","model_name":"code-davinci-002 175B (CodeT)","metrics":{"Competition Pass@1":"6.2%","Interview Pass@1":"14.3%","Introductory Pass@1":"47.3%"},"paper_url":"https://arxiv.org/abs/2207.10397v2","paper_title":"CodeT: Code Generation with Generated Tests","paper_date":"2022-07-21","code_links":[{"title":"microsoft/codet","url":"https://github.com/microsoft/codet"}],"metrics_order":"[\"Introductory Pass@1\", \"Interview Pass@1\", \"Competition Pass@1\", \"Competition Pass@any\", \"Interview Pass@any\", \"Introductory Pass@any\", \"Competition Pass@5\", \"Interview Pass@5\", \"Introductory Pass@5\", \"Competition Pass@1000\", \"Interview Pass@1000\", \"Introductory Pass@1000\", \"Pass@1\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]},{"id":29386,"task":"Code Generation","parent_task":null,"dataset":"APPS","model_name":"deepseek-ai/deepseek-coder-6.7b-instruct","metrics":{"Competition Pass@1":"11.09","Interview Pass@1":"19.70","Introductory Pass@1":"33.80"},"paper_url":"https://arxiv.org/abs/2401.14196v2","paper_title":"DeepSeek-Coder: When the Large Language Model Meets Programming -- The Rise of Code Intelligence","paper_date":"2024-01-25","code_links":[{"title":"deepseek-ai/DeepSeek-Coder","url":"https://github.com/deepseek-ai/DeepSeek-Coder"}],"metrics_order":"[\"Introductory Pass@1\", \"Interview Pass@1\", \"Competition Pass@1\", \"Competition Pass@any\", \"Interview Pass@any\", \"Introductory Pass@any\", \"Competition Pass@5\", \"Interview Pass@5\", \"Introductory Pass@5\", \"Competition Pass@1000\", \"Interview Pass@1000\", \"Introductory Pass@1000\", \"Pass@1\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]},{"id":29387,"task":"Code Generation","parent_task":null,"dataset":"APPS","model_name":"code-davinci-002 175B","metrics":{"Introductory Pass@1":"31.92"},"paper_url":"https://arxiv.org/abs/2207.10397v2","paper_title":"CodeT: Code Generation with Generated Tests","paper_date":"2022-07-21","code_links":[{"title":"microsoft/codet","url":"https://github.com/microsoft/codet"}],"metrics_order":"[\"Introductory Pass@1\", \"Interview Pass@1\", \"Competition Pass@1\", \"Competition Pass@any\", \"Interview Pass@any\", \"Introductory Pass@any\", \"Competition Pass@5\", \"Interview Pass@5\", \"Introductory Pass@5\", \"Competition Pass@1000\", \"Interview Pass@1000\", \"Introductory Pass@1000\", \"Pass@1\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]},{"id":29388,"task":"Code Generation","parent_task":null,"dataset":"APPS","model_name":"CodeChain+WizardCoder-15b","metrics":{"Competition Pass@1":"2.5%","Competition Pass@any":"14.5%","Interview Pass@1":"6.4%","Interview Pass@any":"25.4%","Introductory Pass@1":"29.3%","Introductory Pass@any":"60.9%"},"paper_url":"https://arxiv.org/abs/2310.08992v3","paper_title":"CodeChain: Towards Modular Code Generation Through Chain of Self-revisions with Representative Sub-modules","paper_date":"2023-10-13","code_links":[{"title":"SalesforceAIResearch/CodeChain","url":"https://github.com/SalesforceAIResearch/CodeChain"}],"metrics_order":"[\"Introductory Pass@1\", \"Interview Pass@1\", \"Competition Pass@1\", \"Competition Pass@any\", \"Interview Pass@any\", \"Introductory Pass@any\", \"Competition Pass@5\", \"Interview Pass@5\", \"Introductory Pass@5\", \"Competition Pass@1000\", \"Interview Pass@1000\", \"Introductory Pass@1000\", \"Pass@1\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]},{"id":29389,"task":"Code Generation","parent_task":null,"dataset":"APPS","model_name":"WizardCoder-15b","metrics":{"Competition Pass@1":"3.75","Interview Pass@1":"7.49","Introductory Pass@1":"26.29"},"paper_url":"https://arxiv.org/abs/2310.08992v3","paper_title":"CodeChain: Towards Modular Code Generation Through Chain of Self-revisions with Representative Sub-modules","paper_date":"2023-10-13","code_links":[{"title":"SalesforceAIResearch/CodeChain","url":"https://github.com/SalesforceAIResearch/CodeChain"}],"metrics_order":"[\"Introductory Pass@1\", \"Interview Pass@1\", \"Competition Pass@1\", \"Competition Pass@any\", \"Interview Pass@any\", \"Introductory Pass@any\", \"Competition Pass@5\", \"Interview Pass@5\", \"Introductory Pass@5\", \"Competition Pass@1000\", \"Interview Pass@1000\", \"Introductory Pass@1000\", \"Pass@1\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]},{"id":29390,"task":"Code Generation","parent_task":null,"dataset":"APPS","model_name":"CodeSim (GPT4)","metrics":{"Competition Pass@1":"0.81","Interview Pass@1":"4.21","Introductory Pass@1":"26.04"},"paper_url":"https://arxiv.org/abs/2502.05664v1","paper_title":"CODESIM: Multi-Agent Code Generation and Problem Solving through Simulation-Driven Planning and Debugging","paper_date":"2025-02-08","code_links":[{"title":"kagnlp/CodeGenerator","url":"https://github.com/kagnlp/CodeGenerator"}],"metrics_order":"[\"Introductory Pass@1\", \"Interview Pass@1\", \"Competition Pass@1\", \"Competition Pass@any\", \"Interview Pass@any\", \"Introductory Pass@any\", \"Competition Pass@5\", \"Interview Pass@5\", \"Introductory Pass@5\", \"Competition Pass@1000\", \"Interview Pass@1000\", \"Introductory Pass@1000\", \"Pass@1\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]},{"id":29391,"task":"Code Generation","parent_task":null,"dataset":"APPS","model_name":"CodeRL+CodeT5","metrics":{"Competition Pass@1":"33.3","Interview Pass@1":"13.5","Introductory Pass@1":"20"},"paper_url":"https://arxiv.org/abs/2207.01780v3","paper_title":"CodeRL: Mastering Code Generation through Pretrained Models and Deep Reinforcement Learning","paper_date":"2022-07-05","code_links":[{"title":"salesforce/codet5","url":"https://github.com/salesforce/codet5"},{"title":"salesforce/coderl","url":"https://github.com/salesforce/coderl"}],"metrics_order":"[\"Introductory Pass@1\", \"Interview Pass@1\", \"Competition Pass@1\", \"Competition Pass@any\", \"Interview Pass@any\", \"Introductory Pass@any\", \"Competition Pass@5\", \"Interview Pass@5\", \"Introductory Pass@5\", \"Competition Pass@1000\", \"Interview Pass@1000\", \"Introductory Pass@1000\", \"Pass@1\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]},{"id":29392,"task":"Code Generation","parent_task":null,"dataset":"APPS","model_name":"GPT-J 6B (Finetuned)","metrics":{"Competition Pass@1":"0.69%","Competition Pass@1000":"15.70%","Competition Pass@5":"2.36%","Competition Pass@any":"15.70%","Interview Pass@1":"1.80%","Interview Pass@1000":"14.33%","Interview Pass@5":"4.48%","Interview Pass@any":"14.33%","Introductory Pass@1":"6.77%","Introductory Pass@1000":"38.10%","Introductory Pass@5":"15.27%","Introductory Pass@any":"38.10%"},"paper_url":"https://arxiv.org/abs/2207.01780v3","paper_title":"CodeRL: Mastering Code Generation through Pretrained Models and Deep Reinforcement Learning","paper_date":"2022-07-05","code_links":[{"title":"salesforce/codet5","url":"https://github.com/salesforce/codet5"},{"title":"salesforce/coderl","url":"https://github.com/salesforce/coderl"}],"metrics_order":"[\"Introductory Pass@1\", \"Interview Pass@1\", \"Competition Pass@1\", \"Competition Pass@any\", \"Interview Pass@any\", \"Introductory Pass@any\", \"Competition Pass@5\", \"Interview Pass@5\", \"Introductory Pass@5\", \"Competition Pass@1000\", \"Interview Pass@1000\", \"Introductory Pass@1000\", \"Pass@1\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]},{"id":29393,"task":"Code Generation","parent_task":null,"dataset":"APPS","model_name":"Codex 12B (Raw)","metrics":{"Competition Pass@1":"0.50%","Competition Pass@1000":"13.51%","Competition Pass@5":"1.00%","Competition Pass@any":"13.51%","Interview Pass@1":"1.00%","Interview Pass@1000":"13.15%","Interview Pass@5":"1.73%","Interview Pass@any":"13.15%","Introductory Pass@1":"5.60%","Introductory Pass@1000":"35.20%","Introductory Pass@5":"9.20%","Introductory Pass@any":"35.20%"},"paper_url":"https://arxiv.org/abs/2107.03374v2","paper_title":"Evaluating Large Language Models Trained on Code","paper_date":"2021-07-07","code_links":[{"title":"THUDM/CodeGeeX","url":"https://github.com/THUDM/CodeGeeX"},{"title":"ncoop57/gpt-code-clippy","url":"https://github.com/ncoop57/gpt-code-clippy"},{"title":"codedotal/gpt-code-clippy","url":"https://github.com/codedotal/gpt-code-clippy"},{"title":"openai/human-eval","url":"https://github.com/openai/human-eval"},{"title":"vhellendoorn/code-lms","url":"https://github.com/vhellendoorn/code-lms"},{"title":"glouppe/info8010-deep-learning","url":"https://github.com/glouppe/info8010-deep-learning"},{"title":"microsoft/PythonProgrammingPuzzles","url":"https://github.com/microsoft/PythonProgrammingPuzzles"},{"title":"fsoft-ai4code/codecapybara","url":"https://github.com/fsoft-ai4code/codecapybara"},{"title":"codefuse-ai/codefuse-evaluation","url":"https://github.com/codefuse-ai/codefuse-evaluation"},{"title":"my-other-github-account/llm-humaneval-benchmarks","url":"https://github.com/my-other-github-account/llm-humaneval-benchmarks"},{"title":"superli3/codenavi","url":"https://github.com/superli3/codenavi"},{"title":"superli3/CYRMPR","url":"https://github.com/superli3/CYRMPR"},{"title":"2796gaurav/human-eval","url":"https://github.com/2796gaurav/human-eval"}],"metrics_order":"[\"Introductory Pass@1\", \"Interview Pass@1\", \"Competition Pass@1\", \"Competition Pass@any\", \"Interview Pass@any\", \"Introductory Pass@any\", \"Competition Pass@5\", \"Interview Pass@5\", \"Introductory Pass@5\", \"Competition Pass@1000\", \"Interview Pass@1000\", \"Introductory Pass@1000\", \"Pass@1\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]},{"id":29394,"task":"Code Generation","parent_task":null,"dataset":"APPS","model_name":"GPT-Neo 2.7B (Finetuned)","metrics":{"Competition Pass@1":"0.02%","Competition Pass@1000":"3.23%","Competition Pass@5":"0.09%","Competition Pass@any":"3.32%","Interview Pass@1":"0.14%","Interview Pass@1000":"3.70%","Interview Pass@5":"0.51%","Interview Pass@any":"3.70%","Introductory Pass@1":"4.14%","Introductory Pass@1000":"25.02%","Introductory Pass@5":"9.65%","Introductory Pass@any":"25.02%"},"paper_url":"https://arxiv.org/abs/2207.01780v3","paper_title":"CodeRL: Mastering Code Generation through Pretrained Models and Deep Reinforcement Learning","paper_date":"2022-07-05","code_links":[{"title":"salesforce/codet5","url":"https://github.com/salesforce/codet5"},{"title":"salesforce/coderl","url":"https://github.com/salesforce/coderl"}],"metrics_order":"[\"Introductory Pass@1\", \"Interview Pass@1\", \"Competition Pass@1\", \"Competition Pass@any\", \"Interview Pass@any\", \"Introductory Pass@any\", \"Competition Pass@5\", \"Interview Pass@5\", \"Introductory Pass@5\", \"Competition Pass@1000\", \"Interview Pass@1000\", \"Introductory Pass@1000\", \"Pass@1\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]},{"id":29395,"task":"Code Generation","parent_task":null,"dataset":"APPS","model_name":"GPT-Neo 2.7B","metrics":{"Competition Pass@1":"0.00%","Competition Pass@1000":"11.40%","Competition Pass@5":"0.00%","Competition Pass@any":"11.40%","Interview Pass@1":"0.57%","Interview Pass@1000":"9.83%","Interview Pass@5":"0.80%","Interview Pass@any":"9.83%","Introductory Pass@1":"3.90%","Introductory Pass@1000":"27.90%","Introductory Pass@5":"5.50%","Introductory Pass@any":"27.90%"},"paper_url":"https://arxiv.org/abs/2105.09938v3","paper_title":"Measuring Coding Challenge Competence With APPS","paper_date":"2021-05-20","code_links":[{"title":"ncoop57/gpt-code-clippy","url":"https://github.com/ncoop57/gpt-code-clippy"},{"title":"codedotal/gpt-code-clippy","url":"https://github.com/codedotal/gpt-code-clippy"},{"title":"hendrycks/apps","url":"https://github.com/hendrycks/apps"}],"metrics_order":"[\"Introductory Pass@1\", \"Interview Pass@1\", \"Competition Pass@1\", \"Competition Pass@any\", \"Interview Pass@any\", \"Introductory Pass@any\", \"Competition Pass@5\", \"Interview Pass@5\", \"Introductory Pass@5\", \"Competition Pass@1000\", \"Interview Pass@1000\", \"Introductory Pass@1000\", \"Pass@1\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]},{"id":29396,"task":"Code Generation","parent_task":null,"dataset":"APPS","model_name":"GPT2 1.5B (Finetuned)","metrics":{"Competition Pass@1":"0.00%","Competition Pass@5":"0.00%","Competition Pass@any":"0.0%","Interview Pass@1":"0.57%","Interview Pass@5":"0.80%","Interview Pass@any":"0.80%","Introductory Pass@1":"3.90%","Introductory Pass@5":"5.50%","Introductory Pass@any":"5.50%"},"paper_url":"https://arxiv.org/abs/2207.01780v3","paper_title":"CodeRL: Mastering Code Generation through Pretrained Models and Deep Reinforcement Learning","paper_date":"2022-07-05","code_links":[{"title":"salesforce/codet5","url":"https://github.com/salesforce/codet5"},{"title":"salesforce/coderl","url":"https://github.com/salesforce/coderl"}],"metrics_order":"[\"Introductory Pass@1\", \"Interview Pass@1\", \"Competition Pass@1\", \"Competition Pass@any\", \"Interview Pass@any\", \"Introductory Pass@any\", \"Competition Pass@5\", \"Interview Pass@5\", \"Introductory Pass@5\", \"Competition Pass@1000\", \"Interview Pass@1000\", \"Introductory Pass@1000\", \"Pass@1\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]},{"id":29397,"task":"Code Generation","parent_task":null,"dataset":"APPS","model_name":"MapCoder APPS-150-cherrypicked (GPT-4)","metrics":{"Competition Pass@1":"0.00%","Competition Pass@1000":"8.80%","Competition Pass@5":"0.00%","Competition Pass@any":"8.80%","Interview Pass@1":"0.70%","Interview Pass@1000":"9.27%","Interview Pass@5":"1.03%","Interview Pass@any":"9.27%","Introductory Pass@1":"1.30%","Introductory Pass@1000":"25.00%","Introductory Pass@5":"3.60%","Introductory Pass@any":"25.00%"},"paper_url":"https://arxiv.org/abs/2405.11403v1","paper_title":"MapCoder: Multi-Agent Code Generation for Competitive Problem Solving","paper_date":"2024-05-18","code_links":[{"title":"md-ashraful-pramanik/mapcoder","url":"https://github.com/md-ashraful-pramanik/mapcoder"},{"title":"Luoji-zju/Agents4PLC_release","url":"https://github.com/Luoji-zju/Agents4PLC_release"}],"metrics_order":"[\"Introductory Pass@1\", \"Interview Pass@1\", \"Competition Pass@1\", \"Competition Pass@any\", \"Interview Pass@any\", \"Introductory Pass@any\", \"Competition Pass@5\", \"Interview Pass@5\", \"Introductory Pass@5\", \"Competition Pass@1000\", \"Interview Pass@1000\", \"Introductory Pass@1000\", \"Pass@1\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]},{"id":29398,"task":"Code Generation","parent_task":null,"dataset":"APPS","model_name":"AlphaCode 1B Filtered from  50000","metrics":{"Competition Pass@any":"22.0"},"paper_url":"https://arxiv.org/abs/2203.07814v1","paper_title":"Competition-Level Code Generation with AlphaCode","paper_date":"2022-02-08","code_links":[{"title":"google-deepmind/code_contests","url":"https://github.com/google-deepmind/code_contests"},{"title":"deepmind/code_contests","url":"https://github.com/deepmind/code_contests"}],"metrics_order":"[\"Introductory Pass@1\", \"Interview Pass@1\", \"Competition Pass@1\", \"Competition Pass@any\", \"Interview Pass@any\", \"Introductory Pass@any\", \"Competition Pass@5\", \"Interview Pass@5\", \"Introductory Pass@5\", \"Competition Pass@1000\", \"Interview Pass@1000\", \"Introductory Pass@1000\", \"Pass@1\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]},{"id":29399,"task":"Code Generation","parent_task":null,"dataset":"APPS","model_name":"AlphaCode 1B","metrics":{"Competition Pass@5":"7.75%","Competition Pass@any":"7.75%","Interview Pass@5":"9.66%","Interview Pass@any":"9.66%","Introductory Pass@5":"20.36%","Introductory Pass@any":"20.36%"},"paper_url":"https://arxiv.org/abs/2203.07814v1","paper_title":"Competition-Level Code Generation with AlphaCode","paper_date":"2022-02-08","code_links":[{"title":"google-deepmind/code_contests","url":"https://github.com/google-deepmind/code_contests"},{"title":"deepmind/code_contests","url":"https://github.com/deepmind/code_contests"}],"metrics_order":"[\"Introductory Pass@1\", \"Interview Pass@1\", \"Competition Pass@1\", \"Competition Pass@any\", \"Interview Pass@any\", \"Introductory Pass@any\", \"Competition Pass@5\", \"Interview Pass@5\", \"Introductory Pass@5\", \"Competition Pass@1000\", \"Interview Pass@1000\", \"Introductory Pass@1000\", \"Pass@1\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]},{"id":758928,"task":"Code Generation","parent_task":null,"dataset":"APPS","model_name":"SolidCoder","metrics":{"Pass@1":"77.0"},"paper_url":"https://paperswithcode.com/paper/solidcoder-bridging-the-mental-reality-gap-in-llm-code-generation-through-concrete-execution","paper_title":"SolidCoder: Bridging the Mental-Reality Gap in LLM Code Generation through Concrete Execution","paper_date":"2026-04-20","code_links":[],"metrics_order":null,"area":null,"uses_additional_data":null,"source":"auto","tags":[]}]}