{"task":"Code Generation","dataset":"PECC","metric_names":["Pass@3"],"rows":[{"id":29524,"task":"Code Generation","parent_task":null,"dataset":"PECC","model_name":"Claude 3 Haiku","metrics":{"Pass@3":"27.67"},"paper_url":"https://arxiv.org/abs/2404.18766v1","paper_title":"PECC: Problem Extraction and Coding Challenges","paper_date":"2024-04-29","code_links":[{"title":"hallerpatrick/pecc","url":"https://github.com/hallerpatrick/pecc"}],"metrics_order":"[\"Pass@3\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]},{"id":29525,"task":"Code Generation","parent_task":null,"dataset":"PECC","model_name":"GPT-3.5 Turbo","metrics":{"Pass@3":"23.75"},"paper_url":"https://arxiv.org/abs/2404.18766v1","paper_title":"PECC: Problem Extraction and Coding Challenges","paper_date":"2024-04-29","code_links":[{"title":"hallerpatrick/pecc","url":"https://github.com/hallerpatrick/pecc"}],"metrics_order":"[\"Pass@3\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]},{"id":29526,"task":"Code Generation","parent_task":null,"dataset":"PECC","model_name":"codechat-bison","metrics":{"Pass@3":"11.39"},"paper_url":"https://arxiv.org/abs/2404.18766v1","paper_title":"PECC: Problem Extraction and Coding Challenges","paper_date":"2024-04-29","code_links":[{"title":"hallerpatrick/pecc","url":"https://github.com/hallerpatrick/pecc"}],"metrics_order":"[\"Pass@3\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]},{"id":29527,"task":"Code Generation","parent_task":null,"dataset":"PECC","model_name":"chat-bison","metrics":{"Pass@3":"8.48"},"paper_url":"https://arxiv.org/abs/2404.18766v1","paper_title":"PECC: Problem Extraction and Coding Challenges","paper_date":"2024-04-29","code_links":[{"title":"hallerpatrick/pecc","url":"https://github.com/hallerpatrick/pecc"}],"metrics_order":"[\"Pass@3\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]},{"id":29528,"task":"Code Generation","parent_task":null,"dataset":"PECC","model_name":"Mixtral-8x7B-Instruct","metrics":{"Pass@3":"8.35"},"paper_url":"https://arxiv.org/abs/2404.18766v1","paper_title":"PECC: Problem Extraction and Coding Challenges","paper_date":"2024-04-29","code_links":[{"title":"hallerpatrick/pecc","url":"https://github.com/hallerpatrick/pecc"}],"metrics_order":"[\"Pass@3\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]},{"id":29529,"task":"Code Generation","parent_task":null,"dataset":"PECC","model_name":"Phi-3-mini-128k-instruct","metrics":{"Pass@3":"7.18"},"paper_url":"https://arxiv.org/abs/2404.18766v1","paper_title":"PECC: Problem Extraction and Coding Challenges","paper_date":"2024-04-29","code_links":[{"title":"hallerpatrick/pecc","url":"https://github.com/hallerpatrick/pecc"}],"metrics_order":"[\"Pass@3\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]},{"id":29530,"task":"Code Generation","parent_task":null,"dataset":"PECC","model_name":"WizardLM-2-7B","metrics":{"Pass@3":"3.72"},"paper_url":"https://arxiv.org/abs/2404.18766v1","paper_title":"PECC: Problem Extraction and Coding Challenges","paper_date":"2024-04-29","code_links":[{"title":"hallerpatrick/pecc","url":"https://github.com/hallerpatrick/pecc"}],"metrics_order":"[\"Pass@3\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]},{"id":29531,"task":"Code Generation","parent_task":null,"dataset":"PECC","model_name":"Llama-3-8B-Instruct","metrics":{"Pass@3":"3.1"},"paper_url":"https://arxiv.org/abs/2404.18766v1","paper_title":"PECC: Problem Extraction and Coding Challenges","paper_date":"2024-04-29","code_links":[{"title":"hallerpatrick/pecc","url":"https://github.com/hallerpatrick/pecc"}],"metrics_order":"[\"Pass@3\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]}]}