{"task":"Code Generation","dataset":"RES-Q","metric_names":["pass@1"],"rows":[{"id":29561,"task":"Code Generation","parent_task":null,"dataset":"RES-Q","model_name":"QurrentOS-coder + Claude 3.5 Sonnet","metrics":{"pass@1":"58.0"},"paper_url":"https://arxiv.org/abs/2406.16801v2","paper_title":"RES-Q: Evaluating Code-Editing Large Language Model Systems at the Repository Scale","paper_date":"2024-06-24","code_links":[{"title":"qurrent-ai/res-q","url":"https://github.com/qurrent-ai/res-q"}],"metrics_order":"[\"pass@1\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]},{"id":29562,"task":"Code Generation","parent_task":null,"dataset":"RES-Q","model_name":"QurrentOS-coder + GPT-4o","metrics":{"pass@1":"46.0"},"paper_url":"https://arxiv.org/abs/2406.16801v2","paper_title":"RES-Q: Evaluating Code-Editing Large Language Model Systems at the Repository Scale","paper_date":"2024-06-24","code_links":[{"title":"qurrent-ai/res-q","url":"https://github.com/qurrent-ai/res-q"}],"metrics_order":"[\"pass@1\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]},{"id":29563,"task":"Code Generation","parent_task":null,"dataset":"RES-Q","model_name":"QurrentOS-coder + GPT-4 Turbo","metrics":{"pass@1":"37.0"},"paper_url":"https://arxiv.org/abs/2406.16801v2","paper_title":"RES-Q: Evaluating Code-Editing Large Language Model Systems at the Repository Scale","paper_date":"2024-06-24","code_links":[{"title":"qurrent-ai/res-q","url":"https://github.com/qurrent-ai/res-q"}],"metrics_order":"[\"pass@1\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]},{"id":29564,"task":"Code Generation","parent_task":null,"dataset":"RES-Q","model_name":"QurrentOS-coder + Claude 3 Opus","metrics":{"pass@1":"36.0"},"paper_url":"https://arxiv.org/abs/2406.16801v2","paper_title":"RES-Q: Evaluating Code-Editing Large Language Model Systems at the Repository Scale","paper_date":"2024-06-24","code_links":[{"title":"qurrent-ai/res-q","url":"https://github.com/qurrent-ai/res-q"}],"metrics_order":"[\"pass@1\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]},{"id":29565,"task":"Code Generation","parent_task":null,"dataset":"RES-Q","model_name":"QurrentOS-coder + GPT-4","metrics":{"pass@1":"30.0"},"paper_url":"https://arxiv.org/abs/2406.16801v2","paper_title":"RES-Q: Evaluating Code-Editing Large Language Model Systems at the Repository Scale","paper_date":"2024-06-24","code_links":[{"title":"qurrent-ai/res-q","url":"https://github.com/qurrent-ai/res-q"}],"metrics_order":"[\"pass@1\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]},{"id":29566,"task":"Code Generation","parent_task":null,"dataset":"RES-Q","model_name":"QurrentOS-coder + Gemini 1.5 Pro","metrics":{"pass@1":"30.0"},"paper_url":"https://arxiv.org/abs/2406.16801v2","paper_title":"RES-Q: Evaluating Code-Editing Large Language Model Systems at the Repository Scale","paper_date":"2024-06-24","code_links":[{"title":"qurrent-ai/res-q","url":"https://github.com/qurrent-ai/res-q"}],"metrics_order":"[\"pass@1\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]},{"id":29567,"task":"Code Generation","parent_task":null,"dataset":"RES-Q","model_name":"QurrentOS-coder + DeepSeek-Coder-V2","metrics":{"pass@1":"29.0"},"paper_url":"https://arxiv.org/abs/2406.16801v2","paper_title":"RES-Q: Evaluating Code-Editing Large Language Model Systems at the Repository Scale","paper_date":"2024-06-24","code_links":[{"title":"qurrent-ai/res-q","url":"https://github.com/qurrent-ai/res-q"}],"metrics_order":"[\"pass@1\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]},{"id":29568,"task":"Code Generation","parent_task":null,"dataset":"RES-Q","model_name":"QurrentOS-coder + Llama 3 70b","metrics":{"pass@1":"20.0"},"paper_url":"https://arxiv.org/abs/2406.16801v2","paper_title":"RES-Q: Evaluating Code-Editing Large Language Model Systems at the Repository Scale","paper_date":"2024-06-24","code_links":[{"title":"qurrent-ai/res-q","url":"https://github.com/qurrent-ai/res-q"}],"metrics_order":"[\"pass@1\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]},{"id":29569,"task":"Code Generation","parent_task":null,"dataset":"RES-Q","model_name":"QurrentOS-coder + Qwen-72B-Instruct","metrics":{"pass@1":"18.0"},"paper_url":"https://arxiv.org/abs/2406.16801v2","paper_title":"RES-Q: Evaluating Code-Editing Large Language Model Systems at the Repository Scale","paper_date":"2024-06-24","code_links":[{"title":"qurrent-ai/res-q","url":"https://github.com/qurrent-ai/res-q"}],"metrics_order":"[\"pass@1\"]","area":"Natural Language Processing","uses_additional_data":0,"source":"archive","tags":[]}]}