{"task":"Code Completion","dataset":"SAFIM","metric_names":["Average","Algorithmic","Control","API"],"rows":[{"id":145920,"task":"Code Completion","parent_task":null,"dataset":"SAFIM","model_name":"deepseek-coder-33b-base","metrics":{"API":"75.16","Algorithmic":"60.78","Average":"69.01","Control":"71.10"},"paper_url":"https://arxiv.org/abs/2403.04814v3","paper_title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","paper_date":"2024-03-07","code_links":[{"title":"gonglinyuan/safim","url":"https://github.com/gonglinyuan/safim"}],"metrics_order":"[\"Average\", \"Algorithmic\", \"Control\", \"API\"]","area":"Computer Code","uses_additional_data":0,"source":"archive","tags":[]},{"id":145921,"task":"Code Completion","parent_task":null,"dataset":"SAFIM","model_name":"deepseek-coder-6.7b-base","metrics":{"API":"69.68","Algorithmic":"54.74","Average":"63.40","Control":"65.79"},"paper_url":"https://arxiv.org/abs/2403.04814v3","paper_title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","paper_date":"2024-03-07","code_links":[{"title":"gonglinyuan/safim","url":"https://github.com/gonglinyuan/safim"}],"metrics_order":"[\"Average\", \"Algorithmic\", \"Control\", \"API\"]","area":"Computer Code","uses_additional_data":0,"source":"archive","tags":[]},{"id":145922,"task":"Code Completion","parent_task":null,"dataset":"SAFIM","model_name":"starcoderbase","metrics":{"API":"68.06","Algorithmic":"44.11","Average":"55.54","Control":"54.46"},"paper_url":"https://arxiv.org/abs/2403.04814v3","paper_title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","paper_date":"2024-03-07","code_links":[{"title":"gonglinyuan/safim","url":"https://github.com/gonglinyuan/safim"}],"metrics_order":"[\"Average\", \"Algorithmic\", \"Control\", \"API\"]","area":"Computer Code","uses_additional_data":0,"source":"archive","tags":[]},{"id":145923,"task":"Code Completion","parent_task":null,"dataset":"SAFIM","model_name":"gpt-4-1106-preview","metrics":{"API":"62.58","Algorithmic":"42.11","Average":"53.28","Control":"55.15"},"paper_url":"https://arxiv.org/abs/2403.04814v3","paper_title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","paper_date":"2024-03-07","code_links":[{"title":"gonglinyuan/safim","url":"https://github.com/gonglinyuan/safim"}],"metrics_order":"[\"Average\", \"Algorithmic\", \"Control\", \"API\"]","area":"Computer Code","uses_additional_data":0,"source":"archive","tags":[]},{"id":145924,"task":"Code Completion","parent_task":null,"dataset":"SAFIM","model_name":"CodeLlama-13b-hf","metrics":{"API":"59.68","Algorithmic":"41.41","Average":"52.78","Control":"57.25"},"paper_url":"https://arxiv.org/abs/2403.04814v3","paper_title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","paper_date":"2024-03-07","code_links":[{"title":"gonglinyuan/safim","url":"https://github.com/gonglinyuan/safim"}],"metrics_order":"[\"Average\", \"Algorithmic\", \"Control\", \"API\"]","area":"Computer Code","uses_additional_data":0,"source":"archive","tags":[]},{"id":145925,"task":"Code Completion","parent_task":null,"dataset":"SAFIM","model_name":"deepseek-coder-1.3b-base","metrics":{"API":"62.58","Algorithmic":"41.20","Average":"52.63","Control":"54.10"},"paper_url":"https://arxiv.org/abs/2403.04814v3","paper_title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","paper_date":"2024-03-07","code_links":[{"title":"gonglinyuan/safim","url":"https://github.com/gonglinyuan/safim"}],"metrics_order":"[\"Average\", \"Algorithmic\", \"Control\", \"API\"]","area":"Computer Code","uses_additional_data":0,"source":"archive","tags":[]},{"id":145926,"task":"Code Completion","parent_task":null,"dataset":"SAFIM","model_name":"CodeLlama-34b-hf","metrics":{"API":"56.45","Algorithmic":"38.55","Average":"49.66","Control":"53.98"},"paper_url":"https://arxiv.org/abs/2403.04814v3","paper_title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","paper_date":"2024-03-07","code_links":[{"title":"gonglinyuan/safim","url":"https://github.com/gonglinyuan/safim"}],"metrics_order":"[\"Average\", \"Algorithmic\", \"Control\", \"API\"]","area":"Computer Code","uses_additional_data":0,"source":"archive","tags":[]},{"id":145927,"task":"Code Completion","parent_task":null,"dataset":"SAFIM","model_name":"CodeLlama-7b-hf","metrics":{"API":"46.77","Algorithmic":"34.68","Average":"45.00","Control":"53.56"},"paper_url":"https://arxiv.org/abs/2403.04814v3","paper_title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","paper_date":"2024-03-07","code_links":[{"title":"gonglinyuan/safim","url":"https://github.com/gonglinyuan/safim"}],"metrics_order":"[\"Average\", \"Algorithmic\", \"Control\", \"API\"]","area":"Computer Code","uses_additional_data":0,"source":"archive","tags":[]},{"id":145928,"task":"Code Completion","parent_task":null,"dataset":"SAFIM","model_name":"gpt-3.5-turbo-0301","metrics":{"API":"53.87","Algorithmic":"31.24","Average":"40.86","Control":"37.48"},"paper_url":"https://arxiv.org/abs/2403.04814v3","paper_title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","paper_date":"2024-03-07","code_links":[{"title":"gonglinyuan/safim","url":"https://github.com/gonglinyuan/safim"}],"metrics_order":"[\"Average\", \"Algorithmic\", \"Control\", \"API\"]","area":"Computer Code","uses_additional_data":0,"source":"archive","tags":[]},{"id":145929,"task":"Code Completion","parent_task":null,"dataset":"SAFIM","model_name":"incoder-6B","metrics":{"API":"48.06","Algorithmic":"25.16","Average":"33.79","Control":"28.16"},"paper_url":"https://arxiv.org/abs/2403.04814v3","paper_title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","paper_date":"2024-03-07","code_links":[{"title":"gonglinyuan/safim","url":"https://github.com/gonglinyuan/safim"}],"metrics_order":"[\"Average\", \"Algorithmic\", \"Control\", \"API\"]","area":"Computer Code","uses_additional_data":0,"source":"archive","tags":[]},{"id":145930,"task":"Code Completion","parent_task":null,"dataset":"SAFIM","model_name":"codegen-16B-multi","metrics":{"API":"31.29","Algorithmic":"25.94","Average":"30.99","Control":"35.74"},"paper_url":"https://arxiv.org/abs/2403.04814v3","paper_title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","paper_date":"2024-03-07","code_links":[{"title":"gonglinyuan/safim","url":"https://github.com/gonglinyuan/safim"}],"metrics_order":"[\"Average\", \"Algorithmic\", \"Control\", \"API\"]","area":"Computer Code","uses_additional_data":0,"source":"archive","tags":[]},{"id":145931,"task":"Code Completion","parent_task":null,"dataset":"SAFIM","model_name":"codegen-2B-multi","metrics":{"API":"32.26","Algorithmic":"23.49","Average":"29.55","Control":"32.89"},"paper_url":"https://arxiv.org/abs/2403.04814v3","paper_title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","paper_date":"2024-03-07","code_links":[{"title":"gonglinyuan/safim","url":"https://github.com/gonglinyuan/safim"}],"metrics_order":"[\"Average\", \"Algorithmic\", \"Control\", \"API\"]","area":"Computer Code","uses_additional_data":0,"source":"archive","tags":[]},{"id":145932,"task":"Code Completion","parent_task":null,"dataset":"SAFIM","model_name":"incoder-1B","metrics":{"API":"43.87","Algorithmic":"21.06","Average":"29.27","Control":"22.89"},"paper_url":"https://arxiv.org/abs/2403.04814v3","paper_title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","paper_date":"2024-03-07","code_links":[{"title":"gonglinyuan/safim","url":"https://github.com/gonglinyuan/safim"}],"metrics_order":"[\"Average\", \"Algorithmic\", \"Control\", \"API\"]","area":"Computer Code","uses_additional_data":0,"source":"archive","tags":[]},{"id":145933,"task":"Code Completion","parent_task":null,"dataset":"SAFIM","model_name":"codegen-6B-multi","metrics":{"API":"27.74","Algorithmic":"23.60","Average":"28.71","Control":"34.80"},"paper_url":"https://arxiv.org/abs/2403.04814v3","paper_title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","paper_date":"2024-03-07","code_links":[{"title":"gonglinyuan/safim","url":"https://github.com/gonglinyuan/safim"}],"metrics_order":"[\"Average\", \"Algorithmic\", \"Control\", \"API\"]","area":"Computer Code","uses_additional_data":0,"source":"archive","tags":[]},{"id":145934,"task":"Code Completion","parent_task":null,"dataset":"SAFIM","model_name":"codegen-350M-multi","metrics":{"API":"26.45","Algorithmic":"16.30","Average":"22.94","Control":"26.06"},"paper_url":"https://arxiv.org/abs/2403.04814v3","paper_title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","paper_date":"2024-03-07","code_links":[{"title":"gonglinyuan/safim","url":"https://github.com/gonglinyuan/safim"}],"metrics_order":"[\"Average\", \"Algorithmic\", \"Control\", \"API\"]","area":"Computer Code","uses_additional_data":0,"source":"archive","tags":[]}]}