{"task":"OpenAPI code completion","dataset":"OpenAPI completion refined","metric_names":["Correctness, avg., %","Correctness, max., %","Validness, avg., %","Validness, max., %"],"rows":[{"id":145935,"task":"OpenAPI code completion","parent_task":"Code Completion","dataset":"OpenAPI completion refined","model_name":"Code Llama 7B, fine-tuned with document splitting","metrics":{"Correctness, avg., %":"34","Correctness, max., %":"42","Validness, avg., %":"69.1","Validness, max., %":"76"},"paper_url":"https://arxiv.org/abs/2405.15729v2","paper_title":"Optimizing Large Language Models for OpenAPI Code Completion","paper_date":"2024-05-24","code_links":[{"title":"BohdanPetryshyn/code-llama-fim-fine-tuning","url":"https://github.com/BohdanPetryshyn/code-llama-fim-fine-tuning"},{"title":"BohdanPetryshyn/openapi-completion-benchmark","url":"https://github.com/BohdanPetryshyn/openapi-completion-benchmark"}],"metrics_order":"[\"Correctness, avg., %\", \"Correctness, max., %\", \"Validness, avg., %\", \"Validness, max., %\"]","area":"Computer Code","uses_additional_data":0,"source":"archive","tags":[]},{"id":145936,"task":"OpenAPI code completion","parent_task":"Code Completion","dataset":"OpenAPI completion refined","model_name":"Code Llama 7B, fine-tuned at 4096 tokens","metrics":{"Correctness, avg., %":"32","Correctness, max., %":"45","Validness, avg., %":"63.1","Validness, max., %":"84"},"paper_url":"https://arxiv.org/abs/2405.15729v2","paper_title":"Optimizing Large Language Models for OpenAPI Code Completion","paper_date":"2024-05-24","code_links":[{"title":"BohdanPetryshyn/code-llama-fim-fine-tuning","url":"https://github.com/BohdanPetryshyn/code-llama-fim-fine-tuning"},{"title":"BohdanPetryshyn/openapi-completion-benchmark","url":"https://github.com/BohdanPetryshyn/openapi-completion-benchmark"}],"metrics_order":"[\"Correctness, avg., %\", \"Correctness, max., %\", \"Validness, avg., %\", \"Validness, max., %\"]","area":"Computer Code","uses_additional_data":0,"source":"archive","tags":[]},{"id":145937,"task":"OpenAPI code completion","parent_task":"Code Completion","dataset":"OpenAPI completion refined","model_name":"Code Llama 7B","metrics":{"Correctness, avg., %":"31.1","Correctness, max., %":"36","Validness, avg., %":"60.7","Validness, max., %":"64"},"paper_url":"https://arxiv.org/abs/2405.15729v2","paper_title":"Optimizing Large Language Models for OpenAPI Code Completion","paper_date":"2024-05-24","code_links":[{"title":"BohdanPetryshyn/code-llama-fim-fine-tuning","url":"https://github.com/BohdanPetryshyn/code-llama-fim-fine-tuning"},{"title":"BohdanPetryshyn/openapi-completion-benchmark","url":"https://github.com/BohdanPetryshyn/openapi-completion-benchmark"}],"metrics_order":"[\"Correctness, avg., %\", \"Correctness, max., %\", \"Validness, avg., %\", \"Validness, max., %\"]","area":"Computer Code","uses_additional_data":0,"source":"archive","tags":[]},{"id":145938,"task":"OpenAPI code completion","parent_task":"Code Completion","dataset":"OpenAPI completion refined","model_name":"GitHub Copilot","metrics":{"Correctness, avg., %":"29","Correctness, max., %":"29","Validness, avg., %":"68","Validness, max., %":"68"},"paper_url":"https://arxiv.org/abs/2405.15729v2","paper_title":"Optimizing Large Language Models for OpenAPI Code Completion","paper_date":"2024-05-24","code_links":[{"title":"BohdanPetryshyn/code-llama-fim-fine-tuning","url":"https://github.com/BohdanPetryshyn/code-llama-fim-fine-tuning"},{"title":"BohdanPetryshyn/openapi-completion-benchmark","url":"https://github.com/BohdanPetryshyn/openapi-completion-benchmark"}],"metrics_order":"[\"Correctness, avg., %\", \"Correctness, max., %\", \"Validness, avg., %\", \"Validness, max., %\"]","area":"Computer Code","uses_additional_data":0,"source":"archive","tags":[]},{"id":146515,"task":"OpenAPI code completion","parent_task":null,"dataset":"OpenAPI completion refined","model_name":"Code Llama 7B, fine-tuned with document splitting","metrics":{"Correctness, avg., %":"34","Correctness, max., %":"42","Validness, avg., %":"69.1","Validness, max., %":"76"},"paper_url":"https://arxiv.org/abs/2405.15729v2","paper_title":"Optimizing Large Language Models for OpenAPI Code Completion","paper_date":"2024-05-24","code_links":[{"title":"BohdanPetryshyn/code-llama-fim-fine-tuning","url":"https://github.com/BohdanPetryshyn/code-llama-fim-fine-tuning"},{"title":"BohdanPetryshyn/openapi-completion-benchmark","url":"https://github.com/BohdanPetryshyn/openapi-completion-benchmark"}],"metrics_order":"[\"Correctness, avg., %\", \"Correctness, max., %\", \"Validness, avg., %\", \"Validness, max., %\"]","area":"Computer Code","uses_additional_data":0,"source":"archive","tags":[]},{"id":146516,"task":"OpenAPI code completion","parent_task":null,"dataset":"OpenAPI completion refined","model_name":"Code Llama 7B, fine-tuned at 4096 tokens","metrics":{"Correctness, avg., %":"32","Correctness, max., %":"45","Validness, avg., %":"63.1","Validness, max., %":"84"},"paper_url":"https://arxiv.org/abs/2405.15729v2","paper_title":"Optimizing Large Language Models for OpenAPI Code Completion","paper_date":"2024-05-24","code_links":[{"title":"BohdanPetryshyn/code-llama-fim-fine-tuning","url":"https://github.com/BohdanPetryshyn/code-llama-fim-fine-tuning"},{"title":"BohdanPetryshyn/openapi-completion-benchmark","url":"https://github.com/BohdanPetryshyn/openapi-completion-benchmark"}],"metrics_order":"[\"Correctness, avg., %\", \"Correctness, max., %\", \"Validness, avg., %\", \"Validness, max., %\"]","area":"Computer Code","uses_additional_data":0,"source":"archive","tags":[]},{"id":146517,"task":"OpenAPI code completion","parent_task":null,"dataset":"OpenAPI completion refined","model_name":"Code Llama 7B","metrics":{"Correctness, avg., %":"31.1","Correctness, max., %":"36","Validness, avg., %":"60.7","Validness, max., %":"64"},"paper_url":"https://arxiv.org/abs/2405.15729v2","paper_title":"Optimizing Large Language Models for OpenAPI Code Completion","paper_date":"2024-05-24","code_links":[{"title":"BohdanPetryshyn/code-llama-fim-fine-tuning","url":"https://github.com/BohdanPetryshyn/code-llama-fim-fine-tuning"},{"title":"BohdanPetryshyn/openapi-completion-benchmark","url":"https://github.com/BohdanPetryshyn/openapi-completion-benchmark"}],"metrics_order":"[\"Correctness, avg., %\", \"Correctness, max., %\", \"Validness, avg., %\", \"Validness, max., %\"]","area":"Computer Code","uses_additional_data":0,"source":"archive","tags":[]},{"id":146518,"task":"OpenAPI code completion","parent_task":null,"dataset":"OpenAPI completion refined","model_name":"GitHub Copilot","metrics":{"Correctness, avg., %":"29","Correctness, max., %":"29","Validness, avg., %":"68","Validness, max., %":"68"},"paper_url":"https://arxiv.org/abs/2405.15729v2","paper_title":"Optimizing Large Language Models for OpenAPI Code Completion","paper_date":"2024-05-24","code_links":[{"title":"BohdanPetryshyn/code-llama-fim-fine-tuning","url":"https://github.com/BohdanPetryshyn/code-llama-fim-fine-tuning"},{"title":"BohdanPetryshyn/openapi-completion-benchmark","url":"https://github.com/BohdanPetryshyn/openapi-completion-benchmark"}],"metrics_order":"[\"Correctness, avg., %\", \"Correctness, max., %\", \"Validness, avg., %\", \"Validness, max., %\"]","area":"Computer Code","uses_additional_data":0,"source":"archive","tags":[]}]}