all_data_tasks/46/default.jsonl

{"index":0,"Rank":1,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/GritLM\/GritLM-7B\">GritLM-7B<\/a>","Model Size (Million Parameters)":7240,"Memory Usage (GB, fp32)":26.97,"Average":75.86,"AppsRetrieval (eng-Latn_python-Code)":35.13,"CodeFeedbackMT (c-Code_sql-Code_python-Code_shell-Code_swift-Code_eng-Latn)":50.64,"CodeFeedbackST (python-Code_javascript-Code_go-Code_ruby-Code_java-Code_php-Code_eng-Latn)":84.08,"CodeSearchNetCCRetrieval (go-Code)":85.44,"CodeSearchNetCCRetrieval (java-Code)":87.63,"CodeSearchNetCCRetrieval (javascript-Code)":87.27,"CodeSearchNetCCRetrieval (php-Code)":79.25,"CodeSearchNetCCRetrieval (python-Code)":90.92,"CodeSearchNetCCRetrieval (ruby-Code)":89.63,"CodeSearchNetRetrieval (go-Code)":90.01,"CodeSearchNetRetrieval (java-Code)":89.84,"CodeSearchNetRetrieval (javascript-Code)":81.43,"CodeSearchNetRetrieval (php-Code)":81.57,"CodeSearchNetRetrieval (python-Code)":91.65,"CodeSearchNetRetrieval (ruby-Code)":85.42,"CodeTransOceanContest (python-Code_c++-Code)":89.22,"CodeTransOceanDL":32.98,"CosQA (eng-Latn_python-Code)":31.24,"StackOverflowQA":93.37,"SyntheticText2SQL (eng-Latn_sql-Code)":60.39}
{"index":1,"Rank":2,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/intfloat\/e5-mistral-7b-instruct\">e5-mistral-7b-instruct<\/a>","Model Size (Million Parameters)":7111,"Memory Usage (GB, fp32)":26.49,"Average":73.36,"AppsRetrieval (eng-Latn_python-Code)":23.46,"CodeFeedbackMT (c-Code_sql-Code_python-Code_shell-Code_swift-Code_eng-Latn)":36.4,"CodeFeedbackST (python-Code_javascript-Code_go-Code_ruby-Code_java-Code_php-Code_eng-Latn)":76.41,"CodeSearchNetCCRetrieval (go-Code)":84.05,"CodeSearchNetCCRetrieval (java-Code)":86.21,"CodeSearchNetCCRetrieval (javascript-Code)":86.18,"CodeSearchNetCCRetrieval (php-Code)":75.95,"CodeSearchNetCCRetrieval (python-Code)":90.6,"CodeSearchNetCCRetrieval (ruby-Code)":85.89,"CodeSearchNetRetrieval (go-Code)":93.06,"CodeSearchNetRetrieval (java-Code)":84.08,"CodeSearchNetRetrieval (javascript-Code)":80.93,"CodeSearchNetRetrieval (php-Code)":83.14,"CodeSearchNetRetrieval (python-Code)":91.75,"CodeSearchNetRetrieval (ruby-Code)":85.37,"CodeTransOceanContest (python-Code_c++-Code)":88.58,"CodeTransOceanDL":31.74,"CosQA (eng-Latn_python-Code)":33.1,"StackOverflowQA":91.02,"SyntheticText2SQL (eng-Latn_sql-Code)":59.2}
{"index":3,"Rank":3,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/intfloat\/multilingual-e5-large\">multilingual-e5-large<\/a>","Model Size (Million Parameters)":560,"Memory Usage (GB, fp32)":2.09,"Average":70.17,"AppsRetrieval (eng-Latn_python-Code)":32.55,"CodeFeedbackMT (c-Code_sql-Code_python-Code_shell-Code_swift-Code_eng-Latn)":42.78,"CodeFeedbackST (python-Code_javascript-Code_go-Code_ruby-Code_java-Code_php-Code_eng-Latn)":74.26,"CodeSearchNetCCRetrieval (go-Code)":72.08,"CodeSearchNetCCRetrieval (java-Code)":78.65,"CodeSearchNetCCRetrieval (javascript-Code)":77.67,"CodeSearchNetCCRetrieval (php-Code)":72.21,"CodeSearchNetCCRetrieval (python-Code)":84.45,"CodeSearchNetCCRetrieval (ruby-Code)":81.94,"CodeSearchNetRetrieval (go-Code)":91.8,"CodeSearchNetRetrieval (java-Code)":82.05,"CodeSearchNetRetrieval (javascript-Code)":75.54,"CodeSearchNetRetrieval (php-Code)":84.5,"CodeSearchNetRetrieval (python-Code)":89.42,"CodeSearchNetRetrieval (ruby-Code)":81.43,"CodeTransOceanContest (python-Code_c++-Code)":74.03,"CodeTransOceanDL":31.28,"CosQA (eng-Latn_python-Code)":34.8,"StackOverflowQA":88.89,"SyntheticText2SQL (eng-Latn_sql-Code)":53.07}
{"index":2,"Rank":4,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/intfloat\/multilingual-e5-base\">multilingual-e5-base<\/a>","Model Size (Million Parameters)":278,"Memory Usage (GB, fp32)":1.04,"Average":67.07,"AppsRetrieval (eng-Latn_python-Code)":20.94,"CodeFeedbackMT (c-Code_sql-Code_python-Code_shell-Code_swift-Code_eng-Latn)":43.18,"CodeFeedbackST (python-Code_javascript-Code_go-Code_ruby-Code_java-Code_php-Code_eng-Latn)":72.61,"CodeSearchNetCCRetrieval (go-Code)":72.29,"CodeSearchNetCCRetrieval (java-Code)":78.85,"CodeSearchNetCCRetrieval (javascript-Code)":78.44,"CodeSearchNetCCRetrieval (php-Code)":72.02,"CodeSearchNetCCRetrieval (python-Code)":85.37,"CodeSearchNetCCRetrieval (ruby-Code)":81.92,"CodeSearchNetRetrieval (go-Code)":89.47,"CodeSearchNetRetrieval (java-Code)":78.39,"CodeSearchNetRetrieval (javascript-Code)":71.4,"CodeSearchNetRetrieval (php-Code)":81.05,"CodeSearchNetRetrieval (python-Code)":85.68,"CodeSearchNetRetrieval (ruby-Code)":78.24,"CodeTransOceanContest (python-Code_c++-Code)":51.89,"CodeTransOceanDL":29.88,"CosQA (eng-Latn_python-Code)":31.12,"StackOverflowQA":85.11,"SyntheticText2SQL (eng-Latn_sql-Code)":53.61}
{"index":4,"Rank":5,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/intfloat\/multilingual-e5-small\">multilingual-e5-small<\/a>","Model Size (Million Parameters)":118,"Memory Usage (GB, fp32)":0.44,"Average":65.48,"AppsRetrieval (eng-Latn_python-Code)":12.01,"CodeFeedbackMT (c-Code_sql-Code_python-Code_shell-Code_swift-Code_eng-Latn)":41.32,"CodeFeedbackST (python-Code_javascript-Code_go-Code_ruby-Code_java-Code_php-Code_eng-Latn)":72.67,"CodeSearchNetCCRetrieval (go-Code)":69.66,"CodeSearchNetCCRetrieval (java-Code)":75.58,"CodeSearchNetCCRetrieval (javascript-Code)":77.6,"CodeSearchNetCCRetrieval (php-Code)":69.38,"CodeSearchNetCCRetrieval (python-Code)":80.75,"CodeSearchNetCCRetrieval (ruby-Code)":72.75,"CodeSearchNetRetrieval (go-Code)":90.3,"CodeSearchNetRetrieval (java-Code)":74.96,"CodeSearchNetRetrieval (javascript-Code)":69.53,"CodeSearchNetRetrieval (php-Code)":80.28,"CodeSearchNetRetrieval (python-Code)":86.31,"CodeSearchNetRetrieval (ruby-Code)":76.58,"CodeTransOceanContest (python-Code_c++-Code)":69.76,"CodeTransOceanDL":32.3,"CosQA (eng-Latn_python-Code)":29.61,"StackOverflowQA":81.94,"SyntheticText2SQL (eng-Latn_sql-Code)":46.29}
{"index":8,"Rank":6,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/sentence-transformers\/all-mpnet-base-v2\">all-mpnet-base-v2<\/a>","Model Size (Million Parameters)":110,"Memory Usage (GB, fp32)":0.41,"Average":61.47,"AppsRetrieval (eng-Latn_python-Code)":8.41,"CodeFeedbackMT (c-Code_sql-Code_python-Code_shell-Code_swift-Code_eng-Latn)":37.72,"CodeFeedbackST (python-Code_javascript-Code_go-Code_ruby-Code_java-Code_php-Code_eng-Latn)":69.32,"CodeSearchNetCCRetrieval (go-Code)":61.44,"CodeSearchNetCCRetrieval (java-Code)":68.88,"CodeSearchNetCCRetrieval (javascript-Code)":67.99,"CodeSearchNetCCRetrieval (php-Code)":59.22,"CodeSearchNetCCRetrieval (python-Code)":71.83,"CodeSearchNetCCRetrieval (ruby-Code)":66.42,"CodeSearchNetRetrieval (go-Code)":88.25,"CodeSearchNetRetrieval (java-Code)":65.11,"CodeSearchNetRetrieval (javascript-Code)":65.58,"CodeSearchNetRetrieval (php-Code)":74.47,"CodeSearchNetRetrieval (python-Code)":81.01,"CodeSearchNetRetrieval (ruby-Code)":74.53,"CodeTransOceanContest (python-Code_c++-Code)":70.58,"CodeTransOceanDL":29.45,"CosQA (eng-Latn_python-Code)":33.71,"StackOverflowQA":90.32,"SyntheticText2SQL (eng-Latn_sql-Code)":45.09}
{"index":6,"Rank":7,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/sentence-transformers\/all-MiniLM-L12-v2\">all-MiniLM-L12-v2<\/a>","Model Size (Million Parameters)":33,"Memory Usage (GB, fp32)":0.12,"Average":58.99,"AppsRetrieval (eng-Latn_python-Code)":5.97,"CodeFeedbackMT (c-Code_sql-Code_python-Code_shell-Code_swift-Code_eng-Latn)":27.07,"CodeFeedbackST (python-Code_javascript-Code_go-Code_ruby-Code_java-Code_php-Code_eng-Latn)":61.01,"CodeSearchNetCCRetrieval (go-Code)":63.52,"CodeSearchNetCCRetrieval (java-Code)":67.51,"CodeSearchNetCCRetrieval (javascript-Code)":70.13,"CodeSearchNetCCRetrieval (php-Code)":60.81,"CodeSearchNetCCRetrieval (python-Code)":71.1,"CodeSearchNetCCRetrieval (ruby-Code)":72.28,"CodeSearchNetRetrieval (go-Code)":89.06,"CodeSearchNetRetrieval (java-Code)":53.2,"CodeSearchNetRetrieval (javascript-Code)":67.52,"CodeSearchNetRetrieval (php-Code)":77.36,"CodeSearchNetRetrieval (python-Code)":82.09,"CodeSearchNetRetrieval (ruby-Code)":74.77,"CodeTransOceanContest (python-Code_c++-Code)":59.86,"CodeTransOceanDL":19.94,"CosQA (eng-Latn_python-Code)":32.09,"StackOverflowQA":80.63,"SyntheticText2SQL (eng-Latn_sql-Code)":43.93}
{"index":7,"Rank":8,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/sentence-transformers\/all-MiniLM-L6-v2\">all-MiniLM-L6-v2<\/a>","Model Size (Million Parameters)":23,"Memory Usage (GB, fp32)":0.09,"Average":58.47,"AppsRetrieval (eng-Latn_python-Code)":6.6,"CodeFeedbackMT (c-Code_sql-Code_python-Code_shell-Code_swift-Code_eng-Latn)":35.62,"CodeFeedbackST (python-Code_javascript-Code_go-Code_ruby-Code_java-Code_php-Code_eng-Latn)":65.99,"CodeSearchNetCCRetrieval (go-Code)":60.0,"CodeSearchNetCCRetrieval (java-Code)":66.32,"CodeSearchNetCCRetrieval (javascript-Code)":66.75,"CodeSearchNetCCRetrieval (php-Code)":58.42,"CodeSearchNetCCRetrieval (python-Code)":71.29,"CodeSearchNetCCRetrieval (ruby-Code)":69.32,"CodeSearchNetRetrieval (go-Code)":79.84,"CodeSearchNetRetrieval (java-Code)":51.65,"CodeSearchNetRetrieval (javascript-Code)":63.62,"CodeSearchNetRetrieval (php-Code)":70.82,"CodeSearchNetRetrieval (python-Code)":79.28,"CodeSearchNetRetrieval (ruby-Code)":72.84,"CodeTransOceanContest (python-Code_c++-Code)":61.78,"CodeTransOceanDL":27.94,"CosQA (eng-Latn_python-Code)":33.02,"StackOverflowQA":83.96,"SyntheticText2SQL (eng-Latn_sql-Code)":44.35}
{"index":10,"Rank":9,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/sentence-transformers\/paraphrase-multilingual-mpnet-base-v2\">paraphrase-multilingual-mpnet-base-v2<\/a>","Model Size (Million Parameters)":278,"Memory Usage (GB, fp32)":1.04,"Average":39.41,"AppsRetrieval (eng-Latn_python-Code)":2.34,"CodeFeedbackMT (c-Code_sql-Code_python-Code_shell-Code_swift-Code_eng-Latn)":11.43,"CodeFeedbackST (python-Code_javascript-Code_go-Code_ruby-Code_java-Code_php-Code_eng-Latn)":35.34,"CodeSearchNetCCRetrieval (go-Code)":26.63,"CodeSearchNetCCRetrieval (java-Code)":42.1,"CodeSearchNetCCRetrieval (javascript-Code)":57.46,"CodeSearchNetCCRetrieval (php-Code)":29.8,"CodeSearchNetCCRetrieval (python-Code)":53.43,"CodeSearchNetCCRetrieval (ruby-Code)":55.02,"CodeSearchNetRetrieval (go-Code)":67.23,"CodeSearchNetRetrieval (java-Code)":45.56,"CodeSearchNetRetrieval (javascript-Code)":48.34,"CodeSearchNetRetrieval (php-Code)":48.3,"CodeSearchNetRetrieval (python-Code)":65.54,"CodeSearchNetRetrieval (ruby-Code)":60.19,"CodeTransOceanContest (python-Code_c++-Code)":27.94,"CodeTransOceanDL":16.71,"CosQA (eng-Latn_python-Code)":16.57,"StackOverflowQA":43.11,"SyntheticText2SQL (eng-Latn_sql-Code)":35.22}
{"index":5,"Rank":10,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/sentence-transformers\/LaBSE\">LaBSE<\/a>","Model Size (Million Parameters)":471,"Memory Usage (GB, fp32)":1.75,"Average":37.12,"AppsRetrieval (eng-Latn_python-Code)":2.39,"CodeFeedbackMT (c-Code_sql-Code_python-Code_shell-Code_swift-Code_eng-Latn)":17.98,"CodeFeedbackST (python-Code_javascript-Code_go-Code_ruby-Code_java-Code_php-Code_eng-Latn)":35.98,"CodeSearchNetCCRetrieval (go-Code)":41.73,"CodeSearchNetCCRetrieval (java-Code)":45.83,"CodeSearchNetCCRetrieval (javascript-Code)":52.0,"CodeSearchNetCCRetrieval (php-Code)":31.96,"CodeSearchNetCCRetrieval (python-Code)":40.41,"CodeSearchNetCCRetrieval (ruby-Code)":41.46,"CodeSearchNetRetrieval (go-Code)":48.56,"CodeSearchNetRetrieval (java-Code)":35.05,"CodeSearchNetRetrieval (javascript-Code)":46.24,"CodeSearchNetRetrieval (php-Code)":46.03,"CodeSearchNetRetrieval (python-Code)":60.52,"CodeSearchNetRetrieval (ruby-Code)":51.34,"CodeTransOceanContest (python-Code_c++-Code)":28.1,"CodeTransOceanDL":26.45,"CosQA (eng-Latn_python-Code)":8.77,"StackOverflowQA":38.23,"SyntheticText2SQL (eng-Latn_sql-Code)":43.28}
{"index":9,"Rank":11,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/sentence-transformers\/paraphrase-multilingual-MiniLM-L12-v2\">paraphrase-multilingual-MiniLM-L12-v2<\/a>","Model Size (Million Parameters)":118,"Memory Usage (GB, fp32)":0.44,"Average":33.5,"AppsRetrieval (eng-Latn_python-Code)":2.0,"CodeFeedbackMT (c-Code_sql-Code_python-Code_shell-Code_swift-Code_eng-Latn)":12.53,"CodeFeedbackST (python-Code_javascript-Code_go-Code_ruby-Code_java-Code_php-Code_eng-Latn)":32.53,"CodeSearchNetCCRetrieval (go-Code)":28.54,"CodeSearchNetCCRetrieval (java-Code)":26.23,"CodeSearchNetCCRetrieval (javascript-Code)":43.56,"CodeSearchNetCCRetrieval (php-Code)":16.3,"CodeSearchNetCCRetrieval (python-Code)":44.49,"CodeSearchNetCCRetrieval (ruby-Code)":43.84,"CodeSearchNetRetrieval (go-Code)":61.49,"CodeSearchNetRetrieval (java-Code)":32.49,"CodeSearchNetRetrieval (javascript-Code)":46.41,"CodeSearchNetRetrieval (php-Code)":45.01,"CodeSearchNetRetrieval (python-Code)":58.15,"CodeSearchNetRetrieval (ruby-Code)":56.44,"CodeTransOceanContest (python-Code_c++-Code)":25.67,"CodeTransOceanDL":11.65,"CosQA (eng-Latn_python-Code)":14.24,"StackOverflowQA":37.94,"SyntheticText2SQL (eng-Latn_sql-Code)":30.57}