boards_data/coir/data_tasks/Retrieval/default.jsonl

{"index":0,"Rank":1,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/GritLM\/GritLM-7B\">GritLM-7B<\/a>","Model Size (Million Parameters)":7240,"Memory Usage (GB, fp32)":26.97,"Embedding Dimensions":"","Max Tokens":4096,"Average":75.86,"AppsRetrieval":35.13,"CodeFeedbackMT":50.64,"CodeFeedbackST":84.08,"CodeSearchNetCCRetrieval (go)":85.44,"CodeSearchNetCCRetrieval (java)":87.63,"CodeSearchNetCCRetrieval (javascript)":87.27,"CodeSearchNetCCRetrieval (php)":79.25,"CodeSearchNetCCRetrieval (python)":90.92,"CodeSearchNetCCRetrieval (ruby)":89.63,"CodeSearchNetRetrieval (go)":90.01,"CodeSearchNetRetrieval (java)":89.84,"CodeSearchNetRetrieval (javascript)":81.43,"CodeSearchNetRetrieval (php)":81.57,"CodeSearchNetRetrieval (python)":91.65,"CodeSearchNetRetrieval (ruby)":85.42,"CodeTransOceanContest":89.22,"CodeTransOceanDL":32.98,"CosQA":31.24,"StackOverflowQA":93.37,"SyntheticText2SQL":60.39}
{"index":1,"Rank":2,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/intfloat\/e5-mistral-7b-instruct\">e5-mistral-7b-instruct<\/a>","Model Size (Million Parameters)":7111,"Memory Usage (GB, fp32)":26.49,"Embedding Dimensions":4096,"Max Tokens":32768,"Average":73.36,"AppsRetrieval":23.46,"CodeFeedbackMT":36.4,"CodeFeedbackST":76.41,"CodeSearchNetCCRetrieval (go)":84.05,"CodeSearchNetCCRetrieval (java)":86.21,"CodeSearchNetCCRetrieval (javascript)":86.18,"CodeSearchNetCCRetrieval (php)":75.95,"CodeSearchNetCCRetrieval (python)":90.6,"CodeSearchNetCCRetrieval (ruby)":85.89,"CodeSearchNetRetrieval (go)":93.06,"CodeSearchNetRetrieval (java)":84.08,"CodeSearchNetRetrieval (javascript)":80.93,"CodeSearchNetRetrieval (php)":83.14,"CodeSearchNetRetrieval (python)":91.75,"CodeSearchNetRetrieval (ruby)":85.37,"CodeTransOceanContest":88.58,"CodeTransOceanDL":31.74,"CosQA":33.1,"StackOverflowQA":91.02,"SyntheticText2SQL":59.2}
{"index":4,"Rank":3,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/intfloat\/multilingual-e5-large-instruct\">multilingual-e5-large-instruct<\/a>","Model Size (Million Parameters)":560,"Memory Usage (GB, fp32)":2.09,"Embedding Dimensions":1024,"Max Tokens":514,"Average":70.81,"AppsRetrieval":34.89,"CodeFeedbackMT":39.87,"CodeFeedbackST":75.89,"CodeSearchNetCCRetrieval (go)":72.31,"CodeSearchNetCCRetrieval (java)":80.07,"CodeSearchNetCCRetrieval (javascript)":79.2,"CodeSearchNetCCRetrieval (php)":72.49,"CodeSearchNetCCRetrieval (python)":86.24,"CodeSearchNetCCRetrieval (ruby)":81.9,"CodeSearchNetRetrieval (go)":90.41,"CodeSearchNetRetrieval (java)":80.55,"CodeSearchNetRetrieval (javascript)":75.37,"CodeSearchNetRetrieval (php)":83.7,"CodeSearchNetRetrieval (python)":88.96,"CodeSearchNetRetrieval (ruby)":79.56,"CodeTransOceanContest":83.36,"CodeTransOceanDL":29.01,"CosQA":37.75,"StackOverflowQA":85.82,"SyntheticText2SQL":58.85}
{"index":3,"Rank":4,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/intfloat\/multilingual-e5-large\">multilingual-e5-large<\/a>","Model Size (Million Parameters)":560,"Memory Usage (GB, fp32)":2.09,"Embedding Dimensions":1024,"Max Tokens":514,"Average":70.17,"AppsRetrieval":32.55,"CodeFeedbackMT":42.78,"CodeFeedbackST":74.26,"CodeSearchNetCCRetrieval (go)":72.08,"CodeSearchNetCCRetrieval (java)":78.65,"CodeSearchNetCCRetrieval (javascript)":77.67,"CodeSearchNetCCRetrieval (php)":72.21,"CodeSearchNetCCRetrieval (python)":84.45,"CodeSearchNetCCRetrieval (ruby)":81.94,"CodeSearchNetRetrieval (go)":91.8,"CodeSearchNetRetrieval (java)":82.05,"CodeSearchNetRetrieval (javascript)":75.54,"CodeSearchNetRetrieval (php)":84.5,"CodeSearchNetRetrieval (python)":89.42,"CodeSearchNetRetrieval (ruby)":81.43,"CodeTransOceanContest":74.03,"CodeTransOceanDL":31.28,"CosQA":34.8,"StackOverflowQA":88.89,"SyntheticText2SQL":53.07}
{"index":2,"Rank":5,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/intfloat\/multilingual-e5-base\">multilingual-e5-base<\/a>","Model Size (Million Parameters)":278,"Memory Usage (GB, fp32)":1.04,"Embedding Dimensions":768,"Max Tokens":514,"Average":67.07,"AppsRetrieval":20.94,"CodeFeedbackMT":43.18,"CodeFeedbackST":72.61,"CodeSearchNetCCRetrieval (go)":72.29,"CodeSearchNetCCRetrieval (java)":78.85,"CodeSearchNetCCRetrieval (javascript)":78.44,"CodeSearchNetCCRetrieval (php)":72.02,"CodeSearchNetCCRetrieval (python)":85.37,"CodeSearchNetCCRetrieval (ruby)":81.92,"CodeSearchNetRetrieval (go)":89.47,"CodeSearchNetRetrieval (java)":78.39,"CodeSearchNetRetrieval (javascript)":71.4,"CodeSearchNetRetrieval (php)":81.05,"CodeSearchNetRetrieval (python)":85.68,"CodeSearchNetRetrieval (ruby)":78.24,"CodeTransOceanContest":51.89,"CodeTransOceanDL":29.88,"CosQA":31.12,"StackOverflowQA":85.11,"SyntheticText2SQL":53.61}
{"index":5,"Rank":6,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/intfloat\/multilingual-e5-small\">multilingual-e5-small<\/a>","Model Size (Million Parameters)":118,"Memory Usage (GB, fp32)":0.44,"Embedding Dimensions":384,"Max Tokens":512,"Average":65.48,"AppsRetrieval":12.01,"CodeFeedbackMT":41.32,"CodeFeedbackST":72.67,"CodeSearchNetCCRetrieval (go)":69.66,"CodeSearchNetCCRetrieval (java)":75.58,"CodeSearchNetCCRetrieval (javascript)":77.6,"CodeSearchNetCCRetrieval (php)":69.38,"CodeSearchNetCCRetrieval (python)":80.75,"CodeSearchNetCCRetrieval (ruby)":72.75,"CodeSearchNetRetrieval (go)":90.3,"CodeSearchNetRetrieval (java)":74.96,"CodeSearchNetRetrieval (javascript)":69.53,"CodeSearchNetRetrieval (php)":80.28,"CodeSearchNetRetrieval (python)":86.31,"CodeSearchNetRetrieval (ruby)":76.58,"CodeTransOceanContest":69.76,"CodeTransOceanDL":32.3,"CosQA":29.61,"StackOverflowQA":81.94,"SyntheticText2SQL":46.29}
{"index":9,"Rank":7,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/sentence-transformers\/all-mpnet-base-v2\">all-mpnet-base-v2<\/a>","Model Size (Million Parameters)":110,"Memory Usage (GB, fp32)":0.41,"Embedding Dimensions":768,"Max Tokens":514,"Average":61.47,"AppsRetrieval":8.41,"CodeFeedbackMT":37.72,"CodeFeedbackST":69.32,"CodeSearchNetCCRetrieval (go)":61.44,"CodeSearchNetCCRetrieval (java)":68.88,"CodeSearchNetCCRetrieval (javascript)":67.99,"CodeSearchNetCCRetrieval (php)":59.22,"CodeSearchNetCCRetrieval (python)":71.83,"CodeSearchNetCCRetrieval (ruby)":66.42,"CodeSearchNetRetrieval (go)":88.25,"CodeSearchNetRetrieval (java)":65.11,"CodeSearchNetRetrieval (javascript)":65.58,"CodeSearchNetRetrieval (php)":74.47,"CodeSearchNetRetrieval (python)":81.01,"CodeSearchNetRetrieval (ruby)":74.53,"CodeTransOceanContest":70.58,"CodeTransOceanDL":29.45,"CosQA":33.71,"StackOverflowQA":90.32,"SyntheticText2SQL":45.09}
{"index":7,"Rank":8,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/sentence-transformers\/all-MiniLM-L12-v2\">all-MiniLM-L12-v2<\/a>","Model Size (Million Parameters)":33,"Memory Usage (GB, fp32)":0.12,"Embedding Dimensions":384,"Max Tokens":512,"Average":58.99,"AppsRetrieval":5.97,"CodeFeedbackMT":27.07,"CodeFeedbackST":61.01,"CodeSearchNetCCRetrieval (go)":63.52,"CodeSearchNetCCRetrieval (java)":67.51,"CodeSearchNetCCRetrieval (javascript)":70.13,"CodeSearchNetCCRetrieval (php)":60.81,"CodeSearchNetCCRetrieval (python)":71.1,"CodeSearchNetCCRetrieval (ruby)":72.28,"CodeSearchNetRetrieval (go)":89.06,"CodeSearchNetRetrieval (java)":53.2,"CodeSearchNetRetrieval (javascript)":67.52,"CodeSearchNetRetrieval (php)":77.36,"CodeSearchNetRetrieval (python)":82.09,"CodeSearchNetRetrieval (ruby)":74.77,"CodeTransOceanContest":59.86,"CodeTransOceanDL":19.94,"CosQA":32.09,"StackOverflowQA":80.63,"SyntheticText2SQL":43.93}
{"index":8,"Rank":9,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/sentence-transformers\/all-MiniLM-L6-v2\">all-MiniLM-L6-v2<\/a>","Model Size (Million Parameters)":23,"Memory Usage (GB, fp32)":0.09,"Embedding Dimensions":384,"Max Tokens":512,"Average":58.47,"AppsRetrieval":6.6,"CodeFeedbackMT":35.62,"CodeFeedbackST":65.99,"CodeSearchNetCCRetrieval (go)":60.0,"CodeSearchNetCCRetrieval (java)":66.32,"CodeSearchNetCCRetrieval (javascript)":66.75,"CodeSearchNetCCRetrieval (php)":58.42,"CodeSearchNetCCRetrieval (python)":71.29,"CodeSearchNetCCRetrieval (ruby)":69.32,"CodeSearchNetRetrieval (go)":79.84,"CodeSearchNetRetrieval (java)":51.65,"CodeSearchNetRetrieval (javascript)":63.62,"CodeSearchNetRetrieval (php)":70.82,"CodeSearchNetRetrieval (python)":79.28,"CodeSearchNetRetrieval (ruby)":72.84,"CodeTransOceanContest":61.78,"CodeTransOceanDL":27.94,"CosQA":33.02,"StackOverflowQA":83.96,"SyntheticText2SQL":44.35}
{"index":11,"Rank":10,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/sentence-transformers\/paraphrase-multilingual-mpnet-base-v2\">paraphrase-multilingual-mpnet-base-v2<\/a>","Model Size (Million Parameters)":278,"Memory Usage (GB, fp32)":1.04,"Embedding Dimensions":768,"Max Tokens":514,"Average":39.41,"AppsRetrieval":2.34,"CodeFeedbackMT":11.43,"CodeFeedbackST":35.34,"CodeSearchNetCCRetrieval (go)":26.63,"CodeSearchNetCCRetrieval (java)":42.1,"CodeSearchNetCCRetrieval (javascript)":57.46,"CodeSearchNetCCRetrieval (php)":29.8,"CodeSearchNetCCRetrieval (python)":53.43,"CodeSearchNetCCRetrieval (ruby)":55.02,"CodeSearchNetRetrieval (go)":67.23,"CodeSearchNetRetrieval (java)":45.56,"CodeSearchNetRetrieval (javascript)":48.34,"CodeSearchNetRetrieval (php)":48.3,"CodeSearchNetRetrieval (python)":65.54,"CodeSearchNetRetrieval (ruby)":60.19,"CodeTransOceanContest":27.94,"CodeTransOceanDL":16.71,"CosQA":16.57,"StackOverflowQA":43.11,"SyntheticText2SQL":35.22}
{"index":6,"Rank":11,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/sentence-transformers\/LaBSE\">LaBSE<\/a>","Model Size (Million Parameters)":471,"Memory Usage (GB, fp32)":1.75,"Embedding Dimensions":768,"Max Tokens":512,"Average":37.12,"AppsRetrieval":2.39,"CodeFeedbackMT":17.98,"CodeFeedbackST":35.98,"CodeSearchNetCCRetrieval (go)":41.73,"CodeSearchNetCCRetrieval (java)":45.83,"CodeSearchNetCCRetrieval (javascript)":52.0,"CodeSearchNetCCRetrieval (php)":31.96,"CodeSearchNetCCRetrieval (python)":40.41,"CodeSearchNetCCRetrieval (ruby)":41.46,"CodeSearchNetRetrieval (go)":48.56,"CodeSearchNetRetrieval (java)":35.05,"CodeSearchNetRetrieval (javascript)":46.24,"CodeSearchNetRetrieval (php)":46.03,"CodeSearchNetRetrieval (python)":60.52,"CodeSearchNetRetrieval (ruby)":51.34,"CodeTransOceanContest":28.1,"CodeTransOceanDL":26.45,"CosQA":8.77,"StackOverflowQA":38.23,"SyntheticText2SQL":43.28}
{"index":10,"Rank":12,"Model":"<a target=\"_blank\" style=\"text-decoration: underline\" href=\"https:\/\/huggingface.co\/sentence-transformers\/paraphrase-multilingual-MiniLM-L12-v2\">paraphrase-multilingual-MiniLM-L12-v2<\/a>","Model Size (Million Parameters)":118,"Memory Usage (GB, fp32)":0.44,"Embedding Dimensions":384,"Max Tokens":512,"Average":33.5,"AppsRetrieval":2.0,"CodeFeedbackMT":12.53,"CodeFeedbackST":32.53,"CodeSearchNetCCRetrieval (go)":28.54,"CodeSearchNetCCRetrieval (java)":26.23,"CodeSearchNetCCRetrieval (javascript)":43.56,"CodeSearchNetCCRetrieval (php)":16.3,"CodeSearchNetCCRetrieval (python)":44.49,"CodeSearchNetCCRetrieval (ruby)":43.84,"CodeSearchNetRetrieval (go)":61.49,"CodeSearchNetRetrieval (java)":32.49,"CodeSearchNetRetrieval (javascript)":46.41,"CodeSearchNetRetrieval (php)":45.01,"CodeSearchNetRetrieval (python)":58.15,"CodeSearchNetRetrieval (ruby)":56.44,"CodeTransOceanContest":25.67,"CodeTransOceanDL":11.65,"CosQA":14.24,"StackOverflowQA":37.94,"SyntheticText2SQL":30.57}