diff --git "a/README.MD" "b/README.MD" new file mode 100644--- /dev/null +++ "b/README.MD" @@ -0,0 +1,9233 @@ +--- +pipeline_tag: sentence-similarity +tags: +- sentence-transformers +- feature-extraction +- sentence-similarity +- mteb +- arctic +- snowflake-arctic-embed +- transformers.js +license: apache-2.0 +language: +- af +- ar +- az +- be +- bg +- bn +- ca +- ceb +- cs +- cy +- da +- de +- el +- en +- es +- et +- eu +- fa +- fi +- fr +- gl +- gu +- he +- hi +- hr +- ht +- hu +- hy +- id +- is +- it +- ja +- jv +- ka +- kk +- km +- kn +- ko +- ky +- lo +- lt +- lv +- mk +- ml +- mn +- mr +- ms +- my +- ne +- nl +- pa +- pl +- pt +- qu +- ro +- ru +- si +- sk +- sl +- so +- sq +- sr +- sv +- sw +- ta +- te +- th +- tl +- tr +- uk +- ur +- vi +- yo +- zh +model-index: +- name: snowflake-arctic-embed-l-v2.0 + results: + - dataset: + config: en-ext + name: MTEB AmazonCounterfactualClassification (en-ext) + revision: e8379541af4e31359cca9fbcf4b00f2671dba205 + split: test + type: mteb/amazon_counterfactual + metrics: + - type: accuracy + value: 67.039 + - type: f1 + value: 55.1806 + - type: f1_weighted + value: 73.41149999999999 + - type: ap + value: 17.9914 + - type: ap_weighted + value: 17.9914 + - type: main_score + value: 67.039 + task: + type: Classification + - dataset: + config: en + name: MTEB AmazonCounterfactualClassification (en) + revision: e8379541af4e31359cca9fbcf4b00f2671dba205 + split: test + type: mteb/amazon_counterfactual + metrics: + - type: accuracy + value: 65.59700000000001 + - type: f1 + value: 60.244299999999996 + - type: f1_weighted + value: 68.9975 + - type: ap + value: 29.762100000000004 + - type: ap_weighted + value: 29.762100000000004 + - type: main_score + value: 65.59700000000001 + task: + type: Classification + - dataset: + config: default + name: MTEB AmazonPolarityClassification (default) + revision: e2d317d38cd51312af73b3d32a06d1a08b442046 + split: test + type: mteb/amazon_polarity + metrics: + - type: accuracy + value: 74.2565 + - type: f1 + value: 74.0291 + - type: f1_weighted + value: 74.0291 + - type: ap + value: 68.7595 + - type: ap_weighted + value: 68.7595 + - type: main_score + value: 74.2565 + task: + type: Classification + - dataset: + config: en + name: MTEB AmazonReviewsClassification (en) + revision: 1399c76144fd37290681b995c656ef9b2e06e26d + split: test + type: mteb/amazon_reviews_multi + metrics: + - type: accuracy + value: 34.946 + - type: f1 + value: 34.2853 + - type: f1_weighted + value: 34.2853 + - type: main_score + value: 34.946 + task: + type: Classification + - dataset: + config: default + name: MTEB ArguAna (default) + revision: c22ab2a51041ffd869aaddef7af8d8215647e41a + split: test + type: mteb/arguana + metrics: + - type: ndcg_at_1 + value: 33.286 + - type: ndcg_at_3 + value: 49.051 + - type: ndcg_at_5 + value: 54.107000000000006 + - type: ndcg_at_10 + value: 59.146 + - type: ndcg_at_20 + value: 60.897999999999996 + - type: ndcg_at_100 + value: 61.78399999999999 + - type: ndcg_at_1000 + value: 61.845000000000006 + - type: map_at_1 + value: 33.286 + - type: map_at_3 + value: 45.14 + - type: map_at_5 + value: 47.939 + - type: map_at_10 + value: 50.046 + - type: map_at_20 + value: 50.56 + - type: map_at_100 + value: 50.708 + - type: map_at_1000 + value: 50.712 + - type: recall_at_1 + value: 33.286 + - type: recall_at_3 + value: 60.38400000000001 + - type: recall_at_5 + value: 72.688 + - type: recall_at_10 + value: 88.122 + - type: recall_at_20 + value: 94.808 + - type: recall_at_100 + value: 99.21799999999999 + - type: recall_at_1000 + value: 99.644 + - type: precision_at_1 + value: 33.286 + - type: precision_at_3 + value: 20.128 + - type: precision_at_5 + value: 14.538 + - type: precision_at_10 + value: 8.812000000000001 + - type: precision_at_20 + value: 4.74 + - type: precision_at_100 + value: 0.992 + - type: precision_at_1000 + value: 0.1 + - type: mrr_at_1 + value: 33.926 + - type: mrr_at_3 + value: 45.3414 + - type: mrr_at_5 + value: 48.1828 + - type: mrr_at_10 + value: 50.270700000000005 + - type: mrr_at_20 + value: 50.7844 + - type: mrr_at_100 + value: 50.9259 + - type: mrr_at_1000 + value: 50.9294 + - type: nauc_ndcg_at_1_max + value: -10.305 + - type: nauc_ndcg_at_1_std + value: -15.674199999999999 + - type: nauc_ndcg_at_1_diff1 + value: 18.6355 + - type: nauc_ndcg_at_3_max + value: -7.744 + - type: nauc_ndcg_at_3_std + value: -16.894000000000002 + - type: nauc_ndcg_at_3_diff1 + value: 15.4469 + - type: nauc_ndcg_at_5_max + value: -6.4887 + - type: nauc_ndcg_at_5_std + value: -16.1382 + - type: nauc_ndcg_at_5_diff1 + value: 13.8214 + - type: nauc_ndcg_at_10_max + value: -7.616499999999999 + - type: nauc_ndcg_at_10_std + value: -15.8073 + - type: nauc_ndcg_at_10_diff1 + value: 13.7678 + - type: nauc_ndcg_at_20_max + value: -6.9801 + - type: nauc_ndcg_at_20_std + value: -15.068699999999998 + - type: nauc_ndcg_at_20_diff1 + value: 14.2013 + - type: nauc_ndcg_at_100_max + value: -7.5221 + - type: nauc_ndcg_at_100_std + value: -15.417200000000001 + - type: nauc_ndcg_at_100_diff1 + value: 15.1072 + - type: nauc_ndcg_at_1000_max + value: -7.6931 + - type: nauc_ndcg_at_1000_std + value: -15.5367 + - type: nauc_ndcg_at_1000_diff1 + value: 15.001700000000001 + - type: nauc_map_at_1_max + value: -10.305 + - type: nauc_map_at_1_std + value: -15.674199999999999 + - type: nauc_map_at_1_diff1 + value: 18.6355 + - type: nauc_map_at_3_max + value: -8.4505 + - type: nauc_map_at_3_std + value: -16.5487 + - type: nauc_map_at_3_diff1 + value: 15.965599999999998 + - type: nauc_map_at_5_max + value: -7.8429 + - type: nauc_map_at_5_std + value: -16.1332 + - type: nauc_map_at_5_diff1 + value: 15.0893 + - type: nauc_map_at_10_max + value: -8.3186 + - type: nauc_map_at_10_std + value: -15.979399999999998 + - type: nauc_map_at_10_diff1 + value: 15.136199999999999 + - type: nauc_map_at_20_max + value: -8.1697 + - type: nauc_map_at_20_std + value: -15.8241 + - type: nauc_map_at_20_diff1 + value: 15.260599999999998 + - type: nauc_map_at_100_max + value: -8.2285 + - type: nauc_map_at_100_std + value: -15.8624 + - type: nauc_map_at_100_diff1 + value: 15.412600000000001 + - type: nauc_map_at_1000_max + value: -8.2359 + - type: nauc_map_at_1000_std + value: -15.867 + - type: nauc_map_at_1000_diff1 + value: 15.408 + - type: nauc_recall_at_1_max + value: -10.305 + - type: nauc_recall_at_1_std + value: -15.674199999999999 + - type: nauc_recall_at_1_diff1 + value: 18.6355 + - type: nauc_recall_at_3_max + value: -5.5097 + - type: nauc_recall_at_3_std + value: -17.9896 + - type: nauc_recall_at_3_diff1 + value: 13.9525 + - type: nauc_recall_at_5_max + value: -0.9383 + - type: nauc_recall_at_5_std + value: -16.035 + - type: nauc_recall_at_5_diff1 + value: 8.8431 + - type: nauc_recall_at_10_max + value: -2.8548 + - type: nauc_recall_at_10_std + value: -14.1203 + - type: nauc_recall_at_10_diff1 + value: 3.2265 + - type: nauc_recall_at_20_max + value: 14.2043 + - type: nauc_recall_at_20_std + value: 2.1298999999999997 + - type: nauc_recall_at_20_diff1 + value: -1.9900000000000002 + - type: nauc_recall_at_100_max + value: 44.0173 + - type: nauc_recall_at_100_std + value: 42.131800000000005 + - type: nauc_recall_at_100_diff1 + value: 29.9983 + - type: nauc_recall_at_1000_max + value: 25.9434 + - type: nauc_recall_at_1000_std + value: 53.9252 + - type: nauc_recall_at_1000_diff1 + value: -0.9778 + - type: nauc_precision_at_1_max + value: -10.305 + - type: nauc_precision_at_1_std + value: -15.674199999999999 + - type: nauc_precision_at_1_diff1 + value: 18.6355 + - type: nauc_precision_at_3_max + value: -5.5097 + - type: nauc_precision_at_3_std + value: -17.9896 + - type: nauc_precision_at_3_diff1 + value: 13.9525 + - type: nauc_precision_at_5_max + value: -0.9383 + - type: nauc_precision_at_5_std + value: -16.035 + - type: nauc_precision_at_5_diff1 + value: 8.8431 + - type: nauc_precision_at_10_max + value: -2.8548 + - type: nauc_precision_at_10_std + value: -14.1203 + - type: nauc_precision_at_10_diff1 + value: 3.2265 + - type: nauc_precision_at_20_max + value: 14.2043 + - type: nauc_precision_at_20_std + value: 2.1298999999999997 + - type: nauc_precision_at_20_diff1 + value: -1.9900000000000002 + - type: nauc_precision_at_100_max + value: 44.0173 + - type: nauc_precision_at_100_std + value: 42.131800000000005 + - type: nauc_precision_at_100_diff1 + value: 29.9983 + - type: nauc_precision_at_1000_max + value: 25.9434 + - type: nauc_precision_at_1000_std + value: 53.9252 + - type: nauc_precision_at_1000_diff1 + value: -0.9778 + - type: nauc_mrr_at_1_max + value: -9.833 + - type: nauc_mrr_at_1_std + value: -14.8351 + - type: nauc_mrr_at_1_diff1 + value: 16.7604 + - type: nauc_mrr_at_3_max + value: -9.0116 + - type: nauc_mrr_at_3_std + value: -16.296 + - type: nauc_mrr_at_3_diff1 + value: 14.178199999999999 + - type: nauc_mrr_at_5_max + value: -8.308300000000001 + - type: nauc_mrr_at_5_std + value: -15.751999999999999 + - type: nauc_mrr_at_5_diff1 + value: 13.306299999999998 + - type: nauc_mrr_at_10_max + value: -8.7962 + - type: nauc_mrr_at_10_std + value: -15.688099999999999 + - type: nauc_mrr_at_10_diff1 + value: 13.2589 + - type: nauc_mrr_at_20_max + value: -8.6773 + - type: nauc_mrr_at_20_std + value: -15.479499999999998 + - type: nauc_mrr_at_20_diff1 + value: 13.354 + - type: nauc_mrr_at_100_max + value: -8.7533 + - type: nauc_mrr_at_100_std + value: -15.553600000000001 + - type: nauc_mrr_at_100_diff1 + value: 13.4796 + - type: nauc_mrr_at_1000_max + value: -8.7608 + - type: nauc_mrr_at_1000_std + value: -15.5582 + - type: nauc_mrr_at_1000_diff1 + value: 13.4748 + - type: main_score + value: 59.146 + task: + type: Retrieval + - dataset: + config: default + name: MTEB ArxivClusteringP2P (default) + revision: a122ad7f3f0291bf49cc6f4d32aa80929df69d5d + split: test + type: mteb/arxiv-clustering-p2p + metrics: + - type: v_measure + value: 43.9715 + - type: v_measure_std + value: 13.4325 + - type: main_score + value: 43.9715 + task: + type: Clustering + - dataset: + config: default + name: MTEB ArxivClusteringS2S (default) + revision: f910caf1a6075f7329cdf8c1a6135696f37dbd53 + split: test + type: mteb/arxiv-clustering-s2s + metrics: + - type: v_measure + value: 34.775800000000004 + - type: v_measure_std + value: 13.922799999999999 + - type: main_score + value: 34.775800000000004 + task: + type: Clustering + - dataset: + config: default + name: MTEB AskUbuntuDupQuestions (default) + revision: 2000358ca161889fa9c082cb41daa8dcfb161a54 + split: test + type: mteb/askubuntudupquestions-reranking + metrics: + - type: map + value: 63.3521 + - type: mrr + value: 77.5965 + - type: nAUC_map_max + value: 21.2353 + - type: nAUC_map_std + value: 17.002100000000002 + - type: nAUC_map_diff1 + value: 3.8135000000000003 + - type: nAUC_mrr_max + value: 35.058299999999996 + - type: nAUC_mrr_std + value: 20.432 + - type: nAUC_mrr_diff1 + value: 9.2584 + - type: main_score + value: 63.3521 + task: + type: Reranking + - dataset: + config: default + name: MTEB BIOSSES (default) + revision: d3fb88f8f02e40887cd149695127462bbcf29b4a + split: test + type: mteb/biosses-sts + metrics: + - type: pearson + value: 89.8072 + - type: spearman + value: 87.2875 + - type: cosine_pearson + value: 89.8072 + - type: cosine_spearman + value: 87.2875 + - type: manhattan_pearson + value: 87.9173 + - type: manhattan_spearman + value: 86.7327 + - type: euclidean_pearson + value: 88.21600000000001 + - type: euclidean_spearman + value: 87.2875 + - type: main_score + value: 87.2875 + task: + type: STS + - dataset: + config: default + name: MTEB Banking77Classification (default) + revision: 0fd18e25b25c072e09e0d92ab615fda904d66300 + split: test + type: mteb/banking77 + metrics: + - type: accuracy + value: 81.8149 + - type: f1 + value: 81.2226 + - type: f1_weighted + value: 81.2226 + - type: main_score + value: 81.8149 + task: + type: Classification + - dataset: + config: default + name: MTEB BiorxivClusteringP2P (default) + revision: 65b79d1d13f80053f67aca9498d9402c2d9f1f40 + split: test + type: mteb/biorxiv-clustering-p2p + metrics: + - type: v_measure + value: 35.0927 + - type: v_measure_std + value: 0.7048 + - type: main_score + value: 35.0927 + task: + type: Clustering + - dataset: + config: default + name: MTEB BiorxivClusteringS2S (default) + revision: 258694dd0231531bc1fd9de6ceb52a0853c6d908 + split: test + type: mteb/biorxiv-clustering-s2s + metrics: + - type: v_measure + value: 30.220999999999997 + - type: v_measure_std + value: 1.107 + - type: main_score + value: 30.220999999999997 + task: + type: Clustering + - dataset: + config: default + name: MTEB CQADupstackAndroidRetrieval (default) + revision: f46a197baaae43b4f621051089b82a364682dfeb + split: test + type: mteb/cqadupstack-android + metrics: + - type: ndcg_at_1 + value: 44.349 + - type: ndcg_at_3 + value: 50.109 + - type: ndcg_at_5 + value: 52.88699999999999 + - type: ndcg_at_10 + value: 55.799 + - type: ndcg_at_20 + value: 57.589999999999996 + - type: ndcg_at_100 + value: 60.539 + - type: ndcg_at_1000 + value: 61.897000000000006 + - type: map_at_1 + value: 36.230000000000004 + - type: map_at_3 + value: 44.929 + - type: map_at_5 + value: 47.191 + - type: map_at_10 + value: 48.88 + - type: map_at_20 + value: 49.685 + - type: map_at_100 + value: 50.327 + - type: map_at_1000 + value: 50.431000000000004 + - type: recall_at_1 + value: 36.230000000000004 + - type: recall_at_3 + value: 53.173 + - type: recall_at_5 + value: 60.35 + - type: recall_at_10 + value: 69.07 + - type: recall_at_20 + value: 75.371 + - type: recall_at_100 + value: 88.736 + - type: recall_at_1000 + value: 96.75399999999999 + - type: precision_at_1 + value: 44.349 + - type: precision_at_3 + value: 23.748 + - type: precision_at_5 + value: 17.368 + - type: precision_at_10 + value: 10.629 + - type: precision_at_20 + value: 6.152 + - type: precision_at_100 + value: 1.6150000000000002 + - type: precision_at_1000 + value: 0.201 + - type: mrr_at_1 + value: 44.3491 + - type: mrr_at_3 + value: 52.0744 + - type: mrr_at_5 + value: 53.9628 + - type: mrr_at_10 + value: 54.9072 + - type: mrr_at_20 + value: 55.19539999999999 + - type: mrr_at_100 + value: 55.4537 + - type: mrr_at_1000 + value: 55.4787 + - type: nauc_ndcg_at_1_max + value: 36.404599999999995 + - type: nauc_ndcg_at_1_std + value: -4.5556 + - type: nauc_ndcg_at_1_diff1 + value: 57.4025 + - type: nauc_ndcg_at_3_max + value: 38.0347 + - type: nauc_ndcg_at_3_std + value: -2.2339 + - type: nauc_ndcg_at_3_diff1 + value: 50.9146 + - type: nauc_ndcg_at_5_max + value: 38.2927 + - type: nauc_ndcg_at_5_std + value: -2.3645 + - type: nauc_ndcg_at_5_diff1 + value: 51.638 + - type: nauc_ndcg_at_10_max + value: 38.4619 + - type: nauc_ndcg_at_10_std + value: -2.8955 + - type: nauc_ndcg_at_10_diff1 + value: 51.35849999999999 + - type: nauc_ndcg_at_20_max + value: 38.2122 + - type: nauc_ndcg_at_20_std + value: -1.9339 + - type: nauc_ndcg_at_20_diff1 + value: 50.4981 + - type: nauc_ndcg_at_100_max + value: 39.380900000000004 + - type: nauc_ndcg_at_100_std + value: -0.21889999999999998 + - type: nauc_ndcg_at_100_diff1 + value: 51.5696 + - type: nauc_ndcg_at_1000_max + value: 38.9069 + - type: nauc_ndcg_at_1000_std + value: -0.8251 + - type: nauc_ndcg_at_1000_diff1 + value: 51.605500000000006 + - type: nauc_map_at_1_max + value: 31.694 + - type: nauc_map_at_1_std + value: -4.2857 + - type: nauc_map_at_1_diff1 + value: 57.991400000000006 + - type: nauc_map_at_3_max + value: 36.115399999999994 + - type: nauc_map_at_3_std + value: -3.9859999999999998 + - type: nauc_map_at_3_diff1 + value: 52.394 + - type: nauc_map_at_5_max + value: 36.896499999999996 + - type: nauc_map_at_5_std + value: -3.6282 + - type: nauc_map_at_5_diff1 + value: 52.7023 + - type: nauc_map_at_10_max + value: 37.2695 + - type: nauc_map_at_10_std + value: -3.7142 + - type: nauc_map_at_10_diff1 + value: 52.6081 + - type: nauc_map_at_20_max + value: 37.4097 + - type: nauc_map_at_20_std + value: -3.0479 + - type: nauc_map_at_20_diff1 + value: 52.2999 + - type: nauc_map_at_100_max + value: 37.6608 + - type: nauc_map_at_100_std + value: -2.7363999999999997 + - type: nauc_map_at_100_diff1 + value: 52.5068 + - type: nauc_map_at_1000_max + value: 37.6406 + - type: nauc_map_at_1000_std + value: -2.7695000000000003 + - type: nauc_map_at_1000_diff1 + value: 52.5091 + - type: nauc_recall_at_1_max + value: 31.694 + - type: nauc_recall_at_1_std + value: -4.2857 + - type: nauc_recall_at_1_diff1 + value: 57.991400000000006 + - type: nauc_recall_at_3_max + value: 35.9705 + - type: nauc_recall_at_3_std + value: -2.78 + - type: nauc_recall_at_3_diff1 + value: 44.2342 + - type: nauc_recall_at_5_max + value: 36.3608 + - type: nauc_recall_at_5_std + value: -1.8541999999999998 + - type: nauc_recall_at_5_diff1 + value: 45.0955 + - type: nauc_recall_at_10_max + value: 35.7364 + - type: nauc_recall_at_10_std + value: -3.2479 + - type: nauc_recall_at_10_diff1 + value: 42.3031 + - type: nauc_recall_at_20_max + value: 34.7814 + - type: nauc_recall_at_20_std + value: 0.7642 + - type: nauc_recall_at_20_diff1 + value: 37.3357 + - type: nauc_recall_at_100_max + value: 49.1721 + - type: nauc_recall_at_100_std + value: 27.8334 + - type: nauc_recall_at_100_diff1 + value: 39.549 + - type: nauc_recall_at_1000_max + value: 59.516400000000004 + - type: nauc_recall_at_1000_std + value: 66.1089 + - type: nauc_recall_at_1000_diff1 + value: 31.4818 + - type: nauc_precision_at_1_max + value: 36.404599999999995 + - type: nauc_precision_at_1_std + value: -4.5556 + - type: nauc_precision_at_1_diff1 + value: 57.4025 + - type: nauc_precision_at_3_max + value: 35.7954 + - type: nauc_precision_at_3_std + value: 0.6122 + - type: nauc_precision_at_3_diff1 + value: 29.4346 + - type: nauc_precision_at_5_max + value: 31.322699999999998 + - type: nauc_precision_at_5_std + value: 2.2124 + - type: nauc_precision_at_5_diff1 + value: 21.1992 + - type: nauc_precision_at_10_max + value: 22.6897 + - type: nauc_precision_at_10_std + value: 3.6117999999999997 + - type: nauc_precision_at_10_diff1 + value: 9.0833 + - type: nauc_precision_at_20_max + value: 14.954799999999999 + - type: nauc_precision_at_20_std + value: 7.2373 + - type: nauc_precision_at_20_diff1 + value: -0.544 + - type: nauc_precision_at_100_max + value: 4.2428 + - type: nauc_precision_at_100_std + value: 7.3461 + - type: nauc_precision_at_100_diff1 + value: -11.3684 + - type: nauc_precision_at_1000_max + value: -9.148399999999999 + - type: nauc_precision_at_1000_std + value: -3.5724 + - type: nauc_precision_at_1000_diff1 + value: -19.142400000000002 + - type: nauc_mrr_at_1_max + value: 36.404599999999995 + - type: nauc_mrr_at_1_std + value: -4.5556 + - type: nauc_mrr_at_1_diff1 + value: 57.4025 + - type: nauc_mrr_at_3_max + value: 38.7222 + - type: nauc_mrr_at_3_std + value: -2.3924000000000003 + - type: nauc_mrr_at_3_diff1 + value: 52.7995 + - type: nauc_mrr_at_5_max + value: 38.7579 + - type: nauc_mrr_at_5_std + value: -2.6441 + - type: nauc_mrr_at_5_diff1 + value: 53.547599999999996 + - type: nauc_mrr_at_10_max + value: 38.7832 + - type: nauc_mrr_at_10_std + value: -2.5202999999999998 + - type: nauc_mrr_at_10_diff1 + value: 53.4856 + - type: nauc_mrr_at_20_max + value: 38.6588 + - type: nauc_mrr_at_20_std + value: -2.501 + - type: nauc_mrr_at_20_diff1 + value: 53.3571 + - type: nauc_mrr_at_100_max + value: 38.6456 + - type: nauc_mrr_at_100_std + value: -2.4756 + - type: nauc_mrr_at_100_diff1 + value: 53.455600000000004 + - type: nauc_mrr_at_1000_max + value: 38.6449 + - type: nauc_mrr_at_1000_std + value: -2.4623 + - type: nauc_mrr_at_1000_diff1 + value: 53.45419999999999 + - type: main_score + value: 55.799 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackEnglishRetrieval (default) + revision: ad9991cb51e31e31e430383c75ffb2885547b5f0 + split: test + type: mteb/cqadupstack-english + metrics: + - type: ndcg_at_1 + value: 44.204 + - type: ndcg_at_3 + value: 49.549 + - type: ndcg_at_5 + value: 51.658 + - type: ndcg_at_10 + value: 53.681 + - type: ndcg_at_20 + value: 55.129 + - type: ndcg_at_100 + value: 57.691 + - type: ndcg_at_1000 + value: 59.325 + - type: map_at_1 + value: 35.193000000000005 + - type: map_at_3 + value: 44.005 + - type: map_at_5 + value: 46.043 + - type: map_at_10 + value: 47.491 + - type: map_at_20 + value: 48.169000000000004 + - type: map_at_100 + value: 48.789 + - type: map_at_1000 + value: 48.898 + - type: recall_at_1 + value: 35.193000000000005 + - type: recall_at_3 + value: 51.333 + - type: recall_at_5 + value: 57.436 + - type: recall_at_10 + value: 63.991 + - type: recall_at_20 + value: 69.37100000000001 + - type: recall_at_100 + value: 81.099 + - type: recall_at_1000 + value: 91.363 + - type: precision_at_1 + value: 44.204 + - type: precision_at_3 + value: 24.374000000000002 + - type: precision_at_5 + value: 17.287 + - type: precision_at_10 + value: 10.293 + - type: precision_at_20 + value: 5.943 + - type: precision_at_100 + value: 1.5730000000000002 + - type: precision_at_1000 + value: 0.197 + - type: mrr_at_1 + value: 44.2038 + - type: mrr_at_3 + value: 51.624199999999995 + - type: mrr_at_5 + value: 52.9459 + - type: mrr_at_10 + value: 53.697399999999995 + - type: mrr_at_20 + value: 54.028200000000005 + - type: mrr_at_100 + value: 54.267900000000004 + - type: mrr_at_1000 + value: 54.3028 + - type: nauc_ndcg_at_1_max + value: 45.3525 + - type: nauc_ndcg_at_1_std + value: -2.2124 + - type: nauc_ndcg_at_1_diff1 + value: 59.392100000000006 + - type: nauc_ndcg_at_3_max + value: 46.6258 + - type: nauc_ndcg_at_3_std + value: -2.8042000000000002 + - type: nauc_ndcg_at_3_diff1 + value: 55.0995 + - type: nauc_ndcg_at_5_max + value: 47.3391 + - type: nauc_ndcg_at_5_std + value: -1.8336999999999999 + - type: nauc_ndcg_at_5_diff1 + value: 54.848 + - type: nauc_ndcg_at_10_max + value: 47.713899999999995 + - type: nauc_ndcg_at_10_std + value: -0.6185 + - type: nauc_ndcg_at_10_diff1 + value: 54.6241 + - type: nauc_ndcg_at_20_max + value: 48.072900000000004 + - type: nauc_ndcg_at_20_std + value: -0.21589999999999998 + - type: nauc_ndcg_at_20_diff1 + value: 54.655100000000004 + - type: nauc_ndcg_at_100_max + value: 48.4791 + - type: nauc_ndcg_at_100_std + value: 1.9865000000000002 + - type: nauc_ndcg_at_100_diff1 + value: 54.033 + - type: nauc_ndcg_at_1000_max + value: 48.3686 + - type: nauc_ndcg_at_1000_std + value: 1.8716 + - type: nauc_ndcg_at_1000_diff1 + value: 54.125 + - type: nauc_map_at_1_max + value: 34.797200000000004 + - type: nauc_map_at_1_std + value: -13.140199999999998 + - type: nauc_map_at_1_diff1 + value: 61.197100000000006 + - type: nauc_map_at_3_max + value: 41.4347 + - type: nauc_map_at_3_std + value: -10.0816 + - type: nauc_map_at_3_diff1 + value: 57.8979 + - type: nauc_map_at_5_max + value: 43.1536 + - type: nauc_map_at_5_std + value: -7.8041 + - type: nauc_map_at_5_diff1 + value: 57.1125 + - type: nauc_map_at_10_max + value: 44.243700000000004 + - type: nauc_map_at_10_std + value: -6.047000000000001 + - type: nauc_map_at_10_diff1 + value: 56.688700000000004 + - type: nauc_map_at_20_max + value: 44.7799 + - type: nauc_map_at_20_std + value: -5.2916 + - type: nauc_map_at_20_diff1 + value: 56.565799999999996 + - type: nauc_map_at_100_max + value: 45.3233 + - type: nauc_map_at_100_std + value: -4.287 + - type: nauc_map_at_100_diff1 + value: 56.41460000000001 + - type: nauc_map_at_1000_max + value: 45.3992 + - type: nauc_map_at_1000_std + value: -4.1593 + - type: nauc_map_at_1000_diff1 + value: 56.413599999999995 + - type: nauc_recall_at_1_max + value: 34.797200000000004 + - type: nauc_recall_at_1_std + value: -13.140199999999998 + - type: nauc_recall_at_1_diff1 + value: 61.197100000000006 + - type: nauc_recall_at_3_max + value: 42.7264 + - type: nauc_recall_at_3_std + value: -8.201799999999999 + - type: nauc_recall_at_3_diff1 + value: 52.3494 + - type: nauc_recall_at_5_max + value: 44.6494 + - type: nauc_recall_at_5_std + value: -3.3112999999999997 + - type: nauc_recall_at_5_diff1 + value: 50.1019 + - type: nauc_recall_at_10_max + value: 46.6669 + - type: nauc_recall_at_10_std + value: 2.3359 + - type: nauc_recall_at_10_diff1 + value: 48.1454 + - type: nauc_recall_at_20_max + value: 48.7828 + - type: nauc_recall_at_20_std + value: 6.0266 + - type: nauc_recall_at_20_diff1 + value: 46.786699999999996 + - type: nauc_recall_at_100_max + value: 53.081999999999994 + - type: nauc_recall_at_100_std + value: 24.1569 + - type: nauc_recall_at_100_diff1 + value: 40.4049 + - type: nauc_recall_at_1000_max + value: 55.803000000000004 + - type: nauc_recall_at_1000_std + value: 36.3769 + - type: nauc_recall_at_1000_diff1 + value: 34.336 + - type: nauc_precision_at_1_max + value: 45.3525 + - type: nauc_precision_at_1_std + value: -2.2124 + - type: nauc_precision_at_1_diff1 + value: 59.392100000000006 + - type: nauc_precision_at_3_max + value: 44.2838 + - type: nauc_precision_at_3_std + value: 14.3908 + - type: nauc_precision_at_3_diff1 + value: 27.219700000000003 + - type: nauc_precision_at_5_max + value: 42.9914 + - type: nauc_precision_at_5_std + value: 23.0682 + - type: nauc_precision_at_5_diff1 + value: 16.2263 + - type: nauc_precision_at_10_max + value: 38.5042 + - type: nauc_precision_at_10_std + value: 30.792199999999998 + - type: nauc_precision_at_10_diff1 + value: 5.7691 + - type: nauc_precision_at_20_max + value: 34.417500000000004 + - type: nauc_precision_at_20_std + value: 34.1749 + - type: nauc_precision_at_20_diff1 + value: -0.9022 + - type: nauc_precision_at_100_max + value: 27.4072 + - type: nauc_precision_at_100_std + value: 42.4351 + - type: nauc_precision_at_100_diff1 + value: -11.407 + - type: nauc_precision_at_1000_max + value: 16.142400000000002 + - type: nauc_precision_at_1000_std + value: 36.4482 + - type: nauc_precision_at_1000_diff1 + value: -16.8073 + - type: nauc_mrr_at_1_max + value: 45.3525 + - type: nauc_mrr_at_1_std + value: -2.2124 + - type: nauc_mrr_at_1_diff1 + value: 59.392100000000006 + - type: nauc_mrr_at_3_max + value: 48.7407 + - type: nauc_mrr_at_3_std + value: 0.2074 + - type: nauc_mrr_at_3_diff1 + value: 55.8153 + - type: nauc_mrr_at_5_max + value: 48.9081 + - type: nauc_mrr_at_5_std + value: 0.9781 + - type: nauc_mrr_at_5_diff1 + value: 55.6807 + - type: nauc_mrr_at_10_max + value: 48.7888 + - type: nauc_mrr_at_10_std + value: 1.384 + - type: nauc_mrr_at_10_diff1 + value: 55.5207 + - type: nauc_mrr_at_20_max + value: 48.7371 + - type: nauc_mrr_at_20_std + value: 1.3671 + - type: nauc_mrr_at_20_diff1 + value: 55.508199999999995 + - type: nauc_mrr_at_100_max + value: 48.7472 + - type: nauc_mrr_at_100_std + value: 1.5221 + - type: nauc_mrr_at_100_diff1 + value: 55.5036 + - type: nauc_mrr_at_1000_max + value: 48.7402 + - type: nauc_mrr_at_1000_std + value: 1.5072 + - type: nauc_mrr_at_1000_diff1 + value: 55.507 + - type: main_score + value: 53.681 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackGamingRetrieval (default) + revision: 4885aa143210c98657558c04aaf3dc47cfb54340 + split: test + type: mteb/cqadupstack-gaming + metrics: + - type: ndcg_at_1 + value: 50.345 + - type: ndcg_at_3 + value: 57.776 + - type: ndcg_at_5 + value: 60.477000000000004 + - type: ndcg_at_10 + value: 63.172 + - type: ndcg_at_20 + value: 64.62 + - type: ndcg_at_100 + value: 66.538 + - type: ndcg_at_1000 + value: 67.43 + - type: map_at_1 + value: 44.153 + - type: map_at_3 + value: 53.979 + - type: map_at_5 + value: 55.925000000000004 + - type: map_at_10 + value: 57.32899999999999 + - type: map_at_20 + value: 57.879000000000005 + - type: map_at_100 + value: 58.239 + - type: map_at_1000 + value: 58.285 + - type: recall_at_1 + value: 44.153 + - type: recall_at_3 + value: 62.766999999999996 + - type: recall_at_5 + value: 69.405 + - type: recall_at_10 + value: 77.107 + - type: recall_at_20 + value: 82.337 + - type: recall_at_100 + value: 91.307 + - type: recall_at_1000 + value: 97.586 + - type: precision_at_1 + value: 50.345 + - type: precision_at_3 + value: 25.601000000000003 + - type: precision_at_5 + value: 17.416999999999998 + - type: precision_at_10 + value: 9.994 + - type: precision_at_20 + value: 5.492 + - type: precision_at_100 + value: 1.261 + - type: precision_at_1000 + value: 0.13799999999999998 + - type: mrr_at_1 + value: 50.3448 + - type: mrr_at_3 + value: 58.160900000000005 + - type: mrr_at_5 + value: 59.549600000000005 + - type: mrr_at_10 + value: 60.545899999999996 + - type: mrr_at_20 + value: 60.8453 + - type: mrr_at_100 + value: 61.06120000000001 + - type: mrr_at_1000 + value: 61.083299999999994 + - type: nauc_ndcg_at_1_max + value: 39.467400000000005 + - type: nauc_ndcg_at_1_std + value: -6.512 + - type: nauc_ndcg_at_1_diff1 + value: 57.337700000000005 + - type: nauc_ndcg_at_3_max + value: 42.8884 + - type: nauc_ndcg_at_3_std + value: -6.0156 + - type: nauc_ndcg_at_3_diff1 + value: 54.432 + - type: nauc_ndcg_at_5_max + value: 44.831500000000005 + - type: nauc_ndcg_at_5_std + value: -4.3286999999999995 + - type: nauc_ndcg_at_5_diff1 + value: 54.6971 + - type: nauc_ndcg_at_10_max + value: 44.391799999999996 + - type: nauc_ndcg_at_10_std + value: -3.6792 + - type: nauc_ndcg_at_10_diff1 + value: 53.749199999999995 + - type: nauc_ndcg_at_20_max + value: 44.9459 + - type: nauc_ndcg_at_20_std + value: -2.1965 + - type: nauc_ndcg_at_20_diff1 + value: 53.7261 + - type: nauc_ndcg_at_100_max + value: 45.0603 + - type: nauc_ndcg_at_100_std + value: -1.1026 + - type: nauc_ndcg_at_100_diff1 + value: 54.059900000000006 + - type: nauc_ndcg_at_1000_max + value: 44.9294 + - type: nauc_ndcg_at_1000_std + value: -1.7629 + - type: nauc_ndcg_at_1000_diff1 + value: 54.57189999999999 + - type: nauc_map_at_1_max + value: 34.3031 + - type: nauc_map_at_1_std + value: -8.9637 + - type: nauc_map_at_1_diff1 + value: 57.99100000000001 + - type: nauc_map_at_3_max + value: 40.732 + - type: nauc_map_at_3_std + value: -8.312999999999999 + - type: nauc_map_at_3_diff1 + value: 55.9106 + - type: nauc_map_at_5_max + value: 42.1709 + - type: nauc_map_at_5_std + value: -6.9354 + - type: nauc_map_at_5_diff1 + value: 56.042899999999996 + - type: nauc_map_at_10_max + value: 42.1589 + - type: nauc_map_at_10_std + value: -6.3601 + - type: nauc_map_at_10_diff1 + value: 55.490700000000004 + - type: nauc_map_at_20_max + value: 42.595 + - type: nauc_map_at_20_std + value: -5.5588 + - type: nauc_map_at_20_diff1 + value: 55.4651 + - type: nauc_map_at_100_max + value: 42.6911 + - type: nauc_map_at_100_std + value: -5.2459999999999996 + - type: nauc_map_at_100_diff1 + value: 55.45060000000001 + - type: nauc_map_at_1000_max + value: 42.7134 + - type: nauc_map_at_1000_std + value: -5.2317 + - type: nauc_map_at_1000_diff1 + value: 55.4871 + - type: nauc_recall_at_1_max + value: 34.3031 + - type: nauc_recall_at_1_std + value: -8.9637 + - type: nauc_recall_at_1_diff1 + value: 57.99100000000001 + - type: nauc_recall_at_3_max + value: 43.623400000000004 + - type: nauc_recall_at_3_std + value: -6.2843 + - type: nauc_recall_at_3_diff1 + value: 50.775800000000004 + - type: nauc_recall_at_5_max + value: 48.7222 + - type: nauc_recall_at_5_std + value: -0.9506000000000001 + - type: nauc_recall_at_5_diff1 + value: 50.41480000000001 + - type: nauc_recall_at_10_max + value: 47.6178 + - type: nauc_recall_at_10_std + value: 2.2783 + - type: nauc_recall_at_10_diff1 + value: 45.1663 + - type: nauc_recall_at_20_max + value: 51.454 + - type: nauc_recall_at_20_std + value: 11.8339 + - type: nauc_recall_at_20_diff1 + value: 42.8694 + - type: nauc_recall_at_100_max + value: 58.145500000000006 + - type: nauc_recall_at_100_std + value: 35.4717 + - type: nauc_recall_at_100_diff1 + value: 40.8401 + - type: nauc_recall_at_1000_max + value: 79.9122 + - type: nauc_recall_at_1000_std + value: 64.5076 + - type: nauc_recall_at_1000_diff1 + value: 48.7357 + - type: nauc_precision_at_1_max + value: 39.467400000000005 + - type: nauc_precision_at_1_std + value: -6.512 + - type: nauc_precision_at_1_diff1 + value: 57.337700000000005 + - type: nauc_precision_at_3_max + value: 39.763799999999996 + - type: nauc_precision_at_3_std + value: 2.8881 + - type: nauc_precision_at_3_diff1 + value: 30.5735 + - type: nauc_precision_at_5_max + value: 38.062200000000004 + - type: nauc_precision_at_5_std + value: 10.2952 + - type: nauc_precision_at_5_diff1 + value: 21.2531 + - type: nauc_precision_at_10_max + value: 31.330099999999998 + - type: nauc_precision_at_10_std + value: 16.6561 + - type: nauc_precision_at_10_diff1 + value: 8.4745 + - type: nauc_precision_at_20_max + value: 28.5499 + - type: nauc_precision_at_20_std + value: 25.593300000000003 + - type: nauc_precision_at_20_diff1 + value: 0.8708 + - type: nauc_precision_at_100_max + value: 20.275299999999998 + - type: nauc_precision_at_100_std + value: 31.6878 + - type: nauc_precision_at_100_diff1 + value: -8.8113 + - type: nauc_precision_at_1000_max + value: 15.4133 + - type: nauc_precision_at_1000_std + value: 29.5211 + - type: nauc_precision_at_1000_diff1 + value: -11.061300000000001 + - type: nauc_mrr_at_1_max + value: 39.467400000000005 + - type: nauc_mrr_at_1_std + value: -6.512 + - type: nauc_mrr_at_1_diff1 + value: 57.337700000000005 + - type: nauc_mrr_at_3_max + value: 42.9279 + - type: nauc_mrr_at_3_std + value: -5.251200000000001 + - type: nauc_mrr_at_3_diff1 + value: 54.8802 + - type: nauc_mrr_at_5_max + value: 43.5261 + - type: nauc_mrr_at_5_std + value: -4.4842 + - type: nauc_mrr_at_5_diff1 + value: 54.874500000000005 + - type: nauc_mrr_at_10_max + value: 43.2392 + - type: nauc_mrr_at_10_std + value: -4.2739 + - type: nauc_mrr_at_10_diff1 + value: 54.5466 + - type: nauc_mrr_at_20_max + value: 43.2263 + - type: nauc_mrr_at_20_std + value: -4.122 + - type: nauc_mrr_at_20_diff1 + value: 54.5397 + - type: nauc_mrr_at_100_max + value: 43.2131 + - type: nauc_mrr_at_100_std + value: -4.041 + - type: nauc_mrr_at_100_diff1 + value: 54.586800000000004 + - type: nauc_mrr_at_1000_max + value: 43.2078 + - type: nauc_mrr_at_1000_std + value: -4.0622 + - type: nauc_mrr_at_1000_diff1 + value: 54.606100000000005 + - type: main_score + value: 63.172 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackGisRetrieval (default) + revision: 5003b3064772da1887988e05400cf3806fe491f2 + split: test + type: mteb/cqadupstack-gis + metrics: + - type: ndcg_at_1 + value: 32.429 + - type: ndcg_at_3 + value: 39.639 + - type: ndcg_at_5 + value: 42.051 + - type: ndcg_at_10 + value: 44.759 + - type: ndcg_at_20 + value: 46.588 + - type: ndcg_at_100 + value: 49.457 + - type: ndcg_at_1000 + value: 51.248000000000005 + - type: map_at_1 + value: 30.259999999999998 + - type: map_at_3 + value: 36.998 + - type: map_at_5 + value: 38.452 + - type: map_at_10 + value: 39.653 + - type: map_at_20 + value: 40.199 + - type: map_at_100 + value: 40.63 + - type: map_at_1000 + value: 40.701 + - type: recall_at_1 + value: 30.259999999999998 + - type: recall_at_3 + value: 44.531 + - type: recall_at_5 + value: 50.349999999999994 + - type: recall_at_10 + value: 58.294999999999995 + - type: recall_at_20 + value: 65.19200000000001 + - type: recall_at_100 + value: 79.699 + - type: recall_at_1000 + value: 93.181 + - type: precision_at_1 + value: 32.429 + - type: precision_at_3 + value: 16.61 + - type: precision_at_5 + value: 11.39 + - type: precision_at_10 + value: 6.746 + - type: precision_at_20 + value: 3.8019999999999996 + - type: precision_at_100 + value: 0.963 + - type: precision_at_1000 + value: 0.11399999999999999 + - type: mrr_at_1 + value: 32.4294 + - type: mrr_at_3 + value: 39.265499999999996 + - type: mrr_at_5 + value: 40.6158 + - type: mrr_at_10 + value: 41.7454 + - type: mrr_at_20 + value: 42.187999999999995 + - type: mrr_at_100 + value: 42.530699999999996 + - type: mrr_at_1000 + value: 42.584300000000006 + - type: nauc_ndcg_at_1_max + value: 30.2344 + - type: nauc_ndcg_at_1_std + value: -8.76 + - type: nauc_ndcg_at_1_diff1 + value: 43.3339 + - type: nauc_ndcg_at_3_max + value: 31.300299999999996 + - type: nauc_ndcg_at_3_std + value: -5.2691 + - type: nauc_ndcg_at_3_diff1 + value: 39.6872 + - type: nauc_ndcg_at_5_max + value: 31.844099999999997 + - type: nauc_ndcg_at_5_std + value: -4.228400000000001 + - type: nauc_ndcg_at_5_diff1 + value: 38.2047 + - type: nauc_ndcg_at_10_max + value: 31.664900000000003 + - type: nauc_ndcg_at_10_std + value: -3.2960000000000003 + - type: nauc_ndcg_at_10_diff1 + value: 36.6259 + - type: nauc_ndcg_at_20_max + value: 31.630999999999997 + - type: nauc_ndcg_at_20_std + value: -2.6685 + - type: nauc_ndcg_at_20_diff1 + value: 36.577 + - type: nauc_ndcg_at_100_max + value: 32.283899999999996 + - type: nauc_ndcg_at_100_std + value: -2.1553 + - type: nauc_ndcg_at_100_diff1 + value: 36.3958 + - type: nauc_ndcg_at_1000_max + value: 32.4852 + - type: nauc_ndcg_at_1000_std + value: -2.3408 + - type: nauc_ndcg_at_1000_diff1 + value: 37.0227 + - type: nauc_map_at_1_max + value: 27.620800000000003 + - type: nauc_map_at_1_std + value: -10.7657 + - type: nauc_map_at_1_diff1 + value: 43.7864 + - type: nauc_map_at_3_max + value: 30.0483 + - type: nauc_map_at_3_std + value: -6.9221 + - type: nauc_map_at_3_diff1 + value: 40.826 + - type: nauc_map_at_5_max + value: 30.560399999999998 + - type: nauc_map_at_5_std + value: -6.1894 + - type: nauc_map_at_5_diff1 + value: 40.0042 + - type: nauc_map_at_10_max + value: 30.665100000000002 + - type: nauc_map_at_10_std + value: -5.8472 + - type: nauc_map_at_10_diff1 + value: 39.3857 + - type: nauc_map_at_20_max + value: 30.761699999999998 + - type: nauc_map_at_20_std + value: -5.591 + - type: nauc_map_at_20_diff1 + value: 39.4111 + - type: nauc_map_at_100_max + value: 30.859399999999997 + - type: nauc_map_at_100_std + value: -5.532 + - type: nauc_map_at_100_diff1 + value: 39.3888 + - type: nauc_map_at_1000_max + value: 30.871199999999998 + - type: nauc_map_at_1000_std + value: -5.5322000000000005 + - type: nauc_map_at_1000_diff1 + value: 39.4166 + - type: nauc_recall_at_1_max + value: 27.620800000000003 + - type: nauc_recall_at_1_std + value: -10.7657 + - type: nauc_recall_at_1_diff1 + value: 43.7864 + - type: nauc_recall_at_3_max + value: 31.187199999999997 + - type: nauc_recall_at_3_std + value: -2.5515 + - type: nauc_recall_at_3_diff1 + value: 36.9576 + - type: nauc_recall_at_5_max + value: 32.6827 + - type: nauc_recall_at_5_std + value: -0.4259 + - type: nauc_recall_at_5_diff1 + value: 33.1674 + - type: nauc_recall_at_10_max + value: 31.729400000000002 + - type: nauc_recall_at_10_std + value: 2.8294 + - type: nauc_recall_at_10_diff1 + value: 27.7289 + - type: nauc_recall_at_20_max + value: 30.9251 + - type: nauc_recall_at_20_std + value: 5.9573 + - type: nauc_recall_at_20_diff1 + value: 26.271499999999996 + - type: nauc_recall_at_100_max + value: 35.8557 + - type: nauc_recall_at_100_std + value: 14.478399999999999 + - type: nauc_recall_at_100_diff1 + value: 20.6213 + - type: nauc_recall_at_1000_max + value: 49.7086 + - type: nauc_recall_at_1000_std + value: 36.9282 + - type: nauc_recall_at_1000_diff1 + value: 14.288300000000001 + - type: nauc_precision_at_1_max + value: 30.2344 + - type: nauc_precision_at_1_std + value: -8.76 + - type: nauc_precision_at_1_diff1 + value: 43.3339 + - type: nauc_precision_at_3_max + value: 34.808699999999995 + - type: nauc_precision_at_3_std + value: 0.7861999999999999 + - type: nauc_precision_at_3_diff1 + value: 33.232299999999995 + - type: nauc_precision_at_5_max + value: 35.9325 + - type: nauc_precision_at_5_std + value: 4.1644 + - type: nauc_precision_at_5_diff1 + value: 28.872799999999998 + - type: nauc_precision_at_10_max + value: 34.2471 + - type: nauc_precision_at_10_std + value: 7.2728 + - type: nauc_precision_at_10_diff1 + value: 21.044999999999998 + - type: nauc_precision_at_20_max + value: 31.828200000000002 + - type: nauc_precision_at_20_std + value: 10.2775 + - type: nauc_precision_at_20_diff1 + value: 16.7988 + - type: nauc_precision_at_100_max + value: 26.320100000000004 + - type: nauc_precision_at_100_std + value: 14.0416 + - type: nauc_precision_at_100_diff1 + value: 3.4286999999999996 + - type: nauc_precision_at_1000_max + value: 17.6282 + - type: nauc_precision_at_1000_std + value: 13.1888 + - type: nauc_precision_at_1000_diff1 + value: -6.7075 + - type: nauc_mrr_at_1_max + value: 30.2344 + - type: nauc_mrr_at_1_std + value: -8.76 + - type: nauc_mrr_at_1_diff1 + value: 43.3339 + - type: nauc_mrr_at_3_max + value: 32.2423 + - type: nauc_mrr_at_3_std + value: -4.6264 + - type: nauc_mrr_at_3_diff1 + value: 39.6214 + - type: nauc_mrr_at_5_max + value: 32.496199999999995 + - type: nauc_mrr_at_5_std + value: -4.3406 + - type: nauc_mrr_at_5_diff1 + value: 38.921 + - type: nauc_mrr_at_10_max + value: 32.330799999999996 + - type: nauc_mrr_at_10_std + value: -3.943 + - type: nauc_mrr_at_10_diff1 + value: 38.2251 + - type: nauc_mrr_at_20_max + value: 32.1807 + - type: nauc_mrr_at_20_std + value: -3.9316999999999998 + - type: nauc_mrr_at_20_diff1 + value: 38.2161 + - type: nauc_mrr_at_100_max + value: 32.2413 + - type: nauc_mrr_at_100_std + value: -3.8869000000000002 + - type: nauc_mrr_at_100_diff1 + value: 38.217800000000004 + - type: nauc_mrr_at_1000_max + value: 32.2481 + - type: nauc_mrr_at_1000_std + value: -3.8933000000000004 + - type: nauc_mrr_at_1000_diff1 + value: 38.2515 + - type: main_score + value: 44.759 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackMathematicaRetrieval (default) + revision: 90fceea13679c63fe563ded68f3b6f06e50061de + split: test + type: mteb/cqadupstack-mathematica + metrics: + - type: ndcg_at_1 + value: 22.761 + - type: ndcg_at_3 + value: 27.578999999999997 + - type: ndcg_at_5 + value: 30.067 + - type: ndcg_at_10 + value: 32.823 + - type: ndcg_at_20 + value: 35.129 + - type: ndcg_at_100 + value: 38.903999999999996 + - type: ndcg_at_1000 + value: 41.181 + - type: map_at_1 + value: 18.360000000000003 + - type: map_at_3 + value: 24.264 + - type: map_at_5 + value: 25.844 + - type: map_at_10 + value: 27.093 + - type: map_at_20 + value: 27.839999999999996 + - type: map_at_100 + value: 28.416999999999998 + - type: map_at_1000 + value: 28.517 + - type: recall_at_1 + value: 18.360000000000003 + - type: recall_at_3 + value: 31.044 + - type: recall_at_5 + value: 37.432 + - type: recall_at_10 + value: 45.525999999999996 + - type: recall_at_20 + value: 53.557 + - type: recall_at_100 + value: 72.14500000000001 + - type: recall_at_1000 + value: 88.041 + - type: precision_at_1 + value: 22.761 + - type: precision_at_3 + value: 13.350000000000001 + - type: precision_at_5 + value: 9.801 + - type: precision_at_10 + value: 6.157 + - type: precision_at_20 + value: 3.744 + - type: precision_at_100 + value: 1.055 + - type: precision_at_1000 + value: 0.13799999999999998 + - type: mrr_at_1 + value: 22.761200000000002 + - type: mrr_at_3 + value: 29.187400000000004 + - type: mrr_at_5 + value: 30.866500000000002 + - type: mrr_at_10 + value: 32.0236 + - type: mrr_at_20 + value: 32.5924 + - type: mrr_at_100 + value: 32.995000000000005 + - type: mrr_at_1000 + value: 33.042100000000005 + - type: nauc_ndcg_at_1_max + value: 22.3876 + - type: nauc_ndcg_at_1_std + value: -0.26649999999999996 + - type: nauc_ndcg_at_1_diff1 + value: 42.7688 + - type: nauc_ndcg_at_3_max + value: 24.329 + - type: nauc_ndcg_at_3_std + value: 1.3894 + - type: nauc_ndcg_at_3_diff1 + value: 38.5792 + - type: nauc_ndcg_at_5_max + value: 24.331 + - type: nauc_ndcg_at_5_std + value: 3.1460000000000004 + - type: nauc_ndcg_at_5_diff1 + value: 36.1599 + - type: nauc_ndcg_at_10_max + value: 23.9962 + - type: nauc_ndcg_at_10_std + value: 3.6198 + - type: nauc_ndcg_at_10_diff1 + value: 34.615899999999996 + - type: nauc_ndcg_at_20_max + value: 23.189899999999998 + - type: nauc_ndcg_at_20_std + value: 3.3743000000000003 + - type: nauc_ndcg_at_20_diff1 + value: 34.5344 + - type: nauc_ndcg_at_100_max + value: 24.1644 + - type: nauc_ndcg_at_100_std + value: 5.3245000000000005 + - type: nauc_ndcg_at_100_diff1 + value: 34.1404 + - type: nauc_ndcg_at_1000_max + value: 24.4504 + - type: nauc_ndcg_at_1000_std + value: 5.0385 + - type: nauc_ndcg_at_1000_diff1 + value: 34.3277 + - type: nauc_map_at_1_max + value: 20.5435 + - type: nauc_map_at_1_std + value: -0.1746 + - type: nauc_map_at_1_diff1 + value: 43.252 + - type: nauc_map_at_3_max + value: 23.108999999999998 + - type: nauc_map_at_3_std + value: 0.8848 + - type: nauc_map_at_3_diff1 + value: 39.9259 + - type: nauc_map_at_5_max + value: 23.329900000000002 + - type: nauc_map_at_5_std + value: 1.7795999999999998 + - type: nauc_map_at_5_diff1 + value: 38.448 + - type: nauc_map_at_10_max + value: 23.1789 + - type: nauc_map_at_10_std + value: 2.1036 + - type: nauc_map_at_10_diff1 + value: 37.653 + - type: nauc_map_at_20_max + value: 22.9132 + - type: nauc_map_at_20_std + value: 2.1094 + - type: nauc_map_at_20_diff1 + value: 37.5569 + - type: nauc_map_at_100_max + value: 23.0857 + - type: nauc_map_at_100_std + value: 2.4645 + - type: nauc_map_at_100_diff1 + value: 37.4881 + - type: nauc_map_at_1000_max + value: 23.0988 + - type: nauc_map_at_1000_std + value: 2.4427999999999996 + - type: nauc_map_at_1000_diff1 + value: 37.4707 + - type: nauc_recall_at_1_max + value: 20.5435 + - type: nauc_recall_at_1_std + value: -0.1746 + - type: nauc_recall_at_1_diff1 + value: 43.252 + - type: nauc_recall_at_3_max + value: 24.393500000000003 + - type: nauc_recall_at_3_std + value: 3.3230999999999997 + - type: nauc_recall_at_3_diff1 + value: 34.7983 + - type: nauc_recall_at_5_max + value: 23.4229 + - type: nauc_recall_at_5_std + value: 6.2542 + - type: nauc_recall_at_5_diff1 + value: 28.8147 + - type: nauc_recall_at_10_max + value: 22.6162 + - type: nauc_recall_at_10_std + value: 6.9113 + - type: nauc_recall_at_10_diff1 + value: 24.617900000000002 + - type: nauc_recall_at_20_max + value: 19.8826 + - type: nauc_recall_at_20_std + value: 6.0004 + - type: nauc_recall_at_20_diff1 + value: 24.0887 + - type: nauc_recall_at_100_max + value: 24.428900000000002 + - type: nauc_recall_at_100_std + value: 18.8358 + - type: nauc_recall_at_100_diff1 + value: 18.6841 + - type: nauc_recall_at_1000_max + value: 34.9059 + - type: nauc_recall_at_1000_std + value: 30.6124 + - type: nauc_recall_at_1000_diff1 + value: 11.7067 + - type: nauc_precision_at_1_max + value: 22.3876 + - type: nauc_precision_at_1_std + value: -0.26649999999999996 + - type: nauc_precision_at_1_diff1 + value: 42.7688 + - type: nauc_precision_at_3_max + value: 24.7919 + - type: nauc_precision_at_3_std + value: 1.3971 + - type: nauc_precision_at_3_diff1 + value: 32.175599999999996 + - type: nauc_precision_at_5_max + value: 25.4503 + - type: nauc_precision_at_5_std + value: 4.4636000000000005 + - type: nauc_precision_at_5_diff1 + value: 25.453599999999998 + - type: nauc_precision_at_10_max + value: 21.1404 + - type: nauc_precision_at_10_std + value: 4.7988 + - type: nauc_precision_at_10_diff1 + value: 17.3144 + - type: nauc_precision_at_20_max + value: 16.4733 + - type: nauc_precision_at_20_std + value: 3.7228999999999997 + - type: nauc_precision_at_20_diff1 + value: 12.853 + - type: nauc_precision_at_100_max + value: 12.5551 + - type: nauc_precision_at_100_std + value: 6.2132 + - type: nauc_precision_at_100_diff1 + value: 1.2163 + - type: nauc_precision_at_1000_max + value: 2.706 + - type: nauc_precision_at_1000_std + value: -0.7363999999999999 + - type: nauc_precision_at_1000_diff1 + value: -6.0556 + - type: nauc_mrr_at_1_max + value: 22.3876 + - type: nauc_mrr_at_1_std + value: -0.26649999999999996 + - type: nauc_mrr_at_1_diff1 + value: 42.7688 + - type: nauc_mrr_at_3_max + value: 24.9398 + - type: nauc_mrr_at_3_std + value: 1.5026 + - type: nauc_mrr_at_3_diff1 + value: 39.2078 + - type: nauc_mrr_at_5_max + value: 24.9525 + - type: nauc_mrr_at_5_std + value: 2.2446 + - type: nauc_mrr_at_5_diff1 + value: 37.9502 + - type: nauc_mrr_at_10_max + value: 24.8361 + - type: nauc_mrr_at_10_std + value: 2.1445 + - type: nauc_mrr_at_10_diff1 + value: 37.4108 + - type: nauc_mrr_at_20_max + value: 24.529300000000003 + - type: nauc_mrr_at_20_std + value: 2.0292 + - type: nauc_mrr_at_20_diff1 + value: 37.3959 + - type: nauc_mrr_at_100_max + value: 24.627299999999998 + - type: nauc_mrr_at_100_std + value: 2.2496 + - type: nauc_mrr_at_100_diff1 + value: 37.4236 + - type: nauc_mrr_at_1000_max + value: 24.6481 + - type: nauc_mrr_at_1000_std + value: 2.2540999999999998 + - type: nauc_mrr_at_1000_diff1 + value: 37.4501 + - type: main_score + value: 32.823 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackPhysicsRetrieval (default) + revision: 79531abbd1fb92d06c6d6315a0cbbbf5bb247ea4 + split: test + type: mteb/cqadupstack-physics + metrics: + - type: ndcg_at_1 + value: 40.135 + - type: ndcg_at_3 + value: 45.062999999999995 + - type: ndcg_at_5 + value: 47.674 + - type: ndcg_at_10 + value: 50.312 + - type: ndcg_at_20 + value: 52.349000000000004 + - type: ndcg_at_100 + value: 55.428 + - type: ndcg_at_1000 + value: 57.202 + - type: map_at_1 + value: 32.757 + - type: map_at_3 + value: 40.722 + - type: map_at_5 + value: 42.656 + - type: map_at_10 + value: 44.162 + - type: map_at_20 + value: 44.889 + - type: map_at_100 + value: 45.454 + - type: map_at_1000 + value: 45.562999999999995 + - type: recall_at_1 + value: 32.757 + - type: recall_at_3 + value: 48.120000000000005 + - type: recall_at_5 + value: 54.666000000000004 + - type: recall_at_10 + value: 62.632 + - type: recall_at_20 + value: 69.592 + - type: recall_at_100 + value: 83.863 + - type: recall_at_1000 + value: 95.065 + - type: precision_at_1 + value: 40.135 + - type: precision_at_3 + value: 21.367 + - type: precision_at_5 + value: 15.265 + - type: precision_at_10 + value: 9.057 + - type: precision_at_20 + value: 5.25 + - type: precision_at_100 + value: 1.347 + - type: precision_at_1000 + value: 0.169 + - type: mrr_at_1 + value: 40.1347 + - type: mrr_at_3 + value: 47.3532 + - type: mrr_at_5 + value: 48.8547 + - type: mrr_at_10 + value: 49.9016 + - type: mrr_at_20 + value: 50.31250000000001 + - type: mrr_at_100 + value: 50.6278 + - type: mrr_at_1000 + value: 50.6652 + - type: nauc_ndcg_at_1_max + value: 38.7881 + - type: nauc_ndcg_at_1_std + value: -8.296000000000001 + - type: nauc_ndcg_at_1_diff1 + value: 52.21130000000001 + - type: nauc_ndcg_at_3_max + value: 38.7708 + - type: nauc_ndcg_at_3_std + value: -6.576700000000001 + - type: nauc_ndcg_at_3_diff1 + value: 48.9321 + - type: nauc_ndcg_at_5_max + value: 38.438 + - type: nauc_ndcg_at_5_std + value: -6.2548 + - type: nauc_ndcg_at_5_diff1 + value: 48.0762 + - type: nauc_ndcg_at_10_max + value: 38.365899999999996 + - type: nauc_ndcg_at_10_std + value: -5.7385 + - type: nauc_ndcg_at_10_diff1 + value: 48.158899999999996 + - type: nauc_ndcg_at_20_max + value: 39.0394 + - type: nauc_ndcg_at_20_std + value: -5.0741000000000005 + - type: nauc_ndcg_at_20_diff1 + value: 48.540499999999994 + - type: nauc_ndcg_at_100_max + value: 39.7277 + - type: nauc_ndcg_at_100_std + value: -2.7447 + - type: nauc_ndcg_at_100_diff1 + value: 47.9735 + - type: nauc_ndcg_at_1000_max + value: 40.0211 + - type: nauc_ndcg_at_1000_std + value: -2.7227 + - type: nauc_ndcg_at_1000_diff1 + value: 48.1857 + - type: nauc_map_at_1_max + value: 33.7229 + - type: nauc_map_at_1_std + value: -12.5585 + - type: nauc_map_at_1_diff1 + value: 54.0852 + - type: nauc_map_at_3_max + value: 36.403 + - type: nauc_map_at_3_std + value: -9.1775 + - type: nauc_map_at_3_diff1 + value: 49.7749 + - type: nauc_map_at_5_max + value: 36.804500000000004 + - type: nauc_map_at_5_std + value: -8.4613 + - type: nauc_map_at_5_diff1 + value: 49.1705 + - type: nauc_map_at_10_max + value: 37.3301 + - type: nauc_map_at_10_std + value: -7.706200000000001 + - type: nauc_map_at_10_diff1 + value: 49.3899 + - type: nauc_map_at_20_max + value: 37.541999999999994 + - type: nauc_map_at_20_std + value: -7.4139 + - type: nauc_map_at_20_diff1 + value: 49.4555 + - type: nauc_map_at_100_max + value: 37.7874 + - type: nauc_map_at_100_std + value: -6.8967 + - type: nauc_map_at_100_diff1 + value: 49.336999999999996 + - type: nauc_map_at_1000_max + value: 37.8174 + - type: nauc_map_at_1000_std + value: -6.8435 + - type: nauc_map_at_1000_diff1 + value: 49.3269 + - type: nauc_recall_at_1_max + value: 33.7229 + - type: nauc_recall_at_1_std + value: -12.5585 + - type: nauc_recall_at_1_diff1 + value: 54.0852 + - type: nauc_recall_at_3_max + value: 34.7265 + - type: nauc_recall_at_3_std + value: -8.2544 + - type: nauc_recall_at_3_diff1 + value: 45.2066 + - type: nauc_recall_at_5_max + value: 34.319 + - type: nauc_recall_at_5_std + value: -6.7825 + - type: nauc_recall_at_5_diff1 + value: 41.783 + - type: nauc_recall_at_10_max + value: 34.5308 + - type: nauc_recall_at_10_std + value: -3.8527 + - type: nauc_recall_at_10_diff1 + value: 40.9153 + - type: nauc_recall_at_20_max + value: 36.6563 + - type: nauc_recall_at_20_std + value: -0.6942 + - type: nauc_recall_at_20_diff1 + value: 41.7078 + - type: nauc_recall_at_100_max + value: 38.7406 + - type: nauc_recall_at_100_std + value: 18.8691 + - type: nauc_recall_at_100_diff1 + value: 34.8788 + - type: nauc_recall_at_1000_max + value: 53.96490000000001 + - type: nauc_recall_at_1000_std + value: 46.1526 + - type: nauc_recall_at_1000_diff1 + value: 34.4075 + - type: nauc_precision_at_1_max + value: 38.7881 + - type: nauc_precision_at_1_std + value: -8.296000000000001 + - type: nauc_precision_at_1_diff1 + value: 52.21130000000001 + - type: nauc_precision_at_3_max + value: 38.4296 + - type: nauc_precision_at_3_std + value: 5.1817 + - type: nauc_precision_at_3_diff1 + value: 32.3129 + - type: nauc_precision_at_5_max + value: 33.9238 + - type: nauc_precision_at_5_std + value: 10.5533 + - type: nauc_precision_at_5_diff1 + value: 22.5911 + - type: nauc_precision_at_10_max + value: 30.967 + - type: nauc_precision_at_10_std + value: 16.371 + - type: nauc_precision_at_10_diff1 + value: 15.714 + - type: nauc_precision_at_20_max + value: 27.0551 + - type: nauc_precision_at_20_std + value: 18.2058 + - type: nauc_precision_at_20_diff1 + value: 10.084 + - type: nauc_precision_at_100_max + value: 18.493000000000002 + - type: nauc_precision_at_100_std + value: 25.315199999999997 + - type: nauc_precision_at_100_diff1 + value: -5.4256 + - type: nauc_precision_at_1000_max + value: 6.7 + - type: nauc_precision_at_1000_std + value: 22.2852 + - type: nauc_precision_at_1000_diff1 + value: -14.102 + - type: nauc_mrr_at_1_max + value: 38.7881 + - type: nauc_mrr_at_1_std + value: -8.296000000000001 + - type: nauc_mrr_at_1_diff1 + value: 52.21130000000001 + - type: nauc_mrr_at_3_max + value: 40.9462 + - type: nauc_mrr_at_3_std + value: -5.224 + - type: nauc_mrr_at_3_diff1 + value: 49.9567 + - type: nauc_mrr_at_5_max + value: 40.6606 + - type: nauc_mrr_at_5_std + value: -5.1892000000000005 + - type: nauc_mrr_at_5_diff1 + value: 49.274499999999996 + - type: nauc_mrr_at_10_max + value: 40.7644 + - type: nauc_mrr_at_10_std + value: -4.7934 + - type: nauc_mrr_at_10_diff1 + value: 49.2337 + - type: nauc_mrr_at_20_max + value: 40.8569 + - type: nauc_mrr_at_20_std + value: -4.7076 + - type: nauc_mrr_at_20_diff1 + value: 49.358999999999995 + - type: nauc_mrr_at_100_max + value: 40.8362 + - type: nauc_mrr_at_100_std + value: -4.5678 + - type: nauc_mrr_at_100_diff1 + value: 49.32 + - type: nauc_mrr_at_1000_max + value: 40.827400000000004 + - type: nauc_mrr_at_1000_std + value: -4.5844000000000005 + - type: nauc_mrr_at_1000_diff1 + value: 49.3213 + - type: main_score + value: 50.312 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackProgrammersRetrieval (default) + revision: 6184bc1440d2dbc7612be22b50686b8826d22b32 + split: test + type: mteb/cqadupstack-programmers + metrics: + - type: ndcg_at_1 + value: 38.013999999999996 + - type: ndcg_at_3 + value: 42.824 + - type: ndcg_at_5 + value: 45.074999999999996 + - type: ndcg_at_10 + value: 47.769 + - type: ndcg_at_20 + value: 49.964 + - type: ndcg_at_100 + value: 53.271 + - type: ndcg_at_1000 + value: 55.217000000000006 + - type: map_at_1 + value: 31.751 + - type: map_at_3 + value: 38.95 + - type: map_at_5 + value: 40.681 + - type: map_at_10 + value: 42.097 + - type: map_at_20 + value: 42.892 + - type: map_at_100 + value: 43.472 + - type: map_at_1000 + value: 43.578 + - type: recall_at_1 + value: 31.751 + - type: recall_at_3 + value: 45.409 + - type: recall_at_5 + value: 51.373000000000005 + - type: recall_at_10 + value: 59.168 + - type: recall_at_20 + value: 66.669 + - type: recall_at_100 + value: 82.26400000000001 + - type: recall_at_1000 + value: 95.017 + - type: precision_at_1 + value: 38.013999999999996 + - type: precision_at_3 + value: 19.977 + - type: precision_at_5 + value: 14.11 + - type: precision_at_10 + value: 8.493 + - type: precision_at_20 + value: 5.0 + - type: precision_at_100 + value: 1.312 + - type: precision_at_1000 + value: 0.165 + - type: mrr_at_1 + value: 38.0137 + - type: mrr_at_3 + value: 44.9772 + - type: mrr_at_5 + value: 46.387 + - type: mrr_at_10 + value: 47.384100000000004 + - type: mrr_at_20 + value: 47.8746 + - type: mrr_at_100 + value: 48.2235 + - type: mrr_at_1000 + value: 48.2699 + - type: nauc_ndcg_at_1_max + value: 35.9967 + - type: nauc_ndcg_at_1_std + value: 4.926500000000001 + - type: nauc_ndcg_at_1_diff1 + value: 43.5414 + - type: nauc_ndcg_at_3_max + value: 35.4574 + - type: nauc_ndcg_at_3_std + value: 2.6951 + - type: nauc_ndcg_at_3_diff1 + value: 38.5888 + - type: nauc_ndcg_at_5_max + value: 35.7783 + - type: nauc_ndcg_at_5_std + value: 3.5970000000000004 + - type: nauc_ndcg_at_5_diff1 + value: 38.107 + - type: nauc_ndcg_at_10_max + value: 35.9047 + - type: nauc_ndcg_at_10_std + value: 5.3849 + - type: nauc_ndcg_at_10_diff1 + value: 37.6917 + - type: nauc_ndcg_at_20_max + value: 37.4203 + - type: nauc_ndcg_at_20_std + value: 7.5072 + - type: nauc_ndcg_at_20_diff1 + value: 37.9429 + - type: nauc_ndcg_at_100_max + value: 37.913000000000004 + - type: nauc_ndcg_at_100_std + value: 8.8726 + - type: nauc_ndcg_at_100_diff1 + value: 37.8018 + - type: nauc_ndcg_at_1000_max + value: 37.7521 + - type: nauc_ndcg_at_1000_std + value: 8.0898 + - type: nauc_ndcg_at_1000_diff1 + value: 38.188 + - type: nauc_map_at_1_max + value: 30.6039 + - type: nauc_map_at_1_std + value: -1.1973 + - type: nauc_map_at_1_diff1 + value: 44.4956 + - type: nauc_map_at_3_max + value: 33.79 + - type: nauc_map_at_3_std + value: 0.7224999999999999 + - type: nauc_map_at_3_diff1 + value: 40.5918 + - type: nauc_map_at_5_max + value: 34.799 + - type: nauc_map_at_5_std + value: 1.9663 + - type: nauc_map_at_5_diff1 + value: 40.119 + - type: nauc_map_at_10_max + value: 35.0036 + - type: nauc_map_at_10_std + value: 2.9479 + - type: nauc_map_at_10_diff1 + value: 39.725899999999996 + - type: nauc_map_at_20_max + value: 35.6907 + - type: nauc_map_at_20_std + value: 3.7684 + - type: nauc_map_at_20_diff1 + value: 39.6845 + - type: nauc_map_at_100_max + value: 35.8249 + - type: nauc_map_at_100_std + value: 4.123 + - type: nauc_map_at_100_diff1 + value: 39.6397 + - type: nauc_map_at_1000_max + value: 35.8146 + - type: nauc_map_at_1000_std + value: 4.100899999999999 + - type: nauc_map_at_1000_diff1 + value: 39.6511 + - type: nauc_recall_at_1_max + value: 30.6039 + - type: nauc_recall_at_1_std + value: -1.1973 + - type: nauc_recall_at_1_diff1 + value: 44.4956 + - type: nauc_recall_at_3_max + value: 33.9619 + - type: nauc_recall_at_3_std + value: 1.3599 + - type: nauc_recall_at_3_diff1 + value: 36.673899999999996 + - type: nauc_recall_at_5_max + value: 34.798899999999996 + - type: nauc_recall_at_5_std + value: 3.9083 + - type: nauc_recall_at_5_diff1 + value: 34.2275 + - type: nauc_recall_at_10_max + value: 34.3508 + - type: nauc_recall_at_10_std + value: 8.6454 + - type: nauc_recall_at_10_diff1 + value: 31.9422 + - type: nauc_recall_at_20_max + value: 39.1475 + - type: nauc_recall_at_20_std + value: 17.0303 + - type: nauc_recall_at_20_diff1 + value: 32.138099999999994 + - type: nauc_recall_at_100_max + value: 43.452 + - type: nauc_recall_at_100_std + value: 31.8449 + - type: nauc_recall_at_100_diff1 + value: 27.38 + - type: nauc_recall_at_1000_max + value: 56.720000000000006 + - type: nauc_recall_at_1000_std + value: 51.5088 + - type: nauc_recall_at_1000_diff1 + value: 28.131099999999996 + - type: nauc_precision_at_1_max + value: 35.9967 + - type: nauc_precision_at_1_std + value: 4.926500000000001 + - type: nauc_precision_at_1_diff1 + value: 43.5414 + - type: nauc_precision_at_3_max + value: 36.204 + - type: nauc_precision_at_3_std + value: 9.6793 + - type: nauc_precision_at_3_diff1 + value: 22.8807 + - type: nauc_precision_at_5_max + value: 34.226 + - type: nauc_precision_at_5_std + value: 14.0818 + - type: nauc_precision_at_5_diff1 + value: 16.223000000000003 + - type: nauc_precision_at_10_max + value: 28.3789 + - type: nauc_precision_at_10_std + value: 18.8125 + - type: nauc_precision_at_10_diff1 + value: 7.382700000000001 + - type: nauc_precision_at_20_max + value: 26.151600000000002 + - type: nauc_precision_at_20_std + value: 22.352 + - type: nauc_precision_at_20_diff1 + value: 1.0934 + - type: nauc_precision_at_100_max + value: 13.886399999999998 + - type: nauc_precision_at_100_std + value: 21.5356 + - type: nauc_precision_at_100_diff1 + value: -10.3265 + - type: nauc_precision_at_1000_max + value: -1.5730000000000002 + - type: nauc_precision_at_1000_std + value: 9.9943 + - type: nauc_precision_at_1000_diff1 + value: -18.5193 + - type: nauc_mrr_at_1_max + value: 35.9967 + - type: nauc_mrr_at_1_std + value: 4.926500000000001 + - type: nauc_mrr_at_1_diff1 + value: 43.5414 + - type: nauc_mrr_at_3_max + value: 37.1377 + - type: nauc_mrr_at_3_std + value: 5.6196 + - type: nauc_mrr_at_3_diff1 + value: 38.9643 + - type: nauc_mrr_at_5_max + value: 36.945499999999996 + - type: nauc_mrr_at_5_std + value: 5.9594000000000005 + - type: nauc_mrr_at_5_diff1 + value: 38.431 + - type: nauc_mrr_at_10_max + value: 37.094300000000004 + - type: nauc_mrr_at_10_std + value: 6.6665 + - type: nauc_mrr_at_10_diff1 + value: 38.4148 + - type: nauc_mrr_at_20_max + value: 37.283100000000005 + - type: nauc_mrr_at_20_std + value: 7.0301 + - type: nauc_mrr_at_20_diff1 + value: 38.6425 + - type: nauc_mrr_at_100_max + value: 37.312200000000004 + - type: nauc_mrr_at_100_std + value: 7.0826 + - type: nauc_mrr_at_100_diff1 + value: 38.689800000000005 + - type: nauc_mrr_at_1000_max + value: 37.319 + - type: nauc_mrr_at_1000_std + value: 7.0653999999999995 + - type: nauc_mrr_at_1000_diff1 + value: 38.7106 + - type: main_score + value: 47.769 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackRetrieval (default) + revision: CQADupstackRetrieval_is_a_combined_dataset + split: test + type: CQADupstackRetrieval_is_a_combined_dataset + metrics: + - type: main_score + value: 46.10300000000001 + - type: ndcg_at_10 + value: 46.10300000000001 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackStatsRetrieval (default) + revision: 65ac3a16b8e91f9cee4c9828cc7c335575432a2a + split: test + type: mteb/cqadupstack-stats + metrics: + - type: ndcg_at_1 + value: 32.362 + - type: ndcg_at_3 + value: 36.026 + - type: ndcg_at_5 + value: 38.122 + - type: ndcg_at_10 + value: 40.174 + - type: ndcg_at_20 + value: 41.836 + - type: ndcg_at_100 + value: 44.444 + - type: ndcg_at_1000 + value: 46.929 + - type: map_at_1 + value: 28.871999999999996 + - type: map_at_3 + value: 33.613 + - type: map_at_5 + value: 35.007 + - type: map_at_10 + value: 35.976 + - type: map_at_20 + value: 36.496 + - type: map_at_100 + value: 36.895 + - type: map_at_1000 + value: 36.994 + - type: recall_at_1 + value: 28.871999999999996 + - type: recall_at_3 + value: 38.705 + - type: recall_at_5 + value: 43.821 + - type: recall_at_10 + value: 49.921 + - type: recall_at_20 + value: 56.163 + - type: recall_at_100 + value: 69.084 + - type: recall_at_1000 + value: 87.35000000000001 + - type: precision_at_1 + value: 32.362 + - type: precision_at_3 + value: 15.184000000000001 + - type: precision_at_5 + value: 10.583 + - type: precision_at_10 + value: 6.166 + - type: precision_at_20 + value: 3.512 + - type: precision_at_100 + value: 0.897 + - type: precision_at_1000 + value: 0.11900000000000001 + - type: mrr_at_1 + value: 32.362 + - type: mrr_at_3 + value: 36.937599999999996 + - type: mrr_at_5 + value: 38.1416 + - type: mrr_at_10 + value: 39.012299999999996 + - type: mrr_at_20 + value: 39.4119 + - type: mrr_at_100 + value: 39.745200000000004 + - type: mrr_at_1000 + value: 39.8191 + - type: nauc_ndcg_at_1_max + value: 39.396300000000004 + - type: nauc_ndcg_at_1_std + value: 0.8482 + - type: nauc_ndcg_at_1_diff1 + value: 52.376999999999995 + - type: nauc_ndcg_at_3_max + value: 39.0785 + - type: nauc_ndcg_at_3_std + value: 3.2739 + - type: nauc_ndcg_at_3_diff1 + value: 48.3207 + - type: nauc_ndcg_at_5_max + value: 38.4648 + - type: nauc_ndcg_at_5_std + value: 3.3379 + - type: nauc_ndcg_at_5_diff1 + value: 47.468500000000006 + - type: nauc_ndcg_at_10_max + value: 39.0329 + - type: nauc_ndcg_at_10_std + value: 4.0895 + - type: nauc_ndcg_at_10_diff1 + value: 46.1268 + - type: nauc_ndcg_at_20_max + value: 38.359 + - type: nauc_ndcg_at_20_std + value: 4.2744 + - type: nauc_ndcg_at_20_diff1 + value: 45.1661 + - type: nauc_ndcg_at_100_max + value: 39.461 + - type: nauc_ndcg_at_100_std + value: 7.2038 + - type: nauc_ndcg_at_100_diff1 + value: 44.809 + - type: nauc_ndcg_at_1000_max + value: 39.875699999999995 + - type: nauc_ndcg_at_1000_std + value: 6.9621 + - type: nauc_ndcg_at_1000_diff1 + value: 45.473200000000006 + - type: nauc_map_at_1_max + value: 35.936800000000005 + - type: nauc_map_at_1_std + value: -3.2637 + - type: nauc_map_at_1_diff1 + value: 52.3431 + - type: nauc_map_at_3_max + value: 37.8006 + - type: nauc_map_at_3_std + value: 0.7727999999999999 + - type: nauc_map_at_3_diff1 + value: 49.1872 + - type: nauc_map_at_5_max + value: 37.932300000000005 + - type: nauc_map_at_5_std + value: 1.4745 + - type: nauc_map_at_5_diff1 + value: 48.8466 + - type: nauc_map_at_10_max + value: 38.4041 + - type: nauc_map_at_10_std + value: 2.0481 + - type: nauc_map_at_10_diff1 + value: 48.2292 + - type: nauc_map_at_20_max + value: 38.1992 + - type: nauc_map_at_20_std + value: 2.1198 + - type: nauc_map_at_20_diff1 + value: 47.9169 + - type: nauc_map_at_100_max + value: 38.3504 + - type: nauc_map_at_100_std + value: 2.5100000000000002 + - type: nauc_map_at_100_diff1 + value: 47.8259 + - type: nauc_map_at_1000_max + value: 38.3865 + - type: nauc_map_at_1000_std + value: 2.5181999999999998 + - type: nauc_map_at_1000_diff1 + value: 47.853699999999996 + - type: nauc_recall_at_1_max + value: 35.936800000000005 + - type: nauc_recall_at_1_std + value: -3.2637 + - type: nauc_recall_at_1_diff1 + value: 52.3431 + - type: nauc_recall_at_3_max + value: 37.227700000000006 + - type: nauc_recall_at_3_std + value: 3.8813 + - type: nauc_recall_at_3_diff1 + value: 44.8185 + - type: nauc_recall_at_5_max + value: 35.963 + - type: nauc_recall_at_5_std + value: 4.9497 + - type: nauc_recall_at_5_diff1 + value: 42.6322 + - type: nauc_recall_at_10_max + value: 37.358000000000004 + - type: nauc_recall_at_10_std + value: 6.6888000000000005 + - type: nauc_recall_at_10_diff1 + value: 38.7639 + - type: nauc_recall_at_20_max + value: 34.2341 + - type: nauc_recall_at_20_std + value: 7.0213 + - type: nauc_recall_at_20_diff1 + value: 34.8021 + - type: nauc_recall_at_100_max + value: 39.406600000000005 + - type: nauc_recall_at_100_std + value: 25.7393 + - type: nauc_recall_at_100_diff1 + value: 29.9173 + - type: nauc_recall_at_1000_max + value: 45.287 + - type: nauc_recall_at_1000_std + value: 38.572 + - type: nauc_recall_at_1000_diff1 + value: 26.744 + - type: nauc_precision_at_1_max + value: 39.396300000000004 + - type: nauc_precision_at_1_std + value: 0.8482 + - type: nauc_precision_at_1_diff1 + value: 52.376999999999995 + - type: nauc_precision_at_3_max + value: 42.1919 + - type: nauc_precision_at_3_std + value: 13.9189 + - type: nauc_precision_at_3_diff1 + value: 40.2337 + - type: nauc_precision_at_5_max + value: 39.8644 + - type: nauc_precision_at_5_std + value: 15.656900000000002 + - type: nauc_precision_at_5_diff1 + value: 35.1421 + - type: nauc_precision_at_10_max + value: 40.7678 + - type: nauc_precision_at_10_std + value: 19.5881 + - type: nauc_precision_at_10_diff1 + value: 28.822300000000002 + - type: nauc_precision_at_20_max + value: 35.4842 + - type: nauc_precision_at_20_std + value: 20.6978 + - type: nauc_precision_at_20_diff1 + value: 21.4608 + - type: nauc_precision_at_100_max + value: 33.211400000000005 + - type: nauc_precision_at_100_std + value: 31.5029 + - type: nauc_precision_at_100_diff1 + value: 13.0526 + - type: nauc_precision_at_1000_max + value: 21.6976 + - type: nauc_precision_at_1000_std + value: 26.4203 + - type: nauc_precision_at_1000_diff1 + value: 2.6056 + - type: nauc_mrr_at_1_max + value: 39.396300000000004 + - type: nauc_mrr_at_1_std + value: 0.8482 + - type: nauc_mrr_at_1_diff1 + value: 52.376999999999995 + - type: nauc_mrr_at_3_max + value: 40.191 + - type: nauc_mrr_at_3_std + value: 3.9919999999999995 + - type: nauc_mrr_at_3_diff1 + value: 49.2714 + - type: nauc_mrr_at_5_max + value: 39.9654 + - type: nauc_mrr_at_5_std + value: 4.0258 + - type: nauc_mrr_at_5_diff1 + value: 48.6599 + - type: nauc_mrr_at_10_max + value: 40.1413 + - type: nauc_mrr_at_10_std + value: 4.389 + - type: nauc_mrr_at_10_diff1 + value: 48.0272 + - type: nauc_mrr_at_20_max + value: 39.9265 + - type: nauc_mrr_at_20_std + value: 4.3462 + - type: nauc_mrr_at_20_diff1 + value: 47.8592 + - type: nauc_mrr_at_100_max + value: 40.0623 + - type: nauc_mrr_at_100_std + value: 4.698 + - type: nauc_mrr_at_100_diff1 + value: 47.8456 + - type: nauc_mrr_at_1000_max + value: 40.0698 + - type: nauc_mrr_at_1000_std + value: 4.6803 + - type: nauc_mrr_at_1000_diff1 + value: 47.8659 + - type: main_score + value: 40.174 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackTexRetrieval (default) + revision: 46989137a86843e03a6195de44b09deda022eec7 + split: test + type: mteb/cqadupstack-tex + metrics: + - type: ndcg_at_1 + value: 25.155 + - type: ndcg_at_3 + value: 29.339 + - type: ndcg_at_5 + value: 31.452999999999996 + - type: ndcg_at_10 + value: 33.937 + - type: ndcg_at_20 + value: 36.018 + - type: ndcg_at_100 + value: 39.531 + - type: ndcg_at_1000 + value: 42.22 + - type: map_at_1 + value: 20.874000000000002 + - type: map_at_3 + value: 26.345000000000002 + - type: map_at_5 + value: 27.773999999999997 + - type: map_at_10 + value: 28.965999999999998 + - type: map_at_20 + value: 29.625 + - type: map_at_100 + value: 30.188 + - type: map_at_1000 + value: 30.314000000000004 + - type: recall_at_1 + value: 20.874000000000002 + - type: recall_at_3 + value: 31.984 + - type: recall_at_5 + value: 37.467 + - type: recall_at_10 + value: 44.774 + - type: recall_at_20 + value: 52.323 + - type: recall_at_100 + value: 69.549 + - type: recall_at_1000 + value: 88.419 + - type: precision_at_1 + value: 25.155 + - type: precision_at_3 + value: 13.719000000000001 + - type: precision_at_5 + value: 9.841999999999999 + - type: precision_at_10 + value: 6.069999999999999 + - type: precision_at_20 + value: 3.6799999999999997 + - type: precision_at_100 + value: 1.045 + - type: precision_at_1000 + value: 0.146 + - type: mrr_at_1 + value: 25.1549 + - type: mrr_at_3 + value: 30.7123 + - type: mrr_at_5 + value: 32.0148 + - type: mrr_at_10 + value: 33.035199999999996 + - type: mrr_at_20 + value: 33.5778 + - type: mrr_at_100 + value: 34.0001 + - type: mrr_at_1000 + value: 34.070499999999996 + - type: nauc_ndcg_at_1_max + value: 34.6903 + - type: nauc_ndcg_at_1_std + value: -0.48469999999999996 + - type: nauc_ndcg_at_1_diff1 + value: 41.827799999999996 + - type: nauc_ndcg_at_3_max + value: 34.7107 + - type: nauc_ndcg_at_3_std + value: 1.2525 + - type: nauc_ndcg_at_3_diff1 + value: 36.09 + - type: nauc_ndcg_at_5_max + value: 34.363899999999994 + - type: nauc_ndcg_at_5_std + value: 1.187 + - type: nauc_ndcg_at_5_diff1 + value: 35.5019 + - type: nauc_ndcg_at_10_max + value: 34.1261 + - type: nauc_ndcg_at_10_std + value: 2.0704000000000002 + - type: nauc_ndcg_at_10_diff1 + value: 35.0098 + - type: nauc_ndcg_at_20_max + value: 34.5028 + - type: nauc_ndcg_at_20_std + value: 2.9973 + - type: nauc_ndcg_at_20_diff1 + value: 34.6486 + - type: nauc_ndcg_at_100_max + value: 34.8192 + - type: nauc_ndcg_at_100_std + value: 4.4281 + - type: nauc_ndcg_at_100_diff1 + value: 34.252500000000005 + - type: nauc_ndcg_at_1000_max + value: 34.8293 + - type: nauc_ndcg_at_1000_std + value: 4.2747 + - type: nauc_ndcg_at_1000_diff1 + value: 34.5083 + - type: nauc_map_at_1_max + value: 31.448700000000002 + - type: nauc_map_at_1_std + value: -1.5652 + - type: nauc_map_at_1_diff1 + value: 42.3532 + - type: nauc_map_at_3_max + value: 33.458 + - type: nauc_map_at_3_std + value: 0.372 + - type: nauc_map_at_3_diff1 + value: 37.6257 + - type: nauc_map_at_5_max + value: 33.3902 + - type: nauc_map_at_5_std + value: 0.2957 + - type: nauc_map_at_5_diff1 + value: 37.0708 + - type: nauc_map_at_10_max + value: 33.4473 + - type: nauc_map_at_10_std + value: 0.7451 + - type: nauc_map_at_10_diff1 + value: 36.7872 + - type: nauc_map_at_20_max + value: 33.6705 + - type: nauc_map_at_20_std + value: 1.0755000000000001 + - type: nauc_map_at_20_diff1 + value: 36.6791 + - type: nauc_map_at_100_max + value: 33.772200000000005 + - type: nauc_map_at_100_std + value: 1.308 + - type: nauc_map_at_100_diff1 + value: 36.5896 + - type: nauc_map_at_1000_max + value: 33.7881 + - type: nauc_map_at_1000_std + value: 1.3087 + - type: nauc_map_at_1000_diff1 + value: 36.5978 + - type: nauc_recall_at_1_max + value: 31.448700000000002 + - type: nauc_recall_at_1_std + value: -1.5652 + - type: nauc_recall_at_1_diff1 + value: 42.3532 + - type: nauc_recall_at_3_max + value: 33.7171 + - type: nauc_recall_at_3_std + value: 2.4527 + - type: nauc_recall_at_3_diff1 + value: 32.6832 + - type: nauc_recall_at_5_max + value: 32.7828 + - type: nauc_recall_at_5_std + value: 2.0332 + - type: nauc_recall_at_5_diff1 + value: 30.8446 + - type: nauc_recall_at_10_max + value: 31.6463 + - type: nauc_recall_at_10_std + value: 4.3727 + - type: nauc_recall_at_10_diff1 + value: 29.1731 + - type: nauc_recall_at_20_max + value: 31.968999999999998 + - type: nauc_recall_at_20_std + value: 7.5392 + - type: nauc_recall_at_20_diff1 + value: 26.961299999999998 + - type: nauc_recall_at_100_max + value: 32.9142 + - type: nauc_recall_at_100_std + value: 17.2332 + - type: nauc_recall_at_100_diff1 + value: 22.0707 + - type: nauc_recall_at_1000_max + value: 32.1463 + - type: nauc_recall_at_1000_std + value: 29.664600000000004 + - type: nauc_recall_at_1000_diff1 + value: 13.9131 + - type: nauc_precision_at_1_max + value: 34.6903 + - type: nauc_precision_at_1_std + value: -0.48469999999999996 + - type: nauc_precision_at_1_diff1 + value: 41.827799999999996 + - type: nauc_precision_at_3_max + value: 36.8823 + - type: nauc_precision_at_3_std + value: 3.7052 + - type: nauc_precision_at_3_diff1 + value: 29.505599999999998 + - type: nauc_precision_at_5_max + value: 35.106 + - type: nauc_precision_at_5_std + value: 3.9923 + - type: nauc_precision_at_5_diff1 + value: 25.684099999999997 + - type: nauc_precision_at_10_max + value: 32.1139 + - type: nauc_precision_at_10_std + value: 7.097100000000001 + - type: nauc_precision_at_10_diff1 + value: 20.521 + - type: nauc_precision_at_20_max + value: 30.3506 + - type: nauc_precision_at_20_std + value: 9.7899 + - type: nauc_precision_at_20_diff1 + value: 16.106 + - type: nauc_precision_at_100_max + value: 23.7062 + - type: nauc_precision_at_100_std + value: 12.7852 + - type: nauc_precision_at_100_diff1 + value: 5.9668 + - type: nauc_precision_at_1000_max + value: 13.6273 + - type: nauc_precision_at_1000_std + value: 7.0956 + - type: nauc_precision_at_1000_diff1 + value: -3.6863 + - type: nauc_mrr_at_1_max + value: 34.6903 + - type: nauc_mrr_at_1_std + value: -0.48469999999999996 + - type: nauc_mrr_at_1_diff1 + value: 41.827799999999996 + - type: nauc_mrr_at_3_max + value: 35.826 + - type: nauc_mrr_at_3_std + value: 1.3141999999999998 + - type: nauc_mrr_at_3_diff1 + value: 37.1995 + - type: nauc_mrr_at_5_max + value: 35.6178 + - type: nauc_mrr_at_5_std + value: 1.3211 + - type: nauc_mrr_at_5_diff1 + value: 36.8396 + - type: nauc_mrr_at_10_max + value: 35.4784 + - type: nauc_mrr_at_10_std + value: 1.6153 + - type: nauc_mrr_at_10_diff1 + value: 36.6262 + - type: nauc_mrr_at_20_max + value: 35.5478 + - type: nauc_mrr_at_20_std + value: 1.8614 + - type: nauc_mrr_at_20_diff1 + value: 36.5754 + - type: nauc_mrr_at_100_max + value: 35.5825 + - type: nauc_mrr_at_100_std + value: 1.9792 + - type: nauc_mrr_at_100_diff1 + value: 36.5758 + - type: nauc_mrr_at_1000_max + value: 35.5811 + - type: nauc_mrr_at_1000_std + value: 1.9691 + - type: nauc_mrr_at_1000_diff1 + value: 36.587399999999995 + - type: main_score + value: 33.937 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackUnixRetrieval (default) + revision: 6c6430d3a6d36f8d2a829195bc5dc94d7e063e53 + split: test + type: mteb/cqadupstack-unix + metrics: + - type: ndcg_at_1 + value: 36.381 + - type: ndcg_at_3 + value: 41.605 + - type: ndcg_at_5 + value: 43.854 + - type: ndcg_at_10 + value: 46.831 + - type: ndcg_at_20 + value: 49.114999999999995 + - type: ndcg_at_100 + value: 52.071 + - type: ndcg_at_1000 + value: 53.864999999999995 + - type: map_at_1 + value: 30.957 + - type: map_at_3 + value: 38.074999999999996 + - type: map_at_5 + value: 39.732 + - type: map_at_10 + value: 41.187000000000005 + - type: map_at_20 + value: 41.94 + - type: map_at_100 + value: 42.447 + - type: map_at_1000 + value: 42.536 + - type: recall_at_1 + value: 30.957 + - type: recall_at_3 + value: 45.213 + - type: recall_at_5 + value: 51.196 + - type: recall_at_10 + value: 59.724 + - type: recall_at_20 + value: 67.837 + - type: recall_at_100 + value: 81.843 + - type: recall_at_1000 + value: 93.91000000000001 + - type: precision_at_1 + value: 36.381 + - type: precision_at_3 + value: 18.999 + - type: precision_at_5 + value: 13.172 + - type: precision_at_10 + value: 7.938000000000001 + - type: precision_at_20 + value: 4.6129999999999995 + - type: precision_at_100 + value: 1.172 + - type: precision_at_1000 + value: 0.14300000000000002 + - type: mrr_at_1 + value: 36.3806 + - type: mrr_at_3 + value: 42.7239 + - type: mrr_at_5 + value: 44.0905 + - type: mrr_at_10 + value: 45.2951 + - type: mrr_at_20 + value: 45.8788 + - type: mrr_at_100 + value: 46.1807 + - type: mrr_at_1000 + value: 46.226800000000004 + - type: nauc_ndcg_at_1_max + value: 47.0214 + - type: nauc_ndcg_at_1_std + value: -0.8086 + - type: nauc_ndcg_at_1_diff1 + value: 55.931200000000004 + - type: nauc_ndcg_at_3_max + value: 44.829299999999996 + - type: nauc_ndcg_at_3_std + value: 0.6224000000000001 + - type: nauc_ndcg_at_3_diff1 + value: 49.7765 + - type: nauc_ndcg_at_5_max + value: 44.3325 + - type: nauc_ndcg_at_5_std + value: 0.1854 + - type: nauc_ndcg_at_5_diff1 + value: 49.0426 + - type: nauc_ndcg_at_10_max + value: 44.358599999999996 + - type: nauc_ndcg_at_10_std + value: 0.6905 + - type: nauc_ndcg_at_10_diff1 + value: 48.1902 + - type: nauc_ndcg_at_20_max + value: 45.018 + - type: nauc_ndcg_at_20_std + value: 1.555 + - type: nauc_ndcg_at_20_diff1 + value: 48.2645 + - type: nauc_ndcg_at_100_max + value: 45.3244 + - type: nauc_ndcg_at_100_std + value: 3.0655 + - type: nauc_ndcg_at_100_diff1 + value: 48.1011 + - type: nauc_ndcg_at_1000_max + value: 45.2297 + - type: nauc_ndcg_at_1000_std + value: 2.5452 + - type: nauc_ndcg_at_1000_diff1 + value: 48.4179 + - type: nauc_map_at_1_max + value: 44.1846 + - type: nauc_map_at_1_std + value: -2.661 + - type: nauc_map_at_1_diff1 + value: 58.4395 + - type: nauc_map_at_3_max + value: 44.7697 + - type: nauc_map_at_3_std + value: -0.3776 + - type: nauc_map_at_3_diff1 + value: 52.7119 + - type: nauc_map_at_5_max + value: 44.6708 + - type: nauc_map_at_5_std + value: -0.4622 + - type: nauc_map_at_5_diff1 + value: 51.8622 + - type: nauc_map_at_10_max + value: 44.7631 + - type: nauc_map_at_10_std + value: -0.2403 + - type: nauc_map_at_10_diff1 + value: 51.439299999999996 + - type: nauc_map_at_20_max + value: 45.0612 + - type: nauc_map_at_20_std + value: 0.0038000000000000004 + - type: nauc_map_at_20_diff1 + value: 51.3768 + - type: nauc_map_at_100_max + value: 45.137 + - type: nauc_map_at_100_std + value: 0.2717 + - type: nauc_map_at_100_diff1 + value: 51.316700000000004 + - type: nauc_map_at_1000_max + value: 45.1229 + - type: nauc_map_at_1000_std + value: 0.2513 + - type: nauc_map_at_1000_diff1 + value: 51.3133 + - type: nauc_recall_at_1_max + value: 44.1846 + - type: nauc_recall_at_1_std + value: -2.661 + - type: nauc_recall_at_1_diff1 + value: 58.4395 + - type: nauc_recall_at_3_max + value: 41.656 + - type: nauc_recall_at_3_std + value: 1.6587999999999998 + - type: nauc_recall_at_3_diff1 + value: 44.9322 + - type: nauc_recall_at_5_max + value: 40.501 + - type: nauc_recall_at_5_std + value: 1.1215 + - type: nauc_recall_at_5_diff1 + value: 41.7702 + - type: nauc_recall_at_10_max + value: 39.577400000000004 + - type: nauc_recall_at_10_std + value: 2.172 + - type: nauc_recall_at_10_diff1 + value: 38.0253 + - type: nauc_recall_at_20_max + value: 41.1537 + - type: nauc_recall_at_20_std + value: 6.1195 + - type: nauc_recall_at_20_diff1 + value: 37.391400000000004 + - type: nauc_recall_at_100_max + value: 42.2577 + - type: nauc_recall_at_100_std + value: 20.7745 + - type: nauc_recall_at_100_diff1 + value: 32.8151 + - type: nauc_recall_at_1000_max + value: 43.5594 + - type: nauc_recall_at_1000_std + value: 37.6573 + - type: nauc_recall_at_1000_diff1 + value: 29.7545 + - type: nauc_precision_at_1_max + value: 47.0214 + - type: nauc_precision_at_1_std + value: -0.8086 + - type: nauc_precision_at_1_diff1 + value: 55.931200000000004 + - type: nauc_precision_at_3_max + value: 39.4995 + - type: nauc_precision_at_3_std + value: 5.0051 + - type: nauc_precision_at_3_diff1 + value: 32.0456 + - type: nauc_precision_at_5_max + value: 34.972500000000004 + - type: nauc_precision_at_5_std + value: 5.1238 + - type: nauc_precision_at_5_diff1 + value: 24.2515 + - type: nauc_precision_at_10_max + value: 28.364099999999997 + - type: nauc_precision_at_10_std + value: 6.0539000000000005 + - type: nauc_precision_at_10_diff1 + value: 14.192599999999999 + - type: nauc_precision_at_20_max + value: 25.7353 + - type: nauc_precision_at_20_std + value: 8.860999999999999 + - type: nauc_precision_at_20_diff1 + value: 7.0925 + - type: nauc_precision_at_100_max + value: 11.8965 + - type: nauc_precision_at_100_std + value: 13.143099999999999 + - type: nauc_precision_at_100_diff1 + value: -8.5811 + - type: nauc_precision_at_1000_max + value: -3.7232000000000003 + - type: nauc_precision_at_1000_std + value: 6.392 + - type: nauc_precision_at_1000_diff1 + value: -20.5151 + - type: nauc_mrr_at_1_max + value: 47.0214 + - type: nauc_mrr_at_1_std + value: -0.8086 + - type: nauc_mrr_at_1_diff1 + value: 55.931200000000004 + - type: nauc_mrr_at_3_max + value: 45.6591 + - type: nauc_mrr_at_3_std + value: 0.6383 + - type: nauc_mrr_at_3_diff1 + value: 50.0407 + - type: nauc_mrr_at_5_max + value: 45.7236 + - type: nauc_mrr_at_5_std + value: 0.5502 + - type: nauc_mrr_at_5_diff1 + value: 49.6432 + - type: nauc_mrr_at_10_max + value: 45.6287 + - type: nauc_mrr_at_10_std + value: 0.6239 + - type: nauc_mrr_at_10_diff1 + value: 49.391200000000005 + - type: nauc_mrr_at_20_max + value: 45.704899999999995 + - type: nauc_mrr_at_20_std + value: 0.7987 + - type: nauc_mrr_at_20_diff1 + value: 49.4844 + - type: nauc_mrr_at_100_max + value: 45.708 + - type: nauc_mrr_at_100_std + value: 0.8823 + - type: nauc_mrr_at_100_diff1 + value: 49.5323 + - type: nauc_mrr_at_1000_max + value: 45.7135 + - type: nauc_mrr_at_1000_std + value: 0.8635999999999999 + - type: nauc_mrr_at_1000_diff1 + value: 49.5497 + - type: main_score + value: 46.831 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackWebmastersRetrieval (default) + revision: 160c094312a0e1facb97e55eeddb698c0abe3571 + split: test + type: mteb/cqadupstack-webmasters + metrics: + - type: ndcg_at_1 + value: 34.98 + - type: ndcg_at_3 + value: 39.911 + - type: ndcg_at_5 + value: 42.21 + - type: ndcg_at_10 + value: 45.539 + - type: ndcg_at_20 + value: 47.964 + - type: ndcg_at_100 + value: 51.642999999999994 + - type: ndcg_at_1000 + value: 53.647 + - type: map_at_1 + value: 30.034 + - type: map_at_3 + value: 35.97 + - type: map_at_5 + value: 37.635999999999996 + - type: map_at_10 + value: 39.367999999999995 + - type: map_at_20 + value: 40.328 + - type: map_at_100 + value: 41.158 + - type: map_at_1000 + value: 41.366 + - type: recall_at_1 + value: 30.034 + - type: recall_at_3 + value: 42.006 + - type: recall_at_5 + value: 47.843 + - type: recall_at_10 + value: 57.568 + - type: recall_at_20 + value: 66.493 + - type: recall_at_100 + value: 84.136 + - type: recall_at_1000 + value: 95.631 + - type: precision_at_1 + value: 34.98 + - type: precision_at_3 + value: 18.116 + - type: precision_at_5 + value: 13.202 + - type: precision_at_10 + value: 8.616999999999999 + - type: precision_at_20 + value: 5.425 + - type: precision_at_100 + value: 1.6260000000000001 + - type: precision_at_1000 + value: 0.249 + - type: mrr_at_1 + value: 34.9802 + - type: mrr_at_3 + value: 41.172599999999996 + - type: mrr_at_5 + value: 42.4671 + - type: mrr_at_10 + value: 43.8709 + - type: mrr_at_20 + value: 44.4684 + - type: mrr_at_100 + value: 44.8617 + - type: mrr_at_1000 + value: 44.9033 + - type: nauc_ndcg_at_1_max + value: 36.1514 + - type: nauc_ndcg_at_1_std + value: 6.7383 + - type: nauc_ndcg_at_1_diff1 + value: 49.9936 + - type: nauc_ndcg_at_3_max + value: 38.3225 + - type: nauc_ndcg_at_3_std + value: 8.0985 + - type: nauc_ndcg_at_3_diff1 + value: 42.9416 + - type: nauc_ndcg_at_5_max + value: 39.4299 + - type: nauc_ndcg_at_5_std + value: 9.2335 + - type: nauc_ndcg_at_5_diff1 + value: 43.4214 + - type: nauc_ndcg_at_10_max + value: 39.1123 + - type: nauc_ndcg_at_10_std + value: 9.4134 + - type: nauc_ndcg_at_10_diff1 + value: 42.6415 + - type: nauc_ndcg_at_20_max + value: 38.9531 + - type: nauc_ndcg_at_20_std + value: 9.707 + - type: nauc_ndcg_at_20_diff1 + value: 43.0215 + - type: nauc_ndcg_at_100_max + value: 40.3045 + - type: nauc_ndcg_at_100_std + value: 11.304400000000001 + - type: nauc_ndcg_at_100_diff1 + value: 43.0846 + - type: nauc_ndcg_at_1000_max + value: 39.9421 + - type: nauc_ndcg_at_1000_std + value: 11.1666 + - type: nauc_ndcg_at_1000_diff1 + value: 43.3505 + - type: nauc_map_at_1_max + value: 34.735 + - type: nauc_map_at_1_std + value: 2.9007 + - type: nauc_map_at_1_diff1 + value: 52.495599999999996 + - type: nauc_map_at_3_max + value: 37.5749 + - type: nauc_map_at_3_std + value: 5.1779 + - type: nauc_map_at_3_diff1 + value: 46.536300000000004 + - type: nauc_map_at_5_max + value: 38.4721 + - type: nauc_map_at_5_std + value: 6.0973 + - type: nauc_map_at_5_diff1 + value: 46.434799999999996 + - type: nauc_map_at_10_max + value: 38.744299999999996 + - type: nauc_map_at_10_std + value: 6.7116 + - type: nauc_map_at_10_diff1 + value: 46.0759 + - type: nauc_map_at_20_max + value: 38.756 + - type: nauc_map_at_20_std + value: 7.263699999999999 + - type: nauc_map_at_20_diff1 + value: 46.0274 + - type: nauc_map_at_100_max + value: 38.9362 + - type: nauc_map_at_100_std + value: 8.0227 + - type: nauc_map_at_100_diff1 + value: 45.8767 + - type: nauc_map_at_1000_max + value: 38.7473 + - type: nauc_map_at_1000_std + value: 8.089 + - type: nauc_map_at_1000_diff1 + value: 45.8848 + - type: nauc_recall_at_1_max + value: 34.735 + - type: nauc_recall_at_1_std + value: 2.9007 + - type: nauc_recall_at_1_diff1 + value: 52.495599999999996 + - type: nauc_recall_at_3_max + value: 37.1901 + - type: nauc_recall_at_3_std + value: 6.4211 + - type: nauc_recall_at_3_diff1 + value: 38.846000000000004 + - type: nauc_recall_at_5_max + value: 39.8879 + - type: nauc_recall_at_5_std + value: 9.5204 + - type: nauc_recall_at_5_diff1 + value: 37.9339 + - type: nauc_recall_at_10_max + value: 37.181999999999995 + - type: nauc_recall_at_10_std + value: 9.764100000000001 + - type: nauc_recall_at_10_diff1 + value: 33.4855 + - type: nauc_recall_at_20_max + value: 35.6859 + - type: nauc_recall_at_20_std + value: 13.173599999999999 + - type: nauc_recall_at_20_diff1 + value: 33.254 + - type: nauc_recall_at_100_max + value: 42.728100000000005 + - type: nauc_recall_at_100_std + value: 25.913999999999998 + - type: nauc_recall_at_100_diff1 + value: 28.9205 + - type: nauc_recall_at_1000_max + value: 56.496900000000004 + - type: nauc_recall_at_1000_std + value: 56.183499999999995 + - type: nauc_recall_at_1000_diff1 + value: 24.8659 + - type: nauc_precision_at_1_max + value: 36.1514 + - type: nauc_precision_at_1_std + value: 6.7383 + - type: nauc_precision_at_1_diff1 + value: 49.9936 + - type: nauc_precision_at_3_max + value: 36.5767 + - type: nauc_precision_at_3_std + value: 14.884500000000001 + - type: nauc_precision_at_3_diff1 + value: 26.1181 + - type: nauc_precision_at_5_max + value: 33.7094 + - type: nauc_precision_at_5_std + value: 17.566699999999997 + - type: nauc_precision_at_5_diff1 + value: 20.061799999999998 + - type: nauc_precision_at_10_max + value: 28.034 + - type: nauc_precision_at_10_std + value: 23.1877 + - type: nauc_precision_at_10_diff1 + value: 9.646799999999999 + - type: nauc_precision_at_20_max + value: 17.930699999999998 + - type: nauc_precision_at_20_std + value: 23.0956 + - type: nauc_precision_at_20_diff1 + value: -0.0383 + - type: nauc_precision_at_100_max + value: 0.6149 + - type: nauc_precision_at_100_std + value: 22.7163 + - type: nauc_precision_at_100_diff1 + value: -8.730400000000001 + - type: nauc_precision_at_1000_max + value: -19.8022 + - type: nauc_precision_at_1000_std + value: 8.6017 + - type: nauc_precision_at_1000_diff1 + value: -14.161499999999998 + - type: nauc_mrr_at_1_max + value: 36.1514 + - type: nauc_mrr_at_1_std + value: 6.7383 + - type: nauc_mrr_at_1_diff1 + value: 49.9936 + - type: nauc_mrr_at_3_max + value: 37.894299999999994 + - type: nauc_mrr_at_3_std + value: 8.948599999999999 + - type: nauc_mrr_at_3_diff1 + value: 43.985400000000006 + - type: nauc_mrr_at_5_max + value: 38.8686 + - type: nauc_mrr_at_5_std + value: 9.4464 + - type: nauc_mrr_at_5_diff1 + value: 43.9985 + - type: nauc_mrr_at_10_max + value: 38.419 + - type: nauc_mrr_at_10_std + value: 9.4221 + - type: nauc_mrr_at_10_diff1 + value: 43.621700000000004 + - type: nauc_mrr_at_20_max + value: 38.3933 + - type: nauc_mrr_at_20_std + value: 9.6024 + - type: nauc_mrr_at_20_diff1 + value: 43.8952 + - type: nauc_mrr_at_100_max + value: 38.4371 + - type: nauc_mrr_at_100_std + value: 9.657200000000001 + - type: nauc_mrr_at_100_diff1 + value: 43.9457 + - type: nauc_mrr_at_1000_max + value: 38.4386 + - type: nauc_mrr_at_1000_std + value: 9.6614 + - type: nauc_mrr_at_1000_diff1 + value: 43.9579 + - type: main_score + value: 45.539 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackWordpressRetrieval (default) + revision: 4ffe81d471b1924886b33c7567bfb200e9eec5c4 + split: test + type: mteb/cqadupstack-wordpress + metrics: + - type: ndcg_at_1 + value: 26.987 + - type: ndcg_at_3 + value: 33.056999999999995 + - type: ndcg_at_5 + value: 35.356 + - type: ndcg_at_10 + value: 38.440000000000005 + - type: ndcg_at_20 + value: 40.136 + - type: ndcg_at_100 + value: 43.473 + - type: ndcg_at_1000 + value: 45.687 + - type: map_at_1 + value: 24.651999999999997 + - type: map_at_3 + value: 30.416999999999998 + - type: map_at_5 + value: 31.863999999999997 + - type: map_at_10 + value: 33.253 + - type: map_at_20 + value: 33.756 + - type: map_at_100 + value: 34.257 + - type: map_at_1000 + value: 34.347 + - type: recall_at_1 + value: 24.651999999999997 + - type: recall_at_3 + value: 37.88 + - type: recall_at_5 + value: 43.136 + - type: recall_at_10 + value: 52.06699999999999 + - type: recall_at_20 + value: 58.540000000000006 + - type: recall_at_100 + value: 75.22 + - type: recall_at_1000 + value: 91.774 + - type: precision_at_1 + value: 26.987 + - type: precision_at_3 + value: 14.048 + - type: precision_at_5 + value: 9.871 + - type: precision_at_10 + value: 6.063000000000001 + - type: precision_at_20 + value: 3.4099999999999997 + - type: precision_at_100 + value: 0.922 + - type: precision_at_1000 + value: 0.123 + - type: mrr_at_1 + value: 26.9871 + - type: mrr_at_3 + value: 33.1485 + - type: mrr_at_5 + value: 34.3407 + - type: mrr_at_10 + value: 35.6087 + - type: mrr_at_20 + value: 36.0483 + - type: mrr_at_100 + value: 36.463699999999996 + - type: mrr_at_1000 + value: 36.5278 + - type: nauc_ndcg_at_1_max + value: 26.6537 + - type: nauc_ndcg_at_1_std + value: -3.9813 + - type: nauc_ndcg_at_1_diff1 + value: 47.8302 + - type: nauc_ndcg_at_3_max + value: 27.3661 + - type: nauc_ndcg_at_3_std + value: -2.2132 + - type: nauc_ndcg_at_3_diff1 + value: 39.9424 + - type: nauc_ndcg_at_5_max + value: 27.417799999999996 + - type: nauc_ndcg_at_5_std + value: -1.0684 + - type: nauc_ndcg_at_5_diff1 + value: 39.163599999999995 + - type: nauc_ndcg_at_10_max + value: 26.555400000000002 + - type: nauc_ndcg_at_10_std + value: 0.0103 + - type: nauc_ndcg_at_10_diff1 + value: 38.9487 + - type: nauc_ndcg_at_20_max + value: 25.963900000000002 + - type: nauc_ndcg_at_20_std + value: 0.7779 + - type: nauc_ndcg_at_20_diff1 + value: 38.7279 + - type: nauc_ndcg_at_100_max + value: 26.6365 + - type: nauc_ndcg_at_100_std + value: 3.0018 + - type: nauc_ndcg_at_100_diff1 + value: 38.1326 + - type: nauc_ndcg_at_1000_max + value: 26.52 + - type: nauc_ndcg_at_1000_std + value: 2.6968 + - type: nauc_ndcg_at_1000_diff1 + value: 38.1665 + - type: nauc_map_at_1_max + value: 24.950400000000002 + - type: nauc_map_at_1_std + value: -4.2715000000000005 + - type: nauc_map_at_1_diff1 + value: 48.2994 + - type: nauc_map_at_3_max + value: 26.4208 + - type: nauc_map_at_3_std + value: -3.0675 + - type: nauc_map_at_3_diff1 + value: 41.987 + - type: nauc_map_at_5_max + value: 26.641900000000003 + - type: nauc_map_at_5_std + value: -2.3005 + - type: nauc_map_at_5_diff1 + value: 41.4695 + - type: nauc_map_at_10_max + value: 26.2781 + - type: nauc_map_at_10_std + value: -1.8994 + - type: nauc_map_at_10_diff1 + value: 41.193000000000005 + - type: nauc_map_at_20_max + value: 26.0838 + - type: nauc_map_at_20_std + value: -1.7046999999999999 + - type: nauc_map_at_20_diff1 + value: 41.1128 + - type: nauc_map_at_100_max + value: 26.230199999999996 + - type: nauc_map_at_100_std + value: -1.2565 + - type: nauc_map_at_100_diff1 + value: 41.0271 + - type: nauc_map_at_1000_max + value: 26.2069 + - type: nauc_map_at_1000_std + value: -1.2469 + - type: nauc_map_at_1000_diff1 + value: 41.019 + - type: nauc_recall_at_1_max + value: 24.950400000000002 + - type: nauc_recall_at_1_std + value: -4.2715000000000005 + - type: nauc_recall_at_1_diff1 + value: 48.2994 + - type: nauc_recall_at_3_max + value: 27.2098 + - type: nauc_recall_at_3_std + value: -1.309 + - type: nauc_recall_at_3_diff1 + value: 34.4663 + - type: nauc_recall_at_5_max + value: 27.323700000000002 + - type: nauc_recall_at_5_std + value: 1.7010999999999998 + - type: nauc_recall_at_5_diff1 + value: 32.4911 + - type: nauc_recall_at_10_max + value: 24.6483 + - type: nauc_recall_at_10_std + value: 4.9019 + - type: nauc_recall_at_10_diff1 + value: 32.0585 + - type: nauc_recall_at_20_max + value: 22.556 + - type: nauc_recall_at_20_std + value: 8.1527 + - type: nauc_recall_at_20_diff1 + value: 30.8345 + - type: nauc_recall_at_100_max + value: 25.354300000000002 + - type: nauc_recall_at_100_std + value: 22.8578 + - type: nauc_recall_at_100_diff1 + value: 23.291999999999998 + - type: nauc_recall_at_1000_max + value: 26.523999999999997 + - type: nauc_recall_at_1000_std + value: 44.7733 + - type: nauc_recall_at_1000_diff1 + value: 3.1338 + - type: nauc_precision_at_1_max + value: 26.6537 + - type: nauc_precision_at_1_std + value: -3.9813 + - type: nauc_precision_at_1_diff1 + value: 47.8302 + - type: nauc_precision_at_3_max + value: 30.8201 + - type: nauc_precision_at_3_std + value: 1.7691 + - type: nauc_precision_at_3_diff1 + value: 33.3835 + - type: nauc_precision_at_5_max + value: 29.5433 + - type: nauc_precision_at_5_std + value: 4.4224 + - type: nauc_precision_at_5_diff1 + value: 28.426000000000002 + - type: nauc_precision_at_10_max + value: 26.0888 + - type: nauc_precision_at_10_std + value: 7.8104000000000005 + - type: nauc_precision_at_10_diff1 + value: 24.509800000000002 + - type: nauc_precision_at_20_max + value: 22.218799999999998 + - type: nauc_precision_at_20_std + value: 11.248099999999999 + - type: nauc_precision_at_20_diff1 + value: 20.6056 + - type: nauc_precision_at_100_max + value: 16.4622 + - type: nauc_precision_at_100_std + value: 25.735200000000003 + - type: nauc_precision_at_100_diff1 + value: 6.2566 + - type: nauc_precision_at_1000_max + value: -9.109399999999999 + - type: nauc_precision_at_1000_std + value: 13.820099999999998 + - type: nauc_precision_at_1000_diff1 + value: -7.9046 + - type: nauc_mrr_at_1_max + value: 26.6537 + - type: nauc_mrr_at_1_std + value: -3.9813 + - type: nauc_mrr_at_1_diff1 + value: 47.8302 + - type: nauc_mrr_at_3_max + value: 27.9843 + - type: nauc_mrr_at_3_std + value: -2.3418 + - type: nauc_mrr_at_3_diff1 + value: 41.4877 + - type: nauc_mrr_at_5_max + value: 27.9298 + - type: nauc_mrr_at_5_std + value: -1.7860999999999998 + - type: nauc_mrr_at_5_diff1 + value: 40.9261 + - type: nauc_mrr_at_10_max + value: 27.6814 + - type: nauc_mrr_at_10_std + value: -1.1542000000000001 + - type: nauc_mrr_at_10_diff1 + value: 40.9534 + - type: nauc_mrr_at_20_max + value: 27.507900000000003 + - type: nauc_mrr_at_20_std + value: -0.9558000000000001 + - type: nauc_mrr_at_20_diff1 + value: 41.0046 + - type: nauc_mrr_at_100_max + value: 27.5032 + - type: nauc_mrr_at_100_std + value: -0.7483 + - type: nauc_mrr_at_100_diff1 + value: 40.9239 + - type: nauc_mrr_at_1000_max + value: 27.4957 + - type: nauc_mrr_at_1000_std + value: -0.7642 + - type: nauc_mrr_at_1000_diff1 + value: 40.9219 + - type: main_score + value: 38.440000000000005 + task: + type: Retrieval + - dataset: + config: default + name: MTEB ClimateFEVER (default) + revision: 47f2ac6acb640fc46020b02a5b59fdda04d39380 + split: test + type: mteb/climate-fever + metrics: + - type: ndcg_at_1 + value: 47.231 + - type: ndcg_at_3 + value: 38.605000000000004 + - type: ndcg_at_5 + value: 40.058 + - type: ndcg_at_10 + value: 43.482 + - type: ndcg_at_20 + value: 45.732 + - type: ndcg_at_100 + value: 49.062 + - type: ndcg_at_1000 + value: 51.605000000000004 + - type: map_at_1 + value: 20.674 + - type: map_at_3 + value: 29.375 + - type: map_at_5 + value: 31.872 + - type: map_at_10 + value: 33.846 + - type: map_at_20 + value: 34.733000000000004 + - type: map_at_100 + value: 35.411 + - type: map_at_1000 + value: 35.553000000000004 + - type: recall_at_1 + value: 20.674 + - type: recall_at_3 + value: 33.859 + - type: recall_at_5 + value: 39.76 + - type: recall_at_10 + value: 47.150999999999996 + - type: recall_at_20 + value: 53.522999999999996 + - type: recall_at_100 + value: 66.125 + - type: recall_at_1000 + value: 80.368 + - type: precision_at_1 + value: 47.231 + - type: precision_at_3 + value: 28.534 + - type: precision_at_5 + value: 20.782 + - type: precision_at_10 + value: 12.742999999999999 + - type: precision_at_20 + value: 7.342 + - type: precision_at_100 + value: 1.883 + - type: precision_at_1000 + value: 0.23700000000000002 + - type: mrr_at_1 + value: 47.2313 + - type: mrr_at_3 + value: 55.6352 + - type: mrr_at_5 + value: 56.92509999999999 + - type: mrr_at_10 + value: 57.833400000000005 + - type: mrr_at_20 + value: 58.178700000000006 + - type: mrr_at_100 + value: 58.385 + - type: mrr_at_1000 + value: 58.40919999999999 + - type: nauc_ndcg_at_1_max + value: 41.5456 + - type: nauc_ndcg_at_1_std + value: 19.2734 + - type: nauc_ndcg_at_1_diff1 + value: 38.0868 + - type: nauc_ndcg_at_3_max + value: 41.6105 + - type: nauc_ndcg_at_3_std + value: 19.5917 + - type: nauc_ndcg_at_3_diff1 + value: 29.192800000000002 + - type: nauc_ndcg_at_5_max + value: 42.1893 + - type: nauc_ndcg_at_5_std + value: 21.9984 + - type: nauc_ndcg_at_5_diff1 + value: 27.7412 + - type: nauc_ndcg_at_10_max + value: 42.5633 + - type: nauc_ndcg_at_10_std + value: 24.265700000000002 + - type: nauc_ndcg_at_10_diff1 + value: 27.0287 + - type: nauc_ndcg_at_20_max + value: 43.364200000000004 + - type: nauc_ndcg_at_20_std + value: 26.2174 + - type: nauc_ndcg_at_20_diff1 + value: 26.980500000000003 + - type: nauc_ndcg_at_100_max + value: 43.9582 + - type: nauc_ndcg_at_100_std + value: 28.454 + - type: nauc_ndcg_at_100_diff1 + value: 27.087099999999996 + - type: nauc_ndcg_at_1000_max + value: 44.0356 + - type: nauc_ndcg_at_1000_std + value: 28.64 + - type: nauc_ndcg_at_1000_diff1 + value: 27.1343 + - type: nauc_map_at_1_max + value: 39.2181 + - type: nauc_map_at_1_std + value: 12.4972 + - type: nauc_map_at_1_diff1 + value: 39.5664 + - type: nauc_map_at_3_max + value: 41.5441 + - type: nauc_map_at_3_std + value: 17.333000000000002 + - type: nauc_map_at_3_diff1 + value: 29.9555 + - type: nauc_map_at_5_max + value: 41.0041 + - type: nauc_map_at_5_std + value: 19.3667 + - type: nauc_map_at_5_diff1 + value: 28.0157 + - type: nauc_map_at_10_max + value: 41.2914 + - type: nauc_map_at_10_std + value: 21.051000000000002 + - type: nauc_map_at_10_diff1 + value: 27.387 + - type: nauc_map_at_20_max + value: 41.6964 + - type: nauc_map_at_20_std + value: 21.9338 + - type: nauc_map_at_20_diff1 + value: 27.4326 + - type: nauc_map_at_100_max + value: 41.8592 + - type: nauc_map_at_100_std + value: 22.46 + - type: nauc_map_at_100_diff1 + value: 27.4024 + - type: nauc_map_at_1000_max + value: 41.8737 + - type: nauc_map_at_1000_std + value: 22.4882 + - type: nauc_map_at_1000_diff1 + value: 27.405099999999997 + - type: nauc_recall_at_1_max + value: 39.2181 + - type: nauc_recall_at_1_std + value: 12.4972 + - type: nauc_recall_at_1_diff1 + value: 39.5664 + - type: nauc_recall_at_3_max + value: 41.3571 + - type: nauc_recall_at_3_std + value: 18.607699999999998 + - type: nauc_recall_at_3_diff1 + value: 25.8418 + - type: nauc_recall_at_5_max + value: 39.1225 + - type: nauc_recall_at_5_std + value: 22.2091 + - type: nauc_recall_at_5_diff1 + value: 20.9495 + - type: nauc_recall_at_10_max + value: 38.0045 + - type: nauc_recall_at_10_std + value: 25.584 + - type: nauc_recall_at_10_diff1 + value: 18.489 + - type: nauc_recall_at_20_max + value: 38.0096 + - type: nauc_recall_at_20_std + value: 29.3335 + - type: nauc_recall_at_20_diff1 + value: 17.0106 + - type: nauc_recall_at_100_max + value: 37.7378 + - type: nauc_recall_at_100_std + value: 37.0189 + - type: nauc_recall_at_100_diff1 + value: 14.815900000000001 + - type: nauc_recall_at_1000_max + value: 36.2825 + - type: nauc_recall_at_1000_std + value: 42.1995 + - type: nauc_recall_at_1000_diff1 + value: 10.5182 + - type: nauc_precision_at_1_max + value: 41.5456 + - type: nauc_precision_at_1_std + value: 19.2734 + - type: nauc_precision_at_1_diff1 + value: 38.0868 + - type: nauc_precision_at_3_max + value: 35.72 + - type: nauc_precision_at_3_std + value: 22.8785 + - type: nauc_precision_at_3_diff1 + value: 15.240200000000002 + - type: nauc_precision_at_5_max + value: 30.4643 + - type: nauc_precision_at_5_std + value: 26.2774 + - type: nauc_precision_at_5_diff1 + value: 8.8749 + - type: nauc_precision_at_10_max + value: 25.960299999999997 + - type: nauc_precision_at_10_std + value: 28.3825 + - type: nauc_precision_at_10_diff1 + value: 4.626799999999999 + - type: nauc_precision_at_20_max + value: 24.8278 + - type: nauc_precision_at_20_std + value: 32.1644 + - type: nauc_precision_at_20_diff1 + value: 2.5019 + - type: nauc_precision_at_100_max + value: 17.180999999999997 + - type: nauc_precision_at_100_std + value: 33.955400000000004 + - type: nauc_precision_at_100_diff1 + value: -1.9183 + - type: nauc_precision_at_1000_max + value: 4.8986 + - type: nauc_precision_at_1000_std + value: 26.5376 + - type: nauc_precision_at_1000_diff1 + value: -9.3468 + - type: nauc_mrr_at_1_max + value: 41.5456 + - type: nauc_mrr_at_1_std + value: 19.2734 + - type: nauc_mrr_at_1_diff1 + value: 38.0868 + - type: nauc_mrr_at_3_max + value: 43.7301 + - type: nauc_mrr_at_3_std + value: 22.409100000000002 + - type: nauc_mrr_at_3_diff1 + value: 34.846500000000006 + - type: nauc_mrr_at_5_max + value: 44.0608 + - type: nauc_mrr_at_5_std + value: 23.3812 + - type: nauc_mrr_at_5_diff1 + value: 34.5847 + - type: nauc_mrr_at_10_max + value: 44.026700000000005 + - type: nauc_mrr_at_10_std + value: 23.339399999999998 + - type: nauc_mrr_at_10_diff1 + value: 34.7306 + - type: nauc_mrr_at_20_max + value: 44.1444 + - type: nauc_mrr_at_20_std + value: 23.5132 + - type: nauc_mrr_at_20_diff1 + value: 34.6927 + - type: nauc_mrr_at_100_max + value: 44.1228 + - type: nauc_mrr_at_100_std + value: 23.5783 + - type: nauc_mrr_at_100_diff1 + value: 34.7193 + - type: nauc_mrr_at_1000_max + value: 44.1082 + - type: nauc_mrr_at_1000_std + value: 23.5574 + - type: nauc_mrr_at_1000_diff1 + value: 34.719699999999996 + - type: main_score + value: 43.482 + task: + type: Retrieval + - dataset: + config: default + name: MTEB DBPedia (default) + revision: c0f706b76e590d620bd6618b3ca8efdd34e2d659 + split: test + type: mteb/dbpedia + metrics: + - type: ndcg_at_1 + value: 59.25 + - type: ndcg_at_3 + value: 48.256 + - type: ndcg_at_5 + value: 45.580999999999996 + - type: ndcg_at_10 + value: 43.37 + - type: ndcg_at_20 + value: 43.106 + - type: ndcg_at_100 + value: 47.845 + - type: ndcg_at_1000 + value: 54.974999999999994 + - type: map_at_1 + value: 10.032 + - type: map_at_3 + value: 14.954 + - type: map_at_5 + value: 17.408 + - type: map_at_10 + value: 20.461 + - type: map_at_20 + value: 23.759 + - type: map_at_100 + value: 28.718 + - type: map_at_1000 + value: 30.406 + - type: recall_at_1 + value: 10.032 + - type: recall_at_3 + value: 15.905 + - type: recall_at_5 + value: 19.622999999999998 + - type: recall_at_10 + value: 25.125999999999998 + - type: recall_at_20 + value: 33.262 + - type: recall_at_100 + value: 52.515 + - type: recall_at_1000 + value: 75.224 + - type: precision_at_1 + value: 72.0 + - type: precision_at_3 + value: 50.917 + - type: precision_at_5 + value: 43.4 + - type: precision_at_10 + value: 34.175 + - type: precision_at_20 + value: 26.325 + - type: precision_at_100 + value: 10.893 + - type: precision_at_1000 + value: 2.0549999999999997 + - type: mrr_at_1 + value: 72.0 + - type: mrr_at_3 + value: 77.5417 + - type: mrr_at_5 + value: 78.2042 + - type: mrr_at_10 + value: 78.7173 + - type: mrr_at_20 + value: 78.9521 + - type: mrr_at_100 + value: 79.0382 + - type: mrr_at_1000 + value: 79.0408 + - type: nauc_ndcg_at_1_max + value: 49.778 + - type: nauc_ndcg_at_1_std + value: 20.462 + - type: nauc_ndcg_at_1_diff1 + value: 49.3621 + - type: nauc_ndcg_at_3_max + value: 44.4388 + - type: nauc_ndcg_at_3_std + value: 24.646 + - type: nauc_ndcg_at_3_diff1 + value: 33.3173 + - type: nauc_ndcg_at_5_max + value: 44.2179 + - type: nauc_ndcg_at_5_std + value: 25.597399999999997 + - type: nauc_ndcg_at_5_diff1 + value: 31.0886 + - type: nauc_ndcg_at_10_max + value: 43.7812 + - type: nauc_ndcg_at_10_std + value: 25.61 + - type: nauc_ndcg_at_10_diff1 + value: 30.667699999999996 + - type: nauc_ndcg_at_20_max + value: 39.4779 + - type: nauc_ndcg_at_20_std + value: 20.891000000000002 + - type: nauc_ndcg_at_20_diff1 + value: 29.492600000000003 + - type: nauc_ndcg_at_100_max + value: 41.511900000000004 + - type: nauc_ndcg_at_100_std + value: 27.340999999999998 + - type: nauc_ndcg_at_100_diff1 + value: 30.5701 + - type: nauc_ndcg_at_1000_max + value: 47.0571 + - type: nauc_ndcg_at_1000_std + value: 37.0976 + - type: nauc_ndcg_at_1000_diff1 + value: 31.5615 + - type: nauc_map_at_1_max + value: 0.4743 + - type: nauc_map_at_1_std + value: -23.7532 + - type: nauc_map_at_1_diff1 + value: 26.0851 + - type: nauc_map_at_3_max + value: 8.5131 + - type: nauc_map_at_3_std + value: -18.6015 + - type: nauc_map_at_3_diff1 + value: 21.9172 + - type: nauc_map_at_5_max + value: 12.295499999999999 + - type: nauc_map_at_5_std + value: -13.872100000000001 + - type: nauc_map_at_5_diff1 + value: 21.3319 + - type: nauc_map_at_10_max + value: 17.1428 + - type: nauc_map_at_10_std + value: -6.638199999999999 + - type: nauc_map_at_10_diff1 + value: 20.8671 + - type: nauc_map_at_20_max + value: 21.7306 + - type: nauc_map_at_20_std + value: 2.1404 + - type: nauc_map_at_20_diff1 + value: 20.7929 + - type: nauc_map_at_100_max + value: 29.677799999999998 + - type: nauc_map_at_100_std + value: 16.9458 + - type: nauc_map_at_100_diff1 + value: 22.4101 + - type: nauc_map_at_1000_max + value: 31.5735 + - type: nauc_map_at_1000_std + value: 20.5816 + - type: nauc_map_at_1000_diff1 + value: 22.561400000000003 + - type: nauc_recall_at_1_max + value: 0.4743 + - type: nauc_recall_at_1_std + value: -23.7532 + - type: nauc_recall_at_1_diff1 + value: 26.0851 + - type: nauc_recall_at_3_max + value: 6.851500000000001 + - type: nauc_recall_at_3_std + value: -18.7341 + - type: nauc_recall_at_3_diff1 + value: 19.703699999999998 + - type: nauc_recall_at_5_max + value: 10.0265 + - type: nauc_recall_at_5_std + value: -14.2537 + - type: nauc_recall_at_5_diff1 + value: 18.8765 + - type: nauc_recall_at_10_max + value: 14.1582 + - type: nauc_recall_at_10_std + value: -7.703 + - type: nauc_recall_at_10_diff1 + value: 17.9056 + - type: nauc_recall_at_20_max + value: 15.0343 + - type: nauc_recall_at_20_std + value: -0.9846 + - type: nauc_recall_at_20_diff1 + value: 14.377899999999999 + - type: nauc_recall_at_100_max + value: 27.904600000000002 + - type: nauc_recall_at_100_std + value: 24.6322 + - type: nauc_recall_at_100_diff1 + value: 16.869500000000002 + - type: nauc_recall_at_1000_max + value: 33.7755 + - type: nauc_recall_at_1000_std + value: 42.241800000000005 + - type: nauc_recall_at_1000_diff1 + value: 17.3324 + - type: nauc_precision_at_1_max + value: 62.3459 + - type: nauc_precision_at_1_std + value: 28.3277 + - type: nauc_precision_at_1_diff1 + value: 57.8053 + - type: nauc_precision_at_3_max + value: 45.8296 + - type: nauc_precision_at_3_std + value: 39.8642 + - type: nauc_precision_at_3_diff1 + value: 15.7381 + - type: nauc_precision_at_5_max + value: 45.331900000000005 + - type: nauc_precision_at_5_std + value: 45.1279 + - type: nauc_precision_at_5_diff1 + value: 11.473700000000001 + - type: nauc_precision_at_10_max + value: 42.276399999999995 + - type: nauc_precision_at_10_std + value: 50.9538 + - type: nauc_precision_at_10_diff1 + value: 6.708699999999999 + - type: nauc_precision_at_20_max + value: 37.961600000000004 + - type: nauc_precision_at_20_std + value: 52.0611 + - type: nauc_precision_at_20_diff1 + value: 5.9309 + - type: nauc_precision_at_100_max + value: 29.567 + - type: nauc_precision_at_100_std + value: 50.07 + - type: nauc_precision_at_100_diff1 + value: 3.2583 + - type: nauc_precision_at_1000_max + value: 5.5285 + - type: nauc_precision_at_1000_std + value: 20.5813 + - type: nauc_precision_at_1000_diff1 + value: -6.6333 + - type: nauc_mrr_at_1_max + value: 62.3459 + - type: nauc_mrr_at_1_std + value: 28.3277 + - type: nauc_mrr_at_1_diff1 + value: 57.8053 + - type: nauc_mrr_at_3_max + value: 66.5168 + - type: nauc_mrr_at_3_std + value: 37.4446 + - type: nauc_mrr_at_3_diff1 + value: 57.6125 + - type: nauc_mrr_at_5_max + value: 65.8343 + - type: nauc_mrr_at_5_std + value: 36.6396 + - type: nauc_mrr_at_5_diff1 + value: 56.91589999999999 + - type: nauc_mrr_at_10_max + value: 65.73750000000001 + - type: nauc_mrr_at_10_std + value: 36.4067 + - type: nauc_mrr_at_10_diff1 + value: 56.9594 + - type: nauc_mrr_at_20_max + value: 65.6623 + - type: nauc_mrr_at_20_std + value: 36.0989 + - type: nauc_mrr_at_20_diff1 + value: 56.9662 + - type: nauc_mrr_at_100_max + value: 65.6934 + - type: nauc_mrr_at_100_std + value: 36.0911 + - type: nauc_mrr_at_100_diff1 + value: 57.0541 + - type: nauc_mrr_at_1000_max + value: 65.68929999999999 + - type: nauc_mrr_at_1000_std + value: 36.0838 + - type: nauc_mrr_at_1000_diff1 + value: 57.054300000000005 + - type: main_score + value: 43.37 + task: + type: Retrieval + - dataset: + config: default + name: MTEB EmotionClassification (default) + revision: 4f58c6b202a23cf9a4da393831edf4f9183cad37 + split: test + type: mteb/emotion + metrics: + - type: accuracy + value: 42.53 + - type: f1 + value: 38.4608 + - type: f1_weighted + value: 44.6927 + - type: main_score + value: 42.53 + task: + type: Classification + - dataset: + config: default + name: MTEB FEVER (default) + revision: bea83ef9e8fb933d90a2f1d5515737465d613e12 + split: test + type: mteb/fever + metrics: + - type: ndcg_at_1 + value: 90.519 + - type: ndcg_at_3 + value: 91.387 + - type: ndcg_at_5 + value: 91.644 + - type: ndcg_at_10 + value: 91.91 + - type: ndcg_at_20 + value: 92.136 + - type: ndcg_at_100 + value: 92.406 + - type: ndcg_at_1000 + value: 92.62599999999999 + - type: map_at_1 + value: 83.994 + - type: map_at_3 + value: 88.885 + - type: map_at_5 + value: 89.185 + - type: map_at_10 + value: 89.36500000000001 + - type: map_at_20 + value: 89.458 + - type: map_at_100 + value: 89.515 + - type: map_at_1000 + value: 89.52799999999999 + - type: recall_at_1 + value: 83.994 + - type: recall_at_3 + value: 93.145 + - type: recall_at_5 + value: 94.016 + - type: recall_at_10 + value: 94.836 + - type: recall_at_20 + value: 95.56700000000001 + - type: recall_at_100 + value: 96.711 + - type: recall_at_1000 + value: 98.027 + - type: precision_at_1 + value: 90.519 + - type: precision_at_3 + value: 33.922999999999995 + - type: precision_at_5 + value: 20.636 + - type: precision_at_10 + value: 10.474 + - type: precision_at_20 + value: 5.316 + - type: precision_at_100 + value: 1.0919999999999999 + - type: precision_at_1000 + value: 0.11299999999999999 + - type: mrr_at_1 + value: 90.5191 + - type: mrr_at_3 + value: 94.37440000000001 + - type: mrr_at_5 + value: 94.4832 + - type: mrr_at_10 + value: 94.5215 + - type: mrr_at_20 + value: 94.5365 + - type: mrr_at_100 + value: 94.5422 + - type: mrr_at_1000 + value: 94.54249999999999 + - type: nauc_ndcg_at_1_max + value: 22.1341 + - type: nauc_ndcg_at_1_std + value: -11.1273 + - type: nauc_ndcg_at_1_diff1 + value: 81.8507 + - type: nauc_ndcg_at_3_max + value: 16.8937 + - type: nauc_ndcg_at_3_std + value: -7.1829 + - type: nauc_ndcg_at_3_diff1 + value: 43.892199999999995 + - type: nauc_ndcg_at_5_max + value: 17.9177 + - type: nauc_ndcg_at_5_std + value: -5.2 + - type: nauc_ndcg_at_5_diff1 + value: 41.9608 + - type: nauc_ndcg_at_10_max + value: 17.8222 + - type: nauc_ndcg_at_10_std + value: -3.8736 + - type: nauc_ndcg_at_10_diff1 + value: 41.955 + - type: nauc_ndcg_at_20_max + value: 18.467200000000002 + - type: nauc_ndcg_at_20_std + value: -2.7304 + - type: nauc_ndcg_at_20_diff1 + value: 42.950300000000006 + - type: nauc_ndcg_at_100_max + value: 18.5918 + - type: nauc_ndcg_at_100_std + value: -2.874 + - type: nauc_ndcg_at_100_diff1 + value: 44.182 + - type: nauc_ndcg_at_1000_max + value: 18.9498 + - type: nauc_ndcg_at_1000_std + value: -2.8561 + - type: nauc_ndcg_at_1000_diff1 + value: 45.5587 + - type: nauc_map_at_1_max + value: 14.943600000000002 + - type: nauc_map_at_1_std + value: -6.3744 + - type: nauc_map_at_1_diff1 + value: 51.697700000000005 + - type: nauc_map_at_3_max + value: 15.7558 + - type: nauc_map_at_3_std + value: -5.8517 + - type: nauc_map_at_3_diff1 + value: 41.814 + - type: nauc_map_at_5_max + value: 16.6287 + - type: nauc_map_at_5_std + value: -4.9942 + - type: nauc_map_at_5_diff1 + value: 41.605199999999996 + - type: nauc_map_at_10_max + value: 16.8146 + - type: nauc_map_at_10_std + value: -4.4551 + - type: nauc_map_at_10_diff1 + value: 41.9641 + - type: nauc_map_at_20_max + value: 17.0709 + - type: nauc_map_at_20_std + value: -4.1187000000000005 + - type: nauc_map_at_20_diff1 + value: 42.3292 + - type: nauc_map_at_100_max + value: 17.1076 + - type: nauc_map_at_100_std + value: -4.1089 + - type: nauc_map_at_100_diff1 + value: 42.5101 + - type: nauc_map_at_1000_max + value: 17.1309 + - type: nauc_map_at_1000_std + value: -4.0958000000000006 + - type: nauc_map_at_1000_diff1 + value: 42.5694 + - type: nauc_recall_at_1_max + value: 14.943600000000002 + - type: nauc_recall_at_1_std + value: -6.3744 + - type: nauc_recall_at_1_diff1 + value: 51.697700000000005 + - type: nauc_recall_at_3_max + value: 11.8984 + - type: nauc_recall_at_3_std + value: -4.224 + - type: nauc_recall_at_3_diff1 + value: 13.962 + - type: nauc_recall_at_5_max + value: 16.2434 + - type: nauc_recall_at_5_std + value: 1.6707 + - type: nauc_recall_at_5_diff1 + value: 7.788 + - type: nauc_recall_at_10_max + value: 16.4427 + - type: nauc_recall_at_10_std + value: 8.259 + - type: nauc_recall_at_10_diff1 + value: 4.5507 + - type: nauc_recall_at_20_max + value: 19.0546 + - type: nauc_recall_at_20_std + value: 16.7132 + - type: nauc_recall_at_20_diff1 + value: 3.5242000000000004 + - type: nauc_recall_at_100_max + value: 19.6815 + - type: nauc_recall_at_100_std + value: 21.4767 + - type: nauc_recall_at_100_diff1 + value: 1.4785 + - type: nauc_recall_at_1000_max + value: 26.5748 + - type: nauc_recall_at_1000_std + value: 37.026399999999995 + - type: nauc_recall_at_1000_diff1 + value: 1.512 + - type: nauc_precision_at_1_max + value: 22.1341 + - type: nauc_precision_at_1_std + value: -11.1273 + - type: nauc_precision_at_1_diff1 + value: 81.8507 + - type: nauc_precision_at_3_max + value: 13.6152 + - type: nauc_precision_at_3_std + value: -2.4367 + - type: nauc_precision_at_3_diff1 + value: 1.6237000000000001 + - type: nauc_precision_at_5_max + value: 13.977400000000001 + - type: nauc_precision_at_5_std + value: 4.3391 + - type: nauc_precision_at_5_diff1 + value: -6.660000000000001 + - type: nauc_precision_at_10_max + value: 10.4986 + - type: nauc_precision_at_10_std + value: 8.9132 + - type: nauc_precision_at_10_diff1 + value: -7.5682 + - type: nauc_precision_at_20_max + value: 11.0525 + - type: nauc_precision_at_20_std + value: 12.0579 + - type: nauc_precision_at_20_diff1 + value: -5.0471 + - type: nauc_precision_at_100_max + value: 7.1659 + - type: nauc_precision_at_100_std + value: 8.1754 + - type: nauc_precision_at_100_diff1 + value: -2.7885 + - type: nauc_precision_at_1000_max + value: 4.9776 + - type: nauc_precision_at_1000_std + value: 5.8301 + - type: nauc_precision_at_1000_diff1 + value: 0.18860000000000002 + - type: nauc_mrr_at_1_max + value: 22.1341 + - type: nauc_mrr_at_1_std + value: -11.1273 + - type: nauc_mrr_at_1_diff1 + value: 81.8507 + - type: nauc_mrr_at_3_max + value: 21.6738 + - type: nauc_mrr_at_3_std + value: -15.7016 + - type: nauc_mrr_at_3_diff1 + value: 81.0757 + - type: nauc_mrr_at_5_max + value: 22.6603 + - type: nauc_mrr_at_5_std + value: -14.7345 + - type: nauc_mrr_at_5_diff1 + value: 81.1092 + - type: nauc_mrr_at_10_max + value: 22.4279 + - type: nauc_mrr_at_10_std + value: -14.5002 + - type: nauc_mrr_at_10_diff1 + value: 81.11080000000001 + - type: nauc_mrr_at_20_max + value: 22.3604 + - type: nauc_mrr_at_20_std + value: -14.3058 + - type: nauc_mrr_at_20_diff1 + value: 81.1563 + - type: nauc_mrr_at_100_max + value: 22.311 + - type: nauc_mrr_at_100_std + value: -14.318100000000001 + - type: nauc_mrr_at_100_diff1 + value: 81.1586 + - type: nauc_mrr_at_1000_max + value: 22.307199999999998 + - type: nauc_mrr_at_1000_std + value: -14.3234 + - type: nauc_mrr_at_1000_diff1 + value: 81.1576 + - type: main_score + value: 91.91 + task: + type: Retrieval + - dataset: + config: default + name: MTEB FiQA2018 (default) + revision: 27a168819829fe9bcd655c2df245fb19452e8e06 + split: test + type: mteb/fiqa + metrics: + - type: ndcg_at_1 + value: 44.753 + - type: ndcg_at_3 + value: 41.555 + - type: ndcg_at_5 + value: 42.809999999999995 + - type: ndcg_at_10 + value: 45.49 + - type: ndcg_at_20 + value: 48.287 + - type: ndcg_at_100 + value: 52.115 + - type: ndcg_at_1000 + value: 54.797 + - type: map_at_1 + value: 22.894000000000002 + - type: map_at_3 + value: 32.786 + - type: map_at_5 + value: 35.495 + - type: map_at_10 + value: 37.635000000000005 + - type: map_at_20 + value: 38.771 + - type: map_at_100 + value: 39.56 + - type: map_at_1000 + value: 39.734 + - type: recall_at_1 + value: 22.894000000000002 + - type: recall_at_3 + value: 37.579 + - type: recall_at_5 + value: 44.03 + - type: recall_at_10 + value: 52.61900000000001 + - type: recall_at_20 + value: 61.227 + - type: recall_at_100 + value: 76.88199999999999 + - type: recall_at_1000 + value: 92.534 + - type: precision_at_1 + value: 44.753 + - type: precision_at_3 + value: 27.675 + - type: precision_at_5 + value: 20.556 + - type: precision_at_10 + value: 12.592999999999998 + - type: precision_at_20 + value: 7.507999999999999 + - type: precision_at_100 + value: 1.9369999999999998 + - type: precision_at_1000 + value: 0.242 + - type: mrr_at_1 + value: 44.7531 + - type: mrr_at_3 + value: 50.694399999999995 + - type: mrr_at_5 + value: 51.990700000000004 + - type: mrr_at_10 + value: 52.9925 + - type: mrr_at_20 + value: 53.4612 + - type: mrr_at_100 + value: 53.7889 + - type: mrr_at_1000 + value: 53.8244 + - type: nauc_ndcg_at_1_max + value: 46.679700000000004 + - type: nauc_ndcg_at_1_std + value: -7.8208 + - type: nauc_ndcg_at_1_diff1 + value: 55.9238 + - type: nauc_ndcg_at_3_max + value: 39.761 + - type: nauc_ndcg_at_3_std + value: -7.6645 + - type: nauc_ndcg_at_3_diff1 + value: 43.6641 + - type: nauc_ndcg_at_5_max + value: 37.2506 + - type: nauc_ndcg_at_5_std + value: -7.574300000000001 + - type: nauc_ndcg_at_5_diff1 + value: 41.6025 + - type: nauc_ndcg_at_10_max + value: 38.1464 + - type: nauc_ndcg_at_10_std + value: -6.1288 + - type: nauc_ndcg_at_10_diff1 + value: 42.625 + - type: nauc_ndcg_at_20_max + value: 39.687 + - type: nauc_ndcg_at_20_std + value: -4.6046 + - type: nauc_ndcg_at_20_diff1 + value: 43.2796 + - type: nauc_ndcg_at_100_max + value: 41.4101 + - type: nauc_ndcg_at_100_std + value: -2.1537 + - type: nauc_ndcg_at_100_diff1 + value: 43.980599999999995 + - type: nauc_ndcg_at_1000_max + value: 42.0853 + - type: nauc_ndcg_at_1000_std + value: -2.5 + - type: nauc_ndcg_at_1000_diff1 + value: 44.5636 + - type: nauc_map_at_1_max + value: 21.019299999999998 + - type: nauc_map_at_1_std + value: -10.8832 + - type: nauc_map_at_1_diff1 + value: 45.1685 + - type: nauc_map_at_3_max + value: 29.0524 + - type: nauc_map_at_3_std + value: -9.6495 + - type: nauc_map_at_3_diff1 + value: 41.3844 + - type: nauc_map_at_5_max + value: 31.3813 + - type: nauc_map_at_5_std + value: -8.7888 + - type: nauc_map_at_5_diff1 + value: 40.1699 + - type: nauc_map_at_10_max + value: 33.8361 + - type: nauc_map_at_10_std + value: -7.9594 + - type: nauc_map_at_10_diff1 + value: 40.788999999999994 + - type: nauc_map_at_20_max + value: 34.9439 + - type: nauc_map_at_20_std + value: -7.382700000000001 + - type: nauc_map_at_20_diff1 + value: 41.134100000000004 + - type: nauc_map_at_100_max + value: 35.530899999999995 + - type: nauc_map_at_100_std + value: -6.8411 + - type: nauc_map_at_100_diff1 + value: 41.316 + - type: nauc_map_at_1000_max + value: 35.6246 + - type: nauc_map_at_1000_std + value: -6.828399999999999 + - type: nauc_map_at_1000_diff1 + value: 41.3739 + - type: nauc_recall_at_1_max + value: 21.019299999999998 + - type: nauc_recall_at_1_std + value: -10.8832 + - type: nauc_recall_at_1_diff1 + value: 45.1685 + - type: nauc_recall_at_3_max + value: 25.667499999999997 + - type: nauc_recall_at_3_std + value: -9.3695 + - type: nauc_recall_at_3_diff1 + value: 35.0424 + - type: nauc_recall_at_5_max + value: 26.2285 + - type: nauc_recall_at_5_std + value: -7.6552 + - type: nauc_recall_at_5_diff1 + value: 31.7068 + - type: nauc_recall_at_10_max + value: 29.12 + - type: nauc_recall_at_10_std + value: -3.5869 + - type: nauc_recall_at_10_diff1 + value: 31.952599999999997 + - type: nauc_recall_at_20_max + value: 31.5269 + - type: nauc_recall_at_20_std + value: 2.2824 + - type: nauc_recall_at_20_diff1 + value: 31.4747 + - type: nauc_recall_at_100_max + value: 34.533500000000004 + - type: nauc_recall_at_100_std + value: 18.8398 + - type: nauc_recall_at_100_diff1 + value: 29.525000000000002 + - type: nauc_recall_at_1000_max + value: 38.973600000000005 + - type: nauc_recall_at_1000_std + value: 37.9643 + - type: nauc_recall_at_1000_diff1 + value: 29.247899999999998 + - type: nauc_precision_at_1_max + value: 46.679700000000004 + - type: nauc_precision_at_1_std + value: -7.8208 + - type: nauc_precision_at_1_diff1 + value: 55.9238 + - type: nauc_precision_at_3_max + value: 46.348800000000004 + - type: nauc_precision_at_3_std + value: -2.4303000000000003 + - type: nauc_precision_at_3_diff1 + value: 31.4803 + - type: nauc_precision_at_5_max + value: 45.657 + - type: nauc_precision_at_5_std + value: 0.9887999999999999 + - type: nauc_precision_at_5_diff1 + value: 22.6439 + - type: nauc_precision_at_10_max + value: 48.147099999999995 + - type: nauc_precision_at_10_std + value: 5.313 + - type: nauc_precision_at_10_diff1 + value: 20.7803 + - type: nauc_precision_at_20_max + value: 47.407199999999996 + - type: nauc_precision_at_20_std + value: 8.8254 + - type: nauc_precision_at_20_diff1 + value: 17.7327 + - type: nauc_precision_at_100_max + value: 43.4944 + - type: nauc_precision_at_100_std + value: 14.8423 + - type: nauc_precision_at_100_diff1 + value: 11.7231 + - type: nauc_precision_at_1000_max + value: 36.3175 + - type: nauc_precision_at_1000_std + value: 14.9478 + - type: nauc_precision_at_1000_diff1 + value: 4.9391 + - type: nauc_mrr_at_1_max + value: 46.679700000000004 + - type: nauc_mrr_at_1_std + value: -7.8208 + - type: nauc_mrr_at_1_diff1 + value: 55.9238 + - type: nauc_mrr_at_3_max + value: 48.0241 + - type: nauc_mrr_at_3_std + value: -6.761100000000001 + - type: nauc_mrr_at_3_diff1 + value: 53.5091 + - type: nauc_mrr_at_5_max + value: 48.0965 + - type: nauc_mrr_at_5_std + value: -6.3173 + - type: nauc_mrr_at_5_diff1 + value: 52.9184 + - type: nauc_mrr_at_10_max + value: 48.3523 + - type: nauc_mrr_at_10_std + value: -5.6531 + - type: nauc_mrr_at_10_diff1 + value: 53.209399999999995 + - type: nauc_mrr_at_20_max + value: 48.365700000000004 + - type: nauc_mrr_at_20_std + value: -5.4359 + - type: nauc_mrr_at_20_diff1 + value: 53.16760000000001 + - type: nauc_mrr_at_100_max + value: 48.351699999999994 + - type: nauc_mrr_at_100_std + value: -5.3941 + - type: nauc_mrr_at_100_diff1 + value: 53.2419 + - type: nauc_mrr_at_1000_max + value: 48.343399999999995 + - type: nauc_mrr_at_1000_std + value: -5.4193 + - type: nauc_mrr_at_1000_diff1 + value: 53.264500000000005 + - type: main_score + value: 45.49 + task: + type: Retrieval + - dataset: + config: default + name: MTEB HotpotQA (default) + revision: ab518f4d6fcca38d87c25209f94beba119d02014 + split: test + type: mteb/hotpotqa + metrics: + - type: ndcg_at_1 + value: 86.536 + - type: ndcg_at_3 + value: 64.485 + - type: ndcg_at_5 + value: 66.513 + - type: ndcg_at_10 + value: 68.151 + - type: ndcg_at_20 + value: 69.145 + - type: ndcg_at_100 + value: 70.552 + - type: ndcg_at_1000 + value: 71.772 + - type: map_at_1 + value: 43.268 + - type: map_at_3 + value: 56.013999999999996 + - type: map_at_5 + value: 57.69 + - type: map_at_10 + value: 58.709 + - type: map_at_20 + value: 59.122 + - type: map_at_100 + value: 59.418000000000006 + - type: map_at_1000 + value: 59.480999999999995 + - type: recall_at_1 + value: 43.268 + - type: recall_at_3 + value: 58.831999999999994 + - type: recall_at_5 + value: 62.829 + - type: recall_at_10 + value: 66.94099999999999 + - type: recall_at_20 + value: 70.135 + - type: recall_at_100 + value: 76.34 + - type: recall_at_1000 + value: 84.443 + - type: precision_at_1 + value: 86.536 + - type: precision_at_3 + value: 39.221000000000004 + - type: precision_at_5 + value: 25.131999999999998 + - type: precision_at_10 + value: 13.388 + - type: precision_at_20 + value: 7.013999999999999 + - type: precision_at_100 + value: 1.5270000000000001 + - type: precision_at_1000 + value: 0.169 + - type: mrr_at_1 + value: 86.5361 + - type: mrr_at_3 + value: 89.6151 + - type: mrr_at_5 + value: 89.9521 + - type: mrr_at_10 + value: 90.1301 + - type: mrr_at_20 + value: 90.201 + - type: mrr_at_100 + value: 90.2397 + - type: mrr_at_1000 + value: 90.245 + - type: nauc_ndcg_at_1_max + value: 57.6156 + - type: nauc_ndcg_at_1_std + value: -3.39 + - type: nauc_ndcg_at_1_diff1 + value: 83.0288 + - type: nauc_ndcg_at_3_max + value: 17.758599999999998 + - type: nauc_ndcg_at_3_std + value: 3.3521 + - type: nauc_ndcg_at_3_diff1 + value: 15.4846 + - type: nauc_ndcg_at_5_max + value: 14.6571 + - type: nauc_ndcg_at_5_std + value: 4.2071 + - type: nauc_ndcg_at_5_diff1 + value: 12.3942 + - type: nauc_ndcg_at_10_max + value: 12.5579 + - type: nauc_ndcg_at_10_std + value: 4.7895 + - type: nauc_ndcg_at_10_diff1 + value: 10.2189 + - type: nauc_ndcg_at_20_max + value: 11.5413 + - type: nauc_ndcg_at_20_std + value: 5.0043 + - type: nauc_ndcg_at_20_diff1 + value: 9.3896 + - type: nauc_ndcg_at_100_max + value: 10.6797 + - type: nauc_ndcg_at_100_std + value: 5.7805 + - type: nauc_ndcg_at_100_diff1 + value: 8.5649 + - type: nauc_ndcg_at_1000_max + value: 10.8847 + - type: nauc_ndcg_at_1000_std + value: 6.1945 + - type: nauc_ndcg_at_1000_diff1 + value: 8.539 + - type: nauc_map_at_1_max + value: 57.6156 + - type: nauc_map_at_1_std + value: -3.39 + - type: nauc_map_at_1_diff1 + value: 83.0288 + - type: nauc_map_at_3_max + value: 12.4083 + - type: nauc_map_at_3_std + value: 3.2297 + - type: nauc_map_at_3_diff1 + value: 8.2482 + - type: nauc_map_at_5_max + value: 10.4054 + - type: nauc_map_at_5_std + value: 3.7108000000000003 + - type: nauc_map_at_5_diff1 + value: 6.4539 + - type: nauc_map_at_10_max + value: 9.439300000000001 + - type: nauc_map_at_10_std + value: 4.0356000000000005 + - type: nauc_map_at_10_diff1 + value: 5.502400000000001 + - type: nauc_map_at_20_max + value: 9.141 + - type: nauc_map_at_20_std + value: 4.1145000000000005 + - type: nauc_map_at_20_diff1 + value: 5.2942 + - type: nauc_map_at_100_max + value: 9.0071 + - type: nauc_map_at_100_std + value: 4.2345 + - type: nauc_map_at_100_diff1 + value: 5.1606 + - type: nauc_map_at_1000_max + value: 9.017999999999999 + - type: nauc_map_at_1000_std + value: 4.2501 + - type: nauc_map_at_1000_diff1 + value: 5.162 + - type: nauc_recall_at_1_max + value: 57.6156 + - type: nauc_recall_at_1_std + value: -3.39 + - type: nauc_recall_at_1_diff1 + value: 83.0288 + - type: nauc_recall_at_3_max + value: 8.4358 + - type: nauc_recall_at_3_std + value: 4.925199999999999 + - type: nauc_recall_at_3_diff1 + value: 0.29009999999999997 + - type: nauc_recall_at_5_max + value: 3.2076000000000002 + - type: nauc_recall_at_5_std + value: 6.2316 + - type: nauc_recall_at_5_diff1 + value: -4.6014 + - type: nauc_recall_at_10_max + value: -1.7786 + - type: nauc_recall_at_10_std + value: 7.467300000000001 + - type: nauc_recall_at_10_diff1 + value: -9.6991 + - type: nauc_recall_at_20_max + value: -5.0717 + - type: nauc_recall_at_20_std + value: 8.1128 + - type: nauc_recall_at_20_diff1 + value: -12.5945 + - type: nauc_recall_at_100_max + value: -10.5434 + - type: nauc_recall_at_100_std + value: 11.7719 + - type: nauc_recall_at_100_diff1 + value: -18.394 + - type: nauc_recall_at_1000_max + value: -15.5908 + - type: nauc_recall_at_1000_std + value: 16.842399999999998 + - type: nauc_recall_at_1000_diff1 + value: -27.099400000000003 + - type: nauc_precision_at_1_max + value: 57.6156 + - type: nauc_precision_at_1_std + value: -3.39 + - type: nauc_precision_at_1_diff1 + value: 83.0288 + - type: nauc_precision_at_3_max + value: 8.4358 + - type: nauc_precision_at_3_std + value: 4.925199999999999 + - type: nauc_precision_at_3_diff1 + value: 0.29009999999999997 + - type: nauc_precision_at_5_max + value: 3.2076000000000002 + - type: nauc_precision_at_5_std + value: 6.2316 + - type: nauc_precision_at_5_diff1 + value: -4.6014 + - type: nauc_precision_at_10_max + value: -1.7786 + - type: nauc_precision_at_10_std + value: 7.467300000000001 + - type: nauc_precision_at_10_diff1 + value: -9.6991 + - type: nauc_precision_at_20_max + value: -5.0717 + - type: nauc_precision_at_20_std + value: 8.1128 + - type: nauc_precision_at_20_diff1 + value: -12.5945 + - type: nauc_precision_at_100_max + value: -10.5434 + - type: nauc_precision_at_100_std + value: 11.7719 + - type: nauc_precision_at_100_diff1 + value: -18.394 + - type: nauc_precision_at_1000_max + value: -15.5908 + - type: nauc_precision_at_1000_std + value: 16.842399999999998 + - type: nauc_precision_at_1000_diff1 + value: -27.099400000000003 + - type: nauc_mrr_at_1_max + value: 57.6156 + - type: nauc_mrr_at_1_std + value: -3.39 + - type: nauc_mrr_at_1_diff1 + value: 83.0288 + - type: nauc_mrr_at_3_max + value: 62.074 + - type: nauc_mrr_at_3_std + value: -0.45199999999999996 + - type: nauc_mrr_at_3_diff1 + value: 82.8025 + - type: nauc_mrr_at_5_max + value: 62.157300000000006 + - type: nauc_mrr_at_5_std + value: 0.2829 + - type: nauc_mrr_at_5_diff1 + value: 82.9913 + - type: nauc_mrr_at_10_max + value: 61.9838 + - type: nauc_mrr_at_10_std + value: 0.16670000000000001 + - type: nauc_mrr_at_10_diff1 + value: 82.9452 + - type: nauc_mrr_at_20_max + value: 61.9516 + - type: nauc_mrr_at_20_std + value: 0.18159999999999998 + - type: nauc_mrr_at_20_diff1 + value: 82.9723 + - type: nauc_mrr_at_100_max + value: 61.891600000000004 + - type: nauc_mrr_at_100_std + value: 0.1432 + - type: nauc_mrr_at_100_diff1 + value: 82.97489999999999 + - type: nauc_mrr_at_1000_max + value: 61.88249999999999 + - type: nauc_mrr_at_1000_std + value: 0.1357 + - type: nauc_mrr_at_1000_diff1 + value: 82.9723 + - type: main_score + value: 68.151 + task: + type: Retrieval + - dataset: + config: default + name: MTEB ImdbClassification (default) + revision: 3d86128a09e091d6018b6d26cad27f2739fc2db7 + split: test + type: mteb/imdb + metrics: + - type: accuracy + value: 72.5444 + - type: f1 + value: 72.4069 + - type: f1_weighted + value: 72.4069 + - type: ap + value: 66.8419 + - type: ap_weighted + value: 66.8419 + - type: main_score + value: 72.5444 + task: + type: Classification + - dataset: + config: default + name: MTEB MSMARCO (default) + revision: c5a29a104738b98a9e76336939199e264163d4a0 + split: dev + type: mteb/msmarco + metrics: + - type: ndcg_at_1 + value: 25.516 + - type: ndcg_at_3 + value: 36.687999999999995 + - type: ndcg_at_5 + value: 40.864 + - type: ndcg_at_10 + value: 44.856 + - type: ndcg_at_20 + value: 47.3 + - type: ndcg_at_100 + value: 50.062 + - type: ndcg_at_1000 + value: 51.085 + - type: map_at_1 + value: 24.782 + - type: map_at_3 + value: 33.668 + - type: map_at_5 + value: 36.010999999999996 + - type: map_at_10 + value: 37.702000000000005 + - type: map_at_20 + value: 38.391 + - type: map_at_100 + value: 38.798 + - type: map_at_1000 + value: 38.841 + - type: recall_at_1 + value: 24.782 + - type: recall_at_3 + value: 44.722 + - type: recall_at_5 + value: 54.769999999999996 + - type: recall_at_10 + value: 66.842 + - type: recall_at_20 + value: 76.319 + - type: recall_at_100 + value: 90.761 + - type: recall_at_1000 + value: 98.48 + - type: precision_at_1 + value: 25.516 + - type: precision_at_3 + value: 15.506 + - type: precision_at_5 + value: 11.413 + - type: precision_at_10 + value: 6.99 + - type: precision_at_20 + value: 4.009 + - type: precision_at_100 + value: 0.959 + - type: precision_at_1000 + value: 0.105 + - type: mrr_at_1 + value: 25.5014 + - type: mrr_at_3 + value: 34.3553 + - type: mrr_at_5 + value: 36.666199999999996 + - type: mrr_at_10 + value: 38.3084 + - type: mrr_at_20 + value: 38.9663 + - type: mrr_at_100 + value: 39.341300000000004 + - type: mrr_at_1000 + value: 39.3785 + - type: nauc_ndcg_at_1_max + value: 4.2138 + - type: nauc_ndcg_at_1_std + value: -24.7801 + - type: nauc_ndcg_at_1_diff1 + value: 37.758399999999995 + - type: nauc_ndcg_at_3_max + value: 5.2536 + - type: nauc_ndcg_at_3_std + value: -29.642200000000003 + - type: nauc_ndcg_at_3_diff1 + value: 32.1639 + - type: nauc_ndcg_at_5_max + value: 5.0839 + - type: nauc_ndcg_at_5_std + value: -31.3077 + - type: nauc_ndcg_at_5_diff1 + value: 31.5135 + - type: nauc_ndcg_at_10_max + value: 6.2542 + - type: nauc_ndcg_at_10_std + value: -30.8439 + - type: nauc_ndcg_at_10_diff1 + value: 31.461299999999998 + - type: nauc_ndcg_at_20_max + value: 6.5669 + - type: nauc_ndcg_at_20_std + value: -29.6288 + - type: nauc_ndcg_at_20_diff1 + value: 31.590200000000003 + - type: nauc_ndcg_at_100_max + value: 6.691800000000001 + - type: nauc_ndcg_at_100_std + value: -28.1768 + - type: nauc_ndcg_at_100_diff1 + value: 32.1699 + - type: nauc_ndcg_at_1000_max + value: 6.451700000000001 + - type: nauc_ndcg_at_1000_std + value: -28.2093 + - type: nauc_ndcg_at_1000_diff1 + value: 32.3573 + - type: nauc_map_at_1_max + value: 4.1941 + - type: nauc_map_at_1_std + value: -24.9531 + - type: nauc_map_at_1_diff1 + value: 38.099 + - type: nauc_map_at_3_max + value: 4.9883999999999995 + - type: nauc_map_at_3_std + value: -28.7062 + - type: nauc_map_at_3_diff1 + value: 33.5696 + - type: nauc_map_at_5_max + value: 4.8525 + - type: nauc_map_at_5_std + value: -29.6601 + - type: nauc_map_at_5_diff1 + value: 33.2144 + - type: nauc_map_at_10_max + value: 5.3533 + - type: nauc_map_at_10_std + value: -29.4529 + - type: nauc_map_at_10_diff1 + value: 33.219300000000004 + - type: nauc_map_at_20_max + value: 5.416300000000001 + - type: nauc_map_at_20_std + value: -29.1294 + - type: nauc_map_at_20_diff1 + value: 33.2747 + - type: nauc_map_at_100_max + value: 5.4547 + - type: nauc_map_at_100_std + value: -28.8978 + - type: nauc_map_at_100_diff1 + value: 33.3505 + - type: nauc_map_at_1000_max + value: 5.4512 + - type: nauc_map_at_1000_std + value: -28.8844 + - type: nauc_map_at_1000_diff1 + value: 33.356700000000004 + - type: nauc_recall_at_1_max + value: 4.1941 + - type: nauc_recall_at_1_std + value: -24.9531 + - type: nauc_recall_at_1_diff1 + value: 38.099 + - type: nauc_recall_at_3_max + value: 5.884799999999999 + - type: nauc_recall_at_3_std + value: -32.317 + - type: nauc_recall_at_3_diff1 + value: 28.284399999999998 + - type: nauc_recall_at_5_max + value: 5.4525 + - type: nauc_recall_at_5_std + value: -36.4055 + - type: nauc_recall_at_5_diff1 + value: 26.384200000000003 + - type: nauc_recall_at_10_max + value: 9.403400000000001 + - type: nauc_recall_at_10_std + value: -35.9112 + - type: nauc_recall_at_10_diff1 + value: 25.2415 + - type: nauc_recall_at_20_max + value: 12.0952 + - type: nauc_recall_at_20_std + value: -30.778299999999998 + - type: nauc_recall_at_20_diff1 + value: 24.1866 + - type: nauc_recall_at_100_max + value: 19.6413 + - type: nauc_recall_at_100_std + value: -11.9243 + - type: nauc_recall_at_100_diff1 + value: 24.6153 + - type: nauc_recall_at_1000_max + value: 48.1206 + - type: nauc_recall_at_1000_std + value: 48.0062 + - type: nauc_recall_at_1000_diff1 + value: 16.2543 + - type: nauc_precision_at_1_max + value: 4.2138 + - type: nauc_precision_at_1_std + value: -24.7801 + - type: nauc_precision_at_1_diff1 + value: 37.758399999999995 + - type: nauc_precision_at_3_max + value: 5.7985 + - type: nauc_precision_at_3_std + value: -31.749899999999997 + - type: nauc_precision_at_3_diff1 + value: 27.373399999999997 + - type: nauc_precision_at_5_max + value: 5.390000000000001 + - type: nauc_precision_at_5_std + value: -35.0586 + - type: nauc_precision_at_5_diff1 + value: 25.100099999999998 + - type: nauc_precision_at_10_max + value: 9.248199999999999 + - type: nauc_precision_at_10_std + value: -32.244299999999996 + - type: nauc_precision_at_10_diff1 + value: 22.5684 + - type: nauc_precision_at_20_max + value: 11.495099999999999 + - type: nauc_precision_at_20_std + value: -24.226300000000002 + - type: nauc_precision_at_20_diff1 + value: 19.6528 + - type: nauc_precision_at_100_max + value: 14.3649 + - type: nauc_precision_at_100_std + value: 0.0593 + - type: nauc_precision_at_100_diff1 + value: 10.9596 + - type: nauc_precision_at_1000_max + value: 10.9512 + - type: nauc_precision_at_1000_std + value: 18.288 + - type: nauc_precision_at_1000_diff1 + value: -3.5423000000000004 + - type: nauc_mrr_at_1_max + value: 4.2204 + - type: nauc_mrr_at_1_std + value: -24.7703 + - type: nauc_mrr_at_1_diff1 + value: 37.8126 + - type: nauc_mrr_at_3_max + value: 5.0668 + - type: nauc_mrr_at_3_std + value: -28.2677 + - type: nauc_mrr_at_3_diff1 + value: 33.3724 + - type: nauc_mrr_at_5_max + value: 5.0481 + - type: nauc_mrr_at_5_std + value: -29.133 + - type: nauc_mrr_at_5_diff1 + value: 33.0415 + - type: nauc_mrr_at_10_max + value: 5.5038 + - type: nauc_mrr_at_10_std + value: -28.886200000000002 + - type: nauc_mrr_at_10_diff1 + value: 33.0593 + - type: nauc_mrr_at_20_max + value: 5.5467 + - type: nauc_mrr_at_20_std + value: -28.5678 + - type: nauc_mrr_at_20_diff1 + value: 33.0916 + - type: nauc_mrr_at_100_max + value: 5.5636 + - type: nauc_mrr_at_100_std + value: -28.3877 + - type: nauc_mrr_at_100_diff1 + value: 33.1799 + - type: nauc_mrr_at_1000_max + value: 5.557 + - type: nauc_mrr_at_1000_std + value: -28.3796 + - type: nauc_mrr_at_1000_diff1 + value: 33.184999999999995 + - type: main_score + value: 44.856 + task: + type: Retrieval + - dataset: + config: en + name: MTEB MTOPDomainClassification (en) + revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf + split: test + type: mteb/mtop_domain + metrics: + - type: accuracy + value: 93.5317 + - type: f1 + value: 93.1956 + - type: f1_weighted + value: 93.5431 + - type: main_score + value: 93.5317 + task: + type: Classification + - dataset: + config: en + name: MTEB MTOPIntentClassification (en) + revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba + split: test + type: mteb/mtop_intent + metrics: + - type: accuracy + value: 67.7907 + - type: f1 + value: 48.2877 + - type: f1_weighted + value: 70.3225 + - type: main_score + value: 67.7907 + task: + type: Classification + - dataset: + config: en + name: MTEB MassiveIntentClassification (en) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 71.456 + - type: f1 + value: 68.2268 + - type: f1_weighted + value: 70.4722 + - type: main_score + value: 71.456 + task: + type: Classification + - dataset: + config: en + name: MTEB MassiveScenarioClassification (en) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 76.21719999999999 + - type: f1 + value: 75.14189999999999 + - type: f1_weighted + value: 76.0733 + - type: main_score + value: 76.21719999999999 + task: + type: Classification + - dataset: + config: default + name: MTEB MedrxivClusteringP2P (default) + revision: e7a26af6f3ae46b30dde8737f02c07b1505bcc73 + split: test + type: mteb/medrxiv-clustering-p2p + metrics: + - type: v_measure + value: 31.3917 + - type: v_measure_std + value: 1.4778 + - type: main_score + value: 31.3917 + task: + type: Clustering + - dataset: + config: default + name: MTEB MedrxivClusteringS2S (default) + revision: 35191c8c0dca72d8ff3efcd72aa802307d469663 + split: test + type: mteb/medrxiv-clustering-s2s + metrics: + - type: v_measure + value: 28.2408 + - type: v_measure_std + value: 1.1622999999999999 + - type: main_score + value: 28.2408 + task: + type: Clustering + - dataset: + config: default + name: MTEB MindSmallReranking (default) + revision: 59042f120c80e8afa9cdbb224f67076cec0fc9a7 + split: test + type: mteb/mind_small + metrics: + - type: map + value: 29.5796 + - type: mrr + value: 30.3081 + - type: nAUC_map_max + value: -24.9194 + - type: nAUC_map_std + value: -9.042 + - type: nAUC_map_diff1 + value: 12.1611 + - type: nAUC_mrr_max + value: -19.3867 + - type: nAUC_mrr_std + value: -6.3873 + - type: nAUC_mrr_diff1 + value: 11.8078 + - type: main_score + value: 29.5796 + task: + type: Reranking + - dataset: + config: default + name: MTEB NFCorpus (default) + revision: ec0fa4fe99da2ff19ca1214b7966684033a58814 + split: test + type: mteb/nfcorpus + metrics: + - type: ndcg_at_1 + value: 45.046 + - type: ndcg_at_3 + value: 41.704 + - type: ndcg_at_5 + value: 39.296 + - type: ndcg_at_10 + value: 35.343999999999994 + - type: ndcg_at_20 + value: 32.525999999999996 + - type: ndcg_at_100 + value: 31.352999999999998 + - type: ndcg_at_1000 + value: 39.772 + - type: map_at_1 + value: 5.833 + - type: map_at_3 + value: 9.953 + - type: map_at_5 + value: 11.549 + - type: map_at_10 + value: 13.38 + - type: map_at_20 + value: 14.706 + - type: map_at_100 + value: 16.422 + - type: map_at_1000 + value: 17.777 + - type: recall_at_1 + value: 5.833 + - type: recall_at_3 + value: 11.112 + - type: recall_at_5 + value: 13.834 + - type: recall_at_10 + value: 16.961000000000002 + - type: recall_at_20 + value: 20.294999999999998 + - type: recall_at_100 + value: 30.253000000000004 + - type: recall_at_1000 + value: 60.902 + - type: precision_at_1 + value: 46.44 + - type: precision_at_3 + value: 39.009 + - type: precision_at_5 + value: 33.745999999999995 + - type: precision_at_10 + value: 25.635 + - type: precision_at_20 + value: 18.576 + - type: precision_at_100 + value: 7.731000000000001 + - type: precision_at_1000 + value: 2.037 + - type: mrr_at_1 + value: 46.7492 + - type: mrr_at_3 + value: 54.6956 + - type: mrr_at_5 + value: 55.8875 + - type: mrr_at_10 + value: 56.3913 + - type: mrr_at_20 + value: 56.6265 + - type: mrr_at_100 + value: 56.815599999999996 + - type: mrr_at_1000 + value: 56.8573 + - type: nauc_ndcg_at_1_max + value: 43.3685 + - type: nauc_ndcg_at_1_std + value: 21.6124 + - type: nauc_ndcg_at_1_diff1 + value: 29.0317 + - type: nauc_ndcg_at_3_max + value: 39.8155 + - type: nauc_ndcg_at_3_std + value: 23.2206 + - type: nauc_ndcg_at_3_diff1 + value: 20.7425 + - type: nauc_ndcg_at_5_max + value: 40.951 + - type: nauc_ndcg_at_5_std + value: 24.7184 + - type: nauc_ndcg_at_5_diff1 + value: 19.098599999999998 + - type: nauc_ndcg_at_10_max + value: 41.4733 + - type: nauc_ndcg_at_10_std + value: 27.4588 + - type: nauc_ndcg_at_10_diff1 + value: 17.224800000000002 + - type: nauc_ndcg_at_20_max + value: 40.3519 + - type: nauc_ndcg_at_20_std + value: 27.2947 + - type: nauc_ndcg_at_20_diff1 + value: 16.502 + - type: nauc_ndcg_at_100_max + value: 44.0676 + - type: nauc_ndcg_at_100_std + value: 29.1921 + - type: nauc_ndcg_at_100_diff1 + value: 20.9199 + - type: nauc_ndcg_at_1000_max + value: 48.9082 + - type: nauc_ndcg_at_1000_std + value: 33.799600000000005 + - type: nauc_ndcg_at_1000_diff1 + value: 19.741600000000002 + - type: nauc_map_at_1_max + value: 19.2048 + - type: nauc_map_at_1_std + value: -13.564599999999999 + - type: nauc_map_at_1_diff1 + value: 37.601099999999995 + - type: nauc_map_at_3_max + value: 23.1853 + - type: nauc_map_at_3_std + value: -8.3204 + - type: nauc_map_at_3_diff1 + value: 32.5527 + - type: nauc_map_at_5_max + value: 26.747500000000002 + - type: nauc_map_at_5_std + value: -4.136 + - type: nauc_map_at_5_diff1 + value: 29.041800000000002 + - type: nauc_map_at_10_max + value: 30.492200000000004 + - type: nauc_map_at_10_std + value: 2.2847 + - type: nauc_map_at_10_diff1 + value: 25.949699999999996 + - type: nauc_map_at_20_max + value: 32.628800000000005 + - type: nauc_map_at_20_std + value: 6.2305 + - type: nauc_map_at_20_diff1 + value: 24.0997 + - type: nauc_map_at_100_max + value: 35.0282 + - type: nauc_map_at_100_std + value: 12.181899999999999 + - type: nauc_map_at_100_diff1 + value: 22.6844 + - type: nauc_map_at_1000_max + value: 35.274899999999995 + - type: nauc_map_at_1000_std + value: 14.9827 + - type: nauc_map_at_1000_diff1 + value: 21.4096 + - type: nauc_recall_at_1_max + value: 19.2048 + - type: nauc_recall_at_1_std + value: -13.564599999999999 + - type: nauc_recall_at_1_diff1 + value: 37.601099999999995 + - type: nauc_recall_at_3_max + value: 20.5895 + - type: nauc_recall_at_3_std + value: -7.8295 + - type: nauc_recall_at_3_diff1 + value: 28.4675 + - type: nauc_recall_at_5_max + value: 24.8771 + - type: nauc_recall_at_5_std + value: -2.869 + - type: nauc_recall_at_5_diff1 + value: 23.301 + - type: nauc_recall_at_10_max + value: 28.647299999999998 + - type: nauc_recall_at_10_std + value: 4.4991 + - type: nauc_recall_at_10_diff1 + value: 20.5606 + - type: nauc_recall_at_20_max + value: 30.3525 + - type: nauc_recall_at_20_std + value: 8.712 + - type: nauc_recall_at_20_diff1 + value: 17.4748 + - type: nauc_recall_at_100_max + value: 34.0702 + - type: nauc_recall_at_100_std + value: 23.3319 + - type: nauc_recall_at_100_diff1 + value: 17.2015 + - type: nauc_recall_at_1000_max + value: 27.8011 + - type: nauc_recall_at_1000_std + value: 21.6507 + - type: nauc_recall_at_1000_diff1 + value: 4.4638 + - type: nauc_precision_at_1_max + value: 44.6989 + - type: nauc_precision_at_1_std + value: 22.622 + - type: nauc_precision_at_1_diff1 + value: 28.881400000000003 + - type: nauc_precision_at_3_max + value: 39.4166 + - type: nauc_precision_at_3_std + value: 29.2591 + - type: nauc_precision_at_3_diff1 + value: 12.1577 + - type: nauc_precision_at_5_max + value: 39.6371 + - type: nauc_precision_at_5_std + value: 33.201 + - type: nauc_precision_at_5_diff1 + value: 7.958 + - type: nauc_precision_at_10_max + value: 38.2593 + - type: nauc_precision_at_10_std + value: 40.6097 + - type: nauc_precision_at_10_diff1 + value: 1.376 + - type: nauc_precision_at_20_max + value: 31.375999999999998 + - type: nauc_precision_at_20_std + value: 42.3468 + - type: nauc_precision_at_20_diff1 + value: -4.1699 + - type: nauc_precision_at_100_max + value: 16.628 + - type: nauc_precision_at_100_std + value: 41.800599999999996 + - type: nauc_precision_at_100_diff1 + value: -9.4674 + - type: nauc_precision_at_1000_max + value: 1.6051 + - type: nauc_precision_at_1000_std + value: 29.1306 + - type: nauc_precision_at_1000_diff1 + value: -11.1912 + - type: nauc_mrr_at_1_max + value: 44.4339 + - type: nauc_mrr_at_1_std + value: 23.6489 + - type: nauc_mrr_at_1_diff1 + value: 28.0393 + - type: nauc_mrr_at_3_max + value: 47.780899999999995 + - type: nauc_mrr_at_3_std + value: 31.412499999999998 + - type: nauc_mrr_at_3_diff1 + value: 24.1569 + - type: nauc_mrr_at_5_max + value: 48.732 + - type: nauc_mrr_at_5_std + value: 31.899100000000004 + - type: nauc_mrr_at_5_diff1 + value: 24.4177 + - type: nauc_mrr_at_10_max + value: 48.9748 + - type: nauc_mrr_at_10_std + value: 32.2053 + - type: nauc_mrr_at_10_diff1 + value: 24.0317 + - type: nauc_mrr_at_20_max + value: 49.0832 + - type: nauc_mrr_at_20_std + value: 32.0994 + - type: nauc_mrr_at_20_diff1 + value: 23.9777 + - type: nauc_mrr_at_100_max + value: 49.1731 + - type: nauc_mrr_at_100_std + value: 32.3179 + - type: nauc_mrr_at_100_diff1 + value: 24.081 + - type: nauc_mrr_at_1000_max + value: 49.1387 + - type: nauc_mrr_at_1000_std + value: 32.2738 + - type: nauc_mrr_at_1000_diff1 + value: 24.063200000000002 + - type: main_score + value: 35.343999999999994 + task: + type: Retrieval + - dataset: + config: default + name: MTEB NQ (default) + revision: b774495ed302d8c44a3a7ea25c90dbce03968f31 + split: test + type: mteb/nq + metrics: + - type: ndcg_at_1 + value: 44.93 + - type: ndcg_at_3 + value: 56.003 + - type: ndcg_at_5 + value: 60.150000000000006 + - type: ndcg_at_10 + value: 63.673 + - type: ndcg_at_20 + value: 65.211 + - type: ndcg_at_100 + value: 66.686 + - type: ndcg_at_1000 + value: 67.009 + - type: map_at_1 + value: 40.035 + - type: map_at_3 + value: 51.976 + - type: map_at_5 + value: 54.510999999999996 + - type: map_at_10 + value: 56.17100000000001 + - type: map_at_20 + value: 56.684 + - type: map_at_100 + value: 56.932 + - type: map_at_1000 + value: 56.946 + - type: recall_at_1 + value: 40.035 + - type: recall_at_3 + value: 64.224 + - type: recall_at_5 + value: 73.682 + - type: recall_at_10 + value: 83.809 + - type: recall_at_20 + value: 89.385 + - type: recall_at_100 + value: 96.705 + - type: recall_at_1000 + value: 99.054 + - type: precision_at_1 + value: 44.93 + - type: precision_at_3 + value: 25.019000000000002 + - type: precision_at_5 + value: 17.445 + - type: precision_at_10 + value: 10.043000000000001 + - type: precision_at_20 + value: 5.4 + - type: precision_at_100 + value: 1.174 + - type: precision_at_1000 + value: 0.121 + - type: mrr_at_1 + value: 44.9305 + - type: mrr_at_3 + value: 55.37370000000001 + - type: mrr_at_5 + value: 57.4464 + - type: mrr_at_10 + value: 58.680200000000006 + - type: mrr_at_20 + value: 59.0042 + - type: mrr_at_100 + value: 59.178799999999995 + - type: mrr_at_1000 + value: 59.188700000000004 + - type: nauc_ndcg_at_1_max + value: 23.8396 + - type: nauc_ndcg_at_1_std + value: -3.8885000000000005 + - type: nauc_ndcg_at_1_diff1 + value: 37.971500000000006 + - type: nauc_ndcg_at_3_max + value: 30.025800000000004 + - type: nauc_ndcg_at_3_std + value: -4.9848 + - type: nauc_ndcg_at_3_diff1 + value: 34.324799999999996 + - type: nauc_ndcg_at_5_max + value: 32.2984 + - type: nauc_ndcg_at_5_std + value: -3.263 + - type: nauc_ndcg_at_5_diff1 + value: 35.2865 + - type: nauc_ndcg_at_10_max + value: 32.4173 + - type: nauc_ndcg_at_10_std + value: -2.398 + - type: nauc_ndcg_at_10_diff1 + value: 34.767399999999995 + - type: nauc_ndcg_at_20_max + value: 32.332 + - type: nauc_ndcg_at_20_std + value: -1.7824 + - type: nauc_ndcg_at_20_diff1 + value: 35.0354 + - type: nauc_ndcg_at_100_max + value: 31.3774 + - type: nauc_ndcg_at_100_std + value: -1.4645 + - type: nauc_ndcg_at_100_diff1 + value: 35.255900000000004 + - type: nauc_ndcg_at_1000_max + value: 31.008799999999997 + - type: nauc_ndcg_at_1000_std + value: -1.9499 + - type: nauc_ndcg_at_1000_diff1 + value: 35.3522 + - type: nauc_map_at_1_max + value: 21.296300000000002 + - type: nauc_map_at_1_std + value: -6.0126 + - type: nauc_map_at_1_diff1 + value: 37.9216 + - type: nauc_map_at_3_max + value: 28.1195 + - type: nauc_map_at_3_std + value: -5.3494 + - type: nauc_map_at_3_diff1 + value: 35.0839 + - type: nauc_map_at_5_max + value: 29.365999999999996 + - type: nauc_map_at_5_std + value: -4.410200000000001 + - type: nauc_map_at_5_diff1 + value: 35.6342 + - type: nauc_map_at_10_max + value: 29.378300000000003 + - type: nauc_map_at_10_std + value: -4.0228 + - type: nauc_map_at_10_diff1 + value: 35.451 + - type: nauc_map_at_20_max + value: 29.3604 + - type: nauc_map_at_20_std + value: -3.7953 + - type: nauc_map_at_20_diff1 + value: 35.5496 + - type: nauc_map_at_100_max + value: 29.233199999999997 + - type: nauc_map_at_100_std + value: -3.7321 + - type: nauc_map_at_100_diff1 + value: 35.574099999999994 + - type: nauc_map_at_1000_max + value: 29.2215 + - type: nauc_map_at_1000_std + value: -3.7482 + - type: nauc_map_at_1000_diff1 + value: 35.5805 + - type: nauc_recall_at_1_max + value: 21.296300000000002 + - type: nauc_recall_at_1_std + value: -6.0126 + - type: nauc_recall_at_1_diff1 + value: 37.9216 + - type: nauc_recall_at_3_max + value: 34.2599 + - type: nauc_recall_at_3_std + value: -5.5474000000000006 + - type: nauc_recall_at_3_diff1 + value: 30.7103 + - type: nauc_recall_at_5_max + value: 41.6689 + - type: nauc_recall_at_5_std + value: -0.7705 + - type: nauc_recall_at_5_diff1 + value: 32.6001 + - type: nauc_recall_at_10_max + value: 47.236200000000004 + - type: nauc_recall_at_10_std + value: 3.9309999999999996 + - type: nauc_recall_at_10_diff1 + value: 29.277199999999997 + - type: nauc_recall_at_20_max + value: 53.957100000000004 + - type: nauc_recall_at_20_std + value: 11.282499999999999 + - type: nauc_recall_at_20_diff1 + value: 29.7674 + - type: nauc_recall_at_100_max + value: 66.87039999999999 + - type: nauc_recall_at_100_std + value: 46.8733 + - type: nauc_recall_at_100_diff1 + value: 30.0249 + - type: nauc_recall_at_1000_max + value: 88.33670000000001 + - type: nauc_recall_at_1000_std + value: 77.0724 + - type: nauc_recall_at_1000_diff1 + value: 34.0192 + - type: nauc_precision_at_1_max + value: 23.8396 + - type: nauc_precision_at_1_std + value: -3.8885000000000005 + - type: nauc_precision_at_1_diff1 + value: 37.971500000000006 + - type: nauc_precision_at_3_max + value: 31.053399999999996 + - type: nauc_precision_at_3_std + value: 0.3766 + - type: nauc_precision_at_3_diff1 + value: 21.5732 + - type: nauc_precision_at_5_max + value: 30.816100000000002 + - type: nauc_precision_at_5_std + value: 5.3659 + - type: nauc_precision_at_5_diff1 + value: 17.4728 + - type: nauc_precision_at_10_max + value: 25.204300000000003 + - type: nauc_precision_at_10_std + value: 10.6652 + - type: nauc_precision_at_10_diff1 + value: 7.7665 + - type: nauc_precision_at_20_max + value: 20.3015 + - type: nauc_precision_at_20_std + value: 14.1789 + - type: nauc_precision_at_20_diff1 + value: 3.2251000000000003 + - type: nauc_precision_at_100_max + value: 9.709 + - type: nauc_precision_at_100_std + value: 17.7706 + - type: nauc_precision_at_100_diff1 + value: -5.5258 + - type: nauc_precision_at_1000_max + value: 4.5083 + - type: nauc_precision_at_1000_std + value: 14.754900000000001 + - type: nauc_precision_at_1000_diff1 + value: -8.1761 + - type: nauc_mrr_at_1_max + value: 23.8396 + - type: nauc_mrr_at_1_std + value: -3.8885000000000005 + - type: nauc_mrr_at_1_diff1 + value: 37.971500000000006 + - type: nauc_mrr_at_3_max + value: 28.9257 + - type: nauc_mrr_at_3_std + value: -3.6295 + - type: nauc_mrr_at_3_diff1 + value: 35.390100000000004 + - type: nauc_mrr_at_5_max + value: 29.8503 + - type: nauc_mrr_at_5_std + value: -2.8144 + - type: nauc_mrr_at_5_diff1 + value: 35.8786 + - type: nauc_mrr_at_10_max + value: 29.662899999999997 + - type: nauc_mrr_at_10_std + value: -2.6432 + - type: nauc_mrr_at_10_diff1 + value: 35.708400000000005 + - type: nauc_mrr_at_20_max + value: 29.5659 + - type: nauc_mrr_at_20_std + value: -2.6337 + - type: nauc_mrr_at_20_diff1 + value: 35.761900000000004 + - type: nauc_mrr_at_100_max + value: 29.432399999999998 + - type: nauc_mrr_at_100_std + value: -2.6328 + - type: nauc_mrr_at_100_diff1 + value: 35.8182 + - type: nauc_mrr_at_1000_max + value: 29.4234 + - type: nauc_mrr_at_1000_std + value: -2.6451 + - type: nauc_mrr_at_1000_diff1 + value: 35.8215 + - type: main_score + value: 63.673 + task: + type: Retrieval + - dataset: + config: default + name: MTEB QuoraRetrieval (default) + revision: e4e08e0b7dbe3c8700f0daef558ff32256715259 + split: test + type: mteb/quora + metrics: + - type: ndcg_at_1 + value: 82.27 + - type: ndcg_at_3 + value: 86.28099999999999 + - type: ndcg_at_5 + value: 87.81400000000001 + - type: ndcg_at_10 + value: 89.021 + - type: ndcg_at_20 + value: 89.643 + - type: ndcg_at_100 + value: 90.13 + - type: ndcg_at_1000 + value: 90.226 + - type: map_at_1 + value: 71.43599999999999 + - type: map_at_3 + value: 82.49 + - type: map_at_5 + value: 84.331 + - type: map_at_10 + value: 85.416 + - type: map_at_20 + value: 85.827 + - type: map_at_100 + value: 86.024 + - type: map_at_1000 + value: 86.039 + - type: recall_at_1 + value: 71.43599999999999 + - type: recall_at_3 + value: 87.912 + - type: recall_at_5 + value: 92.30000000000001 + - type: recall_at_10 + value: 95.814 + - type: recall_at_20 + value: 97.80799999999999 + - type: recall_at_100 + value: 99.551 + - type: recall_at_1000 + value: 99.97 + - type: precision_at_1 + value: 82.27 + - type: precision_at_3 + value: 37.747 + - type: precision_at_5 + value: 24.782 + - type: precision_at_10 + value: 13.497 + - type: precision_at_20 + value: 7.147 + - type: precision_at_100 + value: 1.529 + - type: precision_at_1000 + value: 0.157 + - type: mrr_at_1 + value: 82.23 + - type: mrr_at_3 + value: 87.26 + - type: mrr_at_5 + value: 87.9305 + - type: mrr_at_10 + value: 88.20949999999999 + - type: mrr_at_20 + value: 88.2764 + - type: mrr_at_100 + value: 88.2967 + - type: mrr_at_1000 + value: 88.2976 + - type: nauc_ndcg_at_1_max + value: 37.0736 + - type: nauc_ndcg_at_1_std + value: -43.2326 + - type: nauc_ndcg_at_1_diff1 + value: 77.9945 + - type: nauc_ndcg_at_3_max + value: 33.9426 + - type: nauc_ndcg_at_3_std + value: -51.3108 + - type: nauc_ndcg_at_3_diff1 + value: 76.2559 + - type: nauc_ndcg_at_5_max + value: 34.927 + - type: nauc_ndcg_at_5_std + value: -52.50749999999999 + - type: nauc_ndcg_at_5_diff1 + value: 76.578 + - type: nauc_ndcg_at_10_max + value: 35.9905 + - type: nauc_ndcg_at_10_std + value: -51.808699999999995 + - type: nauc_ndcg_at_10_diff1 + value: 76.6957 + - type: nauc_ndcg_at_20_max + value: 36.119299999999996 + - type: nauc_ndcg_at_20_std + value: -50.1628 + - type: nauc_ndcg_at_20_diff1 + value: 76.6659 + - type: nauc_ndcg_at_100_max + value: 36.4315 + - type: nauc_ndcg_at_100_std + value: -48.0358 + - type: nauc_ndcg_at_100_diff1 + value: 76.5866 + - type: nauc_ndcg_at_1000_max + value: 36.459399999999995 + - type: nauc_ndcg_at_1000_std + value: -47.834199999999996 + - type: nauc_ndcg_at_1000_diff1 + value: 76.5791 + - type: nauc_map_at_1_max + value: 25.902199999999997 + - type: nauc_map_at_1_std + value: -44.6605 + - type: nauc_map_at_1_diff1 + value: 80.78070000000001 + - type: nauc_map_at_3_max + value: 31.3371 + - type: nauc_map_at_3_std + value: -53.9334 + - type: nauc_map_at_3_diff1 + value: 77.7089 + - type: nauc_map_at_5_max + value: 33.1663 + - type: nauc_map_at_5_std + value: -53.86919999999999 + - type: nauc_map_at_5_diff1 + value: 77.32430000000001 + - type: nauc_map_at_10_max + value: 34.4253 + - type: nauc_map_at_10_std + value: -52.423500000000004 + - type: nauc_map_at_10_diff1 + value: 77.0479 + - type: nauc_map_at_20_max + value: 34.6738 + - type: nauc_map_at_20_std + value: -51.095400000000005 + - type: nauc_map_at_20_diff1 + value: 76.88810000000001 + - type: nauc_map_at_100_max + value: 34.7984 + - type: nauc_map_at_100_std + value: -50.2705 + - type: nauc_map_at_100_diff1 + value: 76.8083 + - type: nauc_map_at_1000_max + value: 34.8162 + - type: nauc_map_at_1000_std + value: -50.211600000000004 + - type: nauc_map_at_1000_diff1 + value: 76.8047 + - type: nauc_recall_at_1_max + value: 25.902199999999997 + - type: nauc_recall_at_1_std + value: -44.6605 + - type: nauc_recall_at_1_diff1 + value: 80.78070000000001 + - type: nauc_recall_at_3_max + value: 27.693 + - type: nauc_recall_at_3_std + value: -61.799400000000006 + - type: nauc_recall_at_3_diff1 + value: 74.25 + - type: nauc_recall_at_5_max + value: 30.216700000000003 + - type: nauc_recall_at_5_std + value: -68.2919 + - type: nauc_recall_at_5_diff1 + value: 72.8613 + - type: nauc_recall_at_10_max + value: 34.4765 + - type: nauc_recall_at_10_std + value: -74.3633 + - type: nauc_recall_at_10_diff1 + value: 73.0316 + - type: nauc_recall_at_20_max + value: 33.812 + - type: nauc_recall_at_20_std + value: -72.8956 + - type: nauc_recall_at_20_diff1 + value: 73.4475 + - type: nauc_recall_at_100_max + value: 39.0326 + - type: nauc_recall_at_100_std + value: -42.9628 + - type: nauc_recall_at_100_diff1 + value: 72.66669999999999 + - type: nauc_recall_at_1000_max + value: 16.4069 + - type: nauc_recall_at_1000_std + value: 20.353099999999998 + - type: nauc_recall_at_1000_diff1 + value: 72.6857 + - type: nauc_precision_at_1_max + value: 37.0736 + - type: nauc_precision_at_1_std + value: -43.2326 + - type: nauc_precision_at_1_diff1 + value: 77.9945 + - type: nauc_precision_at_3_max + value: 7.225099999999999 + - type: nauc_precision_at_3_std + value: 5.4519 + - type: nauc_precision_at_3_diff1 + value: -20.1979 + - type: nauc_precision_at_5_max + value: 3.1125 + - type: nauc_precision_at_5_std + value: 17.542099999999998 + - type: nauc_precision_at_5_diff1 + value: -32.5768 + - type: nauc_precision_at_10_max + value: -0.3758 + - type: nauc_precision_at_10_std + value: 27.9681 + - type: nauc_precision_at_10_diff1 + value: -39.8065 + - type: nauc_precision_at_20_max + value: -2.7107 + - type: nauc_precision_at_20_std + value: 34.9186 + - type: nauc_precision_at_20_diff1 + value: -42.686800000000005 + - type: nauc_precision_at_100_max + value: -4.587 + - type: nauc_precision_at_100_std + value: 41.415600000000005 + - type: nauc_precision_at_100_diff1 + value: -44.357 + - type: nauc_precision_at_1000_max + value: -5.003 + - type: nauc_precision_at_1000_std + value: 42.5355 + - type: nauc_precision_at_1000_diff1 + value: -44.5697 + - type: nauc_mrr_at_1_max + value: 37.1298 + - type: nauc_mrr_at_1_std + value: -43.2774 + - type: nauc_mrr_at_1_diff1 + value: 78.0714 + - type: nauc_mrr_at_3_max + value: 37.644800000000004 + - type: nauc_mrr_at_3_std + value: -46.231 + - type: nauc_mrr_at_3_diff1 + value: 77.0599 + - type: nauc_mrr_at_5_max + value: 37.994299999999996 + - type: nauc_mrr_at_5_std + value: -46.0511 + - type: nauc_mrr_at_5_diff1 + value: 77.1377 + - type: nauc_mrr_at_10_max + value: 37.9206 + - type: nauc_mrr_at_10_std + value: -45.8065 + - type: nauc_mrr_at_10_diff1 + value: 77.1994 + - type: nauc_mrr_at_20_max + value: 37.8028 + - type: nauc_mrr_at_20_std + value: -45.7095 + - type: nauc_mrr_at_20_diff1 + value: 77.2152 + - type: nauc_mrr_at_100_max + value: 37.7912 + - type: nauc_mrr_at_100_std + value: -45.6767 + - type: nauc_mrr_at_100_diff1 + value: 77.2139 + - type: nauc_mrr_at_1000_max + value: 37.79 + - type: nauc_mrr_at_1000_std + value: -45.6766 + - type: nauc_mrr_at_1000_diff1 + value: 77.2145 + - type: main_score + value: 89.021 + task: + type: Retrieval + - dataset: + config: default + name: MTEB RedditClustering (default) + revision: 24640382cdbf8abc73003fb0fa6d111a705499eb + split: test + type: mteb/reddit-clustering + metrics: + - type: v_measure + value: 51.208600000000004 + - type: v_measure_std + value: 4.2761000000000005 + - type: main_score + value: 51.208600000000004 + task: + type: Clustering + - dataset: + config: default + name: MTEB RedditClusteringP2P (default) + revision: 385e3cb46b4cfa89021f56c4380204149d0efe33 + split: test + type: mteb/reddit-clustering-p2p + metrics: + - type: v_measure + value: 60.372899999999994 + - type: v_measure_std + value: 12.0829 + - type: main_score + value: 60.372899999999994 + task: + type: Clustering + - dataset: + config: default + name: MTEB SCIDOCS (default) + revision: f8c2fcf00f625baaa80f62ec5bd9e1fff3b8ae88 + split: test + type: mteb/scidocs + metrics: + - type: ndcg_at_1 + value: 22.400000000000002 + - type: ndcg_at_3 + value: 19.192 + - type: ndcg_at_5 + value: 16.767000000000003 + - type: ndcg_at_10 + value: 20.238999999999997 + - type: ndcg_at_20 + value: 22.720000000000002 + - type: ndcg_at_100 + value: 27.567999999999998 + - type: ndcg_at_1000 + value: 32.535 + - type: map_at_1 + value: 4.552 + - type: map_at_3 + value: 8.495999999999999 + - type: map_at_5 + value: 10.213999999999999 + - type: map_at_10 + value: 11.985 + - type: map_at_20 + value: 12.937000000000001 + - type: map_at_100 + value: 13.885 + - type: map_at_1000 + value: 14.155999999999999 + - type: recall_at_1 + value: 4.552 + - type: recall_at_3 + value: 11.067 + - type: recall_at_5 + value: 15.052 + - type: recall_at_10 + value: 21.422 + - type: recall_at_20 + value: 27.279999999999998 + - type: recall_at_100 + value: 42.968 + - type: recall_at_1000 + value: 67.232 + - type: precision_at_1 + value: 22.400000000000002 + - type: precision_at_3 + value: 18.2 + - type: precision_at_5 + value: 14.860000000000001 + - type: precision_at_10 + value: 10.58 + - type: precision_at_20 + value: 6.715 + - type: precision_at_100 + value: 2.114 + - type: precision_at_1000 + value: 0.331 + - type: mrr_at_1 + value: 22.400000000000002 + - type: mrr_at_3 + value: 31.0833 + - type: mrr_at_5 + value: 32.853300000000004 + - type: mrr_at_10 + value: 34.2814 + - type: mrr_at_20 + value: 34.814 + - type: mrr_at_100 + value: 35.2576 + - type: mrr_at_1000 + value: 35.322199999999995 + - type: nauc_ndcg_at_1_max + value: 23.7575 + - type: nauc_ndcg_at_1_std + value: 4.1697 + - type: nauc_ndcg_at_1_diff1 + value: 28.3995 + - type: nauc_ndcg_at_3_max + value: 27.5517 + - type: nauc_ndcg_at_3_std + value: 8.8005 + - type: nauc_ndcg_at_3_diff1 + value: 22.334799999999998 + - type: nauc_ndcg_at_5_max + value: 28.607599999999998 + - type: nauc_ndcg_at_5_std + value: 10.0785 + - type: nauc_ndcg_at_5_diff1 + value: 21.4713 + - type: nauc_ndcg_at_10_max + value: 30.812099999999997 + - type: nauc_ndcg_at_10_std + value: 14.4374 + - type: nauc_ndcg_at_10_diff1 + value: 20.5304 + - type: nauc_ndcg_at_20_max + value: 32.3888 + - type: nauc_ndcg_at_20_std + value: 17.8152 + - type: nauc_ndcg_at_20_diff1 + value: 20.2815 + - type: nauc_ndcg_at_100_max + value: 34.402100000000004 + - type: nauc_ndcg_at_100_std + value: 22.3694 + - type: nauc_ndcg_at_100_diff1 + value: 20.9422 + - type: nauc_ndcg_at_1000_max + value: 33.7269 + - type: nauc_ndcg_at_1000_std + value: 23.646700000000003 + - type: nauc_ndcg_at_1000_diff1 + value: 19.7226 + - type: nauc_map_at_1_max + value: 23.5069 + - type: nauc_map_at_1_std + value: 3.8736 + - type: nauc_map_at_1_diff1 + value: 28.231 + - type: nauc_map_at_3_max + value: 27.293 + - type: nauc_map_at_3_std + value: 6.9329 + - type: nauc_map_at_3_diff1 + value: 21.8664 + - type: nauc_map_at_5_max + value: 28.591100000000004 + - type: nauc_map_at_5_std + value: 8.2248 + - type: nauc_map_at_5_diff1 + value: 21.4395 + - type: nauc_map_at_10_max + value: 30.417300000000004 + - type: nauc_map_at_10_std + value: 11.615300000000001 + - type: nauc_map_at_10_diff1 + value: 20.624000000000002 + - type: nauc_map_at_20_max + value: 31.479200000000002 + - type: nauc_map_at_20_std + value: 13.808699999999998 + - type: nauc_map_at_20_diff1 + value: 20.413 + - type: nauc_map_at_100_max + value: 32.2613 + - type: nauc_map_at_100_std + value: 15.5692 + - type: nauc_map_at_100_diff1 + value: 20.5465 + - type: nauc_map_at_1000_max + value: 32.2476 + - type: nauc_map_at_1000_std + value: 15.7471 + - type: nauc_map_at_1000_diff1 + value: 20.4622 + - type: nauc_recall_at_1_max + value: 23.5069 + - type: nauc_recall_at_1_std + value: 3.8736 + - type: nauc_recall_at_1_diff1 + value: 28.231 + - type: nauc_recall_at_3_max + value: 27.970299999999998 + - type: nauc_recall_at_3_std + value: 10.2171 + - type: nauc_recall_at_3_diff1 + value: 19.403699999999997 + - type: nauc_recall_at_5_max + value: 28.4521 + - type: nauc_recall_at_5_std + value: 12.2105 + - type: nauc_recall_at_5_diff1 + value: 17.5747 + - type: nauc_recall_at_10_max + value: 30.6955 + - type: nauc_recall_at_10_std + value: 19.096 + - type: nauc_recall_at_10_diff1 + value: 15.3116 + - type: nauc_recall_at_20_max + value: 32.1047 + - type: nauc_recall_at_20_std + value: 24.823600000000003 + - type: nauc_recall_at_20_diff1 + value: 14.257700000000002 + - type: nauc_recall_at_100_max + value: 33.6062 + - type: nauc_recall_at_100_std + value: 33.8641 + - type: nauc_recall_at_100_diff1 + value: 14.5145 + - type: nauc_recall_at_1000_max + value: 26.848300000000002 + - type: nauc_recall_at_1000_std + value: 38.5884 + - type: nauc_recall_at_1000_diff1 + value: 5.6408 + - type: nauc_precision_at_1_max + value: 23.7575 + - type: nauc_precision_at_1_std + value: 4.1697 + - type: nauc_precision_at_1_diff1 + value: 28.3995 + - type: nauc_precision_at_3_max + value: 28.2504 + - type: nauc_precision_at_3_std + value: 10.6227 + - type: nauc_precision_at_3_diff1 + value: 19.5683 + - type: nauc_precision_at_5_max + value: 28.8134 + - type: nauc_precision_at_5_std + value: 12.518899999999999 + - type: nauc_precision_at_5_diff1 + value: 17.8036 + - type: nauc_precision_at_10_max + value: 30.9813 + - type: nauc_precision_at_10_std + value: 19.3506 + - type: nauc_precision_at_10_diff1 + value: 15.512 + - type: nauc_precision_at_20_max + value: 32.6743 + - type: nauc_precision_at_20_std + value: 24.9974 + - type: nauc_precision_at_20_diff1 + value: 14.794099999999998 + - type: nauc_precision_at_100_max + value: 34.413700000000006 + - type: nauc_precision_at_100_std + value: 34.0889 + - type: nauc_precision_at_100_diff1 + value: 15.252699999999999 + - type: nauc_precision_at_1000_max + value: 27.3954 + - type: nauc_precision_at_1000_std + value: 37.8895 + - type: nauc_precision_at_1000_diff1 + value: 6.587999999999999 + - type: nauc_mrr_at_1_max + value: 23.7575 + - type: nauc_mrr_at_1_std + value: 4.1697 + - type: nauc_mrr_at_1_diff1 + value: 28.3995 + - type: nauc_mrr_at_3_max + value: 26.8324 + - type: nauc_mrr_at_3_std + value: 8.646700000000001 + - type: nauc_mrr_at_3_diff1 + value: 25.5754 + - type: nauc_mrr_at_5_max + value: 26.8274 + - type: nauc_mrr_at_5_std + value: 8.911 + - type: nauc_mrr_at_5_diff1 + value: 25.106 + - type: nauc_mrr_at_10_max + value: 27.073399999999996 + - type: nauc_mrr_at_10_std + value: 9.7624 + - type: nauc_mrr_at_10_diff1 + value: 24.9405 + - type: nauc_mrr_at_20_max + value: 27.1229 + - type: nauc_mrr_at_20_std + value: 10.0676 + - type: nauc_mrr_at_20_diff1 + value: 24.8122 + - type: nauc_mrr_at_100_max + value: 27.1391 + - type: nauc_mrr_at_100_std + value: 9.9628 + - type: nauc_mrr_at_100_diff1 + value: 24.9507 + - type: nauc_mrr_at_1000_max + value: 27.114 + - type: nauc_mrr_at_1000_std + value: 9.9537 + - type: nauc_mrr_at_1000_diff1 + value: 24.9421 + - type: main_score + value: 20.238999999999997 + task: + type: Retrieval + - dataset: + config: default + name: MTEB SICK-R (default) + revision: 20a6d6f312dd54037fe07a32d58e5e168867909d + split: test + type: mteb/sickr-sts + metrics: + - type: pearson + value: 79.5908 + - type: spearman + value: 73.9888 + - type: cosine_pearson + value: 79.5908 + - type: cosine_spearman + value: 73.9888 + - type: manhattan_pearson + value: 77.0623 + - type: manhattan_spearman + value: 73.7724 + - type: euclidean_pearson + value: 77.30890000000001 + - type: euclidean_spearman + value: 73.9888 + - type: main_score + value: 73.9888 + task: + type: STS + - dataset: + config: default + name: MTEB STS12 (default) + revision: a0d554a64d88156834ff5ae9920b964011b16384 + split: test + type: mteb/sts12-sts + metrics: + - type: pearson + value: 74.0752 + - type: spearman + value: 71.22699999999999 + - type: cosine_pearson + value: 74.0752 + - type: cosine_spearman + value: 71.22699999999999 + - type: manhattan_pearson + value: 70.6037 + - type: manhattan_spearman + value: 70.9916 + - type: euclidean_pearson + value: 70.922 + - type: euclidean_spearman + value: 71.22699999999999 + - type: main_score + value: 71.22699999999999 + task: + type: STS + - dataset: + config: default + name: MTEB STS13 (default) + revision: 7e90230a92c190f1bf69ae9002b8cea547a64cca + split: test + type: mteb/sts13-sts + metrics: + - type: pearson + value: 77.8946 + - type: spearman + value: 80.4405 + - type: cosine_pearson + value: 77.8946 + - type: cosine_spearman + value: 80.4405 + - type: manhattan_pearson + value: 79.6856 + - type: manhattan_spearman + value: 80.1236 + - type: euclidean_pearson + value: 80.0315 + - type: euclidean_spearman + value: 80.44059999999999 + - type: main_score + value: 80.4405 + task: + type: STS + - dataset: + config: default + name: MTEB STS14 (default) + revision: 6031580fec1f6af667f0bd2da0a551cf4f0b2375 + split: test + type: mteb/sts14-sts + metrics: + - type: pearson + value: 76.2196 + - type: spearman + value: 75.10419999999999 + - type: cosine_pearson + value: 76.2196 + - type: cosine_spearman + value: 75.10419999999999 + - type: manhattan_pearson + value: 75.4647 + - type: manhattan_spearman + value: 74.81179999999999 + - type: euclidean_pearson + value: 75.8091 + - type: euclidean_spearman + value: 75.10419999999999 + - type: main_score + value: 75.10419999999999 + task: + type: STS + - dataset: + config: default + name: MTEB STS15 (default) + revision: ae752c7c21bf194d8b67fd573edf7ae58183cbe3 + split: test + type: mteb/sts15-sts + metrics: + - type: pearson + value: 81.2455 + - type: spearman + value: 82.8681 + - type: cosine_pearson + value: 81.2455 + - type: cosine_spearman + value: 82.8681 + - type: manhattan_pearson + value: 82.4327 + - type: manhattan_spearman + value: 82.7513 + - type: euclidean_pearson + value: 82.5635 + - type: euclidean_spearman + value: 82.8681 + - type: main_score + value: 82.8681 + task: + type: STS + - dataset: + config: default + name: MTEB STS16 (default) + revision: 4d8694f8f0e0100860b497b999b3dbed754a0513 + split: test + type: mteb/sts16-sts + metrics: + - type: pearson + value: 81.6322 + - type: spearman + value: 83.487 + - type: cosine_pearson + value: 81.6322 + - type: cosine_spearman + value: 83.487 + - type: manhattan_pearson + value: 83.0048 + - type: manhattan_spearman + value: 83.4064 + - type: euclidean_pearson + value: 83.0938 + - type: euclidean_spearman + value: 83.487 + - type: main_score + value: 83.487 + task: + type: STS + - dataset: + config: en-en + name: MTEB STS17 (en-en) + revision: faeb762787bd10488a50c8b5be4a3b82e411949c + split: test + type: mteb/sts17-crosslingual-sts + metrics: + - type: pearson + value: 81.1124 + - type: spearman + value: 84.5436 + - type: cosine_pearson + value: 81.1124 + - type: cosine_spearman + value: 84.5436 + - type: manhattan_pearson + value: 83.5158 + - type: manhattan_spearman + value: 84.596 + - type: euclidean_pearson + value: 83.4429 + - type: euclidean_spearman + value: 84.5436 + - type: main_score + value: 84.5436 + task: + type: STS + - dataset: + config: en-tr + name: MTEB STS17 (en-tr) + revision: faeb762787bd10488a50c8b5be4a3b82e411949c + split: test + type: mteb/sts17-crosslingual-sts + metrics: + - type: pearson + value: 62.0001 + - type: spearman + value: 63.631099999999996 + - type: cosine_pearson + value: 62.0001 + - type: cosine_spearman + value: 63.631099999999996 + - type: manhattan_pearson + value: 62.239599999999996 + - type: manhattan_spearman + value: 62.892199999999995 + - type: euclidean_pearson + value: 62.9809 + - type: euclidean_spearman + value: 63.631099999999996 + - type: main_score + value: 63.631099999999996 + task: + type: STS + - dataset: + config: it-en + name: MTEB STS17 (it-en) + revision: faeb762787bd10488a50c8b5be4a3b82e411949c + split: test + type: mteb/sts17-crosslingual-sts + metrics: + - type: pearson + value: 75.1556 + - type: spearman + value: 76.8807 + - type: cosine_pearson + value: 75.1556 + - type: cosine_spearman + value: 76.8807 + - type: manhattan_pearson + value: 76.2428 + - type: manhattan_spearman + value: 76.8101 + - type: euclidean_pearson + value: 76.107 + - type: euclidean_spearman + value: 76.8807 + - type: main_score + value: 76.8807 + task: + type: STS + - dataset: + config: es-en + name: MTEB STS17 (es-en) + revision: faeb762787bd10488a50c8b5be4a3b82e411949c + split: test + type: mteb/sts17-crosslingual-sts + metrics: + - type: pearson + value: 69.85719999999999 + - type: spearman + value: 71.0489 + - type: cosine_pearson + value: 69.85719999999999 + - type: cosine_spearman + value: 71.0489 + - type: manhattan_pearson + value: 71.08449999999999 + - type: manhattan_spearman + value: 71.0051 + - type: euclidean_pearson + value: 71.19760000000001 + - type: euclidean_spearman + value: 71.0489 + - type: main_score + value: 71.0489 + task: + type: STS + - dataset: + config: nl-en + name: MTEB STS17 (nl-en) + revision: faeb762787bd10488a50c8b5be4a3b82e411949c + split: test + type: mteb/sts17-crosslingual-sts + metrics: + - type: pearson + value: 76.1131 + - type: spearman + value: 78.2714 + - type: cosine_pearson + value: 76.1131 + - type: cosine_spearman + value: 78.2714 + - type: manhattan_pearson + value: 76.70270000000001 + - type: manhattan_spearman + value: 77.7803 + - type: euclidean_pearson + value: 77.14269999999999 + - type: euclidean_spearman + value: 78.2714 + - type: main_score + value: 78.2714 + task: + type: STS + - dataset: + config: fr-en + name: MTEB STS17 (fr-en) + revision: faeb762787bd10488a50c8b5be4a3b82e411949c + split: test + type: mteb/sts17-crosslingual-sts + metrics: + - type: pearson + value: 74.49719999999999 + - type: spearman + value: 76.2747 + - type: cosine_pearson + value: 74.49719999999999 + - type: cosine_spearman + value: 76.2747 + - type: manhattan_pearson + value: 75.071 + - type: manhattan_spearman + value: 75.8969 + - type: euclidean_pearson + value: 75.289 + - type: euclidean_spearman + value: 76.2747 + - type: main_score + value: 76.2747 + task: + type: STS + - dataset: + config: en-de + name: MTEB STS17 (en-de) + revision: faeb762787bd10488a50c8b5be4a3b82e411949c + split: test + type: mteb/sts17-crosslingual-sts + metrics: + - type: pearson + value: 76.7073 + - type: spearman + value: 79.3107 + - type: cosine_pearson + value: 76.7073 + - type: cosine_spearman + value: 79.3107 + - type: manhattan_pearson + value: 77.9578 + - type: manhattan_spearman + value: 79.3195 + - type: euclidean_pearson + value: 77.7386 + - type: euclidean_spearman + value: 79.3107 + - type: main_score + value: 79.3107 + task: + type: STS + - dataset: + config: en-ar + name: MTEB STS17 (en-ar) + revision: faeb762787bd10488a50c8b5be4a3b82e411949c + split: test + type: mteb/sts17-crosslingual-sts + metrics: + - type: pearson + value: 60.5826 + - type: spearman + value: 61.0502 + - type: cosine_pearson + value: 60.5826 + - type: cosine_spearman + value: 61.0502 + - type: manhattan_pearson + value: 61.202 + - type: manhattan_spearman + value: 61.2039 + - type: euclidean_pearson + value: 61.1915 + - type: euclidean_spearman + value: 61.0502 + - type: main_score + value: 61.0502 + task: + type: STS + - dataset: + config: en + name: MTEB STS22 (en) + revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3 + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: pearson + value: 69.2521 + - type: spearman + value: 68.06219999999999 + - type: cosine_pearson + value: 69.2521 + - type: cosine_spearman + value: 68.06219999999999 + - type: manhattan_pearson + value: 70.5115 + - type: manhattan_spearman + value: 67.8705 + - type: euclidean_pearson + value: 70.68480000000001 + - type: euclidean_spearman + value: 68.06219999999999 + - type: main_score + value: 68.06219999999999 + task: + type: STS + - dataset: + config: pl-en + name: MTEB STS22 (pl-en) + revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3 + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: pearson + value: 77.97500000000001 + - type: spearman + value: 76.848 + - type: cosine_pearson + value: 77.97500000000001 + - type: cosine_spearman + value: 76.848 + - type: manhattan_pearson + value: 76.4098 + - type: manhattan_spearman + value: 76.6188 + - type: euclidean_pearson + value: 77.17500000000001 + - type: euclidean_spearman + value: 76.848 + - type: main_score + value: 76.848 + task: + type: STS + - dataset: + config: zh-en + name: MTEB STS22 (zh-en) + revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3 + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: pearson + value: 71.3604 + - type: spearman + value: 70.7891 + - type: cosine_pearson + value: 71.3604 + - type: cosine_spearman + value: 70.7891 + - type: manhattan_pearson + value: 73.0185 + - type: manhattan_spearman + value: 70.79299999999999 + - type: euclidean_pearson + value: 73.17620000000001 + - type: euclidean_spearman + value: 70.7891 + - type: main_score + value: 70.7891 + task: + type: STS + - dataset: + config: es-en + name: MTEB STS22 (es-en) + revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3 + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: pearson + value: 77.58239999999999 + - type: spearman + value: 78.5907 + - type: cosine_pearson + value: 77.58239999999999 + - type: cosine_spearman + value: 78.5907 + - type: manhattan_pearson + value: 79.25720000000001 + - type: manhattan_spearman + value: 78.6249 + - type: euclidean_pearson + value: 79.3724 + - type: euclidean_spearman + value: 78.5907 + - type: main_score + value: 78.5907 + task: + type: STS + - dataset: + config: de-en + name: MTEB STS22 (de-en) + revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3 + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: pearson + value: 63.324000000000005 + - type: spearman + value: 55.1099 + - type: cosine_pearson + value: 63.324000000000005 + - type: cosine_spearman + value: 55.1099 + - type: manhattan_pearson + value: 67.3128 + - type: manhattan_spearman + value: 56.340199999999996 + - type: euclidean_pearson + value: 67.12089999999999 + - type: euclidean_spearman + value: 55.1099 + - type: main_score + value: 55.1099 + task: + type: STS + - dataset: + config: default + name: MTEB STSBenchmark (default) + revision: b0fddb56ed78048fa8b90373c8a3cfc37b684831 + split: test + type: mteb/stsbenchmark-sts + metrics: + - type: pearson + value: 78.02329999999999 + - type: spearman + value: 79.1887 + - type: cosine_pearson + value: 78.02329999999999 + - type: cosine_spearman + value: 79.1887 + - type: manhattan_pearson + value: 78.8951 + - type: manhattan_spearman + value: 78.9444 + - type: euclidean_pearson + value: 79.1499 + - type: euclidean_spearman + value: 79.1888 + - type: main_score + value: 79.1887 + task: + type: STS + - dataset: + config: default + name: MTEB SciDocsRR (default) + revision: d3c5e1fc0b855ab6097bf1cda04dd73947d7caab + split: test + type: mteb/scidocs-reranking + metrics: + - type: map + value: 78.7501 + - type: mrr + value: 93.9748 + - type: nAUC_map_max + value: 54.495599999999996 + - type: nAUC_map_std + value: 70.0377 + - type: nAUC_map_diff1 + value: 6.0146999999999995 + - type: nAUC_mrr_max + value: 81.1486 + - type: nAUC_mrr_std + value: 78.3478 + - type: nAUC_mrr_diff1 + value: 50.7613 + - type: main_score + value: 78.7501 + task: + type: Reranking + - dataset: + config: default + name: MTEB SciFact (default) + revision: 0228b52cf27578f30900b9e5271d331663a030d7 + split: test + type: mteb/scifact + metrics: + - type: ndcg_at_1 + value: 58.667 + - type: ndcg_at_3 + value: 66.022 + - type: ndcg_at_5 + value: 68.508 + - type: ndcg_at_10 + value: 70.586 + - type: ndcg_at_20 + value: 71.714 + - type: ndcg_at_100 + value: 72.81 + - type: ndcg_at_1000 + value: 73.482 + - type: map_at_1 + value: 55.594 + - type: map_at_3 + value: 63.2 + - type: map_at_5 + value: 64.996 + - type: map_at_10 + value: 65.988 + - type: map_at_20 + value: 66.347 + - type: map_at_100 + value: 66.526 + - type: map_at_1000 + value: 66.547 + - type: recall_at_1 + value: 55.594 + - type: recall_at_3 + value: 71.22800000000001 + - type: recall_at_5 + value: 77.078 + - type: recall_at_10 + value: 83.172 + - type: recall_at_20 + value: 87.422 + - type: recall_at_100 + value: 93.167 + - type: recall_at_1000 + value: 98.667 + - type: precision_at_1 + value: 58.667 + - type: precision_at_3 + value: 25.778000000000002 + - type: precision_at_5 + value: 17.333000000000002 + - type: precision_at_10 + value: 9.433 + - type: precision_at_20 + value: 4.967 + - type: precision_at_100 + value: 1.06 + - type: precision_at_1000 + value: 0.11199999999999999 + - type: mrr_at_1 + value: 58.666700000000006 + - type: mrr_at_3 + value: 65.3889 + - type: mrr_at_5 + value: 66.62219999999999 + - type: mrr_at_10 + value: 67.3364 + - type: mrr_at_20 + value: 67.6046 + - type: mrr_at_100 + value: 67.73320000000001 + - type: mrr_at_1000 + value: 67.7526 + - type: nauc_ndcg_at_1_max + value: 60.2511 + - type: nauc_ndcg_at_1_std + value: 12.422 + - type: nauc_ndcg_at_1_diff1 + value: 74.4289 + - type: nauc_ndcg_at_3_max + value: 60.2109 + - type: nauc_ndcg_at_3_std + value: 11.0152 + - type: nauc_ndcg_at_3_diff1 + value: 71.0436 + - type: nauc_ndcg_at_5_max + value: 62.690999999999995 + - type: nauc_ndcg_at_5_std + value: 13.585700000000001 + - type: nauc_ndcg_at_5_diff1 + value: 70.4007 + - type: nauc_ndcg_at_10_max + value: 62.740899999999996 + - type: nauc_ndcg_at_10_std + value: 13.980400000000001 + - type: nauc_ndcg_at_10_diff1 + value: 70.0506 + - type: nauc_ndcg_at_20_max + value: 62.271699999999996 + - type: nauc_ndcg_at_20_std + value: 15.9756 + - type: nauc_ndcg_at_20_diff1 + value: 70.3237 + - type: nauc_ndcg_at_100_max + value: 62.125 + - type: nauc_ndcg_at_100_std + value: 15.5809 + - type: nauc_ndcg_at_100_diff1 + value: 70.4151 + - type: nauc_ndcg_at_1000_max + value: 61.9259 + - type: nauc_ndcg_at_1000_std + value: 15.3462 + - type: nauc_ndcg_at_1000_diff1 + value: 70.7346 + - type: nauc_map_at_1_max + value: 53.6767 + - type: nauc_map_at_1_std + value: 3.7751 + - type: nauc_map_at_1_diff1 + value: 74.60329999999999 + - type: nauc_map_at_3_max + value: 57.0403 + - type: nauc_map_at_3_std + value: 8.2272 + - type: nauc_map_at_3_diff1 + value: 71.7906 + - type: nauc_map_at_5_max + value: 59.6713 + - type: nauc_map_at_5_std + value: 10.8346 + - type: nauc_map_at_5_diff1 + value: 71.3356 + - type: nauc_map_at_10_max + value: 60.0086 + - type: nauc_map_at_10_std + value: 11.4394 + - type: nauc_map_at_10_diff1 + value: 71.14869999999999 + - type: nauc_map_at_20_max + value: 59.940599999999996 + - type: nauc_map_at_20_std + value: 12.0728 + - type: nauc_map_at_20_diff1 + value: 71.31 + - type: nauc_map_at_100_max + value: 59.95589999999999 + - type: nauc_map_at_100_std + value: 12.148299999999999 + - type: nauc_map_at_100_diff1 + value: 71.2142 + - type: nauc_map_at_1000_max + value: 59.9486 + - type: nauc_map_at_1000_std + value: 12.139 + - type: nauc_map_at_1000_diff1 + value: 71.2225 + - type: nauc_recall_at_1_max + value: 53.6767 + - type: nauc_recall_at_1_std + value: 3.7751 + - type: nauc_recall_at_1_diff1 + value: 74.60329999999999 + - type: nauc_recall_at_3_max + value: 60.4078 + - type: nauc_recall_at_3_std + value: 9.038300000000001 + - type: nauc_recall_at_3_diff1 + value: 67.60119999999999 + - type: nauc_recall_at_5_max + value: 68.0179 + - type: nauc_recall_at_5_std + value: 16.061600000000002 + - type: nauc_recall_at_5_diff1 + value: 65.54759999999999 + - type: nauc_recall_at_10_max + value: 68.7372 + - type: nauc_recall_at_10_std + value: 16.8637 + - type: nauc_recall_at_10_diff1 + value: 62.7613 + - type: nauc_recall_at_20_max + value: 67.1403 + - type: nauc_recall_at_20_std + value: 31.3919 + - type: nauc_recall_at_20_diff1 + value: 62.66929999999999 + - type: nauc_recall_at_100_max + value: 68.6366 + - type: nauc_recall_at_100_std + value: 32.4577 + - type: nauc_recall_at_100_diff1 + value: 64.52029999999999 + - type: nauc_recall_at_1000_max + value: 70.7166 + - type: nauc_recall_at_1000_std + value: 70.47149999999999 + - type: nauc_recall_at_1000_diff1 + value: 85.58590000000001 + - type: nauc_precision_at_1_max + value: 60.2511 + - type: nauc_precision_at_1_std + value: 12.422 + - type: nauc_precision_at_1_diff1 + value: 74.4289 + - type: nauc_precision_at_3_max + value: 58.75280000000001 + - type: nauc_precision_at_3_std + value: 27.605400000000003 + - type: nauc_precision_at_3_diff1 + value: 49.1523 + - type: nauc_precision_at_5_max + value: 56.4694 + - type: nauc_precision_at_5_std + value: 39.080799999999996 + - type: nauc_precision_at_5_diff1 + value: 28.8162 + - type: nauc_precision_at_10_max + value: 48.8894 + - type: nauc_precision_at_10_std + value: 43.8149 + - type: nauc_precision_at_10_diff1 + value: 15.0093 + - type: nauc_precision_at_20_max + value: 41.4059 + - type: nauc_precision_at_20_std + value: 50.7143 + - type: nauc_precision_at_20_diff1 + value: 8.3552 + - type: nauc_precision_at_100_max + value: 33.5064 + - type: nauc_precision_at_100_std + value: 52.8775 + - type: nauc_precision_at_100_diff1 + value: -5.0870999999999995 + - type: nauc_precision_at_1000_max + value: 23.9064 + - type: nauc_precision_at_1000_std + value: 57.784800000000004 + - type: nauc_precision_at_1000_diff1 + value: -20.1246 + - type: nauc_mrr_at_1_max + value: 60.2511 + - type: nauc_mrr_at_1_std + value: 12.422 + - type: nauc_mrr_at_1_diff1 + value: 74.4289 + - type: nauc_mrr_at_3_max + value: 62.663199999999996 + - type: nauc_mrr_at_3_std + value: 14.7348 + - type: nauc_mrr_at_3_diff1 + value: 72.1185 + - type: nauc_mrr_at_5_max + value: 63.3871 + - type: nauc_mrr_at_5_std + value: 15.773000000000001 + - type: nauc_mrr_at_5_diff1 + value: 71.6722 + - type: nauc_mrr_at_10_max + value: 62.8474 + - type: nauc_mrr_at_10_std + value: 15.1896 + - type: nauc_mrr_at_10_diff1 + value: 71.64110000000001 + - type: nauc_mrr_at_20_max + value: 62.699400000000004 + - type: nauc_mrr_at_20_std + value: 15.554499999999999 + - type: nauc_mrr_at_20_diff1 + value: 71.6049 + - type: nauc_mrr_at_100_max + value: 62.6665 + - type: nauc_mrr_at_100_std + value: 15.4586 + - type: nauc_mrr_at_100_diff1 + value: 71.6217 + - type: nauc_mrr_at_1000_max + value: 62.6641 + - type: nauc_mrr_at_1000_std + value: 15.4535 + - type: nauc_mrr_at_1000_diff1 + value: 71.6307 + - type: main_score + value: 70.586 + task: + type: Retrieval + - dataset: + config: default + name: MTEB SprintDuplicateQuestions (default) + revision: d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46 + split: test + type: mteb/sprintduplicatequestions-pairclassification + metrics: + - type: similarity_accuracy + value: 99.8416 + - type: similarity_accuracy_threshold + value: 74.52069999999999 + - type: similarity_f1 + value: 92.008 + - type: similarity_f1_threshold + value: 74.4529 + - type: similarity_precision + value: 91.9162 + - type: similarity_recall + value: 92.10000000000001 + - type: similarity_ap + value: 96.54209999999999 + - type: cosine_accuracy + value: 99.8416 + - type: cosine_accuracy_threshold + value: 74.52069999999999 + - type: cosine_f1 + value: 92.008 + - type: cosine_f1_threshold + value: 74.4529 + - type: cosine_precision + value: 91.9162 + - type: cosine_recall + value: 92.10000000000001 + - type: cosine_ap + value: 96.54209999999999 + - type: manhattan_accuracy + value: 99.8446 + - type: manhattan_accuracy_threshold + value: 1784.866 + - type: manhattan_f1 + value: 92.1539 + - type: manhattan_f1_threshold + value: 1787.6774 + - type: manhattan_precision + value: 92.1079 + - type: manhattan_recall + value: 92.2 + - type: manhattan_ap + value: 96.5207 + - type: euclidean_accuracy + value: 99.8416 + - type: euclidean_accuracy_threshold + value: 71.3853 + - type: euclidean_f1 + value: 92.008 + - type: euclidean_f1_threshold + value: 71.4803 + - type: euclidean_precision + value: 91.9162 + - type: euclidean_recall + value: 92.10000000000001 + - type: euclidean_ap + value: 96.54209999999999 + - type: dot_accuracy + value: 99.8416 + - type: dot_accuracy_threshold + value: 74.52069999999999 + - type: dot_f1 + value: 92.008 + - type: dot_f1_threshold + value: 74.4528 + - type: dot_precision + value: 91.9162 + - type: dot_recall + value: 92.10000000000001 + - type: dot_ap + value: 96.54209999999999 + - type: max_accuracy + value: 99.8446 + - type: max_f1 + value: 92.1539 + - type: max_precision + value: 92.1079 + - type: max_recall + value: 92.2 + - type: max_ap + value: 96.54209999999999 + - type: main_score + value: 96.54209999999999 + task: + type: PairClassification + - dataset: + config: default + name: MTEB StackExchangeClustering (default) + revision: 6cbc1f7b2bc0622f2e39d2c77fa502909748c259 + split: test + type: mteb/stackexchange-clustering + metrics: + - type: v_measure + value: 63.4035 + - type: v_measure_std + value: 4.758 + - type: main_score + value: 63.4035 + task: + type: Clustering + - dataset: + config: default + name: MTEB StackExchangeClusteringP2P (default) + revision: 815ca46b2622cec33ccafc3735d572c266efdb44 + split: test + type: mteb/stackexchange-clustering-p2p + metrics: + - type: v_measure + value: 36.288599999999995 + - type: v_measure_std + value: 1.3107 + - type: main_score + value: 36.288599999999995 + task: + type: Clustering + - dataset: + config: default + name: MTEB StackOverflowDupQuestions (default) + revision: e185fbe320c72810689fc5848eb6114e1ef5ec69 + split: test + type: mteb/stackoverflowdupquestions-reranking + metrics: + - type: map + value: 51.457699999999996 + - type: mrr + value: 52.374500000000005 + - type: nAUC_map_max + value: 12.912399999999998 + - type: nAUC_map_std + value: 6.4524 + - type: nAUC_map_diff1 + value: 37.2785 + - type: nAUC_mrr_max + value: 13.333999999999998 + - type: nAUC_mrr_std + value: 7.0440000000000005 + - type: nAUC_mrr_diff1 + value: 37.2993 + - type: main_score + value: 51.457699999999996 + task: + type: Reranking + - dataset: + config: default + name: MTEB SummEval (default) + revision: cda12ad7615edc362dbf25a00fdd61d3b1eaf93c + split: test + type: mteb/summeval + metrics: + - type: pearson + value: 29.7101 + - type: spearman + value: 30.514200000000002 + - type: cosine_spearman + value: 30.514200000000002 + - type: cosine_pearson + value: 29.7101 + - type: dot_spearman + value: 30.514200000000002 + - type: dot_pearson + value: 29.7101 + - type: main_score + value: 30.514200000000002 + task: + type: Summarization + - dataset: + config: default + name: MTEB TRECCOVID (default) + revision: bb9466bac8153a0349341eb1b22e06409e78ef4e + split: test + type: mteb/trec-covid + metrics: + - type: ndcg_at_1 + value: 86.0 + - type: ndcg_at_3 + value: 86.542 + - type: ndcg_at_5 + value: 85.297 + - type: ndcg_at_10 + value: 83.866 + - type: ndcg_at_20 + value: 80.553 + - type: ndcg_at_100 + value: 65.091 + - type: ndcg_at_1000 + value: 57.86900000000001 + - type: map_at_1 + value: 0.23500000000000001 + - type: map_at_3 + value: 0.7100000000000001 + - type: map_at_5 + value: 1.1440000000000001 + - type: map_at_10 + value: 2.185 + - type: map_at_20 + value: 4.004 + - type: map_at_100 + value: 13.25 + - type: map_at_1000 + value: 32.668 + - type: recall_at_1 + value: 0.23500000000000001 + - type: recall_at_3 + value: 0.736 + - type: recall_at_5 + value: 1.191 + - type: recall_at_10 + value: 2.323 + - type: recall_at_20 + value: 4.390000000000001 + - type: recall_at_100 + value: 15.962000000000002 + - type: recall_at_1000 + value: 54.290000000000006 + - type: precision_at_1 + value: 90.0 + - type: precision_at_3 + value: 92.0 + - type: precision_at_5 + value: 90.0 + - type: precision_at_10 + value: 88.6 + - type: precision_at_20 + value: 85.5 + - type: precision_at_100 + value: 67.14 + - type: precision_at_1000 + value: 25.81 + - type: mrr_at_1 + value: 90.0 + - type: mrr_at_3 + value: 94.6667 + - type: mrr_at_5 + value: 94.6667 + - type: mrr_at_10 + value: 94.6667 + - type: mrr_at_20 + value: 94.6667 + - type: mrr_at_100 + value: 94.6667 + - type: mrr_at_1000 + value: 94.6667 + - type: nauc_ndcg_at_1_max + value: -0.0208 + - type: nauc_ndcg_at_1_std + value: 9.228200000000001 + - type: nauc_ndcg_at_1_diff1 + value: -7.4962 + - type: nauc_ndcg_at_3_max + value: 16.5755 + - type: nauc_ndcg_at_3_std + value: 39.0511 + - type: nauc_ndcg_at_3_diff1 + value: -14.5975 + - type: nauc_ndcg_at_5_max + value: 15.326799999999999 + - type: nauc_ndcg_at_5_std + value: 44.2523 + - type: nauc_ndcg_at_5_diff1 + value: -15.004600000000002 + - type: nauc_ndcg_at_10_max + value: 34.5609 + - type: nauc_ndcg_at_10_std + value: 62.8752 + - type: nauc_ndcg_at_10_diff1 + value: -22.9907 + - type: nauc_ndcg_at_20_max + value: 35.7633 + - type: nauc_ndcg_at_20_std + value: 74.1826 + - type: nauc_ndcg_at_20_diff1 + value: -26.3264 + - type: nauc_ndcg_at_100_max + value: 36.939499999999995 + - type: nauc_ndcg_at_100_std + value: 80.702 + - type: nauc_ndcg_at_100_diff1 + value: -41.7784 + - type: nauc_ndcg_at_1000_max + value: 41.3313 + - type: nauc_ndcg_at_1000_std + value: 68.0671 + - type: nauc_ndcg_at_1000_diff1 + value: -14.6009 + - type: nauc_map_at_1_max + value: -15.2873 + - type: nauc_map_at_1_std + value: -24.4781 + - type: nauc_map_at_1_diff1 + value: 35.4803 + - type: nauc_map_at_3_max + value: -14.107700000000001 + - type: nauc_map_at_3_std + value: -23.197699999999998 + - type: nauc_map_at_3_diff1 + value: 37.8596 + - type: nauc_map_at_5_max + value: -12.7588 + - type: nauc_map_at_5_std + value: -20.174400000000002 + - type: nauc_map_at_5_diff1 + value: 39.575700000000005 + - type: nauc_map_at_10_max + value: -4.8804 + - type: nauc_map_at_10_std + value: -11.0753 + - type: nauc_map_at_10_diff1 + value: 38.2457 + - type: nauc_map_at_20_max + value: 0.7396 + - type: nauc_map_at_20_std + value: 0.3599 + - type: nauc_map_at_20_diff1 + value: 35.4735 + - type: nauc_map_at_100_max + value: 20.011000000000003 + - type: nauc_map_at_100_std + value: 45.2654 + - type: nauc_map_at_100_diff1 + value: 3.6394 + - type: nauc_map_at_1000_max + value: 43.317099999999996 + - type: nauc_map_at_1000_std + value: 74.6629 + - type: nauc_map_at_1000_diff1 + value: -22.509 + - type: nauc_recall_at_1_max + value: -15.2873 + - type: nauc_recall_at_1_std + value: -24.4781 + - type: nauc_recall_at_1_diff1 + value: 35.4803 + - type: nauc_recall_at_3_max + value: -14.1509 + - type: nauc_recall_at_3_std + value: -24.7684 + - type: nauc_recall_at_3_diff1 + value: 40.6736 + - type: nauc_recall_at_5_max + value: -13.053899999999999 + - type: nauc_recall_at_5_std + value: -21.7134 + - type: nauc_recall_at_5_diff1 + value: 42.4446 + - type: nauc_recall_at_10_max + value: -7.3492 + - type: nauc_recall_at_10_std + value: -15.7989 + - type: nauc_recall_at_10_diff1 + value: 41.6543 + - type: nauc_recall_at_20_max + value: -4.8004 + - type: nauc_recall_at_20_std + value: -9.6834 + - type: nauc_recall_at_20_diff1 + value: 41.7323 + - type: nauc_recall_at_100_max + value: 11.3356 + - type: nauc_recall_at_100_std + value: 28.1118 + - type: nauc_recall_at_100_diff1 + value: 15.6166 + - type: nauc_recall_at_1000_max + value: 39.9341 + - type: nauc_recall_at_1000_std + value: 54.15410000000001 + - type: nauc_recall_at_1000_diff1 + value: -2.0016 + - type: nauc_precision_at_1_max + value: 12.2035 + - type: nauc_precision_at_1_std + value: 24.1923 + - type: nauc_precision_at_1_diff1 + value: -25.368800000000004 + - type: nauc_precision_at_3_max + value: 31.019600000000004 + - type: nauc_precision_at_3_std + value: 56.08539999999999 + - type: nauc_precision_at_3_diff1 + value: -33.821600000000004 + - type: nauc_precision_at_5_max + value: 26.127699999999997 + - type: nauc_precision_at_5_std + value: 52.8458 + - type: nauc_precision_at_5_diff1 + value: -22.24 + - type: nauc_precision_at_10_max + value: 45.8122 + - type: nauc_precision_at_10_std + value: 71.9086 + - type: nauc_precision_at_10_diff1 + value: -28.500700000000002 + - type: nauc_precision_at_20_max + value: 44.2567 + - type: nauc_precision_at_20_std + value: 80.86410000000001 + - type: nauc_precision_at_20_diff1 + value: -28.518 + - type: nauc_precision_at_100_max + value: 42.8044 + - type: nauc_precision_at_100_std + value: 84.13669999999999 + - type: nauc_precision_at_100_diff1 + value: -47.1098 + - type: nauc_precision_at_1000_max + value: 40.260200000000005 + - type: nauc_precision_at_1000_std + value: 53.53059999999999 + - type: nauc_precision_at_1000_diff1 + value: -41.2652 + - type: nauc_mrr_at_1_max + value: 12.2035 + - type: nauc_mrr_at_1_std + value: 24.1923 + - type: nauc_mrr_at_1_diff1 + value: -25.368800000000004 + - type: nauc_mrr_at_3_max + value: 16.8738 + - type: nauc_mrr_at_3_std + value: 28.113300000000002 + - type: nauc_mrr_at_3_diff1 + value: -20.3198 + - type: nauc_mrr_at_5_max + value: 16.8738 + - type: nauc_mrr_at_5_std + value: 28.113300000000002 + - type: nauc_mrr_at_5_diff1 + value: -20.3198 + - type: nauc_mrr_at_10_max + value: 16.8738 + - type: nauc_mrr_at_10_std + value: 28.113300000000002 + - type: nauc_mrr_at_10_diff1 + value: -20.3198 + - type: nauc_mrr_at_20_max + value: 16.8738 + - type: nauc_mrr_at_20_std + value: 28.113300000000002 + - type: nauc_mrr_at_20_diff1 + value: -20.3198 + - type: nauc_mrr_at_100_max + value: 16.8738 + - type: nauc_mrr_at_100_std + value: 28.113300000000002 + - type: nauc_mrr_at_100_diff1 + value: -20.3198 + - type: nauc_mrr_at_1000_max + value: 16.8738 + - type: nauc_mrr_at_1000_std + value: 28.113300000000002 + - type: nauc_mrr_at_1000_diff1 + value: -20.3198 + - type: main_score + value: 83.866 + task: + type: Retrieval + - dataset: + config: default + name: MTEB Touche2020 (default) + revision: a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f + split: test + type: mteb/touche2020 + metrics: + - type: ndcg_at_1 + value: 38.775999999999996 + - type: ndcg_at_3 + value: 33.664 + - type: ndcg_at_5 + value: 31.61 + - type: ndcg_at_10 + value: 29.499 + - type: ndcg_at_20 + value: 29.772 + - type: ndcg_at_100 + value: 39.845000000000006 + - type: ndcg_at_1000 + value: 51.141999999999996 + - type: map_at_1 + value: 3.004 + - type: map_at_3 + value: 6.027 + - type: map_at_5 + value: 7.993 + - type: map_at_10 + value: 11.546 + - type: map_at_20 + value: 14.185 + - type: map_at_100 + value: 17.698 + - type: map_at_1000 + value: 19.364 + - type: recall_at_1 + value: 3.004 + - type: recall_at_3 + value: 7.178 + - type: recall_at_5 + value: 11.196 + - type: recall_at_10 + value: 18.584999999999997 + - type: recall_at_20 + value: 26.845999999999997 + - type: recall_at_100 + value: 49.025 + - type: recall_at_1000 + value: 82.884 + - type: precision_at_1 + value: 40.816 + - type: precision_at_3 + value: 33.333 + - type: precision_at_5 + value: 30.612000000000002 + - type: precision_at_10 + value: 25.714 + - type: precision_at_20 + value: 19.387999999999998 + - type: precision_at_100 + value: 7.939 + - type: precision_at_1000 + value: 1.545 + - type: mrr_at_1 + value: 40.8163 + - type: mrr_at_3 + value: 53.401399999999995 + - type: mrr_at_5 + value: 56.7687 + - type: mrr_at_10 + value: 57.5421 + - type: mrr_at_20 + value: 58.142 + - type: mrr_at_100 + value: 58.2307 + - type: mrr_at_1000 + value: 58.2307 + - type: nauc_ndcg_at_1_max + value: -18.0584 + - type: nauc_ndcg_at_1_std + value: -25.634600000000002 + - type: nauc_ndcg_at_1_diff1 + value: -1.7021000000000002 + - type: nauc_ndcg_at_3_max + value: -17.8622 + - type: nauc_ndcg_at_3_std + value: -20.119799999999998 + - type: nauc_ndcg_at_3_diff1 + value: -2.399 + - type: nauc_ndcg_at_5_max + value: -22.0829 + - type: nauc_ndcg_at_5_std + value: -22.841 + - type: nauc_ndcg_at_5_diff1 + value: -12.350200000000001 + - type: nauc_ndcg_at_10_max + value: -17.858999999999998 + - type: nauc_ndcg_at_10_std + value: -17.9067 + - type: nauc_ndcg_at_10_diff1 + value: -9.3129 + - type: nauc_ndcg_at_20_max + value: -24.479400000000002 + - type: nauc_ndcg_at_20_std + value: -16.06 + - type: nauc_ndcg_at_20_diff1 + value: -10.57 + - type: nauc_ndcg_at_100_max + value: -20.9167 + - type: nauc_ndcg_at_100_std + value: 9.6051 + - type: nauc_ndcg_at_100_diff1 + value: -0.2363 + - type: nauc_ndcg_at_1000_max + value: -13.6708 + - type: nauc_ndcg_at_1000_std + value: 17.956 + - type: nauc_ndcg_at_1000_diff1 + value: -2.5696 + - type: nauc_map_at_1_max + value: -14.276900000000001 + - type: nauc_map_at_1_std + value: -31.3091 + - type: nauc_map_at_1_diff1 + value: -1.4354 + - type: nauc_map_at_3_max + value: -21.7098 + - type: nauc_map_at_3_std + value: -32.112899999999996 + - type: nauc_map_at_3_diff1 + value: -8.846 + - type: nauc_map_at_5_max + value: -16.700200000000002 + - type: nauc_map_at_5_std + value: -32.643499999999996 + - type: nauc_map_at_5_diff1 + value: -13.9766 + - type: nauc_map_at_10_max + value: -13.415199999999999 + - type: nauc_map_at_10_std + value: -28.459200000000003 + - type: nauc_map_at_10_diff1 + value: -12.4042 + - type: nauc_map_at_20_max + value: -17.8629 + - type: nauc_map_at_20_std + value: -24.5837 + - type: nauc_map_at_20_diff1 + value: -14.9642 + - type: nauc_map_at_100_max + value: -15.6478 + - type: nauc_map_at_100_std + value: -11.4237 + - type: nauc_map_at_100_diff1 + value: -11.542 + - type: nauc_map_at_1000_max + value: -15.2149 + - type: nauc_map_at_1000_std + value: -8.0384 + - type: nauc_map_at_1000_diff1 + value: -12.984000000000002 + - type: nauc_recall_at_1_max + value: -14.276900000000001 + - type: nauc_recall_at_1_std + value: -31.3091 + - type: nauc_recall_at_1_diff1 + value: -1.4354 + - type: nauc_recall_at_3_max + value: -23.021900000000002 + - type: nauc_recall_at_3_std + value: -30.2834 + - type: nauc_recall_at_3_diff1 + value: -11.4226 + - type: nauc_recall_at_5_max + value: -20.596600000000002 + - type: nauc_recall_at_5_std + value: -33.219300000000004 + - type: nauc_recall_at_5_diff1 + value: -17.718999999999998 + - type: nauc_recall_at_10_max + value: -16.1214 + - type: nauc_recall_at_10_std + value: -23.9041 + - type: nauc_recall_at_10_diff1 + value: -11.047 + - type: nauc_recall_at_20_max + value: -25.603399999999997 + - type: nauc_recall_at_20_std + value: -15.8105 + - type: nauc_recall_at_20_diff1 + value: -14.546000000000001 + - type: nauc_recall_at_100_max + value: -16.389400000000002 + - type: nauc_recall_at_100_std + value: 28.5141 + - type: nauc_recall_at_100_diff1 + value: 6.1868 + - type: nauc_recall_at_1000_max + value: 11.022 + - type: nauc_recall_at_1000_std + value: 68.0021 + - type: nauc_recall_at_1000_diff1 + value: 8.426 + - type: nauc_precision_at_1_max + value: -17.1625 + - type: nauc_precision_at_1_std + value: -27.9451 + - type: nauc_precision_at_1_diff1 + value: 1.0831 + - type: nauc_precision_at_3_max + value: -17.2798 + - type: nauc_precision_at_3_std + value: -20.347199999999997 + - type: nauc_precision_at_3_diff1 + value: -5.2689 + - type: nauc_precision_at_5_max + value: -19.6408 + - type: nauc_precision_at_5_std + value: -24.157 + - type: nauc_precision_at_5_diff1 + value: -20.274900000000002 + - type: nauc_precision_at_10_max + value: -11.8033 + - type: nauc_precision_at_10_std + value: -7.2727 + - type: nauc_precision_at_10_diff1 + value: -9.3776 + - type: nauc_precision_at_20_max + value: -20.1541 + - type: nauc_precision_at_20_std + value: 9.0645 + - type: nauc_precision_at_20_diff1 + value: -16.1323 + - type: nauc_precision_at_100_max + value: 0.3701 + - type: nauc_precision_at_100_std + value: 67.6941 + - type: nauc_precision_at_100_diff1 + value: 8.0336 + - type: nauc_precision_at_1000_max + value: 38.8632 + - type: nauc_precision_at_1000_std + value: 38.0504 + - type: nauc_precision_at_1000_diff1 + value: 0.5907 + - type: nauc_mrr_at_1_max + value: -17.1625 + - type: nauc_mrr_at_1_std + value: -27.9451 + - type: nauc_mrr_at_1_diff1 + value: 1.0831 + - type: nauc_mrr_at_3_max + value: -20.479300000000002 + - type: nauc_mrr_at_3_std + value: -21.9225 + - type: nauc_mrr_at_3_diff1 + value: -1.5211000000000001 + - type: nauc_mrr_at_5_max + value: -24.8175 + - type: nauc_mrr_at_5_std + value: -23.805 + - type: nauc_mrr_at_5_diff1 + value: -7.9258 + - type: nauc_mrr_at_10_max + value: -22.53 + - type: nauc_mrr_at_10_std + value: -21.9391 + - type: nauc_mrr_at_10_diff1 + value: -5.7533 + - type: nauc_mrr_at_20_max + value: -22.7064 + - type: nauc_mrr_at_20_std + value: -22.4697 + - type: nauc_mrr_at_20_diff1 + value: -5.7068 + - type: nauc_mrr_at_100_max + value: -23.0016 + - type: nauc_mrr_at_100_std + value: -22.488 + - type: nauc_mrr_at_100_diff1 + value: -5.3738 + - type: nauc_mrr_at_1000_max + value: -23.0016 + - type: nauc_mrr_at_1000_std + value: -22.488 + - type: nauc_mrr_at_1000_diff1 + value: -5.3738 + - type: main_score + value: 29.499 + task: + type: Retrieval + - dataset: + config: default + name: MTEB ToxicConversationsClassification (default) + revision: edfaf9da55d3dd50d43143d90c1ac476895ae6de + split: test + type: mteb/toxic_conversations_50k + metrics: + - type: accuracy + value: 65.8643 + - type: f1 + value: 50.6764 + - type: f1_weighted + value: 73.2472 + - type: ap + value: 12.2658 + - type: ap_weighted + value: 12.2658 + - type: main_score + value: 65.8643 + task: + type: Classification + - dataset: + config: default + name: MTEB TweetSentimentExtractionClassification (default) + revision: d604517c81ca91fe16a244d1248fc021f9ecee7a + split: test + type: mteb/tweet_sentiment_extraction + metrics: + - type: accuracy + value: 59.6633 + - type: f1 + value: 59.935700000000004 + - type: f1_weighted + value: 59.0249 + - type: main_score + value: 59.6633 + task: + type: Classification + - dataset: + config: default + name: MTEB TwentyNewsgroupsClustering (default) + revision: 6125ec4e24fa026cec8a478383ee943acfbd5449 + split: test + type: mteb/twentynewsgroups-clustering + metrics: + - type: v_measure + value: 43.2311 + - type: v_measure_std + value: 2.3994999999999997 + - type: main_score + value: 43.2311 + task: + type: Clustering + - dataset: + config: default + name: MTEB TwitterSemEval2015 (default) + revision: 70970daeab8776df92f5ea462b6173c0b46fd2d1 + split: test + type: mteb/twittersemeval2015-pairclassification + metrics: + - type: similarity_accuracy + value: 83.8469 + - type: similarity_accuracy_threshold + value: 77.6695 + - type: similarity_f1 + value: 62.3159 + - type: similarity_f1_threshold + value: 71.6554 + - type: similarity_precision + value: 59.114599999999996 + - type: similarity_recall + value: 65.8839 + - type: similarity_ap + value: 67.00930000000001 + - type: cosine_accuracy + value: 83.8469 + - type: cosine_accuracy_threshold + value: 77.6695 + - type: cosine_f1 + value: 62.3159 + - type: cosine_f1_threshold + value: 71.6554 + - type: cosine_precision + value: 59.114599999999996 + - type: cosine_recall + value: 65.8839 + - type: cosine_ap + value: 67.00930000000001 + - type: manhattan_accuracy + value: 83.7694 + - type: manhattan_accuracy_threshold + value: 1677.8293999999999 + - type: manhattan_f1 + value: 62.1324 + - type: manhattan_f1_threshold + value: 1848.6641 + - type: manhattan_precision + value: 61.839999999999996 + - type: manhattan_recall + value: 62.4274 + - type: manhattan_ap + value: 66.8849 + - type: euclidean_accuracy + value: 83.8469 + - type: euclidean_accuracy_threshold + value: 66.8288 + - type: euclidean_f1 + value: 62.3159 + - type: euclidean_f1_threshold + value: 75.2922 + - type: euclidean_precision + value: 59.114599999999996 + - type: euclidean_recall + value: 65.8839 + - type: euclidean_ap + value: 67.00930000000001 + - type: dot_accuracy + value: 83.8469 + - type: dot_accuracy_threshold + value: 77.6695 + - type: dot_f1 + value: 62.3159 + - type: dot_f1_threshold + value: 71.6554 + - type: dot_precision + value: 59.114599999999996 + - type: dot_recall + value: 65.8839 + - type: dot_ap + value: 67.00930000000001 + - type: max_accuracy + value: 83.8469 + - type: max_f1 + value: 62.3159 + - type: max_precision + value: 61.839999999999996 + - type: max_recall + value: 65.8839 + - type: max_ap + value: 67.00930000000001 + - type: main_score + value: 67.00930000000001 + task: + type: PairClassification + - dataset: + config: default + name: MTEB TwitterURLCorpus (default) + revision: 8b6510b0b1fa4e4c4f879467980e9be563ec1cdf + split: test + type: mteb/twitterurlcorpus-pairclassification + metrics: + - type: similarity_accuracy + value: 88.8811 + - type: similarity_accuracy_threshold + value: 71.1053 + - type: similarity_f1 + value: 77.9005 + - type: similarity_f1_threshold + value: 67.5068 + - type: similarity_precision + value: 75.5609 + - type: similarity_recall + value: 80.3896 + - type: similarity_ap + value: 85.459 + - type: cosine_accuracy + value: 88.8811 + - type: cosine_accuracy_threshold + value: 71.1053 + - type: cosine_f1 + value: 77.9005 + - type: cosine_f1_threshold + value: 67.5068 + - type: cosine_precision + value: 75.5609 + - type: cosine_recall + value: 80.3896 + - type: cosine_ap + value: 85.459 + - type: manhattan_accuracy + value: 88.8598 + - type: manhattan_accuracy_threshold + value: 1928.9173 + - type: manhattan_f1 + value: 77.9172 + - type: manhattan_f1_threshold + value: 2007.8883999999998 + - type: manhattan_precision + value: 76.29310000000001 + - type: manhattan_recall + value: 79.6119 + - type: manhattan_ap + value: 85.4464 + - type: euclidean_accuracy + value: 88.8811 + - type: euclidean_accuracy_threshold + value: 76.0193 + - type: euclidean_f1 + value: 77.9005 + - type: euclidean_f1_threshold + value: 80.6141 + - type: euclidean_precision + value: 75.5609 + - type: euclidean_recall + value: 80.3896 + - type: euclidean_ap + value: 85.459 + - type: dot_accuracy + value: 88.8811 + - type: dot_accuracy_threshold + value: 71.1053 + - type: dot_f1 + value: 77.9005 + - type: dot_f1_threshold + value: 67.5068 + - type: dot_precision + value: 75.5609 + - type: dot_recall + value: 80.3896 + - type: dot_ap + value: 85.459 + - type: max_accuracy + value: 88.8811 + - type: max_f1 + value: 77.9172 + - type: max_precision + value: 76.29310000000001 + - type: max_recall + value: 80.3896 + - type: max_ap + value: 85.459 + - type: main_score + value: 85.459 + task: + type: PairClassification +--- + +

Snowflake's Arctic-embed-l-v2.0

+

+

+ News | + Models | + Usage | + Evaluation | + Contact | + FAQ + License | + Acknowledgement +

+

+ + +## News +12/04/2024: Release of [snowflake-arctic-embed-l-v2.0](https://huggingface.co/Snowflake/snowflake-arctic-embed-l-v2.0) and [snowflake-arctic-embed-m-v2.0](https://huggingface.co/Snowflake/snowflake-arctic-embed-m-v2.0) our newest models with multilingual workloads in mind. + +## Models +Snowflake arctic-embed-l-v2.0 is the newest addition to the suite of embedding models Snowflake has released optimizing for retrieval performance and inference efficiency. +Arctic Embed 2.0 introduces a new standard for multilingual embedding models, combining high-quality multilingual text retrieval without sacrificing performance in English. +Released under the permissive Apache 2.0 license, Arctic Embed 2.0 is ideal for applications that demand reliable, enterprise-grade multilingual search and retrieval at scale. + +Key Features: + +1. Multilingual without compromise: Excels in English and non-English retrieval, outperforming leading open-source and proprietary models on benchmarks like MTEB Retrieval, CLEF, and MIRACL. + +2. Inference efficiency: Its 303m non-embedding parameters inference is fast and efficient for any scale. + +3. Compression-friendly: Achieves high-quality retrieval with embeddings as small as 128 bytes/vector using Matryoshka Representation Learning (MRL) and quantization-aware embedding training. + +4. Drop-In Replacement: arctic-embed-l-v2.0 builds on BAAI/bge-m3-retromae](https://huggingface.co/BAAI/bge-m3-retromae) which allows direct drop-in inference replacement with any form of new libraries, kernels, inference engines etc. + +5. Long Context Support: arctic-embed-l-v2.0 builds on [BAAI/bge-m3-retromae](https://huggingface.co/BAAI/bge-m3-retromae) which can support a context window of up to 8192 via the use of RoPE. + + +### Quality Benchmarks +Unlike most other open-source models, Arctic-embed-l-v2.0 excels across English (via MTEB Retrieval) and multilingual (via MIRACL and CLEF). +You no longer need to support models to empower high-quality English and multilingual retrieval. All numbers mentioned below are the average NDCG@10 across the dataset being discussed. + +| Model Name | # params | # non-emb params | # dimensions | BEIR (15) | MIRACL (4) | CLEF (Focused) | CLEF (Full) | +|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| **snowflake-arctic-l-v2.0** | 568M | 303M | 1024 | **55.6** | 55.8 | **52.9** | **54.3** | +| snowflake-arctic-m | 109M | 86M | 768 | 54.9 | 24.9 | 34.4 | 29.1 | +| snowflake-arctic-l | 335M | 303M | 1024 | 56.0 | 34.8 | 38.2 | 33.7 | +| me5 base | 560M | 303M | 1024 | 51.4 | 54.0 | 43.0 | 34.6 | +| bge-m3 (BAAI) | 568M | 303M | 1024 | 48.8 | **56.8** | 40.8 | 41.3 | +| gte (Alibaba) | 305M | 113M | 768 | 51.1 | 52.3 | 47.7 | 53.1 | + +Aside from high-quality retrieval arctic delivers embeddings that are easily compressible. Leverage vector truncation via MRL to decrease vector size by 4x with less than 3% degredation in quality. +Combine MRLed vectors with vector compression (Int4) to power retrieval in 128 bytes per doc. + +| Model | | BEIR (15) | Relative Performance | MIRACL (4) | Relative Performance | CLEF (5) | Relative Performance | CLEF (Full) | Relative Performance | +|---|---|:---:|:---:|:---:|:---:|:---:|---|---|---| +| snowflake-arctic-l-v2.0 | 1024 | 55.6 | N/A | 55.8 | N/A | 52.9 | N/A | 54.3 | N/A | +| snowflake-arctic-l-v2.0 | 256 | 54.3 | -0.18% | 54.3 | -2.70% | 51.9 | -1.81% | 53.4 | -1.53% | + +## Usage + +### Using Sentence Transformers + +```python +from sentence_transformers import SentenceTransformer + +# Load the model +model_name = 'Snowflake/snowflake-arctic-embed-l-v2.0' +model = SentenceTransformer(model_name) + +# Define the queries and documents +queries = ['what is snowflake?', 'Where can I get the best tacos?'] +documents = ['The Data Cloud!', 'Mexico City of Course!'] + +# Compute embeddings: use `prompt_name="query"` to encode queries! +query_embeddings = model.encode(queries, prompt_name="query") +document_embeddings = model.encode(documents) + +# Compute cosine similarity scores +scores = model.similarity(query_embeddings, document_embeddings) + +# Output the results +for query, query_scores in zip(queries, scores): + doc_score_pairs = list(zip(documents, query_scores)) + doc_score_pairs = sorted(doc_score_pairs, key=lambda x: x[1], reverse=True) + print("Query:", query) + for document, score in doc_score_pairs: + print(score, document) + +``` + + + +### Using Huggingface Transformers + + +You can use the transformers package to use Snowflake's arctic-embed model, as shown below. For optimal retrieval quality, use the CLS token to embed each text portion and use the query prefix below (just on the query). + +```python +import torch +from transformers import AutoModel, AutoTokenizer + +model_name = 'Snowflake/snowflake-arctic-embed-l-v2.0' +tokenizer = AutoTokenizer.from_pretrained(model_name) +model = AutoModel.from_pretrained(model_name, add_pooling_layer=False) +model.eval() + +query_prefix = 'query: ' +queries = ['what is snowflake?', 'Where can I get the best tacos?'] +queries_with_prefix = ["{}{}".format(query_prefix, i) for i in queries] +query_tokens = tokenizer(queries_with_prefix, padding=True, truncation=True, return_tensors='pt', max_length=8192) + +documents = ['The Data Cloud!', 'Mexico City of Course!'] +document_tokens = tokenizer(documents, padding=True, truncation=True, return_tensors='pt', max_length=8192) + +# Compute token embeddings +with torch.no_grad(): + query_embeddings = model(**query_tokens)[0][:, 0] + document_embeddings = model(**document_tokens)[0][:, 0] + + +# normalize embeddings +query_embeddings = torch.nn.functional.normalize(query_embeddings, p=2, dim=1) +document_embeddings = torch.nn.functional.normalize(document_embeddings, p=2, dim=1) + +scores = torch.mm(query_embeddings, document_embeddings.transpose(0, 1)) +for query, query_scores in zip(queries, scores): + doc_score_pairs = list(zip(documents, query_scores)) + doc_score_pairs = sorted(doc_score_pairs, key=lambda x: x[1], reverse=True) + #Output passages & scores + print("Query:", query) + for document, score in doc_score_pairs: + print(score, document) +``` + + +This should produce the following scores + +``` +Query: what is snowflake? +tensor(0.2715) The Data Cloud! +tensor(0.0661) Mexico City of Course! +Query: Where can I get the best tacos? +tensor(0.2797) Mexico City of Course! +tensor(0.1250) The Data Cloud! +``` + +### Using Huggingface Transformers.js + +If you haven't already, you can install the [Transformers.js](https://huggingface.co/docs/transformers.js) JavaScript library from [NPM](https://www.npmjs.com/package/@huggingface/transformers) using: +```bash +npm i @huggingface/transformers +``` + +You can then use the model for retrieval, as follows: + +```js +import { pipeline, dot } from '@huggingface/transformers'; + +// Create feature extraction pipeline +const extractor = await pipeline('feature-extraction', 'Snowflake/snowflake-arctic-embed-m-v2.0', { + dtype: 'q8', +}); + +// Generate sentence embeddings +const sentences = [ + 'query: what is snowflake?', + 'The Data Cloud!', + 'Mexico City of Course!', +] +const output = await extractor(sentences, { normalize: true, pooling: 'cls' }); + +// Compute similarity scores +const [source_embeddings, ...document_embeddings ] = output.tolist(); +const similarities = document_embeddings.map(x => dot(source_embeddings, x)); +console.log(similarities); // [0.24783534471401417, 0.05313122704326892] +``` + + +## Contact + + +Feel free to open an issue or pull request if you have any questions or suggestions about this project. +You also can email Daniel Campos(daniel.campos@snowflake.com). + + +## License +Arctic is licensed under the [Apache-2](https://www.apache.org/licenses/LICENSE-2.0). The released models can be used for commercial purposes free of charge. \ No newline at end of file