diff --git "a/README.md" "b/README.md" --- "a/README.md" +++ "b/README.md" @@ -1,3 +1,9087 @@ --- -license: mit +model-index: +- name: stella_en_400M_v5 + results: + - dataset: + config: en + name: MTEB AmazonCounterfactualClassification (en) + revision: e8379541af4e31359cca9fbcf4b00f2671dba205 + split: test + type: mteb/amazon_counterfactual + metrics: + - type: accuracy + value: 92.35820895522387 + - type: ap + value: 70.81322736988783 + - type: ap_weighted + value: 70.81322736988783 + - type: f1 + value: 88.9505466159595 + - type: f1_weighted + value: 92.68630932872613 + - type: main_score + value: 92.35820895522387 + task: + type: Classification + - dataset: + config: default + name: MTEB AmazonPolarityClassification + revision: e2d317d38cd51312af73b3d32a06d1a08b442046 + split: test + type: mteb/amazon_polarity + metrics: + - type: accuracy + value: 97.1945 + - type: ap + value: 96.08192192244094 + - type: ap_weighted + value: 96.08192192244094 + - type: f1 + value: 97.1936887167346 + - type: f1_weighted + value: 97.1936887167346 + - type: main_score + value: 97.1945 + task: + type: Classification + - dataset: + config: en + name: MTEB AmazonReviewsClassification (en) + revision: 1399c76144fd37290681b995c656ef9b2e06e26d + split: test + type: mteb/amazon_reviews_multi + metrics: + - type: accuracy + value: 59.528000000000006 + - type: f1 + value: 59.21016819840188 + - type: f1_weighted + value: 59.21016819840188 + - type: main_score + value: 59.528000000000006 + task: + type: Classification + - dataset: + config: en + name: MTEB AmazonReviewsClassification (en) + revision: 1399c76144fd37290681b995c656ef9b2e06e26d + split: validation + type: mteb/amazon_reviews_multi + metrics: + - type: accuracy + value: 59.64 + - type: f1 + value: 59.26937685495699 + - type: f1_weighted + value: 59.26937685495699 + - type: main_score + value: 59.64 + task: + type: Classification + - dataset: + config: default + name: MTEB ArguAna + revision: c22ab2a51041ffd869aaddef7af8d8215647e41a + split: test + type: mteb/arguana + metrics: + - type: main_score + value: 64.24 + - type: map_at_1 + value: 40.398 + - type: map_at_10 + value: 56.215 + - type: map_at_100 + value: 56.833999999999996 + - type: map_at_1000 + value: 56.835 + - type: map_at_20 + value: 56.747 + - type: map_at_3 + value: 52.181 + - type: map_at_5 + value: 54.628 + - type: mrr_at_1 + value: 41.25177809388336 + - type: mrr_at_10 + value: 56.570762491815216 + - type: mrr_at_100 + value: 57.17548614361504 + - type: mrr_at_1000 + value: 57.176650626377466 + - type: mrr_at_20 + value: 57.08916253512566 + - type: mrr_at_3 + value: 52.47747747747754 + - type: mrr_at_5 + value: 54.94547178757718 + - type: nauc_map_at_1000_diff1 + value: 22.408086887100158 + - type: nauc_map_at_1000_max + value: -8.730419096847543 + - type: nauc_map_at_1000_std + value: -17.789262741255737 + - type: nauc_map_at_100_diff1 + value: 22.407371684274025 + - type: nauc_map_at_100_max + value: -8.732263549026266 + - type: nauc_map_at_100_std + value: -17.79550515579994 + - type: nauc_map_at_10_diff1 + value: 21.925005073301246 + - type: nauc_map_at_10_max + value: -8.990323944492134 + - type: nauc_map_at_10_std + value: -18.199246301671458 + - type: nauc_map_at_1_diff1 + value: 26.23276644969203 + - type: nauc_map_at_1_max + value: -12.376511389571245 + - type: nauc_map_at_1_std + value: -18.11411715207284 + - type: nauc_map_at_20_diff1 + value: 22.32455790850922 + - type: nauc_map_at_20_max + value: -8.664671547236034 + - type: nauc_map_at_20_std + value: -17.8290016125137 + - type: nauc_map_at_3_diff1 + value: 22.395462147465064 + - type: nauc_map_at_3_max + value: -8.206580750918844 + - type: nauc_map_at_3_std + value: -17.604490446911484 + - type: nauc_map_at_5_diff1 + value: 21.95307379904799 + - type: nauc_map_at_5_max + value: -8.03958102978443 + - type: nauc_map_at_5_std + value: -17.36578866595004 + - type: nauc_mrr_at_1000_diff1 + value: 20.124236798365587 + - type: nauc_mrr_at_1000_max + value: -9.587376069575898 + - type: nauc_mrr_at_1000_std + value: -17.79191612151833 + - type: nauc_mrr_at_100_diff1 + value: 20.123612603474033 + - type: nauc_mrr_at_100_max + value: -9.589187218607831 + - type: nauc_mrr_at_100_std + value: -17.7981617777748 + - type: nauc_mrr_at_10_diff1 + value: 19.723683875738075 + - type: nauc_mrr_at_10_max + value: -9.774151729178815 + - type: nauc_mrr_at_10_std + value: -18.168668675495162 + - type: nauc_mrr_at_1_diff1 + value: 23.945332059908132 + - type: nauc_mrr_at_1_max + value: -12.260461466152819 + - type: nauc_mrr_at_1_std + value: -18.007194922921148 + - type: nauc_mrr_at_20_diff1 + value: 20.04819461810257 + - type: nauc_mrr_at_20_max + value: -9.518368283588936 + - type: nauc_mrr_at_20_std + value: -17.831608149836136 + - type: nauc_mrr_at_3_diff1 + value: 19.8571785245832 + - type: nauc_mrr_at_3_max + value: -9.464375021240478 + - type: nauc_mrr_at_3_std + value: -17.728533927330453 + - type: nauc_mrr_at_5_diff1 + value: 19.670313652167827 + - type: nauc_mrr_at_5_max + value: -8.966372585728434 + - type: nauc_mrr_at_5_std + value: -17.468955834324817 + - type: nauc_ndcg_at_1000_diff1 + value: 21.863049281767417 + - type: nauc_ndcg_at_1000_max + value: -8.18698520924057 + - type: nauc_ndcg_at_1000_std + value: -17.634483364794804 + - type: nauc_ndcg_at_100_diff1 + value: 21.849924385738586 + - type: nauc_ndcg_at_100_max + value: -8.226437560889345 + - type: nauc_ndcg_at_100_std + value: -17.774648478087002 + - type: nauc_ndcg_at_10_diff1 + value: 19.888395590413573 + - type: nauc_ndcg_at_10_max + value: -8.968706085632382 + - type: nauc_ndcg_at_10_std + value: -19.31386964628115 + - type: nauc_ndcg_at_1_diff1 + value: 26.23276644969203 + - type: nauc_ndcg_at_1_max + value: -12.376511389571245 + - type: nauc_ndcg_at_1_std + value: -18.11411715207284 + - type: nauc_ndcg_at_20_diff1 + value: 21.38413342416933 + - type: nauc_ndcg_at_20_max + value: -7.636238194084164 + - type: nauc_ndcg_at_20_std + value: -17.946390844693028 + - type: nauc_ndcg_at_3_diff1 + value: 21.29169165029195 + - type: nauc_ndcg_at_3_max + value: -6.793840499730093 + - type: nauc_ndcg_at_3_std + value: -17.52359001586737 + - type: nauc_ndcg_at_5_diff1 + value: 20.238297656671364 + - type: nauc_ndcg_at_5_max + value: -6.424992706950072 + - type: nauc_ndcg_at_5_std + value: -17.082391132291356 + - type: nauc_precision_at_1000_diff1 + value: -7.05195108528572 + - type: nauc_precision_at_1000_max + value: 34.439879624882145 + - type: nauc_precision_at_1000_std + value: 68.72436351659353 + - type: nauc_precision_at_100_diff1 + value: -2.769464113932605 + - type: nauc_precision_at_100_max + value: 9.89562961226698 + - type: nauc_precision_at_100_std + value: -0.5880967482224028 + - type: nauc_precision_at_10_diff1 + value: 2.1371544726832323 + - type: nauc_precision_at_10_max + value: -11.93051325147756 + - type: nauc_precision_at_10_std + value: -30.83144187392059 + - type: nauc_precision_at_1_diff1 + value: 26.23276644969203 + - type: nauc_precision_at_1_max + value: -12.376511389571245 + - type: nauc_precision_at_1_std + value: -18.11411715207284 + - type: nauc_precision_at_20_diff1 + value: 3.780146814257504 + - type: nauc_precision_at_20_max + value: 17.06527540214615 + - type: nauc_precision_at_20_std + value: -20.36832563035565 + - type: nauc_precision_at_3_diff1 + value: 17.63894384012077 + - type: nauc_precision_at_3_max + value: -2.0220490624638887 + - type: nauc_precision_at_3_std + value: -17.285601413493918 + - type: nauc_precision_at_5_diff1 + value: 12.557855071944601 + - type: nauc_precision_at_5_max + value: 0.5840236463956658 + - type: nauc_precision_at_5_std + value: -15.827224420217846 + - type: nauc_recall_at_1000_diff1 + value: -7.051951085286463 + - type: nauc_recall_at_1000_max + value: 34.43987962487738 + - type: nauc_recall_at_1000_std + value: 68.724363516591 + - type: nauc_recall_at_100_diff1 + value: -2.769464113930314 + - type: nauc_recall_at_100_max + value: 9.895629612270017 + - type: nauc_recall_at_100_std + value: -0.58809674821745 + - type: nauc_recall_at_10_diff1 + value: 2.1371544726834495 + - type: nauc_recall_at_10_max + value: -11.930513251477253 + - type: nauc_recall_at_10_std + value: -30.83144187392047 + - type: nauc_recall_at_1_diff1 + value: 26.23276644969203 + - type: nauc_recall_at_1_max + value: -12.376511389571245 + - type: nauc_recall_at_1_std + value: -18.11411715207284 + - type: nauc_recall_at_20_diff1 + value: 3.7801468142575922 + - type: nauc_recall_at_20_max + value: 17.0652754021456 + - type: nauc_recall_at_20_std + value: -20.36832563035559 + - type: nauc_recall_at_3_diff1 + value: 17.63894384012074 + - type: nauc_recall_at_3_max + value: -2.02204906246383 + - type: nauc_recall_at_3_std + value: -17.28560141349386 + - type: nauc_recall_at_5_diff1 + value: 12.55785507194463 + - type: nauc_recall_at_5_max + value: 0.5840236463957296 + - type: nauc_recall_at_5_std + value: -15.827224420217856 + - type: ndcg_at_1 + value: 40.398 + - type: ndcg_at_10 + value: 64.24 + - type: ndcg_at_100 + value: 66.631 + - type: ndcg_at_1000 + value: 66.65100000000001 + - type: ndcg_at_20 + value: 66.086 + - type: ndcg_at_3 + value: 55.938 + - type: ndcg_at_5 + value: 60.370000000000005 + - type: precision_at_1 + value: 40.398 + - type: precision_at_10 + value: 8.962 + - type: precision_at_100 + value: 0.9950000000000001 + - type: precision_at_1000 + value: 0.1 + - type: precision_at_20 + value: 4.836 + - type: precision_at_3 + value: 22.262 + - type: precision_at_5 + value: 15.519 + - type: recall_at_1 + value: 40.398 + - type: recall_at_10 + value: 89.616 + - type: recall_at_100 + value: 99.502 + - type: recall_at_1000 + value: 99.644 + - type: recall_at_20 + value: 96.72800000000001 + - type: recall_at_3 + value: 66.78500000000001 + - type: recall_at_5 + value: 77.596 + task: + type: Retrieval + - dataset: + config: default + name: MTEB ArxivClusteringP2P + revision: a122ad7f3f0291bf49cc6f4d32aa80929df69d5d + split: test + type: mteb/arxiv-clustering-p2p + metrics: + - type: main_score + value: 55.1564333205451 + - type: v_measure + value: 55.1564333205451 + - type: v_measure_std + value: 14.696883012214512 + task: + type: Clustering + - dataset: + config: default + name: MTEB ArxivClusteringS2S + revision: f910caf1a6075f7329cdf8c1a6135696f37dbd53 + split: test + type: mteb/arxiv-clustering-s2s + metrics: + - type: main_score + value: 49.823698316694795 + - type: v_measure + value: 49.823698316694795 + - type: v_measure_std + value: 14.951660654298186 + task: + type: Clustering + - dataset: + config: default + name: MTEB AskUbuntuDupQuestions + revision: 2000358ca161889fa9c082cb41daa8dcfb161a54 + split: test + type: mteb/askubuntudupquestions-reranking + metrics: + - type: main_score + value: 66.15294503553424 + - type: map + value: 66.15294503553424 + - type: mrr + value: 78.53438420612935 + - type: nAUC_map_diff1 + value: 12.569697092717997 + - type: nAUC_map_max + value: 21.50670312412572 + - type: nAUC_map_std + value: 16.943786429229064 + - type: nAUC_mrr_diff1 + value: 15.590272897361238 + - type: nAUC_mrr_max + value: 34.96072022474653 + - type: nAUC_mrr_std + value: 21.649217605241045 + task: + type: Reranking + - dataset: + config: default + name: MTEB BIOSSES + revision: d3fb88f8f02e40887cd149695127462bbcf29b4a + split: test + type: mteb/biosses-sts + metrics: + - type: cosine_pearson + value: 85.7824546319275 + - type: cosine_spearman + value: 83.29587385660628 + - type: euclidean_pearson + value: 84.58764190565167 + - type: euclidean_spearman + value: 83.30069324352772 + - type: main_score + value: 83.29587385660628 + - type: manhattan_pearson + value: 84.95996839947179 + - type: manhattan_spearman + value: 83.87480271054358 + - type: pearson + value: 85.7824546319275 + - type: spearman + value: 83.29587385660628 + task: + type: STS + - dataset: + config: default + name: MTEB Banking77Classification + revision: 0fd18e25b25c072e09e0d92ab615fda904d66300 + split: test + type: mteb/banking77 + metrics: + - type: accuracy + value: 89.30194805194806 + - type: f1 + value: 89.26182507266391 + - type: f1_weighted + value: 89.26182507266391 + - type: main_score + value: 89.30194805194806 + task: + type: Classification + - dataset: + config: default + name: MTEB BiorxivClusteringP2P + revision: 65b79d1d13f80053f67aca9498d9402c2d9f1f40 + split: test + type: mteb/biorxiv-clustering-p2p + metrics: + - type: main_score + value: 50.67972171889736 + - type: v_measure + value: 50.67972171889736 + - type: v_measure_std + value: 0.7687409980036303 + task: + type: Clustering + - dataset: + config: default + name: MTEB BiorxivClusteringS2S + revision: 258694dd0231531bc1fd9de6ceb52a0853c6d908 + split: test + type: mteb/biorxiv-clustering-s2s + metrics: + - type: main_score + value: 45.80539715556144 + - type: v_measure + value: 45.80539715556144 + - type: v_measure_std + value: 0.9601346216579142 + task: + type: Clustering + - dataset: + config: default + name: MTEB CQADupstackAndroidRetrieval + revision: f46a197baaae43b4f621051089b82a364682dfeb + split: test + type: mteb/cqadupstack-android + metrics: + - type: main_score + value: 53.974 + - type: map_at_1 + value: 34.383 + - type: map_at_10 + value: 46.993 + - type: map_at_100 + value: 48.608000000000004 + - type: map_at_1000 + value: 48.722 + - type: map_at_20 + value: 47.904 + - type: map_at_3 + value: 43.0 + - type: map_at_5 + value: 45.181 + - type: mrr_at_1 + value: 43.204577968526465 + - type: mrr_at_10 + value: 53.45005336421645 + - type: mrr_at_100 + value: 54.14225008838758 + - type: mrr_at_1000 + value: 54.179415122755145 + - type: mrr_at_20 + value: 53.87577091647368 + - type: mrr_at_3 + value: 51.00143061516454 + - type: mrr_at_5 + value: 52.46065808297566 + - type: nauc_map_at_1000_diff1 + value: 53.44956217485285 + - type: nauc_map_at_1000_max + value: 36.6088085691723 + - type: nauc_map_at_1000_std + value: -3.330166926757272 + - type: nauc_map_at_100_diff1 + value: 53.44869744603097 + - type: nauc_map_at_100_max + value: 36.601153049081674 + - type: nauc_map_at_100_std + value: -3.2828051195172945 + - type: nauc_map_at_10_diff1 + value: 53.40190283742897 + - type: nauc_map_at_10_max + value: 36.06545095751328 + - type: nauc_map_at_10_std + value: -4.046389855494228 + - type: nauc_map_at_1_diff1 + value: 57.75005364303626 + - type: nauc_map_at_1_max + value: 30.23016814650989 + - type: nauc_map_at_1_std + value: -5.0871153033102186 + - type: nauc_map_at_20_diff1 + value: 53.43843903317814 + - type: nauc_map_at_20_max + value: 36.206301825584234 + - type: nauc_map_at_20_std + value: -3.6640331685570517 + - type: nauc_map_at_3_diff1 + value: 53.50565186265725 + - type: nauc_map_at_3_max + value: 35.384484613413726 + - type: nauc_map_at_3_std + value: -6.406625419686271 + - type: nauc_map_at_5_diff1 + value: 53.59098491288354 + - type: nauc_map_at_5_max + value: 35.35885622742766 + - type: nauc_map_at_5_std + value: -5.297241297637247 + - type: nauc_mrr_at_1000_diff1 + value: 53.33018892980987 + - type: nauc_mrr_at_1000_max + value: 38.35651284839087 + - type: nauc_mrr_at_1000_std + value: -2.690216921525478 + - type: nauc_mrr_at_100_diff1 + value: 53.321721430402526 + - type: nauc_mrr_at_100_max + value: 38.36700375555358 + - type: nauc_mrr_at_100_std + value: -2.668471180993299 + - type: nauc_mrr_at_10_diff1 + value: 53.072843228803734 + - type: nauc_mrr_at_10_max + value: 38.29899178434874 + - type: nauc_mrr_at_10_std + value: -2.731232588566702 + - type: nauc_mrr_at_1_diff1 + value: 58.215027100156334 + - type: nauc_mrr_at_1_max + value: 38.079566722240095 + - type: nauc_mrr_at_1_std + value: -2.968455825921349 + - type: nauc_mrr_at_20_diff1 + value: 53.28426352493095 + - type: nauc_mrr_at_20_max + value: 38.27277677619666 + - type: nauc_mrr_at_20_std + value: -2.733892286409146 + - type: nauc_mrr_at_3_diff1 + value: 53.42410923109592 + - type: nauc_mrr_at_3_max + value: 39.24436286127795 + - type: nauc_mrr_at_3_std + value: -4.27367227169996 + - type: nauc_mrr_at_5_diff1 + value: 53.23376388309158 + - type: nauc_mrr_at_5_max + value: 38.5217011527492 + - type: nauc_mrr_at_5_std + value: -3.2680426608503823 + - type: nauc_ndcg_at_1000_diff1 + value: 52.54333660516546 + - type: nauc_ndcg_at_1000_max + value: 37.43214526041539 + - type: nauc_ndcg_at_1000_std + value: -1.1960682452610014 + - type: nauc_ndcg_at_100_diff1 + value: 52.19817001244995 + - type: nauc_ndcg_at_100_max + value: 37.65455567519698 + - type: nauc_ndcg_at_100_std + value: -0.32029340172128024 + - type: nauc_ndcg_at_10_diff1 + value: 51.38122777714283 + - type: nauc_ndcg_at_10_max + value: 36.70046496568422 + - type: nauc_ndcg_at_10_std + value: -2.462175966250386 + - type: nauc_ndcg_at_1_diff1 + value: 58.215027100156334 + - type: nauc_ndcg_at_1_max + value: 38.079566722240095 + - type: nauc_ndcg_at_1_std + value: -2.968455825921349 + - type: nauc_ndcg_at_20_diff1 + value: 51.80095733174004 + - type: nauc_ndcg_at_20_max + value: 36.09583445493353 + - type: nauc_ndcg_at_20_std + value: -2.0508715409334783 + - type: nauc_ndcg_at_3_diff1 + value: 51.354237228730184 + - type: nauc_ndcg_at_3_max + value: 38.588938416538696 + - type: nauc_ndcg_at_3_std + value: -5.145993631601536 + - type: nauc_ndcg_at_5_diff1 + value: 51.73747522351185 + - type: nauc_ndcg_at_5_max + value: 36.798364610155815 + - type: nauc_ndcg_at_5_std + value: -4.084023387854836 + - type: nauc_precision_at_1000_diff1 + value: -17.516707759823454 + - type: nauc_precision_at_1000_max + value: -0.1207553822170436 + - type: nauc_precision_at_1000_std + value: -6.030577232368578 + - type: nauc_precision_at_100_diff1 + value: -9.145819130810498 + - type: nauc_precision_at_100_max + value: 11.96368132178629 + - type: nauc_precision_at_100_std + value: 7.71967340295656 + - type: nauc_precision_at_10_diff1 + value: 7.9931993925793865 + - type: nauc_precision_at_10_max + value: 26.484313749084095 + - type: nauc_precision_at_10_std + value: 8.712786019080635 + - type: nauc_precision_at_1_diff1 + value: 58.215027100156334 + - type: nauc_precision_at_1_max + value: 38.079566722240095 + - type: nauc_precision_at_1_std + value: -2.968455825921349 + - type: nauc_precision_at_20_diff1 + value: 0.8393669364662555 + - type: nauc_precision_at_20_max + value: 19.83491875645947 + - type: nauc_precision_at_20_std + value: 9.461124186482857 + - type: nauc_precision_at_3_diff1 + value: 27.641754672670572 + - type: nauc_precision_at_3_max + value: 38.411100973307164 + - type: nauc_precision_at_3_std + value: -0.9720843380376243 + - type: nauc_precision_at_5_diff1 + value: 18.705446120839266 + - type: nauc_precision_at_5_max + value: 31.543523656202453 + - type: nauc_precision_at_5_std + value: 3.849693074719181 + - type: nauc_recall_at_1000_diff1 + value: 44.34280209610548 + - type: nauc_recall_at_1000_max + value: 27.469913293945602 + - type: nauc_recall_at_1000_std + value: 60.340541133251755 + - type: nauc_recall_at_100_diff1 + value: 41.629821738762466 + - type: nauc_recall_at_100_max + value: 37.17236528492244 + - type: nauc_recall_at_100_std + value: 25.94520878194288 + - type: nauc_recall_at_10_diff1 + value: 42.05079825021677 + - type: nauc_recall_at_10_max + value: 29.32597215510977 + - type: nauc_recall_at_10_std + value: 1.0977496609409771 + - type: nauc_recall_at_1_diff1 + value: 57.75005364303626 + - type: nauc_recall_at_1_max + value: 30.23016814650989 + - type: nauc_recall_at_1_std + value: -5.0871153033102186 + - type: nauc_recall_at_20_diff1 + value: 42.41360152005307 + - type: nauc_recall_at_20_max + value: 26.592849999722972 + - type: nauc_recall_at_20_std + value: 3.3724018291774778 + - type: nauc_recall_at_3_diff1 + value: 46.14060500159665 + - type: nauc_recall_at_3_max + value: 33.986937349996445 + - type: nauc_recall_at_3_std + value: -8.037365765322734 + - type: nauc_recall_at_5_diff1 + value: 44.764082419958854 + - type: nauc_recall_at_5_max + value: 30.39820421813806 + - type: nauc_recall_at_5_std + value: -4.001142031691271 + - type: ndcg_at_1 + value: 43.205 + - type: ndcg_at_10 + value: 53.974 + - type: ndcg_at_100 + value: 59.156 + - type: ndcg_at_1000 + value: 60.662000000000006 + - type: ndcg_at_20 + value: 56.118 + - type: ndcg_at_3 + value: 48.737 + - type: ndcg_at_5 + value: 51.099000000000004 + - type: precision_at_1 + value: 43.205 + - type: precision_at_10 + value: 10.83 + - type: precision_at_100 + value: 1.66 + - type: precision_at_1000 + value: 0.209 + - type: precision_at_20 + value: 6.366 + - type: precision_at_3 + value: 24.32 + - type: precision_at_5 + value: 17.596999999999998 + - type: recall_at_1 + value: 34.383 + - type: recall_at_10 + value: 66.69 + - type: recall_at_100 + value: 88.378 + - type: recall_at_1000 + value: 97.675 + - type: recall_at_20 + value: 74.612 + - type: recall_at_3 + value: 50.736000000000004 + - type: recall_at_5 + value: 57.655 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackEnglishRetrieval + revision: ad9991cb51e31e31e430383c75ffb2885547b5f0 + split: test + type: mteb/cqadupstack-english + metrics: + - type: main_score + value: 49.553000000000004 + - type: map_at_1 + value: 32.51 + - type: map_at_10 + value: 43.589 + - type: map_at_100 + value: 44.763999999999996 + - type: map_at_1000 + value: 44.891999999999996 + - type: map_at_20 + value: 44.212 + - type: map_at_3 + value: 40.425 + - type: map_at_5 + value: 42.109 + - type: mrr_at_1 + value: 41.52866242038217 + - type: mrr_at_10 + value: 49.90862905671824 + - type: mrr_at_100 + value: 50.51591181736842 + - type: mrr_at_1000 + value: 50.55866867651289 + - type: mrr_at_20 + value: 50.24923478276552 + - type: mrr_at_3 + value: 47.80254777070064 + - type: mrr_at_5 + value: 48.968152866242086 + - type: nauc_map_at_1000_diff1 + value: 53.566776065340605 + - type: nauc_map_at_1000_max + value: 31.365048046996492 + - type: nauc_map_at_1000_std + value: 6.5977438656877245 + - type: nauc_map_at_100_diff1 + value: 53.526516574455094 + - type: nauc_map_at_100_max + value: 31.259149771772616 + - type: nauc_map_at_100_std + value: 6.506640128645118 + - type: nauc_map_at_10_diff1 + value: 53.78105912041269 + - type: nauc_map_at_10_max + value: 30.399146609411332 + - type: nauc_map_at_10_std + value: 5.387253617793277 + - type: nauc_map_at_1_diff1 + value: 58.2120368710498 + - type: nauc_map_at_1_max + value: 27.847746672276042 + - type: nauc_map_at_1_std + value: 0.4123973289371852 + - type: nauc_map_at_20_diff1 + value: 53.57985635821413 + - type: nauc_map_at_20_max + value: 30.855245672321356 + - type: nauc_map_at_20_std + value: 5.899871547943132 + - type: nauc_map_at_3_diff1 + value: 54.44898012949102 + - type: nauc_map_at_3_max + value: 29.046842973909335 + - type: nauc_map_at_3_std + value: 3.525384201614754 + - type: nauc_map_at_5_diff1 + value: 53.85690116890884 + - type: nauc_map_at_5_max + value: 29.814219828226424 + - type: nauc_map_at_5_std + value: 4.702418731354826 + - type: nauc_mrr_at_1000_diff1 + value: 53.54318987010153 + - type: nauc_mrr_at_1000_max + value: 35.7577371258958 + - type: nauc_mrr_at_1000_std + value: 9.944626674145326 + - type: nauc_mrr_at_100_diff1 + value: 53.534287375399394 + - type: nauc_mrr_at_100_max + value: 35.7410660950445 + - type: nauc_mrr_at_100_std + value: 9.940292629068242 + - type: nauc_mrr_at_10_diff1 + value: 53.65256412924659 + - type: nauc_mrr_at_10_max + value: 35.727892006017086 + - type: nauc_mrr_at_10_std + value: 9.675769833118597 + - type: nauc_mrr_at_1_diff1 + value: 57.774977251447346 + - type: nauc_mrr_at_1_max + value: 38.32422292262853 + - type: nauc_mrr_at_1_std + value: 7.527792796951038 + - type: nauc_mrr_at_20_diff1 + value: 53.58172577427318 + - type: nauc_mrr_at_20_max + value: 35.70063356547134 + - type: nauc_mrr_at_20_std + value: 9.836439405079968 + - type: nauc_mrr_at_3_diff1 + value: 54.119421127324365 + - type: nauc_mrr_at_3_max + value: 36.188791759322065 + - type: nauc_mrr_at_3_std + value: 9.317423000852521 + - type: nauc_mrr_at_5_diff1 + value: 53.60354036689715 + - type: nauc_mrr_at_5_max + value: 35.88687876321913 + - type: nauc_mrr_at_5_std + value: 9.627845249694138 + - type: nauc_ndcg_at_1000_diff1 + value: 51.372556936792535 + - type: nauc_ndcg_at_1000_max + value: 32.595629954014505 + - type: nauc_ndcg_at_1000_std + value: 11.19788327225347 + - type: nauc_ndcg_at_100_diff1 + value: 51.19804034942206 + - type: nauc_ndcg_at_100_max + value: 31.876207149967396 + - type: nauc_ndcg_at_100_std + value: 10.734530072047418 + - type: nauc_ndcg_at_10_diff1 + value: 52.28823560783261 + - type: nauc_ndcg_at_10_max + value: 31.427090088357133 + - type: nauc_ndcg_at_10_std + value: 8.299594910583458 + - type: nauc_ndcg_at_1_diff1 + value: 57.774977251447346 + - type: nauc_ndcg_at_1_max + value: 38.32422292262853 + - type: nauc_ndcg_at_1_std + value: 7.527792796951038 + - type: nauc_ndcg_at_20_diff1 + value: 51.8717158000014 + - type: nauc_ndcg_at_20_max + value: 31.544028766384535 + - type: nauc_ndcg_at_20_std + value: 9.169876491018004 + - type: nauc_ndcg_at_3_diff1 + value: 52.37877478284309 + - type: nauc_ndcg_at_3_max + value: 32.74142482367942 + - type: nauc_ndcg_at_3_std + value: 7.042805092280206 + - type: nauc_ndcg_at_5_diff1 + value: 52.078182897086776 + - type: nauc_ndcg_at_5_max + value: 32.08226995182918 + - type: nauc_ndcg_at_5_std + value: 7.743370178150149 + - type: nauc_precision_at_1000_diff1 + value: -10.07643946633187 + - type: nauc_precision_at_1000_max + value: 18.750577972318993 + - type: nauc_precision_at_1000_std + value: 20.942704378017748 + - type: nauc_precision_at_100_diff1 + value: -2.0875217447578875 + - type: nauc_precision_at_100_max + value: 26.17224296959061 + - type: nauc_precision_at_100_std + value: 26.19862902456053 + - type: nauc_precision_at_10_diff1 + value: 12.991254729268743 + - type: nauc_precision_at_10_max + value: 30.383882238623922 + - type: nauc_precision_at_10_std + value: 19.88583596931918 + - type: nauc_precision_at_1_diff1 + value: 57.774977251447346 + - type: nauc_precision_at_1_max + value: 38.32422292262853 + - type: nauc_precision_at_1_std + value: 7.527792796951038 + - type: nauc_precision_at_20_diff1 + value: 5.747515623809232 + - type: nauc_precision_at_20_max + value: 29.45039755934858 + - type: nauc_precision_at_20_std + value: 23.16332592002045 + - type: nauc_precision_at_3_diff1 + value: 30.3123784938194 + - type: nauc_precision_at_3_max + value: 33.24762574785096 + - type: nauc_precision_at_3_std + value: 15.029708795650699 + - type: nauc_precision_at_5_diff1 + value: 21.71162166245634 + - type: nauc_precision_at_5_max + value: 32.782622946462965 + - type: nauc_precision_at_5_std + value: 18.308290529011693 + - type: nauc_recall_at_1000_diff1 + value: 29.957649999575885 + - type: nauc_recall_at_1000_max + value: 24.253210847263354 + - type: nauc_recall_at_1000_std + value: 47.373918259119314 + - type: nauc_recall_at_100_diff1 + value: 35.74046014841824 + - type: nauc_recall_at_100_max + value: 21.89583929709974 + - type: nauc_recall_at_100_std + value: 25.553247112667183 + - type: nauc_recall_at_10_diff1 + value: 45.21101064557515 + - type: nauc_recall_at_10_max + value: 22.87872828174405 + - type: nauc_recall_at_10_std + value: 9.46063127203539 + - type: nauc_recall_at_1_diff1 + value: 58.2120368710498 + - type: nauc_recall_at_1_max + value: 27.847746672276042 + - type: nauc_recall_at_1_std + value: 0.4123973289371852 + - type: nauc_recall_at_20_diff1 + value: 42.38239986216361 + - type: nauc_recall_at_20_max + value: 23.023522286353426 + - type: nauc_recall_at_20_std + value: 13.116387349072658 + - type: nauc_recall_at_3_diff1 + value: 49.08071204851084 + - type: nauc_recall_at_3_max + value: 25.145380870221718 + - type: nauc_recall_at_3_std + value: 5.009272584230173 + - type: nauc_recall_at_5_diff1 + value: 46.69875153919115 + - type: nauc_recall_at_5_max + value: 24.369082274282835 + - type: nauc_recall_at_5_std + value: 7.321739741670011 + - type: ndcg_at_1 + value: 41.528999999999996 + - type: ndcg_at_10 + value: 49.553000000000004 + - type: ndcg_at_100 + value: 53.49699999999999 + - type: ndcg_at_1000 + value: 55.538 + - type: ndcg_at_20 + value: 51.026 + - type: ndcg_at_3 + value: 45.349000000000004 + - type: ndcg_at_5 + value: 47.14 + - type: precision_at_1 + value: 41.528999999999996 + - type: precision_at_10 + value: 9.554 + - type: precision_at_100 + value: 1.4749999999999999 + - type: precision_at_1000 + value: 0.196 + - type: precision_at_20 + value: 5.545 + - type: precision_at_3 + value: 22.251 + - type: precision_at_5 + value: 15.706999999999999 + - type: recall_at_1 + value: 32.51 + - type: recall_at_10 + value: 59.78099999999999 + - type: recall_at_100 + value: 76.66 + - type: recall_at_1000 + value: 89.48899999999999 + - type: recall_at_20 + value: 65.226 + - type: recall_at_3 + value: 46.837 + - type: recall_at_5 + value: 52.042 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackGamingRetrieval + revision: 4885aa143210c98657558c04aaf3dc47cfb54340 + split: test + type: mteb/cqadupstack-gaming + metrics: + - type: main_score + value: 61.482000000000006 + - type: map_at_1 + value: 42.984 + - type: map_at_10 + value: 55.547000000000004 + - type: map_at_100 + value: 56.503 + - type: map_at_1000 + value: 56.555 + - type: map_at_20 + value: 56.094 + - type: map_at_3 + value: 52.006 + - type: map_at_5 + value: 54.029 + - type: mrr_at_1 + value: 49.46708463949843 + - type: mrr_at_10 + value: 59.17781260884719 + - type: mrr_at_100 + value: 59.750584002528285 + - type: mrr_at_1000 + value: 59.77686594272671 + - type: mrr_at_20 + value: 59.508073865685795 + - type: mrr_at_3 + value: 56.68756530825504 + - type: mrr_at_5 + value: 58.23928944618611 + - type: nauc_map_at_1000_diff1 + value: 52.314842283949815 + - type: nauc_map_at_1000_max + value: 37.63956784641385 + - type: nauc_map_at_1000_std + value: -3.824372006192471 + - type: nauc_map_at_100_diff1 + value: 52.28520655858172 + - type: nauc_map_at_100_max + value: 37.6182525826383 + - type: nauc_map_at_100_std + value: -3.8283816497118153 + - type: nauc_map_at_10_diff1 + value: 52.46043108685517 + - type: nauc_map_at_10_max + value: 37.16012835791356 + - type: nauc_map_at_10_std + value: -4.460007302392867 + - type: nauc_map_at_1_diff1 + value: 55.69200231728936 + - type: nauc_map_at_1_max + value: 31.372964149706405 + - type: nauc_map_at_1_std + value: -7.0713651554373484 + - type: nauc_map_at_20_diff1 + value: 52.36172331060843 + - type: nauc_map_at_20_max + value: 37.47156416438901 + - type: nauc_map_at_20_std + value: -4.0709430724796345 + - type: nauc_map_at_3_diff1 + value: 53.44182983230819 + - type: nauc_map_at_3_max + value: 36.64605766472148 + - type: nauc_map_at_3_std + value: -6.086835967931243 + - type: nauc_map_at_5_diff1 + value: 52.84132400204189 + - type: nauc_map_at_5_max + value: 36.593785570245366 + - type: nauc_map_at_5_std + value: -5.38747484767931 + - type: nauc_mrr_at_1000_diff1 + value: 51.22860794336571 + - type: nauc_mrr_at_1000_max + value: 38.52198438884325 + - type: nauc_mrr_at_1000_std + value: -3.59177256451736 + - type: nauc_mrr_at_100_diff1 + value: 51.213620616907406 + - type: nauc_mrr_at_100_max + value: 38.53134391043693 + - type: nauc_mrr_at_100_std + value: -3.582237940900262 + - type: nauc_mrr_at_10_diff1 + value: 51.09367921852942 + - type: nauc_mrr_at_10_max + value: 38.37888478090058 + - type: nauc_mrr_at_10_std + value: -3.6931789780306694 + - type: nauc_mrr_at_1_diff1 + value: 54.564324471744875 + - type: nauc_mrr_at_1_max + value: 37.34896714077177 + - type: nauc_mrr_at_1_std + value: -5.633720379384613 + - type: nauc_mrr_at_20_diff1 + value: 51.18587010115815 + - type: nauc_mrr_at_20_max + value: 38.539226695044036 + - type: nauc_mrr_at_20_std + value: -3.5723180309182583 + - type: nauc_mrr_at_3_diff1 + value: 51.40138188217392 + - type: nauc_mrr_at_3_max + value: 38.45429931100304 + - type: nauc_mrr_at_3_std + value: -4.627389372621268 + - type: nauc_mrr_at_5_diff1 + value: 51.16972709196802 + - type: nauc_mrr_at_5_max + value: 38.257331089023694 + - type: nauc_mrr_at_5_std + value: -4.160349719834441 + - type: nauc_ndcg_at_1000_diff1 + value: 51.17507832690963 + - type: nauc_ndcg_at_1000_max + value: 39.07065049927589 + - type: nauc_ndcg_at_1000_std + value: -1.3625584227069216 + - type: nauc_ndcg_at_100_diff1 + value: 50.715843467416036 + - type: nauc_ndcg_at_100_max + value: 39.14568870815261 + - type: nauc_ndcg_at_100_std + value: -0.9531436891792046 + - type: nauc_ndcg_at_10_diff1 + value: 50.626988488110094 + - type: nauc_ndcg_at_10_max + value: 38.03522123739489 + - type: nauc_ndcg_at_10_std + value: -2.109858643466042 + - type: nauc_ndcg_at_1_diff1 + value: 54.564324471744875 + - type: nauc_ndcg_at_1_max + value: 37.34896714077177 + - type: nauc_ndcg_at_1_std + value: -5.633720379384613 + - type: nauc_ndcg_at_20_diff1 + value: 50.61641057331065 + - type: nauc_ndcg_at_20_max + value: 38.63468305031056 + - type: nauc_ndcg_at_20_std + value: -1.5436822489115078 + - type: nauc_ndcg_at_3_diff1 + value: 51.943117007546135 + - type: nauc_ndcg_at_3_max + value: 37.79874879746648 + - type: nauc_ndcg_at_3_std + value: -4.711417057142626 + - type: nauc_ndcg_at_5_diff1 + value: 51.22149770995894 + - type: nauc_ndcg_at_5_max + value: 37.31345620274675 + - type: nauc_ndcg_at_5_std + value: -3.852735749981374 + - type: nauc_precision_at_1000_diff1 + value: -15.394599072903759 + - type: nauc_precision_at_1000_max + value: 15.4007811167538 + - type: nauc_precision_at_1000_std + value: 20.513393475858653 + - type: nauc_precision_at_100_diff1 + value: -12.191648425138062 + - type: nauc_precision_at_100_max + value: 18.590980227479317 + - type: nauc_precision_at_100_std + value: 21.347960429888968 + - type: nauc_precision_at_10_diff1 + value: 6.702604624515554 + - type: nauc_precision_at_10_max + value: 27.008700778787027 + - type: nauc_precision_at_10_std + value: 13.020115923291453 + - type: nauc_precision_at_1_diff1 + value: 54.564324471744875 + - type: nauc_precision_at_1_max + value: 37.34896714077177 + - type: nauc_precision_at_1_std + value: -5.633720379384613 + - type: nauc_precision_at_20_diff1 + value: -0.8585421908503704 + - type: nauc_precision_at_20_max + value: 24.371781222149117 + - type: nauc_precision_at_20_std + value: 16.803662129166703 + - type: nauc_precision_at_3_diff1 + value: 29.873656502152418 + - type: nauc_precision_at_3_max + value: 37.4137425915249 + - type: nauc_precision_at_3_std + value: 2.031215687506434 + - type: nauc_precision_at_5_diff1 + value: 18.72136831168126 + - type: nauc_precision_at_5_max + value: 31.07228793623305 + - type: nauc_precision_at_5_std + value: 5.707077671480514 + - type: nauc_recall_at_1000_diff1 + value: 51.558197415750875 + - type: nauc_recall_at_1000_max + value: 62.91600739331539 + - type: nauc_recall_at_1000_std + value: 65.44061798991277 + - type: nauc_recall_at_100_diff1 + value: 39.75335825100584 + - type: nauc_recall_at_100_max + value: 47.345197875987836 + - type: nauc_recall_at_100_std + value: 26.644988235991995 + - type: nauc_recall_at_10_diff1 + value: 43.145318028199824 + - type: nauc_recall_at_10_max + value: 35.990174276389475 + - type: nauc_recall_at_10_std + value: 5.287013278975453 + - type: nauc_recall_at_1_diff1 + value: 55.69200231728936 + - type: nauc_recall_at_1_max + value: 31.372964149706405 + - type: nauc_recall_at_1_std + value: -7.0713651554373484 + - type: nauc_recall_at_20_diff1 + value: 41.86522952137498 + - type: nauc_recall_at_20_max + value: 39.468049246546414 + - type: nauc_recall_at_20_std + value: 10.523046173739804 + - type: nauc_recall_at_3_diff1 + value: 48.734111878635815 + - type: nauc_recall_at_3_max + value: 35.803450080013775 + - type: nauc_recall_at_3_std + value: -4.845734876173088 + - type: nauc_recall_at_5_diff1 + value: 45.918448807422415 + - type: nauc_recall_at_5_max + value: 34.30505608137996 + - type: nauc_recall_at_5_std + value: -1.6514368945560158 + - type: ndcg_at_1 + value: 49.467 + - type: ndcg_at_10 + value: 61.482000000000006 + - type: ndcg_at_100 + value: 65.148 + - type: ndcg_at_1000 + value: 66.11 + - type: ndcg_at_20 + value: 62.965 + - type: ndcg_at_3 + value: 55.75599999999999 + - type: ndcg_at_5 + value: 58.648999999999994 + - type: precision_at_1 + value: 49.467 + - type: precision_at_10 + value: 9.905999999999999 + - type: precision_at_100 + value: 1.264 + - type: precision_at_1000 + value: 0.13799999999999998 + - type: precision_at_20 + value: 5.423 + - type: precision_at_3 + value: 24.807000000000002 + - type: precision_at_5 + value: 17.091 + - type: recall_at_1 + value: 42.984 + - type: recall_at_10 + value: 75.26899999999999 + - type: recall_at_100 + value: 91.146 + - type: recall_at_1000 + value: 97.87599999999999 + - type: recall_at_20 + value: 80.807 + - type: recall_at_3 + value: 60.007999999999996 + - type: recall_at_5 + value: 67.14 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackGisRetrieval + revision: 5003b3064772da1887988e05400cf3806fe491f2 + split: test + type: mteb/cqadupstack-gis + metrics: + - type: main_score + value: 41.119 + - type: map_at_1 + value: 26.093 + - type: map_at_10 + value: 35.661 + - type: map_at_100 + value: 36.77 + - type: map_at_1000 + value: 36.847 + - type: map_at_20 + value: 36.266999999999996 + - type: map_at_3 + value: 32.719 + - type: map_at_5 + value: 34.458 + - type: mrr_at_1 + value: 28.474576271186443 + - type: mrr_at_10 + value: 37.91579230562277 + - type: mrr_at_100 + value: 38.843502843271146 + - type: mrr_at_1000 + value: 38.89590111801632 + - type: mrr_at_20 + value: 38.437384724483906 + - type: mrr_at_3 + value: 35.084745762711854 + - type: mrr_at_5 + value: 36.77966101694913 + - type: nauc_map_at_1000_diff1 + value: 38.42004646151798 + - type: nauc_map_at_1000_max + value: 24.55070631824853 + - type: nauc_map_at_1000_std + value: -0.3504784017542892 + - type: nauc_map_at_100_diff1 + value: 38.39678049708717 + - type: nauc_map_at_100_max + value: 24.544444813328823 + - type: nauc_map_at_100_std + value: -0.35299091693706613 + - type: nauc_map_at_10_diff1 + value: 38.54685907537762 + - type: nauc_map_at_10_max + value: 24.330052501473148 + - type: nauc_map_at_10_std + value: -0.6012082597130156 + - type: nauc_map_at_1_diff1 + value: 44.51875722625461 + - type: nauc_map_at_1_max + value: 23.272125285988235 + - type: nauc_map_at_1_std + value: -4.148940843666968 + - type: nauc_map_at_20_diff1 + value: 38.43155304512796 + - type: nauc_map_at_20_max + value: 24.4940679088991 + - type: nauc_map_at_20_std + value: -0.46203150906252805 + - type: nauc_map_at_3_diff1 + value: 39.57417230011771 + - type: nauc_map_at_3_max + value: 23.47397251626197 + - type: nauc_map_at_3_std + value: -1.8815758174772557 + - type: nauc_map_at_5_diff1 + value: 38.87284926527319 + - type: nauc_map_at_5_max + value: 24.14613777288936 + - type: nauc_map_at_5_std + value: -1.528959975262841 + - type: nauc_mrr_at_1000_diff1 + value: 37.32796203185405 + - type: nauc_mrr_at_1000_max + value: 25.988613521850358 + - type: nauc_mrr_at_1000_std + value: -0.07900066966767322 + - type: nauc_mrr_at_100_diff1 + value: 37.29234843812534 + - type: nauc_mrr_at_100_max + value: 25.99372555718925 + - type: nauc_mrr_at_100_std + value: -0.06884301339938263 + - type: nauc_mrr_at_10_diff1 + value: 37.52646273988614 + - type: nauc_mrr_at_10_max + value: 25.895420253660433 + - type: nauc_mrr_at_10_std + value: -0.16790004214456672 + - type: nauc_mrr_at_1_diff1 + value: 42.69686182822761 + - type: nauc_mrr_at_1_max + value: 26.050165605491742 + - type: nauc_mrr_at_1_std + value: -3.2029081347234265 + - type: nauc_mrr_at_20_diff1 + value: 37.27883648577863 + - type: nauc_mrr_at_20_max + value: 25.976584491011074 + - type: nauc_mrr_at_20_std + value: -0.112059943717117 + - type: nauc_mrr_at_3_diff1 + value: 38.489341099350284 + - type: nauc_mrr_at_3_max + value: 25.62245902436551 + - type: nauc_mrr_at_3_std + value: -1.1084899794154743 + - type: nauc_mrr_at_5_diff1 + value: 37.77953502953197 + - type: nauc_mrr_at_5_max + value: 25.766145481628612 + - type: nauc_mrr_at_5_std + value: -0.7784473182662061 + - type: nauc_ndcg_at_1000_diff1 + value: 35.71935794006166 + - type: nauc_ndcg_at_1000_max + value: 25.488881881457743 + - type: nauc_ndcg_at_1000_std + value: 1.7711090863506873 + - type: nauc_ndcg_at_100_diff1 + value: 34.996988207102106 + - type: nauc_ndcg_at_100_max + value: 25.505354534065354 + - type: nauc_ndcg_at_100_std + value: 2.061525857156352 + - type: nauc_ndcg_at_10_diff1 + value: 35.923551649851255 + - type: nauc_ndcg_at_10_max + value: 24.769126683216662 + - type: nauc_ndcg_at_10_std + value: 1.3289815466801196 + - type: nauc_ndcg_at_1_diff1 + value: 42.69686182822761 + - type: nauc_ndcg_at_1_max + value: 26.050165605491742 + - type: nauc_ndcg_at_1_std + value: -3.2029081347234265 + - type: nauc_ndcg_at_20_diff1 + value: 35.23510291106356 + - type: nauc_ndcg_at_20_max + value: 25.235372251979605 + - type: nauc_ndcg_at_20_std + value: 1.6154316053586737 + - type: nauc_ndcg_at_3_diff1 + value: 38.0131047498137 + - type: nauc_ndcg_at_3_max + value: 23.91711550019696 + - type: nauc_ndcg_at_3_std + value: -1.2380678434693104 + - type: nauc_ndcg_at_5_diff1 + value: 36.82060817797453 + - type: nauc_ndcg_at_5_max + value: 24.463714963600854 + - type: nauc_ndcg_at_5_std + value: -0.7287713654086785 + - type: nauc_precision_at_1000_diff1 + value: -8.943912359532389 + - type: nauc_precision_at_1000_max + value: 9.926329101642303 + - type: nauc_precision_at_1000_std + value: 13.133631111469878 + - type: nauc_precision_at_100_diff1 + value: 1.2108361444877151 + - type: nauc_precision_at_100_max + value: 19.51234562265741 + - type: nauc_precision_at_100_std + value: 14.013168984699531 + - type: nauc_precision_at_10_diff1 + value: 20.824091581134507 + - type: nauc_precision_at_10_max + value: 25.830773262834416 + - type: nauc_precision_at_10_std + value: 11.42086226866109 + - type: nauc_precision_at_1_diff1 + value: 42.69686182822761 + - type: nauc_precision_at_1_max + value: 26.050165605491742 + - type: nauc_precision_at_1_std + value: -3.2029081347234265 + - type: nauc_precision_at_20_diff1 + value: 15.306310531056377 + - type: nauc_precision_at_20_max + value: 25.90125011294581 + - type: nauc_precision_at_20_std + value: 13.11125029318942 + - type: nauc_precision_at_3_diff1 + value: 30.284445671371945 + - type: nauc_precision_at_3_max + value: 25.83731378670555 + - type: nauc_precision_at_3_std + value: 2.897645699098519 + - type: nauc_precision_at_5_diff1 + value: 25.7981575302161 + - type: nauc_precision_at_5_max + value: 26.187388553843316 + - type: nauc_precision_at_5_std + value: 4.226853440660339 + - type: nauc_recall_at_1000_diff1 + value: 9.078599994381939 + - type: nauc_recall_at_1000_max + value: 32.56232099511339 + - type: nauc_recall_at_1000_std + value: 25.309709495982734 + - type: nauc_recall_at_100_diff1 + value: 17.075888968477617 + - type: nauc_recall_at_100_max + value: 28.095249414650198 + - type: nauc_recall_at_100_std + value: 14.29668213569113 + - type: nauc_recall_at_10_diff1 + value: 27.361710883346568 + - type: nauc_recall_at_10_max + value: 23.55485165860555 + - type: nauc_recall_at_10_std + value: 6.574662384536404 + - type: nauc_recall_at_1_diff1 + value: 44.51875722625461 + - type: nauc_recall_at_1_max + value: 23.272125285988235 + - type: nauc_recall_at_1_std + value: -4.148940843666968 + - type: nauc_recall_at_20_diff1 + value: 23.2567937245563 + - type: nauc_recall_at_20_max + value: 25.121587399929197 + - type: nauc_recall_at_20_std + value: 8.01934607194124 + - type: nauc_recall_at_3_diff1 + value: 33.7684886988769 + - type: nauc_recall_at_3_max + value: 21.473481929333797 + - type: nauc_recall_at_3_std + value: -0.5300775162871144 + - type: nauc_recall_at_5_diff1 + value: 30.943683335367076 + - type: nauc_recall_at_5_max + value: 23.07703627332008 + - type: nauc_recall_at_5_std + value: 0.4693000530683178 + - type: ndcg_at_1 + value: 28.475 + - type: ndcg_at_10 + value: 41.119 + - type: ndcg_at_100 + value: 46.405 + - type: ndcg_at_1000 + value: 48.301 + - type: ndcg_at_20 + value: 43.206 + - type: ndcg_at_3 + value: 35.42 + - type: ndcg_at_5 + value: 38.349 + - type: precision_at_1 + value: 28.475 + - type: precision_at_10 + value: 6.429 + - type: precision_at_100 + value: 0.9560000000000001 + - type: precision_at_1000 + value: 0.116 + - type: precision_at_20 + value: 3.712 + - type: precision_at_3 + value: 15.140999999999998 + - type: precision_at_5 + value: 10.802 + - type: recall_at_1 + value: 26.093 + - type: recall_at_10 + value: 55.718 + - type: recall_at_100 + value: 79.56299999999999 + - type: recall_at_1000 + value: 93.815 + - type: recall_at_20 + value: 63.629000000000005 + - type: recall_at_3 + value: 40.483999999999995 + - type: recall_at_5 + value: 47.475 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackMathematicaRetrieval + revision: 90fceea13679c63fe563ded68f3b6f06e50061de + split: test + type: mteb/cqadupstack-mathematica + metrics: + - type: main_score + value: 32.015 + - type: map_at_1 + value: 17.044 + - type: map_at_10 + value: 26.253999999999998 + - type: map_at_100 + value: 27.589000000000002 + - type: map_at_1000 + value: 27.714 + - type: map_at_20 + value: 26.931 + - type: map_at_3 + value: 23.321 + - type: map_at_5 + value: 24.929000000000002 + - type: mrr_at_1 + value: 21.144278606965177 + - type: mrr_at_10 + value: 30.78427307904919 + - type: mrr_at_100 + value: 31.7862255263549 + - type: mrr_at_1000 + value: 31.849990331440765 + - type: mrr_at_20 + value: 31.321638074777375 + - type: mrr_at_3 + value: 27.92288557213932 + - type: mrr_at_5 + value: 29.633084577114438 + - type: nauc_map_at_1000_diff1 + value: 34.57225445729808 + - type: nauc_map_at_1000_max + value: 10.66145970457799 + - type: nauc_map_at_1000_std + value: 7.0809881413798585 + - type: nauc_map_at_100_diff1 + value: 34.53504572959364 + - type: nauc_map_at_100_max + value: 10.605393967602557 + - type: nauc_map_at_100_std + value: 7.079006904917323 + - type: nauc_map_at_10_diff1 + value: 34.71241351032732 + - type: nauc_map_at_10_max + value: 10.658551571500164 + - type: nauc_map_at_10_std + value: 6.616094717921069 + - type: nauc_map_at_1_diff1 + value: 40.03056900966001 + - type: nauc_map_at_1_max + value: 9.270626026654664 + - type: nauc_map_at_1_std + value: 6.4401895045344855 + - type: nauc_map_at_20_diff1 + value: 34.58339109714446 + - type: nauc_map_at_20_max + value: 10.60633819186475 + - type: nauc_map_at_20_std + value: 6.724527519182256 + - type: nauc_map_at_3_diff1 + value: 35.913629313185695 + - type: nauc_map_at_3_max + value: 9.458438444112614 + - type: nauc_map_at_3_std + value: 6.60674965667657 + - type: nauc_map_at_5_diff1 + value: 34.830830182795005 + - type: nauc_map_at_5_max + value: 10.440388055223321 + - type: nauc_map_at_5_std + value: 5.802754153009861 + - type: nauc_mrr_at_1000_diff1 + value: 35.25228940968282 + - type: nauc_mrr_at_1000_max + value: 13.694077462160168 + - type: nauc_mrr_at_1000_std + value: 5.591486832792639 + - type: nauc_mrr_at_100_diff1 + value: 35.23589347965209 + - type: nauc_mrr_at_100_max + value: 13.684262696205659 + - type: nauc_mrr_at_100_std + value: 5.580183691998299 + - type: nauc_mrr_at_10_diff1 + value: 35.43718377777631 + - type: nauc_mrr_at_10_max + value: 13.864605660657137 + - type: nauc_mrr_at_10_std + value: 5.528487657733224 + - type: nauc_mrr_at_1_diff1 + value: 40.18797805533981 + - type: nauc_mrr_at_1_max + value: 12.524219192047385 + - type: nauc_mrr_at_1_std + value: 4.129057810670297 + - type: nauc_mrr_at_20_diff1 + value: 35.23331124375955 + - type: nauc_mrr_at_20_max + value: 13.730879245177494 + - type: nauc_mrr_at_20_std + value: 5.477929971733812 + - type: nauc_mrr_at_3_diff1 + value: 35.8446184010843 + - type: nauc_mrr_at_3_max + value: 13.049767486625374 + - type: nauc_mrr_at_3_std + value: 5.367274422620354 + - type: nauc_mrr_at_5_diff1 + value: 35.189721084914254 + - type: nauc_mrr_at_5_max + value: 13.787169162028457 + - type: nauc_mrr_at_5_std + value: 4.83688435784119 + - type: nauc_ndcg_at_1000_diff1 + value: 32.74934849290088 + - type: nauc_ndcg_at_1000_max + value: 12.63220195288599 + - type: nauc_ndcg_at_1000_std + value: 9.41404553916723 + - type: nauc_ndcg_at_100_diff1 + value: 32.097329725003654 + - type: nauc_ndcg_at_100_max + value: 11.286376050496692 + - type: nauc_ndcg_at_100_std + value: 9.673407201526521 + - type: nauc_ndcg_at_10_diff1 + value: 33.07063223608012 + - type: nauc_ndcg_at_10_max + value: 12.082128040981049 + - type: nauc_ndcg_at_10_std + value: 7.562639224641771 + - type: nauc_ndcg_at_1_diff1 + value: 40.18797805533981 + - type: nauc_ndcg_at_1_max + value: 12.524219192047385 + - type: nauc_ndcg_at_1_std + value: 4.129057810670297 + - type: nauc_ndcg_at_20_diff1 + value: 32.564254816910285 + - type: nauc_ndcg_at_20_max + value: 11.73076439229411 + - type: nauc_ndcg_at_20_std + value: 7.79318294813014 + - type: nauc_ndcg_at_3_diff1 + value: 35.01457038147314 + - type: nauc_ndcg_at_3_max + value: 10.72848160762736 + - type: nauc_ndcg_at_3_std + value: 6.157231570866382 + - type: nauc_ndcg_at_5_diff1 + value: 33.20816487121954 + - type: nauc_ndcg_at_5_max + value: 11.851644260190005 + - type: nauc_ndcg_at_5_std + value: 5.387773386577467 + - type: nauc_precision_at_1000_diff1 + value: 2.8372544162006936 + - type: nauc_precision_at_1000_max + value: 10.858462734068324 + - type: nauc_precision_at_1000_std + value: -0.24555720407002804 + - type: nauc_precision_at_100_diff1 + value: 7.128712637235406 + - type: nauc_precision_at_100_max + value: 9.073365664295839 + - type: nauc_precision_at_100_std + value: 9.230296295017718 + - type: nauc_precision_at_10_diff1 + value: 20.834933601227526 + - type: nauc_precision_at_10_max + value: 15.691671464023486 + - type: nauc_precision_at_10_std + value: 7.150053993682323 + - type: nauc_precision_at_1_diff1 + value: 40.18797805533981 + - type: nauc_precision_at_1_max + value: 12.524219192047385 + - type: nauc_precision_at_1_std + value: 4.129057810670297 + - type: nauc_precision_at_20_diff1 + value: 16.23013203360854 + - type: nauc_precision_at_20_max + value: 13.426062435114744 + - type: nauc_precision_at_20_std + value: 7.469697094042914 + - type: nauc_precision_at_3_diff1 + value: 30.205885104799478 + - type: nauc_precision_at_3_max + value: 12.948592630950785 + - type: nauc_precision_at_3_std + value: 6.088037170177383 + - type: nauc_precision_at_5_diff1 + value: 25.848216806599055 + - type: nauc_precision_at_5_max + value: 16.380415265106556 + - type: nauc_precision_at_5_std + value: 3.1573546153448775 + - type: nauc_recall_at_1000_diff1 + value: 7.257846626852283 + - type: nauc_recall_at_1000_max + value: 29.09412574133125 + - type: nauc_recall_at_1000_std + value: 48.4425501760214 + - type: nauc_recall_at_100_diff1 + value: 17.664452583241257 + - type: nauc_recall_at_100_max + value: 5.7055534880151 + - type: nauc_recall_at_100_std + value: 23.335585558939478 + - type: nauc_recall_at_10_diff1 + value: 26.410894513625216 + - type: nauc_recall_at_10_max + value: 11.947422949559943 + - type: nauc_recall_at_10_std + value: 10.002102681154467 + - type: nauc_recall_at_1_diff1 + value: 40.03056900966001 + - type: nauc_recall_at_1_max + value: 9.270626026654664 + - type: nauc_recall_at_1_std + value: 6.4401895045344855 + - type: nauc_recall_at_20_diff1 + value: 24.191303161320207 + - type: nauc_recall_at_20_max + value: 10.461312380813835 + - type: nauc_recall_at_20_std + value: 10.49749517692461 + - type: nauc_recall_at_3_diff1 + value: 30.298367653230386 + - type: nauc_recall_at_3_max + value: 8.142827258414421 + - type: nauc_recall_at_3_std + value: 7.943180287513221 + - type: nauc_recall_at_5_diff1 + value: 26.6334429954341 + - type: nauc_recall_at_5_max + value: 11.145603106344137 + - type: nauc_recall_at_5_std + value: 4.968015089323 + - type: ndcg_at_1 + value: 21.144 + - type: ndcg_at_10 + value: 32.015 + - type: ndcg_at_100 + value: 38.369 + - type: ndcg_at_1000 + value: 41.102 + - type: ndcg_at_20 + value: 34.21 + - type: ndcg_at_3 + value: 26.623 + - type: ndcg_at_5 + value: 29.176999999999996 + - type: precision_at_1 + value: 21.144 + - type: precision_at_10 + value: 6.231 + - type: precision_at_100 + value: 1.085 + - type: precision_at_1000 + value: 0.145 + - type: precision_at_20 + value: 3.7310000000000003 + - type: precision_at_3 + value: 13.059999999999999 + - type: precision_at_5 + value: 9.677 + - type: recall_at_1 + value: 17.044 + - type: recall_at_10 + value: 44.874 + - type: recall_at_100 + value: 73.0 + - type: recall_at_1000 + value: 92.193 + - type: recall_at_20 + value: 52.686 + - type: recall_at_3 + value: 30.422 + - type: recall_at_5 + value: 36.811 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackPhysicsRetrieval + revision: 79531abbd1fb92d06c6d6315a0cbbbf5bb247ea4 + split: test + type: mteb/cqadupstack-physics + metrics: + - type: main_score + value: 50.158 + - type: map_at_1 + value: 30.529 + - type: map_at_10 + value: 43.269999999999996 + - type: map_at_100 + value: 44.739000000000004 + - type: map_at_1000 + value: 44.830999999999996 + - type: map_at_20 + value: 44.155 + - type: map_at_3 + value: 39.37 + - type: map_at_5 + value: 41.446 + - type: mrr_at_1 + value: 38.69104908565929 + - type: mrr_at_10 + value: 49.29033564019124 + - type: mrr_at_100 + value: 50.16126024892191 + - type: mrr_at_1000 + value: 50.18374960814055 + - type: mrr_at_20 + value: 49.87234825950972 + - type: mrr_at_3 + value: 46.56721206288094 + - type: mrr_at_5 + value: 48.01572024382413 + - type: nauc_map_at_1000_diff1 + value: 48.575670612926736 + - type: nauc_map_at_1000_max + value: 31.82036593180807 + - type: nauc_map_at_1000_std + value: 3.369865718086696 + - type: nauc_map_at_100_diff1 + value: 48.5611123427273 + - type: nauc_map_at_100_max + value: 31.75709781237859 + - type: nauc_map_at_100_std + value: 3.321864938430637 + - type: nauc_map_at_10_diff1 + value: 48.4543403210739 + - type: nauc_map_at_10_max + value: 31.32852656830552 + - type: nauc_map_at_10_std + value: 2.506314295812317 + - type: nauc_map_at_1_diff1 + value: 54.841891793012145 + - type: nauc_map_at_1_max + value: 31.230149664353835 + - type: nauc_map_at_1_std + value: -2.221705314009157 + - type: nauc_map_at_20_diff1 + value: 48.46228321125242 + - type: nauc_map_at_20_max + value: 31.55683955582787 + - type: nauc_map_at_20_std + value: 3.2177933233709073 + - type: nauc_map_at_3_diff1 + value: 48.8438551764958 + - type: nauc_map_at_3_max + value: 30.68091164257478 + - type: nauc_map_at_3_std + value: 1.6111603105331231 + - type: nauc_map_at_5_diff1 + value: 48.4263676267222 + - type: nauc_map_at_5_max + value: 30.770879706742416 + - type: nauc_map_at_5_std + value: 2.4742797888673818 + - type: nauc_mrr_at_1000_diff1 + value: 47.55290657468302 + - type: nauc_mrr_at_1000_max + value: 33.67362321793355 + - type: nauc_mrr_at_1000_std + value: 3.7911583989878914 + - type: nauc_mrr_at_100_diff1 + value: 47.543795741247294 + - type: nauc_mrr_at_100_max + value: 33.66375805916915 + - type: nauc_mrr_at_100_std + value: 3.8030115590090348 + - type: nauc_mrr_at_10_diff1 + value: 47.23768889356714 + - type: nauc_mrr_at_10_max + value: 33.43570404075205 + - type: nauc_mrr_at_10_std + value: 3.558389110580555 + - type: nauc_mrr_at_1_diff1 + value: 52.44419091092719 + - type: nauc_mrr_at_1_max + value: 35.30830914409924 + - type: nauc_mrr_at_1_std + value: 0.689803902465474 + - type: nauc_mrr_at_20_diff1 + value: 47.41985223328362 + - type: nauc_mrr_at_20_max + value: 33.63792808264595 + - type: nauc_mrr_at_20_std + value: 3.859294417245836 + - type: nauc_mrr_at_3_diff1 + value: 47.633013243942436 + - type: nauc_mrr_at_3_max + value: 33.44779870404666 + - type: nauc_mrr_at_3_std + value: 3.230568846028269 + - type: nauc_mrr_at_5_diff1 + value: 47.46701148305837 + - type: nauc_mrr_at_5_max + value: 33.22226684417501 + - type: nauc_mrr_at_5_std + value: 3.4734248804804526 + - type: nauc_ndcg_at_1000_diff1 + value: 47.43557285271372 + - type: nauc_ndcg_at_1000_max + value: 32.98714031630084 + - type: nauc_ndcg_at_1000_std + value: 5.855925940817803 + - type: nauc_ndcg_at_100_diff1 + value: 47.102075784902624 + - type: nauc_ndcg_at_100_max + value: 32.32322975437478 + - type: nauc_ndcg_at_100_std + value: 6.001483927743542 + - type: nauc_ndcg_at_10_diff1 + value: 46.145593327588955 + - type: nauc_ndcg_at_10_max + value: 31.078552401475708 + - type: nauc_ndcg_at_10_std + value: 3.842442704226773 + - type: nauc_ndcg_at_1_diff1 + value: 52.44419091092719 + - type: nauc_ndcg_at_1_max + value: 35.30830914409924 + - type: nauc_ndcg_at_1_std + value: 0.689803902465474 + - type: nauc_ndcg_at_20_diff1 + value: 46.42845689007039 + - type: nauc_ndcg_at_20_max + value: 31.558401932719782 + - type: nauc_ndcg_at_20_std + value: 5.732285001077974 + - type: nauc_ndcg_at_3_diff1 + value: 46.70360115824387 + - type: nauc_ndcg_at_3_max + value: 31.67975186201027 + - type: nauc_ndcg_at_3_std + value: 3.720641115941153 + - type: nauc_ndcg_at_5_diff1 + value: 46.25987714133794 + - type: nauc_ndcg_at_5_max + value: 30.702659077798916 + - type: nauc_ndcg_at_5_std + value: 4.283836239794501 + - type: nauc_precision_at_1000_diff1 + value: -10.587724242959737 + - type: nauc_precision_at_1000_max + value: 6.604170682826986 + - type: nauc_precision_at_1000_std + value: 13.892706075528869 + - type: nauc_precision_at_100_diff1 + value: -5.3168989874515615 + - type: nauc_precision_at_100_max + value: 11.1488025056379 + - type: nauc_precision_at_100_std + value: 16.636084116690697 + - type: nauc_precision_at_10_diff1 + value: 12.108498751088915 + - type: nauc_precision_at_10_max + value: 22.036386721990468 + - type: nauc_precision_at_10_std + value: 11.667178206324394 + - type: nauc_precision_at_1_diff1 + value: 52.44419091092719 + - type: nauc_precision_at_1_max + value: 35.30830914409924 + - type: nauc_precision_at_1_std + value: 0.689803902465474 + - type: nauc_precision_at_20_diff1 + value: 5.123874774732612 + - type: nauc_precision_at_20_max + value: 17.37860606386887 + - type: nauc_precision_at_20_std + value: 16.532726222738358 + - type: nauc_precision_at_3_diff1 + value: 27.863832415801305 + - type: nauc_precision_at_3_max + value: 27.568337770887286 + - type: nauc_precision_at_3_std + value: 9.3221947160408 + - type: nauc_precision_at_5_diff1 + value: 19.874842708314482 + - type: nauc_precision_at_5_max + value: 23.830093498481794 + - type: nauc_precision_at_5_std + value: 11.792743038682273 + - type: nauc_recall_at_1000_diff1 + value: 40.71312926275486 + - type: nauc_recall_at_1000_max + value: 42.8985177949371 + - type: nauc_recall_at_1000_std + value: 36.566555425851284 + - type: nauc_recall_at_100_diff1 + value: 35.04182139191391 + - type: nauc_recall_at_100_max + value: 25.08963759436396 + - type: nauc_recall_at_100_std + value: 22.393101461796046 + - type: nauc_recall_at_10_diff1 + value: 36.68683208653224 + - type: nauc_recall_at_10_max + value: 24.840225268252837 + - type: nauc_recall_at_10_std + value: 6.421070179932187 + - type: nauc_recall_at_1_diff1 + value: 54.841891793012145 + - type: nauc_recall_at_1_max + value: 31.230149664353835 + - type: nauc_recall_at_1_std + value: -2.221705314009157 + - type: nauc_recall_at_20_diff1 + value: 35.69983469273023 + - type: nauc_recall_at_20_max + value: 25.16182934858501 + - type: nauc_recall_at_20_std + value: 15.030622978372302 + - type: nauc_recall_at_3_diff1 + value: 41.238001408076514 + - type: nauc_recall_at_3_max + value: 26.478921555909285 + - type: nauc_recall_at_3_std + value: 3.6458401270022422 + - type: nauc_recall_at_5_diff1 + value: 39.028304422239806 + - type: nauc_recall_at_5_max + value: 24.611319533297525 + - type: nauc_recall_at_5_std + value: 5.984705005755768 + - type: ndcg_at_1 + value: 38.690999999999995 + - type: ndcg_at_10 + value: 50.158 + - type: ndcg_at_100 + value: 55.784 + - type: ndcg_at_1000 + value: 57.249 + - type: ndcg_at_20 + value: 52.708999999999996 + - type: ndcg_at_3 + value: 44.219 + - type: ndcg_at_5 + value: 46.808 + - type: precision_at_1 + value: 38.690999999999995 + - type: precision_at_10 + value: 9.538 + - type: precision_at_100 + value: 1.429 + - type: precision_at_1000 + value: 0.174 + - type: precision_at_20 + value: 5.621 + - type: precision_at_3 + value: 21.816 + - type: precision_at_5 + value: 15.553 + - type: recall_at_1 + value: 30.529 + - type: recall_at_10 + value: 64.557 + - type: recall_at_100 + value: 87.895 + - type: recall_at_1000 + value: 96.922 + - type: recall_at_20 + value: 73.547 + - type: recall_at_3 + value: 47.542 + - type: recall_at_5 + value: 54.356 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackProgrammersRetrieval + revision: 6184bc1440d2dbc7612be22b50686b8826d22b32 + split: test + type: mteb/cqadupstack-programmers + metrics: + - type: main_score + value: 47.400999999999996 + - type: map_at_1 + value: 29.217 + - type: map_at_10 + value: 40.981 + - type: map_at_100 + value: 42.498000000000005 + - type: map_at_1000 + value: 42.596000000000004 + - type: map_at_20 + value: 41.927 + - type: map_at_3 + value: 37.408 + - type: map_at_5 + value: 39.395 + - type: mrr_at_1 + value: 36.64383561643836 + - type: mrr_at_10 + value: 46.7149108501848 + - type: mrr_at_100 + value: 47.63430578999128 + - type: mrr_at_1000 + value: 47.68124555514414 + - type: mrr_at_20 + value: 47.32335630087596 + - type: mrr_at_3 + value: 44.121004566210026 + - type: mrr_at_5 + value: 45.59931506849312 + - type: nauc_map_at_1000_diff1 + value: 47.636248312030254 + - type: nauc_map_at_1000_max + value: 41.347235458973906 + - type: nauc_map_at_1000_std + value: 7.678119443869429 + - type: nauc_map_at_100_diff1 + value: 47.60961829666386 + - type: nauc_map_at_100_max + value: 41.3491851268752 + - type: nauc_map_at_100_std + value: 7.718686741871067 + - type: nauc_map_at_10_diff1 + value: 47.55528726733137 + - type: nauc_map_at_10_max + value: 40.803624204426754 + - type: nauc_map_at_10_std + value: 6.877917986199969 + - type: nauc_map_at_1_diff1 + value: 51.50668476702385 + - type: nauc_map_at_1_max + value: 37.74817918702292 + - type: nauc_map_at_1_std + value: 1.466836241107892 + - type: nauc_map_at_20_diff1 + value: 47.56087525155653 + - type: nauc_map_at_20_max + value: 41.227728942164894 + - type: nauc_map_at_20_std + value: 7.3847714638766435 + - type: nauc_map_at_3_diff1 + value: 47.71455711143182 + - type: nauc_map_at_3_max + value: 38.82691929947111 + - type: nauc_map_at_3_std + value: 5.382447104679012 + - type: nauc_map_at_5_diff1 + value: 47.58482437317151 + - type: nauc_map_at_5_max + value: 40.572426131016485 + - type: nauc_map_at_5_std + value: 6.713601936827272 + - type: nauc_mrr_at_1000_diff1 + value: 48.55731582508632 + - type: nauc_mrr_at_1000_max + value: 44.05526539350372 + - type: nauc_mrr_at_1000_std + value: 8.301047055275673 + - type: nauc_mrr_at_100_diff1 + value: 48.54018042803715 + - type: nauc_mrr_at_100_max + value: 44.05044171200567 + - type: nauc_mrr_at_100_std + value: 8.329731307642941 + - type: nauc_mrr_at_10_diff1 + value: 48.441603732736354 + - type: nauc_mrr_at_10_max + value: 43.8745917330144 + - type: nauc_mrr_at_10_std + value: 8.045051573323754 + - type: nauc_mrr_at_1_diff1 + value: 53.8210033752627 + - type: nauc_mrr_at_1_max + value: 45.064606547858155 + - type: nauc_mrr_at_1_std + value: 5.3256899829601 + - type: nauc_mrr_at_20_diff1 + value: 48.60416000954831 + - type: nauc_mrr_at_20_max + value: 44.09829821823393 + - type: nauc_mrr_at_20_std + value: 8.152946405900163 + - type: nauc_mrr_at_3_diff1 + value: 48.886188610464956 + - type: nauc_mrr_at_3_max + value: 43.78513267387858 + - type: nauc_mrr_at_3_std + value: 7.351375891435474 + - type: nauc_mrr_at_5_diff1 + value: 48.257835089345406 + - type: nauc_mrr_at_5_max + value: 44.304076062424215 + - type: nauc_mrr_at_5_std + value: 8.150091351957732 + - type: nauc_ndcg_at_1000_diff1 + value: 46.913912334653084 + - type: nauc_ndcg_at_1000_max + value: 42.59980659657984 + - type: nauc_ndcg_at_1000_std + value: 10.21827254320915 + - type: nauc_ndcg_at_100_diff1 + value: 46.325915618209834 + - type: nauc_ndcg_at_100_max + value: 42.70556519079187 + - type: nauc_ndcg_at_100_std + value: 11.451414106072582 + - type: nauc_ndcg_at_10_diff1 + value: 46.39364034924719 + - type: nauc_ndcg_at_10_max + value: 41.39755203174421 + - type: nauc_ndcg_at_10_std + value: 8.377584271286084 + - type: nauc_ndcg_at_1_diff1 + value: 53.8210033752627 + - type: nauc_ndcg_at_1_max + value: 45.064606547858155 + - type: nauc_ndcg_at_1_std + value: 5.3256899829601 + - type: nauc_ndcg_at_20_diff1 + value: 46.51546562158014 + - type: nauc_ndcg_at_20_max + value: 42.45402567498778 + - type: nauc_ndcg_at_20_std + value: 9.538391068587613 + - type: nauc_ndcg_at_3_diff1 + value: 46.67796763185899 + - type: nauc_ndcg_at_3_max + value: 40.33078691738594 + - type: nauc_ndcg_at_3_std + value: 6.622690050383229 + - type: nauc_ndcg_at_5_diff1 + value: 46.17466073411471 + - type: nauc_ndcg_at_5_max + value: 41.68560605679634 + - type: nauc_ndcg_at_5_std + value: 8.278894530428783 + - type: nauc_precision_at_1000_diff1 + value: -4.134331387385582 + - type: nauc_precision_at_1000_max + value: -0.14605548786941924 + - type: nauc_precision_at_1000_std + value: -1.9544799888371407 + - type: nauc_precision_at_100_diff1 + value: 2.0012849208801375 + - type: nauc_precision_at_100_max + value: 13.831938245949285 + - type: nauc_precision_at_100_std + value: 11.211218771941438 + - type: nauc_precision_at_10_diff1 + value: 19.80743832460212 + - type: nauc_precision_at_10_max + value: 32.11569527640909 + - type: nauc_precision_at_10_std + value: 12.013922731618264 + - type: nauc_precision_at_1_diff1 + value: 53.8210033752627 + - type: nauc_precision_at_1_max + value: 45.064606547858155 + - type: nauc_precision_at_1_std + value: 5.3256899829601 + - type: nauc_precision_at_20_diff1 + value: 12.619954792926297 + - type: nauc_precision_at_20_max + value: 26.493484402746542 + - type: nauc_precision_at_20_std + value: 13.363275030294158 + - type: nauc_precision_at_3_diff1 + value: 33.77938309135373 + - type: nauc_precision_at_3_max + value: 38.88468758379097 + - type: nauc_precision_at_3_std + value: 11.707583322884002 + - type: nauc_precision_at_5_diff1 + value: 26.183678843283907 + - type: nauc_precision_at_5_max + value: 37.56754335142728 + - type: nauc_precision_at_5_std + value: 14.518813619124952 + - type: nauc_recall_at_1000_diff1 + value: 43.420905851085834 + - type: nauc_recall_at_1000_max + value: 43.98427590351494 + - type: nauc_recall_at_1000_std + value: 58.323286379858196 + - type: nauc_recall_at_100_diff1 + value: 33.004621959984135 + - type: nauc_recall_at_100_max + value: 41.20967143478881 + - type: nauc_recall_at_100_std + value: 37.92168632164788 + - type: nauc_recall_at_10_diff1 + value: 39.41006234629036 + - type: nauc_recall_at_10_max + value: 36.57670659511164 + - type: nauc_recall_at_10_std + value: 11.855499190084762 + - type: nauc_recall_at_1_diff1 + value: 51.50668476702385 + - type: nauc_recall_at_1_max + value: 37.74817918702292 + - type: nauc_recall_at_1_std + value: 1.466836241107892 + - type: nauc_recall_at_20_diff1 + value: 38.91703792345513 + - type: nauc_recall_at_20_max + value: 40.337286481871075 + - type: nauc_recall_at_20_std + value: 16.46636303434243 + - type: nauc_recall_at_3_diff1 + value: 41.632185131115165 + - type: nauc_recall_at_3_max + value: 34.991130024297654 + - type: nauc_recall_at_3_std + value: 7.011470080999405 + - type: nauc_recall_at_5_diff1 + value: 40.00945449677146 + - type: nauc_recall_at_5_max + value: 38.58807752224573 + - type: nauc_recall_at_5_std + value: 10.611240066579937 + - type: ndcg_at_1 + value: 36.644 + - type: ndcg_at_10 + value: 47.400999999999996 + - type: ndcg_at_100 + value: 53.21900000000001 + - type: ndcg_at_1000 + value: 55.015 + - type: ndcg_at_20 + value: 50.174 + - type: ndcg_at_3 + value: 42.025 + - type: ndcg_at_5 + value: 44.382 + - type: precision_at_1 + value: 36.644 + - type: precision_at_10 + value: 8.938 + - type: precision_at_100 + value: 1.376 + - type: precision_at_1000 + value: 0.16999999999999998 + - type: precision_at_20 + value: 5.365 + - type: precision_at_3 + value: 20.662 + - type: precision_at_5 + value: 14.749 + - type: recall_at_1 + value: 29.217 + - type: recall_at_10 + value: 60.668 + - type: recall_at_100 + value: 84.63300000000001 + - type: recall_at_1000 + value: 96.517 + - type: recall_at_20 + value: 70.464 + - type: recall_at_3 + value: 45.202999999999996 + - type: recall_at_5 + value: 51.635 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackRetrieval + revision: 4ffe81d471b1924886b33c7567bfb200e9eec5c4 + split: test + type: mteb/cqadupstack + metrics: + - type: main_score + value: 44.361250000000005 + - type: map_at_1 + value: 28.304499999999997 + - type: map_at_10 + value: 38.54841666666666 + - type: map_at_100 + value: 39.83141666666667 + - type: map_at_1000 + value: 39.944750000000006 + - type: map_at_20 + value: 39.25341666666667 + - type: map_at_3 + value: 35.406749999999995 + - type: map_at_5 + value: 37.15558333333333 + - type: mrr_at_1 + value: 34.09077232860122 + - type: mrr_at_10 + value: 43.15445393211421 + - type: mrr_at_100 + value: 43.98645286848257 + - type: mrr_at_1000 + value: 44.037631313469404 + - type: mrr_at_20 + value: 43.64045813249614 + - type: mrr_at_3 + value: 40.674138648480486 + - type: mrr_at_5 + value: 42.106251182620255 + - type: nauc_map_at_1000_diff1 + value: 46.250011739434996 + - type: nauc_map_at_1000_max + value: 30.13664446260598 + - type: nauc_map_at_1000_std + value: 5.422301791618935 + - type: nauc_map_at_100_diff1 + value: 46.253631351999395 + - type: nauc_map_at_100_max + value: 30.12612918885181 + - type: nauc_map_at_100_std + value: 5.367077019987172 + - type: nauc_map_at_10_diff1 + value: 46.328171341741346 + - type: nauc_map_at_10_max + value: 29.80274612581464 + - type: nauc_map_at_10_std + value: 4.62996685176396 + - type: nauc_map_at_1_diff1 + value: 51.56118117729493 + - type: nauc_map_at_1_max + value: 27.94885243863768 + - type: nauc_map_at_1_std + value: 1.700366508927356 + - type: nauc_map_at_20_diff1 + value: 46.286750260299094 + - type: nauc_map_at_20_max + value: 29.979205290353278 + - type: nauc_map_at_20_std + value: 5.010588412441873 + - type: nauc_map_at_3_diff1 + value: 47.10018183619064 + - type: nauc_map_at_3_max + value: 29.062318206078753 + - type: nauc_map_at_3_std + value: 3.2235696254694197 + - type: nauc_map_at_5_diff1 + value: 46.41971733050039 + - type: nauc_map_at_5_max + value: 29.456798617695657 + - type: nauc_map_at_5_std + value: 4.0921691023077145 + - type: nauc_mrr_at_1000_diff1 + value: 45.88888977975723 + - type: nauc_mrr_at_1000_max + value: 32.162138978089544 + - type: nauc_mrr_at_1000_std + value: 6.2811943424217915 + - type: nauc_mrr_at_100_diff1 + value: 45.87480433011124 + - type: nauc_mrr_at_100_max + value: 32.16011334212834 + - type: nauc_mrr_at_100_std + value: 6.2865717772421785 + - type: nauc_mrr_at_10_diff1 + value: 45.849652904658825 + - type: nauc_mrr_at_10_max + value: 32.13847916232293 + - type: nauc_mrr_at_10_std + value: 6.105718728141999 + - type: nauc_mrr_at_1_diff1 + value: 51.013730325062156 + - type: nauc_mrr_at_1_max + value: 32.77457396492779 + - type: nauc_mrr_at_1_std + value: 4.415684893471724 + - type: nauc_mrr_at_20_diff1 + value: 45.86663046255274 + - type: nauc_mrr_at_20_max + value: 32.15219360697865 + - type: nauc_mrr_at_20_std + value: 6.19603046412763 + - type: nauc_mrr_at_3_diff1 + value: 46.522376582423185 + - type: nauc_mrr_at_3_max + value: 32.18259009733714 + - type: nauc_mrr_at_3_std + value: 5.288000648220897 + - type: nauc_mrr_at_5_diff1 + value: 45.86611481369745 + - type: nauc_mrr_at_5_max + value: 32.14261639054921 + - type: nauc_mrr_at_5_std + value: 5.8811238177073735 + - type: nauc_ndcg_at_1000_diff1 + value: 44.5055097547565 + - type: nauc_ndcg_at_1000_max + value: 31.149682057975458 + - type: nauc_ndcg_at_1000_std + value: 8.157937194901333 + - type: nauc_ndcg_at_100_diff1 + value: 44.12398363638596 + - type: nauc_ndcg_at_100_max + value: 30.878064321409994 + - type: nauc_ndcg_at_100_std + value: 8.40493441452808 + - type: nauc_ndcg_at_10_diff1 + value: 44.200093505221474 + - type: nauc_ndcg_at_10_max + value: 30.15267107733158 + - type: nauc_ndcg_at_10_std + value: 6.407495361566107 + - type: nauc_ndcg_at_1_diff1 + value: 51.013730325062156 + - type: nauc_ndcg_at_1_max + value: 32.77457396492779 + - type: nauc_ndcg_at_1_std + value: 4.415684893471724 + - type: nauc_ndcg_at_20_diff1 + value: 44.16988321564116 + - type: nauc_ndcg_at_20_max + value: 30.333532500651213 + - type: nauc_ndcg_at_20_std + value: 7.10024701386895 + - type: nauc_ndcg_at_3_diff1 + value: 45.35982873879988 + - type: nauc_ndcg_at_3_max + value: 30.288312457948702 + - type: nauc_ndcg_at_3_std + value: 4.653900898293395 + - type: nauc_ndcg_at_5_diff1 + value: 44.324558115380185 + - type: nauc_ndcg_at_5_max + value: 30.048149698941373 + - type: nauc_ndcg_at_5_std + value: 5.6684459618413205 + - type: nauc_precision_at_1000_diff1 + value: -7.282175798304458 + - type: nauc_precision_at_1000_max + value: 7.820142031765352 + - type: nauc_precision_at_1000_std + value: 11.736131836431172 + - type: nauc_precision_at_100_diff1 + value: 1.0222940256506976 + - type: nauc_precision_at_100_max + value: 16.12346497070298 + - type: nauc_precision_at_100_std + value: 18.202607395247874 + - type: nauc_precision_at_10_diff1 + value: 18.289439185857837 + - type: nauc_precision_at_10_max + value: 26.116517399154375 + - type: nauc_precision_at_10_std + value: 13.921214069982302 + - type: nauc_precision_at_1_diff1 + value: 51.013730325062156 + - type: nauc_precision_at_1_max + value: 32.77457396492779 + - type: nauc_precision_at_1_std + value: 4.415684893471724 + - type: nauc_precision_at_20_diff1 + value: 12.365165405210886 + - type: nauc_precision_at_20_max + value: 22.946297258937367 + - type: nauc_precision_at_20_std + value: 16.13862870358933 + - type: nauc_precision_at_3_diff1 + value: 32.063423642849685 + - type: nauc_precision_at_3_max + value: 30.140965811989407 + - type: nauc_precision_at_3_std + value: 8.501746262550146 + - type: nauc_precision_at_5_diff1 + value: 24.777203357717948 + - type: nauc_precision_at_5_max + value: 28.401579566848472 + - type: nauc_precision_at_5_std + value: 11.643246774390914 + - type: nauc_recall_at_1000_diff1 + value: 30.04216463401409 + - type: nauc_recall_at_1000_max + value: 34.98067760563842 + - type: nauc_recall_at_1000_std + value: 48.01453905250591 + - type: nauc_recall_at_100_diff1 + value: 31.193415507513972 + - type: nauc_recall_at_100_max + value: 28.69740149270981 + - type: nauc_recall_at_100_std + value: 25.20960758920368 + - type: nauc_recall_at_10_diff1 + value: 36.18870823636506 + - type: nauc_recall_at_10_max + value: 26.005625231341238 + - type: nauc_recall_at_10_std + value: 8.891983977041376 + - type: nauc_recall_at_1_diff1 + value: 51.56118117729493 + - type: nauc_recall_at_1_max + value: 27.94885243863768 + - type: nauc_recall_at_1_std + value: 1.700366508927356 + - type: nauc_recall_at_20_diff1 + value: 34.93996118564803 + - type: nauc_recall_at_20_max + value: 26.149961715956138 + - type: nauc_recall_at_20_std + value: 12.0657502367633 + - type: nauc_recall_at_3_diff1 + value: 40.80743946709512 + - type: nauc_recall_at_3_max + value: 26.443127773025783 + - type: nauc_recall_at_3_std + value: 3.7011448604241477 + - type: nauc_recall_at_5_diff1 + value: 37.608535157055776 + - type: nauc_recall_at_5_max + value: 26.168016189725822 + - type: nauc_recall_at_5_std + value: 6.344191564595316 + - type: ndcg_at_1 + value: 34.09083333333333 + - type: ndcg_at_10 + value: 44.361250000000005 + - type: ndcg_at_100 + value: 49.586166666666664 + - type: ndcg_at_1000 + value: 51.623583333333336 + - type: ndcg_at_20 + value: 46.40158333333333 + - type: ndcg_at_3 + value: 39.27733333333333 + - type: ndcg_at_5 + value: 41.662333333333336 + - type: precision_at_1 + value: 34.09083333333333 + - type: precision_at_10 + value: 7.957000000000002 + - type: precision_at_100 + value: 1.2521666666666669 + - type: precision_at_1000 + value: 0.16125 + - type: precision_at_20 + value: 4.6755 + - type: precision_at_3 + value: 18.402083333333334 + - type: precision_at_5 + value: 13.104333333333335 + - type: recall_at_1 + value: 28.304499999999997 + - type: recall_at_10 + value: 56.80666666666667 + - type: recall_at_100 + value: 79.66208333333334 + - type: recall_at_1000 + value: 93.6455 + - type: recall_at_20 + value: 64.2495 + - type: recall_at_3 + value: 42.431333333333335 + - type: recall_at_5 + value: 48.665416666666665 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackStatsRetrieval + revision: 65ac3a16b8e91f9cee4c9828cc7c335575432a2a + split: test + type: mteb/cqadupstack-stats + metrics: + - type: main_score + value: 37.901 + - type: map_at_1 + value: 26.266000000000002 + - type: map_at_10 + value: 33.6 + - type: map_at_100 + value: 34.603 + - type: map_at_1000 + value: 34.708 + - type: map_at_20 + value: 34.133 + - type: map_at_3 + value: 31.237 + - type: map_at_5 + value: 32.707 + - type: mrr_at_1 + value: 29.9079754601227 + - type: mrr_at_10 + value: 36.75960414840782 + - type: mrr_at_100 + value: 37.597864424554295 + - type: mrr_at_1000 + value: 37.67621935122073 + - type: mrr_at_20 + value: 37.24171791793564 + - type: mrr_at_3 + value: 34.68813905930471 + - type: mrr_at_5 + value: 35.95347648261759 + - type: nauc_map_at_1000_diff1 + value: 49.7020197608868 + - type: nauc_map_at_1000_max + value: 38.14067261437235 + - type: nauc_map_at_1000_std + value: 13.344611852565905 + - type: nauc_map_at_100_diff1 + value: 49.72273375346987 + - type: nauc_map_at_100_max + value: 38.11974195288931 + - type: nauc_map_at_100_std + value: 13.32628220222871 + - type: nauc_map_at_10_diff1 + value: 49.61336648979491 + - type: nauc_map_at_10_max + value: 37.99270823687552 + - type: nauc_map_at_10_std + value: 12.962750231530526 + - type: nauc_map_at_1_diff1 + value: 54.69257880372346 + - type: nauc_map_at_1_max + value: 38.609162979616976 + - type: nauc_map_at_1_std + value: 10.186051365979914 + - type: nauc_map_at_20_diff1 + value: 49.80314290856952 + - type: nauc_map_at_20_max + value: 38.03662021258161 + - type: nauc_map_at_20_std + value: 13.08328471675429 + - type: nauc_map_at_3_diff1 + value: 50.23590713418514 + - type: nauc_map_at_3_max + value: 37.807633421314065 + - type: nauc_map_at_3_std + value: 11.076439870682464 + - type: nauc_map_at_5_diff1 + value: 49.49950665856434 + - type: nauc_map_at_5_max + value: 37.772558381356916 + - type: nauc_map_at_5_std + value: 13.072608152661532 + - type: nauc_mrr_at_1000_diff1 + value: 49.905676117236055 + - type: nauc_mrr_at_1000_max + value: 40.868891312053236 + - type: nauc_mrr_at_1000_std + value: 14.235640158058075 + - type: nauc_mrr_at_100_diff1 + value: 49.90995441536491 + - type: nauc_mrr_at_100_max + value: 40.86107436263684 + - type: nauc_mrr_at_100_std + value: 14.229534876079228 + - type: nauc_mrr_at_10_diff1 + value: 49.86757469223346 + - type: nauc_mrr_at_10_max + value: 40.88378059039097 + - type: nauc_mrr_at_10_std + value: 13.937699637927214 + - type: nauc_mrr_at_1_diff1 + value: 54.97246875202937 + - type: nauc_mrr_at_1_max + value: 43.21949977348923 + - type: nauc_mrr_at_1_std + value: 12.757586516037337 + - type: nauc_mrr_at_20_diff1 + value: 49.99976981173147 + - type: nauc_mrr_at_20_max + value: 40.8707720726509 + - type: nauc_mrr_at_20_std + value: 14.057989841360383 + - type: nauc_mrr_at_3_diff1 + value: 50.680871756122016 + - type: nauc_mrr_at_3_max + value: 41.23248478084467 + - type: nauc_mrr_at_3_std + value: 12.905348231690184 + - type: nauc_mrr_at_5_diff1 + value: 49.87308482032918 + - type: nauc_mrr_at_5_max + value: 40.830256851862224 + - type: nauc_mrr_at_5_std + value: 14.223085632591898 + - type: nauc_ndcg_at_1000_diff1 + value: 47.64787646785112 + - type: nauc_ndcg_at_1000_max + value: 39.08202768044249 + - type: nauc_ndcg_at_1000_std + value: 16.446745452324198 + - type: nauc_ndcg_at_100_diff1 + value: 47.7692822135391 + - type: nauc_ndcg_at_100_max + value: 38.56463305854455 + - type: nauc_ndcg_at_100_std + value: 16.19108846061072 + - type: nauc_ndcg_at_10_diff1 + value: 47.768546405489346 + - type: nauc_ndcg_at_10_max + value: 37.93126542366707 + - type: nauc_ndcg_at_10_std + value: 13.90597936021141 + - type: nauc_ndcg_at_1_diff1 + value: 54.97246875202937 + - type: nauc_ndcg_at_1_max + value: 43.21949977348923 + - type: nauc_ndcg_at_1_std + value: 12.757586516037337 + - type: nauc_ndcg_at_20_diff1 + value: 48.31887885171088 + - type: nauc_ndcg_at_20_max + value: 37.96907369878272 + - type: nauc_ndcg_at_20_std + value: 14.361569920214562 + - type: nauc_ndcg_at_3_diff1 + value: 49.41166078995533 + - type: nauc_ndcg_at_3_max + value: 38.4962713292873 + - type: nauc_ndcg_at_3_std + value: 11.647198208903689 + - type: nauc_ndcg_at_5_diff1 + value: 47.79298367906792 + - type: nauc_ndcg_at_5_max + value: 37.59798273170525 + - type: nauc_ndcg_at_5_std + value: 14.369220517370913 + - type: nauc_precision_at_1000_diff1 + value: 4.779290277609319 + - type: nauc_precision_at_1000_max + value: 22.181043405691963 + - type: nauc_precision_at_1000_std + value: 14.73247174367766 + - type: nauc_precision_at_100_diff1 + value: 20.690215573934548 + - type: nauc_precision_at_100_max + value: 31.519613396282605 + - type: nauc_precision_at_100_std + value: 22.258879545517008 + - type: nauc_precision_at_10_diff1 + value: 37.18469921478606 + - type: nauc_precision_at_10_max + value: 38.80312310884728 + - type: nauc_precision_at_10_std + value: 20.548480478974312 + - type: nauc_precision_at_1_diff1 + value: 54.97246875202937 + - type: nauc_precision_at_1_max + value: 43.21949977348923 + - type: nauc_precision_at_1_std + value: 12.757586516037337 + - type: nauc_precision_at_20_diff1 + value: 34.13612072510413 + - type: nauc_precision_at_20_max + value: 36.33974409113382 + - type: nauc_precision_at_20_std + value: 19.40715666658738 + - type: nauc_precision_at_3_diff1 + value: 44.52308297046978 + - type: nauc_precision_at_3_max + value: 40.32105173693474 + - type: nauc_precision_at_3_std + value: 15.012376385333855 + - type: nauc_precision_at_5_diff1 + value: 38.95952631711529 + - type: nauc_precision_at_5_max + value: 38.73803476644105 + - type: nauc_precision_at_5_std + value: 22.534363384686547 + - type: nauc_recall_at_1000_diff1 + value: 23.530169479478676 + - type: nauc_recall_at_1000_max + value: 41.08310224321662 + - type: nauc_recall_at_1000_std + value: 52.157876195038156 + - type: nauc_recall_at_100_diff1 + value: 36.81146016189628 + - type: nauc_recall_at_100_max + value: 34.742906812186256 + - type: nauc_recall_at_100_std + value: 28.470956037508554 + - type: nauc_recall_at_10_diff1 + value: 39.91433768103735 + - type: nauc_recall_at_10_max + value: 33.474764892584474 + - type: nauc_recall_at_10_std + value: 14.683676644107656 + - type: nauc_recall_at_1_diff1 + value: 54.69257880372346 + - type: nauc_recall_at_1_max + value: 38.609162979616976 + - type: nauc_recall_at_1_std + value: 10.186051365979914 + - type: nauc_recall_at_20_diff1 + value: 41.453493281903015 + - type: nauc_recall_at_20_max + value: 32.47416777238201 + - type: nauc_recall_at_20_std + value: 16.060473738451822 + - type: nauc_recall_at_3_diff1 + value: 44.26329717724608 + - type: nauc_recall_at_3_max + value: 34.65501168113369 + - type: nauc_recall_at_3_std + value: 10.17026796706714 + - type: nauc_recall_at_5_diff1 + value: 40.209903370140296 + - type: nauc_recall_at_5_max + value: 32.73075112914612 + - type: nauc_recall_at_5_std + value: 16.238093039480795 + - type: ndcg_at_1 + value: 29.908 + - type: ndcg_at_10 + value: 37.901 + - type: ndcg_at_100 + value: 42.669000000000004 + - type: ndcg_at_1000 + value: 45.324 + - type: ndcg_at_20 + value: 39.646 + - type: ndcg_at_3 + value: 33.726 + - type: ndcg_at_5 + value: 35.961 + - type: precision_at_1 + value: 29.908 + - type: precision_at_10 + value: 5.859 + - type: precision_at_100 + value: 0.9079999999999999 + - type: precision_at_1000 + value: 0.121 + - type: precision_at_20 + value: 3.405 + - type: precision_at_3 + value: 14.366000000000001 + - type: precision_at_5 + value: 10.184 + - type: recall_at_1 + value: 26.266000000000002 + - type: recall_at_10 + value: 48.059000000000005 + - type: recall_at_100 + value: 69.45700000000001 + - type: recall_at_1000 + value: 89.11200000000001 + - type: recall_at_20 + value: 54.452999999999996 + - type: recall_at_3 + value: 36.422 + - type: recall_at_5 + value: 42.035 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackTexRetrieval + revision: 46989137a86843e03a6195de44b09deda022eec7 + split: test + type: mteb/cqadupstack-tex + metrics: + - type: main_score + value: 32.42 + - type: map_at_1 + value: 18.936 + - type: map_at_10 + value: 27.236 + - type: map_at_100 + value: 28.451999999999998 + - type: map_at_1000 + value: 28.579 + - type: map_at_20 + value: 27.906 + - type: map_at_3 + value: 24.633 + - type: map_at_5 + value: 26.018 + - type: mrr_at_1 + value: 23.5719201651755 + - type: mrr_at_10 + value: 31.648559630321536 + - type: mrr_at_100 + value: 32.59499394633238 + - type: mrr_at_1000 + value: 32.6655128458643 + - type: mrr_at_20 + value: 32.20562525813067 + - type: mrr_at_3 + value: 29.421885753613246 + - type: mrr_at_5 + value: 30.617687543014526 + - type: nauc_map_at_1000_diff1 + value: 36.94323263959507 + - type: nauc_map_at_1000_max + value: 23.771599100231374 + - type: nauc_map_at_1000_std + value: 7.1222723931932945 + - type: nauc_map_at_100_diff1 + value: 36.91494551237442 + - type: nauc_map_at_100_max + value: 23.733346527251147 + - type: nauc_map_at_100_std + value: 7.090204071351739 + - type: nauc_map_at_10_diff1 + value: 37.04097820878135 + - type: nauc_map_at_10_max + value: 23.803971277888216 + - type: nauc_map_at_10_std + value: 6.39831219027254 + - type: nauc_map_at_1_diff1 + value: 42.8969146431479 + - type: nauc_map_at_1_max + value: 22.51288060492358 + - type: nauc_map_at_1_std + value: 3.7880059589050643 + - type: nauc_map_at_20_diff1 + value: 36.880310083350004 + - type: nauc_map_at_20_max + value: 23.721978963544746 + - type: nauc_map_at_20_std + value: 6.8720799504919095 + - type: nauc_map_at_3_diff1 + value: 38.379233284113326 + - type: nauc_map_at_3_max + value: 23.368545890408264 + - type: nauc_map_at_3_std + value: 5.342157811678566 + - type: nauc_map_at_5_diff1 + value: 37.61779195289775 + - type: nauc_map_at_5_max + value: 23.414139560213314 + - type: nauc_map_at_5_std + value: 5.995478708910173 + - type: nauc_mrr_at_1000_diff1 + value: 37.46436680808877 + - type: nauc_mrr_at_1000_max + value: 25.237760419475187 + - type: nauc_mrr_at_1000_std + value: 8.504455979740074 + - type: nauc_mrr_at_100_diff1 + value: 37.45158220895872 + - type: nauc_mrr_at_100_max + value: 25.22343267664494 + - type: nauc_mrr_at_100_std + value: 8.507739174312585 + - type: nauc_mrr_at_10_diff1 + value: 37.441303680613714 + - type: nauc_mrr_at_10_max + value: 25.329154251224768 + - type: nauc_mrr_at_10_std + value: 8.236842519370176 + - type: nauc_mrr_at_1_diff1 + value: 43.14895579136297 + - type: nauc_mrr_at_1_max + value: 24.708412178193115 + - type: nauc_mrr_at_1_std + value: 6.617634022393408 + - type: nauc_mrr_at_20_diff1 + value: 37.36511956264013 + - type: nauc_mrr_at_20_max + value: 25.27240486447333 + - type: nauc_mrr_at_20_std + value: 8.464655061933227 + - type: nauc_mrr_at_3_diff1 + value: 38.67516429996044 + - type: nauc_mrr_at_3_max + value: 25.306987038005275 + - type: nauc_mrr_at_3_std + value: 7.355554700412325 + - type: nauc_mrr_at_5_diff1 + value: 38.10984280323206 + - type: nauc_mrr_at_5_max + value: 25.300728066173374 + - type: nauc_mrr_at_5_std + value: 7.976467409780732 + - type: nauc_ndcg_at_1000_diff1 + value: 34.98282133204461 + - type: nauc_ndcg_at_1000_max + value: 24.5828439287868 + - type: nauc_ndcg_at_1000_std + value: 10.371064197055073 + - type: nauc_ndcg_at_100_diff1 + value: 34.67788955650232 + - type: nauc_ndcg_at_100_max + value: 24.073740369120806 + - type: nauc_ndcg_at_100_std + value: 10.31564507027441 + - type: nauc_ndcg_at_10_diff1 + value: 34.85587928047769 + - type: nauc_ndcg_at_10_max + value: 24.43044490299822 + - type: nauc_ndcg_at_10_std + value: 8.053424009139828 + - type: nauc_ndcg_at_1_diff1 + value: 43.14895579136297 + - type: nauc_ndcg_at_1_max + value: 24.708412178193115 + - type: nauc_ndcg_at_1_std + value: 6.617634022393408 + - type: nauc_ndcg_at_20_diff1 + value: 34.515917631732115 + - type: nauc_ndcg_at_20_max + value: 24.19694248762929 + - type: nauc_ndcg_at_20_std + value: 9.320771030174605 + - type: nauc_ndcg_at_3_diff1 + value: 37.03512297714074 + - type: nauc_ndcg_at_3_max + value: 23.96938009162311 + - type: nauc_ndcg_at_3_std + value: 6.575669478741932 + - type: nauc_ndcg_at_5_diff1 + value: 36.060639653815954 + - type: nauc_ndcg_at_5_max + value: 24.043379966375195 + - type: nauc_ndcg_at_5_std + value: 7.45050440194733 + - type: nauc_precision_at_1000_diff1 + value: 11.153913795780158 + - type: nauc_precision_at_1000_max + value: 15.420229300566431 + - type: nauc_precision_at_1000_std + value: 12.034329129634239 + - type: nauc_precision_at_100_diff1 + value: 13.608906739151951 + - type: nauc_precision_at_100_max + value: 17.73797738765977 + - type: nauc_precision_at_100_std + value: 17.790571644445205 + - type: nauc_precision_at_10_diff1 + value: 20.897406252665437 + - type: nauc_precision_at_10_max + value: 23.11918551320126 + - type: nauc_precision_at_10_std + value: 12.935124111201942 + - type: nauc_precision_at_1_diff1 + value: 43.14895579136297 + - type: nauc_precision_at_1_max + value: 24.708412178193115 + - type: nauc_precision_at_1_std + value: 6.617634022393408 + - type: nauc_precision_at_20_diff1 + value: 17.717295118461898 + - type: nauc_precision_at_20_max + value: 20.716771929346326 + - type: nauc_precision_at_20_std + value: 16.68450524188211 + - type: nauc_precision_at_3_diff1 + value: 30.785637103625753 + - type: nauc_precision_at_3_max + value: 24.29204993772703 + - type: nauc_precision_at_3_std + value: 9.498012045096026 + - type: nauc_precision_at_5_diff1 + value: 26.688727640225636 + - type: nauc_precision_at_5_max + value: 23.694763472251363 + - type: nauc_precision_at_5_std + value: 11.3853592072625 + - type: nauc_recall_at_1000_diff1 + value: 22.356409562371514 + - type: nauc_recall_at_1000_max + value: 27.366326658034634 + - type: nauc_recall_at_1000_std + value: 33.74625442546299 + - type: nauc_recall_at_100_diff1 + value: 24.5991223782448 + - type: nauc_recall_at_100_max + value: 20.44736011515427 + - type: nauc_recall_at_100_std + value: 20.827429233732868 + - type: nauc_recall_at_10_diff1 + value: 26.669736246752258 + - type: nauc_recall_at_10_max + value: 22.785409673805805 + - type: nauc_recall_at_10_std + value: 9.905753045529723 + - type: nauc_recall_at_1_diff1 + value: 42.8969146431479 + - type: nauc_recall_at_1_max + value: 22.51288060492358 + - type: nauc_recall_at_1_std + value: 3.7880059589050643 + - type: nauc_recall_at_20_diff1 + value: 25.08321378411484 + - type: nauc_recall_at_20_max + value: 21.556457547258834 + - type: nauc_recall_at_20_std + value: 14.14509388470547 + - type: nauc_recall_at_3_diff1 + value: 32.942999125142606 + - type: nauc_recall_at_3_max + value: 22.34920448389651 + - type: nauc_recall_at_3_std + value: 6.152767607197115 + - type: nauc_recall_at_5_diff1 + value: 30.280311448364195 + - type: nauc_recall_at_5_max + value: 22.232709619804815 + - type: nauc_recall_at_5_std + value: 8.317838538960972 + - type: ndcg_at_1 + value: 23.572000000000003 + - type: ndcg_at_10 + value: 32.42 + - type: ndcg_at_100 + value: 37.97 + - type: ndcg_at_1000 + value: 40.736 + - type: ndcg_at_20 + value: 34.528 + - type: ndcg_at_3 + value: 28.081 + - type: ndcg_at_5 + value: 29.975 + - type: precision_at_1 + value: 23.572000000000003 + - type: precision_at_10 + value: 6.087 + - type: precision_at_100 + value: 1.035 + - type: precision_at_1000 + value: 0.146 + - type: precision_at_20 + value: 3.675 + - type: precision_at_3 + value: 13.661000000000001 + - type: precision_at_5 + value: 9.8 + - type: recall_at_1 + value: 18.936 + - type: recall_at_10 + value: 43.458000000000006 + - type: recall_at_100 + value: 68.458 + - type: recall_at_1000 + value: 87.847 + - type: recall_at_20 + value: 51.214000000000006 + - type: recall_at_3 + value: 30.903999999999996 + - type: recall_at_5 + value: 36.086 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackUnixRetrieval + revision: 6c6430d3a6d36f8d2a829195bc5dc94d7e063e53 + split: test + type: mteb/cqadupstack-unix + metrics: + - type: main_score + value: 43.669999999999995 + - type: map_at_1 + value: 27.840999999999998 + - type: map_at_10 + value: 38.018 + - type: map_at_100 + value: 39.317 + - type: map_at_1000 + value: 39.414 + - type: map_at_20 + value: 38.744 + - type: map_at_3 + value: 35.022 + - type: map_at_5 + value: 36.754999999999995 + - type: mrr_at_1 + value: 33.30223880597015 + - type: mrr_at_10 + value: 42.468905472636784 + - type: mrr_at_100 + value: 43.35921217251144 + - type: mrr_at_1000 + value: 43.41192409720638 + - type: mrr_at_20 + value: 42.98945154987357 + - type: mrr_at_3 + value: 39.98756218905469 + - type: mrr_at_5 + value: 41.391480099502424 + - type: nauc_map_at_1000_diff1 + value: 51.37310738164103 + - type: nauc_map_at_1000_max + value: 36.82544641632401 + - type: nauc_map_at_1000_std + value: 3.055725621118122 + - type: nauc_map_at_100_diff1 + value: 51.36341958716978 + - type: nauc_map_at_100_max + value: 36.834169105016564 + - type: nauc_map_at_100_std + value: 3.064020943669763 + - type: nauc_map_at_10_diff1 + value: 51.44270224027026 + - type: nauc_map_at_10_max + value: 36.44835383011058 + - type: nauc_map_at_10_std + value: 2.5141207325864143 + - type: nauc_map_at_1_diff1 + value: 56.6372792866915 + - type: nauc_map_at_1_max + value: 36.1803755535215 + - type: nauc_map_at_1_std + value: 0.31977094347955537 + - type: nauc_map_at_20_diff1 + value: 51.43068019320809 + - type: nauc_map_at_20_max + value: 36.716831883867265 + - type: nauc_map_at_20_std + value: 2.873054267872677 + - type: nauc_map_at_3_diff1 + value: 51.64487864432652 + - type: nauc_map_at_3_max + value: 35.95556315927097 + - type: nauc_map_at_3_std + value: 1.2140189340738128 + - type: nauc_map_at_5_diff1 + value: 51.24716514919114 + - type: nauc_map_at_5_max + value: 35.96236292229056 + - type: nauc_map_at_5_std + value: 2.0228946045126635 + - type: nauc_mrr_at_1000_diff1 + value: 50.4709938187239 + - type: nauc_mrr_at_1000_max + value: 38.17036558544397 + - type: nauc_mrr_at_1000_std + value: 3.9495748558741663 + - type: nauc_mrr_at_100_diff1 + value: 50.45913534847875 + - type: nauc_mrr_at_100_max + value: 38.16853913679008 + - type: nauc_mrr_at_100_std + value: 3.95690023298836 + - type: nauc_mrr_at_10_diff1 + value: 50.41092187542978 + - type: nauc_mrr_at_10_max + value: 38.21402534320531 + - type: nauc_mrr_at_10_std + value: 3.8821360168065007 + - type: nauc_mrr_at_1_diff1 + value: 56.184793793739296 + - type: nauc_mrr_at_1_max + value: 39.029842976782426 + - type: nauc_mrr_at_1_std + value: 2.7404115941177807 + - type: nauc_mrr_at_20_diff1 + value: 50.44213341941401 + - type: nauc_mrr_at_20_max + value: 38.21572816127193 + - type: nauc_mrr_at_20_std + value: 3.8996521743689803 + - type: nauc_mrr_at_3_diff1 + value: 50.4214955440644 + - type: nauc_mrr_at_3_max + value: 37.919708586037345 + - type: nauc_mrr_at_3_std + value: 2.690223971711943 + - type: nauc_mrr_at_5_diff1 + value: 50.20495450033871 + - type: nauc_mrr_at_5_max + value: 37.98100598405518 + - type: nauc_mrr_at_5_std + value: 3.4647793898610937 + - type: nauc_ndcg_at_1000_diff1 + value: 49.85355102619098 + - type: nauc_ndcg_at_1000_max + value: 37.31224794997029 + - type: nauc_ndcg_at_1000_std + value: 5.2888358551197925 + - type: nauc_ndcg_at_100_diff1 + value: 49.37442680731448 + - type: nauc_ndcg_at_100_max + value: 37.26475120961565 + - type: nauc_ndcg_at_100_std + value: 5.840448008443884 + - type: nauc_ndcg_at_10_diff1 + value: 49.588294004839916 + - type: nauc_ndcg_at_10_max + value: 36.47242070557319 + - type: nauc_ndcg_at_10_std + value: 3.792525444937786 + - type: nauc_ndcg_at_1_diff1 + value: 56.184793793739296 + - type: nauc_ndcg_at_1_max + value: 39.029842976782426 + - type: nauc_ndcg_at_1_std + value: 2.7404115941177807 + - type: nauc_ndcg_at_20_diff1 + value: 49.61331333561641 + - type: nauc_ndcg_at_20_max + value: 36.98443040121314 + - type: nauc_ndcg_at_20_std + value: 4.684757661793056 + - type: nauc_ndcg_at_3_diff1 + value: 49.38375093072672 + - type: nauc_ndcg_at_3_max + value: 36.15725360341771 + - type: nauc_ndcg_at_3_std + value: 1.8424211295522894 + - type: nauc_ndcg_at_5_diff1 + value: 48.97411191818124 + - type: nauc_ndcg_at_5_max + value: 35.738364587228695 + - type: nauc_ndcg_at_5_std + value: 2.811998354199763 + - type: nauc_precision_at_1000_diff1 + value: -7.947321550782299 + - type: nauc_precision_at_1000_max + value: 4.3548311540194184 + - type: nauc_precision_at_1000_std + value: 5.917685639450147 + - type: nauc_precision_at_100_diff1 + value: 4.168436727910857 + - type: nauc_precision_at_100_max + value: 17.514268193372455 + - type: nauc_precision_at_100_std + value: 14.63664211004551 + - type: nauc_precision_at_10_diff1 + value: 27.86233517168723 + - type: nauc_precision_at_10_max + value: 31.84484914461883 + - type: nauc_precision_at_10_std + value: 8.027299857521172 + - type: nauc_precision_at_1_diff1 + value: 56.184793793739296 + - type: nauc_precision_at_1_max + value: 39.029842976782426 + - type: nauc_precision_at_1_std + value: 2.7404115941177807 + - type: nauc_precision_at_20_diff1 + value: 20.794855774041064 + - type: nauc_precision_at_20_max + value: 28.974122166664102 + - type: nauc_precision_at_20_std + value: 11.311840343204429 + - type: nauc_precision_at_3_diff1 + value: 37.39252955092301 + - type: nauc_precision_at_3_max + value: 34.55872037917925 + - type: nauc_precision_at_3_std + value: 3.0298517108977343 + - type: nauc_precision_at_5_diff1 + value: 31.805150162258382 + - type: nauc_precision_at_5_max + value: 32.68205915951889 + - type: nauc_precision_at_5_std + value: 5.674953992396121 + - type: nauc_recall_at_1000_diff1 + value: 37.15789350006901 + - type: nauc_recall_at_1000_max + value: 31.082533600261304 + - type: nauc_recall_at_1000_std + value: 38.710619061772626 + - type: nauc_recall_at_100_diff1 + value: 37.24034995517936 + - type: nauc_recall_at_100_max + value: 32.5752032068577 + - type: nauc_recall_at_100_std + value: 21.354866373015753 + - type: nauc_recall_at_10_diff1 + value: 42.77906909919547 + - type: nauc_recall_at_10_max + value: 31.883100193463015 + - type: nauc_recall_at_10_std + value: 6.250310586319574 + - type: nauc_recall_at_1_diff1 + value: 56.6372792866915 + - type: nauc_recall_at_1_max + value: 36.1803755535215 + - type: nauc_recall_at_1_std + value: 0.31977094347955537 + - type: nauc_recall_at_20_diff1 + value: 41.784525889463886 + - type: nauc_recall_at_20_max + value: 32.51490410125763 + - type: nauc_recall_at_20_std + value: 9.402706224269968 + - type: nauc_recall_at_3_diff1 + value: 43.85638741223575 + - type: nauc_recall_at_3_max + value: 31.983364030153243 + - type: nauc_recall_at_3_std + value: 0.7541680018178514 + - type: nauc_recall_at_5_diff1 + value: 42.12392907664656 + - type: nauc_recall_at_5_max + value: 30.85659257187195 + - type: nauc_recall_at_5_std + value: 3.7005663344473776 + - type: ndcg_at_1 + value: 33.302 + - type: ndcg_at_10 + value: 43.669999999999995 + - type: ndcg_at_100 + value: 49.345 + - type: ndcg_at_1000 + value: 51.388 + - type: ndcg_at_20 + value: 45.915 + - type: ndcg_at_3 + value: 38.621 + - type: ndcg_at_5 + value: 41.042 + - type: precision_at_1 + value: 33.302 + - type: precision_at_10 + value: 7.4719999999999995 + - type: precision_at_100 + value: 1.169 + - type: precision_at_1000 + value: 0.14400000000000002 + - type: precision_at_20 + value: 4.403 + - type: precision_at_3 + value: 17.879 + - type: precision_at_5 + value: 12.612000000000002 + - type: recall_at_1 + value: 27.840999999999998 + - type: recall_at_10 + value: 56.31699999999999 + - type: recall_at_100 + value: 80.76299999999999 + - type: recall_at_1000 + value: 94.878 + - type: recall_at_20 + value: 64.33 + - type: recall_at_3 + value: 42.266 + - type: recall_at_5 + value: 48.64 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackWebmastersRetrieval + revision: 160c094312a0e1facb97e55eeddb698c0abe3571 + split: test + type: mteb/cqadupstack-webmasters + metrics: + - type: main_score + value: 45.843 + - type: map_at_1 + value: 29.652 + - type: map_at_10 + value: 39.629999999999995 + - type: map_at_100 + value: 41.352 + - type: map_at_1000 + value: 41.593 + - type: map_at_20 + value: 40.471000000000004 + - type: map_at_3 + value: 36.541000000000004 + - type: map_at_5 + value: 38.156 + - type: mrr_at_1 + value: 36.16600790513834 + - type: mrr_at_10 + value: 44.91083192170149 + - type: mrr_at_100 + value: 45.844961766323934 + - type: mrr_at_1000 + value: 45.89287424796409 + - type: mrr_at_20 + value: 45.45520778317772 + - type: mrr_at_3 + value: 42.45718050065877 + - type: mrr_at_5 + value: 43.91963109354414 + - type: nauc_map_at_1000_diff1 + value: 48.242111460837314 + - type: nauc_map_at_1000_max + value: 27.413876794445972 + - type: nauc_map_at_1000_std + value: 17.5826534525386 + - type: nauc_map_at_100_diff1 + value: 48.466036243559174 + - type: nauc_map_at_100_max + value: 27.572201454486333 + - type: nauc_map_at_100_std + value: 17.034069650932963 + - type: nauc_map_at_10_diff1 + value: 48.71924818812899 + - type: nauc_map_at_10_max + value: 27.209651957410223 + - type: nauc_map_at_10_std + value: 15.103718660403553 + - type: nauc_map_at_1_diff1 + value: 53.809138735063215 + - type: nauc_map_at_1_max + value: 24.772619683471756 + - type: nauc_map_at_1_std + value: 11.230244685516237 + - type: nauc_map_at_20_diff1 + value: 48.707225553714885 + - type: nauc_map_at_20_max + value: 27.526341054923005 + - type: nauc_map_at_20_std + value: 15.822375460519453 + - type: nauc_map_at_3_diff1 + value: 49.43811138822573 + - type: nauc_map_at_3_max + value: 26.93413023133727 + - type: nauc_map_at_3_std + value: 12.71923235166143 + - type: nauc_map_at_5_diff1 + value: 48.4889096850373 + - type: nauc_map_at_5_max + value: 26.89285162428735 + - type: nauc_map_at_5_std + value: 14.044508913439588 + - type: nauc_mrr_at_1000_diff1 + value: 46.92383783594224 + - type: nauc_mrr_at_1000_max + value: 29.77420658773981 + - type: nauc_mrr_at_1000_std + value: 19.360170257563937 + - type: nauc_mrr_at_100_diff1 + value: 46.90559759648834 + - type: nauc_mrr_at_100_max + value: 29.781897661280066 + - type: nauc_mrr_at_100_std + value: 19.364017980637527 + - type: nauc_mrr_at_10_diff1 + value: 46.99157319712585 + - type: nauc_mrr_at_10_max + value: 29.88951317858189 + - type: nauc_mrr_at_10_std + value: 19.24420177623068 + - type: nauc_mrr_at_1_diff1 + value: 51.59659445736797 + - type: nauc_mrr_at_1_max + value: 30.24079058048848 + - type: nauc_mrr_at_1_std + value: 18.593800172605583 + - type: nauc_mrr_at_20_diff1 + value: 46.95969840830725 + - type: nauc_mrr_at_20_max + value: 29.80012273355407 + - type: nauc_mrr_at_20_std + value: 19.229519307602708 + - type: nauc_mrr_at_3_diff1 + value: 47.720258881525545 + - type: nauc_mrr_at_3_max + value: 29.737796980507564 + - type: nauc_mrr_at_3_std + value: 17.87040303195628 + - type: nauc_mrr_at_5_diff1 + value: 46.543743638677526 + - type: nauc_mrr_at_5_max + value: 29.731379637453166 + - type: nauc_mrr_at_5_std + value: 19.067338276646986 + - type: nauc_ndcg_at_1000_diff1 + value: 46.5641454446737 + - type: nauc_ndcg_at_1000_max + value: 28.615641814617724 + - type: nauc_ndcg_at_1000_std + value: 20.68576562770653 + - type: nauc_ndcg_at_100_diff1 + value: 46.34746988200411 + - type: nauc_ndcg_at_100_max + value: 28.64543150686983 + - type: nauc_ndcg_at_100_std + value: 20.579705686992643 + - type: nauc_ndcg_at_10_diff1 + value: 45.90486789468765 + - type: nauc_ndcg_at_10_max + value: 26.830813337994858 + - type: nauc_ndcg_at_10_std + value: 19.23770258587883 + - type: nauc_ndcg_at_1_diff1 + value: 51.59659445736797 + - type: nauc_ndcg_at_1_max + value: 30.24079058048848 + - type: nauc_ndcg_at_1_std + value: 18.593800172605583 + - type: nauc_ndcg_at_20_diff1 + value: 46.09422270134306 + - type: nauc_ndcg_at_20_max + value: 27.454797668927455 + - type: nauc_ndcg_at_20_std + value: 19.181389192275475 + - type: nauc_ndcg_at_3_diff1 + value: 46.71006928524964 + - type: nauc_ndcg_at_3_max + value: 27.916112539349903 + - type: nauc_ndcg_at_3_std + value: 17.31361800908132 + - type: nauc_ndcg_at_5_diff1 + value: 45.234453532794475 + - type: nauc_ndcg_at_5_max + value: 26.933584065195664 + - type: nauc_ndcg_at_5_std + value: 18.936947077270887 + - type: nauc_precision_at_1000_diff1 + value: -23.10090750610695 + - type: nauc_precision_at_1000_max + value: -8.405320792259879 + - type: nauc_precision_at_1000_std + value: 46.152078898060374 + - type: nauc_precision_at_100_diff1 + value: -13.443767854498608 + - type: nauc_precision_at_100_max + value: 1.1501918120691448 + - type: nauc_precision_at_100_std + value: 44.277553577668385 + - type: nauc_precision_at_10_diff1 + value: 13.27441909897881 + - type: nauc_precision_at_10_max + value: 19.977211396862753 + - type: nauc_precision_at_10_std + value: 33.06204878507605 + - type: nauc_precision_at_1_diff1 + value: 51.59659445736797 + - type: nauc_precision_at_1_max + value: 30.24079058048848 + - type: nauc_precision_at_1_std + value: 18.593800172605583 + - type: nauc_precision_at_20_diff1 + value: 4.527260454589361 + - type: nauc_precision_at_20_max + value: 16.554094089838006 + - type: nauc_precision_at_20_std + value: 37.06501546581287 + - type: nauc_precision_at_3_diff1 + value: 31.366659847405998 + - type: nauc_precision_at_3_max + value: 27.647011088055645 + - type: nauc_precision_at_3_std + value: 20.75536063536343 + - type: nauc_precision_at_5_diff1 + value: 21.96099801384215 + - type: nauc_precision_at_5_max + value: 24.75155156846284 + - type: nauc_precision_at_5_std + value: 27.88548717173277 + - type: nauc_recall_at_1000_diff1 + value: 34.68388954282424 + - type: nauc_recall_at_1000_max + value: 32.426242708262656 + - type: nauc_recall_at_1000_std + value: 69.53359054076861 + - type: nauc_recall_at_100_diff1 + value: 32.851235248077046 + - type: nauc_recall_at_100_max + value: 28.611353716064226 + - type: nauc_recall_at_100_std + value: 34.040196877896896 + - type: nauc_recall_at_10_diff1 + value: 39.1596658072276 + - type: nauc_recall_at_10_max + value: 22.078692047937317 + - type: nauc_recall_at_10_std + value: 17.416076033161563 + - type: nauc_recall_at_1_diff1 + value: 53.809138735063215 + - type: nauc_recall_at_1_max + value: 24.772619683471756 + - type: nauc_recall_at_1_std + value: 11.230244685516237 + - type: nauc_recall_at_20_diff1 + value: 37.30710292116189 + - type: nauc_recall_at_20_max + value: 22.715190241204702 + - type: nauc_recall_at_20_std + value: 19.576272964488798 + - type: nauc_recall_at_3_diff1 + value: 43.81556416336881 + - type: nauc_recall_at_3_max + value: 23.76912155560393 + - type: nauc_recall_at_3_std + value: 11.621642370048967 + - type: nauc_recall_at_5_diff1 + value: 38.92654417865752 + - type: nauc_recall_at_5_max + value: 22.577305464525583 + - type: nauc_recall_at_5_std + value: 15.619874581468562 + - type: ndcg_at_1 + value: 36.166 + - type: ndcg_at_10 + value: 45.843 + - type: ndcg_at_100 + value: 51.638 + - type: ndcg_at_1000 + value: 53.766000000000005 + - type: ndcg_at_20 + value: 47.894999999999996 + - type: ndcg_at_3 + value: 41.105999999999995 + - type: ndcg_at_5 + value: 43.236999999999995 + - type: precision_at_1 + value: 36.166 + - type: precision_at_10 + value: 8.854 + - type: precision_at_100 + value: 1.763 + - type: precision_at_1000 + value: 0.253 + - type: precision_at_20 + value: 5.524 + - type: precision_at_3 + value: 19.368 + - type: precision_at_5 + value: 13.794 + - type: recall_at_1 + value: 29.652 + - type: recall_at_10 + value: 56.87 + - type: recall_at_100 + value: 82.636 + - type: recall_at_1000 + value: 96.015 + - type: recall_at_20 + value: 64.559 + - type: recall_at_3 + value: 43.018 + - type: recall_at_5 + value: 48.898 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackWordpressRetrieval + revision: 4ffe81d471b1924886b33c7567bfb200e9eec5c4 + split: test + type: mteb/cqadupstack-wordpress + metrics: + - type: main_score + value: 36.799 + - type: map_at_1 + value: 24.199 + - type: map_at_10 + value: 31.802000000000003 + - type: map_at_100 + value: 32.782 + - type: map_at_1000 + value: 32.885999999999996 + - type: map_at_20 + value: 32.297 + - type: map_at_3 + value: 29.199 + - type: map_at_5 + value: 30.684 + - type: mrr_at_1 + value: 26.98706099815157 + - type: mrr_at_10 + value: 34.82373910747295 + - type: mrr_at_100 + value: 35.60636179524524 + - type: mrr_at_1000 + value: 35.67920886464087 + - type: mrr_at_20 + value: 35.205688156264166 + - type: mrr_at_3 + value: 32.3475046210721 + - type: mrr_at_5 + value: 33.69685767097968 + - type: nauc_map_at_1000_diff1 + value: 40.20426926234343 + - type: nauc_map_at_1000_max + value: 21.494946749706873 + - type: nauc_map_at_1000_std + value: 6.740658345691629 + - type: nauc_map_at_100_diff1 + value: 40.21346368227976 + - type: nauc_map_at_100_max + value: 21.519414102900587 + - type: nauc_map_at_100_std + value: 6.728326343964929 + - type: nauc_map_at_10_diff1 + value: 40.20946775511352 + - type: nauc_map_at_10_max + value: 21.4327874369473 + - type: nauc_map_at_10_std + value: 6.300725206247967 + - type: nauc_map_at_1_diff1 + value: 48.14626703158692 + - type: nauc_map_at_1_max + value: 22.33923130960631 + - type: nauc_map_at_1_std + value: 5.090028695091633 + - type: nauc_map_at_20_diff1 + value: 40.20152307766448 + - type: nauc_map_at_20_max + value: 21.33060510827152 + - type: nauc_map_at_20_std + value: 6.446310449390428 + - type: nauc_map_at_3_diff1 + value: 42.061375857749546 + - type: nauc_map_at_3_max + value: 21.164318616149437 + - type: nauc_map_at_3_std + value: 5.580282469128072 + - type: nauc_map_at_5_diff1 + value: 40.17915298851805 + - type: nauc_map_at_5_max + value: 21.74297763242879 + - type: nauc_map_at_5_std + value: 6.491160358688669 + - type: nauc_mrr_at_1000_diff1 + value: 39.10934219251255 + - type: nauc_mrr_at_1000_max + value: 21.846629873784625 + - type: nauc_mrr_at_1000_std + value: 8.057162052334247 + - type: nauc_mrr_at_100_diff1 + value: 39.08953488227305 + - type: nauc_mrr_at_100_max + value: 21.854814482583407 + - type: nauc_mrr_at_100_std + value: 8.047002010462869 + - type: nauc_mrr_at_10_diff1 + value: 39.02243568995744 + - type: nauc_mrr_at_10_max + value: 21.869186325121877 + - type: nauc_mrr_at_10_std + value: 7.752358221355221 + - type: nauc_mrr_at_1_diff1 + value: 46.55758811314048 + - type: nauc_mrr_at_1_max + value: 23.39628479504325 + - type: nauc_mrr_at_1_std + value: 6.411526263489063 + - type: nauc_mrr_at_20_diff1 + value: 39.04482497580765 + - type: nauc_mrr_at_20_max + value: 21.71096837801306 + - type: nauc_mrr_at_20_std + value: 7.792209245351021 + - type: nauc_mrr_at_3_diff1 + value: 40.972654911969606 + - type: nauc_mrr_at_3_max + value: 22.201491962131563 + - type: nauc_mrr_at_3_std + value: 7.377387305680109 + - type: nauc_mrr_at_5_diff1 + value: 38.96061797298516 + - type: nauc_mrr_at_5_max + value: 22.122457591798288 + - type: nauc_mrr_at_5_std + value: 7.960408962585282 + - type: nauc_ndcg_at_1000_diff1 + value: 37.108559297120806 + - type: nauc_ndcg_at_1000_max + value: 21.396966860957964 + - type: nauc_ndcg_at_1000_std + value: 9.204225492779988 + - type: nauc_ndcg_at_100_diff1 + value: 36.684372012765145 + - type: nauc_ndcg_at_100_max + value: 21.49123864972334 + - type: nauc_ndcg_at_100_std + value: 9.283401674369356 + - type: nauc_ndcg_at_10_diff1 + value: 36.45366504131012 + - type: nauc_ndcg_at_10_max + value: 20.676973108891747 + - type: nauc_ndcg_at_10_std + value: 7.061104890923634 + - type: nauc_ndcg_at_1_diff1 + value: 46.55758811314048 + - type: nauc_ndcg_at_1_max + value: 23.39628479504325 + - type: nauc_ndcg_at_1_std + value: 6.411526263489063 + - type: nauc_ndcg_at_20_diff1 + value: 36.46390212261509 + - type: nauc_ndcg_at_20_max + value: 20.144035227652054 + - type: nauc_ndcg_at_20_std + value: 7.399863037642277 + - type: nauc_ndcg_at_3_diff1 + value: 39.691967942016994 + - type: nauc_ndcg_at_3_max + value: 21.13548400680126 + - type: nauc_ndcg_at_3_std + value: 6.020014655984021 + - type: nauc_ndcg_at_5_diff1 + value: 36.332041845498324 + - type: nauc_ndcg_at_5_max + value: 21.366769913673863 + - type: nauc_ndcg_at_5_std + value: 7.4243373596009254 + - type: nauc_precision_at_1000_diff1 + value: -8.454624723417623 + - type: nauc_precision_at_1000_max + value: -0.9825894243576654 + - type: nauc_precision_at_1000_std + value: 1.7451960107522466 + - type: nauc_precision_at_100_diff1 + value: 5.644791706864375 + - type: nauc_precision_at_100_max + value: 15.266172301655173 + - type: nauc_precision_at_100_std + value: 13.110610839542943 + - type: nauc_precision_at_10_diff1 + value: 18.992389487759702 + - type: nauc_precision_at_10_max + value: 20.10241613456987 + - type: nauc_precision_at_10_std + value: 8.610860495036796 + - type: nauc_precision_at_1_diff1 + value: 46.55758811314048 + - type: nauc_precision_at_1_max + value: 23.39628479504325 + - type: nauc_precision_at_1_std + value: 6.411526263489063 + - type: nauc_precision_at_20_diff1 + value: 16.19784028858522 + - type: nauc_precision_at_20_max + value: 15.914334277633 + - type: nauc_precision_at_20_std + value: 9.289965849650313 + - type: nauc_precision_at_3_diff1 + value: 30.73183828980282 + - type: nauc_precision_at_3_max + value: 20.561355516958585 + - type: nauc_precision_at_3_std + value: 7.621053320590477 + - type: nauc_precision_at_5_diff1 + value: 21.068706175783465 + - type: nauc_precision_at_5_max + value: 21.58867062775011 + - type: nauc_precision_at_5_std + value: 10.67797154758921 + - type: nauc_recall_at_1000_diff1 + value: 16.448482276918444 + - type: nauc_recall_at_1000_max + value: 24.631554088464693 + - type: nauc_recall_at_1000_std + value: 40.22894954703105 + - type: nauc_recall_at_100_diff1 + value: 22.90839330496672 + - type: nauc_recall_at_100_max + value: 21.478479672427163 + - type: nauc_recall_at_100_std + value: 21.731342939613473 + - type: nauc_recall_at_10_diff1 + value: 25.46506324838181 + - type: nauc_recall_at_10_max + value: 16.731454783530985 + - type: nauc_recall_at_10_std + value: 7.749262767718348 + - type: nauc_recall_at_1_diff1 + value: 48.14626703158692 + - type: nauc_recall_at_1_max + value: 22.33923130960631 + - type: nauc_recall_at_1_std + value: 5.090028695091633 + - type: nauc_recall_at_20_diff1 + value: 24.924997945479284 + - type: nauc_recall_at_20_max + value: 14.37238378554847 + - type: nauc_recall_at_20_std + value: 8.578793415673006 + - type: nauc_recall_at_3_diff1 + value: 33.91855390710597 + - type: nauc_recall_at_3_max + value: 18.538702457334903 + - type: nauc_recall_at_3_std + value: 5.518307456996586 + - type: nauc_recall_at_5_diff1 + value: 25.76556579447583 + - type: nauc_recall_at_5_max + value: 19.124456482353093 + - type: nauc_recall_at_5_std + value: 8.55150525063633 + - type: ndcg_at_1 + value: 26.987 + - type: ndcg_at_10 + value: 36.799 + - type: ndcg_at_100 + value: 41.833999999999996 + - type: ndcg_at_1000 + value: 44.292 + - type: ndcg_at_20 + value: 38.427 + - type: ndcg_at_3 + value: 31.665 + - type: ndcg_at_5 + value: 34.129 + - type: precision_at_1 + value: 26.987 + - type: precision_at_10 + value: 5.7860000000000005 + - type: precision_at_100 + value: 0.906 + - type: precision_at_1000 + value: 0.123 + - type: precision_at_20 + value: 3.3360000000000003 + - type: precision_at_3 + value: 13.494 + - type: precision_at_5 + value: 9.686 + - type: recall_at_1 + value: 24.199 + - type: recall_at_10 + value: 49.419000000000004 + - type: recall_at_100 + value: 73.356 + - type: recall_at_1000 + value: 91.40700000000001 + - type: recall_at_20 + value: 55.467 + - type: recall_at_3 + value: 35.333999999999996 + - type: recall_at_5 + value: 41.211999999999996 + task: + type: Retrieval + - dataset: + config: default + name: MTEB ClimateFEVER + revision: 47f2ac6acb640fc46020b02a5b59fdda04d39380 + split: test + type: mteb/climate-fever + metrics: + - type: main_score + value: 43.525999999999996 + - type: map_at_1 + value: 19.291 + - type: map_at_10 + value: 33.471000000000004 + - type: map_at_100 + value: 35.388999999999996 + - type: map_at_1000 + value: 35.568 + - type: map_at_20 + value: 34.496 + - type: map_at_3 + value: 28.713 + - type: map_at_5 + value: 31.384 + - type: mrr_at_1 + value: 43.77850162866449 + - type: mrr_at_10 + value: 56.28576598934912 + - type: mrr_at_100 + value: 56.8588518168194 + - type: mrr_at_1000 + value: 56.878236725973544 + - type: mrr_at_20 + value: 56.6409328120183 + - type: mrr_at_3 + value: 53.56134636264935 + - type: mrr_at_5 + value: 55.27795874049956 + - type: nauc_map_at_1000_diff1 + value: 27.262513153363876 + - type: nauc_map_at_1000_max + value: 40.099398684385584 + - type: nauc_map_at_1000_std + value: 18.847812394005512 + - type: nauc_map_at_100_diff1 + value: 27.238993503030745 + - type: nauc_map_at_100_max + value: 40.07730434492169 + - type: nauc_map_at_100_std + value: 18.795349250833684 + - type: nauc_map_at_10_diff1 + value: 27.70929180366227 + - type: nauc_map_at_10_max + value: 39.55987024970173 + - type: nauc_map_at_10_std + value: 17.214881544648996 + - type: nauc_map_at_1_diff1 + value: 43.34155892182403 + - type: nauc_map_at_1_max + value: 38.23324890148018 + - type: nauc_map_at_1_std + value: 6.0781444393516075 + - type: nauc_map_at_20_diff1 + value: 27.311577477800103 + - type: nauc_map_at_20_max + value: 39.624414083413456 + - type: nauc_map_at_20_std + value: 18.149811054163287 + - type: nauc_map_at_3_diff1 + value: 30.475965062734367 + - type: nauc_map_at_3_max + value: 38.49324825043695 + - type: nauc_map_at_3_std + value: 13.357656038648487 + - type: nauc_map_at_5_diff1 + value: 28.425110095017747 + - type: nauc_map_at_5_max + value: 39.017894870747796 + - type: nauc_map_at_5_std + value: 15.543817194122564 + - type: nauc_mrr_at_1000_diff1 + value: 33.16689354701644 + - type: nauc_mrr_at_1000_max + value: 41.70755363247148 + - type: nauc_mrr_at_1000_std + value: 24.61667417463176 + - type: nauc_mrr_at_100_diff1 + value: 33.147229262917506 + - type: nauc_mrr_at_100_max + value: 41.712455697170725 + - type: nauc_mrr_at_100_std + value: 24.6418922043652 + - type: nauc_mrr_at_10_diff1 + value: 32.94185191112572 + - type: nauc_mrr_at_10_max + value: 41.64272730141954 + - type: nauc_mrr_at_10_std + value: 24.663391015702707 + - type: nauc_mrr_at_1_diff1 + value: 39.571969559016395 + - type: nauc_mrr_at_1_max + value: 39.396249211263495 + - type: nauc_mrr_at_1_std + value: 16.984149923258357 + - type: nauc_mrr_at_20_diff1 + value: 33.10040770334742 + - type: nauc_mrr_at_20_max + value: 41.807565560083034 + - type: nauc_mrr_at_20_std + value: 24.8064180365271 + - type: nauc_mrr_at_3_diff1 + value: 33.065406161485704 + - type: nauc_mrr_at_3_max + value: 41.049510969934694 + - type: nauc_mrr_at_3_std + value: 23.18371458928609 + - type: nauc_mrr_at_5_diff1 + value: 33.2389593543916 + - type: nauc_mrr_at_5_max + value: 41.629486918949915 + - type: nauc_mrr_at_5_std + value: 24.5777253036149 + - type: nauc_ndcg_at_1000_diff1 + value: 25.868840609197637 + - type: nauc_ndcg_at_1000_max + value: 42.79564910784761 + - type: nauc_ndcg_at_1000_std + value: 27.035091271680113 + - type: nauc_ndcg_at_100_diff1 + value: 25.019789319579942 + - type: nauc_ndcg_at_100_max + value: 42.482345143533735 + - type: nauc_ndcg_at_100_std + value: 26.76872010731345 + - type: nauc_ndcg_at_10_diff1 + value: 25.949464660653238 + - type: nauc_ndcg_at_10_max + value: 40.79769544643906 + - type: nauc_ndcg_at_10_std + value: 22.486116508973204 + - type: nauc_ndcg_at_1_diff1 + value: 39.571969559016395 + - type: nauc_ndcg_at_1_max + value: 39.396249211263495 + - type: nauc_ndcg_at_1_std + value: 16.984149923258357 + - type: nauc_ndcg_at_20_diff1 + value: 25.173455685962214 + - type: nauc_ndcg_at_20_max + value: 40.88873540662413 + - type: nauc_ndcg_at_20_std + value: 24.4451041955519 + - type: nauc_ndcg_at_3_diff1 + value: 28.185416070726333 + - type: nauc_ndcg_at_3_max + value: 39.10600031163912 + - type: nauc_ndcg_at_3_std + value: 18.42694044215541 + - type: nauc_ndcg_at_5_diff1 + value: 27.112647584005583 + - type: nauc_ndcg_at_5_max + value: 40.154045682322526 + - type: nauc_ndcg_at_5_std + value: 20.26822517176828 + - type: nauc_precision_at_1000_diff1 + value: -16.42087927044017 + - type: nauc_precision_at_1000_max + value: 3.5326295053913 + - type: nauc_precision_at_1000_std + value: 24.406810708493197 + - type: nauc_precision_at_100_diff1 + value: -12.17648135724982 + - type: nauc_precision_at_100_max + value: 15.895489260126183 + - type: nauc_precision_at_100_std + value: 32.48346122610907 + - type: nauc_precision_at_10_diff1 + value: -1.2493131347748072 + - type: nauc_precision_at_10_max + value: 26.409459305604376 + - type: nauc_precision_at_10_std + value: 31.115432019300016 + - type: nauc_precision_at_1_diff1 + value: 39.571969559016395 + - type: nauc_precision_at_1_max + value: 39.396249211263495 + - type: nauc_precision_at_1_std + value: 16.984149923258357 + - type: nauc_precision_at_20_diff1 + value: -6.597509397240593 + - type: nauc_precision_at_20_max + value: 21.461984620659695 + - type: nauc_precision_at_20_std + value: 32.9450259748889 + - type: nauc_precision_at_3_diff1 + value: 9.46378764865453 + - type: nauc_precision_at_3_max + value: 32.03650819375425 + - type: nauc_precision_at_3_std + value: 26.489382638510765 + - type: nauc_precision_at_5_diff1 + value: 3.5987036728169537 + - type: nauc_precision_at_5_max + value: 30.633955978579703 + - type: nauc_precision_at_5_std + value: 30.532430088014443 + - type: nauc_recall_at_1000_diff1 + value: 10.714633106872254 + - type: nauc_recall_at_1000_max + value: 43.94958623961 + - type: nauc_recall_at_1000_std + value: 51.78914468954123 + - type: nauc_recall_at_100_diff1 + value: 9.63781472255557 + - type: nauc_recall_at_100_max + value: 38.50917465255336 + - type: nauc_recall_at_100_std + value: 37.78623984642377 + - type: nauc_recall_at_10_diff1 + value: 16.480342820841688 + - type: nauc_recall_at_10_max + value: 35.982566867357406 + - type: nauc_recall_at_10_std + value: 23.30688188788895 + - type: nauc_recall_at_1_diff1 + value: 43.34155892182403 + - type: nauc_recall_at_1_max + value: 38.23324890148018 + - type: nauc_recall_at_1_std + value: 6.0781444393516075 + - type: nauc_recall_at_20_diff1 + value: 13.521048985146367 + - type: nauc_recall_at_20_max + value: 34.62462209239834 + - type: nauc_recall_at_20_std + value: 27.85924191501618 + - type: nauc_recall_at_3_diff1 + value: 23.57032748533523 + - type: nauc_recall_at_3_max + value: 36.32703197635613 + - type: nauc_recall_at_3_std + value: 15.730238734014337 + - type: nauc_recall_at_5_diff1 + value: 19.61387036368584 + - type: nauc_recall_at_5_max + value: 36.22030835529556 + - type: nauc_recall_at_5_std + value: 19.76310648649897 + - type: ndcg_at_1 + value: 43.779 + - type: ndcg_at_10 + value: 43.525999999999996 + - type: ndcg_at_100 + value: 50.138000000000005 + - type: ndcg_at_1000 + value: 52.991 + - type: ndcg_at_20 + value: 46.083 + - type: ndcg_at_3 + value: 38.002 + - type: ndcg_at_5 + value: 39.842 + - type: precision_at_1 + value: 43.779 + - type: precision_at_10 + value: 13.205 + - type: precision_at_100 + value: 2.051 + - type: precision_at_1000 + value: 0.259 + - type: precision_at_20 + value: 7.722999999999999 + - type: precision_at_3 + value: 28.903000000000002 + - type: precision_at_5 + value: 21.368000000000002 + - type: recall_at_1 + value: 19.291 + - type: recall_at_10 + value: 48.754 + - type: recall_at_100 + value: 70.97200000000001 + - type: recall_at_1000 + value: 86.611 + - type: recall_at_20 + value: 55.884 + - type: recall_at_3 + value: 34.101 + - type: recall_at_5 + value: 40.784 + task: + type: Retrieval + - dataset: + config: default + name: MTEB DBPedia + revision: c0f706b76e590d620bd6618b3ca8efdd34e2d659 + split: test + type: mteb/dbpedia + metrics: + - type: main_score + value: 49.884 + - type: map_at_1 + value: 9.913 + - type: map_at_10 + value: 23.186999999999998 + - type: map_at_100 + value: 34.207 + - type: map_at_1000 + value: 36.318 + - type: map_at_20 + value: 27.419 + - type: map_at_3 + value: 15.656 + - type: map_at_5 + value: 18.945999999999998 + - type: mrr_at_1 + value: 75.75 + - type: mrr_at_10 + value: 82.16279761904761 + - type: mrr_at_100 + value: 82.48445635330299 + - type: mrr_at_1000 + value: 82.4870246719901 + - type: mrr_at_20 + value: 82.36203632968338 + - type: mrr_at_3 + value: 81.29166666666666 + - type: mrr_at_5 + value: 82.02916666666667 + - type: nauc_map_at_1000_diff1 + value: 17.0739966990996 + - type: nauc_map_at_1000_max + value: 28.440065298437133 + - type: nauc_map_at_1000_std + value: 20.83498154003865 + - type: nauc_map_at_100_diff1 + value: 17.75982086107111 + - type: nauc_map_at_100_max + value: 26.87850835673573 + - type: nauc_map_at_100_std + value: 18.350282298599275 + - type: nauc_map_at_10_diff1 + value: 17.15984258564116 + - type: nauc_map_at_10_max + value: 10.846179132675553 + - type: nauc_map_at_10_std + value: -6.263534464094614 + - type: nauc_map_at_1_diff1 + value: 24.014897777973694 + - type: nauc_map_at_1_max + value: -4.556638938723358 + - type: nauc_map_at_1_std + value: -22.7844467526989 + - type: nauc_map_at_20_diff1 + value: 16.3179372493187 + - type: nauc_map_at_20_max + value: 17.176378915498915 + - type: nauc_map_at_20_std + value: 1.9378637630340372 + - type: nauc_map_at_3_diff1 + value: 19.12786794046792 + - type: nauc_map_at_3_max + value: 0.09063919305677291 + - type: nauc_map_at_3_std + value: -16.713143158330492 + - type: nauc_map_at_5_diff1 + value: 18.76504725420023 + - type: nauc_map_at_5_max + value: 5.040867712207419 + - type: nauc_map_at_5_std + value: -12.382578318931165 + - type: nauc_mrr_at_1000_diff1 + value: 54.61266255011247 + - type: nauc_mrr_at_1000_max + value: 60.83961280977112 + - type: nauc_mrr_at_1000_std + value: 32.70429260443016 + - type: nauc_mrr_at_100_diff1 + value: 54.61346236538542 + - type: nauc_mrr_at_100_max + value: 60.8407974416647 + - type: nauc_mrr_at_100_std + value: 32.69272843993462 + - type: nauc_mrr_at_10_diff1 + value: 54.74633685810871 + - type: nauc_mrr_at_10_max + value: 61.084525933097865 + - type: nauc_mrr_at_10_std + value: 33.001220210025565 + - type: nauc_mrr_at_1_diff1 + value: 56.12708423835806 + - type: nauc_mrr_at_1_max + value: 58.9314540998289 + - type: nauc_mrr_at_1_std + value: 27.39422607651012 + - type: nauc_mrr_at_20_diff1 + value: 54.58896150245695 + - type: nauc_mrr_at_20_max + value: 60.890929983464815 + - type: nauc_mrr_at_20_std + value: 32.65559641276393 + - type: nauc_mrr_at_3_diff1 + value: 54.38229071443791 + - type: nauc_mrr_at_3_max + value: 59.987849044098596 + - type: nauc_mrr_at_3_std + value: 33.439813880719974 + - type: nauc_mrr_at_5_diff1 + value: 54.961790262449824 + - type: nauc_mrr_at_5_max + value: 61.17705173908951 + - type: nauc_mrr_at_5_std + value: 33.30939850734856 + - type: nauc_ndcg_at_1000_diff1 + value: 29.27465932507067 + - type: nauc_ndcg_at_1000_max + value: 47.952543312315214 + - type: nauc_ndcg_at_1000_std + value: 36.17132236391485 + - type: nauc_ndcg_at_100_diff1 + value: 28.63072328980134 + - type: nauc_ndcg_at_100_max + value: 41.460833419186564 + - type: nauc_ndcg_at_100_std + value: 27.157100358988135 + - type: nauc_ndcg_at_10_diff1 + value: 23.41488013023301 + - type: nauc_ndcg_at_10_max + value: 39.27798133072349 + - type: nauc_ndcg_at_10_std + value: 21.979241438928312 + - type: nauc_ndcg_at_1_diff1 + value: 46.12120543657642 + - type: nauc_ndcg_at_1_max + value: 47.28452124039853 + - type: nauc_ndcg_at_1_std + value: 19.799884708952543 + - type: nauc_ndcg_at_20_diff1 + value: 23.627669045115574 + - type: nauc_ndcg_at_20_max + value: 35.88225062457673 + - type: nauc_ndcg_at_20_std + value: 18.218628030529498 + - type: nauc_ndcg_at_3_diff1 + value: 25.37309228946118 + - type: nauc_ndcg_at_3_max + value: 40.64426332992231 + - type: nauc_ndcg_at_3_std + value: 24.608330645901482 + - type: nauc_ndcg_at_5_diff1 + value: 24.055798594999654 + - type: nauc_ndcg_at_5_max + value: 41.16180524175431 + - type: nauc_ndcg_at_5_std + value: 24.048305528761315 + - type: nauc_precision_at_1000_diff1 + value: -18.234943251015576 + - type: nauc_precision_at_1000_max + value: 0.48708502364659184 + - type: nauc_precision_at_1000_std + value: 2.4473601543134027 + - type: nauc_precision_at_100_diff1 + value: -3.0077810947381227 + - type: nauc_precision_at_100_max + value: 25.27249321108913 + - type: nauc_precision_at_100_std + value: 37.36575792126928 + - type: nauc_precision_at_10_diff1 + value: -0.2393778190297635 + - type: nauc_precision_at_10_max + value: 36.40513293547299 + - type: nauc_precision_at_10_std + value: 37.4827885766009 + - type: nauc_precision_at_1_diff1 + value: 56.12708423835806 + - type: nauc_precision_at_1_max + value: 58.9314540998289 + - type: nauc_precision_at_1_std + value: 27.39422607651012 + - type: nauc_precision_at_20_diff1 + value: -1.2010133229402933 + - type: nauc_precision_at_20_max + value: 34.117541814385966 + - type: nauc_precision_at_20_std + value: 39.13273254177449 + - type: nauc_precision_at_3_diff1 + value: 11.757378092198486 + - type: nauc_precision_at_3_max + value: 42.637962482588875 + - type: nauc_precision_at_3_std + value: 37.42465077352342 + - type: nauc_precision_at_5_diff1 + value: 7.233177203405101 + - type: nauc_precision_at_5_max + value: 43.1663582897407 + - type: nauc_precision_at_5_std + value: 38.848449220750055 + - type: nauc_recall_at_1000_diff1 + value: 27.33938551969145 + - type: nauc_recall_at_1000_max + value: 45.5614254479334 + - type: nauc_recall_at_1000_std + value: 50.58528916250458 + - type: nauc_recall_at_100_diff1 + value: 23.610383761920097 + - type: nauc_recall_at_100_max + value: 31.422168485847184 + - type: nauc_recall_at_100_std + value: 25.58649926458304 + - type: nauc_recall_at_10_diff1 + value: 14.62495111808408 + - type: nauc_recall_at_10_max + value: 7.4295041277681095 + - type: nauc_recall_at_10_std + value: -9.32297089600654 + - type: nauc_recall_at_1_diff1 + value: 24.014897777973694 + - type: nauc_recall_at_1_max + value: -4.556638938723358 + - type: nauc_recall_at_1_std + value: -22.7844467526989 + - type: nauc_recall_at_20_diff1 + value: 14.027862330014662 + - type: nauc_recall_at_20_max + value: 12.437478731690844 + - type: nauc_recall_at_20_std + value: -3.0740743798103676 + - type: nauc_recall_at_3_diff1 + value: 16.354018356566712 + - type: nauc_recall_at_3_max + value: -2.9812231240997917 + - type: nauc_recall_at_3_std + value: -18.27746460743442 + - type: nauc_recall_at_5_diff1 + value: 16.81486583473587 + - type: nauc_recall_at_5_max + value: 2.420128513974744 + - type: nauc_recall_at_5_std + value: -14.441820321214108 + - type: ndcg_at_1 + value: 63.87500000000001 + - type: ndcg_at_10 + value: 49.884 + - type: ndcg_at_100 + value: 54.738 + - type: ndcg_at_1000 + value: 61.635 + - type: ndcg_at_20 + value: 48.894999999999996 + - type: ndcg_at_3 + value: 54.287 + - type: ndcg_at_5 + value: 52.40899999999999 + - type: precision_at_1 + value: 75.75 + - type: precision_at_10 + value: 40.9 + - type: precision_at_100 + value: 13.139999999999999 + - type: precision_at_1000 + value: 2.533 + - type: precision_at_20 + value: 30.8 + - type: precision_at_3 + value: 57.667 + - type: precision_at_5 + value: 51.05 + - type: recall_at_1 + value: 9.913 + - type: recall_at_10 + value: 28.591 + - type: recall_at_100 + value: 61.017999999999994 + - type: recall_at_1000 + value: 83.383 + - type: recall_at_20 + value: 37.834 + - type: recall_at_3 + value: 17.049 + - type: recall_at_5 + value: 21.685 + task: + type: Retrieval + - dataset: + config: default + name: MTEB EmotionClassification + revision: 4f58c6b202a23cf9a4da393831edf4f9183cad37 + split: test + type: mteb/emotion + metrics: + - type: accuracy + value: 78.77499999999999 + - type: f1 + value: 73.74058240799386 + - type: f1_weighted + value: 79.78804377638227 + - type: main_score + value: 78.77499999999999 + task: + type: Classification + - dataset: + config: default + name: MTEB FEVER + revision: bea83ef9e8fb933d90a2f1d5515737465d613e12 + split: test + type: mteb/fever + metrics: + - type: main_score + value: 90.986 + - type: map_at_1 + value: 81.601 + - type: map_at_10 + value: 88.242 + - type: map_at_100 + value: 88.46000000000001 + - type: map_at_1000 + value: 88.472 + - type: map_at_20 + value: 88.375 + - type: map_at_3 + value: 87.237 + - type: map_at_5 + value: 87.85300000000001 + - type: mrr_at_1 + value: 87.81878187818782 + - type: mrr_at_10 + value: 92.20301196786335 + - type: mrr_at_100 + value: 92.24884236673292 + - type: mrr_at_1000 + value: 92.2496338899362 + - type: mrr_at_20 + value: 92.23112073283473 + - type: mrr_at_3 + value: 91.77417741774165 + - type: mrr_at_5 + value: 92.03970397039689 + - type: nauc_map_at_1000_diff1 + value: 56.54670664910505 + - type: nauc_map_at_1000_max + value: 33.08375749975477 + - type: nauc_map_at_1000_std + value: 2.7491595418252865 + - type: nauc_map_at_100_diff1 + value: 56.50887688686924 + - type: nauc_map_at_100_max + value: 33.075487189958494 + - type: nauc_map_at_100_std + value: 2.7675869969253375 + - type: nauc_map_at_10_diff1 + value: 56.08080806610569 + - type: nauc_map_at_10_max + value: 32.776972098819066 + - type: nauc_map_at_10_std + value: 2.5904846711290097 + - type: nauc_map_at_1_diff1 + value: 60.645344065853145 + - type: nauc_map_at_1_max + value: 31.232776777514797 + - type: nauc_map_at_1_std + value: -1.1946138176109171 + - type: nauc_map_at_20_diff1 + value: 56.28378454162355 + - type: nauc_map_at_20_max + value: 32.98207150385811 + - type: nauc_map_at_20_std + value: 2.8469814040214025 + - type: nauc_map_at_3_diff1 + value: 55.81958007095375 + - type: nauc_map_at_3_max + value: 31.602707711038313 + - type: nauc_map_at_3_std + value: 0.8117019292273401 + - type: nauc_map_at_5_diff1 + value: 55.706025752316535 + - type: nauc_map_at_5_max + value: 32.16032683604737 + - type: nauc_map_at_5_std + value: 1.8853201503498669 + - type: nauc_mrr_at_1000_diff1 + value: 75.4997173366251 + - type: nauc_mrr_at_1000_max + value: 41.49117135484116 + - type: nauc_mrr_at_1000_std + value: -2.0636172883680852 + - type: nauc_mrr_at_100_diff1 + value: 75.50118860648519 + - type: nauc_mrr_at_100_max + value: 41.49490161517194 + - type: nauc_mrr_at_100_std + value: -2.057024385178682 + - type: nauc_mrr_at_10_diff1 + value: 75.47295153099428 + - type: nauc_mrr_at_10_max + value: 41.55003304042536 + - type: nauc_mrr_at_10_std + value: -2.0353663198929253 + - type: nauc_mrr_at_1_diff1 + value: 76.632058433229 + - type: nauc_mrr_at_1_max + value: 39.754483718891656 + - type: nauc_mrr_at_1_std + value: -2.962241058101701 + - type: nauc_mrr_at_20_diff1 + value: 75.47221882396194 + - type: nauc_mrr_at_20_max + value: 41.50779280480839 + - type: nauc_mrr_at_20_std + value: -1.9620212266426307 + - type: nauc_mrr_at_3_diff1 + value: 75.5682297897137 + - type: nauc_mrr_at_3_max + value: 41.53543801506081 + - type: nauc_mrr_at_3_std + value: -3.391681195945978 + - type: nauc_mrr_at_5_diff1 + value: 75.37562775183947 + - type: nauc_mrr_at_5_max + value: 41.42028509006753 + - type: nauc_mrr_at_5_std + value: -2.418698675622726 + - type: nauc_ndcg_at_1000_diff1 + value: 59.364557011624 + - type: nauc_ndcg_at_1000_max + value: 35.4112238125149 + - type: nauc_ndcg_at_1000_std + value: 3.717516193303376 + - type: nauc_ndcg_at_100_diff1 + value: 58.55706703023122 + - type: nauc_ndcg_at_100_max + value: 35.352285999934594 + - type: nauc_ndcg_at_100_std + value: 4.273437944266781 + - type: nauc_ndcg_at_10_diff1 + value: 56.77422701267037 + - type: nauc_ndcg_at_10_max + value: 34.24909893882957 + - type: nauc_ndcg_at_10_std + value: 4.178151434006727 + - type: nauc_ndcg_at_1_diff1 + value: 76.632058433229 + - type: nauc_ndcg_at_1_max + value: 39.754483718891656 + - type: nauc_ndcg_at_1_std + value: -2.962241058101701 + - type: nauc_ndcg_at_20_diff1 + value: 57.27343398231262 + - type: nauc_ndcg_at_20_max + value: 34.7416626740278 + - type: nauc_ndcg_at_20_std + value: 4.955858766014002 + - type: nauc_ndcg_at_3_diff1 + value: 57.69267803121093 + - type: nauc_ndcg_at_3_max + value: 33.13744317023105 + - type: nauc_ndcg_at_3_std + value: 0.40380284030057023 + - type: nauc_ndcg_at_5_diff1 + value: 56.57461019113917 + - type: nauc_ndcg_at_5_max + value: 33.244657840804386 + - type: nauc_ndcg_at_5_std + value: 2.5121440827702046 + - type: nauc_precision_at_1000_diff1 + value: -14.54492513449718 + - type: nauc_precision_at_1000_max + value: -5.94552147573623 + - type: nauc_precision_at_1000_std + value: 1.2446209816057374 + - type: nauc_precision_at_100_diff1 + value: -15.452676132568344 + - type: nauc_precision_at_100_max + value: -3.760241749847617 + - type: nauc_precision_at_100_std + value: 4.623534605290865 + - type: nauc_precision_at_10_diff1 + value: -12.712908026086176 + - type: nauc_precision_at_10_max + value: 0.45241316994816805 + - type: nauc_precision_at_10_std + value: 7.849478570138391 + - type: nauc_precision_at_1_diff1 + value: 76.632058433229 + - type: nauc_precision_at_1_max + value: 39.754483718891656 + - type: nauc_precision_at_1_std + value: -2.962241058101701 + - type: nauc_precision_at_20_diff1 + value: -14.514618673172041 + - type: nauc_precision_at_20_max + value: -1.113635490621818 + - type: nauc_precision_at_20_std + value: 8.599811730457576 + - type: nauc_precision_at_3_diff1 + value: 6.1367799850003815 + - type: nauc_precision_at_3_max + value: 8.466271950897857 + - type: nauc_precision_at_3_std + value: 1.7458051543195068 + - type: nauc_precision_at_5_diff1 + value: -5.804548945783379 + - type: nauc_precision_at_5_max + value: 3.4060251839074818 + - type: nauc_precision_at_5_std + value: 5.583410511782371 + - type: nauc_recall_at_1000_diff1 + value: 19.329432953574095 + - type: nauc_recall_at_1000_max + value: 43.260442595158736 + - type: nauc_recall_at_1000_std + value: 53.89644660661804 + - type: nauc_recall_at_100_diff1 + value: 21.265326296051235 + - type: nauc_recall_at_100_max + value: 38.573000195373695 + - type: nauc_recall_at_100_std + value: 42.169391082152785 + - type: nauc_recall_at_10_diff1 + value: 29.785129558987432 + - type: nauc_recall_at_10_max + value: 28.379657867558034 + - type: nauc_recall_at_10_std + value: 21.132574624091973 + - type: nauc_recall_at_1_diff1 + value: 60.645344065853145 + - type: nauc_recall_at_1_max + value: 31.232776777514797 + - type: nauc_recall_at_1_std + value: -1.1946138176109171 + - type: nauc_recall_at_20_diff1 + value: 25.88845612373954 + - type: nauc_recall_at_20_max + value: 30.24785945821152 + - type: nauc_recall_at_20_std + value: 31.73911437468067 + - type: nauc_recall_at_3_diff1 + value: 42.2968464797395 + - type: nauc_recall_at_3_max + value: 26.494318009870018 + - type: nauc_recall_at_3_std + value: 2.6045977160467544 + - type: nauc_recall_at_5_diff1 + value: 35.81340094401374 + - type: nauc_recall_at_5_max + value: 25.91082947510634 + - type: nauc_recall_at_5_std + value: 9.759404930864779 + - type: ndcg_at_1 + value: 87.819 + - type: ndcg_at_10 + value: 90.986 + - type: ndcg_at_100 + value: 91.69 + - type: ndcg_at_1000 + value: 91.863 + - type: ndcg_at_20 + value: 91.293 + - type: ndcg_at_3 + value: 89.621 + - type: ndcg_at_5 + value: 90.333 + - type: precision_at_1 + value: 87.819 + - type: precision_at_10 + value: 10.753 + - type: precision_at_100 + value: 1.138 + - type: precision_at_1000 + value: 0.117 + - type: precision_at_20 + value: 5.4879999999999995 + - type: precision_at_3 + value: 33.703 + - type: precision_at_5 + value: 20.831 + - type: recall_at_1 + value: 81.601 + - type: recall_at_10 + value: 95.44200000000001 + - type: recall_at_100 + value: 98.14399999999999 + - type: recall_at_1000 + value: 99.157 + - type: recall_at_20 + value: 96.43 + - type: recall_at_3 + value: 91.729 + - type: recall_at_5 + value: 93.552 + task: + type: Retrieval + - dataset: + config: default + name: MTEB FiQA2018 + revision: 27a168819829fe9bcd655c2df245fb19452e8e06 + split: test + type: mteb/fiqa + metrics: + - type: main_score + value: 56.056 + - type: map_at_1 + value: 28.666000000000004 + - type: map_at_10 + value: 47.437000000000005 + - type: map_at_100 + value: 49.537 + - type: map_at_1000 + value: 49.665 + - type: map_at_20 + value: 48.618 + - type: map_at_3 + value: 41.355 + - type: map_at_5 + value: 44.525 + - type: mrr_at_1 + value: 55.55555555555556 + - type: mrr_at_10 + value: 63.705173427395614 + - type: mrr_at_100 + value: 64.25449940779741 + - type: mrr_at_1000 + value: 64.27635581092147 + - type: mrr_at_20 + value: 64.03796029079103 + - type: mrr_at_3 + value: 61.49691358024688 + - type: mrr_at_5 + value: 62.73148148148143 + - type: nauc_map_at_1000_diff1 + value: 43.24282910397747 + - type: nauc_map_at_1000_max + value: 28.506093180265644 + - type: nauc_map_at_1000_std + value: -13.040508386155054 + - type: nauc_map_at_100_diff1 + value: 43.23650442904607 + - type: nauc_map_at_100_max + value: 28.470565635459156 + - type: nauc_map_at_100_std + value: -12.988098780714935 + - type: nauc_map_at_10_diff1 + value: 43.393840733087686 + - type: nauc_map_at_10_max + value: 26.637302062720153 + - type: nauc_map_at_10_std + value: -14.47500292113762 + - type: nauc_map_at_1_diff1 + value: 47.705150227211725 + - type: nauc_map_at_1_max + value: 15.354189686550129 + - type: nauc_map_at_1_std + value: -14.559819859039067 + - type: nauc_map_at_20_diff1 + value: 43.14121075706104 + - type: nauc_map_at_20_max + value: 27.811170590408395 + - type: nauc_map_at_20_std + value: -13.459413585283583 + - type: nauc_map_at_3_diff1 + value: 44.33938667720801 + - type: nauc_map_at_3_max + value: 21.785619884549398 + - type: nauc_map_at_3_std + value: -15.569980103071593 + - type: nauc_map_at_5_diff1 + value: 43.39280905665027 + - type: nauc_map_at_5_max + value: 25.021492190645017 + - type: nauc_map_at_5_std + value: -14.48856622187443 + - type: nauc_mrr_at_1000_diff1 + value: 52.971563939946286 + - type: nauc_mrr_at_1000_max + value: 38.88019486172324 + - type: nauc_mrr_at_1000_std + value: -12.412991642381616 + - type: nauc_mrr_at_100_diff1 + value: 52.978468139876945 + - type: nauc_mrr_at_100_max + value: 38.89751787948751 + - type: nauc_mrr_at_100_std + value: -12.3677876252269 + - type: nauc_mrr_at_10_diff1 + value: 52.78507148048174 + - type: nauc_mrr_at_10_max + value: 38.55079809310022 + - type: nauc_mrr_at_10_std + value: -12.944127025078755 + - type: nauc_mrr_at_1_diff1 + value: 55.52626805861546 + - type: nauc_mrr_at_1_max + value: 40.49306809164979 + - type: nauc_mrr_at_1_std + value: -12.886607701317681 + - type: nauc_mrr_at_20_diff1 + value: 52.9592152665678 + - type: nauc_mrr_at_20_max + value: 38.88514014589964 + - type: nauc_mrr_at_20_std + value: -12.434464359819444 + - type: nauc_mrr_at_3_diff1 + value: 52.73696844091174 + - type: nauc_mrr_at_3_max + value: 38.61018727252859 + - type: nauc_mrr_at_3_std + value: -13.123989867364166 + - type: nauc_mrr_at_5_diff1 + value: 53.037110010188 + - type: nauc_mrr_at_5_max + value: 38.44770729849151 + - type: nauc_mrr_at_5_std + value: -13.49318771828972 + - type: nauc_ndcg_at_1000_diff1 + value: 44.73813840091289 + - type: nauc_ndcg_at_1000_max + value: 33.70113904685389 + - type: nauc_ndcg_at_1000_std + value: -10.328687058192742 + - type: nauc_ndcg_at_100_diff1 + value: 44.595174119928835 + - type: nauc_ndcg_at_100_max + value: 33.4788285112467 + - type: nauc_ndcg_at_100_std + value: -8.695355259716946 + - type: nauc_ndcg_at_10_diff1 + value: 44.39837225263 + - type: nauc_ndcg_at_10_max + value: 29.188289725593393 + - type: nauc_ndcg_at_10_std + value: -13.67608323673103 + - type: nauc_ndcg_at_1_diff1 + value: 55.52626805861546 + - type: nauc_ndcg_at_1_max + value: 40.49306809164979 + - type: nauc_ndcg_at_1_std + value: -12.886607701317681 + - type: nauc_ndcg_at_20_diff1 + value: 44.24661739902305 + - type: nauc_ndcg_at_20_max + value: 31.667868318249965 + - type: nauc_ndcg_at_20_std + value: -10.65470780066342 + - type: nauc_ndcg_at_3_diff1 + value: 43.39857166975522 + - type: nauc_ndcg_at_3_max + value: 31.764668313577495 + - type: nauc_ndcg_at_3_std + value: -14.494866954678152 + - type: nauc_ndcg_at_5_diff1 + value: 43.16976647347281 + - type: nauc_ndcg_at_5_max + value: 29.878329062643143 + - type: nauc_ndcg_at_5_std + value: -13.987689089179739 + - type: nauc_precision_at_1000_diff1 + value: -9.807973252625484 + - type: nauc_precision_at_1000_max + value: 26.6279603849494 + - type: nauc_precision_at_1000_std + value: 7.113187103520632 + - type: nauc_precision_at_100_diff1 + value: -4.777149603323976 + - type: nauc_precision_at_100_max + value: 31.03410463692187 + - type: nauc_precision_at_100_std + value: 10.463144150275435 + - type: nauc_precision_at_10_diff1 + value: 8.691528703215962 + - type: nauc_precision_at_10_max + value: 33.329579434123374 + - type: nauc_precision_at_10_std + value: -0.8002015226329403 + - type: nauc_precision_at_1_diff1 + value: 55.52626805861546 + - type: nauc_precision_at_1_max + value: 40.49306809164979 + - type: nauc_precision_at_1_std + value: -12.886607701317681 + - type: nauc_precision_at_20_diff1 + value: 3.4564653474184284 + - type: nauc_precision_at_20_max + value: 34.401070158471136 + - type: nauc_precision_at_20_std + value: 5.813431200164549 + - type: nauc_precision_at_3_diff1 + value: 22.463219705462187 + - type: nauc_precision_at_3_max + value: 34.77413976546924 + - type: nauc_precision_at_3_std + value: -7.083890789741479 + - type: nauc_precision_at_5_diff1 + value: 14.011006004883154 + - type: nauc_precision_at_5_max + value: 35.73655466853702 + - type: nauc_precision_at_5_std + value: -2.8395172077771598 + - type: nauc_recall_at_1000_diff1 + value: 16.478046357391555 + - type: nauc_recall_at_1000_max + value: 43.231704288282344 + - type: nauc_recall_at_1000_std + value: 38.430684937573645 + - type: nauc_recall_at_100_diff1 + value: 30.764718344602436 + - type: nauc_recall_at_100_max + value: 31.769050487166655 + - type: nauc_recall_at_100_std + value: 23.48468311677149 + - type: nauc_recall_at_10_diff1 + value: 34.47339565324045 + - type: nauc_recall_at_10_max + value: 19.054212335800454 + - type: nauc_recall_at_10_std + value: -11.039734015330437 + - type: nauc_recall_at_1_diff1 + value: 47.705150227211725 + - type: nauc_recall_at_1_max + value: 15.354189686550129 + - type: nauc_recall_at_1_std + value: -14.559819859039067 + - type: nauc_recall_at_20_diff1 + value: 32.1011474016873 + - type: nauc_recall_at_20_max + value: 25.546372988304423 + - type: nauc_recall_at_20_std + value: -0.007233471152482897 + - type: nauc_recall_at_3_diff1 + value: 37.5708138019065 + - type: nauc_recall_at_3_max + value: 16.66410785756736 + - type: nauc_recall_at_3_std + value: -15.404817020108966 + - type: nauc_recall_at_5_diff1 + value: 35.714519648479595 + - type: nauc_recall_at_5_max + value: 19.02075233009296 + - type: nauc_recall_at_5_std + value: -13.180963359760725 + - type: ndcg_at_1 + value: 55.556000000000004 + - type: ndcg_at_10 + value: 56.056 + - type: ndcg_at_100 + value: 62.44 + - type: ndcg_at_1000 + value: 64.263 + - type: ndcg_at_20 + value: 58.638999999999996 + - type: ndcg_at_3 + value: 51.722 + - type: ndcg_at_5 + value: 52.701 + - type: precision_at_1 + value: 55.556000000000004 + - type: precision_at_10 + value: 15.679000000000002 + - type: precision_at_100 + value: 2.252 + - type: precision_at_1000 + value: 0.257 + - type: precision_at_20 + value: 9.02 + - type: precision_at_3 + value: 34.619 + - type: precision_at_5 + value: 25.093 + - type: recall_at_1 + value: 28.666000000000004 + - type: recall_at_10 + value: 63.717999999999996 + - type: recall_at_100 + value: 86.938 + - type: recall_at_1000 + value: 97.603 + - type: recall_at_20 + value: 71.649 + - type: recall_at_3 + value: 46.663 + - type: recall_at_5 + value: 53.313 + task: + type: Retrieval + - dataset: + config: default + name: MTEB HotpotQA + revision: ab518f4d6fcca38d87c25209f94beba119d02014 + split: test + type: mteb/hotpotqa + metrics: + - type: main_score + value: 71.74199999999999 + - type: map_at_1 + value: 41.729 + - type: map_at_10 + value: 63.168 + - type: map_at_100 + value: 64.132 + - type: map_at_1000 + value: 64.199 + - type: map_at_20 + value: 63.736000000000004 + - type: map_at_3 + value: 59.826 + - type: map_at_5 + value: 61.882000000000005 + - type: mrr_at_1 + value: 83.45712356515868 + - type: mrr_at_10 + value: 87.850342432719 + - type: mrr_at_100 + value: 88.0016320691113 + - type: mrr_at_1000 + value: 88.00576596968136 + - type: mrr_at_20 + value: 87.94463253190389 + - type: mrr_at_3 + value: 87.13706954760278 + - type: mrr_at_5 + value: 87.59419311276136 + - type: nauc_map_at_1000_diff1 + value: 13.635446621095054 + - type: nauc_map_at_1000_max + value: 18.670632529445633 + - type: nauc_map_at_1000_std + value: 10.444842636150575 + - type: nauc_map_at_100_diff1 + value: 13.599262398010783 + - type: nauc_map_at_100_max + value: 18.636389405484806 + - type: nauc_map_at_100_std + value: 10.460027483576043 + - type: nauc_map_at_10_diff1 + value: 13.235053919323942 + - type: nauc_map_at_10_max + value: 18.252140477080047 + - type: nauc_map_at_10_std + value: 9.9075337042203 + - type: nauc_map_at_1_diff1 + value: 76.51940497836482 + - type: nauc_map_at_1_max + value: 51.251419487235474 + - type: nauc_map_at_1_std + value: 0.16714896857146574 + - type: nauc_map_at_20_diff1 + value: 13.4178245722222 + - type: nauc_map_at_20_max + value: 18.40988771210718 + - type: nauc_map_at_20_std + value: 10.216685163366282 + - type: nauc_map_at_3_diff1 + value: 13.38370761663418 + - type: nauc_map_at_3_max + value: 17.760962555456537 + - type: nauc_map_at_3_std + value: 7.15741965624388 + - type: nauc_map_at_5_diff1 + value: 13.138133309724855 + - type: nauc_map_at_5_max + value: 17.871761295251044 + - type: nauc_map_at_5_std + value: 8.475147426940074 + - type: nauc_mrr_at_1000_diff1 + value: 75.82650818891959 + - type: nauc_mrr_at_1000_max + value: 53.6736100668434 + - type: nauc_mrr_at_1000_std + value: 1.8025016349213916 + - type: nauc_mrr_at_100_diff1 + value: 75.82530574210111 + - type: nauc_mrr_at_100_max + value: 53.68067545829002 + - type: nauc_mrr_at_100_std + value: 1.8147470536495791 + - type: nauc_mrr_at_10_diff1 + value: 75.8330135686799 + - type: nauc_mrr_at_10_max + value: 53.78626885349077 + - type: nauc_mrr_at_10_std + value: 1.7975782717226636 + - type: nauc_mrr_at_1_diff1 + value: 76.51940497836482 + - type: nauc_mrr_at_1_max + value: 51.251419487235474 + - type: nauc_mrr_at_1_std + value: 0.16714896857146574 + - type: nauc_mrr_at_20_diff1 + value: 75.82783382464166 + - type: nauc_mrr_at_20_max + value: 53.68364567043885 + - type: nauc_mrr_at_20_std + value: 1.742037904463963 + - type: nauc_mrr_at_3_diff1 + value: 75.6944609768663 + - type: nauc_mrr_at_3_max + value: 53.803941340341666 + - type: nauc_mrr_at_3_std + value: 1.1849945458077804 + - type: nauc_mrr_at_5_diff1 + value: 75.73006960604903 + - type: nauc_mrr_at_5_max + value: 53.62223096420106 + - type: nauc_mrr_at_5_std + value: 1.6144067563410909 + - type: nauc_ndcg_at_1000_diff1 + value: 21.58025241642726 + - type: nauc_ndcg_at_1000_max + value: 24.675747527001153 + - type: nauc_ndcg_at_1000_std + value: 13.075943547492718 + - type: nauc_ndcg_at_100_diff1 + value: 20.30260137544846 + - type: nauc_ndcg_at_100_max + value: 23.757528813872018 + - type: nauc_ndcg_at_100_std + value: 13.648994687574062 + - type: nauc_ndcg_at_10_diff1 + value: 18.995052360997818 + - type: nauc_ndcg_at_10_max + value: 22.254260808196037 + - type: nauc_ndcg_at_10_std + value: 11.27212390633054 + - type: nauc_ndcg_at_1_diff1 + value: 76.51940497836482 + - type: nauc_ndcg_at_1_max + value: 51.251419487235474 + - type: nauc_ndcg_at_1_std + value: 0.16714896857146574 + - type: nauc_ndcg_at_20_diff1 + value: 19.333742380695757 + - type: nauc_ndcg_at_20_max + value: 22.527779834633364 + - type: nauc_ndcg_at_20_std + value: 12.161009000707917 + - type: nauc_ndcg_at_3_diff1 + value: 20.013329040965534 + - type: nauc_ndcg_at_3_max + value: 21.99692460311921 + - type: nauc_ndcg_at_3_std + value: 6.8076290638386165 + - type: nauc_ndcg_at_5_diff1 + value: 19.08226315942471 + - type: nauc_ndcg_at_5_max + value: 21.71185964294168 + - type: nauc_ndcg_at_5_std + value: 8.671911269518214 + - type: nauc_precision_at_1000_diff1 + value: 2.4462475489446764 + - type: nauc_precision_at_1000_max + value: 29.145662064268578 + - type: nauc_precision_at_1000_std + value: 49.20704909525856 + - type: nauc_precision_at_100_diff1 + value: 0.11271196725540299 + - type: nauc_precision_at_100_max + value: 17.37584606388067 + - type: nauc_precision_at_100_std + value: 34.66099346244071 + - type: nauc_precision_at_10_diff1 + value: 2.9923183951227825 + - type: nauc_precision_at_10_max + value: 14.261884731124264 + - type: nauc_precision_at_10_std + value: 18.084188795498378 + - type: nauc_precision_at_1_diff1 + value: 76.51940497836482 + - type: nauc_precision_at_1_max + value: 51.251419487235474 + - type: nauc_precision_at_1_std + value: 0.16714896857146574 + - type: nauc_precision_at_20_diff1 + value: 1.9180293008303761 + - type: nauc_precision_at_20_max + value: 13.832269193468512 + - type: nauc_precision_at_20_std + value: 21.65284406055607 + - type: nauc_precision_at_3_diff1 + value: 7.226609484731811 + - type: nauc_precision_at_3_max + value: 15.162908526977272 + - type: nauc_precision_at_3_std + value: 8.451859972962776 + - type: nauc_precision_at_5_diff1 + value: 4.705236845538159 + - type: nauc_precision_at_5_max + value: 14.022910843582666 + - type: nauc_precision_at_5_std + value: 11.777269322821605 + - type: nauc_recall_at_1000_diff1 + value: 2.446247548945172 + - type: nauc_recall_at_1000_max + value: 29.14566206426889 + - type: nauc_recall_at_1000_std + value: 49.20704909525879 + - type: nauc_recall_at_100_diff1 + value: 0.1127119672553316 + - type: nauc_recall_at_100_max + value: 17.37584606388062 + - type: nauc_recall_at_100_std + value: 34.660993462440686 + - type: nauc_recall_at_10_diff1 + value: 2.9923183951227927 + - type: nauc_recall_at_10_max + value: 14.261884731124299 + - type: nauc_recall_at_10_std + value: 18.08418879549837 + - type: nauc_recall_at_1_diff1 + value: 76.51940497836482 + - type: nauc_recall_at_1_max + value: 51.251419487235474 + - type: nauc_recall_at_1_std + value: 0.16714896857146574 + - type: nauc_recall_at_20_diff1 + value: 1.918029300830432 + - type: nauc_recall_at_20_max + value: 13.832269193468566 + - type: nauc_recall_at_20_std + value: 21.65284406055605 + - type: nauc_recall_at_3_diff1 + value: 7.226609484731802 + - type: nauc_recall_at_3_max + value: 15.162908526977182 + - type: nauc_recall_at_3_std + value: 8.451859972962634 + - type: nauc_recall_at_5_diff1 + value: 4.705236845538197 + - type: nauc_recall_at_5_max + value: 14.02291084358265 + - type: nauc_recall_at_5_std + value: 11.777269322821638 + - type: ndcg_at_1 + value: 83.45700000000001 + - type: ndcg_at_10 + value: 71.74199999999999 + - type: ndcg_at_100 + value: 75.008 + - type: ndcg_at_1000 + value: 76.242 + - type: ndcg_at_20 + value: 73.114 + - type: ndcg_at_3 + value: 67.128 + - type: ndcg_at_5 + value: 69.645 + - type: precision_at_1 + value: 83.45700000000001 + - type: precision_at_10 + value: 14.747 + - type: precision_at_100 + value: 1.73 + - type: precision_at_1000 + value: 0.189 + - type: precision_at_20 + value: 7.8149999999999995 + - type: precision_at_3 + value: 42.323 + - type: precision_at_5 + value: 27.381 + - type: recall_at_1 + value: 41.729 + - type: recall_at_10 + value: 73.734 + - type: recall_at_100 + value: 86.502 + - type: recall_at_1000 + value: 94.60499999999999 + - type: recall_at_20 + value: 78.14999999999999 + - type: recall_at_3 + value: 63.483999999999995 + - type: recall_at_5 + value: 68.45400000000001 + task: + type: Retrieval + - dataset: + config: default + name: MTEB ImdbClassification + revision: 3d86128a09e091d6018b6d26cad27f2739fc2db7 + split: test + type: mteb/imdb + metrics: + - type: accuracy + value: 96.4904 + - type: ap + value: 94.85481918794709 + - type: ap_weighted + value: 94.85481918794709 + - type: f1 + value: 96.4898592305707 + - type: f1_weighted + value: 96.4898592305707 + - type: main_score + value: 96.4904 + task: + type: Classification + - dataset: + config: default + name: MTEB MSMARCO + revision: c5a29a104738b98a9e76336939199e264163d4a0 + split: dev + type: mteb/msmarco + metrics: + - type: main_score + value: 43.692 + - type: map_at_1 + value: 23.751 + - type: map_at_10 + value: 36.553999999999995 + - type: map_at_100 + value: 37.721 + - type: map_at_1000 + value: 37.763999999999996 + - type: map_at_20 + value: 37.289 + - type: map_at_3 + value: 32.643 + - type: map_at_5 + value: 34.851 + - type: mrr_at_1 + value: 24.455587392550143 + - type: mrr_at_10 + value: 37.18388706963206 + - type: mrr_at_100 + value: 38.28330737932916 + - type: mrr_at_1000 + value: 38.32054399710817 + - type: mrr_at_20 + value: 37.8818001216278 + - type: mrr_at_3 + value: 33.35721107927405 + - type: mrr_at_5 + value: 35.52483285577843 + - type: nauc_map_at_1000_diff1 + value: 36.3576177260684 + - type: nauc_map_at_1000_max + value: 7.854511605962703 + - type: nauc_map_at_1000_std + value: -17.701121059746878 + - type: nauc_map_at_100_diff1 + value: 36.356075649230505 + - type: nauc_map_at_100_max + value: 7.862168042999533 + - type: nauc_map_at_100_std + value: -17.670102459097233 + - type: nauc_map_at_10_diff1 + value: 36.22122978875574 + - type: nauc_map_at_10_max + value: 7.80848606967416 + - type: nauc_map_at_10_std + value: -18.3265151386167 + - type: nauc_map_at_1_diff1 + value: 39.28605466408357 + - type: nauc_map_at_1_max + value: 6.20202977590459 + - type: nauc_map_at_1_std + value: -15.734334090045026 + - type: nauc_map_at_20_diff1 + value: 36.33637880909657 + - type: nauc_map_at_20_max + value: 7.843437969476022 + - type: nauc_map_at_20_std + value: -17.917533363025996 + - type: nauc_map_at_3_diff1 + value: 36.24864976076741 + - type: nauc_map_at_3_max + value: 7.420345251835957 + - type: nauc_map_at_3_std + value: -18.71678497722944 + - type: nauc_map_at_5_diff1 + value: 36.0789619291824 + - type: nauc_map_at_5_max + value: 7.7314285669514495 + - type: nauc_map_at_5_std + value: -18.748688764538706 + - type: nauc_mrr_at_1000_diff1 + value: 36.23912675623378 + - type: nauc_mrr_at_1000_max + value: 7.690553436255147 + - type: nauc_mrr_at_1000_std + value: -17.609526070212304 + - type: nauc_mrr_at_100_diff1 + value: 36.23782651189002 + - type: nauc_mrr_at_100_max + value: 7.70075095171647 + - type: nauc_mrr_at_100_std + value: -17.575714144960184 + - type: nauc_mrr_at_10_diff1 + value: 36.125229472534215 + - type: nauc_mrr_at_10_max + value: 7.635472248755658 + - type: nauc_mrr_at_10_std + value: -18.208166616511086 + - type: nauc_mrr_at_1_diff1 + value: 39.20986875554532 + - type: nauc_mrr_at_1_max + value: 6.062668487561363 + - type: nauc_mrr_at_1_std + value: -16.04130340817602 + - type: nauc_mrr_at_20_diff1 + value: 36.21207088739667 + - type: nauc_mrr_at_20_max + value: 7.699610250145951 + - type: nauc_mrr_at_20_std + value: -17.778245221724028 + - type: nauc_mrr_at_3_diff1 + value: 36.03957583885305 + - type: nauc_mrr_at_3_max + value: 7.225515576504581 + - type: nauc_mrr_at_3_std + value: -18.74478742943741 + - type: nauc_mrr_at_5_diff1 + value: 35.969152496648974 + - type: nauc_mrr_at_5_max + value: 7.584059789018233 + - type: nauc_mrr_at_5_std + value: -18.569374723129332 + - type: nauc_ndcg_at_1000_diff1 + value: 35.894655529841806 + - type: nauc_ndcg_at_1000_max + value: 8.579327424366236 + - type: nauc_ndcg_at_1000_std + value: -16.359677367747896 + - type: nauc_ndcg_at_100_diff1 + value: 35.89861902483983 + - type: nauc_ndcg_at_100_max + value: 8.830873623962242 + - type: nauc_ndcg_at_100_std + value: -15.173125564722978 + - type: nauc_ndcg_at_10_diff1 + value: 35.36499811105169 + - type: nauc_ndcg_at_10_max + value: 8.449267180956992 + - type: nauc_ndcg_at_10_std + value: -18.41978802362402 + - type: nauc_ndcg_at_1_diff1 + value: 39.15422481210622 + - type: nauc_ndcg_at_1_max + value: 6.055515791928331 + - type: nauc_ndcg_at_1_std + value: -16.042779610876252 + - type: nauc_ndcg_at_20_diff1 + value: 35.73402868264468 + - type: nauc_ndcg_at_20_max + value: 8.695705518210847 + - type: nauc_ndcg_at_20_std + value: -16.7735829470466 + - type: nauc_ndcg_at_3_diff1 + value: 35.31358242856231 + - type: nauc_ndcg_at_3_max + value: 7.645692789058997 + - type: nauc_ndcg_at_3_std + value: -19.460003734786874 + - type: nauc_ndcg_at_5_diff1 + value: 35.05216588927143 + - type: nauc_ndcg_at_5_max + value: 8.216690520604715 + - type: nauc_ndcg_at_5_std + value: -19.3982054492159 + - type: nauc_precision_at_1000_diff1 + value: -4.440002625111349 + - type: nauc_precision_at_1000_max + value: 7.886988951901723 + - type: nauc_precision_at_1000_std + value: 9.88111187048247 + - type: nauc_precision_at_100_diff1 + value: 15.728286119463325 + - type: nauc_precision_at_100_max + value: 13.218650824470654 + - type: nauc_precision_at_100_std + value: 16.113245895522553 + - type: nauc_precision_at_10_diff1 + value: 29.51218489610567 + - type: nauc_precision_at_10_max + value: 10.197432401942912 + - type: nauc_precision_at_10_std + value: -16.950603431359493 + - type: nauc_precision_at_1_diff1 + value: 39.15422481210622 + - type: nauc_precision_at_1_max + value: 6.055515791928331 + - type: nauc_precision_at_1_std + value: -16.042779610876252 + - type: nauc_precision_at_20_diff1 + value: 27.825993070397338 + - type: nauc_precision_at_20_max + value: 11.437632287846007 + - type: nauc_precision_at_20_std + value: -7.450353566405601 + - type: nauc_precision_at_3_diff1 + value: 32.14135556796588 + - type: nauc_precision_at_3_max + value: 7.989252443574163 + - type: nauc_precision_at_3_std + value: -21.566254595671055 + - type: nauc_precision_at_5_diff1 + value: 30.68778685307082 + - type: nauc_precision_at_5_max + value: 9.332160758499892 + - type: nauc_precision_at_5_std + value: -20.928554713448914 + - type: nauc_recall_at_1000_diff1 + value: 25.00810478716878 + - type: nauc_recall_at_1000_max + value: 46.518165765201644 + - type: nauc_recall_at_1000_std + value: 61.4734635576085 + - type: nauc_recall_at_100_diff1 + value: 33.895581318261726 + - type: nauc_recall_at_100_max + value: 20.10706035872801 + - type: nauc_recall_at_100_std + value: 24.204226584457047 + - type: nauc_recall_at_10_diff1 + value: 32.363127359576296 + - type: nauc_recall_at_10_max + value: 10.729923804989545 + - type: nauc_recall_at_10_std + value: -18.1335370184202 + - type: nauc_recall_at_1_diff1 + value: 39.28605466408357 + - type: nauc_recall_at_1_max + value: 6.20202977590459 + - type: nauc_recall_at_1_std + value: -15.734334090045026 + - type: nauc_recall_at_20_diff1 + value: 33.47804003169795 + - type: nauc_recall_at_20_max + value: 12.781494765263382 + - type: nauc_recall_at_20_std + value: -9.263970132202658 + - type: nauc_recall_at_3_diff1 + value: 32.71001429428999 + - type: nauc_recall_at_3_max + value: 8.353439197382693 + - type: nauc_recall_at_3_std + value: -21.235097744366954 + - type: nauc_recall_at_5_diff1 + value: 31.87451464963415 + - type: nauc_recall_at_5_max + value: 9.635051450907305 + - type: nauc_recall_at_5_std + value: -21.113235357132794 + - type: ndcg_at_1 + value: 24.47 + - type: ndcg_at_10 + value: 43.692 + - type: ndcg_at_100 + value: 49.211 + - type: ndcg_at_1000 + value: 50.244 + - type: ndcg_at_20 + value: 46.278000000000006 + - type: ndcg_at_3 + value: 35.719 + - type: ndcg_at_5 + value: 39.652 + - type: precision_at_1 + value: 24.47 + - type: precision_at_10 + value: 6.857 + - type: precision_at_100 + value: 0.9610000000000001 + - type: precision_at_1000 + value: 0.105 + - type: precision_at_20 + value: 3.968 + - type: precision_at_3 + value: 15.181000000000001 + - type: precision_at_5 + value: 11.117 + - type: recall_at_1 + value: 23.751 + - type: recall_at_10 + value: 65.64 + - type: recall_at_100 + value: 90.967 + - type: recall_at_1000 + value: 98.738 + - type: recall_at_20 + value: 75.639 + - type: recall_at_3 + value: 43.927 + - type: recall_at_5 + value: 53.366 + task: + type: Retrieval + - dataset: + config: en + name: MTEB MTOPDomainClassification (en) + revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf + split: test + type: mteb/mtop_domain + metrics: + - type: accuracy + value: 98.82580939352485 + - type: f1 + value: 98.75201754333801 + - type: f1_weighted + value: 98.82795205108245 + - type: main_score + value: 98.82580939352485 + task: + type: Classification + - dataset: + config: en + name: MTEB MTOPIntentClassification (en) + revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba + split: test + type: mteb/mtop_intent + metrics: + - type: accuracy + value: 92.29822161422709 + - type: f1 + value: 77.75210224871594 + - type: f1_weighted + value: 93.58661422540348 + - type: main_score + value: 92.29822161422709 + task: + type: Classification + - dataset: + config: en + name: MTEB MassiveIntentClassification (en) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 85.17484868863484 + - type: f1 + value: 81.94484244487094 + - type: f1_weighted + value: 85.21022593423332 + - type: main_score + value: 85.17484868863484 + task: + type: Classification + - dataset: + config: en + name: MTEB MassiveIntentClassification (en) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: validation + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 85.45499262174128 + - type: f1 + value: 82.75179260642076 + - type: f1_weighted + value: 85.24021944366758 + - type: main_score + value: 85.45499262174128 + task: + type: Classification + - dataset: + config: en + name: MTEB MassiveScenarioClassification (en) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 89.61667787491594 + - type: f1 + value: 89.02701927621264 + - type: f1_weighted + value: 89.56306982022801 + - type: main_score + value: 89.61667787491594 + task: + type: Classification + - dataset: + config: en + name: MTEB MassiveScenarioClassification (en) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: validation + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 89.39006394490899 + - type: f1 + value: 88.60886270543065 + - type: f1_weighted + value: 89.31865719898514 + - type: main_score + value: 89.39006394490899 + task: + type: Classification + - dataset: + config: default + name: MTEB MedrxivClusteringP2P + revision: e7a26af6f3ae46b30dde8737f02c07b1505bcc73 + split: test + type: mteb/medrxiv-clustering-p2p + metrics: + - type: main_score + value: 46.318282423948574 + - type: v_measure + value: 46.318282423948574 + - type: v_measure_std + value: 0.9729055662461538 + task: + type: Clustering + - dataset: + config: default + name: MTEB MedrxivClusteringS2S + revision: 35191c8c0dca72d8ff3efcd72aa802307d469663 + split: test + type: mteb/medrxiv-clustering-s2s + metrics: + - type: main_score + value: 44.29033625273981 + - type: v_measure + value: 44.29033625273981 + - type: v_measure_std + value: 1.0596383629128594 + task: + type: Clustering + - dataset: + config: default + name: MTEB MindSmallReranking + revision: 59042f120c80e8afa9cdbb224f67076cec0fc9a7 + split: test + type: mteb/mind_small + metrics: + - type: main_score + value: 33.0526129239962 + - type: map + value: 33.0526129239962 + - type: mrr + value: 34.29260046890935 + - type: nAUC_map_diff1 + value: 12.579738077238032 + - type: nAUC_map_max + value: -20.936629344962 + - type: nAUC_map_std + value: -1.6096805784945216 + - type: nAUC_mrr_diff1 + value: 11.597584463580807 + - type: nAUC_mrr_max + value: -15.723702838537504 + - type: nAUC_mrr_std + value: 0.2719172965777737 + task: + type: Reranking + - dataset: + config: default + name: MTEB NFCorpus + revision: ec0fa4fe99da2ff19ca1214b7966684033a58814 + split: test + type: mteb/nfcorpus + metrics: + - type: main_score + value: 41.486000000000004 + - type: map_at_1 + value: 6.866 + - type: map_at_10 + value: 15.895999999999999 + - type: map_at_100 + value: 21.093 + - type: map_at_1000 + value: 23.067 + - type: map_at_20 + value: 18.125 + - type: map_at_3 + value: 11.421000000000001 + - type: map_at_5 + value: 13.415 + - type: mrr_at_1 + value: 52.63157894736842 + - type: mrr_at_10 + value: 61.486805248415166 + - type: mrr_at_100 + value: 62.08211009182091 + - type: mrr_at_1000 + value: 62.10828701365016 + - type: mrr_at_20 + value: 61.904411187915784 + - type: mrr_at_3 + value: 59.90712074303407 + - type: mrr_at_5 + value: 60.91331269349847 + - type: nauc_map_at_1000_diff1 + value: 25.484625278529403 + - type: nauc_map_at_1000_max + value: 31.206600396418853 + - type: nauc_map_at_1000_std + value: 15.569448072357156 + - type: nauc_map_at_100_diff1 + value: 27.636750226316764 + - type: nauc_map_at_100_max + value: 29.66992681250722 + - type: nauc_map_at_100_std + value: 10.570600484002671 + - type: nauc_map_at_10_diff1 + value: 32.76642525548697 + - type: nauc_map_at_10_max + value: 21.459225397237663 + - type: nauc_map_at_10_std + value: -3.546494734209264 + - type: nauc_map_at_1_diff1 + value: 48.8002894871328 + - type: nauc_map_at_1_max + value: 5.7236722609868815 + - type: nauc_map_at_1_std + value: -13.283554044471352 + - type: nauc_map_at_20_diff1 + value: 30.57169701502308 + - type: nauc_map_at_20_max + value: 25.79666139518404 + - type: nauc_map_at_20_std + value: 1.781732492989651 + - type: nauc_map_at_3_diff1 + value: 40.076315947201095 + - type: nauc_map_at_3_max + value: 12.862524429140054 + - type: nauc_map_at_3_std + value: -9.188349777126817 + - type: nauc_map_at_5_diff1 + value: 36.9918718052938 + - type: nauc_map_at_5_max + value: 16.74234374361876 + - type: nauc_map_at_5_std + value: -7.818523349307494 + - type: nauc_mrr_at_1000_diff1 + value: 26.88183002609805 + - type: nauc_mrr_at_1000_max + value: 47.10209348428658 + - type: nauc_mrr_at_1000_std + value: 32.067825924992924 + - type: nauc_mrr_at_100_diff1 + value: 26.871482491566745 + - type: nauc_mrr_at_100_max + value: 47.11303868498556 + - type: nauc_mrr_at_100_std + value: 32.08961428818868 + - type: nauc_mrr_at_10_diff1 + value: 26.6356914977722 + - type: nauc_mrr_at_10_max + value: 47.091624558810366 + - type: nauc_mrr_at_10_std + value: 31.942424120660164 + - type: nauc_mrr_at_1_diff1 + value: 28.19774198483673 + - type: nauc_mrr_at_1_max + value: 41.44380927834253 + - type: nauc_mrr_at_1_std + value: 25.18222691885917 + - type: nauc_mrr_at_20_diff1 + value: 26.86487347109452 + - type: nauc_mrr_at_20_max + value: 47.1987778214726 + - type: nauc_mrr_at_20_std + value: 32.143517921610034 + - type: nauc_mrr_at_3_diff1 + value: 27.34340373236422 + - type: nauc_mrr_at_3_max + value: 46.358726506276646 + - type: nauc_mrr_at_3_std + value: 31.74924155572593 + - type: nauc_mrr_at_5_diff1 + value: 27.209667205060672 + - type: nauc_mrr_at_5_max + value: 46.79883369072009 + - type: nauc_mrr_at_5_std + value: 31.655605306670758 + - type: nauc_ndcg_at_1000_diff1 + value: 18.940195769769687 + - type: nauc_ndcg_at_1000_max + value: 46.48551313937331 + - type: nauc_ndcg_at_1000_std + value: 33.64819502089232 + - type: nauc_ndcg_at_100_diff1 + value: 19.50885253809146 + - type: nauc_ndcg_at_100_max + value: 40.53174462354878 + - type: nauc_ndcg_at_100_std + value: 28.516152877751118 + - type: nauc_ndcg_at_10_diff1 + value: 16.01699218096564 + - type: nauc_ndcg_at_10_max + value: 41.17322878314514 + - type: nauc_ndcg_at_10_std + value: 29.002233224832196 + - type: nauc_ndcg_at_1_diff1 + value: 27.443547710102205 + - type: nauc_ndcg_at_1_max + value: 40.66529763309582 + - type: nauc_ndcg_at_1_std + value: 24.15016766225869 + - type: nauc_ndcg_at_20_diff1 + value: 17.541197675685062 + - type: nauc_ndcg_at_20_max + value: 40.53231266973844 + - type: nauc_ndcg_at_20_std + value: 29.54096347876548 + - type: nauc_ndcg_at_3_diff1 + value: 18.649628357473716 + - type: nauc_ndcg_at_3_max + value: 41.18603570171764 + - type: nauc_ndcg_at_3_std + value: 27.125524188420396 + - type: nauc_ndcg_at_5_diff1 + value: 17.519593751448483 + - type: nauc_ndcg_at_5_max + value: 42.715997890377345 + - type: nauc_ndcg_at_5_std + value: 27.902627839899868 + - type: nauc_precision_at_1000_diff1 + value: -15.528797630565155 + - type: nauc_precision_at_1000_max + value: 13.741640921778671 + - type: nauc_precision_at_1000_std + value: 44.50896053788372 + - type: nauc_precision_at_100_diff1 + value: -14.491464489721887 + - type: nauc_precision_at_100_max + value: 23.136434418999457 + - type: nauc_precision_at_100_std + value: 49.73145147863128 + - type: nauc_precision_at_10_diff1 + value: -4.829188942994277 + - type: nauc_precision_at_10_max + value: 40.327612559528866 + - type: nauc_precision_at_10_std + value: 39.34919529635044 + - type: nauc_precision_at_1_diff1 + value: 28.19774198483673 + - type: nauc_precision_at_1_max + value: 41.44380927834253 + - type: nauc_precision_at_1_std + value: 25.18222691885917 + - type: nauc_precision_at_20_diff1 + value: -7.210726293112847 + - type: nauc_precision_at_20_max + value: 37.195679576636984 + - type: nauc_precision_at_20_std + value: 45.4597096418357 + - type: nauc_precision_at_3_diff1 + value: 7.578219537774854 + - type: nauc_precision_at_3_max + value: 41.59775233475654 + - type: nauc_precision_at_3_std + value: 30.764584790895118 + - type: nauc_precision_at_5_diff1 + value: 1.655451789039598 + - type: nauc_precision_at_5_max + value: 43.435739407610455 + - type: nauc_precision_at_5_std + value: 33.42552263325999 + - type: nauc_recall_at_1000_diff1 + value: 5.030705700690516 + - type: nauc_recall_at_1000_max + value: 19.108072570815583 + - type: nauc_recall_at_1000_std + value: 14.697734974217308 + - type: nauc_recall_at_100_diff1 + value: 14.746540318132407 + - type: nauc_recall_at_100_max + value: 21.798705033854795 + - type: nauc_recall_at_100_std + value: 11.416195108842587 + - type: nauc_recall_at_10_diff1 + value: 25.548642427860486 + - type: nauc_recall_at_10_max + value: 18.711677681987474 + - type: nauc_recall_at_10_std + value: -5.988904818971677 + - type: nauc_recall_at_1_diff1 + value: 48.8002894871328 + - type: nauc_recall_at_1_max + value: 5.7236722609868815 + - type: nauc_recall_at_1_std + value: -13.283554044471352 + - type: nauc_recall_at_20_diff1 + value: 23.39140739154809 + - type: nauc_recall_at_20_max + value: 19.351150636155474 + - type: nauc_recall_at_20_std + value: -2.757280266915132 + - type: nauc_recall_at_3_diff1 + value: 38.17453576012812 + - type: nauc_recall_at_3_max + value: 13.47003839643972 + - type: nauc_recall_at_3_std + value: -8.75780163862688 + - type: nauc_recall_at_5_diff1 + value: 33.02812855226899 + - type: nauc_recall_at_5_max + value: 15.477626408978477 + - type: nauc_recall_at_5_std + value: -9.072206441070708 + - type: ndcg_at_1 + value: 50.773999999999994 + - type: ndcg_at_10 + value: 41.486000000000004 + - type: ndcg_at_100 + value: 39.051 + - type: ndcg_at_1000 + value: 48.106 + - type: ndcg_at_20 + value: 39.432 + - type: ndcg_at_3 + value: 47.428 + - type: ndcg_at_5 + value: 45.227000000000004 + - type: precision_at_1 + value: 52.632 + - type: precision_at_10 + value: 31.146 + - type: precision_at_100 + value: 10.328 + - type: precision_at_1000 + value: 2.432 + - type: precision_at_20 + value: 23.793 + - type: precision_at_3 + value: 45.201 + - type: precision_at_5 + value: 39.876 + - type: recall_at_1 + value: 6.866 + - type: recall_at_10 + value: 20.447000000000003 + - type: recall_at_100 + value: 40.607 + - type: recall_at_1000 + value: 73.411 + - type: recall_at_20 + value: 26.082 + - type: recall_at_3 + value: 12.484 + - type: recall_at_5 + value: 15.847 + task: + type: Retrieval + - dataset: + config: default + name: MTEB NQ + revision: b774495ed302d8c44a3a7ea25c90dbce03968f31 + split: test + type: mteb/nq + metrics: + - type: main_score + value: 69.072 + - type: map_at_1 + value: 45.483000000000004 + - type: map_at_10 + value: 62.050000000000004 + - type: map_at_100 + value: 62.693 + - type: map_at_1000 + value: 62.702999999999996 + - type: map_at_20 + value: 62.498 + - type: map_at_3 + value: 58.285 + - type: map_at_5 + value: 60.711000000000006 + - type: mrr_at_1 + value: 50.840092699884124 + - type: mrr_at_10 + value: 64.54635224116673 + - type: mrr_at_100 + value: 64.9526548702289 + - type: mrr_at_1000 + value: 64.95908460752281 + - type: mrr_at_20 + value: 64.82949565799959 + - type: mrr_at_3 + value: 61.89165701042856 + - type: mrr_at_5 + value: 63.632676709154026 + - type: nauc_map_at_1000_diff1 + value: 43.187285304185224 + - type: nauc_map_at_1000_max + value: 32.39921659632756 + - type: nauc_map_at_1000_std + value: -5.780901333066553 + - type: nauc_map_at_100_diff1 + value: 43.184487221204456 + - type: nauc_map_at_100_max + value: 32.41176116347982 + - type: nauc_map_at_100_std + value: -5.76422606662383 + - type: nauc_map_at_10_diff1 + value: 42.967066814031746 + - type: nauc_map_at_10_max + value: 32.489617364418514 + - type: nauc_map_at_10_std + value: -6.029045531102664 + - type: nauc_map_at_1_diff1 + value: 46.16376563218624 + - type: nauc_map_at_1_max + value: 26.342624776802232 + - type: nauc_map_at_1_std + value: -7.142171388751972 + - type: nauc_map_at_20_diff1 + value: 43.15894358608328 + - type: nauc_map_at_20_max + value: 32.46492198956245 + - type: nauc_map_at_20_std + value: -5.788373305449195 + - type: nauc_map_at_3_diff1 + value: 43.231752344608545 + - type: nauc_map_at_3_max + value: 31.68003009949564 + - type: nauc_map_at_3_std + value: -8.015235132765458 + - type: nauc_map_at_5_diff1 + value: 42.86197608819917 + - type: nauc_map_at_5_max + value: 32.363857571094485 + - type: nauc_map_at_5_std + value: -6.780487416387977 + - type: nauc_mrr_at_1000_diff1 + value: 43.40542912045782 + - type: nauc_mrr_at_1000_max + value: 32.8461770324533 + - type: nauc_mrr_at_1000_std + value: -3.6505425530008204 + - type: nauc_mrr_at_100_diff1 + value: 43.40233508014468 + - type: nauc_mrr_at_100_max + value: 32.85598538385942 + - type: nauc_mrr_at_100_std + value: -3.637477352635459 + - type: nauc_mrr_at_10_diff1 + value: 43.260179162806054 + - type: nauc_mrr_at_10_max + value: 32.942643527040474 + - type: nauc_mrr_at_10_std + value: -3.712052825320437 + - type: nauc_mrr_at_1_diff1 + value: 46.354919460881206 + - type: nauc_mrr_at_1_max + value: 29.1760258591106 + - type: nauc_mrr_at_1_std + value: -4.107225031227406 + - type: nauc_mrr_at_20_diff1 + value: 43.37092385434311 + - type: nauc_mrr_at_20_max + value: 32.93390254712846 + - type: nauc_mrr_at_20_std + value: -3.5719056112132006 + - type: nauc_mrr_at_3_diff1 + value: 43.1744474040527 + - type: nauc_mrr_at_3_max + value: 32.741290559777994 + - type: nauc_mrr_at_3_std + value: -4.72677925120697 + - type: nauc_mrr_at_5_diff1 + value: 43.108396819975674 + - type: nauc_mrr_at_5_max + value: 32.970519514893084 + - type: nauc_mrr_at_5_std + value: -4.090906158975974 + - type: nauc_ndcg_at_1000_diff1 + value: 42.786664193638714 + - type: nauc_ndcg_at_1000_max + value: 33.65554095609296 + - type: nauc_ndcg_at_1000_std + value: -4.024030130584482 + - type: nauc_ndcg_at_100_diff1 + value: 42.691246775210814 + - type: nauc_ndcg_at_100_max + value: 34.063232335110875 + - type: nauc_ndcg_at_100_std + value: -3.477813807415248 + - type: nauc_ndcg_at_10_diff1 + value: 41.90988990571757 + - type: nauc_ndcg_at_10_max + value: 34.58934812881633 + - type: nauc_ndcg_at_10_std + value: -4.3295110195497655 + - type: nauc_ndcg_at_1_diff1 + value: 46.354919460881206 + - type: nauc_ndcg_at_1_max + value: 29.1760258591106 + - type: nauc_ndcg_at_1_std + value: -4.107225031227406 + - type: nauc_ndcg_at_20_diff1 + value: 42.493206675867114 + - type: nauc_ndcg_at_20_max + value: 34.562441307459544 + - type: nauc_ndcg_at_20_std + value: -3.4456116866749107 + - type: nauc_ndcg_at_3_diff1 + value: 42.24180336502808 + - type: nauc_ndcg_at_3_max + value: 33.064267018100594 + - type: nauc_ndcg_at_3_std + value: -7.786248093572142 + - type: nauc_ndcg_at_5_diff1 + value: 41.692714787779565 + - type: nauc_ndcg_at_5_max + value: 34.20502498949156 + - type: nauc_ndcg_at_5_std + value: -5.979557859282785 + - type: nauc_precision_at_1000_diff1 + value: -13.779832506640702 + - type: nauc_precision_at_1000_max + value: 1.243001688631421 + - type: nauc_precision_at_1000_std + value: 17.351623398622323 + - type: nauc_precision_at_100_diff1 + value: -11.310526816290297 + - type: nauc_precision_at_100_max + value: 5.771669506192959 + - type: nauc_precision_at_100_std + value: 19.917795079540113 + - type: nauc_precision_at_10_diff1 + value: 2.163699384635286 + - type: nauc_precision_at_10_max + value: 19.66440698458386 + - type: nauc_precision_at_10_std + value: 13.689876348315726 + - type: nauc_precision_at_1_diff1 + value: 46.354919460881206 + - type: nauc_precision_at_1_max + value: 29.1760258591106 + - type: nauc_precision_at_1_std + value: -4.107225031227406 + - type: nauc_precision_at_20_diff1 + value: -3.038735879584471 + - type: nauc_precision_at_20_max + value: 14.132968299701695 + - type: nauc_precision_at_20_std + value: 17.78069734664346 + - type: nauc_precision_at_3_diff1 + value: 21.783760758070095 + - type: nauc_precision_at_3_max + value: 30.244127986404497 + - type: nauc_precision_at_3_std + value: -0.12411163467738723 + - type: nauc_precision_at_5_diff1 + value: 10.980635723302418 + - type: nauc_precision_at_5_max + value: 25.302293738975575 + - type: nauc_precision_at_5_std + value: 6.4740817488722024 + - type: nauc_recall_at_1000_diff1 + value: 34.10343772356593 + - type: nauc_recall_at_1000_max + value: 80.72497340357538 + - type: nauc_recall_at_1000_std + value: 69.54564103264093 + - type: nauc_recall_at_100_diff1 + value: 33.427719956774126 + - type: nauc_recall_at_100_max + value: 71.54086768335449 + - type: nauc_recall_at_100_std + value: 49.66157377654885 + - type: nauc_recall_at_10_diff1 + value: 33.70139560054039 + - type: nauc_recall_at_10_max + value: 45.47878072860151 + - type: nauc_recall_at_10_std + value: 1.4188516615716378 + - type: nauc_recall_at_1_diff1 + value: 46.16376563218624 + - type: nauc_recall_at_1_max + value: 26.342624776802232 + - type: nauc_recall_at_1_std + value: -7.142171388751972 + - type: nauc_recall_at_20_diff1 + value: 35.805379874970086 + - type: nauc_recall_at_20_max + value: 51.80479822253392 + - type: nauc_recall_at_20_std + value: 13.531467576460143 + - type: nauc_recall_at_3_diff1 + value: 37.288500141631616 + - type: nauc_recall_at_3_max + value: 35.07078243516728 + - type: nauc_recall_at_3_std + value: -10.452926441410405 + - type: nauc_recall_at_5_diff1 + value: 34.83186104526897 + - type: nauc_recall_at_5_max + value: 39.58488976496973 + - type: nauc_recall_at_5_std + value: -6.3049292065708835 + - type: ndcg_at_1 + value: 50.839999999999996 + - type: ndcg_at_10 + value: 69.072 + - type: ndcg_at_100 + value: 71.538 + - type: ndcg_at_1000 + value: 71.77799999999999 + - type: ndcg_at_20 + value: 70.41 + - type: ndcg_at_3 + value: 62.544999999999995 + - type: ndcg_at_5 + value: 66.33099999999999 + - type: precision_at_1 + value: 50.839999999999996 + - type: precision_at_10 + value: 10.495000000000001 + - type: precision_at_100 + value: 1.1900000000000002 + - type: precision_at_1000 + value: 0.121 + - type: precision_at_20 + value: 5.5809999999999995 + - type: precision_at_3 + value: 27.636 + - type: precision_at_5 + value: 18.864 + - type: recall_at_1 + value: 45.483000000000004 + - type: recall_at_10 + value: 87.483 + - type: recall_at_100 + value: 97.844 + - type: recall_at_1000 + value: 99.66199999999999 + - type: recall_at_20 + value: 92.294 + - type: recall_at_3 + value: 71.2 + - type: recall_at_5 + value: 79.753 + task: + type: Retrieval + - dataset: + config: default + name: MTEB QuoraRetrieval + revision: e4e08e0b7dbe3c8700f0daef558ff32256715259 + split: test + type: mteb/quora + metrics: + - type: main_score + value: 89.58 + - type: map_at_1 + value: 71.819 + - type: map_at_10 + value: 86.04899999999999 + - type: map_at_100 + value: 86.648 + - type: map_at_1000 + value: 86.66199999999999 + - type: map_at_20 + value: 86.441 + - type: map_at_3 + value: 83.114 + - type: map_at_5 + value: 84.981 + - type: mrr_at_1 + value: 82.62 + - type: mrr_at_10 + value: 88.62899999999979 + - type: mrr_at_100 + value: 88.70918591324215 + - type: mrr_at_1000 + value: 88.70973091492397 + - type: mrr_at_20 + value: 88.68914765317221 + - type: mrr_at_3 + value: 87.74999999999979 + - type: mrr_at_5 + value: 88.36799999999974 + - type: nauc_map_at_1000_diff1 + value: 77.89207709760448 + - type: nauc_map_at_1000_max + value: 29.63371361495422 + - type: nauc_map_at_1000_std + value: -48.628180385874344 + - type: nauc_map_at_100_diff1 + value: 77.89592179104915 + - type: nauc_map_at_100_max + value: 29.617171506130756 + - type: nauc_map_at_100_std + value: -48.66057170774648 + - type: nauc_map_at_10_diff1 + value: 78.0618161228185 + - type: nauc_map_at_10_max + value: 29.178490609366737 + - type: nauc_map_at_10_std + value: -50.74755004592002 + - type: nauc_map_at_1_diff1 + value: 81.64335579973574 + - type: nauc_map_at_1_max + value: 21.813832226652174 + - type: nauc_map_at_1_std + value: -42.57570978190876 + - type: nauc_map_at_20_diff1 + value: 77.9299081005938 + - type: nauc_map_at_20_max + value: 29.458718470003888 + - type: nauc_map_at_20_std + value: -49.63337236763102 + - type: nauc_map_at_3_diff1 + value: 78.72941448509229 + - type: nauc_map_at_3_max + value: 26.600997896960056 + - type: nauc_map_at_3_std + value: -51.889002227479885 + - type: nauc_map_at_5_diff1 + value: 78.31466610917171 + - type: nauc_map_at_5_max + value: 28.09863984582896 + - type: nauc_map_at_5_std + value: -52.14058096096497 + - type: nauc_mrr_at_1000_diff1 + value: 78.42667263739992 + - type: nauc_mrr_at_1000_max + value: 31.98996235127974 + - type: nauc_mrr_at_1000_std + value: -44.380439148429296 + - type: nauc_mrr_at_100_diff1 + value: 78.42661032698115 + - type: nauc_mrr_at_100_max + value: 31.991652631740102 + - type: nauc_mrr_at_100_std + value: -44.37854108460535 + - type: nauc_mrr_at_10_diff1 + value: 78.39126022544136 + - type: nauc_mrr_at_10_max + value: 32.02023484451197 + - type: nauc_mrr_at_10_std + value: -44.561252349176954 + - type: nauc_mrr_at_1_diff1 + value: 79.21630894647448 + - type: nauc_mrr_at_1_max + value: 31.526303156060177 + - type: nauc_mrr_at_1_std + value: -41.887504422443136 + - type: nauc_mrr_at_20_diff1 + value: 78.42548039170424 + - type: nauc_mrr_at_20_max + value: 31.99588275070137 + - type: nauc_mrr_at_20_std + value: -44.44957722627042 + - type: nauc_mrr_at_3_diff1 + value: 78.26165151833735 + - type: nauc_mrr_at_3_max + value: 32.18028826126801 + - type: nauc_mrr_at_3_std + value: -44.6998237213182 + - type: nauc_mrr_at_5_diff1 + value: 78.34786430903962 + - type: nauc_mrr_at_5_max + value: 32.168476272879566 + - type: nauc_mrr_at_5_std + value: -44.7915919956712 + - type: nauc_ndcg_at_1000_diff1 + value: 77.79198355957816 + - type: nauc_ndcg_at_1000_max + value: 31.14363511518406 + - type: nauc_ndcg_at_1000_std + value: -46.69335151274275 + - type: nauc_ndcg_at_100_diff1 + value: 77.79898090286419 + - type: nauc_ndcg_at_100_max + value: 31.115103811629215 + - type: nauc_ndcg_at_100_std + value: -46.73078913421965 + - type: nauc_ndcg_at_10_diff1 + value: 77.74856635461343 + - type: nauc_ndcg_at_10_max + value: 30.279584686212747 + - type: nauc_ndcg_at_10_std + value: -50.23514662356807 + - type: nauc_ndcg_at_1_diff1 + value: 79.17833000040999 + - type: nauc_ndcg_at_1_max + value: 31.703788144510746 + - type: nauc_ndcg_at_1_std + value: -41.854817402870715 + - type: nauc_ndcg_at_20_diff1 + value: 77.7380353804671 + - type: nauc_ndcg_at_20_max + value: 30.622294129001553 + - type: nauc_ndcg_at_20_std + value: -49.035794761065254 + - type: nauc_ndcg_at_3_diff1 + value: 77.41476880573593 + - type: nauc_ndcg_at_3_max + value: 29.015949978243032 + - type: nauc_ndcg_at_3_std + value: -49.78627087622648 + - type: nauc_ndcg_at_5_diff1 + value: 77.64439137502896 + - type: nauc_ndcg_at_5_max + value: 29.444684897492206 + - type: nauc_ndcg_at_5_std + value: -51.21908400252501 + - type: nauc_precision_at_1000_diff1 + value: -44.92396459446822 + - type: nauc_precision_at_1000_max + value: -3.674153720989045 + - type: nauc_precision_at_1000_std + value: 39.56552468277785 + - type: nauc_precision_at_100_diff1 + value: -44.75143023259094 + - type: nauc_precision_at_100_max + value: -3.705280025140011 + - type: nauc_precision_at_100_std + value: 39.433619999113326 + - type: nauc_precision_at_10_diff1 + value: -41.0651074726579 + - type: nauc_precision_at_10_max + value: -0.21097985601783667 + - type: nauc_precision_at_10_std + value: 26.24652824589493 + - type: nauc_precision_at_1_diff1 + value: 79.17833000040999 + - type: nauc_precision_at_1_max + value: 31.703788144510746 + - type: nauc_precision_at_1_std + value: -41.854817402870715 + - type: nauc_precision_at_20_diff1 + value: -43.368001340920294 + - type: nauc_precision_at_20_max + value: -2.036990010399129 + - type: nauc_precision_at_20_std + value: 32.37747041406297 + - type: nauc_precision_at_3_diff1 + value: -22.089307548346877 + - type: nauc_precision_at_3_max + value: 6.2280973175296 + - type: nauc_precision_at_3_std + value: 5.323992514036145 + - type: nauc_precision_at_5_diff1 + value: -34.07115055244003 + - type: nauc_precision_at_5_max + value: 2.5955315789198834 + - type: nauc_precision_at_5_std + value: 16.26096689407332 + - type: nauc_recall_at_1000_diff1 + value: 58.27703860947467 + - type: nauc_recall_at_1000_max + value: 68.59835835315768 + - type: nauc_recall_at_1000_std + value: 77.96687006056064 + - type: nauc_recall_at_100_diff1 + value: 73.24371223081737 + - type: nauc_recall_at_100_max + value: 39.55925344664591 + - type: nauc_recall_at_100_std + value: -32.25605030215798 + - type: nauc_recall_at_10_diff1 + value: 73.41261201339202 + - type: nauc_recall_at_10_max + value: 26.822979434062926 + - type: nauc_recall_at_10_std + value: -74.2909332592806 + - type: nauc_recall_at_1_diff1 + value: 81.64335579973574 + - type: nauc_recall_at_1_max + value: 21.813832226652174 + - type: nauc_recall_at_1_std + value: -42.57570978190876 + - type: nauc_recall_at_20_diff1 + value: 72.7621297920656 + - type: nauc_recall_at_20_max + value: 26.02492304096079 + - type: nauc_recall_at_20_std + value: -77.8724532438279 + - type: nauc_recall_at_3_diff1 + value: 75.25149312810714 + - type: nauc_recall_at_3_max + value: 23.20545662481487 + - type: nauc_recall_at_3_std + value: -59.69689982140521 + - type: nauc_recall_at_5_diff1 + value: 73.69807273001406 + - type: nauc_recall_at_5_max + value: 24.073666798066057 + - type: nauc_recall_at_5_std + value: -67.91121268130719 + - type: ndcg_at_1 + value: 82.64 + - type: ndcg_at_10 + value: 89.58 + - type: ndcg_at_100 + value: 90.606 + - type: ndcg_at_1000 + value: 90.676 + - type: ndcg_at_20 + value: 90.132 + - type: ndcg_at_3 + value: 86.88 + - type: ndcg_at_5 + value: 88.40299999999999 + - type: precision_at_1 + value: 82.64 + - type: precision_at_10 + value: 13.604 + - type: precision_at_100 + value: 1.539 + - type: precision_at_1000 + value: 0.157 + - type: precision_at_20 + value: 7.188 + - type: precision_at_3 + value: 38.083 + - type: precision_at_5 + value: 25.018 + - type: recall_at_1 + value: 71.819 + - type: recall_at_10 + value: 96.34700000000001 + - type: recall_at_100 + value: 99.715 + - type: recall_at_1000 + value: 99.995 + - type: recall_at_20 + value: 98.073 + - type: recall_at_3 + value: 88.57300000000001 + - type: recall_at_5 + value: 92.908 + task: + type: Retrieval + - dataset: + config: default + name: MTEB RedditClustering + revision: 24640382cdbf8abc73003fb0fa6d111a705499eb + split: test + type: mteb/reddit-clustering + metrics: + - type: main_score + value: 71.18966762070158 + - type: v_measure + value: 71.18966762070158 + - type: v_measure_std + value: 2.7498969054457048 + task: + type: Clustering + - dataset: + config: default + name: MTEB RedditClusteringP2P + revision: 385e3cb46b4cfa89021f56c4380204149d0efe33 + split: test + type: mteb/reddit-clustering-p2p + metrics: + - type: main_score + value: 74.42014716862516 + - type: v_measure + value: 74.42014716862516 + - type: v_measure_std + value: 9.909739891410648 + task: + type: Clustering + - dataset: + config: default + name: MTEB SCIDOCS + revision: f8c2fcf00f625baaa80f62ec5bd9e1fff3b8ae88 + split: test + type: mteb/scidocs + metrics: + - type: main_score + value: 25.041999999999998 + - type: map_at_1 + value: 5.893000000000001 + - type: map_at_10 + value: 15.260000000000002 + - type: map_at_100 + value: 18.084 + - type: map_at_1000 + value: 18.467 + - type: map_at_20 + value: 16.675 + - type: map_at_3 + value: 10.526 + - type: map_at_5 + value: 12.775 + - type: mrr_at_1 + value: 28.999999999999996 + - type: mrr_at_10 + value: 41.03575396825395 + - type: mrr_at_100 + value: 42.136771862785835 + - type: mrr_at_1000 + value: 42.16698555415099 + - type: mrr_at_20 + value: 41.707493696104315 + - type: mrr_at_3 + value: 37.34999999999998 + - type: mrr_at_5 + value: 39.59999999999995 + - type: nauc_map_at_1000_diff1 + value: 12.080002654911883 + - type: nauc_map_at_1000_max + value: 29.813563682286276 + - type: nauc_map_at_1000_std + value: 20.36659817908673 + - type: nauc_map_at_100_diff1 + value: 12.108735517749706 + - type: nauc_map_at_100_max + value: 29.76830671710955 + - type: nauc_map_at_100_std + value: 20.3433621032846 + - type: nauc_map_at_10_diff1 + value: 12.91575031185637 + - type: nauc_map_at_10_max + value: 29.427600958386318 + - type: nauc_map_at_10_std + value: 16.89867275177153 + - type: nauc_map_at_1_diff1 + value: 19.353069488987916 + - type: nauc_map_at_1_max + value: 17.093914951159693 + - type: nauc_map_at_1_std + value: 8.19886078055046 + - type: nauc_map_at_20_diff1 + value: 11.977233457943113 + - type: nauc_map_at_20_max + value: 29.171812822948805 + - type: nauc_map_at_20_std + value: 18.780517506173965 + - type: nauc_map_at_3_diff1 + value: 14.453129464176092 + - type: nauc_map_at_3_max + value: 25.801958649112077 + - type: nauc_map_at_3_std + value: 11.572823684429643 + - type: nauc_map_at_5_diff1 + value: 13.167155808104997 + - type: nauc_map_at_5_max + value: 27.355626948365792 + - type: nauc_map_at_5_std + value: 14.414151839192183 + - type: nauc_mrr_at_1000_diff1 + value: 17.262104643988636 + - type: nauc_mrr_at_1000_max + value: 23.991373837217058 + - type: nauc_mrr_at_1000_std + value: 12.44755488671623 + - type: nauc_mrr_at_100_diff1 + value: 17.267280132318703 + - type: nauc_mrr_at_100_max + value: 24.022189287889294 + - type: nauc_mrr_at_100_std + value: 12.480695500214788 + - type: nauc_mrr_at_10_diff1 + value: 17.012383998246268 + - type: nauc_mrr_at_10_max + value: 24.192637911171722 + - type: nauc_mrr_at_10_std + value: 12.524608847408917 + - type: nauc_mrr_at_1_diff1 + value: 19.43518811038007 + - type: nauc_mrr_at_1_max + value: 17.747482933395602 + - type: nauc_mrr_at_1_std + value: 8.410779775558684 + - type: nauc_mrr_at_20_diff1 + value: 17.202663281407446 + - type: nauc_mrr_at_20_max + value: 24.091991130543118 + - type: nauc_mrr_at_20_std + value: 12.503814263019908 + - type: nauc_mrr_at_3_diff1 + value: 17.52733013432995 + - type: nauc_mrr_at_3_max + value: 23.569459518780214 + - type: nauc_mrr_at_3_std + value: 11.770846827520726 + - type: nauc_mrr_at_5_diff1 + value: 17.10817561975543 + - type: nauc_mrr_at_5_max + value: 23.945141435234678 + - type: nauc_mrr_at_5_std + value: 12.034468615317719 + - type: nauc_ndcg_at_1000_diff1 + value: 12.317811393346936 + - type: nauc_ndcg_at_1000_max + value: 30.809991350156103 + - type: nauc_ndcg_at_1000_std + value: 24.517501065205067 + - type: nauc_ndcg_at_100_diff1 + value: 12.824804203182936 + - type: nauc_ndcg_at_100_max + value: 30.895499817010748 + - type: nauc_ndcg_at_100_std + value: 25.424376279745402 + - type: nauc_ndcg_at_10_diff1 + value: 13.32724552457439 + - type: nauc_ndcg_at_10_max + value: 30.409088666807456 + - type: nauc_ndcg_at_10_std + value: 18.216330475714113 + - type: nauc_ndcg_at_1_diff1 + value: 19.43518811038007 + - type: nauc_ndcg_at_1_max + value: 17.747482933395602 + - type: nauc_ndcg_at_1_std + value: 8.410779775558684 + - type: nauc_ndcg_at_20_diff1 + value: 12.224399111852902 + - type: nauc_ndcg_at_20_max + value: 29.86352330445272 + - type: nauc_ndcg_at_20_std + value: 21.196937851331807 + - type: nauc_ndcg_at_3_diff1 + value: 15.367489533734027 + - type: nauc_ndcg_at_3_max + value: 26.76486390741532 + - type: nauc_ndcg_at_3_std + value: 12.606077508789923 + - type: nauc_ndcg_at_5_diff1 + value: 13.831157482390935 + - type: nauc_ndcg_at_5_max + value: 28.070226983968904 + - type: nauc_ndcg_at_5_std + value: 15.236787943125435 + - type: nauc_precision_at_1000_diff1 + value: 0.016122957101357048 + - type: nauc_precision_at_1000_max + value: 24.380929903557334 + - type: nauc_precision_at_1000_std + value: 34.54045112720052 + - type: nauc_precision_at_100_diff1 + value: 7.255224788507301 + - type: nauc_precision_at_100_max + value: 27.98453788447542 + - type: nauc_precision_at_100_std + value: 35.38999555441665 + - type: nauc_precision_at_10_diff1 + value: 9.69185099834181 + - type: nauc_precision_at_10_max + value: 32.532315522580454 + - type: nauc_precision_at_10_std + value: 21.48948348473612 + - type: nauc_precision_at_1_diff1 + value: 19.43518811038007 + - type: nauc_precision_at_1_max + value: 17.747482933395602 + - type: nauc_precision_at_1_std + value: 8.410779775558684 + - type: nauc_precision_at_20_diff1 + value: 6.964076536695672 + - type: nauc_precision_at_20_max + value: 29.30087236410044 + - type: nauc_precision_at_20_std + value: 26.413625895571986 + - type: nauc_precision_at_3_diff1 + value: 14.145134359925155 + - type: nauc_precision_at_3_max + value: 29.915650960808303 + - type: nauc_precision_at_3_std + value: 14.095370019867797 + - type: nauc_precision_at_5_diff1 + value: 11.043933558522692 + - type: nauc_precision_at_5_max + value: 30.93016505807111 + - type: nauc_precision_at_5_std + value: 17.749256196062603 + - type: nauc_recall_at_1000_diff1 + value: -0.7776817772090345 + - type: nauc_recall_at_1000_max + value: 23.094717340324518 + - type: nauc_recall_at_1000_std + value: 37.189908681396425 + - type: nauc_recall_at_100_diff1 + value: 6.887748742013364 + - type: nauc_recall_at_100_max + value: 27.00798435230277 + - type: nauc_recall_at_100_std + value: 35.908147807345344 + - type: nauc_recall_at_10_diff1 + value: 9.605632017480751 + - type: nauc_recall_at_10_max + value: 31.845202901168655 + - type: nauc_recall_at_10_std + value: 21.497414586634683 + - type: nauc_recall_at_1_diff1 + value: 19.353069488987916 + - type: nauc_recall_at_1_max + value: 17.093914951159693 + - type: nauc_recall_at_1_std + value: 8.19886078055046 + - type: nauc_recall_at_20_diff1 + value: 6.927503731844782 + - type: nauc_recall_at_20_max + value: 28.611698183338202 + - type: nauc_recall_at_20_std + value: 26.69018660149911 + - type: nauc_recall_at_3_diff1 + value: 14.043724087062268 + - type: nauc_recall_at_3_max + value: 29.269835821380465 + - type: nauc_recall_at_3_std + value: 14.104419605998094 + - type: nauc_recall_at_5_diff1 + value: 11.017319452873336 + - type: nauc_recall_at_5_max + value: 30.295720628306228 + - type: nauc_recall_at_5_std + value: 17.758048545573825 + - type: ndcg_at_1 + value: 28.999999999999996 + - type: ndcg_at_10 + value: 25.041999999999998 + - type: ndcg_at_100 + value: 35.045 + - type: ndcg_at_1000 + value: 40.803 + - type: ndcg_at_20 + value: 28.584 + - type: ndcg_at_3 + value: 23.249 + - type: ndcg_at_5 + value: 20.533 + - type: precision_at_1 + value: 28.999999999999996 + - type: precision_at_10 + value: 13.120000000000001 + - type: precision_at_100 + value: 2.7470000000000003 + - type: precision_at_1000 + value: 0.41200000000000003 + - type: precision_at_20 + value: 8.584999999999999 + - type: precision_at_3 + value: 21.633 + - type: precision_at_5 + value: 18.099999999999998 + - type: recall_at_1 + value: 5.893000000000001 + - type: recall_at_10 + value: 26.567 + - type: recall_at_100 + value: 55.800000000000004 + - type: recall_at_1000 + value: 83.608 + - type: recall_at_20 + value: 34.86 + - type: recall_at_3 + value: 13.153 + - type: recall_at_5 + value: 18.323 + task: + type: Retrieval + - dataset: + config: default + name: MTEB SICK-R + revision: 20a6d6f312dd54037fe07a32d58e5e168867909d + split: test + type: mteb/sickr-sts + metrics: + - type: cosine_pearson + value: 86.57284584320382 + - type: cosine_spearman + value: 82.20531642680812 + - type: euclidean_pearson + value: 83.94261758556554 + - type: euclidean_spearman + value: 82.20721497738559 + - type: main_score + value: 82.20531642680812 + - type: manhattan_pearson + value: 84.15902154703083 + - type: manhattan_spearman + value: 82.19506027155957 + - type: pearson + value: 86.57284584320382 + - type: spearman + value: 82.20531642680812 + task: + type: STS + - dataset: + config: default + name: MTEB STS12 + revision: a0d554a64d88156834ff5ae9920b964011b16384 + split: test + type: mteb/sts12-sts + metrics: + - type: cosine_pearson + value: 86.28047602146931 + - type: cosine_spearman + value: 79.51504881448884 + - type: euclidean_pearson + value: 83.10545189967856 + - type: euclidean_spearman + value: 79.50586960492797 + - type: main_score + value: 79.51504881448884 + - type: manhattan_pearson + value: 83.44244457500889 + - type: manhattan_spearman + value: 79.730303339846 + - type: pearson + value: 86.28047602146931 + - type: spearman + value: 79.51504881448884 + task: + type: STS + - dataset: + config: default + name: MTEB STS13 + revision: 7e90230a92c190f1bf69ae9002b8cea547a64cca + split: test + type: mteb/sts13-sts + metrics: + - type: cosine_pearson + value: 88.74723553048702 + - type: cosine_spearman + value: 89.18936052329725 + - type: euclidean_pearson + value: 88.90400878928668 + - type: euclidean_spearman + value: 89.19174821431281 + - type: main_score + value: 89.18936052329725 + - type: manhattan_pearson + value: 88.81504628424054 + - type: manhattan_spearman + value: 89.18063294142597 + - type: pearson + value: 88.74723553048702 + - type: spearman + value: 89.18936052329725 + task: + type: STS + - dataset: + config: default + name: MTEB STS14 + revision: 6031580fec1f6af667f0bd2da0a551cf4f0b2375 + split: test + type: mteb/sts14-sts + metrics: + - type: cosine_pearson + value: 86.45403437836023 + - type: cosine_spearman + value: 85.14654611519086 + - type: euclidean_pearson + value: 85.87509624462743 + - type: euclidean_spearman + value: 85.1391108856681 + - type: main_score + value: 85.14654611519086 + - type: manhattan_pearson + value: 85.96635794953866 + - type: manhattan_spearman + value: 85.3271371527667 + - type: pearson + value: 86.45403437836023 + - type: spearman + value: 85.14654611519086 + task: + type: STS + - dataset: + config: default + name: MTEB STS15 + revision: ae752c7c21bf194d8b67fd573edf7ae58183cbe3 + split: test + type: mteb/sts15-sts + metrics: + - type: cosine_pearson + value: 87.84742260009705 + - type: cosine_spearman + value: 89.10215217191254 + - type: euclidean_pearson + value: 88.97393286325477 + - type: euclidean_spearman + value: 89.1014105509662 + - type: main_score + value: 89.10215217191254 + - type: manhattan_pearson + value: 89.31698781090151 + - type: manhattan_spearman + value: 89.53000001764433 + - type: pearson + value: 87.84742260009705 + - type: spearman + value: 89.10215217191254 + task: + type: STS + - dataset: + config: default + name: MTEB STS16 + revision: 4d8694f8f0e0100860b497b999b3dbed754a0513 + split: test + type: mteb/sts16-sts + metrics: + - type: cosine_pearson + value: 85.22397535461835 + - type: cosine_spearman + value: 87.14066355879785 + - type: euclidean_pearson + value: 86.31393364087295 + - type: euclidean_spearman + value: 87.14018892702765 + - type: main_score + value: 87.14066355879785 + - type: manhattan_pearson + value: 86.36366855248434 + - type: manhattan_spearman + value: 87.20858630423012 + - type: pearson + value: 85.22397535461835 + - type: spearman + value: 87.14066355879785 + task: + type: STS + - dataset: + config: en-en + name: MTEB STS17 (en-en) + revision: faeb762787bd10488a50c8b5be4a3b82e411949c + split: test + type: mteb/sts17-crosslingual-sts + metrics: + - type: cosine_pearson + value: 90.66131612061355 + - type: cosine_spearman + value: 90.97082650129164 + - type: euclidean_pearson + value: 90.98181906744969 + - type: euclidean_spearman + value: 90.99008476850047 + - type: main_score + value: 90.97082650129164 + - type: manhattan_pearson + value: 90.75245040709021 + - type: manhattan_spearman + value: 90.6199877691265 + - type: pearson + value: 90.66131612061355 + - type: spearman + value: 90.97082650129164 + task: + type: STS + - dataset: + config: en + name: MTEB STS22 (en) + revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3 + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: cosine_pearson + value: 67.270656447085 + - type: cosine_spearman + value: 67.82870469746828 + - type: euclidean_pearson + value: 69.03857775285664 + - type: euclidean_spearman + value: 67.74455108773341 + - type: main_score + value: 67.82870469746828 + - type: manhattan_pearson + value: 69.25304172245812 + - type: manhattan_spearman + value: 68.00987097916055 + - type: pearson + value: 67.270656447085 + - type: spearman + value: 67.82870469746828 + task: + type: STS + - dataset: + config: default + name: MTEB STSBenchmark + revision: b0fddb56ed78048fa8b90373c8a3cfc37b684831 + split: test + type: mteb/stsbenchmark-sts + metrics: + - type: cosine_pearson + value: 87.17245205384889 + - type: cosine_spearman + value: 87.7360146030987 + - type: euclidean_pearson + value: 87.48919412794656 + - type: euclidean_spearman + value: 87.7312047878383 + - type: main_score + value: 87.7360146030987 + - type: manhattan_pearson + value: 87.61476224354806 + - type: manhattan_spearman + value: 87.95220889254693 + - type: pearson + value: 87.17245205384889 + - type: spearman + value: 87.7360146030987 + task: + type: STS + - dataset: + config: default + name: MTEB SciDocsRR + revision: d3c5e1fc0b855ab6097bf1cda04dd73947d7caab + split: test + type: mteb/scidocs-reranking + metrics: + - type: main_score + value: 88.43547871921146 + - type: map + value: 88.43547871921146 + - type: mrr + value: 96.5564473652709 + - type: nAUC_map_diff1 + value: -13.66029392579231 + - type: nAUC_map_max + value: 50.325613574053506 + - type: nAUC_map_std + value: 60.02986231275796 + - type: nAUC_mrr_diff1 + value: 23.83821476411125 + - type: nAUC_mrr_max + value: 86.72643311769906 + - type: nAUC_mrr_std + value: 72.12741063469213 + task: + type: Reranking + - dataset: + config: default + name: MTEB SciFact + revision: 0228b52cf27578f30900b9e5271d331663a030d7 + split: test + type: mteb/scifact + metrics: + - type: main_score + value: 78.233 + - type: map_at_1 + value: 61.49400000000001 + - type: map_at_10 + value: 73.30600000000001 + - type: map_at_100 + value: 73.719 + - type: map_at_1000 + value: 73.724 + - type: map_at_20 + value: 73.611 + - type: map_at_3 + value: 70.626 + - type: map_at_5 + value: 72.417 + - type: mrr_at_1 + value: 64.66666666666666 + - type: mrr_at_10 + value: 74.30357142857143 + - type: mrr_at_100 + value: 74.56950898079988 + - type: mrr_at_1000 + value: 74.57295833098681 + - type: mrr_at_20 + value: 74.46165223665226 + - type: mrr_at_3 + value: 72.3888888888889 + - type: mrr_at_5 + value: 73.60555555555557 + - type: nauc_map_at_1000_diff1 + value: 76.51524604780636 + - type: nauc_map_at_1000_max + value: 53.48521938401881 + - type: nauc_map_at_1000_std + value: -7.347799382158861 + - type: nauc_map_at_100_diff1 + value: 76.5122888096236 + - type: nauc_map_at_100_max + value: 53.49221847471618 + - type: nauc_map_at_100_std + value: -7.329683735681086 + - type: nauc_map_at_10_diff1 + value: 76.30928630674504 + - type: nauc_map_at_10_max + value: 53.00102977185941 + - type: nauc_map_at_10_std + value: -7.7467740085108705 + - type: nauc_map_at_1_diff1 + value: 79.54189281784247 + - type: nauc_map_at_1_max + value: 46.630071622109526 + - type: nauc_map_at_1_std + value: -14.395943134644112 + - type: nauc_map_at_20_diff1 + value: 76.41604361947962 + - type: nauc_map_at_20_max + value: 53.578883876146875 + - type: nauc_map_at_20_std + value: -7.403103451288041 + - type: nauc_map_at_3_diff1 + value: 76.25911617571941 + - type: nauc_map_at_3_max + value: 49.140287380513605 + - type: nauc_map_at_3_std + value: -11.35992449218983 + - type: nauc_map_at_5_diff1 + value: 76.35122077770336 + - type: nauc_map_at_5_max + value: 52.1744367901208 + - type: nauc_map_at_5_std + value: -7.85753955055384 + - type: nauc_mrr_at_1000_diff1 + value: 76.97223309515867 + - type: nauc_mrr_at_1000_max + value: 57.263787498613326 + - type: nauc_mrr_at_1000_std + value: -4.884090708840035 + - type: nauc_mrr_at_100_diff1 + value: 76.97312970894603 + - type: nauc_mrr_at_100_max + value: 57.26850730446478 + - type: nauc_mrr_at_100_std + value: -4.875200894216617 + - type: nauc_mrr_at_10_diff1 + value: 76.65927674223613 + - type: nauc_mrr_at_10_max + value: 57.30979763941454 + - type: nauc_mrr_at_10_std + value: -4.863331094022142 + - type: nauc_mrr_at_1_diff1 + value: 80.0454932568644 + - type: nauc_mrr_at_1_max + value: 56.76038421319305 + - type: nauc_mrr_at_1_std + value: -4.101939392632653 + - type: nauc_mrr_at_20_diff1 + value: 76.87237970440503 + - type: nauc_mrr_at_20_max + value: 57.33843605225869 + - type: nauc_mrr_at_20_std + value: -4.96248984417978 + - type: nauc_mrr_at_3_diff1 + value: 76.74130186666727 + - type: nauc_mrr_at_3_max + value: 56.19313244846155 + - type: nauc_mrr_at_3_std + value: -5.684365934009136 + - type: nauc_mrr_at_5_diff1 + value: 76.66406918799962 + - type: nauc_mrr_at_5_max + value: 57.56110093228628 + - type: nauc_mrr_at_5_std + value: -3.7464413085588073 + - type: nauc_ndcg_at_1000_diff1 + value: 76.19194173971773 + - type: nauc_ndcg_at_1000_max + value: 55.57464600170693 + - type: nauc_ndcg_at_1000_std + value: -6.0761689532372625 + - type: nauc_ndcg_at_100_diff1 + value: 76.14631273843654 + - type: nauc_ndcg_at_100_max + value: 55.72246565373382 + - type: nauc_ndcg_at_100_std + value: -5.595160698860595 + - type: nauc_ndcg_at_10_diff1 + value: 75.0108223611192 + - type: nauc_ndcg_at_10_max + value: 55.27894212877493 + - type: nauc_ndcg_at_10_std + value: -6.968331740214591 + - type: nauc_ndcg_at_1_diff1 + value: 80.0454932568644 + - type: nauc_ndcg_at_1_max + value: 56.76038421319305 + - type: nauc_ndcg_at_1_std + value: -4.101939392632653 + - type: nauc_ndcg_at_20_diff1 + value: 75.54887755702472 + - type: nauc_ndcg_at_20_max + value: 56.406879417251496 + - type: nauc_ndcg_at_20_std + value: -6.495231061329629 + - type: nauc_ndcg_at_3_diff1 + value: 75.03620356688509 + - type: nauc_ndcg_at_3_max + value: 52.147381077773424 + - type: nauc_ndcg_at_3_std + value: -8.448005688956199 + - type: nauc_ndcg_at_5_diff1 + value: 75.1195898074229 + - type: nauc_ndcg_at_5_max + value: 54.2321033861173 + - type: nauc_ndcg_at_5_std + value: -5.882690780895338 + - type: nauc_precision_at_1000_diff1 + value: -28.081979732100532 + - type: nauc_precision_at_1000_max + value: 35.055348014832916 + - type: nauc_precision_at_1000_std + value: 59.61280468927384 + - type: nauc_precision_at_100_diff1 + value: -25.112740730587458 + - type: nauc_precision_at_100_max + value: 38.26331300116496 + - type: nauc_precision_at_100_std + value: 62.46316222328831 + - type: nauc_precision_at_10_diff1 + value: -2.6766206473658833 + - type: nauc_precision_at_10_max + value: 45.95321867204845 + - type: nauc_precision_at_10_std + value: 45.07212468670564 + - type: nauc_precision_at_1_diff1 + value: 80.0454932568644 + - type: nauc_precision_at_1_max + value: 56.76038421319305 + - type: nauc_precision_at_1_std + value: -4.101939392632653 + - type: nauc_precision_at_20_diff1 + value: -10.698911116738385 + - type: nauc_precision_at_20_max + value: 43.467275950182994 + - type: nauc_precision_at_20_std + value: 48.00467321991766 + - type: nauc_precision_at_3_diff1 + value: 33.6344708541193 + - type: nauc_precision_at_3_max + value: 49.309242331670504 + - type: nauc_precision_at_3_std + value: 21.02940391379915 + - type: nauc_precision_at_5_diff1 + value: 13.560415600596318 + - type: nauc_precision_at_5_max + value: 48.918726500100085 + - type: nauc_precision_at_5_std + value: 39.940930429172184 + - type: nauc_recall_at_1000_diff1 + value: .nan + - type: nauc_recall_at_1000_max + value: .nan + - type: nauc_recall_at_1000_std + value: .nan + - type: nauc_recall_at_100_diff1 + value: 70.82166199813196 + - type: nauc_recall_at_100_max + value: 76.6106442577042 + - type: nauc_recall_at_100_std + value: 66.47992530345513 + - type: nauc_recall_at_10_diff1 + value: 62.68908885556092 + - type: nauc_recall_at_10_max + value: 58.14262437741839 + - type: nauc_recall_at_10_std + value: -12.946717875063369 + - type: nauc_recall_at_1_diff1 + value: 79.54189281784247 + - type: nauc_recall_at_1_max + value: 46.630071622109526 + - type: nauc_recall_at_1_std + value: -14.395943134644112 + - type: nauc_recall_at_20_diff1 + value: 65.79470497876567 + - type: nauc_recall_at_20_max + value: 71.68308183488456 + - type: nauc_recall_at_20_std + value: -12.556850697268453 + - type: nauc_recall_at_3_diff1 + value: 68.3240211318129 + - type: nauc_recall_at_3_max + value: 45.05998217275036 + - type: nauc_recall_at_3_std + value: -14.23179772593869 + - type: nauc_recall_at_5_diff1 + value: 67.53366869904056 + - type: nauc_recall_at_5_max + value: 53.57935627081027 + - type: nauc_recall_at_5_std + value: -3.3271112904853393 + - type: ndcg_at_1 + value: 64.667 + - type: ndcg_at_10 + value: 78.233 + - type: ndcg_at_100 + value: 79.806 + - type: ndcg_at_1000 + value: 79.92099999999999 + - type: ndcg_at_20 + value: 79.006 + - type: ndcg_at_3 + value: 74.018 + - type: ndcg_at_5 + value: 76.334 + - type: precision_at_1 + value: 64.667 + - type: precision_at_10 + value: 10.4 + - type: precision_at_100 + value: 1.1199999999999999 + - type: precision_at_1000 + value: 0.11299999999999999 + - type: precision_at_20 + value: 5.383 + - type: precision_at_3 + value: 29.444 + - type: precision_at_5 + value: 19.467000000000002 + - type: recall_at_1 + value: 61.49400000000001 + - type: recall_at_10 + value: 92.156 + - type: recall_at_100 + value: 99.167 + - type: recall_at_1000 + value: 100.0 + - type: recall_at_20 + value: 94.833 + - type: recall_at_3 + value: 80.833 + - type: recall_at_5 + value: 86.6 + task: + type: Retrieval + - dataset: + config: default + name: MTEB SprintDuplicateQuestions + revision: d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46 + split: test + type: mteb/sprintduplicatequestions-pairclassification + metrics: + - type: cosine_accuracy + value: 99.8039603960396 + - type: cosine_accuracy_threshold + value: 84.54211950302124 + - type: cosine_ap + value: 95.59056372734358 + - type: cosine_f1 + value: 90.1394422310757 + - type: cosine_f1_threshold + value: 84.54211950302124 + - type: cosine_precision + value: 89.78174603174604 + - type: cosine_recall + value: 90.5 + - type: dot_accuracy + value: 99.80594059405941 + - type: dot_accuracy_threshold + value: 85.57180166244507 + - type: dot_ap + value: 95.53453431914399 + - type: dot_f1 + value: 90.10442565887618 + - type: dot_f1_threshold + value: 84.59715843200684 + - type: dot_precision + value: 89.61424332344214 + - type: dot_recall + value: 90.60000000000001 + - type: euclidean_accuracy + value: 99.8039603960396 + - type: euclidean_accuracy_threshold + value: 53.253382444381714 + - type: euclidean_ap + value: 95.5850992402159 + - type: euclidean_f1 + value: 90.09457441513192 + - type: euclidean_f1_threshold + value: 55.725520849227905 + - type: euclidean_precision + value: 89.69276511397423 + - type: euclidean_recall + value: 90.5 + - type: main_score + value: 95.7485189884476 + - type: manhattan_accuracy + value: 99.81485148514851 + - type: manhattan_accuracy_threshold + value: 3491.29638671875 + - type: manhattan_ap + value: 95.7485189884476 + - type: manhattan_f1 + value: 90.464048954615 + - type: manhattan_f1_threshold + value: 3491.29638671875 + - type: manhattan_precision + value: 92.2996878251821 + - type: manhattan_recall + value: 88.7 + - type: max_ap + value: 95.7485189884476 + - type: max_f1 + value: 90.464048954615 + - type: max_precision + value: 92.2996878251821 + - type: max_recall + value: 90.60000000000001 + - type: similarity_accuracy + value: 99.8039603960396 + - type: similarity_accuracy_threshold + value: 84.54211950302124 + - type: similarity_ap + value: 95.59056372734358 + - type: similarity_f1 + value: 90.1394422310757 + - type: similarity_f1_threshold + value: 84.54211950302124 + - type: similarity_precision + value: 89.78174603174604 + - type: similarity_recall + value: 90.5 + task: + type: PairClassification + - dataset: + config: default + name: MTEB StackExchangeClustering + revision: 6cbc1f7b2bc0622f2e39d2c77fa502909748c259 + split: test + type: mteb/stackexchange-clustering + metrics: + - type: main_score + value: 78.49205191950675 + - type: v_measure + value: 78.49205191950675 + - type: v_measure_std + value: 2.84869550699959 + task: + type: Clustering + - dataset: + config: default + name: MTEB StackExchangeClusteringP2P + revision: 815ca46b2622cec33ccafc3735d572c266efdb44 + split: test + type: mteb/stackexchange-clustering-p2p + metrics: + - type: main_score + value: 48.90421736513028 + - type: v_measure + value: 48.90421736513028 + - type: v_measure_std + value: 1.6875865714471023 + task: + type: Clustering + - dataset: + config: default + name: MTEB StackOverflowDupQuestions + revision: e185fbe320c72810689fc5848eb6114e1ef5ec69 + split: test + type: mteb/stackoverflowdupquestions-reranking + metrics: + - type: main_score + value: 52.9874730481696 + - type: map + value: 52.9874730481696 + - type: mrr + value: 53.85867604617604 + - type: nAUC_map_diff1 + value: 39.633429293407616 + - type: nAUC_map_max + value: 10.236807988858546 + - type: nAUC_map_std + value: 10.276522217929674 + - type: nAUC_mrr_diff1 + value: 40.0543079218377 + - type: nAUC_mrr_max + value: 10.96209807382042 + - type: nAUC_mrr_std + value: 10.524400196109918 + task: + type: Reranking + - dataset: + config: default + name: MTEB SummEval + revision: cda12ad7615edc362dbf25a00fdd61d3b1eaf93c + split: test + type: mteb/summeval + metrics: + - type: cosine_pearson + value: 30.727801109114232 + - type: cosine_spearman + value: 31.66058223980157 + - type: dot_pearson + value: 30.78818248622866 + - type: dot_spearman + value: 31.525158776890265 + - type: main_score + value: 31.66058223980157 + - type: pearson + value: 30.727801109114232 + - type: spearman + value: 31.66058223980157 + task: + type: Summarization + - dataset: + config: default + name: MTEB TRECCOVID + revision: bb9466bac8153a0349341eb1b22e06409e78ef4e + split: test + type: mteb/trec-covid + metrics: + - type: main_score + value: 85.206 + - type: map_at_1 + value: 0.246 + - type: map_at_10 + value: 2.1950000000000003 + - type: map_at_100 + value: 14.179 + - type: map_at_1000 + value: 35.037 + - type: map_at_20 + value: 4.143 + - type: map_at_3 + value: 0.7100000000000001 + - type: map_at_5 + value: 1.135 + - type: mrr_at_1 + value: 94.0 + - type: mrr_at_10 + value: 96.66666666666666 + - type: mrr_at_100 + value: 96.66666666666666 + - type: mrr_at_1000 + value: 96.66666666666666 + - type: mrr_at_20 + value: 96.66666666666666 + - type: mrr_at_3 + value: 96.66666666666666 + - type: mrr_at_5 + value: 96.66666666666666 + - type: nauc_map_at_1000_diff1 + value: -4.6264497624527525 + - type: nauc_map_at_1000_max + value: 44.594457564749355 + - type: nauc_map_at_1000_std + value: 73.17642341400133 + - type: nauc_map_at_100_diff1 + value: 23.451335157405726 + - type: nauc_map_at_100_max + value: 25.426398857299525 + - type: nauc_map_at_100_std + value: 64.07416694472633 + - type: nauc_map_at_10_diff1 + value: 46.57568738568346 + - type: nauc_map_at_10_max + value: 9.693233249079238 + - type: nauc_map_at_10_std + value: 28.549530265164357 + - type: nauc_map_at_1_diff1 + value: 53.48238396620123 + - type: nauc_map_at_1_max + value: 0.33476619393733076 + - type: nauc_map_at_1_std + value: 8.906362219128463 + - type: nauc_map_at_20_diff1 + value: 39.40719602207749 + - type: nauc_map_at_20_max + value: 9.635915072074045 + - type: nauc_map_at_20_std + value: 35.15634791346394 + - type: nauc_map_at_3_diff1 + value: 53.11784737840137 + - type: nauc_map_at_3_max + value: 3.059682761072153 + - type: nauc_map_at_3_std + value: 21.310633086556617 + - type: nauc_map_at_5_diff1 + value: 49.91570701185436 + - type: nauc_map_at_5_max + value: 8.045082896244576 + - type: nauc_map_at_5_std + value: 20.597686235051647 + - type: nauc_mrr_at_1000_diff1 + value: 41.98412698412726 + - type: nauc_mrr_at_1000_max + value: 78.24463118580779 + - type: nauc_mrr_at_1000_std + value: 0.30812324930028195 + - type: nauc_mrr_at_100_diff1 + value: 41.98412698412726 + - type: nauc_mrr_at_100_max + value: 78.24463118580779 + - type: nauc_mrr_at_100_std + value: 0.30812324930028195 + - type: nauc_mrr_at_10_diff1 + value: 41.98412698412726 + - type: nauc_mrr_at_10_max + value: 78.24463118580779 + - type: nauc_mrr_at_10_std + value: 0.30812324930028195 + - type: nauc_mrr_at_1_diff1 + value: 38.62433862433873 + - type: nauc_mrr_at_1_max + value: 80.78120136943666 + - type: nauc_mrr_at_1_std + value: -10.768751945222197 + - type: nauc_mrr_at_20_diff1 + value: 41.98412698412726 + - type: nauc_mrr_at_20_max + value: 78.24463118580779 + - type: nauc_mrr_at_20_std + value: 0.30812324930028195 + - type: nauc_mrr_at_3_diff1 + value: 41.98412698412726 + - type: nauc_mrr_at_3_max + value: 78.24463118580779 + - type: nauc_mrr_at_3_std + value: 0.30812324930028195 + - type: nauc_mrr_at_5_diff1 + value: 41.98412698412726 + - type: nauc_mrr_at_5_max + value: 78.24463118580779 + - type: nauc_mrr_at_5_std + value: 0.30812324930028195 + - type: nauc_ndcg_at_1000_diff1 + value: 0.5174948602880207 + - type: nauc_ndcg_at_1000_max + value: 48.60686602077053 + - type: nauc_ndcg_at_1000_std + value: 75.72456343175277 + - type: nauc_ndcg_at_100_diff1 + value: -20.747252137999254 + - type: nauc_ndcg_at_100_max + value: 49.985132618254994 + - type: nauc_ndcg_at_100_std + value: 61.096383293836574 + - type: nauc_ndcg_at_10_diff1 + value: 6.791377920463332 + - type: nauc_ndcg_at_10_max + value: 57.50019332833286 + - type: nauc_ndcg_at_10_std + value: 49.201028841219426 + - type: nauc_ndcg_at_1_diff1 + value: 54.92683440362145 + - type: nauc_ndcg_at_1_max + value: 83.8667228129276 + - type: nauc_ndcg_at_1_std + value: 1.6738604063586122 + - type: nauc_ndcg_at_20_diff1 + value: -5.1948699196314925 + - type: nauc_ndcg_at_20_max + value: 54.483087684806556 + - type: nauc_ndcg_at_20_std + value: 50.54823818118781 + - type: nauc_ndcg_at_3_diff1 + value: 26.267246500164372 + - type: nauc_ndcg_at_3_max + value: 63.0173212926611 + - type: nauc_ndcg_at_3_std + value: 41.025597406368256 + - type: nauc_ndcg_at_5_diff1 + value: 16.910185454343036 + - type: nauc_ndcg_at_5_max + value: 60.9328683868778 + - type: nauc_ndcg_at_5_std + value: 36.70169905857712 + - type: nauc_precision_at_1000_diff1 + value: -46.374447765983525 + - type: nauc_precision_at_1000_max + value: 35.36052337813863 + - type: nauc_precision_at_1000_std + value: 14.219220668161018 + - type: nauc_precision_at_100_diff1 + value: -29.7838083657744 + - type: nauc_precision_at_100_max + value: 43.93589400385112 + - type: nauc_precision_at_100_std + value: 55.425045718579945 + - type: nauc_precision_at_10_diff1 + value: -12.016613405227687 + - type: nauc_precision_at_10_max + value: 57.79924427743131 + - type: nauc_precision_at_10_std + value: 49.022036703550675 + - type: nauc_precision_at_1_diff1 + value: 38.62433862433873 + - type: nauc_precision_at_1_max + value: 80.78120136943666 + - type: nauc_precision_at_1_std + value: -10.768751945222197 + - type: nauc_precision_at_20_diff1 + value: -23.95633847880195 + - type: nauc_precision_at_20_max + value: 48.34715917258276 + - type: nauc_precision_at_20_std + value: 48.82198285255887 + - type: nauc_precision_at_3_diff1 + value: 6.871296905858807 + - type: nauc_precision_at_3_max + value: 70.54805793285054 + - type: nauc_precision_at_3_std + value: 44.65108624094803 + - type: nauc_precision_at_5_diff1 + value: -9.074932448759695 + - type: nauc_precision_at_5_max + value: 67.41284242437573 + - type: nauc_precision_at_5_std + value: 23.876891983919577 + - type: nauc_recall_at_1000_diff1 + value: 8.142288830293255 + - type: nauc_recall_at_1000_max + value: 38.85182826835104 + - type: nauc_recall_at_1000_std + value: 68.60783819217335 + - type: nauc_recall_at_100_diff1 + value: 34.262914076287466 + - type: nauc_recall_at_100_max + value: 12.87009658528838 + - type: nauc_recall_at_100_std + value: 56.21330603762995 + - type: nauc_recall_at_10_diff1 + value: 49.33830945338758 + - type: nauc_recall_at_10_max + value: 0.3539875530671406 + - type: nauc_recall_at_10_std + value: 26.85864465557644 + - type: nauc_recall_at_1_diff1 + value: 53.48238396620123 + - type: nauc_recall_at_1_max + value: 0.33476619393733076 + - type: nauc_recall_at_1_std + value: 8.906362219128463 + - type: nauc_recall_at_20_diff1 + value: 44.21928181266254 + - type: nauc_recall_at_20_max + value: -0.9198356057088594 + - type: nauc_recall_at_20_std + value: 31.484376992896784 + - type: nauc_recall_at_3_diff1 + value: 53.038093080990876 + - type: nauc_recall_at_3_max + value: -1.4170895916973003 + - type: nauc_recall_at_3_std + value: 21.890202855574497 + - type: nauc_recall_at_5_diff1 + value: 49.39742214825278 + - type: nauc_recall_at_5_max + value: 2.8412267611894517 + - type: nauc_recall_at_5_std + value: 18.01598921859512 + - type: ndcg_at_1 + value: 91.0 + - type: ndcg_at_10 + value: 85.206 + - type: ndcg_at_100 + value: 67.29 + - type: ndcg_at_1000 + value: 60.584 + - type: ndcg_at_20 + value: 82.321 + - type: ndcg_at_3 + value: 88.642 + - type: ndcg_at_5 + value: 87.063 + - type: precision_at_1 + value: 94.0 + - type: precision_at_10 + value: 89.8 + - type: precision_at_100 + value: 69.78 + - type: precision_at_1000 + value: 26.738 + - type: precision_at_20 + value: 87.2 + - type: precision_at_3 + value: 92.0 + - type: precision_at_5 + value: 90.8 + - type: recall_at_1 + value: 0.246 + - type: recall_at_10 + value: 2.344 + - type: recall_at_100 + value: 16.962 + - type: recall_at_1000 + value: 57.325 + - type: recall_at_20 + value: 4.517 + - type: recall_at_3 + value: 0.731 + - type: recall_at_5 + value: 1.1780000000000002 + task: + type: Retrieval + - dataset: + config: default + name: MTEB Touche2020 + revision: a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f + split: test + type: mteb/touche2020 + metrics: + - type: main_score + value: 31.455 + - type: map_at_1 + value: 2.9739999999999998 + - type: map_at_10 + value: 12.183 + - type: map_at_100 + value: 18.772 + - type: map_at_1000 + value: 20.415 + - type: map_at_20 + value: 14.451 + - type: map_at_3 + value: 6.507000000000001 + - type: map_at_5 + value: 8.66 + - type: mrr_at_1 + value: 40.816326530612244 + - type: mrr_at_10 + value: 57.70975056689341 + - type: mrr_at_100 + value: 58.18379126542391 + - type: mrr_at_1000 + value: 58.18379126542391 + - type: mrr_at_20 + value: 57.85552316164561 + - type: mrr_at_3 + value: 54.08163265306123 + - type: mrr_at_5 + value: 56.42857142857143 + - type: nauc_map_at_1000_diff1 + value: 3.1567471051481437 + - type: nauc_map_at_1000_max + value: -1.5882060729791523 + - type: nauc_map_at_1000_std + value: 18.69622198722074 + - type: nauc_map_at_100_diff1 + value: 3.3449677678147536 + - type: nauc_map_at_100_max + value: -2.8928606866168405 + - type: nauc_map_at_100_std + value: 15.789984947653412 + - type: nauc_map_at_10_diff1 + value: 2.9696743570444264 + - type: nauc_map_at_10_max + value: -9.096749212011876 + - type: nauc_map_at_10_std + value: -5.38545817258353 + - type: nauc_map_at_1_diff1 + value: 20.680780404542546 + - type: nauc_map_at_1_max + value: -7.04722927447817 + - type: nauc_map_at_1_std + value: -7.062494733973898 + - type: nauc_map_at_20_diff1 + value: 4.070437790119271 + - type: nauc_map_at_20_max + value: -4.84491434686032 + - type: nauc_map_at_20_std + value: 0.5846341109021014 + - type: nauc_map_at_3_diff1 + value: 11.9634978045925 + - type: nauc_map_at_3_max + value: -8.27834591046608 + - type: nauc_map_at_3_std + value: -8.687615453381065 + - type: nauc_map_at_5_diff1 + value: 0.9195191526009436 + - type: nauc_map_at_5_max + value: -1.673813362719489 + - type: nauc_map_at_5_std + value: -6.67549753473631 + - type: nauc_mrr_at_1000_diff1 + value: 19.877993208719573 + - type: nauc_mrr_at_1000_max + value: -10.37776706406218 + - type: nauc_mrr_at_1000_std + value: 7.132169578056367 + - type: nauc_mrr_at_100_diff1 + value: 19.877993208719573 + - type: nauc_mrr_at_100_max + value: -10.37776706406218 + - type: nauc_mrr_at_100_std + value: 7.132169578056367 + - type: nauc_mrr_at_10_diff1 + value: 20.414285568401457 + - type: nauc_mrr_at_10_max + value: -9.677800295687861 + - type: nauc_mrr_at_10_std + value: 8.001103690180859 + - type: nauc_mrr_at_1_diff1 + value: 22.393284073955723 + - type: nauc_mrr_at_1_max + value: -5.889370191243167 + - type: nauc_mrr_at_1_std + value: -1.5183536173658247 + - type: nauc_mrr_at_20_diff1 + value: 20.455564720604055 + - type: nauc_mrr_at_20_max + value: -10.230642830103074 + - type: nauc_mrr_at_20_std + value: 7.863582453266621 + - type: nauc_mrr_at_3_diff1 + value: 17.554895390732618 + - type: nauc_mrr_at_3_max + value: -15.618463505555052 + - type: nauc_mrr_at_3_std + value: 5.913231577966864 + - type: nauc_mrr_at_5_diff1 + value: 18.393678507779914 + - type: nauc_mrr_at_5_max + value: -11.903593353147762 + - type: nauc_mrr_at_5_std + value: 7.580745996262831 + - type: nauc_ndcg_at_1000_diff1 + value: 13.746937095530473 + - type: nauc_ndcg_at_1000_max + value: -0.9319249687895838 + - type: nauc_ndcg_at_1000_std + value: 38.56328031451904 + - type: nauc_ndcg_at_100_diff1 + value: 13.854865944415895 + - type: nauc_ndcg_at_100_max + value: -7.142142012591404 + - type: nauc_ndcg_at_100_std + value: 35.61341954818848 + - type: nauc_ndcg_at_10_diff1 + value: 9.010144273248759 + - type: nauc_ndcg_at_10_max + value: -15.320014897424574 + - type: nauc_ndcg_at_10_std + value: 2.84883880489144 + - type: nauc_ndcg_at_1_diff1 + value: 20.939533945592967 + - type: nauc_ndcg_at_1_max + value: -6.387319972188946 + - type: nauc_ndcg_at_1_std + value: -0.5258673122126726 + - type: nauc_ndcg_at_20_diff1 + value: 14.660827309009496 + - type: nauc_ndcg_at_20_max + value: -13.476196120145994 + - type: nauc_ndcg_at_20_std + value: 8.22391881710838 + - type: nauc_ndcg_at_3_diff1 + value: 13.429985227235935 + - type: nauc_ndcg_at_3_max + value: -14.904544592570247 + - type: nauc_ndcg_at_3_std + value: 1.599779998183342 + - type: nauc_ndcg_at_5_diff1 + value: 8.085466231900622 + - type: nauc_ndcg_at_5_max + value: -9.09591969526831 + - type: nauc_ndcg_at_5_std + value: 3.5794092637248505 + - type: nauc_precision_at_1000_diff1 + value: -9.31941215946743 + - type: nauc_precision_at_1000_max + value: 31.52913520470716 + - type: nauc_precision_at_1000_std + value: 22.720784312185856 + - type: nauc_precision_at_100_diff1 + value: 8.958548406995279 + - type: nauc_precision_at_100_max + value: 15.100597910674104 + - type: nauc_precision_at_100_std + value: 71.04548238175113 + - type: nauc_precision_at_10_diff1 + value: 12.4698194690008 + - type: nauc_precision_at_10_max + value: -15.84870544871496 + - type: nauc_precision_at_10_std + value: 7.575297622501928 + - type: nauc_precision_at_1_diff1 + value: 22.393284073955723 + - type: nauc_precision_at_1_max + value: -5.889370191243167 + - type: nauc_precision_at_1_std + value: -1.5183536173658247 + - type: nauc_precision_at_20_diff1 + value: 15.393505718138758 + - type: nauc_precision_at_20_max + value: -3.70684298539384 + - type: nauc_precision_at_20_std + value: 29.426137824970304 + - type: nauc_precision_at_3_diff1 + value: 9.997768085465394 + - type: nauc_precision_at_3_max + value: -17.12224314347674 + - type: nauc_precision_at_3_std + value: -1.343018166772313 + - type: nauc_precision_at_5_diff1 + value: 3.8936997437913554 + - type: nauc_precision_at_5_max + value: -5.689104289687632 + - type: nauc_precision_at_5_std + value: 3.181098051304285 + - type: nauc_recall_at_1000_diff1 + value: 9.908303508158387 + - type: nauc_recall_at_1000_max + value: 6.174506592699848 + - type: nauc_recall_at_1000_std + value: 77.41931114780012 + - type: nauc_recall_at_100_diff1 + value: 10.286839241876192 + - type: nauc_recall_at_100_max + value: -6.6138697026666815 + - type: nauc_recall_at_100_std + value: 49.608313692633224 + - type: nauc_recall_at_10_diff1 + value: 2.215545846659851 + - type: nauc_recall_at_10_max + value: -17.83025802478445 + - type: nauc_recall_at_10_std + value: -3.3784768673705465 + - type: nauc_recall_at_1_diff1 + value: 20.680780404542546 + - type: nauc_recall_at_1_max + value: -7.04722927447817 + - type: nauc_recall_at_1_std + value: -7.062494733973898 + - type: nauc_recall_at_20_diff1 + value: 6.974410239251615 + - type: nauc_recall_at_20_max + value: -14.161147924731646 + - type: nauc_recall_at_20_std + value: 9.328412057721454 + - type: nauc_recall_at_3_diff1 + value: 7.904589805754212 + - type: nauc_recall_at_3_max + value: -12.1912388648593 + - type: nauc_recall_at_3_std + value: -9.221542013385555 + - type: nauc_recall_at_5_diff1 + value: -3.2604132752706914 + - type: nauc_recall_at_5_max + value: -6.886351441658915 + - type: nauc_recall_at_5_std + value: -7.014252851712789 + - type: ndcg_at_1 + value: 39.796 + - type: ndcg_at_10 + value: 31.455 + - type: ndcg_at_100 + value: 42.388999999999996 + - type: ndcg_at_1000 + value: 53.556000000000004 + - type: ndcg_at_20 + value: 30.808000000000003 + - type: ndcg_at_3 + value: 35.831 + - type: ndcg_at_5 + value: 32.845 + - type: precision_at_1 + value: 40.816 + - type: precision_at_10 + value: 27.143 + - type: precision_at_100 + value: 8.449 + - type: precision_at_1000 + value: 1.6179999999999999 + - type: precision_at_20 + value: 19.387999999999998 + - type: precision_at_3 + value: 35.374 + - type: precision_at_5 + value: 31.019999999999996 + - type: recall_at_1 + value: 2.9739999999999998 + - type: recall_at_10 + value: 19.39 + - type: recall_at_100 + value: 51.636 + - type: recall_at_1000 + value: 86.99900000000001 + - type: recall_at_20 + value: 26.478 + - type: recall_at_3 + value: 7.703 + - type: recall_at_5 + value: 11.42 + task: + type: Retrieval + - dataset: + config: default + name: MTEB ToxicConversationsClassification + revision: edfaf9da55d3dd50d43143d90c1ac476895ae6de + split: test + type: mteb/toxic_conversations_50k + metrics: + - type: accuracy + value: 86.9384765625 + - type: ap + value: 31.737513704141552 + - type: ap_weighted + value: 31.737513704141552 + - type: f1 + value: 71.5490757306975 + - type: f1_weighted + value: 89.14632533489856 + - type: main_score + value: 86.9384765625 + task: + type: Classification + - dataset: + config: default + name: MTEB TweetSentimentExtractionClassification + revision: d604517c81ca91fe16a244d1248fc021f9ecee7a + split: test + type: mteb/tweet_sentiment_extraction + metrics: + - type: accuracy + value: 73.57668364459535 + - type: f1 + value: 73.90467103648074 + - type: f1_weighted + value: 73.42158415034704 + - type: main_score + value: 73.57668364459535 + task: + type: Classification + - dataset: + config: default + name: MTEB TwentyNewsgroupsClustering + revision: 6125ec4e24fa026cec8a478383ee943acfbd5449 + split: test + type: mteb/twentynewsgroups-clustering + metrics: + - type: main_score + value: 58.574148097494685 + - type: v_measure + value: 58.574148097494685 + - type: v_measure_std + value: 0.9443161637490822 + task: + type: Clustering + - dataset: + config: default + name: MTEB TwitterSemEval2015 + revision: 70970daeab8776df92f5ea462b6173c0b46fd2d1 + split: test + type: mteb/twittersemeval2015-pairclassification + metrics: + - type: cosine_accuracy + value: 88.1385229778864 + - type: cosine_accuracy_threshold + value: 83.86307954788208 + - type: cosine_ap + value: 80.17965893449055 + - type: cosine_f1 + value: 73.0614300100705 + - type: cosine_f1_threshold + value: 80.7942807674408 + - type: cosine_precision + value: 69.8603755416466 + - type: cosine_recall + value: 76.56992084432717 + - type: dot_accuracy + value: 88.2100494724921 + - type: dot_accuracy_threshold + value: 83.84793996810913 + - type: dot_ap + value: 80.18603932881858 + - type: dot_f1 + value: 73.07643714466204 + - type: dot_f1_threshold + value: 80.87586164474487 + - type: dot_precision + value: 70.10909090909091 + - type: dot_recall + value: 76.3060686015831 + - type: euclidean_accuracy + value: 88.1385229778864 + - type: euclidean_accuracy_threshold + value: 56.77661895751953 + - type: euclidean_ap + value: 80.1784070881624 + - type: euclidean_f1 + value: 73.04830369529574 + - type: euclidean_f1_threshold + value: 61.91838979721069 + - type: euclidean_precision + value: 69.96859144720948 + - type: euclidean_recall + value: 76.41160949868075 + - type: main_score + value: 80.18603932881858 + - type: manhattan_accuracy + value: 88.0431543184121 + - type: manhattan_accuracy_threshold + value: 3755.6137084960938 + - type: manhattan_ap + value: 79.98270453664578 + - type: manhattan_f1 + value: 72.68242015061023 + - type: manhattan_f1_threshold + value: 3892.494583129883 + - type: manhattan_precision + value: 71.54907975460122 + - type: manhattan_recall + value: 73.85224274406332 + - type: max_ap + value: 80.18603932881858 + - type: max_f1 + value: 73.07643714466204 + - type: max_precision + value: 71.54907975460122 + - type: max_recall + value: 76.56992084432717 + - type: similarity_accuracy + value: 88.1385229778864 + - type: similarity_accuracy_threshold + value: 83.86307954788208 + - type: similarity_ap + value: 80.17965893449055 + - type: similarity_f1 + value: 73.0614300100705 + - type: similarity_f1_threshold + value: 80.7942807674408 + - type: similarity_precision + value: 69.8603755416466 + - type: similarity_recall + value: 76.56992084432717 + task: + type: PairClassification + - dataset: + config: default + name: MTEB TwitterURLCorpus + revision: 8b6510b0b1fa4e4c4f879467980e9be563ec1cdf + split: test + type: mteb/twitterurlcorpus-pairclassification + metrics: + - type: cosine_accuracy + value: 89.7892653393876 + - type: cosine_accuracy_threshold + value: 79.69566583633423 + - type: cosine_ap + value: 87.4579867302024 + - type: cosine_f1 + value: 79.91620843152658 + - type: cosine_f1_threshold + value: 78.53609323501587 + - type: cosine_precision + value: 77.7155329210622 + - type: cosine_recall + value: 82.24514936864799 + - type: dot_accuracy + value: 89.78732487289945 + - type: dot_accuracy_threshold + value: 80.05315661430359 + - type: dot_ap + value: 87.44916182456272 + - type: dot_f1 + value: 79.90419878751591 + - type: dot_f1_threshold + value: 78.57890725135803 + - type: dot_precision + value: 77.73409057812728 + - type: dot_recall + value: 82.19895287958116 + - type: euclidean_accuracy + value: 89.78538440641131 + - type: euclidean_accuracy_threshold + value: 62.29925751686096 + - type: euclidean_ap + value: 87.45904868911386 + - type: euclidean_f1 + value: 79.93127404474657 + - type: euclidean_f1_threshold + value: 65.61101078987122 + - type: euclidean_precision + value: 77.62060210373595 + - type: euclidean_recall + value: 82.38373883584848 + - type: main_score + value: 87.46554314325058 + - type: manhattan_accuracy + value: 89.76597974152986 + - type: manhattan_accuracy_threshold + value: 3988.5299682617188 + - type: manhattan_ap + value: 87.46554314325058 + - type: manhattan_f1 + value: 79.97181740645973 + - type: manhattan_f1_threshold + value: 4235.905838012695 + - type: manhattan_precision + value: 77.13713427283783 + - type: manhattan_recall + value: 83.02279026793964 + - type: max_ap + value: 87.46554314325058 + - type: max_f1 + value: 79.97181740645973 + - type: max_precision + value: 77.73409057812728 + - type: max_recall + value: 83.02279026793964 + - type: similarity_accuracy + value: 89.7892653393876 + - type: similarity_accuracy_threshold + value: 79.69566583633423 + - type: similarity_ap + value: 87.4579867302024 + - type: similarity_f1 + value: 79.91620843152658 + - type: similarity_f1_threshold + value: 78.53609323501587 + - type: similarity_precision + value: 77.7155329210622 + - type: similarity_recall + value: 82.24514936864799 + task: + type: PairClassification +tags: +- mteb --- + + +# Updates + +New open-source models and ToDoList will be listed on https://github.com/DunZhang/Stella/blob/main/news_and_todo.md. + +You can also find these models on my [homepage](https://huggingface.co/infgrad). + +# Introduction + +The models are trained based on `Alibaba-NLP/gte-large-en-v1.5` and `Alibaba-NLP/gte-Qwen2-1.5B-instruct`. Thanks for +their contributions! + +**We simplify usage of prompts, providing two prompts for most general tasks, one is for s2p, another one is for s2s.** + +Prompt of s2p task(e.g. retrieve task): + +```text +Instruct: Given a web search query, retrieve relevant passages that answer the query.\nQuery: {query} +``` + +Prompt of s2s task(e.g. semantic textual similarity task): + +```text +Instruct: Retrieve semantically similar text.\nQuery: {query} +``` + +The models are finally trained by [MRL]((https://arxiv.org/abs/2205.13147)), so they have multiple dimensions: 512, 768, +1024, 2048, 4096, 6144 and 8192. + +The higher the dimension, the better the performance. +**Generally speaking, 1024d is good enough.** The MTEB score of 1024d is only 0.001 lower than 8192d. + +# Model directory structure + +The model directory structure is very simple, it is a standard SentenceTransformer directory **with a series +of `2_Dense_{dims}` +folders**, where `dims` represents the final vector dimension. + +For example, the `2_Dense_256` folder stores Linear weights that convert vector dimensions to 256 dimensions. +Please refer to the following chapters for specific instructions on how to use them. + +# Usage + +You can use `SentenceTransformer` or `transformer` library to encode text. + +```python +import os +import torch +from transformers import AutoModel, AutoTokenizer +from sentence_transformers import SentenceTransformer +from sklearn.preprocessing import normalize + +prompt = "Instruct: Given a web search query, retrieve relevant passages that answer the query.\nQuery: {query}" +queries = ["query1", "query2"] +queries = [prompt.replace("{query}", query) for query in queries] + +# doc do not need any prompts +docs = ["doc1", "doc2"] + +model_dir = "{Your MODEL_PATH}" + +#### method1: SentenceTransformer +# ！！！！The default dimension is 8192，if you need other dimensions, please copy the files from the `2_Dense_{dims}` folder to overwrite them. For example, `copy -r ./2_Dense_1024/* ./2_Dense/` ！！！！ +model = SentenceTransformer(model_dir, trust_remote_code=True).cuda() +vectors = model.encode(queries, convert_to_numpy=True, normalize_embeddings=True) +print(vectors.shape) +print(vectors[:, :4]) + +#### method2：transformers +vector_linear_directory = "2_Dense" +vector_dim = 8192 +model = AutoModel.from_pretrained(model_dir, trust_remote_code=True).cuda().eval() +tokenizer = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True) +vector_linear = torch.nn.Linear(in_features=model.config.hidden_size, out_features=vector_dim) +vector_linear_dict = { + k.replace("linear.", ""): v for k, v in + torch.load(os.path.join(model_dir, f"{vector_linear_directory}/pytorch_model.bin")).items() +} +vector_linear.load_state_dict(vector_linear_dict) +vector_linear.cuda() +with torch.no_grad(): + input_data = tokenizer(queries, padding="longest", truncation=True, max_length=512, return_tensors="pt") + input_data = {k: v.cuda() for k, v in input_data.items()} + attention_mask = input_data["attention_mask"] + last_hidden_state = model(**input_data)[0] + last_hidden = last_hidden_state.masked_fill(~attention_mask[..., None].bool(), 0.0) + vectors = last_hidden.sum(dim=1) / attention_mask.sum(dim=1)[..., None] + vectors = normalize(vector_linear(vectors).cpu().numpy()) +print(vectors.shape) +print(vectors[:, :4]) + +``` + +# FAQ + +Q: The details of training? + +A: The training method and datasets will be released in the future. (specific time unknown, may be provided in a paper) + +Q: How to choose a suitable prompt for my own task? + +A: In most cases, please use the s2p and s2s prompts. These two prompts account for the vast majority of the training +data. + +Q: How to reproduce MTEB results? + +A: Please use evaluation scripts in `Alibaba-NLP/gte-Qwen2-1.5B-instruct` or `intfloat/e5-mistral-7b-instruct` + +Q: Why each dimension has a linear weight? + +A: MRL has multiple training methods, we choose this method which has the best performance. + +Q: What is the sequence length of models? + +A: 512 is recommended, in our experiments, almost all models perform poorly on specialized long text retrieval datasets. Besides, the +model is trained on datasets of 512 length. This may be an optimization term. + +If you have any questions, please start a discussion on community. \ No newline at end of file