diff --git "a/README.md" "b/README.md" --- "a/README.md" +++ "b/README.md" @@ -1,199 +1,9030 @@ --- -library_name: transformers -tags: [] +library_name: peft +license: mit +language: +- en +pipeline_tag: sentence-similarity +tags: +- text-embedding +- embeddings +- information-retrieval +- beir +- text-classification +- language-model +- text-clustering +- text-semantic-similarity +- text-evaluation +- text-reranking +- feature-extraction +- sentence-similarity +- Sentence Similarity +- natural_questions +- ms_marco +- fever +- hotpot_qa +- mteb +model-index: +- name: no_model_name_available + results: + - dataset: + config: en-ext + name: MTEB AmazonCounterfactualClassification (en-ext) + revision: e8379541af4e31359cca9fbcf4b00f2671dba205 + split: test + type: mteb/amazon_counterfactual + metrics: + - type: accuracy + value: 80.75712143928035 + - type: ap + value: 27.99426423345347 + - type: ap_weighted + value: 27.99426423345347 + - type: f1 + value: 67.21155397213585 + - type: f1_weighted + value: 83.87137520108915 + - type: main_score + value: 80.75712143928035 + task: + type: Classification + - dataset: + config: en + name: MTEB AmazonCounterfactualClassification (en) + revision: e8379541af4e31359cca9fbcf4b00f2671dba205 + split: test + type: mteb/amazon_counterfactual + metrics: + - type: accuracy + value: 79.04477611940298 + - type: ap + value: 42.89052934237333 + - type: ap_weighted + value: 42.89052934237333 + - type: f1 + value: 73.01836842535451 + - type: f1_weighted + value: 80.74846934823574 + - type: main_score + value: 79.04477611940298 + task: + type: Classification + - dataset: + config: default + name: MTEB AmazonPolarityClassification (default) + revision: e2d317d38cd51312af73b3d32a06d1a08b442046 + split: test + type: mteb/amazon_polarity + metrics: + - type: accuracy + value: 82.2419 + - type: ap + value: 76.23727309983535 + - type: ap_weighted + value: 76.23727309983535 + - type: f1 + value: 82.16811276024266 + - type: f1_weighted + value: 82.16811276024266 + - type: main_score + value: 82.2419 + task: + type: Classification + - dataset: + config: en + name: MTEB AmazonReviewsClassification (en) + revision: 1399c76144fd37290681b995c656ef9b2e06e26d + split: test + type: mteb/amazon_reviews_multi + metrics: + - type: accuracy + value: 44.906 + - type: f1 + value: 42.82204511203004 + - type: f1_weighted + value: 42.82204511203004 + - type: main_score + value: 44.906 + task: + type: Classification + - dataset: + config: default + name: MTEB ArguAna (default) + revision: c22ab2a51041ffd869aaddef7af8d8215647e41a + split: test + type: mteb/arguana + metrics: + - type: main_score + value: 55.569 + - type: map_at_1 + value: 29.445 + - type: map_at_10 + value: 46.211999999999996 + - type: map_at_100 + value: 47.047 + - type: map_at_1000 + value: 47.052 + - type: map_at_20 + value: 46.93 + - type: map_at_3 + value: 41.027 + - type: map_at_5 + value: 44.01 + - type: mrr_at_1 + value: 30.08534850640114 + - type: mrr_at_10 + value: 46.46554449186029 + - type: mrr_at_100 + value: 47.293414507793265 + - type: mrr_at_1000 + value: 47.2986099865725 + - type: mrr_at_20 + value: 47.17689408794596 + - type: mrr_at_3 + value: 41.216216216216225 + - type: mrr_at_5 + value: 44.256756756756786 + - type: nauc_map_at_1000_diff1 + value: 8.7128370975647 + - type: nauc_map_at_1000_max + value: -12.46770480658987 + - type: nauc_map_at_1000_std + value: -14.629266044478303 + - type: nauc_map_at_100_diff1 + value: 8.712642875182372 + - type: nauc_map_at_100_max + value: -12.459767496544163 + - type: nauc_map_at_100_std + value: -14.61980473618478 + - type: nauc_map_at_10_diff1 + value: 8.48750597132097 + - type: nauc_map_at_10_max + value: -12.505824795851028 + - type: nauc_map_at_10_std + value: -14.554407222062911 + - type: nauc_map_at_1_diff1 + value: 12.491538775069934 + - type: nauc_map_at_1_max + value: -12.788126014384831 + - type: nauc_map_at_1_std + value: -14.10434631751783 + - type: nauc_map_at_20_diff1 + value: 8.676755185149517 + - type: nauc_map_at_20_max + value: -12.447567803981954 + - type: nauc_map_at_20_std + value: -14.722657335646158 + - type: nauc_map_at_3_diff1 + value: 8.83471581307392 + - type: nauc_map_at_3_max + value: -12.727502534904946 + - type: nauc_map_at_3_std + value: -15.123340129701315 + - type: nauc_map_at_5_diff1 + value: 8.39937988275747 + - type: nauc_map_at_5_max + value: -12.719070301967573 + - type: nauc_map_at_5_std + value: -14.63203354573712 + - type: nauc_mrr_at_1000_diff1 + value: 6.7787914734885835 + - type: nauc_mrr_at_1000_max + value: -13.36509565575714 + - type: nauc_mrr_at_1000_std + value: -14.586410316419027 + - type: nauc_mrr_at_100_diff1 + value: 6.778930090485158 + - type: nauc_mrr_at_100_max + value: -13.35705111818527 + - type: nauc_mrr_at_100_std + value: -14.577011999491866 + - type: nauc_mrr_at_10_diff1 + value: 6.572273873632861 + - type: nauc_mrr_at_10_max + value: -13.415556078809503 + - type: nauc_mrr_at_10_std + value: -14.517085337882651 + - type: nauc_mrr_at_1_diff1 + value: 10.519718146866714 + - type: nauc_mrr_at_1_max + value: -13.027199868632824 + - type: nauc_mrr_at_1_std + value: -13.776773223152732 + - type: nauc_mrr_at_20_diff1 + value: 6.752163320009284 + - type: nauc_mrr_at_20_max + value: -13.340280011349392 + - type: nauc_mrr_at_20_std + value: -14.679100601387947 + - type: nauc_mrr_at_3_diff1 + value: 6.805687007057483 + - type: nauc_mrr_at_3_max + value: -13.872319982698727 + - type: nauc_mrr_at_3_std + value: -15.155184992010936 + - type: nauc_mrr_at_5_diff1 + value: 6.552358739263904 + - type: nauc_mrr_at_5_max + value: -13.595785623097994 + - type: nauc_mrr_at_5_std + value: -14.643649476433257 + - type: nauc_ndcg_at_1000_diff1 + value: 8.301588171370765 + - type: nauc_ndcg_at_1000_max + value: -12.15980131357072 + - type: nauc_ndcg_at_1000_std + value: -14.434989241447779 + - type: nauc_ndcg_at_100_diff1 + value: 8.288557571138536 + - type: nauc_ndcg_at_100_max + value: -11.951112405400583 + - type: nauc_ndcg_at_100_std + value: -14.180909956165099 + - type: nauc_ndcg_at_10_diff1 + value: 7.361969224591534 + - type: nauc_ndcg_at_10_max + value: -12.041097921622976 + - type: nauc_ndcg_at_10_std + value: -14.51178165670093 + - type: nauc_ndcg_at_1_diff1 + value: 12.491538775069934 + - type: nauc_ndcg_at_1_max + value: -12.788126014384831 + - type: nauc_ndcg_at_1_std + value: -14.10434631751783 + - type: nauc_ndcg_at_20_diff1 + value: 8.015270611313673 + - type: nauc_ndcg_at_20_max + value: -11.772813060149886 + - type: nauc_ndcg_at_20_std + value: -15.008203598323092 + - type: nauc_ndcg_at_3_diff1 + value: 8.001389776163744 + - type: nauc_ndcg_at_3_max + value: -12.6462403118482 + - type: nauc_ndcg_at_3_std + value: -15.680792381483922 + - type: nauc_ndcg_at_5_diff1 + value: 7.130955353457466 + - type: nauc_ndcg_at_5_max + value: -12.638655320672548 + - type: nauc_ndcg_at_5_std + value: -14.723950364557798 + - type: nauc_precision_at_1000_diff1 + value: 7.322384873180169 + - type: nauc_precision_at_1000_max + value: 10.593718776325648 + - type: nauc_precision_at_1000_std + value: 78.10201038518204 + - type: nauc_precision_at_100_diff1 + value: 5.529960966538224 + - type: nauc_precision_at_100_max + value: 27.475488379257477 + - type: nauc_precision_at_100_std + value: 54.31978467071259 + - type: nauc_precision_at_10_diff1 + value: 0.6850448190432343 + - type: nauc_precision_at_10_max + value: -8.483074632198973 + - type: nauc_precision_at_10_std + value: -13.746210635105511 + - type: nauc_precision_at_1_diff1 + value: 12.491538775069934 + - type: nauc_precision_at_1_max + value: -12.788126014384831 + - type: nauc_precision_at_1_std + value: -14.10434631751783 + - type: nauc_precision_at_20_diff1 + value: 0.6323766426516709 + - type: nauc_precision_at_20_max + value: 2.9752824486321234 + - type: nauc_precision_at_20_std + value: -19.69925172268677 + - type: nauc_precision_at_3_diff1 + value: 5.624921600811906 + - type: nauc_precision_at_3_max + value: -12.38381501403816 + - type: nauc_precision_at_3_std + value: -17.418484643959566 + - type: nauc_precision_at_5_diff1 + value: 2.602397819545678 + - type: nauc_precision_at_5_max + value: -12.338196378977745 + - type: nauc_precision_at_5_std + value: -14.902871091163888 + - type: nauc_recall_at_1000_diff1 + value: 7.322384873173298 + - type: nauc_recall_at_1000_max + value: 10.593718776322545 + - type: nauc_recall_at_1000_std + value: 78.10201038517896 + - type: nauc_recall_at_100_diff1 + value: 5.529960966537435 + - type: nauc_recall_at_100_max + value: 27.47548837926007 + - type: nauc_recall_at_100_std + value: 54.319784670712544 + - type: nauc_recall_at_10_diff1 + value: 0.6850448190432565 + - type: nauc_recall_at_10_max + value: -8.483074632198887 + - type: nauc_recall_at_10_std + value: -13.746210635105417 + - type: nauc_recall_at_1_diff1 + value: 12.491538775069934 + - type: nauc_recall_at_1_max + value: -12.788126014384831 + - type: nauc_recall_at_1_std + value: -14.10434631751783 + - type: nauc_recall_at_20_diff1 + value: 0.6323766426518039 + - type: nauc_recall_at_20_max + value: 2.9752824486321896 + - type: nauc_recall_at_20_std + value: -19.699251722686707 + - type: nauc_recall_at_3_diff1 + value: 5.6249216008119625 + - type: nauc_recall_at_3_max + value: -12.383815014038143 + - type: nauc_recall_at_3_std + value: -17.41848464395952 + - type: nauc_recall_at_5_diff1 + value: 2.6023978195456627 + - type: nauc_recall_at_5_max + value: -12.338196378977738 + - type: nauc_recall_at_5_std + value: -14.902871091163878 + - type: ndcg_at_1 + value: 29.445 + - type: ndcg_at_10 + value: 55.569 + - type: ndcg_at_100 + value: 58.797999999999995 + - type: ndcg_at_1000 + value: 58.904999999999994 + - type: ndcg_at_20 + value: 58.06099999999999 + - type: ndcg_at_3 + value: 44.861000000000004 + - type: ndcg_at_5 + value: 50.275000000000006 + - type: precision_at_1 + value: 29.445 + - type: precision_at_10 + value: 8.549 + - type: precision_at_100 + value: 0.989 + - type: precision_at_1000 + value: 0.1 + - type: precision_at_20 + value: 4.755 + - type: precision_at_3 + value: 18.658 + - type: precision_at_5 + value: 13.841000000000001 + - type: recall_at_1 + value: 29.445 + - type: recall_at_10 + value: 85.491 + - type: recall_at_100 + value: 98.86200000000001 + - type: recall_at_1000 + value: 99.644 + - type: recall_at_20 + value: 95.092 + - type: recall_at_3 + value: 55.974000000000004 + - type: recall_at_5 + value: 69.203 + task: + type: Retrieval + - dataset: + config: default + name: MTEB ArxivClusteringP2P (default) + revision: a122ad7f3f0291bf49cc6f4d32aa80929df69d5d + split: test + type: mteb/arxiv-clustering-p2p + metrics: + - type: main_score + value: 45.458244333942666 + - type: v_measure + value: 45.458244333942666 + - type: v_measure_std + value: 13.637121793311572 + task: + type: Clustering + - dataset: + config: default + name: MTEB ArxivClusteringS2S (default) + revision: f910caf1a6075f7329cdf8c1a6135696f37dbd53 + split: test + type: mteb/arxiv-clustering-s2s + metrics: + - type: main_score + value: 39.06962877368677 + - type: v_measure + value: 39.06962877368677 + - type: v_measure_std + value: 14.42789184527441 + task: + type: Clustering + - dataset: + config: default + name: MTEB AskUbuntuDupQuestions (default) + revision: 2000358ca161889fa9c082cb41daa8dcfb161a54 + split: test + type: mteb/askubuntudupquestions-reranking + metrics: + - type: main_score + value: 59.939354668355364 + - type: map + value: 59.939354668355364 + - type: mrr + value: 73.68860748362134 + - type: nAUC_map_diff1 + value: 3.7964968084590196 + - type: nAUC_map_max + value: 20.116557832806915 + - type: nAUC_map_std + value: 18.678435533703443 + - type: nAUC_mrr_diff1 + value: 4.638409460404129 + - type: nAUC_mrr_max + value: 29.777302380570898 + - type: nAUC_mrr_std + value: 18.530964158566512 + task: + type: Reranking + - dataset: + config: default + name: MTEB BIOSSES (default) + revision: d3fb88f8f02e40887cd149695127462bbcf29b4a + split: test + type: mteb/biosses-sts + metrics: + - type: cosine_pearson + value: 85.81569608480319 + - type: cosine_spearman + value: 83.8476936591105 + - type: euclidean_pearson + value: 84.28194643004147 + - type: euclidean_spearman + value: 83.8476936591105 + - type: main_score + value: 83.8476936591105 + - type: manhattan_pearson + value: 84.42049554832897 + - type: manhattan_spearman + value: 84.15673684544747 + - type: pearson + value: 85.81569366450125 + - type: spearman + value: 83.8476936591105 + task: + type: STS + - dataset: + config: default + name: MTEB Banking77Classification (default) + revision: 0fd18e25b25c072e09e0d92ab615fda904d66300 + split: test + type: mteb/banking77 + metrics: + - type: accuracy + value: 82.20779220779221 + - type: f1 + value: 81.70253746810248 + - type: f1_weighted + value: 81.70253746810249 + - type: main_score + value: 82.20779220779221 + task: + type: Classification + - dataset: + config: default + name: MTEB BiorxivClusteringP2P (default) + revision: 65b79d1d13f80053f67aca9498d9402c2d9f1f40 + split: test + type: mteb/biorxiv-clustering-p2p + metrics: + - type: main_score + value: 35.405443435574846 + - type: v_measure + value: 35.405443435574846 + - type: v_measure_std + value: 1.198244389873123 + task: + type: Clustering + - dataset: + config: default + name: MTEB BiorxivClusteringS2S (default) + revision: 258694dd0231531bc1fd9de6ceb52a0853c6d908 + split: test + type: mteb/biorxiv-clustering-s2s + metrics: + - type: main_score + value: 34.04342789697788 + - type: v_measure + value: 34.04342789697788 + - type: v_measure_std + value: 0.4502432972457361 + task: + type: Clustering + - dataset: + config: default + name: MTEB CQADupstackAndroidRetrieval (default) + revision: f46a197baaae43b4f621051089b82a364682dfeb + split: test + type: mteb/cqadupstack-android + metrics: + - type: main_score + value: 51.205 + - type: map_at_1 + value: 34.44 + - type: map_at_10 + value: 44.99 + - type: map_at_100 + value: 46.487 + - type: map_at_1000 + value: 46.614 + - type: map_at_20 + value: 45.831 + - type: map_at_3 + value: 41.203 + - type: map_at_5 + value: 43.442 + - type: mrr_at_1 + value: 42.34620886981402 + - type: mrr_at_10 + value: 51.04906101687218 + - type: mrr_at_100 + value: 51.78713205902966 + - type: mrr_at_1000 + value: 51.828740696302454 + - type: mrr_at_20 + value: 51.51487685161397 + - type: mrr_at_3 + value: 48.28326180257512 + - type: mrr_at_5 + value: 49.94992846924176 + - type: nauc_map_at_1000_diff1 + value: 46.53937728309425 + - type: nauc_map_at_1000_max + value: 34.99839845756243 + - type: nauc_map_at_1000_std + value: -4.110263778571436 + - type: nauc_map_at_100_diff1 + value: 46.541902950226344 + - type: nauc_map_at_100_max + value: 34.99909733941229 + - type: nauc_map_at_100_std + value: -4.077306314476497 + - type: nauc_map_at_10_diff1 + value: 46.98936643123771 + - type: nauc_map_at_10_max + value: 34.618337137103666 + - type: nauc_map_at_10_std + value: -4.919400110557668 + - type: nauc_map_at_1_diff1 + value: 50.30532672215466 + - type: nauc_map_at_1_max + value: 32.16910117282255 + - type: nauc_map_at_1_std + value: -6.754777617016533 + - type: nauc_map_at_20_diff1 + value: 46.52827563449433 + - type: nauc_map_at_20_max + value: 34.91949942013315 + - type: nauc_map_at_20_std + value: -4.350282054451956 + - type: nauc_map_at_3_diff1 + value: 47.034843887333786 + - type: nauc_map_at_3_max + value: 34.43868744170203 + - type: nauc_map_at_3_std + value: -7.472070847814205 + - type: nauc_map_at_5_diff1 + value: 47.12947907340844 + - type: nauc_map_at_5_max + value: 34.283543727801714 + - type: nauc_map_at_5_std + value: -6.138722259119489 + - type: nauc_mrr_at_1000_diff1 + value: 44.962025109369144 + - type: nauc_mrr_at_1000_max + value: 36.36401678834248 + - type: nauc_mrr_at_1000_std + value: -3.773093703154531 + - type: nauc_mrr_at_100_diff1 + value: 44.93719384489801 + - type: nauc_mrr_at_100_max + value: 36.34719994304288 + - type: nauc_mrr_at_100_std + value: -3.776338522884868 + - type: nauc_mrr_at_10_diff1 + value: 45.15813198285134 + - type: nauc_mrr_at_10_max + value: 36.37556686117443 + - type: nauc_mrr_at_10_std + value: -3.7840437249777015 + - type: nauc_mrr_at_1_diff1 + value: 47.660939188810424 + - type: nauc_mrr_at_1_max + value: 37.468075856884326 + - type: nauc_mrr_at_1_std + value: -4.45481275697827 + - type: nauc_mrr_at_20_diff1 + value: 44.88913673158708 + - type: nauc_mrr_at_20_max + value: 36.380649515021105 + - type: nauc_mrr_at_20_std + value: -3.8403703741993396 + - type: nauc_mrr_at_3_diff1 + value: 45.1391690111807 + - type: nauc_mrr_at_3_max + value: 36.99127987948575 + - type: nauc_mrr_at_3_std + value: -5.098986866578028 + - type: nauc_mrr_at_5_diff1 + value: 45.42341148882821 + - type: nauc_mrr_at_5_max + value: 36.345011374271124 + - type: nauc_mrr_at_5_std + value: -4.320819772748555 + - type: nauc_ndcg_at_1000_diff1 + value: 44.983283560600036 + - type: nauc_ndcg_at_1000_max + value: 35.17294675661479 + - type: nauc_ndcg_at_1000_std + value: -2.1451898313423614 + - type: nauc_ndcg_at_100_diff1 + value: 44.4289234782609 + - type: nauc_ndcg_at_100_max + value: 34.83741338968021 + - type: nauc_ndcg_at_100_std + value: -1.7278311845500716 + - type: nauc_ndcg_at_10_diff1 + value: 45.68521147665236 + - type: nauc_ndcg_at_10_max + value: 34.81228159026973 + - type: nauc_ndcg_at_10_std + value: -3.1093850237377163 + - type: nauc_ndcg_at_1_diff1 + value: 47.660939188810424 + - type: nauc_ndcg_at_1_max + value: 37.468075856884326 + - type: nauc_ndcg_at_1_std + value: -4.45481275697827 + - type: nauc_ndcg_at_20_diff1 + value: 44.33222970478049 + - type: nauc_ndcg_at_20_max + value: 34.815215799960946 + - type: nauc_ndcg_at_20_std + value: -2.6596461346300964 + - type: nauc_ndcg_at_3_diff1 + value: 45.60366209400674 + - type: nauc_ndcg_at_3_max + value: 35.74162925221817 + - type: nauc_ndcg_at_3_std + value: -6.220596871893804 + - type: nauc_ndcg_at_5_diff1 + value: 46.142209074547466 + - type: nauc_ndcg_at_5_max + value: 34.854430713792524 + - type: nauc_ndcg_at_5_std + value: -4.6161075460257655 + - type: nauc_precision_at_1000_diff1 + value: -18.854796090415267 + - type: nauc_precision_at_1000_max + value: -5.700494386093807 + - type: nauc_precision_at_1000_std + value: -0.1781599506360375 + - type: nauc_precision_at_100_diff1 + value: -10.555917979055366 + - type: nauc_precision_at_100_max + value: 4.669858623808923 + - type: nauc_precision_at_100_std + value: 10.971620720126348 + - type: nauc_precision_at_10_diff1 + value: 10.102757895318067 + - type: nauc_precision_at_10_max + value: 20.39903387813051 + - type: nauc_precision_at_10_std + value: 11.063886711650785 + - type: nauc_precision_at_1_diff1 + value: 47.660939188810424 + - type: nauc_precision_at_1_max + value: 37.468075856884326 + - type: nauc_precision_at_1_std + value: -4.45481275697827 + - type: nauc_precision_at_20_diff1 + value: -0.07434334179234021 + - type: nauc_precision_at_20_max + value: 16.176429406043127 + - type: nauc_precision_at_20_std + value: 14.316043361670374 + - type: nauc_precision_at_3_diff1 + value: 26.79563825209919 + - type: nauc_precision_at_3_max + value: 32.010127955554466 + - type: nauc_precision_at_3_std + value: -2.0713046049263517 + - type: nauc_precision_at_5_diff1 + value: 20.366296886151794 + - type: nauc_precision_at_5_max + value: 27.17197239816728 + - type: nauc_precision_at_5_std + value: 5.24046844744102 + - type: nauc_recall_at_1000_diff1 + value: 37.75513261767943 + - type: nauc_recall_at_1000_max + value: 38.925757818551794 + - type: nauc_recall_at_1000_std + value: 39.99723054997409 + - type: nauc_recall_at_100_diff1 + value: 29.42253289101055 + - type: nauc_recall_at_100_max + value: 27.867712530671596 + - type: nauc_recall_at_100_std + value: 13.823874126586459 + - type: nauc_recall_at_10_diff1 + value: 41.13903334713538 + - type: nauc_recall_at_10_max + value: 30.016792424985677 + - type: nauc_recall_at_10_std + value: -0.11625356835654711 + - type: nauc_recall_at_1_diff1 + value: 50.30532672215466 + - type: nauc_recall_at_1_max + value: 32.16910117282255 + - type: nauc_recall_at_1_std + value: -6.754777617016533 + - type: nauc_recall_at_20_diff1 + value: 33.7678490196236 + - type: nauc_recall_at_20_max + value: 29.196821111661002 + - type: nauc_recall_at_20_std + value: 1.8621138765437453 + - type: nauc_recall_at_3_diff1 + value: 43.049429690088786 + - type: nauc_recall_at_3_max + value: 32.65187505922956 + - type: nauc_recall_at_3_std + value: -8.881660133204786 + - type: nauc_recall_at_5_diff1 + value: 43.23693722668031 + - type: nauc_recall_at_5_max + value: 29.946679253231217 + - type: nauc_recall_at_5_std + value: -5.620321560342215 + - type: ndcg_at_1 + value: 42.346000000000004 + - type: ndcg_at_10 + value: 51.205 + - type: ndcg_at_100 + value: 56.53 + - type: ndcg_at_1000 + value: 58.357000000000006 + - type: ndcg_at_20 + value: 53.315999999999995 + - type: ndcg_at_3 + value: 45.809 + - type: ndcg_at_5 + value: 48.526 + - type: precision_at_1 + value: 42.346000000000004 + - type: precision_at_10 + value: 9.714 + - type: precision_at_100 + value: 1.524 + - type: precision_at_1000 + value: 0.201 + - type: precision_at_20 + value: 5.7299999999999995 + - type: precision_at_3 + value: 21.745 + - type: precision_at_5 + value: 15.794 + - type: recall_at_1 + value: 34.44 + - type: recall_at_10 + value: 62.726000000000006 + - type: recall_at_100 + value: 84.956 + - type: recall_at_1000 + value: 96.438 + - type: recall_at_20 + value: 70.258 + - type: recall_at_3 + value: 47.292 + - type: recall_at_5 + value: 54.919 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackEnglishRetrieval (default) + revision: ad9991cb51e31e31e430383c75ffb2885547b5f0 + split: test + type: mteb/cqadupstack-english + metrics: + - type: main_score + value: 46.726 + - type: map_at_1 + value: 30.044999999999998 + - type: map_at_10 + value: 40.909 + - type: map_at_100 + value: 42.163000000000004 + - type: map_at_1000 + value: 42.289 + - type: map_at_20 + value: 41.565000000000005 + - type: map_at_3 + value: 37.973 + - type: map_at_5 + value: 39.667 + - type: mrr_at_1 + value: 38.28025477707007 + - type: mrr_at_10 + value: 47.13358103326261 + - type: mrr_at_100 + value: 47.78086527818363 + - type: mrr_at_1000 + value: 47.82476596537942 + - type: mrr_at_20 + value: 47.49249714568881 + - type: mrr_at_3 + value: 44.915074309978785 + - type: mrr_at_5 + value: 46.30360934182596 + - type: nauc_map_at_1000_diff1 + value: 54.43164390623409 + - type: nauc_map_at_1000_max + value: 41.19895131862085 + - type: nauc_map_at_1000_std + value: -2.053223004822684 + - type: nauc_map_at_100_diff1 + value: 54.406556910559814 + - type: nauc_map_at_100_max + value: 41.086150694127795 + - type: nauc_map_at_100_std + value: -2.1928486753067786 + - type: nauc_map_at_10_diff1 + value: 54.18519298950384 + - type: nauc_map_at_10_max + value: 40.31102567726707 + - type: nauc_map_at_10_std + value: -3.5029971232184693 + - type: nauc_map_at_1_diff1 + value: 61.173122285834836 + - type: nauc_map_at_1_max + value: 36.97827772925505 + - type: nauc_map_at_1_std + value: -6.378456132357965 + - type: nauc_map_at_20_diff1 + value: 54.231210502223625 + - type: nauc_map_at_20_max + value: 40.62415342559933 + - type: nauc_map_at_20_std + value: -2.7289755085928094 + - type: nauc_map_at_3_diff1 + value: 55.525773055963036 + - type: nauc_map_at_3_max + value: 39.514721438127125 + - type: nauc_map_at_3_std + value: -5.671000326858591 + - type: nauc_map_at_5_diff1 + value: 54.509443684217885 + - type: nauc_map_at_5_max + value: 39.8197417990347 + - type: nauc_map_at_5_std + value: -4.765831031074912 + - type: nauc_mrr_at_1000_diff1 + value: 54.03748329782883 + - type: nauc_mrr_at_1000_max + value: 43.78615723591744 + - type: nauc_mrr_at_1000_std + value: 1.8340224956221913 + - type: nauc_mrr_at_100_diff1 + value: 54.03301996137554 + - type: nauc_mrr_at_100_max + value: 43.78068702686736 + - type: nauc_mrr_at_100_std + value: 1.8304848397213038 + - type: nauc_mrr_at_10_diff1 + value: 53.88032032701196 + - type: nauc_mrr_at_10_max + value: 43.83486118237796 + - type: nauc_mrr_at_10_std + value: 1.7710094244916572 + - type: nauc_mrr_at_1_diff1 + value: 58.20304265554071 + - type: nauc_mrr_at_1_max + value: 44.394531718233196 + - type: nauc_mrr_at_1_std + value: 0.3055386317060775 + - type: nauc_mrr_at_20_diff1 + value: 53.9256203276955 + - type: nauc_mrr_at_20_max + value: 43.780993542756725 + - type: nauc_mrr_at_20_std + value: 1.8057106744117806 + - type: nauc_mrr_at_3_diff1 + value: 54.86623753060233 + - type: nauc_mrr_at_3_max + value: 43.967093712431875 + - type: nauc_mrr_at_3_std + value: 0.9057430569539976 + - type: nauc_mrr_at_5_diff1 + value: 54.14989136427999 + - type: nauc_mrr_at_5_max + value: 43.91309348235134 + - type: nauc_mrr_at_5_std + value: 1.4243656479276336 + - type: nauc_ndcg_at_1000_diff1 + value: 52.717678547608834 + - type: nauc_ndcg_at_1000_max + value: 42.17524810534243 + - type: nauc_ndcg_at_1000_std + value: 2.3460781781821916 + - type: nauc_ndcg_at_100_diff1 + value: 52.69740834722126 + - type: nauc_ndcg_at_100_max + value: 41.76014251868014 + - type: nauc_ndcg_at_100_std + value: 1.6421909106485757 + - type: nauc_ndcg_at_10_diff1 + value: 52.126822949276466 + - type: nauc_ndcg_at_10_max + value: 41.86994133270259 + - type: nauc_ndcg_at_10_std + value: -0.067923935333631 + - type: nauc_ndcg_at_1_diff1 + value: 58.20304265554071 + - type: nauc_ndcg_at_1_max + value: 44.394531718233196 + - type: nauc_ndcg_at_1_std + value: 0.3055386317060775 + - type: nauc_ndcg_at_20_diff1 + value: 52.28882242246128 + - type: nauc_ndcg_at_20_max + value: 41.73007556127451 + - type: nauc_ndcg_at_20_std + value: 0.8706246193291374 + - type: nauc_ndcg_at_3_diff1 + value: 53.42940123639542 + - type: nauc_ndcg_at_3_max + value: 42.44856261579197 + - type: nauc_ndcg_at_3_std + value: -1.4866775153475087 + - type: nauc_ndcg_at_5_diff1 + value: 52.454498981350014 + - type: nauc_ndcg_at_5_max + value: 42.018984350887365 + - type: nauc_ndcg_at_5_std + value: -1.3677296618204733 + - type: nauc_precision_at_1000_diff1 + value: -4.6319666175507965 + - type: nauc_precision_at_1000_max + value: 22.938016171475528 + - type: nauc_precision_at_1000_std + value: 34.20667762370421 + - type: nauc_precision_at_100_diff1 + value: 2.8528979773686998 + - type: nauc_precision_at_100_max + value: 29.624564251798898 + - type: nauc_precision_at_100_std + value: 34.07783836555192 + - type: nauc_precision_at_10_diff1 + value: 15.059548074604928 + - type: nauc_precision_at_10_max + value: 35.92390470554715 + - type: nauc_precision_at_10_std + value: 21.981633681811196 + - type: nauc_precision_at_1_diff1 + value: 58.20304265554071 + - type: nauc_precision_at_1_max + value: 44.394531718233196 + - type: nauc_precision_at_1_std + value: 0.3055386317060775 + - type: nauc_precision_at_20_diff1 + value: 9.420231868544562 + - type: nauc_precision_at_20_max + value: 33.56533338593658 + - type: nauc_precision_at_20_std + value: 28.77837828985691 + - type: nauc_precision_at_3_diff1 + value: 32.38429413111333 + - type: nauc_precision_at_3_max + value: 41.282092866807645 + - type: nauc_precision_at_3_std + value: 8.674785842633357 + - type: nauc_precision_at_5_diff1 + value: 23.035073948843873 + - type: nauc_precision_at_5_max + value: 38.79879296453544 + - type: nauc_precision_at_5_std + value: 14.18967952470774 + - type: nauc_recall_at_1000_diff1 + value: 37.90199128162664 + - type: nauc_recall_at_1000_max + value: 32.97607648792403 + - type: nauc_recall_at_1000_std + value: 23.0902823474613 + - type: nauc_recall_at_100_diff1 + value: 42.21849957817069 + - type: nauc_recall_at_100_max + value: 34.460564169598115 + - type: nauc_recall_at_100_std + value: 10.723109458658877 + - type: nauc_recall_at_10_diff1 + value: 42.7683175943683 + - type: nauc_recall_at_10_max + value: 35.843493661616485 + - type: nauc_recall_at_10_std + value: 0.5289956183241737 + - type: nauc_recall_at_1_diff1 + value: 61.173122285834836 + - type: nauc_recall_at_1_max + value: 36.97827772925505 + - type: nauc_recall_at_1_std + value: -6.378456132357965 + - type: nauc_recall_at_20_diff1 + value: 42.55881596998839 + - type: nauc_recall_at_20_max + value: 35.81818350989878 + - type: nauc_recall_at_20_std + value: 4.367410065726315 + - type: nauc_recall_at_3_diff1 + value: 49.35290076445215 + - type: nauc_recall_at_3_max + value: 36.966759981220655 + - type: nauc_recall_at_3_std + value: -5.707497221028364 + - type: nauc_recall_at_5_diff1 + value: 45.36659239330872 + - type: nauc_recall_at_5_max + value: 36.10276428936472 + - type: nauc_recall_at_5_std + value: -4.057651737263702 + - type: ndcg_at_1 + value: 38.279999999999994 + - type: ndcg_at_10 + value: 46.726 + - type: ndcg_at_100 + value: 51.158 + - type: ndcg_at_1000 + value: 53.108999999999995 + - type: ndcg_at_20 + value: 48.317 + - type: ndcg_at_3 + value: 42.824 + - type: ndcg_at_5 + value: 44.714 + - type: precision_at_1 + value: 38.279999999999994 + - type: precision_at_10 + value: 8.904 + - type: precision_at_100 + value: 1.432 + - type: precision_at_1000 + value: 0.189 + - type: precision_at_20 + value: 5.194 + - type: precision_at_3 + value: 20.913 + - type: precision_at_5 + value: 14.764 + - type: recall_at_1 + value: 30.044999999999998 + - type: recall_at_10 + value: 56.623000000000005 + - type: recall_at_100 + value: 75.643 + - type: recall_at_1000 + value: 87.993 + - type: recall_at_20 + value: 62.499 + - type: recall_at_3 + value: 44.713 + - type: recall_at_5 + value: 50.43299999999999 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackGamingRetrieval (default) + revision: 4885aa143210c98657558c04aaf3dc47cfb54340 + split: test + type: mteb/cqadupstack-gaming + metrics: + - type: main_score + value: 58.160999999999994 + - type: map_at_1 + value: 39.654 + - type: map_at_10 + value: 52.38699999999999 + - type: map_at_100 + value: 53.520999999999994 + - type: map_at_1000 + value: 53.574999999999996 + - type: map_at_20 + value: 53.116 + - type: map_at_3 + value: 49.288 + - type: map_at_5 + value: 51.202999999999996 + - type: mrr_at_1 + value: 45.26645768025078 + - type: mrr_at_10 + value: 55.60038811762959 + - type: mrr_at_100 + value: 56.325267304495185 + - type: mrr_at_1000 + value: 56.35295254324715 + - type: mrr_at_20 + value: 56.08029095378797 + - type: mrr_at_3 + value: 53.166144200627016 + - type: mrr_at_5 + value: 54.736677115987575 + - type: nauc_map_at_1000_diff1 + value: 53.73132151615392 + - type: nauc_map_at_1000_max + value: 41.68388882732696 + - type: nauc_map_at_1000_std + value: -10.370074572724949 + - type: nauc_map_at_100_diff1 + value: 53.711248842923496 + - type: nauc_map_at_100_max + value: 41.661337108540216 + - type: nauc_map_at_100_std + value: -10.371131156674043 + - type: nauc_map_at_10_diff1 + value: 53.83236039197653 + - type: nauc_map_at_10_max + value: 41.13561200383782 + - type: nauc_map_at_10_std + value: -11.457482223865018 + - type: nauc_map_at_1_diff1 + value: 56.199828454245846 + - type: nauc_map_at_1_max + value: 36.43341946718662 + - type: nauc_map_at_1_std + value: -10.787349686912423 + - type: nauc_map_at_20_diff1 + value: 53.79070452533048 + - type: nauc_map_at_20_max + value: 41.45918373076048 + - type: nauc_map_at_20_std + value: -10.734554449537557 + - type: nauc_map_at_3_diff1 + value: 53.60248931734135 + - type: nauc_map_at_3_max + value: 39.78202775531023 + - type: nauc_map_at_3_std + value: -13.051581264439685 + - type: nauc_map_at_5_diff1 + value: 53.84761613784422 + - type: nauc_map_at_5_max + value: 40.76979989234891 + - type: nauc_map_at_5_std + value: -11.680052510192473 + - type: nauc_mrr_at_1000_diff1 + value: 53.52159081330875 + - type: nauc_mrr_at_1000_max + value: 42.86157132225623 + - type: nauc_mrr_at_1000_std + value: -8.273468342242158 + - type: nauc_mrr_at_100_diff1 + value: 53.509470185069475 + - type: nauc_mrr_at_100_max + value: 42.87159408747719 + - type: nauc_mrr_at_100_std + value: -8.253367179864671 + - type: nauc_mrr_at_10_diff1 + value: 53.44305686059674 + - type: nauc_mrr_at_10_max + value: 42.74115712982764 + - type: nauc_mrr_at_10_std + value: -8.67771769345137 + - type: nauc_mrr_at_1_diff1 + value: 57.13432720058654 + - type: nauc_mrr_at_1_max + value: 42.30829459320884 + - type: nauc_mrr_at_1_std + value: -8.12499906797024 + - type: nauc_mrr_at_20_diff1 + value: 53.569030033097654 + - type: nauc_mrr_at_20_max + value: 42.86689133151307 + - type: nauc_mrr_at_20_std + value: -8.31447720397405 + - type: nauc_mrr_at_3_diff1 + value: 53.4185569021448 + - type: nauc_mrr_at_3_max + value: 42.20069062003412 + - type: nauc_mrr_at_3_std + value: -9.421116662858031 + - type: nauc_mrr_at_5_diff1 + value: 53.33476556579716 + - type: nauc_mrr_at_5_max + value: 42.65406340705883 + - type: nauc_mrr_at_5_std + value: -8.636415569317174 + - type: nauc_ndcg_at_1000_diff1 + value: 52.58114709663067 + - type: nauc_ndcg_at_1000_max + value: 43.14410233559511 + - type: nauc_ndcg_at_1000_std + value: -7.493813961826964 + - type: nauc_ndcg_at_100_diff1 + value: 52.10736464202627 + - type: nauc_ndcg_at_100_max + value: 43.14795475648977 + - type: nauc_ndcg_at_100_std + value: -7.168346956558692 + - type: nauc_ndcg_at_10_diff1 + value: 52.627807895220066 + - type: nauc_ndcg_at_10_max + value: 42.23625830664404 + - type: nauc_ndcg_at_10_std + value: -10.346313697058182 + - type: nauc_ndcg_at_1_diff1 + value: 57.13432720058654 + - type: nauc_ndcg_at_1_max + value: 42.30829459320884 + - type: nauc_ndcg_at_1_std + value: -8.12499906797024 + - type: nauc_ndcg_at_20_diff1 + value: 52.62888487441878 + - type: nauc_ndcg_at_20_max + value: 42.725467823242 + - type: nauc_ndcg_at_20_std + value: -8.560269204268273 + - type: nauc_ndcg_at_3_diff1 + value: 52.288089741048026 + - type: nauc_ndcg_at_3_max + value: 40.63140490780548 + - type: nauc_ndcg_at_3_std + value: -12.252858978276397 + - type: nauc_ndcg_at_5_diff1 + value: 52.58472816383196 + - type: nauc_ndcg_at_5_max + value: 41.9349671091071 + - type: nauc_ndcg_at_5_std + value: -10.311510068367328 + - type: nauc_precision_at_1000_diff1 + value: -11.200512597389848 + - type: nauc_precision_at_1000_max + value: 16.598621971441354 + - type: nauc_precision_at_1000_std + value: 27.399188600898544 + - type: nauc_precision_at_100_diff1 + value: -5.925754267209147 + - type: nauc_precision_at_100_max + value: 20.905017426614837 + - type: nauc_precision_at_100_std + value: 26.33195793899955 + - type: nauc_precision_at_10_diff1 + value: 17.410308582985163 + - type: nauc_precision_at_10_max + value: 30.657181015507472 + - type: nauc_precision_at_10_std + value: 6.073241021647797 + - type: nauc_precision_at_1_diff1 + value: 57.13432720058654 + - type: nauc_precision_at_1_max + value: 42.30829459320884 + - type: nauc_precision_at_1_std + value: -8.12499906797024 + - type: nauc_precision_at_20_diff1 + value: 7.964510602623282 + - type: nauc_precision_at_20_max + value: 27.52808581836903 + - type: nauc_precision_at_20_std + value: 16.306139009874094 + - type: nauc_precision_at_3_diff1 + value: 34.2518693507898 + - type: nauc_precision_at_3_max + value: 37.035338699191364 + - type: nauc_precision_at_3_std + value: -6.37687725832473 + - type: nauc_precision_at_5_diff1 + value: 26.457255777710813 + - type: nauc_precision_at_5_max + value: 34.673691436277856 + - type: nauc_precision_at_5_std + value: 1.6309393310209783 + - type: nauc_recall_at_1000_diff1 + value: 29.162594500111506 + - type: nauc_recall_at_1000_max + value: 59.63020540632363 + - type: nauc_recall_at_1000_std + value: 59.20150387497538 + - type: nauc_recall_at_100_diff1 + value: 34.05939380734733 + - type: nauc_recall_at_100_max + value: 49.011219854541025 + - type: nauc_recall_at_100_std + value: 20.47168621047079 + - type: nauc_recall_at_10_diff1 + value: 46.94971437950854 + - type: nauc_recall_at_10_max + value: 41.32864314779422 + - type: nauc_recall_at_10_std + value: -10.991268678494666 + - type: nauc_recall_at_1_diff1 + value: 56.199828454245846 + - type: nauc_recall_at_1_max + value: 36.43341946718662 + - type: nauc_recall_at_1_std + value: -10.787349686912423 + - type: nauc_recall_at_20_diff1 + value: 45.62110511390014 + - type: nauc_recall_at_20_max + value: 44.20113226999224 + - type: nauc_recall_at_20_std + value: -0.6593176451099925 + - type: nauc_recall_at_3_diff1 + value: 47.78906181890755 + - type: nauc_recall_at_3_max + value: 37.80089529081245 + - type: nauc_recall_at_3_std + value: -15.632992108786617 + - type: nauc_recall_at_5_diff1 + value: 47.44592538133222 + - type: nauc_recall_at_5_max + value: 40.30564168613615 + - type: nauc_recall_at_5_std + value: -10.844596469396702 + - type: ndcg_at_1 + value: 45.266 + - type: ndcg_at_10 + value: 58.160999999999994 + - type: ndcg_at_100 + value: 62.437 + - type: ndcg_at_1000 + value: 63.478 + - type: ndcg_at_20 + value: 60.236000000000004 + - type: ndcg_at_3 + value: 53.04 + - type: ndcg_at_5 + value: 55.85099999999999 + - type: precision_at_1 + value: 45.266 + - type: precision_at_10 + value: 9.31 + - type: precision_at_100 + value: 1.237 + - type: precision_at_1000 + value: 0.13699999999999998 + - type: precision_at_20 + value: 5.301 + - type: precision_at_3 + value: 23.762 + - type: precision_at_5 + value: 16.414 + - type: recall_at_1 + value: 39.654 + - type: recall_at_10 + value: 71.83699999999999 + - type: recall_at_100 + value: 89.974 + - type: recall_at_1000 + value: 97.178 + - type: recall_at_20 + value: 79.434 + - type: recall_at_3 + value: 58.211 + - type: recall_at_5 + value: 65.078 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackGisRetrieval (default) + revision: 5003b3064772da1887988e05400cf3806fe491f2 + split: test + type: mteb/cqadupstack-gis + metrics: + - type: main_score + value: 34.245 + - type: map_at_1 + value: 20.861 + - type: map_at_10 + value: 29.414 + - type: map_at_100 + value: 30.380000000000003 + - type: map_at_1000 + value: 30.468 + - type: map_at_20 + value: 29.931 + - type: map_at_3 + value: 27.176000000000002 + - type: map_at_5 + value: 28.351 + - type: mrr_at_1 + value: 22.598870056497177 + - type: mrr_at_10 + value: 31.16828087167071 + - type: mrr_at_100 + value: 32.06304085091414 + - type: mrr_at_1000 + value: 32.13140870528097 + - type: mrr_at_20 + value: 31.664265740051544 + - type: mrr_at_3 + value: 28.945386064030153 + - type: mrr_at_5 + value: 30.120527306967993 + - type: nauc_map_at_1000_diff1 + value: 40.930750370941105 + - type: nauc_map_at_1000_max + value: 26.09095804812465 + - type: nauc_map_at_1000_std + value: -5.7047660598999395 + - type: nauc_map_at_100_diff1 + value: 40.89160486700697 + - type: nauc_map_at_100_max + value: 26.0499990388732 + - type: nauc_map_at_100_std + value: -5.730050623548015 + - type: nauc_map_at_10_diff1 + value: 41.05319455710241 + - type: nauc_map_at_10_max + value: 25.974602522413452 + - type: nauc_map_at_10_std + value: -6.076751935301229 + - type: nauc_map_at_1_diff1 + value: 49.33894051910933 + - type: nauc_map_at_1_max + value: 27.030728385670024 + - type: nauc_map_at_1_std + value: -7.092618383196039 + - type: nauc_map_at_20_diff1 + value: 40.99783434685901 + - type: nauc_map_at_20_max + value: 25.9978362578142 + - type: nauc_map_at_20_std + value: -6.012236161789563 + - type: nauc_map_at_3_diff1 + value: 42.48395281415212 + - type: nauc_map_at_3_max + value: 25.7735029217949 + - type: nauc_map_at_3_std + value: -7.211811190725744 + - type: nauc_map_at_5_diff1 + value: 41.49478026582303 + - type: nauc_map_at_5_max + value: 26.2970067341789 + - type: nauc_map_at_5_std + value: -7.265607759257126 + - type: nauc_mrr_at_1000_diff1 + value: 40.547276681410246 + - type: nauc_mrr_at_1000_max + value: 26.5325905811423 + - type: nauc_mrr_at_1000_std + value: -4.455467259587759 + - type: nauc_mrr_at_100_diff1 + value: 40.50648761082374 + - type: nauc_mrr_at_100_max + value: 26.496096169513763 + - type: nauc_mrr_at_100_std + value: -4.4697946027983715 + - type: nauc_mrr_at_10_diff1 + value: 40.65665403075134 + - type: nauc_mrr_at_10_max + value: 26.560974909409484 + - type: nauc_mrr_at_10_std + value: -4.677109315749782 + - type: nauc_mrr_at_1_diff1 + value: 47.92810759231813 + - type: nauc_mrr_at_1_max + value: 28.654386036120915 + - type: nauc_mrr_at_1_std + value: -5.606633895916787 + - type: nauc_mrr_at_20_diff1 + value: 40.616181033303256 + - type: nauc_mrr_at_20_max + value: 26.50360137290702 + - type: nauc_mrr_at_20_std + value: -4.666302107767106 + - type: nauc_mrr_at_3_diff1 + value: 41.7874916356228 + - type: nauc_mrr_at_3_max + value: 26.264741012995778 + - type: nauc_mrr_at_3_std + value: -5.937559902627668 + - type: nauc_mrr_at_5_diff1 + value: 41.07843358850583 + - type: nauc_mrr_at_5_max + value: 26.724559610543615 + - type: nauc_mrr_at_5_std + value: -5.8464111110535875 + - type: nauc_ndcg_at_1000_diff1 + value: 37.94097150743181 + - type: nauc_ndcg_at_1000_max + value: 26.897100222737414 + - type: nauc_ndcg_at_1000_std + value: -1.8302629212970287 + - type: nauc_ndcg_at_100_diff1 + value: 36.858184410983625 + - type: nauc_ndcg_at_100_max + value: 25.74680855995642 + - type: nauc_ndcg_at_100_std + value: -2.232029634184348 + - type: nauc_ndcg_at_10_diff1 + value: 37.767548109643975 + - type: nauc_ndcg_at_10_max + value: 25.61259693293087 + - type: nauc_ndcg_at_10_std + value: -4.055308005180942 + - type: nauc_ndcg_at_1_diff1 + value: 47.92810759231813 + - type: nauc_ndcg_at_1_max + value: 28.654386036120915 + - type: nauc_ndcg_at_1_std + value: -5.606633895916787 + - type: nauc_ndcg_at_20_diff1 + value: 37.44216468461217 + - type: nauc_ndcg_at_20_max + value: 25.45309768558073 + - type: nauc_ndcg_at_20_std + value: -3.8103316409295247 + - type: nauc_ndcg_at_3_diff1 + value: 40.51415926018581 + - type: nauc_ndcg_at_3_max + value: 25.439627154613444 + - type: nauc_ndcg_at_3_std + value: -6.871516601799714 + - type: nauc_ndcg_at_5_diff1 + value: 38.85862596448815 + - type: nauc_ndcg_at_5_max + value: 26.261748522151642 + - type: nauc_ndcg_at_5_std + value: -6.729315175832122 + - type: nauc_precision_at_1000_diff1 + value: 1.122174688906938 + - type: nauc_precision_at_1000_max + value: 17.996156016949627 + - type: nauc_precision_at_1000_std + value: 17.77509694105009 + - type: nauc_precision_at_100_diff1 + value: 11.301850032927534 + - type: nauc_precision_at_100_max + value: 21.415614397967712 + - type: nauc_precision_at_100_std + value: 12.582187769564031 + - type: nauc_precision_at_10_diff1 + value: 23.801938484677017 + - type: nauc_precision_at_10_max + value: 23.314553967017083 + - type: nauc_precision_at_10_std + value: 2.552927138610775 + - type: nauc_precision_at_1_diff1 + value: 47.92810759231813 + - type: nauc_precision_at_1_max + value: 28.654386036120915 + - type: nauc_precision_at_1_std + value: -5.606633895916787 + - type: nauc_precision_at_20_diff1 + value: 19.76766318985895 + - type: nauc_precision_at_20_max + value: 22.32755988950598 + - type: nauc_precision_at_20_std + value: 4.128353830068603 + - type: nauc_precision_at_3_diff1 + value: 32.946456569794556 + - type: nauc_precision_at_3_max + value: 24.846705267531906 + - type: nauc_precision_at_3_std + value: -5.323566174161561 + - type: nauc_precision_at_5_diff1 + value: 28.614365233016226 + - type: nauc_precision_at_5_max + value: 25.78350809072447 + - type: nauc_precision_at_5_std + value: -5.654535273917199 + - type: nauc_recall_at_1000_diff1 + value: 22.540493463839038 + - type: nauc_recall_at_1000_max + value: 42.35187769583485 + - type: nauc_recall_at_1000_std + value: 34.5803390515436 + - type: nauc_recall_at_100_diff1 + value: 19.133573411633726 + - type: nauc_recall_at_100_max + value: 22.59595955627463 + - type: nauc_recall_at_100_std + value: 12.925344614467594 + - type: nauc_recall_at_10_diff1 + value: 28.117200160362664 + - type: nauc_recall_at_10_max + value: 23.698604441777203 + - type: nauc_recall_at_10_std + value: 1.0139123824382068 + - type: nauc_recall_at_1_diff1 + value: 49.33894051910933 + - type: nauc_recall_at_1_max + value: 27.030728385670024 + - type: nauc_recall_at_1_std + value: -7.092618383196039 + - type: nauc_recall_at_20_diff1 + value: 26.019676266994157 + - type: nauc_recall_at_20_max + value: 22.305379536575668 + - type: nauc_recall_at_20_std + value: 2.2060349954805214 + - type: nauc_recall_at_3_diff1 + value: 34.873586906167816 + - type: nauc_recall_at_3_max + value: 23.34959112780764 + - type: nauc_recall_at_3_std + value: -6.5921472256897236 + - type: nauc_recall_at_5_diff1 + value: 31.45737717489477 + - type: nauc_recall_at_5_max + value: 25.65467415334357 + - type: nauc_recall_at_5_std + value: -6.011566491704533 + - type: ndcg_at_1 + value: 22.599 + - type: ndcg_at_10 + value: 34.245 + - type: ndcg_at_100 + value: 39.233000000000004 + - type: ndcg_at_1000 + value: 41.519 + - type: ndcg_at_20 + value: 36.061 + - type: ndcg_at_3 + value: 29.718 + - type: ndcg_at_5 + value: 31.721 + - type: precision_at_1 + value: 22.599 + - type: precision_at_10 + value: 5.401 + - type: precision_at_100 + value: 0.835 + - type: precision_at_1000 + value: 0.107 + - type: precision_at_20 + value: 3.1300000000000003 + - type: precision_at_3 + value: 12.994 + - type: precision_at_5 + value: 9.017 + - type: recall_at_1 + value: 20.861 + - type: recall_at_10 + value: 47.5 + - type: recall_at_100 + value: 70.785 + - type: recall_at_1000 + value: 87.997 + - type: recall_at_20 + value: 54.321 + - type: recall_at_3 + value: 35.064 + - type: recall_at_5 + value: 39.828 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackMathematicaRetrieval (default) + revision: 90fceea13679c63fe563ded68f3b6f06e50061de + split: test + type: mteb/cqadupstack-mathematica + metrics: + - type: main_score + value: 26.733 + - type: map_at_1 + value: 13.669999999999998 + - type: map_at_10 + value: 21.483 + - type: map_at_100 + value: 22.644000000000002 + - type: map_at_1000 + value: 22.78 + - type: map_at_20 + value: 22.032 + - type: map_at_3 + value: 18.515 + - type: map_at_5 + value: 20.372 + - type: mrr_at_1 + value: 16.29353233830846 + - type: mrr_at_10 + value: 25.15448550896312 + - type: mrr_at_100 + value: 26.074244375073192 + - type: mrr_at_1000 + value: 26.16388105631302 + - type: mrr_at_20 + value: 25.623812174601174 + - type: mrr_at_3 + value: 22.077114427860707 + - type: mrr_at_5 + value: 24.023631840796025 + - type: nauc_map_at_1000_diff1 + value: 28.221417213991117 + - type: nauc_map_at_1000_max + value: 15.530305364421856 + - type: nauc_map_at_1000_std + value: 0.09038561520786896 + - type: nauc_map_at_100_diff1 + value: 28.222309675506054 + - type: nauc_map_at_100_max + value: 15.54970271397305 + - type: nauc_map_at_100_std + value: 0.08952275420380759 + - type: nauc_map_at_10_diff1 + value: 28.22096862918755 + - type: nauc_map_at_10_max + value: 15.3542158492742 + - type: nauc_map_at_10_std + value: -0.7879474137448759 + - type: nauc_map_at_1_diff1 + value: 37.24736274884242 + - type: nauc_map_at_1_max + value: 20.65612352041869 + - type: nauc_map_at_1_std + value: 2.63928793263391 + - type: nauc_map_at_20_diff1 + value: 28.231803603473594 + - type: nauc_map_at_20_max + value: 15.258455521289676 + - type: nauc_map_at_20_std + value: -0.1936987895009549 + - type: nauc_map_at_3_diff1 + value: 29.39021412390121 + - type: nauc_map_at_3_max + value: 15.915339583494276 + - type: nauc_map_at_3_std + value: -0.2559033108299609 + - type: nauc_map_at_5_diff1 + value: 28.851081922069643 + - type: nauc_map_at_5_max + value: 14.92081348276245 + - type: nauc_map_at_5_std + value: -1.619937919070068 + - type: nauc_mrr_at_1000_diff1 + value: 27.848872570230398 + - type: nauc_mrr_at_1000_max + value: 17.124819110760566 + - type: nauc_mrr_at_1000_std + value: -1.4162636468654464 + - type: nauc_mrr_at_100_diff1 + value: 27.845093898795003 + - type: nauc_mrr_at_100_max + value: 17.11649334638679 + - type: nauc_mrr_at_100_std + value: -1.4260896840003132 + - type: nauc_mrr_at_10_diff1 + value: 27.76809250077834 + - type: nauc_mrr_at_10_max + value: 17.12656115107917 + - type: nauc_mrr_at_10_std + value: -1.8618103364501557 + - type: nauc_mrr_at_1_diff1 + value: 36.2608882866825 + - type: nauc_mrr_at_1_max + value: 21.458568300578396 + - type: nauc_mrr_at_1_std + value: -0.1493212786529107 + - type: nauc_mrr_at_20_diff1 + value: 27.796503522833955 + - type: nauc_mrr_at_20_max + value: 16.98795098684613 + - type: nauc_mrr_at_20_std + value: -1.6536890266296103 + - type: nauc_mrr_at_3_diff1 + value: 28.78470347909928 + - type: nauc_mrr_at_3_max + value: 17.274518035951097 + - type: nauc_mrr_at_3_std + value: -2.0538931424834055 + - type: nauc_mrr_at_5_diff1 + value: 28.57041423082867 + - type: nauc_mrr_at_5_max + value: 16.597903099177326 + - type: nauc_mrr_at_5_std + value: -2.849054995151847 + - type: nauc_ndcg_at_1000_diff1 + value: 26.08409525721658 + - type: nauc_ndcg_at_1000_max + value: 16.25482660742473 + - type: nauc_ndcg_at_1000_std + value: 2.4531627181843825 + - type: nauc_ndcg_at_100_diff1 + value: 25.773535459334056 + - type: nauc_ndcg_at_100_max + value: 16.252246090966192 + - type: nauc_ndcg_at_100_std + value: 2.489526659269772 + - type: nauc_ndcg_at_10_diff1 + value: 25.372603735449413 + - type: nauc_ndcg_at_10_max + value: 15.209635894769995 + - type: nauc_ndcg_at_10_std + value: -0.775738469848411 + - type: nauc_ndcg_at_1_diff1 + value: 36.2608882866825 + - type: nauc_ndcg_at_1_max + value: 21.458568300578396 + - type: nauc_ndcg_at_1_std + value: -0.1493212786529107 + - type: nauc_ndcg_at_20_diff1 + value: 25.44291164131007 + - type: nauc_ndcg_at_20_max + value: 14.852871889051993 + - type: nauc_ndcg_at_20_std + value: 0.8289825416874446 + - type: nauc_ndcg_at_3_diff1 + value: 27.305156273467013 + - type: nauc_ndcg_at_3_max + value: 15.997943627637925 + - type: nauc_ndcg_at_3_std + value: -1.3543640326847346 + - type: nauc_ndcg_at_5_diff1 + value: 26.75986115136007 + - type: nauc_ndcg_at_5_max + value: 14.24212804981233 + - type: nauc_ndcg_at_5_std + value: -2.952674469110462 + - type: nauc_precision_at_1000_diff1 + value: -0.7938092832438348 + - type: nauc_precision_at_1000_max + value: 1.6100389191190358 + - type: nauc_precision_at_1000_std + value: -0.293625518453535 + - type: nauc_precision_at_100_diff1 + value: 7.499911676254109 + - type: nauc_precision_at_100_max + value: 10.276908044055633 + - type: nauc_precision_at_100_std + value: 5.604685385219145 + - type: nauc_precision_at_10_diff1 + value: 13.051829966440598 + - type: nauc_precision_at_10_max + value: 11.451351452157708 + - type: nauc_precision_at_10_std + value: -3.9111849544623976 + - type: nauc_precision_at_1_diff1 + value: 36.2608882866825 + - type: nauc_precision_at_1_max + value: 21.458568300578396 + - type: nauc_precision_at_1_std + value: -0.1493212786529107 + - type: nauc_precision_at_20_diff1 + value: 11.423146658428001 + - type: nauc_precision_at_20_max + value: 9.107312695097471 + - type: nauc_precision_at_20_std + value: 2.0120583971771056 + - type: nauc_precision_at_3_diff1 + value: 20.38286265906537 + - type: nauc_precision_at_3_max + value: 12.768368952431494 + - type: nauc_precision_at_3_std + value: -4.848668515087723 + - type: nauc_precision_at_5_diff1 + value: 19.093474965134124 + - type: nauc_precision_at_5_max + value: 10.780854753523434 + - type: nauc_precision_at_5_std + value: -7.892519024194415 + - type: nauc_recall_at_1000_diff1 + value: 19.415211825768043 + - type: nauc_recall_at_1000_max + value: 23.295123511450658 + - type: nauc_recall_at_1000_std + value: 27.880265834599793 + - type: nauc_recall_at_100_diff1 + value: 18.496829500539764 + - type: nauc_recall_at_100_max + value: 17.975806339706217 + - type: nauc_recall_at_100_std + value: 13.857781042303802 + - type: nauc_recall_at_10_diff1 + value: 18.27583721829487 + - type: nauc_recall_at_10_max + value: 13.622920139186368 + - type: nauc_recall_at_10_std + value: 0.7347327651834099 + - type: nauc_recall_at_1_diff1 + value: 37.24736274884242 + - type: nauc_recall_at_1_max + value: 20.65612352041869 + - type: nauc_recall_at_1_std + value: 2.63928793263391 + - type: nauc_recall_at_20_diff1 + value: 18.156949630775923 + - type: nauc_recall_at_20_max + value: 12.295074513776836 + - type: nauc_recall_at_20_std + value: 5.160412855725281 + - type: nauc_recall_at_3_diff1 + value: 22.707879624312362 + - type: nauc_recall_at_3_max + value: 14.075829880170676 + - type: nauc_recall_at_3_std + value: -0.9247366317132225 + - type: nauc_recall_at_5_diff1 + value: 21.01344659429256 + - type: nauc_recall_at_5_max + value: 10.441651089831321 + - type: nauc_recall_at_5_std + value: -4.628675158493911 + - type: ndcg_at_1 + value: 16.294 + - type: ndcg_at_10 + value: 26.733 + - type: ndcg_at_100 + value: 32.415 + - type: ndcg_at_1000 + value: 35.556 + - type: ndcg_at_20 + value: 28.593000000000004 + - type: ndcg_at_3 + value: 21.129 + - type: ndcg_at_5 + value: 24.236 + - type: precision_at_1 + value: 16.294 + - type: precision_at_10 + value: 5.261 + - type: precision_at_100 + value: 0.9299999999999999 + - type: precision_at_1000 + value: 0.135 + - type: precision_at_20 + value: 3.1719999999999997 + - type: precision_at_3 + value: 10.323 + - type: precision_at_5 + value: 8.184 + - type: recall_at_1 + value: 13.669999999999998 + - type: recall_at_10 + value: 39.056000000000004 + - type: recall_at_100 + value: 64.298 + - type: recall_at_1000 + value: 86.278 + - type: recall_at_20 + value: 45.706 + - type: recall_at_3 + value: 24.193 + - type: recall_at_5 + value: 31.948 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackPhysicsRetrieval (default) + revision: 79531abbd1fb92d06c6d6315a0cbbbf5bb247ea4 + split: test + type: mteb/cqadupstack-physics + metrics: + - type: main_score + value: 45.538000000000004 + - type: map_at_1 + value: 28.68 + - type: map_at_10 + value: 39.524 + - type: map_at_100 + value: 40.858 + - type: map_at_1000 + value: 40.96 + - type: map_at_20 + value: 40.298 + - type: map_at_3 + value: 36.226 + - type: map_at_5 + value: 38.116 + - type: mrr_at_1 + value: 34.93743984600577 + - type: mrr_at_10 + value: 44.84023862994023 + - type: mrr_at_100 + value: 45.70525432445239 + - type: mrr_at_1000 + value: 45.74790482687698 + - type: mrr_at_20 + value: 45.35629819944036 + - type: mrr_at_3 + value: 42.39653512993261 + - type: mrr_at_5 + value: 43.80654475457166 + - type: nauc_map_at_1000_diff1 + value: 45.540378200668066 + - type: nauc_map_at_1000_max + value: 35.609273973958224 + - type: nauc_map_at_1000_std + value: -1.7919595480551342 + - type: nauc_map_at_100_diff1 + value: 45.52117341914365 + - type: nauc_map_at_100_max + value: 35.60630252652701 + - type: nauc_map_at_100_std + value: -1.8528609120925232 + - type: nauc_map_at_10_diff1 + value: 45.401994008669554 + - type: nauc_map_at_10_max + value: 35.14772595579072 + - type: nauc_map_at_10_std + value: -2.5090043581275987 + - type: nauc_map_at_1_diff1 + value: 51.89583640253883 + - type: nauc_map_at_1_max + value: 34.342282865158786 + - type: nauc_map_at_1_std + value: -6.606869072850554 + - type: nauc_map_at_20_diff1 + value: 45.4894951227973 + - type: nauc_map_at_20_max + value: 35.53590407401343 + - type: nauc_map_at_20_std + value: -2.206734662042172 + - type: nauc_map_at_3_diff1 + value: 46.088019072147766 + - type: nauc_map_at_3_max + value: 34.45521906572965 + - type: nauc_map_at_3_std + value: -3.0262346258170965 + - type: nauc_map_at_5_diff1 + value: 45.5093679023141 + - type: nauc_map_at_5_max + value: 34.99359823322432 + - type: nauc_map_at_5_std + value: -2.706105621668906 + - type: nauc_mrr_at_1000_diff1 + value: 47.18568890087256 + - type: nauc_mrr_at_1000_max + value: 37.20465260904721 + - type: nauc_mrr_at_1000_std + value: -0.46367399574799145 + - type: nauc_mrr_at_100_diff1 + value: 47.16525768323949 + - type: nauc_mrr_at_100_max + value: 37.20811683597186 + - type: nauc_mrr_at_100_std + value: -0.4567796217361991 + - type: nauc_mrr_at_10_diff1 + value: 47.14510075293479 + - type: nauc_mrr_at_10_max + value: 37.07472857897546 + - type: nauc_mrr_at_10_std + value: -0.7748361049232515 + - type: nauc_mrr_at_1_diff1 + value: 51.6925095161796 + - type: nauc_mrr_at_1_max + value: 36.737042386765914 + - type: nauc_mrr_at_1_std + value: -2.5559306545985 + - type: nauc_mrr_at_20_diff1 + value: 47.09911007512393 + - type: nauc_mrr_at_20_max + value: 37.17982387431314 + - type: nauc_mrr_at_20_std + value: -0.572954896536082 + - type: nauc_mrr_at_3_diff1 + value: 48.08467530055374 + - type: nauc_mrr_at_3_max + value: 36.8468982864687 + - type: nauc_mrr_at_3_std + value: -0.7588497005490253 + - type: nauc_mrr_at_5_diff1 + value: 47.57849579078835 + - type: nauc_mrr_at_5_max + value: 37.29921381765654 + - type: nauc_mrr_at_5_std + value: -0.8628323000617983 + - type: nauc_ndcg_at_1000_diff1 + value: 44.01127995400056 + - type: nauc_ndcg_at_1000_max + value: 36.520216646769185 + - type: nauc_ndcg_at_1000_std + value: 1.3570584123016256 + - type: nauc_ndcg_at_100_diff1 + value: 43.58506948132083 + - type: nauc_ndcg_at_100_max + value: 36.43290771936801 + - type: nauc_ndcg_at_100_std + value: 1.0663570929035426 + - type: nauc_ndcg_at_10_diff1 + value: 43.34204702532026 + - type: nauc_ndcg_at_10_max + value: 35.3209018153949 + - type: nauc_ndcg_at_10_std + value: -1.6302063681605896 + - type: nauc_ndcg_at_1_diff1 + value: 51.6925095161796 + - type: nauc_ndcg_at_1_max + value: 36.737042386765914 + - type: nauc_ndcg_at_1_std + value: -2.5559306545985 + - type: nauc_ndcg_at_20_diff1 + value: 43.45880868758326 + - type: nauc_ndcg_at_20_max + value: 36.36538424795932 + - type: nauc_ndcg_at_20_std + value: -0.6531353376861906 + - type: nauc_ndcg_at_3_diff1 + value: 45.028222019793795 + - type: nauc_ndcg_at_3_max + value: 35.2838611119301 + - type: nauc_ndcg_at_3_std + value: -1.1194247158058406 + - type: nauc_ndcg_at_5_diff1 + value: 44.10869842059044 + - type: nauc_ndcg_at_5_max + value: 35.69553449340272 + - type: nauc_ndcg_at_5_std + value: -1.3685749717474223 + - type: nauc_precision_at_1000_diff1 + value: -8.14237713160761 + - type: nauc_precision_at_1000_max + value: 1.3015785738749102 + - type: nauc_precision_at_1000_std + value: 22.631480967557373 + - type: nauc_precision_at_100_diff1 + value: 0.7204300254080606 + - type: nauc_precision_at_100_max + value: 11.888071890259901 + - type: nauc_precision_at_100_std + value: 20.036872837873926 + - type: nauc_precision_at_10_diff1 + value: 17.71073979482443 + - type: nauc_precision_at_10_max + value: 25.320701928320016 + - type: nauc_precision_at_10_std + value: 10.42449521624285 + - type: nauc_precision_at_1_diff1 + value: 51.6925095161796 + - type: nauc_precision_at_1_max + value: 36.737042386765914 + - type: nauc_precision_at_1_std + value: -2.5559306545985 + - type: nauc_precision_at_20_diff1 + value: 12.709902470984586 + - type: nauc_precision_at_20_max + value: 23.88587822660469 + - type: nauc_precision_at_20_std + value: 13.606726981492429 + - type: nauc_precision_at_3_diff1 + value: 31.94823438742197 + - type: nauc_precision_at_3_max + value: 32.13678119642854 + - type: nauc_precision_at_3_std + value: 7.160205411890389 + - type: nauc_precision_at_5_diff1 + value: 24.585061516114877 + - type: nauc_precision_at_5_max + value: 29.217424865304302 + - type: nauc_precision_at_5_std + value: 7.78543533001093 + - type: nauc_recall_at_1000_diff1 + value: 17.102532216435616 + - type: nauc_recall_at_1000_max + value: 31.40940365819655 + - type: nauc_recall_at_1000_std + value: 33.40942585514109 + - type: nauc_recall_at_100_diff1 + value: 25.96217357713677 + - type: nauc_recall_at_100_max + value: 32.1790454600331 + - type: nauc_recall_at_100_std + value: 16.247066984563716 + - type: nauc_recall_at_10_diff1 + value: 32.62660484386981 + - type: nauc_recall_at_10_max + value: 31.044994892358428 + - type: nauc_recall_at_10_std + value: -1.534523284624985 + - type: nauc_recall_at_1_diff1 + value: 51.89583640253883 + - type: nauc_recall_at_1_max + value: 34.342282865158786 + - type: nauc_recall_at_1_std + value: -6.606869072850554 + - type: nauc_recall_at_20_diff1 + value: 31.51921321228521 + - type: nauc_recall_at_20_max + value: 34.15960884354325 + - type: nauc_recall_at_20_std + value: 2.852618186057463 + - type: nauc_recall_at_3_diff1 + value: 38.98359915040439 + - type: nauc_recall_at_3_max + value: 32.12267174992241 + - type: nauc_recall_at_3_std + value: -1.3785705910221837 + - type: nauc_recall_at_5_diff1 + value: 36.16498916721275 + - type: nauc_recall_at_5_max + value: 33.16317035401434 + - type: nauc_recall_at_5_std + value: -1.1398157637294843 + - type: ndcg_at_1 + value: 34.937000000000005 + - type: ndcg_at_10 + value: 45.538000000000004 + - type: ndcg_at_100 + value: 51.029 + - type: ndcg_at_1000 + value: 52.900000000000006 + - type: ndcg_at_20 + value: 47.799 + - type: ndcg_at_3 + value: 40.446 + - type: ndcg_at_5 + value: 42.928 + - type: precision_at_1 + value: 34.937000000000005 + - type: precision_at_10 + value: 8.325000000000001 + - type: precision_at_100 + value: 1.2850000000000001 + - type: precision_at_1000 + value: 0.161 + - type: precision_at_20 + value: 4.904 + - type: precision_at_3 + value: 19.442 + - type: precision_at_5 + value: 13.84 + - type: recall_at_1 + value: 28.68 + - type: recall_at_10 + value: 58.031 + - type: recall_at_100 + value: 80.967 + - type: recall_at_1000 + value: 93.001 + - type: recall_at_20 + value: 65.958 + - type: recall_at_3 + value: 43.712 + - type: recall_at_5 + value: 50.158 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackProgrammersRetrieval (default) + revision: 6184bc1440d2dbc7612be22b50686b8826d22b32 + split: test + type: mteb/cqadupstack-programmers + metrics: + - type: main_score + value: 41.796 + - type: map_at_1 + value: 24.573 + - type: map_at_10 + value: 35.638999999999996 + - type: map_at_100 + value: 37.012 + - type: map_at_1000 + value: 37.13 + - type: map_at_20 + value: 36.396 + - type: map_at_3 + value: 32.394 + - type: map_at_5 + value: 34.244 + - type: mrr_at_1 + value: 30.70776255707763 + - type: mrr_at_10 + value: 41.02889215046749 + - type: mrr_at_100 + value: 41.95464821945586 + - type: mrr_at_1000 + value: 42.0046063984793 + - type: mrr_at_20 + value: 41.59433185226793 + - type: mrr_at_3 + value: 38.527397260273965 + - type: mrr_at_5 + value: 39.897260273972584 + - type: nauc_map_at_1000_diff1 + value: 36.7692768536943 + - type: nauc_map_at_1000_max + value: 26.847959451924737 + - type: nauc_map_at_1000_std + value: 1.2856964110033477 + - type: nauc_map_at_100_diff1 + value: 36.7358266741215 + - type: nauc_map_at_100_max + value: 26.82757207681197 + - type: nauc_map_at_100_std + value: 1.2716949040100032 + - type: nauc_map_at_10_diff1 + value: 36.58972682357357 + - type: nauc_map_at_10_max + value: 26.411220199726245 + - type: nauc_map_at_10_std + value: 0.41468040763730957 + - type: nauc_map_at_1_diff1 + value: 43.925409893290826 + - type: nauc_map_at_1_max + value: 25.37514074309528 + - type: nauc_map_at_1_std + value: -3.282848444289815 + - type: nauc_map_at_20_diff1 + value: 36.70554873434887 + - type: nauc_map_at_20_max + value: 26.556736454224744 + - type: nauc_map_at_20_std + value: 0.837932916272811 + - type: nauc_map_at_3_diff1 + value: 37.60990955315632 + - type: nauc_map_at_3_max + value: 26.05100736078808 + - type: nauc_map_at_3_std + value: -1.075240445586956 + - type: nauc_map_at_5_diff1 + value: 37.02202535579905 + - type: nauc_map_at_5_max + value: 26.202241292942013 + - type: nauc_map_at_5_std + value: -0.8077096897769526 + - type: nauc_mrr_at_1000_diff1 + value: 35.77388799524249 + - type: nauc_mrr_at_1000_max + value: 28.065934859067376 + - type: nauc_mrr_at_1000_std + value: 2.7265303792419324 + - type: nauc_mrr_at_100_diff1 + value: 35.75212384276989 + - type: nauc_mrr_at_100_max + value: 28.08041768718836 + - type: nauc_mrr_at_100_std + value: 2.7711197047800185 + - type: nauc_mrr_at_10_diff1 + value: 35.51702972516797 + - type: nauc_mrr_at_10_max + value: 27.791611404051984 + - type: nauc_mrr_at_10_std + value: 2.2877905993622965 + - type: nauc_mrr_at_1_diff1 + value: 42.24041385647434 + - type: nauc_mrr_at_1_max + value: 28.177064307236805 + - type: nauc_mrr_at_1_std + value: 0.11832266444237129 + - type: nauc_mrr_at_20_diff1 + value: 35.68381829398048 + - type: nauc_mrr_at_20_max + value: 27.884058117799892 + - type: nauc_mrr_at_20_std + value: 2.6284317932739087 + - type: nauc_mrr_at_3_diff1 + value: 36.48102894704183 + - type: nauc_mrr_at_3_max + value: 28.211976441694368 + - type: nauc_mrr_at_3_std + value: 1.5477985287047984 + - type: nauc_mrr_at_5_diff1 + value: 35.7585415335929 + - type: nauc_mrr_at_5_max + value: 27.96983257165152 + - type: nauc_mrr_at_5_std + value: 1.2407798055514567 + - type: nauc_ndcg_at_1000_diff1 + value: 35.40624761326608 + - type: nauc_ndcg_at_1000_max + value: 28.390702778443522 + - type: nauc_ndcg_at_1000_std + value: 5.265721439960366 + - type: nauc_ndcg_at_100_diff1 + value: 34.96082564648268 + - type: nauc_ndcg_at_100_max + value: 28.518954951452326 + - type: nauc_ndcg_at_100_std + value: 5.941973379261584 + - type: nauc_ndcg_at_10_diff1 + value: 34.05270108923721 + - type: nauc_ndcg_at_10_max + value: 26.51991960121645 + - type: nauc_ndcg_at_10_std + value: 2.298355822186611 + - type: nauc_ndcg_at_1_diff1 + value: 42.24041385647434 + - type: nauc_ndcg_at_1_max + value: 28.177064307236805 + - type: nauc_ndcg_at_1_std + value: 0.11832266444237129 + - type: nauc_ndcg_at_20_diff1 + value: 34.480098491703586 + - type: nauc_ndcg_at_20_max + value: 26.826869171672495 + - type: nauc_ndcg_at_20_std + value: 3.6216122201914134 + - type: nauc_ndcg_at_3_diff1 + value: 35.364868057857315 + - type: nauc_ndcg_at_3_max + value: 26.671699349470035 + - type: nauc_ndcg_at_3_std + value: 0.6355522792606674 + - type: nauc_ndcg_at_5_diff1 + value: 34.7955302448707 + - type: nauc_ndcg_at_5_max + value: 26.40195365209001 + - type: nauc_ndcg_at_5_std + value: -0.03633498778776194 + - type: nauc_precision_at_1000_diff1 + value: -7.6870989588278995 + - type: nauc_precision_at_1000_max + value: 3.804917711130948 + - type: nauc_precision_at_1000_std + value: 11.926191914551726 + - type: nauc_precision_at_100_diff1 + value: -1.2981555256339568 + - type: nauc_precision_at_100_max + value: 13.77026997807174 + - type: nauc_precision_at_100_std + value: 20.282991398976456 + - type: nauc_precision_at_10_diff1 + value: 11.241327502703399 + - type: nauc_precision_at_10_max + value: 21.080127815602427 + - type: nauc_precision_at_10_std + value: 13.473538645423536 + - type: nauc_precision_at_1_diff1 + value: 42.24041385647434 + - type: nauc_precision_at_1_max + value: 28.177064307236805 + - type: nauc_precision_at_1_std + value: 0.11832266444237129 + - type: nauc_precision_at_20_diff1 + value: 7.873799383051996 + - type: nauc_precision_at_20_max + value: 19.13594602500818 + - type: nauc_precision_at_20_std + value: 16.58121638621799 + - type: nauc_precision_at_3_diff1 + value: 22.668521544821726 + - type: nauc_precision_at_3_max + value: 25.575159625969757 + - type: nauc_precision_at_3_std + value: 6.977004075884658 + - type: nauc_precision_at_5_diff1 + value: 17.579650101764052 + - type: nauc_precision_at_5_max + value: 23.239877595903753 + - type: nauc_precision_at_5_std + value: 7.153549848513488 + - type: nauc_recall_at_1000_diff1 + value: 35.44896649152339 + - type: nauc_recall_at_1000_max + value: 50.134726555781064 + - type: nauc_recall_at_1000_std + value: 55.488897572741855 + - type: nauc_recall_at_100_diff1 + value: 28.11855482894462 + - type: nauc_recall_at_100_max + value: 36.037412126598554 + - type: nauc_recall_at_100_std + value: 31.679662551318028 + - type: nauc_recall_at_10_diff1 + value: 25.402137870499036 + - type: nauc_recall_at_10_max + value: 23.32732420574465 + - type: nauc_recall_at_10_std + value: 6.3695873328264065 + - type: nauc_recall_at_1_diff1 + value: 43.925409893290826 + - type: nauc_recall_at_1_max + value: 25.37514074309528 + - type: nauc_recall_at_1_std + value: -3.282848444289815 + - type: nauc_recall_at_20_diff1 + value: 26.319380144111104 + - type: nauc_recall_at_20_max + value: 23.966795471870007 + - type: nauc_recall_at_20_std + value: 11.396415613592835 + - type: nauc_recall_at_3_diff1 + value: 31.081873898555816 + - type: nauc_recall_at_3_max + value: 24.186620719378897 + - type: nauc_recall_at_3_std + value: -0.9631804535229787 + - type: nauc_recall_at_5_diff1 + value: 28.47634238398339 + - type: nauc_recall_at_5_max + value: 23.74536300018273 + - type: nauc_recall_at_5_std + value: -0.9507949304318117 + - type: ndcg_at_1 + value: 30.708000000000002 + - type: ndcg_at_10 + value: 41.796 + - type: ndcg_at_100 + value: 47.575 + - type: ndcg_at_1000 + value: 49.742 + - type: ndcg_at_20 + value: 44.066 + - type: ndcg_at_3 + value: 36.706 + - type: ndcg_at_5 + value: 38.977000000000004 + - type: precision_at_1 + value: 30.708000000000002 + - type: precision_at_10 + value: 7.922 + - type: precision_at_100 + value: 1.256 + - type: precision_at_1000 + value: 0.163 + - type: precision_at_20 + value: 4.675 + - type: precision_at_3 + value: 18.189 + - type: precision_at_5 + value: 12.967999999999998 + - type: recall_at_1 + value: 24.573 + - type: recall_at_10 + value: 54.7 + - type: recall_at_100 + value: 79.561 + - type: recall_at_1000 + value: 93.861 + - type: recall_at_20 + value: 62.78 + - type: recall_at_3 + value: 40.228 + - type: recall_at_5 + value: 46.326 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackRetrieval (default) + revision: CQADupstackRetrieval_is_a_combined_dataset + split: test + type: CQADupstackRetrieval_is_a_combined_dataset + metrics: + - type: main_score + value: 39.19233333333332 + - type: ndcg_at_10 + value: 39.19233333333332 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackStatsRetrieval (default) + revision: 65ac3a16b8e91f9cee4c9828cc7c335575432a2a + split: test + type: mteb/cqadupstack-stats + metrics: + - type: main_score + value: 32.612 + - type: map_at_1 + value: 21.364 + - type: map_at_10 + value: 28.291 + - type: map_at_100 + value: 29.241 + - type: map_at_1000 + value: 29.330000000000002 + - type: map_at_20 + value: 28.877999999999997 + - type: map_at_3 + value: 26.011 + - type: map_at_5 + value: 27.04 + - type: mrr_at_1 + value: 24.079754601226995 + - type: mrr_at_10 + value: 30.85335719154737 + - type: mrr_at_100 + value: 31.72319295349122 + - type: mrr_at_1000 + value: 31.789693895698907 + - type: mrr_at_20 + value: 31.409198994207728 + - type: mrr_at_3 + value: 28.7321063394683 + - type: mrr_at_5 + value: 29.729038854805733 + - type: nauc_map_at_1000_diff1 + value: 54.66321846156984 + - type: nauc_map_at_1000_max + value: 38.51925419692489 + - type: nauc_map_at_1000_std + value: 5.311425877366216 + - type: nauc_map_at_100_diff1 + value: 54.65252016011802 + - type: nauc_map_at_100_max + value: 38.47628397127589 + - type: nauc_map_at_100_std + value: 5.321160622644175 + - type: nauc_map_at_10_diff1 + value: 55.28903840370793 + - type: nauc_map_at_10_max + value: 38.65316119774072 + - type: nauc_map_at_10_std + value: 4.725299450879962 + - type: nauc_map_at_1_diff1 + value: 61.286808053765206 + - type: nauc_map_at_1_max + value: 39.923842681828134 + - type: nauc_map_at_1_std + value: 0.3632977020996134 + - type: nauc_map_at_20_diff1 + value: 54.75301676575286 + - type: nauc_map_at_20_max + value: 38.39502868792125 + - type: nauc_map_at_20_std + value: 5.1128741581460755 + - type: nauc_map_at_3_diff1 + value: 57.02410102274832 + - type: nauc_map_at_3_max + value: 38.61723975568939 + - type: nauc_map_at_3_std + value: 2.7064030125410734 + - type: nauc_map_at_5_diff1 + value: 56.044441763601164 + - type: nauc_map_at_5_max + value: 38.71368502442094 + - type: nauc_map_at_5_std + value: 3.445028327502005 + - type: nauc_mrr_at_1000_diff1 + value: 51.88856345795892 + - type: nauc_mrr_at_1000_max + value: 37.1364769884927 + - type: nauc_mrr_at_1000_std + value: 7.508186150441637 + - type: nauc_mrr_at_100_diff1 + value: 51.85343205160575 + - type: nauc_mrr_at_100_max + value: 37.1142401737359 + - type: nauc_mrr_at_100_std + value: 7.525821696938287 + - type: nauc_mrr_at_10_diff1 + value: 52.36601725878158 + - type: nauc_mrr_at_10_max + value: 37.21176078227791 + - type: nauc_mrr_at_10_std + value: 7.061089465459671 + - type: nauc_mrr_at_1_diff1 + value: 58.143161634438144 + - type: nauc_mrr_at_1_max + value: 39.07929312048979 + - type: nauc_mrr_at_1_std + value: 3.658362094316155 + - type: nauc_mrr_at_20_diff1 + value: 51.938137549131305 + - type: nauc_mrr_at_20_max + value: 37.10739608642369 + - type: nauc_mrr_at_20_std + value: 7.406074286394455 + - type: nauc_mrr_at_3_diff1 + value: 53.625445752585534 + - type: nauc_mrr_at_3_max + value: 36.98022102495354 + - type: nauc_mrr_at_3_std + value: 5.81354139265593 + - type: nauc_mrr_at_5_diff1 + value: 52.81177524938679 + - type: nauc_mrr_at_5_max + value: 37.10869722980057 + - type: nauc_mrr_at_5_std + value: 6.463447979318842 + - type: nauc_ndcg_at_1000_diff1 + value: 50.08062567725874 + - type: nauc_ndcg_at_1000_max + value: 37.7811301398 + - type: nauc_ndcg_at_1000_std + value: 9.620454153790758 + - type: nauc_ndcg_at_100_diff1 + value: 49.547194351900714 + - type: nauc_ndcg_at_100_max + value: 36.885646158693106 + - type: nauc_ndcg_at_100_std + value: 9.928853010439004 + - type: nauc_ndcg_at_10_diff1 + value: 52.02771611109392 + - type: nauc_ndcg_at_10_max + value: 37.572688175648395 + - type: nauc_ndcg_at_10_std + value: 7.563824200459868 + - type: nauc_ndcg_at_1_diff1 + value: 58.143161634438144 + - type: nauc_ndcg_at_1_max + value: 39.07929312048979 + - type: nauc_ndcg_at_1_std + value: 3.658362094316155 + - type: nauc_ndcg_at_20_diff1 + value: 50.28151868405584 + - type: nauc_ndcg_at_20_max + value: 36.81633873667909 + - type: nauc_ndcg_at_20_std + value: 8.862570442221514 + - type: nauc_ndcg_at_3_diff1 + value: 54.47516467658079 + - type: nauc_ndcg_at_3_max + value: 37.532065337186125 + - type: nauc_ndcg_at_3_std + value: 4.5725239582001445 + - type: nauc_ndcg_at_5_diff1 + value: 53.22180703290253 + - type: nauc_ndcg_at_5_max + value: 37.59616752850715 + - type: nauc_ndcg_at_5_std + value: 5.275130889190682 + - type: nauc_precision_at_1000_diff1 + value: -4.101889231462178 + - type: nauc_precision_at_1000_max + value: 16.113686606494042 + - type: nauc_precision_at_1000_std + value: 18.67462908599342 + - type: nauc_precision_at_100_diff1 + value: 9.925121176879674 + - type: nauc_precision_at_100_max + value: 22.055179177569002 + - type: nauc_precision_at_100_std + value: 25.294412007212365 + - type: nauc_precision_at_10_diff1 + value: 30.459151247707105 + - type: nauc_precision_at_10_max + value: 29.594624289478492 + - type: nauc_precision_at_10_std + value: 20.96340383222405 + - type: nauc_precision_at_1_diff1 + value: 58.143161634438144 + - type: nauc_precision_at_1_max + value: 39.07929312048979 + - type: nauc_precision_at_1_std + value: 3.658362094316155 + - type: nauc_precision_at_20_diff1 + value: 20.92851614744852 + - type: nauc_precision_at_20_max + value: 25.235663219025266 + - type: nauc_precision_at_20_std + value: 23.592947196949787 + - type: nauc_precision_at_3_diff1 + value: 44.77467190178675 + - type: nauc_precision_at_3_max + value: 33.824987790974276 + - type: nauc_precision_at_3_std + value: 10.63283013472072 + - type: nauc_precision_at_5_diff1 + value: 38.42013972387805 + - type: nauc_precision_at_5_max + value: 32.366568562705574 + - type: nauc_precision_at_5_std + value: 13.254415606865344 + - type: nauc_recall_at_1000_diff1 + value: 26.72653852273919 + - type: nauc_recall_at_1000_max + value: 35.110917727237705 + - type: nauc_recall_at_1000_std + value: 29.541699408887773 + - type: nauc_recall_at_100_diff1 + value: 31.42719789032615 + - type: nauc_recall_at_100_max + value: 29.028285988440615 + - type: nauc_recall_at_100_std + value: 23.893936774361528 + - type: nauc_recall_at_10_diff1 + value: 44.501264636818775 + - type: nauc_recall_at_10_max + value: 34.70764871477816 + - type: nauc_recall_at_10_std + value: 12.879734778729377 + - type: nauc_recall_at_1_diff1 + value: 61.286808053765206 + - type: nauc_recall_at_1_max + value: 39.923842681828134 + - type: nauc_recall_at_1_std + value: 0.3632977020996134 + - type: nauc_recall_at_20_diff1 + value: 37.76446912117247 + - type: nauc_recall_at_20_max + value: 31.393851250209526 + - type: nauc_recall_at_20_std + value: 17.413011800702076 + - type: nauc_recall_at_3_diff1 + value: 51.50295145458307 + - type: nauc_recall_at_3_max + value: 35.37836540860319 + - type: nauc_recall_at_3_std + value: 4.911960139618243 + - type: nauc_recall_at_5_diff1 + value: 48.42193245902022 + - type: nauc_recall_at_5_max + value: 35.269638012676026 + - type: nauc_recall_at_5_std + value: 6.925694327958159 + - type: ndcg_at_1 + value: 24.08 + - type: ndcg_at_10 + value: 32.612 + - type: ndcg_at_100 + value: 37.07 + - type: ndcg_at_1000 + value: 39.585 + - type: ndcg_at_20 + value: 34.618 + - type: ndcg_at_3 + value: 28.168 + - type: ndcg_at_5 + value: 29.839 + - type: precision_at_1 + value: 24.08 + - type: precision_at_10 + value: 5.322 + - type: precision_at_100 + value: 0.808 + - type: precision_at_1000 + value: 0.11 + - type: precision_at_20 + value: 3.167 + - type: precision_at_3 + value: 12.168 + - type: precision_at_5 + value: 8.405 + - type: recall_at_1 + value: 21.364 + - type: recall_at_10 + value: 43.423 + - type: recall_at_100 + value: 63.193999999999996 + - type: recall_at_1000 + value: 82.27499999999999 + - type: recall_at_20 + value: 50.907000000000004 + - type: recall_at_3 + value: 31.083 + - type: recall_at_5 + value: 35.165 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackTexRetrieval (default) + revision: 46989137a86843e03a6195de44b09deda022eec7 + split: test + type: mteb/cqadupstack-tex + metrics: + - type: main_score + value: 24.492 + - type: map_at_1 + value: 14.292 + - type: map_at_10 + value: 20.403 + - type: map_at_100 + value: 21.46 + - type: map_at_1000 + value: 21.590999999999998 + - type: map_at_20 + value: 20.973 + - type: map_at_3 + value: 18.34 + - type: map_at_5 + value: 19.545 + - type: mrr_at_1 + value: 17.412250516173433 + - type: mrr_at_10 + value: 23.85980292552902 + - type: mrr_at_100 + value: 24.807382554477137 + - type: mrr_at_1000 + value: 24.895492171680196 + - type: mrr_at_20 + value: 24.3898759799139 + - type: mrr_at_3 + value: 21.771048405597618 + - type: mrr_at_5 + value: 23.0012617572838 + - type: nauc_map_at_1000_diff1 + value: 33.995711385656804 + - type: nauc_map_at_1000_max + value: 25.16673937307302 + - type: nauc_map_at_1000_std + value: -0.8740755355242664 + - type: nauc_map_at_100_diff1 + value: 33.99537291847843 + - type: nauc_map_at_100_max + value: 25.16308416434281 + - type: nauc_map_at_100_std + value: -0.903252876033793 + - type: nauc_map_at_10_diff1 + value: 34.163656537419875 + - type: nauc_map_at_10_max + value: 25.153405332329488 + - type: nauc_map_at_10_std + value: -1.5064188630994542 + - type: nauc_map_at_1_diff1 + value: 40.91264195434544 + - type: nauc_map_at_1_max + value: 25.107414759912196 + - type: nauc_map_at_1_std + value: -4.238206964379598 + - type: nauc_map_at_20_diff1 + value: 34.101921762396216 + - type: nauc_map_at_20_max + value: 25.12970872477412 + - type: nauc_map_at_20_std + value: -1.1903440675022818 + - type: nauc_map_at_3_diff1 + value: 35.4881466698112 + - type: nauc_map_at_3_max + value: 24.705846425258464 + - type: nauc_map_at_3_std + value: -2.1361773100190447 + - type: nauc_map_at_5_diff1 + value: 34.521249584925044 + - type: nauc_map_at_5_max + value: 25.088971418997314 + - type: nauc_map_at_5_std + value: -1.9455485694774537 + - type: nauc_mrr_at_1000_diff1 + value: 32.73831439970557 + - type: nauc_mrr_at_1000_max + value: 24.8676644643215 + - type: nauc_mrr_at_1000_std + value: -1.4994736914733964 + - type: nauc_mrr_at_100_diff1 + value: 32.74081307790701 + - type: nauc_mrr_at_100_max + value: 24.870195219804017 + - type: nauc_mrr_at_100_std + value: -1.5056458011593927 + - type: nauc_mrr_at_10_diff1 + value: 32.904869588799016 + - type: nauc_mrr_at_10_max + value: 24.88512211730381 + - type: nauc_mrr_at_10_std + value: -2.0086410743047853 + - type: nauc_mrr_at_1_diff1 + value: 38.65639209567485 + - type: nauc_mrr_at_1_max + value: 24.662155409452435 + - type: nauc_mrr_at_1_std + value: -4.4096226126176195 + - type: nauc_mrr_at_20_diff1 + value: 32.76294701700251 + - type: nauc_mrr_at_20_max + value: 24.826702703169722 + - type: nauc_mrr_at_20_std + value: -1.7170820124027029 + - type: nauc_mrr_at_3_diff1 + value: 34.25774239843736 + - type: nauc_mrr_at_3_max + value: 24.90355679088423 + - type: nauc_mrr_at_3_std + value: -2.775477525763894 + - type: nauc_mrr_at_5_diff1 + value: 33.18498430652641 + - type: nauc_mrr_at_5_max + value: 24.851868217803215 + - type: nauc_mrr_at_5_std + value: -2.6663788685260417 + - type: nauc_ndcg_at_1000_diff1 + value: 30.72967346462846 + - type: nauc_ndcg_at_1000_max + value: 25.014067441522425 + - type: nauc_ndcg_at_1000_std + value: 2.8953064620586066 + - type: nauc_ndcg_at_100_diff1 + value: 30.859149462771935 + - type: nauc_ndcg_at_100_max + value: 25.0787456079416 + - type: nauc_ndcg_at_100_std + value: 2.544212558815528 + - type: nauc_ndcg_at_10_diff1 + value: 31.582609602455953 + - type: nauc_ndcg_at_10_max + value: 25.29533124530945 + - type: nauc_ndcg_at_10_std + value: -0.2326991971121844 + - type: nauc_ndcg_at_1_diff1 + value: 38.65639209567485 + - type: nauc_ndcg_at_1_max + value: 24.662155409452435 + - type: nauc_ndcg_at_1_std + value: -4.4096226126176195 + - type: nauc_ndcg_at_20_diff1 + value: 31.24640809492941 + - type: nauc_ndcg_at_20_max + value: 25.083626002540214 + - type: nauc_ndcg_at_20_std + value: 0.7664243801662968 + - type: nauc_ndcg_at_3_diff1 + value: 33.875633253562214 + - type: nauc_ndcg_at_3_max + value: 24.93590899342811 + - type: nauc_ndcg_at_3_std + value: -1.8775982086080036 + - type: nauc_ndcg_at_5_diff1 + value: 32.41653165109768 + - type: nauc_ndcg_at_5_max + value: 25.341107903576297 + - type: nauc_ndcg_at_5_std + value: -1.4155754633547732 + - type: nauc_precision_at_1000_diff1 + value: 4.2280321215542145 + - type: nauc_precision_at_1000_max + value: 11.264815476757484 + - type: nauc_precision_at_1000_std + value: 7.861919022013922 + - type: nauc_precision_at_100_diff1 + value: 12.578151563110515 + - type: nauc_precision_at_100_max + value: 17.77987418666521 + - type: nauc_precision_at_100_std + value: 9.570625873881804 + - type: nauc_precision_at_10_diff1 + value: 21.40488105863851 + - type: nauc_precision_at_10_max + value: 23.584911345070953 + - type: nauc_precision_at_10_std + value: 2.3491117045645327 + - type: nauc_precision_at_1_diff1 + value: 38.65639209567485 + - type: nauc_precision_at_1_max + value: 24.662155409452435 + - type: nauc_precision_at_1_std + value: -4.4096226126176195 + - type: nauc_precision_at_20_diff1 + value: 18.979129809789885 + - type: nauc_precision_at_20_max + value: 21.776861645291827 + - type: nauc_precision_at_20_std + value: 4.9762646043043635 + - type: nauc_precision_at_3_diff1 + value: 26.953151233365176 + - type: nauc_precision_at_3_max + value: 24.07585347393912 + - type: nauc_precision_at_3_std + value: -1.5463392094380226 + - type: nauc_precision_at_5_diff1 + value: 24.255143771697853 + - type: nauc_precision_at_5_max + value: 25.064019283351826 + - type: nauc_precision_at_5_std + value: -0.474052406563324 + - type: nauc_recall_at_1000_diff1 + value: 14.119118543241923 + - type: nauc_recall_at_1000_max + value: 21.35495906914254 + - type: nauc_recall_at_1000_std + value: 26.622579983732226 + - type: nauc_recall_at_100_diff1 + value: 20.693908776479706 + - type: nauc_recall_at_100_max + value: 22.058743353297373 + - type: nauc_recall_at_100_std + value: 14.65896882307795 + - type: nauc_recall_at_10_diff1 + value: 24.298145054404692 + - type: nauc_recall_at_10_max + value: 23.87028352436605 + - type: nauc_recall_at_10_std + value: 3.361268820025586 + - type: nauc_recall_at_1_diff1 + value: 40.91264195434544 + - type: nauc_recall_at_1_max + value: 25.107414759912196 + - type: nauc_recall_at_1_std + value: -4.238206964379598 + - type: nauc_recall_at_20_diff1 + value: 22.983966955981174 + - type: nauc_recall_at_20_max + value: 22.746347029229046 + - type: nauc_recall_at_20_std + value: 6.274536201444075 + - type: nauc_recall_at_3_diff1 + value: 29.928123909565663 + - type: nauc_recall_at_3_max + value: 23.87162736435659 + - type: nauc_recall_at_3_std + value: 0.19477697246241965 + - type: nauc_recall_at_5_diff1 + value: 26.676061459659667 + - type: nauc_recall_at_5_max + value: 24.542080227532317 + - type: nauc_recall_at_5_std + value: 0.6827233030307879 + - type: ndcg_at_1 + value: 17.412 + - type: ndcg_at_10 + value: 24.492 + - type: ndcg_at_100 + value: 29.682 + - type: ndcg_at_1000 + value: 32.861000000000004 + - type: ndcg_at_20 + value: 26.418999999999997 + - type: ndcg_at_3 + value: 20.742 + - type: ndcg_at_5 + value: 22.588 + - type: precision_at_1 + value: 17.412 + - type: precision_at_10 + value: 4.504 + - type: precision_at_100 + value: 0.826 + - type: precision_at_1000 + value: 0.127 + - type: precision_at_20 + value: 2.786 + - type: precision_at_3 + value: 9.853000000000002 + - type: precision_at_5 + value: 7.24 + - type: recall_at_1 + value: 14.292 + - type: recall_at_10 + value: 33.432 + - type: recall_at_100 + value: 57.071000000000005 + - type: recall_at_1000 + value: 79.769 + - type: recall_at_20 + value: 40.634 + - type: recall_at_3 + value: 22.927 + - type: recall_at_5 + value: 27.712999999999997 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackUnixRetrieval (default) + revision: 6c6430d3a6d36f8d2a829195bc5dc94d7e063e53 + split: test + type: mteb/cqadupstack-unix + metrics: + - type: main_score + value: 37.439 + - type: map_at_1 + value: 24.923000000000002 + - type: map_at_10 + value: 32.647 + - type: map_at_100 + value: 33.804 + - type: map_at_1000 + value: 33.927 + - type: map_at_20 + value: 33.263 + - type: map_at_3 + value: 29.857 + - type: map_at_5 + value: 31.476 + - type: mrr_at_1 + value: 28.638059701492537 + - type: mrr_at_10 + value: 36.17030176498454 + - type: mrr_at_100 + value: 37.10060893852815 + - type: mrr_at_1000 + value: 37.17548149633938 + - type: mrr_at_20 + value: 36.69176686594067 + - type: mrr_at_3 + value: 33.72201492537312 + - type: mrr_at_5 + value: 35.26585820895517 + - type: nauc_map_at_1000_diff1 + value: 48.30095671615492 + - type: nauc_map_at_1000_max + value: 32.14571207795211 + - type: nauc_map_at_1000_std + value: -5.313381735612477 + - type: nauc_map_at_100_diff1 + value: 48.313469118117666 + - type: nauc_map_at_100_max + value: 32.12214732965732 + - type: nauc_map_at_100_std + value: -5.336656359244637 + - type: nauc_map_at_10_diff1 + value: 48.50584732553582 + - type: nauc_map_at_10_max + value: 31.76705669266789 + - type: nauc_map_at_10_std + value: -5.738455146249136 + - type: nauc_map_at_1_diff1 + value: 54.340322742093235 + - type: nauc_map_at_1_max + value: 32.932186507473716 + - type: nauc_map_at_1_std + value: -8.60811863166111 + - type: nauc_map_at_20_diff1 + value: 48.33222474283193 + - type: nauc_map_at_20_max + value: 32.04064448794949 + - type: nauc_map_at_20_std + value: -5.528961962374197 + - type: nauc_map_at_3_diff1 + value: 49.28843087601103 + - type: nauc_map_at_3_max + value: 32.06998209758247 + - type: nauc_map_at_3_std + value: -6.5150265078480665 + - type: nauc_map_at_5_diff1 + value: 48.71204737965494 + - type: nauc_map_at_5_max + value: 32.18531774936822 + - type: nauc_map_at_5_std + value: -6.146900356528973 + - type: nauc_mrr_at_1000_diff1 + value: 48.00423333393432 + - type: nauc_mrr_at_1000_max + value: 33.320318046531504 + - type: nauc_mrr_at_1000_std + value: -5.29460579367888 + - type: nauc_mrr_at_100_diff1 + value: 47.99792705566481 + - type: nauc_mrr_at_100_max + value: 33.306833624526305 + - type: nauc_mrr_at_100_std + value: -5.3058269587706155 + - type: nauc_mrr_at_10_diff1 + value: 48.021680214174204 + - type: nauc_mrr_at_10_max + value: 33.13796213034413 + - type: nauc_mrr_at_10_std + value: -5.533038283575727 + - type: nauc_mrr_at_1_diff1 + value: 54.941779639365286 + - type: nauc_mrr_at_1_max + value: 34.9046666902043 + - type: nauc_mrr_at_1_std + value: -8.095236779418917 + - type: nauc_mrr_at_20_diff1 + value: 47.89785380396409 + - type: nauc_mrr_at_20_max + value: 33.26330812739873 + - type: nauc_mrr_at_20_std + value: -5.2966309484325365 + - type: nauc_mrr_at_3_diff1 + value: 48.51073683004861 + - type: nauc_mrr_at_3_max + value: 33.49211414455151 + - type: nauc_mrr_at_3_std + value: -6.2139805296187305 + - type: nauc_mrr_at_5_diff1 + value: 48.07983994995121 + - type: nauc_mrr_at_5_max + value: 33.59833022595908 + - type: nauc_mrr_at_5_std + value: -5.6727714888381096 + - type: nauc_ndcg_at_1000_diff1 + value: 45.87602395111172 + - type: nauc_ndcg_at_1000_max + value: 32.46364188391631 + - type: nauc_ndcg_at_1000_std + value: -2.4724877651919757 + - type: nauc_ndcg_at_100_diff1 + value: 45.68889689518964 + - type: nauc_ndcg_at_100_max + value: 32.22235094753741 + - type: nauc_ndcg_at_100_std + value: -2.497592959166756 + - type: nauc_ndcg_at_10_diff1 + value: 46.00785703894346 + - type: nauc_ndcg_at_10_max + value: 31.160658007982477 + - type: nauc_ndcg_at_10_std + value: -4.392666983417567 + - type: nauc_ndcg_at_1_diff1 + value: 54.941779639365286 + - type: nauc_ndcg_at_1_max + value: 34.9046666902043 + - type: nauc_ndcg_at_1_std + value: -8.095236779418917 + - type: nauc_ndcg_at_20_diff1 + value: 45.45452397149595 + - type: nauc_ndcg_at_20_max + value: 31.916734049691403 + - type: nauc_ndcg_at_20_std + value: -3.52213495990334 + - type: nauc_ndcg_at_3_diff1 + value: 47.07696577513113 + - type: nauc_ndcg_at_3_max + value: 32.22730009640157 + - type: nauc_ndcg_at_3_std + value: -5.481375273861623 + - type: nauc_ndcg_at_5_diff1 + value: 46.30522384858372 + - type: nauc_ndcg_at_5_max + value: 32.185465277837736 + - type: nauc_ndcg_at_5_std + value: -4.998929945649875 + - type: nauc_precision_at_1000_diff1 + value: -8.568501807192467 + - type: nauc_precision_at_1000_max + value: 2.4692314705407314 + - type: nauc_precision_at_1000_std + value: 4.813424836195821 + - type: nauc_precision_at_100_diff1 + value: 7.4117756716258105 + - type: nauc_precision_at_100_max + value: 15.932994507950504 + - type: nauc_precision_at_100_std + value: 8.256600547365489 + - type: nauc_precision_at_10_diff1 + value: 25.872601239073408 + - type: nauc_precision_at_10_max + value: 23.61257206391287 + - type: nauc_precision_at_10_std + value: 0.8883286817063326 + - type: nauc_precision_at_1_diff1 + value: 54.941779639365286 + - type: nauc_precision_at_1_max + value: 34.9046666902043 + - type: nauc_precision_at_1_std + value: -8.095236779418917 + - type: nauc_precision_at_20_diff1 + value: 18.742233859964088 + - type: nauc_precision_at_20_max + value: 23.54694539506335 + - type: nauc_precision_at_20_std + value: 3.5389387501976994 + - type: nauc_precision_at_3_diff1 + value: 37.63476918226607 + - type: nauc_precision_at_3_max + value: 30.231886238656998 + - type: nauc_precision_at_3_std + value: -2.094985663960558 + - type: nauc_precision_at_5_diff1 + value: 32.75363772164373 + - type: nauc_precision_at_5_max + value: 29.378091009461155 + - type: nauc_precision_at_5_std + value: -0.17631428642006344 + - type: nauc_recall_at_1000_diff1 + value: 26.747102344289146 + - type: nauc_recall_at_1000_max + value: 30.46476094621769 + - type: nauc_recall_at_1000_std + value: 35.81853768566178 + - type: nauc_recall_at_100_diff1 + value: 34.04849593257575 + - type: nauc_recall_at_100_max + value: 28.663268275638877 + - type: nauc_recall_at_100_std + value: 11.711424681536876 + - type: nauc_recall_at_10_diff1 + value: 37.83993365748118 + - type: nauc_recall_at_10_max + value: 26.29938111360548 + - type: nauc_recall_at_10_std + value: -0.6696450412433098 + - type: nauc_recall_at_1_diff1 + value: 54.340322742093235 + - type: nauc_recall_at_1_max + value: 32.932186507473716 + - type: nauc_recall_at_1_std + value: -8.60811863166111 + - type: nauc_recall_at_20_diff1 + value: 35.3029731655994 + - type: nauc_recall_at_20_max + value: 28.518869416412578 + - type: nauc_recall_at_20_std + value: 2.992245421131814 + - type: nauc_recall_at_3_diff1 + value: 41.98653465070893 + - type: nauc_recall_at_3_max + value: 30.1211948490127 + - type: nauc_recall_at_3_std + value: -4.005789853665969 + - type: nauc_recall_at_5_diff1 + value: 39.06609334880806 + - type: nauc_recall_at_5_max + value: 29.61576335631887 + - type: nauc_recall_at_5_std + value: -2.3994275743042524 + - type: ndcg_at_1 + value: 28.638 + - type: ndcg_at_10 + value: 37.439 + - type: ndcg_at_100 + value: 42.872 + - type: ndcg_at_1000 + value: 45.586 + - type: ndcg_at_20 + value: 39.451 + - type: ndcg_at_3 + value: 32.461 + - type: ndcg_at_5 + value: 34.999 + - type: precision_at_1 + value: 28.638 + - type: precision_at_10 + value: 6.334 + - type: precision_at_100 + value: 1.006 + - type: precision_at_1000 + value: 0.136 + - type: precision_at_20 + value: 3.7310000000000003 + - type: precision_at_3 + value: 14.366000000000001 + - type: precision_at_5 + value: 10.448 + - type: recall_at_1 + value: 24.923000000000002 + - type: recall_at_10 + value: 48.327999999999996 + - type: recall_at_100 + value: 72.34 + - type: recall_at_1000 + value: 91.143 + - type: recall_at_20 + value: 55.594 + - type: recall_at_3 + value: 35.052 + - type: recall_at_5 + value: 41.421 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackWebmastersRetrieval (default) + revision: 160c094312a0e1facb97e55eeddb698c0abe3571 + split: test + type: mteb/cqadupstack-webmasters + metrics: + - type: main_score + value: 40.971999999999994 + - type: map_at_1 + value: 25.819 + - type: map_at_10 + value: 34.852 + - type: map_at_100 + value: 36.46 + - type: map_at_1000 + value: 36.675000000000004 + - type: map_at_20 + value: 35.625 + - type: map_at_3 + value: 31.695 + - type: map_at_5 + value: 33.287 + - type: mrr_at_1 + value: 31.225296442687743 + - type: mrr_at_10 + value: 39.80786122090468 + - type: mrr_at_100 + value: 40.76982919865368 + - type: mrr_at_1000 + value: 40.82069922798057 + - type: mrr_at_20 + value: 40.266891698680794 + - type: mrr_at_3 + value: 37.12121212121212 + - type: mrr_at_5 + value: 38.55401844532279 + - type: nauc_map_at_1000_diff1 + value: 44.41832334091821 + - type: nauc_map_at_1000_max + value: 28.590682727865 + - type: nauc_map_at_1000_std + value: 9.883491043313123 + - type: nauc_map_at_100_diff1 + value: 44.54296031654382 + - type: nauc_map_at_100_max + value: 28.742103134917024 + - type: nauc_map_at_100_std + value: 9.733062831449288 + - type: nauc_map_at_10_diff1 + value: 44.70007560322151 + - type: nauc_map_at_10_max + value: 27.83713399644896 + - type: nauc_map_at_10_std + value: 8.048213669508982 + - type: nauc_map_at_1_diff1 + value: 51.64770324793262 + - type: nauc_map_at_1_max + value: 28.39051256372256 + - type: nauc_map_at_1_std + value: 5.755403543795948 + - type: nauc_map_at_20_diff1 + value: 44.63367902086573 + - type: nauc_map_at_20_max + value: 28.562353073000896 + - type: nauc_map_at_20_std + value: 8.869112860461934 + - type: nauc_map_at_3_diff1 + value: 45.83400663886595 + - type: nauc_map_at_3_max + value: 26.662955929989874 + - type: nauc_map_at_3_std + value: 5.4477974551366835 + - type: nauc_map_at_5_diff1 + value: 45.628471368524465 + - type: nauc_map_at_5_max + value: 28.081835901832235 + - type: nauc_map_at_5_std + value: 7.210196688831867 + - type: nauc_mrr_at_1000_diff1 + value: 42.33448473232043 + - type: nauc_mrr_at_1000_max + value: 28.188966177260827 + - type: nauc_mrr_at_1000_std + value: 11.044260896044007 + - type: nauc_mrr_at_100_diff1 + value: 42.332679947562355 + - type: nauc_mrr_at_100_max + value: 28.189831437939134 + - type: nauc_mrr_at_100_std + value: 11.062792948764583 + - type: nauc_mrr_at_10_diff1 + value: 42.17413684059611 + - type: nauc_mrr_at_10_max + value: 27.88821469571922 + - type: nauc_mrr_at_10_std + value: 10.459587578196137 + - type: nauc_mrr_at_1_diff1 + value: 48.849453022532266 + - type: nauc_mrr_at_1_max + value: 29.16717689397821 + - type: nauc_mrr_at_1_std + value: 9.214255236404052 + - type: nauc_mrr_at_20_diff1 + value: 42.261619452704366 + - type: nauc_mrr_at_20_max + value: 28.20674776233738 + - type: nauc_mrr_at_20_std + value: 10.902233065072556 + - type: nauc_mrr_at_3_diff1 + value: 42.7628693699471 + - type: nauc_mrr_at_3_max + value: 27.49303475755309 + - type: nauc_mrr_at_3_std + value: 9.431618611139278 + - type: nauc_mrr_at_5_diff1 + value: 42.69000874569754 + - type: nauc_mrr_at_5_max + value: 28.25921590909174 + - type: nauc_mrr_at_5_std + value: 10.353652589791503 + - type: nauc_ndcg_at_1000_diff1 + value: 41.51368691269013 + - type: nauc_ndcg_at_1000_max + value: 28.87279839879139 + - type: nauc_ndcg_at_1000_std + value: 13.828211548096123 + - type: nauc_ndcg_at_100_diff1 + value: 41.547755924192074 + - type: nauc_ndcg_at_100_max + value: 28.991867288106977 + - type: nauc_ndcg_at_100_std + value: 14.391284248603393 + - type: nauc_ndcg_at_10_diff1 + value: 41.70444627590904 + - type: nauc_ndcg_at_10_max + value: 27.410552487346862 + - type: nauc_ndcg_at_10_std + value: 11.395784155275281 + - type: nauc_ndcg_at_1_diff1 + value: 48.849453022532266 + - type: nauc_ndcg_at_1_max + value: 29.16717689397821 + - type: nauc_ndcg_at_1_std + value: 9.214255236404052 + - type: nauc_ndcg_at_20_diff1 + value: 41.54554883052188 + - type: nauc_ndcg_at_20_max + value: 28.758808127596286 + - type: nauc_ndcg_at_20_std + value: 12.33602670046564 + - type: nauc_ndcg_at_3_diff1 + value: 43.168643123233046 + - type: nauc_ndcg_at_3_max + value: 26.755462747248558 + - type: nauc_ndcg_at_3_std + value: 8.254132602200425 + - type: nauc_ndcg_at_5_diff1 + value: 43.25566403171218 + - type: nauc_ndcg_at_5_max + value: 28.15169745024929 + - type: nauc_ndcg_at_5_std + value: 10.562950196094619 + - type: nauc_precision_at_1000_diff1 + value: -17.25268934755889 + - type: nauc_precision_at_1000_max + value: -16.594992756235026 + - type: nauc_precision_at_1000_std + value: 18.988240250966065 + - type: nauc_precision_at_100_diff1 + value: -6.4080172471828165 + - type: nauc_precision_at_100_max + value: 1.549372957534864 + - type: nauc_precision_at_100_std + value: 28.156677811579232 + - type: nauc_precision_at_10_diff1 + value: 13.499382364692746 + - type: nauc_precision_at_10_max + value: 20.203453673714588 + - type: nauc_precision_at_10_std + value: 21.354578818956327 + - type: nauc_precision_at_1_diff1 + value: 48.849453022532266 + - type: nauc_precision_at_1_max + value: 29.16717689397821 + - type: nauc_precision_at_1_std + value: 9.214255236404052 + - type: nauc_precision_at_20_diff1 + value: 8.54499607905094 + - type: nauc_precision_at_20_max + value: 21.5565916729337 + - type: nauc_precision_at_20_std + value: 26.122000885611936 + - type: nauc_precision_at_3_diff1 + value: 29.07245212734616 + - type: nauc_precision_at_3_max + value: 23.470614687640065 + - type: nauc_precision_at_3_std + value: 11.886442567033525 + - type: nauc_precision_at_5_diff1 + value: 23.16544739935118 + - type: nauc_precision_at_5_max + value: 24.888106744984942 + - type: nauc_precision_at_5_std + value: 17.44205064556238 + - type: nauc_recall_at_1000_diff1 + value: 21.59276697504114 + - type: nauc_recall_at_1000_max + value: 31.904473132298428 + - type: nauc_recall_at_1000_std + value: 59.536504255317546 + - type: nauc_recall_at_100_diff1 + value: 29.503736280541343 + - type: nauc_recall_at_100_max + value: 27.015971784145883 + - type: nauc_recall_at_100_std + value: 38.12485785717784 + - type: nauc_recall_at_10_diff1 + value: 32.89780700786117 + - type: nauc_recall_at_10_max + value: 22.950434042715205 + - type: nauc_recall_at_10_std + value: 13.685256300826348 + - type: nauc_recall_at_1_diff1 + value: 51.64770324793262 + - type: nauc_recall_at_1_max + value: 28.39051256372256 + - type: nauc_recall_at_1_std + value: 5.755403543795948 + - type: nauc_recall_at_20_diff1 + value: 31.27470699988394 + - type: nauc_recall_at_20_max + value: 26.86011801452227 + - type: nauc_recall_at_20_std + value: 18.89768460393433 + - type: nauc_recall_at_3_diff1 + value: 39.046051560675274 + - type: nauc_recall_at_3_max + value: 22.28587309217963 + - type: nauc_recall_at_3_std + value: 5.194800798991162 + - type: nauc_recall_at_5_diff1 + value: 38.7757269231933 + - type: nauc_recall_at_5_max + value: 25.904511141768978 + - type: nauc_recall_at_5_std + value: 10.24976063323478 + - type: ndcg_at_1 + value: 31.225 + - type: ndcg_at_10 + value: 40.971999999999994 + - type: ndcg_at_100 + value: 46.732 + - type: ndcg_at_1000 + value: 49.120999999999995 + - type: ndcg_at_20 + value: 42.829 + - type: ndcg_at_3 + value: 35.787 + - type: ndcg_at_5 + value: 37.937 + - type: precision_at_1 + value: 31.225 + - type: precision_at_10 + value: 8.043 + - type: precision_at_100 + value: 1.5630000000000002 + - type: precision_at_1000 + value: 0.23800000000000002 + - type: precision_at_20 + value: 4.931 + - type: precision_at_3 + value: 16.864 + - type: precision_at_5 + value: 12.332 + - type: recall_at_1 + value: 25.819 + - type: recall_at_10 + value: 52.62 + - type: recall_at_100 + value: 78.245 + - type: recall_at_1000 + value: 93.435 + - type: recall_at_20 + value: 59.765 + - type: recall_at_3 + value: 37.688 + - type: recall_at_5 + value: 43.187999999999995 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackWordpressRetrieval (default) + revision: 4ffe81d471b1924886b33c7567bfb200e9eec5c4 + split: test + type: mteb/cqadupstack-wordpress + metrics: + - type: main_score + value: 30.389 + - type: map_at_1 + value: 16.771 + - type: map_at_10 + value: 25.207 + - type: map_at_100 + value: 26.228 + - type: map_at_1000 + value: 26.331 + - type: map_at_20 + value: 25.784000000000002 + - type: map_at_3 + value: 22.372 + - type: map_at_5 + value: 24.092 + - type: mrr_at_1 + value: 18.11460258780037 + - type: mrr_at_10 + value: 26.978992459583946 + - type: mrr_at_100 + value: 27.8793216583687 + - type: mrr_at_1000 + value: 27.953238807610255 + - type: mrr_at_20 + value: 27.499613148322965 + - type: mrr_at_3 + value: 24.245224892174978 + - type: mrr_at_5 + value: 25.862600123228585 + - type: nauc_map_at_1000_diff1 + value: 24.04814927347588 + - type: nauc_map_at_1000_max + value: 10.238701398269171 + - type: nauc_map_at_1000_std + value: -2.693382357147996 + - type: nauc_map_at_100_diff1 + value: 24.04384809427671 + - type: nauc_map_at_100_max + value: 10.27271113851825 + - type: nauc_map_at_100_std + value: -2.6726278148876226 + - type: nauc_map_at_10_diff1 + value: 24.22196472158768 + - type: nauc_map_at_10_max + value: 10.266391885856105 + - type: nauc_map_at_10_std + value: -3.47547014318069 + - type: nauc_map_at_1_diff1 + value: 35.10765155338275 + - type: nauc_map_at_1_max + value: 12.91884814701213 + - type: nauc_map_at_1_std + value: -6.037760610507437 + - type: nauc_map_at_20_diff1 + value: 24.080051454796035 + - type: nauc_map_at_20_max + value: 10.245571431559112 + - type: nauc_map_at_20_std + value: -3.236956459674347 + - type: nauc_map_at_3_diff1 + value: 26.490728083470472 + - type: nauc_map_at_3_max + value: 11.976787701635484 + - type: nauc_map_at_3_std + value: -2.5657073987758694 + - type: nauc_map_at_5_diff1 + value: 25.21072506230328 + - type: nauc_map_at_5_max + value: 10.62317033427519 + - type: nauc_map_at_5_std + value: -3.25537153809809 + - type: nauc_mrr_at_1000_diff1 + value: 23.728787172359166 + - type: nauc_mrr_at_1000_max + value: 10.017916714660302 + - type: nauc_mrr_at_1000_std + value: -0.811252652952945 + - type: nauc_mrr_at_100_diff1 + value: 23.686070874531737 + - type: nauc_mrr_at_100_max + value: 10.005311881820392 + - type: nauc_mrr_at_100_std + value: -0.7779645655320664 + - type: nauc_mrr_at_10_diff1 + value: 23.67796038320212 + - type: nauc_mrr_at_10_max + value: 9.94860885511836 + - type: nauc_mrr_at_10_std + value: -1.268683638693139 + - type: nauc_mrr_at_1_diff1 + value: 34.363701987967445 + - type: nauc_mrr_at_1_max + value: 13.776484599881666 + - type: nauc_mrr_at_1_std + value: -4.408306823968417 + - type: nauc_mrr_at_20_diff1 + value: 23.631697208751973 + - type: nauc_mrr_at_20_max + value: 10.023696131050706 + - type: nauc_mrr_at_20_std + value: -1.1402724592127906 + - type: nauc_mrr_at_3_diff1 + value: 25.699300483314317 + - type: nauc_mrr_at_3_max + value: 12.10361567398046 + - type: nauc_mrr_at_3_std + value: -0.4015531816352707 + - type: nauc_mrr_at_5_diff1 + value: 24.603801176485334 + - type: nauc_mrr_at_5_max + value: 10.345741179557379 + - type: nauc_mrr_at_5_std + value: -0.9959320338172304 + - type: nauc_ndcg_at_1000_diff1 + value: 20.08808262424723 + - type: nauc_ndcg_at_1000_max + value: 8.222087624049657 + - type: nauc_ndcg_at_1000_std + value: 1.327525904130526 + - type: nauc_ndcg_at_100_diff1 + value: 19.465109238912508 + - type: nauc_ndcg_at_100_max + value: 8.2067398378089 + - type: nauc_ndcg_at_100_std + value: 1.6138162865671664 + - type: nauc_ndcg_at_10_diff1 + value: 19.981283960490448 + - type: nauc_ndcg_at_10_max + value: 7.757982560160586 + - type: nauc_ndcg_at_10_std + value: -2.357062554063349 + - type: nauc_ndcg_at_1_diff1 + value: 34.363701987967445 + - type: nauc_ndcg_at_1_max + value: 13.776484599881666 + - type: nauc_ndcg_at_1_std + value: -4.408306823968417 + - type: nauc_ndcg_at_20_diff1 + value: 19.661404661620548 + - type: nauc_ndcg_at_20_max + value: 7.900555901097286 + - type: nauc_ndcg_at_20_std + value: -1.6703428388289487 + - type: nauc_ndcg_at_3_diff1 + value: 24.10426250093583 + - type: nauc_ndcg_at_3_max + value: 11.344636656749707 + - type: nauc_ndcg_at_3_std + value: -0.6130768236450457 + - type: nauc_ndcg_at_5_diff1 + value: 22.27115670375605 + - type: nauc_ndcg_at_5_max + value: 8.704689152279247 + - type: nauc_ndcg_at_5_std + value: -2.006969370686767 + - type: nauc_precision_at_1000_diff1 + value: -19.487100252861033 + - type: nauc_precision_at_1000_max + value: -13.371854365731187 + - type: nauc_precision_at_1000_std + value: 12.824955606499753 + - type: nauc_precision_at_100_diff1 + value: -7.648182326484629 + - type: nauc_precision_at_100_max + value: -0.5460939740053596 + - type: nauc_precision_at_100_std + value: 20.40810859031953 + - type: nauc_precision_at_10_diff1 + value: 4.685552694300163 + - type: nauc_precision_at_10_max + value: 2.9918102543535845 + - type: nauc_precision_at_10_std + value: 2.4858209000809803 + - type: nauc_precision_at_1_diff1 + value: 34.363701987967445 + - type: nauc_precision_at_1_max + value: 13.776484599881666 + - type: nauc_precision_at_1_std + value: -4.408306823968417 + - type: nauc_precision_at_20_diff1 + value: 1.1006984323376434 + - type: nauc_precision_at_20_max + value: 2.84531410406988 + - type: nauc_precision_at_20_std + value: 6.011220421468565 + - type: nauc_precision_at_3_diff1 + value: 17.490939346736646 + - type: nauc_precision_at_3_max + value: 10.492514766447565 + - type: nauc_precision_at_3_std + value: 4.192766305848096 + - type: nauc_precision_at_5_diff1 + value: 11.96443944627016 + - type: nauc_precision_at_5_max + value: 6.182363288089504 + - type: nauc_precision_at_5_std + value: 2.4895654024365417 + - type: nauc_recall_at_1000_diff1 + value: 2.2275808053289916 + - type: nauc_recall_at_1000_max + value: 1.5376259594744983 + - type: nauc_recall_at_1000_std + value: 31.684523779954805 + - type: nauc_recall_at_100_diff1 + value: 4.321971811102423 + - type: nauc_recall_at_100_max + value: 2.468310623042368 + - type: nauc_recall_at_100_std + value: 17.72871230769181 + - type: nauc_recall_at_10_diff1 + value: 9.240379121559593 + - type: nauc_recall_at_10_max + value: 1.0883166828845507 + - type: nauc_recall_at_10_std + value: -1.5638033036017451 + - type: nauc_recall_at_1_diff1 + value: 35.10765155338275 + - type: nauc_recall_at_1_max + value: 12.91884814701213 + - type: nauc_recall_at_1_std + value: -6.037760610507437 + - type: nauc_recall_at_20_diff1 + value: 7.97928372102725 + - type: nauc_recall_at_20_max + value: 1.4798555210595863 + - type: nauc_recall_at_20_std + value: 0.3927790632899301 + - type: nauc_recall_at_3_diff1 + value: 18.880980538524206 + - type: nauc_recall_at_3_max + value: 9.150408391199012 + - type: nauc_recall_at_3_std + value: 1.9667650269595802 + - type: nauc_recall_at_5_diff1 + value: 15.521193207426704 + - type: nauc_recall_at_5_max + value: 3.7230515953301215 + - type: nauc_recall_at_5_std + value: -0.9105133696356293 + - type: ndcg_at_1 + value: 18.115000000000002 + - type: ndcg_at_10 + value: 30.389 + - type: ndcg_at_100 + value: 35.477 + - type: ndcg_at_1000 + value: 37.994 + - type: ndcg_at_20 + value: 32.336 + - type: ndcg_at_3 + value: 24.77 + - type: ndcg_at_5 + value: 27.699 + - type: precision_at_1 + value: 18.115000000000002 + - type: precision_at_10 + value: 5.157 + - type: precision_at_100 + value: 0.843 + - type: precision_at_1000 + value: 0.11499999999999999 + - type: precision_at_20 + value: 3.0589999999999997 + - type: precision_at_3 + value: 11.029 + - type: precision_at_5 + value: 8.355 + - type: recall_at_1 + value: 16.771 + - type: recall_at_10 + value: 44.629999999999995 + - type: recall_at_100 + value: 68.181 + - type: recall_at_1000 + value: 87.009 + - type: recall_at_20 + value: 51.9 + - type: recall_at_3 + value: 29.593000000000004 + - type: recall_at_5 + value: 36.571 + task: + type: Retrieval + - dataset: + config: default + name: MTEB ClimateFEVER (default) + revision: 47f2ac6acb640fc46020b02a5b59fdda04d39380 + split: test + type: mteb/climate-fever + metrics: + - type: main_score + value: 29.046 + - type: map_at_1 + value: 11.463 + - type: map_at_10 + value: 20.195 + - type: map_at_100 + value: 22.304 + - type: map_at_1000 + value: 22.471 + - type: map_at_20 + value: 21.447 + - type: map_at_3 + value: 16.271 + - type: map_at_5 + value: 18.248 + - type: mrr_at_1 + value: 25.01628664495114 + - type: mrr_at_10 + value: 37.63735070575461 + - type: mrr_at_100 + value: 38.50746938708273 + - type: mrr_at_1000 + value: 38.541762330715585 + - type: mrr_at_20 + value: 38.20921027420054 + - type: mrr_at_3 + value: 33.38762214983708 + - type: mrr_at_5 + value: 35.98045602605858 + - type: nauc_map_at_1000_diff1 + value: 19.507654521207428 + - type: nauc_map_at_1000_max + value: 31.46439173085021 + - type: nauc_map_at_1000_std + value: 13.409561984400517 + - type: nauc_map_at_100_diff1 + value: 19.50817254934652 + - type: nauc_map_at_100_max + value: 31.48635658390998 + - type: nauc_map_at_100_std + value: 13.399098672938612 + - type: nauc_map_at_10_diff1 + value: 19.559679731086323 + - type: nauc_map_at_10_max + value: 30.851483608248692 + - type: nauc_map_at_10_std + value: 12.09833006291563 + - type: nauc_map_at_1_diff1 + value: 27.262086152131054 + - type: nauc_map_at_1_max + value: 26.72800697130423 + - type: nauc_map_at_1_std + value: 3.110933137884969 + - type: nauc_map_at_20_diff1 + value: 19.387879957575045 + - type: nauc_map_at_20_max + value: 31.294670275708597 + - type: nauc_map_at_20_std + value: 13.287046356530464 + - type: nauc_map_at_3_diff1 + value: 20.818164913762388 + - type: nauc_map_at_3_max + value: 27.99114219292798 + - type: nauc_map_at_3_std + value: 7.543272315623365 + - type: nauc_map_at_5_diff1 + value: 20.039927104549545 + - type: nauc_map_at_5_max + value: 28.647877157240693 + - type: nauc_map_at_5_std + value: 8.443346117567257 + - type: nauc_mrr_at_1000_diff1 + value: 16.92297635760836 + - type: nauc_mrr_at_1000_max + value: 28.068248889248014 + - type: nauc_mrr_at_1000_std + value: 12.637743512593394 + - type: nauc_mrr_at_100_diff1 + value: 16.919051402642268 + - type: nauc_mrr_at_100_max + value: 28.083709908313974 + - type: nauc_mrr_at_100_std + value: 12.659919475936485 + - type: nauc_mrr_at_10_diff1 + value: 16.879497360631376 + - type: nauc_mrr_at_10_max + value: 28.221975538679867 + - type: nauc_mrr_at_10_std + value: 12.80935873980007 + - type: nauc_mrr_at_1_diff1 + value: 22.696805730346043 + - type: nauc_mrr_at_1_max + value: 26.089179648556904 + - type: nauc_mrr_at_1_std + value: 5.3578289522563365 + - type: nauc_mrr_at_20_diff1 + value: 16.822019965659923 + - type: nauc_mrr_at_20_max + value: 28.157297390210623 + - type: nauc_mrr_at_20_std + value: 12.836601180954052 + - type: nauc_mrr_at_3_diff1 + value: 16.631505043529117 + - type: nauc_mrr_at_3_max + value: 27.085963528774297 + - type: nauc_mrr_at_3_std + value: 11.328628874846704 + - type: nauc_mrr_at_5_diff1 + value: 16.370831216591984 + - type: nauc_mrr_at_5_max + value: 27.28590913517736 + - type: nauc_mrr_at_5_std + value: 12.08474307799185 + - type: nauc_ndcg_at_1000_diff1 + value: 16.74422460610516 + - type: nauc_ndcg_at_1000_max + value: 32.24430599944698 + - type: nauc_ndcg_at_1000_std + value: 18.73622515200571 + - type: nauc_ndcg_at_100_diff1 + value: 16.970449416556733 + - type: nauc_ndcg_at_100_max + value: 32.89586008508324 + - type: nauc_ndcg_at_100_std + value: 18.760299289777866 + - type: nauc_ndcg_at_10_diff1 + value: 16.809172470951093 + - type: nauc_ndcg_at_10_max + value: 32.24150506439938 + - type: nauc_ndcg_at_10_std + value: 16.68933148863763 + - type: nauc_ndcg_at_1_diff1 + value: 22.696805730346043 + - type: nauc_ndcg_at_1_max + value: 26.089179648556904 + - type: nauc_ndcg_at_1_std + value: 5.3578289522563365 + - type: nauc_ndcg_at_20_diff1 + value: 16.467671642323808 + - type: nauc_ndcg_at_20_max + value: 32.852857561929305 + - type: nauc_ndcg_at_20_std + value: 18.790450091911627 + - type: nauc_ndcg_at_3_diff1 + value: 17.523437274115565 + - type: nauc_ndcg_at_3_max + value: 27.897220626307572 + - type: nauc_ndcg_at_3_std + value: 10.421056820759153 + - type: nauc_ndcg_at_5_diff1 + value: 16.859910947145252 + - type: nauc_ndcg_at_5_max + value: 28.57074591671292 + - type: nauc_ndcg_at_5_std + value: 11.499632316008643 + - type: nauc_precision_at_1000_diff1 + value: -3.650227052883629 + - type: nauc_precision_at_1000_max + value: 7.145147724537197 + - type: nauc_precision_at_1000_std + value: 17.296452887334564 + - type: nauc_precision_at_100_diff1 + value: 3.2079783253892358 + - type: nauc_precision_at_100_max + value: 20.499202797388676 + - type: nauc_precision_at_100_std + value: 23.07266198335745 + - type: nauc_precision_at_10_diff1 + value: 6.45217657474587 + - type: nauc_precision_at_10_max + value: 32.0112100625466 + - type: nauc_precision_at_10_std + value: 28.405070739800838 + - type: nauc_precision_at_1_diff1 + value: 22.696805730346043 + - type: nauc_precision_at_1_max + value: 26.089179648556904 + - type: nauc_precision_at_1_std + value: 5.3578289522563365 + - type: nauc_precision_at_20_diff1 + value: 4.002607627487089 + - type: nauc_precision_at_20_max + value: 28.091415543546283 + - type: nauc_precision_at_20_std + value: 29.472239790237033 + - type: nauc_precision_at_3_diff1 + value: 9.36276959538351 + - type: nauc_precision_at_3_max + value: 28.755396373466912 + - type: nauc_precision_at_3_std + value: 17.30892077938441 + - type: nauc_precision_at_5_diff1 + value: 7.21168683426709 + - type: nauc_precision_at_5_max + value: 28.61046876375105 + - type: nauc_precision_at_5_std + value: 19.227662922262294 + - type: nauc_recall_at_1000_diff1 + value: 6.161761326132134 + - type: nauc_recall_at_1000_max + value: 25.22586002178256 + - type: nauc_recall_at_1000_std + value: 29.014300471250966 + - type: nauc_recall_at_100_diff1 + value: 9.8074241423904 + - type: nauc_recall_at_100_max + value: 29.247187556552817 + - type: nauc_recall_at_100_std + value: 24.909419830858777 + - type: nauc_recall_at_10_diff1 + value: 11.005931004138086 + - type: nauc_recall_at_10_max + value: 31.425071866830784 + - type: nauc_recall_at_10_std + value: 21.389283709177406 + - type: nauc_recall_at_1_diff1 + value: 27.262086152131054 + - type: nauc_recall_at_1_max + value: 26.72800697130423 + - type: nauc_recall_at_1_std + value: 3.110933137884969 + - type: nauc_recall_at_20_diff1 + value: 9.721945529846538 + - type: nauc_recall_at_20_max + value: 31.138422443633974 + - type: nauc_recall_at_20_std + value: 25.131236002997127 + - type: nauc_recall_at_3_diff1 + value: 13.870342355168733 + - type: nauc_recall_at_3_max + value: 26.276765510292698 + - type: nauc_recall_at_3_std + value: 11.588223223237314 + - type: nauc_recall_at_5_diff1 + value: 11.709630794252144 + - type: nauc_recall_at_5_max + value: 25.60603633193481 + - type: nauc_recall_at_5_std + value: 12.4718086596069 + - type: ndcg_at_1 + value: 25.016 + - type: ndcg_at_10 + value: 29.046 + - type: ndcg_at_100 + value: 36.547000000000004 + - type: ndcg_at_1000 + value: 39.702 + - type: ndcg_at_20 + value: 32.353 + - type: ndcg_at_3 + value: 22.409000000000002 + - type: ndcg_at_5 + value: 24.945999999999998 + - type: precision_at_1 + value: 25.016 + - type: precision_at_10 + value: 9.433 + - type: precision_at_100 + value: 1.7340000000000002 + - type: precision_at_1000 + value: 0.233 + - type: precision_at_20 + value: 6.143 + - type: precision_at_3 + value: 16.569 + - type: precision_at_5 + value: 13.459 + - type: recall_at_1 + value: 11.463 + - type: recall_at_10 + value: 36.52 + - type: recall_at_100 + value: 61.648 + - type: recall_at_1000 + value: 79.464 + - type: recall_at_20 + value: 45.708 + - type: recall_at_3 + value: 20.942 + - type: recall_at_5 + value: 27.400000000000002 + task: + type: Retrieval + - dataset: + config: default + name: MTEB DBPedia (default) + revision: c0f706b76e590d620bd6618b3ca8efdd34e2d659 + split: test + type: mteb/dbpedia + metrics: + - type: main_score + value: 43.478 + - type: map_at_1 + value: 9.533 + - type: map_at_10 + value: 20.419999999999998 + - type: map_at_100 + value: 28.021 + - type: map_at_1000 + value: 29.847 + - type: map_at_20 + value: 23.352999999999998 + - type: map_at_3 + value: 14.538 + - type: map_at_5 + value: 17.02 + - type: mrr_at_1 + value: 72.0 + - type: mrr_at_10 + value: 78.87450396825396 + - type: mrr_at_100 + value: 79.11617511086757 + - type: mrr_at_1000 + value: 79.11880145542081 + - type: mrr_at_20 + value: 79.03381395973307 + - type: mrr_at_3 + value: 77.125 + - type: mrr_at_5 + value: 78.325 + - type: nauc_map_at_1000_diff1 + value: 23.57318828044527 + - type: nauc_map_at_1000_max + value: 35.89095271996008 + - type: nauc_map_at_1000_std + value: 20.973924248831587 + - type: nauc_map_at_100_diff1 + value: 23.987244820409593 + - type: nauc_map_at_100_max + value: 35.26271748774156 + - type: nauc_map_at_100_std + value: 17.892785619372955 + - type: nauc_map_at_10_diff1 + value: 26.78355123492037 + - type: nauc_map_at_10_max + value: 30.13327930151061 + - type: nauc_map_at_10_std + value: -1.8992482170330651 + - type: nauc_map_at_1_diff1 + value: 33.68916236643283 + - type: nauc_map_at_1_max + value: 24.3207019942468 + - type: nauc_map_at_1_std + value: -14.809581500087651 + - type: nauc_map_at_20_diff1 + value: 25.469736251409223 + - type: nauc_map_at_20_max + value: 32.49339898056429 + - type: nauc_map_at_20_std + value: 4.838969439128736 + - type: nauc_map_at_3_diff1 + value: 28.479418344158336 + - type: nauc_map_at_3_max + value: 27.612502069214468 + - type: nauc_map_at_3_std + value: -9.838424931303104 + - type: nauc_map_at_5_diff1 + value: 28.753200233879706 + - type: nauc_map_at_5_max + value: 28.993880753761708 + - type: nauc_map_at_5_std + value: -6.915952030386515 + - type: nauc_mrr_at_1000_diff1 + value: 55.622066341219444 + - type: nauc_mrr_at_1000_max + value: 59.78738557921911 + - type: nauc_mrr_at_1000_std + value: 38.316621490948144 + - type: nauc_mrr_at_100_diff1 + value: 55.623447857127914 + - type: nauc_mrr_at_100_max + value: 59.78956573362609 + - type: nauc_mrr_at_100_std + value: 38.324373044439525 + - type: nauc_mrr_at_10_diff1 + value: 55.67089980489662 + - type: nauc_mrr_at_10_max + value: 59.91474505352129 + - type: nauc_mrr_at_10_std + value: 38.31018306556157 + - type: nauc_mrr_at_1_diff1 + value: 55.78067765567767 + - type: nauc_mrr_at_1_max + value: 57.31360653235653 + - type: nauc_mrr_at_1_std + value: 36.0462454212454 + - type: nauc_mrr_at_20_diff1 + value: 55.553375818822715 + - type: nauc_mrr_at_20_max + value: 59.841306240314864 + - type: nauc_mrr_at_20_std + value: 38.37809451974795 + - type: nauc_mrr_at_3_diff1 + value: 54.3706467168059 + - type: nauc_mrr_at_3_max + value: 59.08341389062919 + - type: nauc_mrr_at_3_std + value: 38.251627918843546 + - type: nauc_mrr_at_5_diff1 + value: 55.95584990017196 + - type: nauc_mrr_at_5_max + value: 60.17748470726553 + - type: nauc_mrr_at_5_std + value: 38.48817979680824 + - type: nauc_ndcg_at_1000_diff1 + value: 29.753532031977688 + - type: nauc_ndcg_at_1000_max + value: 44.17138792809622 + - type: nauc_ndcg_at_1000_std + value: 35.975396673080404 + - type: nauc_ndcg_at_100_diff1 + value: 29.58964546785338 + - type: nauc_ndcg_at_100_max + value: 41.32806717304176 + - type: nauc_ndcg_at_100_std + value: 27.94930554763625 + - type: nauc_ndcg_at_10_diff1 + value: 29.759334038319736 + - type: nauc_ndcg_at_10_max + value: 44.295331428686765 + - type: nauc_ndcg_at_10_std + value: 23.33760953731664 + - type: nauc_ndcg_at_1_diff1 + value: 45.5358139480234 + - type: nauc_ndcg_at_1_max + value: 47.86459689848099 + - type: nauc_ndcg_at_1_std + value: 25.787178377489205 + - type: nauc_ndcg_at_20_diff1 + value: 28.50441656642591 + - type: nauc_ndcg_at_20_max + value: 42.060196311879714 + - type: nauc_ndcg_at_20_std + value: 21.191727258166967 + - type: nauc_ndcg_at_3_diff1 + value: 28.016520174892527 + - type: nauc_ndcg_at_3_max + value: 43.8486565828547 + - type: nauc_ndcg_at_3_std + value: 24.245058100263346 + - type: nauc_ndcg_at_5_diff1 + value: 29.296069402143193 + - type: nauc_ndcg_at_5_max + value: 44.715472695706225 + - type: nauc_ndcg_at_5_std + value: 24.063706029123967 + - type: nauc_precision_at_1000_diff1 + value: -12.916198821356243 + - type: nauc_precision_at_1000_max + value: -2.6091687316040706 + - type: nauc_precision_at_1000_std + value: 14.550381061319397 + - type: nauc_precision_at_100_diff1 + value: -7.356097121205582 + - type: nauc_precision_at_100_max + value: 12.26796837368589 + - type: nauc_precision_at_100_std + value: 42.98828966458228 + - type: nauc_precision_at_10_diff1 + value: 3.1020350837768564 + - type: nauc_precision_at_10_max + value: 25.67598769680203 + - type: nauc_precision_at_10_std + value: 37.49666437269138 + - type: nauc_precision_at_1_diff1 + value: 55.78067765567767 + - type: nauc_precision_at_1_max + value: 57.31360653235653 + - type: nauc_precision_at_1_std + value: 36.0462454212454 + - type: nauc_precision_at_20_diff1 + value: -2.0226181764518367 + - type: nauc_precision_at_20_max + value: 22.06446708307979 + - type: nauc_precision_at_20_std + value: 41.550338846169744 + - type: nauc_precision_at_3_diff1 + value: 8.181498661367101 + - type: nauc_precision_at_3_max + value: 31.698913906595106 + - type: nauc_precision_at_3_std + value: 29.52267720794877 + - type: nauc_precision_at_5_diff1 + value: 6.749425651741525 + - type: nauc_precision_at_5_max + value: 28.54392447308322 + - type: nauc_precision_at_5_std + value: 33.49699560528142 + - type: nauc_recall_at_1000_diff1 + value: 18.280838269391005 + - type: nauc_recall_at_1000_max + value: 24.510824462700857 + - type: nauc_recall_at_1000_std + value: 42.605278747690264 + - type: nauc_recall_at_100_diff1 + value: 18.449965856234087 + - type: nauc_recall_at_100_max + value: 23.71800401019445 + - type: nauc_recall_at_100_std + value: 23.414866896270784 + - type: nauc_recall_at_10_diff1 + value: 21.784314520358684 + - type: nauc_recall_at_10_max + value: 23.274507877574916 + - type: nauc_recall_at_10_std + value: -4.7758028966385435 + - type: nauc_recall_at_1_diff1 + value: 33.68916236643283 + - type: nauc_recall_at_1_max + value: 24.3207019942468 + - type: nauc_recall_at_1_std + value: -14.809581500087651 + - type: nauc_recall_at_20_diff1 + value: 19.25528082113324 + - type: nauc_recall_at_20_max + value: 24.360978914950774 + - type: nauc_recall_at_20_std + value: 2.0442058267254684 + - type: nauc_recall_at_3_diff1 + value: 24.680856858067777 + - type: nauc_recall_at_3_max + value: 23.954730032567884 + - type: nauc_recall_at_3_std + value: -10.37273906616298 + - type: nauc_recall_at_5_diff1 + value: 26.268701832826192 + - type: nauc_recall_at_5_max + value: 25.202567512147983 + - type: nauc_recall_at_5_std + value: -9.02451003373801 + - type: ndcg_at_1 + value: 60.62499999999999 + - type: ndcg_at_10 + value: 43.478 + - type: ndcg_at_100 + value: 47.415 + - type: ndcg_at_1000 + value: 55.263 + - type: ndcg_at_20 + value: 42.503 + - type: ndcg_at_3 + value: 48.157 + - type: ndcg_at_5 + value: 45.542 + - type: precision_at_1 + value: 72.0 + - type: precision_at_10 + value: 33.575 + - type: precision_at_100 + value: 10.424999999999999 + - type: precision_at_1000 + value: 2.196 + - type: precision_at_20 + value: 25.037 + - type: precision_at_3 + value: 50.5 + - type: precision_at_5 + value: 43.15 + - type: recall_at_1 + value: 9.533 + - type: recall_at_10 + value: 26.504 + - type: recall_at_100 + value: 53.157 + - type: recall_at_1000 + value: 77.228 + - type: recall_at_20 + value: 33.488 + - type: recall_at_3 + value: 15.857 + - type: recall_at_5 + value: 20.024 + task: + type: Retrieval + - dataset: + config: default + name: MTEB EmotionClassification (default) + revision: 4f58c6b202a23cf9a4da393831edf4f9183cad37 + split: test + type: mteb/emotion + metrics: + - type: accuracy + value: 46.785 + - type: f1 + value: 40.736157067189076 + - type: f1_weighted + value: 49.22859255962354 + - type: main_score + value: 46.785 + task: + type: Classification + - dataset: + config: default + name: MTEB FEVER (default) + revision: bea83ef9e8fb933d90a2f1d5515737465d613e12 + split: test + type: mteb/fever + metrics: + - type: main_score + value: 89.19 + - type: map_at_1 + value: 80.094 + - type: map_at_10 + value: 86.10799999999999 + - type: map_at_100 + value: 86.261 + - type: map_at_1000 + value: 86.272 + - type: map_at_20 + value: 86.198 + - type: map_at_3 + value: 85.455 + - type: map_at_5 + value: 85.89 + - type: mrr_at_1 + value: 86.3036303630363 + - type: mrr_at_10 + value: 91.58619433371898 + - type: mrr_at_100 + value: 91.64138433577838 + - type: mrr_at_1000 + value: 91.64163154385662 + - type: mrr_at_20 + value: 91.62463568397531 + - type: mrr_at_3 + value: 91.16411641164105 + - type: mrr_at_5 + value: 91.44464446444628 + - type: nauc_map_at_1000_diff1 + value: 44.7579438043752 + - type: nauc_map_at_1000_max + value: 21.54146596260254 + - type: nauc_map_at_1000_std + value: -13.104925986464972 + - type: nauc_map_at_100_diff1 + value: 44.726726129848274 + - type: nauc_map_at_100_max + value: 21.52817643661516 + - type: nauc_map_at_100_std + value: -13.093960461866006 + - type: nauc_map_at_10_diff1 + value: 44.63681204670047 + - type: nauc_map_at_10_max + value: 21.60607819947736 + - type: nauc_map_at_10_std + value: -13.260351207649082 + - type: nauc_map_at_1_diff1 + value: 52.90212380063111 + - type: nauc_map_at_1_max + value: 23.807162516255616 + - type: nauc_map_at_1_std + value: -14.579072924962947 + - type: nauc_map_at_20_diff1 + value: 44.6848433216619 + - type: nauc_map_at_20_max + value: 21.55005476160233 + - type: nauc_map_at_20_std + value: -13.080279816341337 + - type: nauc_map_at_3_diff1 + value: 44.91770992154857 + - type: nauc_map_at_3_max + value: 21.89808704190864 + - type: nauc_map_at_3_std + value: -13.650716102160734 + - type: nauc_map_at_5_diff1 + value: 44.70088308352055 + - type: nauc_map_at_5_max + value: 21.399888403032797 + - type: nauc_map_at_5_std + value: -13.75318234600381 + - type: nauc_mrr_at_1000_diff1 + value: 73.58384696836679 + - type: nauc_mrr_at_1000_max + value: 33.97804287673037 + - type: nauc_mrr_at_1000_std + value: -26.134425344247532 + - type: nauc_mrr_at_100_diff1 + value: 73.58355080121942 + - type: nauc_mrr_at_100_max + value: 33.978090588261786 + - type: nauc_mrr_at_100_std + value: -26.134077335250065 + - type: nauc_mrr_at_10_diff1 + value: 73.56919790566825 + - type: nauc_mrr_at_10_max + value: 34.05006861889356 + - type: nauc_mrr_at_10_std + value: -26.385093328316444 + - type: nauc_mrr_at_1_diff1 + value: 74.95645171867119 + - type: nauc_mrr_at_1_max + value: 32.82637675408828 + - type: nauc_mrr_at_1_std + value: -24.536778466433486 + - type: nauc_mrr_at_20_diff1 + value: 73.57564493573263 + - type: nauc_mrr_at_20_max + value: 34.02636582545889 + - type: nauc_mrr_at_20_std + value: -26.093421950213262 + - type: nauc_mrr_at_3_diff1 + value: 73.40668466986698 + - type: nauc_mrr_at_3_max + value: 34.67883943938702 + - type: nauc_mrr_at_3_std + value: -26.526548588749016 + - type: nauc_mrr_at_5_diff1 + value: 73.50742215693361 + - type: nauc_mrr_at_5_max + value: 33.93274451907254 + - type: nauc_mrr_at_5_std + value: -26.808596611171236 + - type: nauc_ndcg_at_1000_diff1 + value: 45.807423394488 + - type: nauc_ndcg_at_1000_max + value: 22.347317856778474 + - type: nauc_ndcg_at_1000_std + value: -13.17421507903504 + - type: nauc_ndcg_at_100_diff1 + value: 45.0493277369051 + - type: nauc_ndcg_at_100_max + value: 22.005531300792693 + - type: nauc_ndcg_at_100_std + value: -12.835048500918274 + - type: nauc_ndcg_at_10_diff1 + value: 44.78386521144556 + - type: nauc_ndcg_at_10_max + value: 22.419765542544486 + - type: nauc_ndcg_at_10_std + value: -13.582954637994415 + - type: nauc_ndcg_at_1_diff1 + value: 74.95645171867119 + - type: nauc_ndcg_at_1_max + value: 32.82637675408828 + - type: nauc_ndcg_at_1_std + value: -24.536778466433486 + - type: nauc_ndcg_at_20_diff1 + value: 44.801555858829914 + - type: nauc_ndcg_at_20_max + value: 22.191768817134005 + - type: nauc_ndcg_at_20_std + value: -12.702314555588861 + - type: nauc_ndcg_at_3_diff1 + value: 46.84378493308696 + - type: nauc_ndcg_at_3_max + value: 23.572300143781955 + - type: nauc_ndcg_at_3_std + value: -15.39318126363012 + - type: nauc_ndcg_at_5_diff1 + value: 45.441428704740034 + - type: nauc_ndcg_at_5_max + value: 22.07150886528209 + - type: nauc_ndcg_at_5_std + value: -15.107166131904659 + - type: nauc_precision_at_1000_diff1 + value: -6.708559049401239 + - type: nauc_precision_at_1000_max + value: -4.694555976973598 + - type: nauc_precision_at_1000_std + value: -0.013521786398392596 + - type: nauc_precision_at_100_diff1 + value: -8.659359973424788 + - type: nauc_precision_at_100_max + value: -5.86271322264399 + - type: nauc_precision_at_100_std + value: 1.7877140648461642 + - type: nauc_precision_at_10_diff1 + value: -1.3221091533272413 + - type: nauc_precision_at_10_max + value: 3.0971418194864304 + - type: nauc_precision_at_10_std + value: -5.894144296086863 + - type: nauc_precision_at_1_diff1 + value: 74.95645171867119 + - type: nauc_precision_at_1_max + value: 32.82637675408828 + - type: nauc_precision_at_1_std + value: -24.536778466433486 + - type: nauc_precision_at_20_diff1 + value: -5.834371936120651 + - type: nauc_precision_at_20_max + value: -1.5694242012393542 + - type: nauc_precision_at_20_std + value: 1.2551477115410459 + - type: nauc_precision_at_3_diff1 + value: 22.830780586406853 + - type: nauc_precision_at_3_max + value: 14.041172292217045 + - type: nauc_precision_at_3_std + value: -17.7624763463396 + - type: nauc_precision_at_5_diff1 + value: 9.188166810529339 + - type: nauc_precision_at_5_max + value: 4.160934309649076 + - type: nauc_precision_at_5_std + value: -15.247235106093449 + - type: nauc_recall_at_1000_diff1 + value: -9.248979573074552 + - type: nauc_recall_at_1000_max + value: 1.22683254059056 + - type: nauc_recall_at_1000_std + value: 17.82822947318761 + - type: nauc_recall_at_100_diff1 + value: -2.7743488495403823 + - type: nauc_recall_at_100_max + value: 3.006516600109272 + - type: nauc_recall_at_100_std + value: 12.988526382848796 + - type: nauc_recall_at_10_diff1 + value: 11.119581326065385 + - type: nauc_recall_at_10_max + value: 11.460697625972378 + - type: nauc_recall_at_10_std + value: 0.29982070082114604 + - type: nauc_recall_at_1_diff1 + value: 52.90212380063111 + - type: nauc_recall_at_1_max + value: 23.807162516255616 + - type: nauc_recall_at_1_std + value: -14.579072924962947 + - type: nauc_recall_at_20_diff1 + value: 6.050248111732341 + - type: nauc_recall_at_20_max + value: 8.574891553915348 + - type: nauc_recall_at_20_std + value: 8.143132557614821 + - type: nauc_recall_at_3_diff1 + value: 24.891402326018326 + - type: nauc_recall_at_3_max + value: 17.04027628364904 + - type: nauc_recall_at_3_std + value: -8.626144350391291 + - type: nauc_recall_at_5_diff1 + value: 18.482860227632898 + - type: nauc_recall_at_5_max + value: 11.714814803548023 + - type: nauc_recall_at_5_std + value: -7.632106956510073 + - type: ndcg_at_1 + value: 86.304 + - type: ndcg_at_10 + value: 89.19 + - type: ndcg_at_100 + value: 89.744 + - type: ndcg_at_1000 + value: 89.96300000000001 + - type: ndcg_at_20 + value: 89.43900000000001 + - type: ndcg_at_3 + value: 88.253 + - type: ndcg_at_5 + value: 88.777 + - type: precision_at_1 + value: 86.304 + - type: precision_at_10 + value: 10.245999999999999 + - type: precision_at_100 + value: 1.0670000000000002 + - type: precision_at_1000 + value: 0.11 + - type: precision_at_20 + value: 5.196 + - type: precision_at_3 + value: 32.903 + - type: precision_at_5 + value: 20.141000000000002 + - type: recall_at_1 + value: 80.094 + - type: recall_at_10 + value: 93.513 + - type: recall_at_100 + value: 95.705 + - type: recall_at_1000 + value: 97.074 + - type: recall_at_20 + value: 94.37299999999999 + - type: recall_at_3 + value: 90.706 + - type: recall_at_5 + value: 92.207 + task: + type: Retrieval + - dataset: + config: default + name: MTEB FiQA2018 (default) + revision: 27a168819829fe9bcd655c2df245fb19452e8e06 + split: test + type: mteb/fiqa + metrics: + - type: main_score + value: 43.562 + - type: map_at_1 + value: 22.091 + - type: map_at_10 + value: 35.802 + - type: map_at_100 + value: 37.781 + - type: map_at_1000 + value: 37.958 + - type: map_at_20 + value: 36.884 + - type: map_at_3 + value: 30.947000000000003 + - type: map_at_5 + value: 33.825 + - type: mrr_at_1 + value: 40.89506172839506 + - type: mrr_at_10 + value: 49.951927787575926 + - type: mrr_at_100 + value: 50.746308556063646 + - type: mrr_at_1000 + value: 50.78678961788158 + - type: mrr_at_20 + value: 50.41125770957138 + - type: mrr_at_3 + value: 47.14506172839506 + - type: mrr_at_5 + value: 48.97376543209873 + - type: nauc_map_at_1000_diff1 + value: 42.12670040381569 + - type: nauc_map_at_1000_max + value: 22.71752695113445 + - type: nauc_map_at_1000_std + value: -11.075142789670126 + - type: nauc_map_at_100_diff1 + value: 42.105073379985576 + - type: nauc_map_at_100_max + value: 22.601360991774097 + - type: nauc_map_at_100_std + value: -11.14811247003027 + - type: nauc_map_at_10_diff1 + value: 42.15868905725145 + - type: nauc_map_at_10_max + value: 21.32305211610425 + - type: nauc_map_at_10_std + value: -12.417753964396388 + - type: nauc_map_at_1_diff1 + value: 46.038051316078004 + - type: nauc_map_at_1_max + value: 13.682999715407396 + - type: nauc_map_at_1_std + value: -15.677552946705589 + - type: nauc_map_at_20_diff1 + value: 42.16591873450413 + - type: nauc_map_at_20_max + value: 21.995452042737906 + - type: nauc_map_at_20_std + value: -11.778589286868003 + - type: nauc_map_at_3_diff1 + value: 42.7692091773026 + - type: nauc_map_at_3_max + value: 18.955801090765373 + - type: nauc_map_at_3_std + value: -12.890482122716996 + - type: nauc_map_at_5_diff1 + value: 42.440874854399 + - type: nauc_map_at_5_max + value: 20.797433869273 + - type: nauc_map_at_5_std + value: -12.65105349121336 + - type: nauc_mrr_at_1000_diff1 + value: 49.55660704444496 + - type: nauc_mrr_at_1000_max + value: 32.140496759990334 + - type: nauc_mrr_at_1000_std + value: -9.526100187265591 + - type: nauc_mrr_at_100_diff1 + value: 49.53330675990377 + - type: nauc_mrr_at_100_max + value: 32.14334990005496 + - type: nauc_mrr_at_100_std + value: -9.512577982625467 + - type: nauc_mrr_at_10_diff1 + value: 49.44764070783239 + - type: nauc_mrr_at_10_max + value: 32.02441474083664 + - type: nauc_mrr_at_10_std + value: -9.695690567410614 + - type: nauc_mrr_at_1_diff1 + value: 54.42958130570017 + - type: nauc_mrr_at_1_max + value: 31.83950498073706 + - type: nauc_mrr_at_1_std + value: -12.116918485711826 + - type: nauc_mrr_at_20_diff1 + value: 49.53152965929537 + - type: nauc_mrr_at_20_max + value: 32.08417623051881 + - type: nauc_mrr_at_20_std + value: -9.45078967171175 + - type: nauc_mrr_at_3_diff1 + value: 49.678234835962535 + - type: nauc_mrr_at_3_max + value: 31.912854624375143 + - type: nauc_mrr_at_3_std + value: -11.275134559029071 + - type: nauc_mrr_at_5_diff1 + value: 49.477518934951206 + - type: nauc_mrr_at_5_max + value: 32.221990510809704 + - type: nauc_mrr_at_5_std + value: -10.234710063555815 + - type: nauc_ndcg_at_1000_diff1 + value: 43.0835178969525 + - type: nauc_ndcg_at_1000_max + value: 27.74038467560217 + - type: nauc_ndcg_at_1000_std + value: -6.690085957059853 + - type: nauc_ndcg_at_100_diff1 + value: 42.59465373544716 + - type: nauc_ndcg_at_100_max + value: 26.44864815048616 + - type: nauc_ndcg_at_100_std + value: -7.092736215011223 + - type: nauc_ndcg_at_10_diff1 + value: 42.348075532202074 + - type: nauc_ndcg_at_10_max + value: 23.706172401769983 + - type: nauc_ndcg_at_10_std + value: -10.820669710805781 + - type: nauc_ndcg_at_1_diff1 + value: 54.42958130570017 + - type: nauc_ndcg_at_1_max + value: 31.83950498073706 + - type: nauc_ndcg_at_1_std + value: -12.116918485711826 + - type: nauc_ndcg_at_20_diff1 + value: 42.59450341683887 + - type: nauc_ndcg_at_20_max + value: 24.790355252218532 + - type: nauc_ndcg_at_20_std + value: -9.185773048546144 + - type: nauc_ndcg_at_3_diff1 + value: 41.65146985455856 + - type: nauc_ndcg_at_3_max + value: 25.523553574835013 + - type: nauc_ndcg_at_3_std + value: -10.934658030133301 + - type: nauc_ndcg_at_5_diff1 + value: 42.01964705012472 + - type: nauc_ndcg_at_5_max + value: 25.078758063491513 + - type: nauc_ndcg_at_5_std + value: -11.008288668593785 + - type: nauc_precision_at_1000_diff1 + value: -6.352766079863446 + - type: nauc_precision_at_1000_max + value: 28.839718901513216 + - type: nauc_precision_at_1000_std + value: 18.933229610528358 + - type: nauc_precision_at_100_diff1 + value: 1.6327201037045054 + - type: nauc_precision_at_100_max + value: 29.93200717684826 + - type: nauc_precision_at_100_std + value: 15.810321565444028 + - type: nauc_precision_at_10_diff1 + value: 15.480015557749423 + - type: nauc_precision_at_10_max + value: 29.91600689556612 + - type: nauc_precision_at_10_std + value: 2.693344283989494 + - type: nauc_precision_at_1_diff1 + value: 54.42958130570017 + - type: nauc_precision_at_1_max + value: 31.83950498073706 + - type: nauc_precision_at_1_std + value: -12.116918485711826 + - type: nauc_precision_at_20_diff1 + value: 10.803692154113373 + - type: nauc_precision_at_20_max + value: 30.954391518784153 + - type: nauc_precision_at_20_std + value: 8.682098026578561 + - type: nauc_precision_at_3_diff1 + value: 28.580798097365097 + - type: nauc_precision_at_3_max + value: 31.215466457852653 + - type: nauc_precision_at_3_std + value: -4.469717500289138 + - type: nauc_precision_at_5_diff1 + value: 22.358459008970144 + - type: nauc_precision_at_5_max + value: 32.002241516347475 + - type: nauc_precision_at_5_std + value: -1.303320989302817 + - type: nauc_recall_at_1000_diff1 + value: 32.424195900349936 + - type: nauc_recall_at_1000_max + value: 31.461127485778412 + - type: nauc_recall_at_1000_std + value: 32.98462463603415 + - type: nauc_recall_at_100_diff1 + value: 29.163111450523594 + - type: nauc_recall_at_100_max + value: 19.51882475047066 + - type: nauc_recall_at_100_std + value: 7.652104246450264 + - type: nauc_recall_at_10_diff1 + value: 32.710671644254305 + - type: nauc_recall_at_10_max + value: 15.769519699203736 + - type: nauc_recall_at_10_std + value: -8.774284322717515 + - type: nauc_recall_at_1_diff1 + value: 46.038051316078004 + - type: nauc_recall_at_1_max + value: 13.682999715407396 + - type: nauc_recall_at_1_std + value: -15.677552946705589 + - type: nauc_recall_at_20_diff1 + value: 32.23436954576624 + - type: nauc_recall_at_20_max + value: 17.69006279483891 + - type: nauc_recall_at_20_std + value: -3.261497059216767 + - type: nauc_recall_at_3_diff1 + value: 36.6515580690554 + - type: nauc_recall_at_3_max + value: 16.47495889709967 + - type: nauc_recall_at_3_std + value: -11.772595219362387 + - type: nauc_recall_at_5_diff1 + value: 33.98256991321259 + - type: nauc_recall_at_5_max + value: 18.102746049460468 + - type: nauc_recall_at_5_std + value: -9.978644827138524 + - type: ndcg_at_1 + value: 40.894999999999996 + - type: ndcg_at_10 + value: 43.562 + - type: ndcg_at_100 + value: 50.468 + - type: ndcg_at_1000 + value: 53.290000000000006 + - type: ndcg_at_20 + value: 46.329 + - type: ndcg_at_3 + value: 38.769999999999996 + - type: ndcg_at_5 + value: 40.932 + - type: precision_at_1 + value: 40.894999999999996 + - type: precision_at_10 + value: 12.222 + - type: precision_at_100 + value: 1.934 + - type: precision_at_1000 + value: 0.243 + - type: precision_at_20 + value: 7.315 + - type: precision_at_3 + value: 25.617 + - type: precision_at_5 + value: 19.414 + - type: recall_at_1 + value: 22.091 + - type: recall_at_10 + value: 51.098 + - type: recall_at_100 + value: 76.325 + - type: recall_at_1000 + value: 93.094 + - type: recall_at_20 + value: 59.501000000000005 + - type: recall_at_3 + value: 35.476 + - type: recall_at_5 + value: 43.275000000000006 + task: + type: Retrieval + - dataset: + config: default + name: MTEB HotpotQA (default) + revision: ab518f4d6fcca38d87c25209f94beba119d02014 + split: test + type: mteb/hotpotqa + metrics: + - type: main_score + value: 69.328 + - type: map_at_1 + value: 37.961 + - type: map_at_10 + value: 61.024 + - type: map_at_100 + value: 61.922 + - type: map_at_1000 + value: 61.986 + - type: map_at_20 + value: 61.541000000000004 + - type: map_at_3 + value: 57.489999999999995 + - type: map_at_5 + value: 59.732 + - type: mrr_at_1 + value: 75.92167454422687 + - type: mrr_at_10 + value: 82.35863798591666 + - type: mrr_at_100 + value: 82.55233598798793 + - type: mrr_at_1000 + value: 82.55973830505918 + - type: mrr_at_20 + value: 82.48429833303429 + - type: mrr_at_3 + value: 81.29191987395885 + - type: mrr_at_5 + value: 82.01102858428956 + - type: nauc_map_at_1000_diff1 + value: 20.48554940736609 + - type: nauc_map_at_1000_max + value: 27.085185468402408 + - type: nauc_map_at_1000_std + value: 4.416972493829815 + - type: nauc_map_at_100_diff1 + value: 20.46340964766847 + - type: nauc_map_at_100_max + value: 27.06772646203986 + - type: nauc_map_at_100_std + value: 4.4244636012500775 + - type: nauc_map_at_10_diff1 + value: 20.316537400383133 + - type: nauc_map_at_10_max + value: 27.067030089086103 + - type: nauc_map_at_10_std + value: 3.8499319099786224 + - type: nauc_map_at_1_diff1 + value: 64.79738679447125 + - type: nauc_map_at_1_max + value: 49.40909740671419 + - type: nauc_map_at_1_std + value: -7.953248153850845 + - type: nauc_map_at_20_diff1 + value: 20.256485063019007 + - type: nauc_map_at_20_max + value: 27.008360003669964 + - type: nauc_map_at_20_std + value: 4.285719079014462 + - type: nauc_map_at_3_diff1 + value: 20.224899836855386 + - type: nauc_map_at_3_max + value: 26.819477720031294 + - type: nauc_map_at_3_std + value: 1.4988998842409027 + - type: nauc_map_at_5_diff1 + value: 20.235309134868686 + - type: nauc_map_at_5_max + value: 27.183271273988534 + - type: nauc_map_at_5_std + value: 2.983998347796308 + - type: nauc_mrr_at_1000_diff1 + value: 63.37635429689601 + - type: nauc_mrr_at_1000_max + value: 52.17941779263576 + - type: nauc_mrr_at_1000_std + value: -4.4222253901102055 + - type: nauc_mrr_at_100_diff1 + value: 63.37525520963241 + - type: nauc_mrr_at_100_max + value: 52.18249622868274 + - type: nauc_mrr_at_100_std + value: -4.411554421519852 + - type: nauc_mrr_at_10_diff1 + value: 63.3502703827035 + - type: nauc_mrr_at_10_max + value: 52.291056644371935 + - type: nauc_mrr_at_10_std + value: -4.39135755315105 + - type: nauc_mrr_at_1_diff1 + value: 64.79738679447125 + - type: nauc_mrr_at_1_max + value: 49.40909740671419 + - type: nauc_mrr_at_1_std + value: -7.953248153850845 + - type: nauc_mrr_at_20_diff1 + value: 63.369874957725294 + - type: nauc_mrr_at_20_max + value: 52.214658729761744 + - type: nauc_mrr_at_20_std + value: -4.366230475196012 + - type: nauc_mrr_at_3_diff1 + value: 63.249766666763975 + - type: nauc_mrr_at_3_max + value: 52.32630355954077 + - type: nauc_mrr_at_3_std + value: -4.841133869882257 + - type: nauc_mrr_at_5_diff1 + value: 63.432455507793115 + - type: nauc_mrr_at_5_max + value: 52.41448476682814 + - type: nauc_mrr_at_5_std + value: -4.474924260245147 + - type: nauc_ndcg_at_1000_diff1 + value: 26.773135663443664 + - type: nauc_ndcg_at_1000_max + value: 31.066452064579465 + - type: nauc_ndcg_at_1000_std + value: 6.857809164367065 + - type: nauc_ndcg_at_100_diff1 + value: 26.041528825611028 + - type: nauc_ndcg_at_100_max + value: 30.56025095655811 + - type: nauc_ndcg_at_100_std + value: 7.310340015016684 + - type: nauc_ndcg_at_10_diff1 + value: 25.09103804079387 + - type: nauc_ndcg_at_10_max + value: 30.531764043759896 + - type: nauc_ndcg_at_10_std + value: 5.251553742112263 + - type: nauc_ndcg_at_1_diff1 + value: 64.79738679447125 + - type: nauc_ndcg_at_1_max + value: 49.40909740671419 + - type: nauc_ndcg_at_1_std + value: -7.953248153850845 + - type: nauc_ndcg_at_20_diff1 + value: 24.82225221856848 + - type: nauc_ndcg_at_20_max + value: 30.259902561831343 + - type: nauc_ndcg_at_20_std + value: 6.548327863236633 + - type: nauc_ndcg_at_3_diff1 + value: 25.66768801183447 + - type: nauc_ndcg_at_3_max + value: 30.75093290266145 + - type: nauc_ndcg_at_3_std + value: 1.4653670667244483 + - type: nauc_ndcg_at_5_diff1 + value: 25.273637774398654 + - type: nauc_ndcg_at_5_max + value: 30.96617100513975 + - type: nauc_ndcg_at_5_std + value: 3.633888556741369 + - type: nauc_precision_at_1000_diff1 + value: 4.13169728284232 + - type: nauc_precision_at_1000_max + value: 15.135616796018974 + - type: nauc_precision_at_1000_std + value: 41.9291067619151 + - type: nauc_precision_at_100_diff1 + value: 7.341332237947484 + - type: nauc_precision_at_100_max + value: 17.03071625999757 + - type: nauc_precision_at_100_std + value: 29.008746154653092 + - type: nauc_precision_at_10_diff1 + value: 9.57063901739569 + - type: nauc_precision_at_10_max + value: 21.744195800807866 + - type: nauc_precision_at_10_std + value: 13.811680383737801 + - type: nauc_precision_at_1_diff1 + value: 64.79738679447125 + - type: nauc_precision_at_1_max + value: 49.40909740671419 + - type: nauc_precision_at_1_std + value: -7.953248153850845 + - type: nauc_precision_at_20_diff1 + value: 6.378732192529984 + - type: nauc_precision_at_20_max + value: 19.214638830386555 + - type: nauc_precision_at_20_std + value: 19.328560365010244 + - type: nauc_precision_at_3_diff1 + value: 14.069445697859784 + - type: nauc_precision_at_3_max + value: 24.97832449377698 + - type: nauc_precision_at_3_std + value: 4.71053634059164 + - type: nauc_precision_at_5_diff1 + value: 12.07394147291554 + - type: nauc_precision_at_5_max + value: 24.422756430965137 + - type: nauc_precision_at_5_std + value: 9.20113009038941 + - type: nauc_recall_at_1000_diff1 + value: 4.131697282842698 + - type: nauc_recall_at_1000_max + value: 15.135616796019436 + - type: nauc_recall_at_1000_std + value: 41.92910676191518 + - type: nauc_recall_at_100_diff1 + value: 7.3413322379474995 + - type: nauc_recall_at_100_max + value: 17.030716259997412 + - type: nauc_recall_at_100_std + value: 29.008746154653114 + - type: nauc_recall_at_10_diff1 + value: 9.570639017395651 + - type: nauc_recall_at_10_max + value: 21.74419580080782 + - type: nauc_recall_at_10_std + value: 13.811680383737743 + - type: nauc_recall_at_1_diff1 + value: 64.79738679447125 + - type: nauc_recall_at_1_max + value: 49.40909740671419 + - type: nauc_recall_at_1_std + value: -7.953248153850845 + - type: nauc_recall_at_20_diff1 + value: 6.378732192529887 + - type: nauc_recall_at_20_max + value: 19.214638830386658 + - type: nauc_recall_at_20_std + value: 19.328560365010397 + - type: nauc_recall_at_3_diff1 + value: 14.069445697859786 + - type: nauc_recall_at_3_max + value: 24.978324493776924 + - type: nauc_recall_at_3_std + value: 4.710536340591549 + - type: nauc_recall_at_5_diff1 + value: 12.0739414729155 + - type: nauc_recall_at_5_max + value: 24.422756430965077 + - type: nauc_recall_at_5_std + value: 9.20113009038933 + - type: ndcg_at_1 + value: 75.922 + - type: ndcg_at_10 + value: 69.328 + - type: ndcg_at_100 + value: 72.439 + - type: ndcg_at_1000 + value: 73.673 + - type: ndcg_at_20 + value: 70.621 + - type: ndcg_at_3 + value: 64.30900000000001 + - type: ndcg_at_5 + value: 67.169 + - type: precision_at_1 + value: 75.922 + - type: precision_at_10 + value: 14.635000000000002 + - type: precision_at_100 + value: 1.706 + - type: precision_at_1000 + value: 0.187 + - type: precision_at_20 + value: 7.734000000000001 + - type: precision_at_3 + value: 41.418 + - type: precision_at_5 + value: 27.106 + - type: recall_at_1 + value: 37.961 + - type: recall_at_10 + value: 73.17399999999999 + - type: recall_at_100 + value: 85.294 + - type: recall_at_1000 + value: 93.43 + - type: recall_at_20 + value: 77.34 + - type: recall_at_3 + value: 62.126999999999995 + - type: recall_at_5 + value: 67.765 + task: + type: Retrieval + - dataset: + config: default + name: MTEB ImdbClassification (default) + revision: 3d86128a09e091d6018b6d26cad27f2739fc2db7 + split: test + type: mteb/imdb + metrics: + - type: accuracy + value: 74.8692 + - type: ap + value: 68.75461771078795 + - type: ap_weighted + value: 68.75461771078795 + - type: f1 + value: 74.75077784856869 + - type: f1_weighted + value: 74.75077784856869 + - type: main_score + value: 74.8692 + task: + type: Classification + - dataset: + config: default + name: MTEB MSMARCO (default) + revision: c5a29a104738b98a9e76336939199e264163d4a0 + split: dev + type: mteb/msmarco + metrics: + - type: main_score + value: 40.913 + - type: map_at_1 + value: 21.412 + - type: map_at_10 + value: 33.828 + - type: map_at_100 + value: 35.04 + - type: map_at_1000 + value: 35.089 + - type: map_at_20 + value: 34.581 + - type: map_at_3 + value: 29.749 + - type: map_at_5 + value: 32.086 + - type: mrr_at_1 + value: 22.005730659025787 + - type: mrr_at_10 + value: 34.38339132214486 + - type: mrr_at_100 + value: 35.531082006689324 + - type: mrr_at_1000 + value: 35.57400846682949 + - type: mrr_at_20 + value: 35.10533223608268 + - type: mrr_at_3 + value: 30.365329512893823 + - type: mrr_at_5 + value: 32.68051575931231 + - type: nauc_map_at_1000_diff1 + value: 33.79696476529548 + - type: nauc_map_at_1000_max + value: 23.664544484006537 + - type: nauc_map_at_1000_std + value: -20.974583892117455 + - type: nauc_map_at_100_diff1 + value: 33.78608412473617 + - type: nauc_map_at_100_max + value: 23.700224667468767 + - type: nauc_map_at_100_std + value: -20.956935328067196 + - type: nauc_map_at_10_diff1 + value: 33.775849743172415 + - type: nauc_map_at_10_max + value: 23.71995203836684 + - type: nauc_map_at_10_std + value: -21.69912263727474 + - type: nauc_map_at_1_diff1 + value: 36.83482988041847 + - type: nauc_map_at_1_max + value: 18.284850585245508 + - type: nauc_map_at_1_std + value: -18.410236954200357 + - type: nauc_map_at_20_diff1 + value: 33.77846093627085 + - type: nauc_map_at_20_max + value: 23.823696731930497 + - type: nauc_map_at_20_std + value: -21.216557786213706 + - type: nauc_map_at_3_diff1 + value: 34.04189586486102 + - type: nauc_map_at_3_max + value: 21.82494379041376 + - type: nauc_map_at_3_std + value: -21.28716470947222 + - type: nauc_map_at_5_diff1 + value: 34.000833555013074 + - type: nauc_map_at_5_max + value: 23.129780050479496 + - type: nauc_map_at_5_std + value: -22.110964076473905 + - type: nauc_mrr_at_1000_diff1 + value: 33.50771740159871 + - type: nauc_mrr_at_1000_max + value: 23.2840717090797 + - type: nauc_mrr_at_1000_std + value: -20.81472107620837 + - type: nauc_mrr_at_100_diff1 + value: 33.49768840019982 + - type: nauc_mrr_at_100_max + value: 23.316818541225917 + - type: nauc_mrr_at_100_std + value: -20.795019311135302 + - type: nauc_mrr_at_10_diff1 + value: 33.47029755258697 + - type: nauc_mrr_at_10_max + value: 23.335004828475554 + - type: nauc_mrr_at_10_std + value: -21.495413643986303 + - type: nauc_mrr_at_1_diff1 + value: 36.38981193092678 + - type: nauc_mrr_at_1_max + value: 17.95945233630547 + - type: nauc_mrr_at_1_std + value: -18.578142780060155 + - type: nauc_mrr_at_20_diff1 + value: 33.46770646987267 + - type: nauc_mrr_at_20_max + value: 23.430058665443234 + - type: nauc_mrr_at_20_std + value: -21.01281241611354 + - type: nauc_mrr_at_3_diff1 + value: 33.685652185346534 + - type: nauc_mrr_at_3_max + value: 21.490712167715685 + - type: nauc_mrr_at_3_std + value: -21.22401255026763 + - type: nauc_mrr_at_5_diff1 + value: 33.68618723517232 + - type: nauc_mrr_at_5_max + value: 22.800251136912898 + - type: nauc_mrr_at_5_std + value: -21.96423632186793 + - type: nauc_ndcg_at_1000_diff1 + value: 32.96850170402295 + - type: nauc_ndcg_at_1000_max + value: 25.33005869991171 + - type: nauc_ndcg_at_1000_std + value: -19.691472502675143 + - type: nauc_ndcg_at_100_diff1 + value: 32.734421472147574 + - type: nauc_ndcg_at_100_max + value: 26.287474932815158 + - type: nauc_ndcg_at_100_std + value: -18.837822199467112 + - type: nauc_ndcg_at_10_diff1 + value: 32.640182551829774 + - type: nauc_ndcg_at_10_max + value: 26.502101048138744 + - type: nauc_ndcg_at_10_std + value: -22.381882423915613 + - type: nauc_ndcg_at_1_diff1 + value: 36.38981193092678 + - type: nauc_ndcg_at_1_max + value: 17.95945233630547 + - type: nauc_ndcg_at_1_std + value: -18.578142780060155 + - type: nauc_ndcg_at_20_diff1 + value: 32.61487622733105 + - type: nauc_ndcg_at_20_max + value: 27.06175470260795 + - type: nauc_ndcg_at_20_std + value: -20.56260955513618 + - type: nauc_ndcg_at_3_diff1 + value: 33.27219295282536 + - type: nauc_ndcg_at_3_max + value: 22.798164381085634 + - type: nauc_ndcg_at_3_std + value: -21.97590708219718 + - type: nauc_ndcg_at_5_diff1 + value: 33.21340296792794 + - type: nauc_ndcg_at_5_max + value: 25.088363620672993 + - type: nauc_ndcg_at_5_std + value: -23.345211712553887 + - type: nauc_precision_at_1000_diff1 + value: -7.881189172825498 + - type: nauc_precision_at_1000_max + value: -0.36172641750577517 + - type: nauc_precision_at_1000_std + value: 12.353312008995491 + - type: nauc_precision_at_100_diff1 + value: 9.59397782689476 + - type: nauc_precision_at_100_max + value: 25.51106500095386 + - type: nauc_precision_at_100_std + value: 10.010056195504736 + - type: nauc_precision_at_10_diff1 + value: 25.70706302979518 + - type: nauc_precision_at_10_max + value: 33.01814934143395 + - type: nauc_precision_at_10_std + value: -22.87162235923727 + - type: nauc_precision_at_1_diff1 + value: 36.38981193092678 + - type: nauc_precision_at_1_max + value: 17.95945233630547 + - type: nauc_precision_at_1_std + value: -18.578142780060155 + - type: nauc_precision_at_20_diff1 + value: 22.49753455298279 + - type: nauc_precision_at_20_max + value: 35.289801422565326 + - type: nauc_precision_at_20_std + value: -13.173027862734177 + - type: nauc_precision_at_3_diff1 + value: 30.29614162002589 + - type: nauc_precision_at_3_max + value: 24.654971029585017 + - type: nauc_precision_at_3_std + value: -23.68751533801935 + - type: nauc_precision_at_5_diff1 + value: 29.30462218368561 + - type: nauc_precision_at_5_max + value: 29.217272703755327 + - type: nauc_precision_at_5_std + value: -26.455737517840067 + - type: nauc_recall_at_1000_diff1 + value: 15.110738934268412 + - type: nauc_recall_at_1000_max + value: 51.22302534067236 + - type: nauc_recall_at_1000_std + value: 56.223515047045034 + - type: nauc_recall_at_100_diff1 + value: 25.227373685916927 + - type: nauc_recall_at_100_max + value: 48.64633501006257 + - type: nauc_recall_at_100_std + value: 11.987292157093282 + - type: nauc_recall_at_10_diff1 + value: 28.897676026478553 + - type: nauc_recall_at_10_max + value: 35.92714867516096 + - type: nauc_recall_at_10_std + value: -24.17817018456719 + - type: nauc_recall_at_1_diff1 + value: 36.83482988041847 + - type: nauc_recall_at_1_max + value: 18.284850585245508 + - type: nauc_recall_at_1_std + value: -18.410236954200357 + - type: nauc_recall_at_20_diff1 + value: 27.915951496472776 + - type: nauc_recall_at_20_max + value: 41.319088558806015 + - type: nauc_recall_at_20_std + value: -15.731440350211345 + - type: nauc_recall_at_3_diff1 + value: 31.42633085277067 + - type: nauc_recall_at_3_max + value: 25.52282157983466 + - type: nauc_recall_at_3_std + value: -23.720501119226135 + - type: nauc_recall_at_5_diff1 + value: 31.172788500117036 + - type: nauc_recall_at_5_max + value: 30.751081560202326 + - type: nauc_recall_at_5_std + value: -26.697555089484 + - type: ndcg_at_1 + value: 22.006 + - type: ndcg_at_10 + value: 40.913 + - type: ndcg_at_100 + value: 46.694 + - type: ndcg_at_1000 + value: 47.863 + - type: ndcg_at_20 + value: 43.586000000000006 + - type: ndcg_at_3 + value: 32.592999999999996 + - type: ndcg_at_5 + value: 36.769 + - type: precision_at_1 + value: 22.006 + - type: precision_at_10 + value: 6.576 + - type: precision_at_100 + value: 0.946 + - type: precision_at_1000 + value: 0.105 + - type: precision_at_20 + value: 3.842 + - type: precision_at_3 + value: 13.905999999999999 + - type: precision_at_5 + value: 10.467 + - type: recall_at_1 + value: 21.412 + - type: recall_at_10 + value: 62.822 + - type: recall_at_100 + value: 89.426 + - type: recall_at_1000 + value: 98.29299999999999 + - type: recall_at_20 + value: 73.212 + - type: recall_at_3 + value: 40.205 + - type: recall_at_5 + value: 50.227999999999994 + task: + type: Retrieval + - dataset: + config: en + name: MTEB MTOPDomainClassification (en) + revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf + split: test + type: mteb/mtop_domain + metrics: + - type: accuracy + value: 91.8376652986776 + - type: f1 + value: 91.47210327577605 + - type: f1_weighted + value: 91.8109209343245 + - type: main_score + value: 91.8376652986776 + task: + type: Classification + - dataset: + config: en + name: MTEB MTOPIntentClassification (en) + revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba + split: test + type: mteb/mtop_intent + metrics: + - type: accuracy + value: 66.63018695850434 + - type: f1 + value: 49.85368792059109 + - type: f1_weighted + value: 68.68364917845707 + - type: main_score + value: 66.63018695850434 + task: + type: Classification + - dataset: + config: en + name: MTEB MassiveIntentClassification (en) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 74.18964357767317 + - type: f1 + value: 72.57133220649258 + - type: f1_weighted + value: 73.2041886682309 + - type: main_score + value: 74.18964357767317 + task: + type: Classification + - dataset: + config: en + name: MTEB MassiveScenarioClassification (en) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 77.7404169468729 + - type: f1 + value: 77.10704453864466 + - type: f1_weighted + value: 77.69961168140244 + - type: main_score + value: 77.7404169468729 + task: + type: Classification + - dataset: + config: default + name: MTEB MedrxivClusteringP2P (default) + revision: e7a26af6f3ae46b30dde8737f02c07b1505bcc73 + split: test + type: mteb/medrxiv-clustering-p2p + metrics: + - type: main_score + value: 31.920806889241494 + - type: v_measure + value: 31.920806889241494 + - type: v_measure_std + value: 1.4276839965197128 + task: + type: Clustering + - dataset: + config: default + name: MTEB MedrxivClusteringS2S (default) + revision: 35191c8c0dca72d8ff3efcd72aa802307d469663 + split: test + type: mteb/medrxiv-clustering-s2s + metrics: + - type: main_score + value: 31.4052823506609 + - type: v_measure + value: 31.4052823506609 + - type: v_measure_std + value: 1.0206186959314234 + task: + type: Clustering + - dataset: + config: default + name: MTEB MindSmallReranking (default) + revision: 59042f120c80e8afa9cdbb224f67076cec0fc9a7 + split: test + type: mteb/mind_small + metrics: + - type: main_score + value: 32.16720935506015 + - type: map + value: 32.16720935506015 + - type: mrr + value: 33.41354086532975 + - type: nAUC_map_diff1 + value: 13.49892142611781 + - type: nAUC_map_max + value: -18.797620819003768 + - type: nAUC_map_std + value: -1.9344501580674716 + - type: nAUC_mrr_diff1 + value: 12.506075963569138 + - type: nAUC_mrr_max + value: -13.769643226717607 + - type: nAUC_mrr_std + value: 0.19527743054643182 + task: + type: Reranking + - dataset: + config: default + name: MTEB NFCorpus (default) + revision: ec0fa4fe99da2ff19ca1214b7966684033a58814 + split: test + type: mteb/nfcorpus + metrics: + - type: main_score + value: 36.01 + - type: map_at_1 + value: 5.918 + - type: map_at_10 + value: 13.575999999999999 + - type: map_at_100 + value: 17.532 + - type: map_at_1000 + value: 19.162000000000003 + - type: map_at_20 + value: 15.262 + - type: map_at_3 + value: 9.931 + - type: map_at_5 + value: 11.756 + - type: mrr_at_1 + value: 45.82043343653251 + - type: mrr_at_10 + value: 55.65973757924218 + - type: mrr_at_100 + value: 56.23259196555049 + - type: mrr_at_1000 + value: 56.26088661498459 + - type: mrr_at_20 + value: 56.055531951447726 + - type: mrr_at_3 + value: 52.99277605779154 + - type: mrr_at_5 + value: 54.912280701754355 + - type: nauc_map_at_1000_diff1 + value: 24.757803803709 + - type: nauc_map_at_1000_max + value: 38.12203490581986 + - type: nauc_map_at_1000_std + value: 13.321448701258095 + - type: nauc_map_at_100_diff1 + value: 25.891576411095972 + - type: nauc_map_at_100_max + value: 37.49282420085297 + - type: nauc_map_at_100_std + value: 9.244069982763213 + - type: nauc_map_at_10_diff1 + value: 27.798365244154173 + - type: nauc_map_at_10_max + value: 32.597616488742794 + - type: nauc_map_at_10_std + value: -1.9064562861660292 + - type: nauc_map_at_1_diff1 + value: 38.54714681450678 + - type: nauc_map_at_1_max + value: 18.220486173123522 + - type: nauc_map_at_1_std + value: -17.97066065482943 + - type: nauc_map_at_20_diff1 + value: 27.184262736165305 + - type: nauc_map_at_20_max + value: 35.3436748755543 + - type: nauc_map_at_20_std + value: 2.8612253087863992 + - type: nauc_map_at_3_diff1 + value: 32.497034130826734 + - type: nauc_map_at_3_max + value: 24.46819789246231 + - type: nauc_map_at_3_std + value: -11.384756633811284 + - type: nauc_map_at_5_diff1 + value: 28.750106302620942 + - type: nauc_map_at_5_max + value: 28.405589971620653 + - type: nauc_map_at_5_std + value: -8.014093237013602 + - type: nauc_mrr_at_1000_diff1 + value: 29.200727547933536 + - type: nauc_mrr_at_1000_max + value: 52.09178106390233 + - type: nauc_mrr_at_1000_std + value: 27.31641684256782 + - type: nauc_mrr_at_100_diff1 + value: 29.19149257077949 + - type: nauc_mrr_at_100_max + value: 52.1105717362922 + - type: nauc_mrr_at_100_std + value: 27.343604950676077 + - type: nauc_mrr_at_10_diff1 + value: 29.49077294773766 + - type: nauc_mrr_at_10_max + value: 51.92640807358807 + - type: nauc_mrr_at_10_std + value: 26.970908831839523 + - type: nauc_mrr_at_1_diff1 + value: 30.539512161001138 + - type: nauc_mrr_at_1_max + value: 48.61597681902167 + - type: nauc_mrr_at_1_std + value: 22.006579582133064 + - type: nauc_mrr_at_20_diff1 + value: 29.23487695923463 + - type: nauc_mrr_at_20_max + value: 52.17656836863323 + - type: nauc_mrr_at_20_std + value: 27.36767730350206 + - type: nauc_mrr_at_3_diff1 + value: 28.938191385197147 + - type: nauc_mrr_at_3_max + value: 50.517994195660776 + - type: nauc_mrr_at_3_std + value: 25.37522712215449 + - type: nauc_mrr_at_5_diff1 + value: 28.78652774130156 + - type: nauc_mrr_at_5_max + value: 51.7894366111625 + - type: nauc_mrr_at_5_std + value: 26.628095126330344 + - type: nauc_ndcg_at_1000_diff1 + value: 22.098100468352808 + - type: nauc_ndcg_at_1000_max + value: 49.41736728283494 + - type: nauc_ndcg_at_1000_std + value: 30.053397219753496 + - type: nauc_ndcg_at_100_diff1 + value: 20.911988787000176 + - type: nauc_ndcg_at_100_max + value: 44.282222145794954 + - type: nauc_ndcg_at_100_std + value: 24.670739156333887 + - type: nauc_ndcg_at_10_diff1 + value: 18.765479168259073 + - type: nauc_ndcg_at_10_max + value: 45.1056894165556 + - type: nauc_ndcg_at_10_std + value: 27.48780086040404 + - type: nauc_ndcg_at_1_diff1 + value: 30.603271423208827 + - type: nauc_ndcg_at_1_max + value: 46.27847247605851 + - type: nauc_ndcg_at_1_std + value: 22.131298910421584 + - type: nauc_ndcg_at_20_diff1 + value: 18.90204504898861 + - type: nauc_ndcg_at_20_max + value: 43.05183053430342 + - type: nauc_ndcg_at_20_std + value: 25.807516520305683 + - type: nauc_ndcg_at_3_diff1 + value: 21.687312102969262 + - type: nauc_ndcg_at_3_max + value: 44.667108992659436 + - type: nauc_ndcg_at_3_std + value: 24.289089131876633 + - type: nauc_ndcg_at_5_diff1 + value: 18.395148276385086 + - type: nauc_ndcg_at_5_max + value: 47.01400976057379 + - type: nauc_ndcg_at_5_std + value: 26.104627717960234 + - type: nauc_precision_at_1000_diff1 + value: -11.482660777829508 + - type: nauc_precision_at_1000_max + value: 8.635495742870654 + - type: nauc_precision_at_1000_std + value: 41.72491856998992 + - type: nauc_precision_at_100_diff1 + value: -7.447161341798219 + - type: nauc_precision_at_100_max + value: 19.67544690401704 + - type: nauc_precision_at_100_std + value: 46.548749750008966 + - type: nauc_precision_at_10_diff1 + value: 3.754448374420886 + - type: nauc_precision_at_10_max + value: 40.64329592199495 + - type: nauc_precision_at_10_std + value: 41.11600178445942 + - type: nauc_precision_at_1_diff1 + value: 30.539512161001138 + - type: nauc_precision_at_1_max + value: 48.61597681902167 + - type: nauc_precision_at_1_std + value: 22.006579582133064 + - type: nauc_precision_at_20_diff1 + value: 0.3123684046217722 + - type: nauc_precision_at_20_max + value: 34.6895321561208 + - type: nauc_precision_at_20_std + value: 44.939190047578954 + - type: nauc_precision_at_3_diff1 + value: 13.261202008291962 + - type: nauc_precision_at_3_max + value: 43.51844800043113 + - type: nauc_precision_at_3_std + value: 28.475194526427227 + - type: nauc_precision_at_5_diff1 + value: 5.076916464289792 + - type: nauc_precision_at_5_max + value: 45.499914504765485 + - type: nauc_precision_at_5_std + value: 33.953875872119774 + - type: nauc_recall_at_1000_diff1 + value: 4.31367028884273 + - type: nauc_recall_at_1000_max + value: 21.898463722872133 + - type: nauc_recall_at_1000_std + value: 12.004092782606994 + - type: nauc_recall_at_100_diff1 + value: 12.776078635390187 + - type: nauc_recall_at_100_max + value: 26.696299651930293 + - type: nauc_recall_at_100_std + value: 11.515219809522295 + - type: nauc_recall_at_10_diff1 + value: 22.513567554725288 + - type: nauc_recall_at_10_max + value: 29.113927232397053 + - type: nauc_recall_at_10_std + value: -2.2895940396268353 + - type: nauc_recall_at_1_diff1 + value: 38.54714681450678 + - type: nauc_recall_at_1_max + value: 18.220486173123522 + - type: nauc_recall_at_1_std + value: -17.97066065482943 + - type: nauc_recall_at_20_diff1 + value: 18.320486240005067 + - type: nauc_recall_at_20_max + value: 26.51882526313221 + - type: nauc_recall_at_20_std + value: 0.877818287179115 + - type: nauc_recall_at_3_diff1 + value: 29.032934408227035 + - type: nauc_recall_at_3_max + value: 22.04639669161234 + - type: nauc_recall_at_3_std + value: -11.080522908760907 + - type: nauc_recall_at_5_diff1 + value: 21.581754021983283 + - type: nauc_recall_at_5_max + value: 24.945806092849388 + - type: nauc_recall_at_5_std + value: -8.436248539997395 + - type: ndcg_at_1 + value: 43.963 + - type: ndcg_at_10 + value: 36.01 + - type: ndcg_at_100 + value: 34.139 + - type: ndcg_at_1000 + value: 43.161 + - type: ndcg_at_20 + value: 34.160000000000004 + - type: ndcg_at_3 + value: 41.187000000000005 + - type: ndcg_at_5 + value: 39.207 + - type: precision_at_1 + value: 45.82 + - type: precision_at_10 + value: 26.966 + - type: precision_at_100 + value: 8.963000000000001 + - type: precision_at_1000 + value: 2.2190000000000003 + - type: precision_at_20 + value: 20.263 + - type: precision_at_3 + value: 38.906 + - type: precision_at_5 + value: 34.489 + - type: recall_at_1 + value: 5.918 + - type: recall_at_10 + value: 17.65 + - type: recall_at_100 + value: 35.628 + - type: recall_at_1000 + value: 68.245 + - type: recall_at_20 + value: 22.607 + - type: recall_at_3 + value: 10.929 + - type: recall_at_5 + value: 14.157 + task: + type: Retrieval + - dataset: + config: default + name: MTEB NQ (default) + revision: b774495ed302d8c44a3a7ea25c90dbce03968f31 + split: test + type: mteb/nq + metrics: + - type: main_score + value: 57.035000000000004 + - type: map_at_1 + value: 32.619 + - type: map_at_10 + value: 49.045 + - type: map_at_100 + value: 50.074 + - type: map_at_1000 + value: 50.092999999999996 + - type: map_at_20 + value: 49.738 + - type: map_at_3 + value: 44.567 + - type: map_at_5 + value: 47.229 + - type: mrr_at_1 + value: 36.87717265353419 + - type: mrr_at_10 + value: 51.516270760911375 + - type: mrr_at_100 + value: 52.22966519542097 + - type: mrr_at_1000 + value: 52.242469506815794 + - type: mrr_at_20 + value: 51.99746561230684 + - type: mrr_at_3 + value: 47.85148706064101 + - type: mrr_at_5 + value: 50.071938972576206 + - type: nauc_map_at_1000_diff1 + value: 31.115519931814944 + - type: nauc_map_at_1000_max + value: 21.624450537783105 + - type: nauc_map_at_1000_std + value: -7.575018696337878 + - type: nauc_map_at_100_diff1 + value: 31.106555788570983 + - type: nauc_map_at_100_max + value: 21.645140716470088 + - type: nauc_map_at_100_std + value: -7.552897043464501 + - type: nauc_map_at_10_diff1 + value: 31.069699066299517 + - type: nauc_map_at_10_max + value: 21.897656157204253 + - type: nauc_map_at_10_std + value: -7.814644460476085 + - type: nauc_map_at_1_diff1 + value: 34.452428651446944 + - type: nauc_map_at_1_max + value: 15.174110168280322 + - type: nauc_map_at_1_std + value: -9.643252389134126 + - type: nauc_map_at_20_diff1 + value: 31.062317717363623 + - type: nauc_map_at_20_max + value: 21.798964108711353 + - type: nauc_map_at_20_std + value: -7.677138781520821 + - type: nauc_map_at_3_diff1 + value: 31.361766097805155 + - type: nauc_map_at_3_max + value: 20.491267286126316 + - type: nauc_map_at_3_std + value: -8.945619438641113 + - type: nauc_map_at_5_diff1 + value: 31.12442114841814 + - type: nauc_map_at_5_max + value: 21.754421924177493 + - type: nauc_map_at_5_std + value: -8.553657792287101 + - type: nauc_mrr_at_1000_diff1 + value: 30.909173759174617 + - type: nauc_mrr_at_1000_max + value: 20.568396982936047 + - type: nauc_mrr_at_1000_std + value: -6.46298691960934 + - type: nauc_mrr_at_100_diff1 + value: 30.90039882546381 + - type: nauc_mrr_at_100_max + value: 20.58108089330148 + - type: nauc_mrr_at_100_std + value: -6.447970527630041 + - type: nauc_mrr_at_10_diff1 + value: 30.774067169817908 + - type: nauc_mrr_at_10_max + value: 20.83321245420721 + - type: nauc_mrr_at_10_std + value: -6.531504339359364 + - type: nauc_mrr_at_1_diff1 + value: 33.86515857453312 + - type: nauc_mrr_at_1_max + value: 15.448417869967718 + - type: nauc_mrr_at_1_std + value: -7.748406564947059 + - type: nauc_mrr_at_20_diff1 + value: 30.85746403269301 + - type: nauc_mrr_at_20_max + value: 20.699852776986564 + - type: nauc_mrr_at_20_std + value: -6.481465063293769 + - type: nauc_mrr_at_3_diff1 + value: 30.91149775189778 + - type: nauc_mrr_at_3_max + value: 19.815258225751684 + - type: nauc_mrr_at_3_std + value: -7.303603772513045 + - type: nauc_mrr_at_5_diff1 + value: 30.711634931844777 + - type: nauc_mrr_at_5_max + value: 20.696123579906338 + - type: nauc_mrr_at_5_std + value: -7.024165858434535 + - type: nauc_ndcg_at_1000_diff1 + value: 30.47613696068262 + - type: nauc_ndcg_at_1000_max + value: 22.789503184706447 + - type: nauc_ndcg_at_1000_std + value: -5.885385795349818 + - type: nauc_ndcg_at_100_diff1 + value: 30.19419495914157 + - type: nauc_ndcg_at_100_max + value: 23.336601996838077 + - type: nauc_ndcg_at_100_std + value: -5.215148912208092 + - type: nauc_ndcg_at_10_diff1 + value: 29.85485180901643 + - type: nauc_ndcg_at_10_max + value: 24.677789132430973 + - type: nauc_ndcg_at_10_std + value: -6.236189883843304 + - type: nauc_ndcg_at_1_diff1 + value: 33.86515857453312 + - type: nauc_ndcg_at_1_max + value: 15.448417869967718 + - type: nauc_ndcg_at_1_std + value: -7.748406564947059 + - type: nauc_ndcg_at_20_diff1 + value: 29.959504312055923 + - type: nauc_ndcg_at_20_max + value: 24.437574889496364 + - type: nauc_ndcg_at_20_std + value: -5.747409752283779 + - type: nauc_ndcg_at_3_diff1 + value: 30.43170036009983 + - type: nauc_ndcg_at_3_max + value: 21.823109483300893 + - type: nauc_ndcg_at_3_std + value: -8.512084720940525 + - type: nauc_ndcg_at_5_diff1 + value: 29.974548514353106 + - type: nauc_ndcg_at_5_max + value: 24.09590088782208 + - type: nauc_ndcg_at_5_std + value: -7.888847634937826 + - type: nauc_precision_at_1000_diff1 + value: -8.870521347525568 + - type: nauc_precision_at_1000_max + value: -1.3322281302290309 + - type: nauc_precision_at_1000_std + value: 13.389384983386218 + - type: nauc_precision_at_100_diff1 + value: -7.375424866590398 + - type: nauc_precision_at_100_max + value: 5.109702473964345 + - type: nauc_precision_at_100_std + value: 17.279471298116995 + - type: nauc_precision_at_10_diff1 + value: 7.771485691389087 + - type: nauc_precision_at_10_max + value: 22.18752973565226 + - type: nauc_precision_at_10_std + value: 7.42848856589157 + - type: nauc_precision_at_1_diff1 + value: 33.86515857453312 + - type: nauc_precision_at_1_max + value: 15.448417869967718 + - type: nauc_precision_at_1_std + value: -7.748406564947059 + - type: nauc_precision_at_20_diff1 + value: 1.6415938015728504 + - type: nauc_precision_at_20_max + value: 17.16551452342103 + - type: nauc_precision_at_20_std + value: 11.48958778766897 + - type: nauc_precision_at_3_diff1 + value: 21.114613931447234 + - type: nauc_precision_at_3_max + value: 23.734755775194724 + - type: nauc_precision_at_3_std + value: -3.6102313848531 + - type: nauc_precision_at_5_diff1 + value: 15.115879345520364 + - type: nauc_precision_at_5_max + value: 25.557241903331185 + - type: nauc_precision_at_5_std + value: -0.10256165952353796 + - type: nauc_recall_at_1000_diff1 + value: 39.88651032601705 + - type: nauc_recall_at_1000_max + value: 70.24597257126761 + - type: nauc_recall_at_1000_std + value: 79.95113994180014 + - type: nauc_recall_at_100_diff1 + value: 15.691981385897133 + - type: nauc_recall_at_100_max + value: 58.6004226516443 + - type: nauc_recall_at_100_std + value: 45.361558371782195 + - type: nauc_recall_at_10_diff1 + value: 23.78102715508277 + - type: nauc_recall_at_10_max + value: 38.42592516673584 + - type: nauc_recall_at_10_std + value: -0.6387938100179615 + - type: nauc_recall_at_1_diff1 + value: 34.452428651446944 + - type: nauc_recall_at_1_max + value: 15.174110168280322 + - type: nauc_recall_at_1_std + value: -9.643252389134126 + - type: nauc_recall_at_20_diff1 + value: 22.58051889458828 + - type: nauc_recall_at_20_max + value: 44.22102293855757 + - type: nauc_recall_at_20_std + value: 5.417713940634653 + - type: nauc_recall_at_3_diff1 + value: 27.21987010101224 + - type: nauc_recall_at_3_max + value: 25.622754288300975 + - type: nauc_recall_at_3_std + value: -8.545345964265326 + - type: nauc_recall_at_5_diff1 + value: 25.5539610760959 + - type: nauc_recall_at_5_max + value: 32.34527623982731 + - type: nauc_recall_at_5_std + value: -7.241846674441728 + - type: ndcg_at_1 + value: 36.876999999999995 + - type: ndcg_at_10 + value: 57.035000000000004 + - type: ndcg_at_100 + value: 61.159 + - type: ndcg_at_1000 + value: 61.56 + - type: ndcg_at_20 + value: 59.194 + - type: ndcg_at_3 + value: 48.684 + - type: ndcg_at_5 + value: 53.088 + - type: precision_at_1 + value: 36.876999999999995 + - type: precision_at_10 + value: 9.435 + - type: precision_at_100 + value: 1.172 + - type: precision_at_1000 + value: 0.121 + - type: precision_at_20 + value: 5.235 + - type: precision_at_3 + value: 22.325 + - type: precision_at_5 + value: 15.943999999999999 + - type: recall_at_1 + value: 32.619 + - type: recall_at_10 + value: 78.834 + - type: recall_at_100 + value: 96.456 + - type: recall_at_1000 + value: 99.382 + - type: recall_at_20 + value: 86.78099999999999 + - type: recall_at_3 + value: 57.298 + - type: recall_at_5 + value: 67.413 + task: + type: Retrieval + - dataset: + config: default + name: MTEB QuoraRetrieval (default) + revision: e4e08e0b7dbe3c8700f0daef558ff32256715259 + split: test + type: mteb/quora + metrics: + - type: main_score + value: 89.415 + - type: map_at_1 + value: 71.58200000000001 + - type: map_at_10 + value: 85.841 + - type: map_at_100 + value: 86.451 + - type: map_at_1000 + value: 86.463 + - type: map_at_20 + value: 86.246 + - type: map_at_3 + value: 82.819 + - type: map_at_5 + value: 84.69 + - type: mrr_at_1 + value: 82.44 + - type: mrr_at_10 + value: 88.44180555555535 + - type: mrr_at_100 + value: 88.5259878177649 + - type: mrr_at_1000 + value: 88.52646525699359 + - type: mrr_at_20 + value: 88.50788583080123 + - type: mrr_at_3 + value: 87.56999999999981 + - type: mrr_at_5 + value: 88.15649999999971 + - type: nauc_map_at_1000_diff1 + value: 77.02347735380722 + - type: nauc_map_at_1000_max + value: 52.50528969537401 + - type: nauc_map_at_1000_std + value: -59.15658805657328 + - type: nauc_map_at_100_diff1 + value: 77.02743886363463 + - type: nauc_map_at_100_max + value: 52.53106759298396 + - type: nauc_map_at_100_std + value: -59.20242993039629 + - type: nauc_map_at_10_diff1 + value: 77.33137216898336 + - type: nauc_map_at_10_max + value: 53.062379899702215 + - type: nauc_map_at_10_std + value: -61.37809972228543 + - type: nauc_map_at_1_diff1 + value: 81.2961730531142 + - type: nauc_map_at_1_max + value: 40.78570417682092 + - type: nauc_map_at_1_std + value: -51.502144011218256 + - type: nauc_map_at_20_diff1 + value: 77.12910356944846 + - type: nauc_map_at_20_max + value: 52.81383565976242 + - type: nauc_map_at_20_std + value: -60.12499749631829 + - type: nauc_map_at_3_diff1 + value: 77.96896210205782 + - type: nauc_map_at_3_max + value: 50.94916192710113 + - type: nauc_map_at_3_std + value: -62.69621326487791 + - type: nauc_map_at_5_diff1 + value: 77.6350142444304 + - type: nauc_map_at_5_max + value: 52.666236034196125 + - type: nauc_map_at_5_std + value: -62.77810131126452 + - type: nauc_mrr_at_1000_diff1 + value: 77.64165650846945 + - type: nauc_mrr_at_1000_max + value: 53.381878335917996 + - type: nauc_mrr_at_1000_std + value: -54.6435148786942 + - type: nauc_mrr_at_100_diff1 + value: 77.64123122931281 + - type: nauc_mrr_at_100_max + value: 53.38307144160531 + - type: nauc_mrr_at_100_std + value: -54.64425729988428 + - type: nauc_mrr_at_10_diff1 + value: 77.64793515865411 + - type: nauc_mrr_at_10_max + value: 53.509861622234624 + - type: nauc_mrr_at_10_std + value: -54.839545580537184 + - type: nauc_mrr_at_1_diff1 + value: 78.30258683055786 + - type: nauc_mrr_at_1_max + value: 51.32420685629546 + - type: nauc_mrr_at_1_std + value: -51.6241218906987 + - type: nauc_mrr_at_20_diff1 + value: 77.65433326417731 + - type: nauc_mrr_at_20_max + value: 53.41947976154459 + - type: nauc_mrr_at_20_std + value: -54.69092571190093 + - type: nauc_mrr_at_3_diff1 + value: 77.41224313226685 + - type: nauc_mrr_at_3_max + value: 53.5653203560158 + - type: nauc_mrr_at_3_std + value: -55.152426376121646 + - type: nauc_mrr_at_5_diff1 + value: 77.59534745183089 + - type: nauc_mrr_at_5_max + value: 53.69406790085286 + - type: nauc_mrr_at_5_std + value: -55.17130332175976 + - type: nauc_ndcg_at_1000_diff1 + value: 76.89044514943116 + - type: nauc_ndcg_at_1000_max + value: 53.02123077346147 + - type: nauc_ndcg_at_1000_std + value: -57.04415682674073 + - type: nauc_ndcg_at_100_diff1 + value: 76.89926387011819 + - type: nauc_ndcg_at_100_max + value: 53.18291144905412 + - type: nauc_ndcg_at_100_std + value: -57.27063408619188 + - type: nauc_ndcg_at_10_diff1 + value: 77.14008962226934 + - type: nauc_ndcg_at_10_max + value: 54.262936357206314 + - type: nauc_ndcg_at_10_std + value: -61.27204318160817 + - type: nauc_ndcg_at_1_diff1 + value: 78.24468706459153 + - type: nauc_ndcg_at_1_max + value: 51.47597998468433 + - type: nauc_ndcg_at_1_std + value: -51.59976366261999 + - type: nauc_ndcg_at_20_diff1 + value: 77.03388178639365 + - type: nauc_ndcg_at_20_max + value: 53.874260462208824 + - type: nauc_ndcg_at_20_std + value: -59.3986850177709 + - type: nauc_ndcg_at_3_diff1 + value: 76.47273825311073 + - type: nauc_ndcg_at_3_max + value: 52.92968301991306 + - type: nauc_ndcg_at_3_std + value: -60.52113274738622 + - type: nauc_ndcg_at_5_diff1 + value: 76.9512485429525 + - type: nauc_ndcg_at_5_max + value: 54.05480007105696 + - type: nauc_ndcg_at_5_std + value: -62.101739581125095 + - type: nauc_precision_at_1000_diff1 + value: -45.482326263015544 + - type: nauc_precision_at_1000_max + value: -23.53270930061391 + - type: nauc_precision_at_1000_std + value: 44.86415952225683 + - type: nauc_precision_at_100_diff1 + value: -45.36791160696435 + - type: nauc_precision_at_100_max + value: -22.41480915177284 + - type: nauc_precision_at_100_std + value: 44.06098297365805 + - type: nauc_precision_at_10_diff1 + value: -41.02215426458915 + - type: nauc_precision_at_10_max + value: -11.954140132948595 + - type: nauc_precision_at_10_std + value: 29.876246822345028 + - type: nauc_precision_at_1_diff1 + value: 78.24468706459153 + - type: nauc_precision_at_1_max + value: 51.47597998468433 + - type: nauc_precision_at_1_std + value: -51.59976366261999 + - type: nauc_precision_at_20_diff1 + value: -43.7802413074972 + - type: nauc_precision_at_20_max + value: -17.108686030966407 + - type: nauc_precision_at_20_std + value: 37.07842954746043 + - type: nauc_precision_at_3_diff1 + value: -22.12253574553909 + - type: nauc_precision_at_3_max + value: 5.630524893868261 + - type: nauc_precision_at_3_std + value: 5.11003779556345 + - type: nauc_precision_at_5_diff1 + value: -33.72286875553289 + - type: nauc_precision_at_5_max + value: -3.8038422308600275 + - type: nauc_precision_at_5_std + value: 18.140228052542206 + - type: nauc_recall_at_1000_diff1 + value: 86.92810457515144 + - type: nauc_recall_at_1000_max + value: 35.807656396192186 + - type: nauc_recall_at_1000_std + value: -115.17273576291149 + - type: nauc_recall_at_100_diff1 + value: 72.77594352646085 + - type: nauc_recall_at_100_max + value: 74.60619074499881 + - type: nauc_recall_at_100_std + value: -81.56676626789539 + - type: nauc_recall_at_10_diff1 + value: 75.24537480810442 + - type: nauc_recall_at_10_max + value: 64.38890082633442 + - type: nauc_recall_at_10_std + value: -89.44819841132292 + - type: nauc_recall_at_1_diff1 + value: 81.2961730531142 + - type: nauc_recall_at_1_max + value: 40.78570417682092 + - type: nauc_recall_at_1_std + value: -51.502144011218256 + - type: nauc_recall_at_20_diff1 + value: 75.348250677366 + - type: nauc_recall_at_20_max + value: 67.44998693528419 + - type: nauc_recall_at_20_std + value: -90.77232272507436 + - type: nauc_recall_at_3_diff1 + value: 74.35919793460943 + - type: nauc_recall_at_3_max + value: 51.61715987358313 + - type: nauc_recall_at_3_std + value: -71.40752838357291 + - type: nauc_recall_at_5_diff1 + value: 73.93784267992228 + - type: nauc_recall_at_5_max + value: 57.228558742462596 + - type: nauc_recall_at_5_std + value: -79.04746057018535 + - type: ndcg_at_1 + value: 82.47 + - type: ndcg_at_10 + value: 89.415 + - type: ndcg_at_100 + value: 90.462 + - type: ndcg_at_1000 + value: 90.52 + - type: ndcg_at_20 + value: 90.012 + - type: ndcg_at_3 + value: 86.619 + - type: ndcg_at_5 + value: 88.126 + - type: precision_at_1 + value: 82.47 + - type: precision_at_10 + value: 13.635 + - type: precision_at_100 + value: 1.544 + - type: precision_at_1000 + value: 0.157 + - type: precision_at_20 + value: 7.217999999999999 + - type: precision_at_3 + value: 37.967 + - type: precision_at_5 + value: 24.968 + - type: recall_at_1 + value: 71.58200000000001 + - type: recall_at_10 + value: 96.322 + - type: recall_at_100 + value: 99.763 + - type: recall_at_1000 + value: 99.99900000000001 + - type: recall_at_20 + value: 98.225 + - type: recall_at_3 + value: 88.274 + - type: recall_at_5 + value: 92.55 + task: + type: Retrieval + - dataset: + config: default + name: MTEB RedditClustering (default) + revision: 24640382cdbf8abc73003fb0fa6d111a705499eb + split: test + type: mteb/reddit-clustering + metrics: + - type: main_score + value: 57.43849582318883 + - type: v_measure + value: 57.43849582318883 + - type: v_measure_std + value: 4.938915046379781 + task: + type: Clustering + - dataset: + config: default + name: MTEB RedditClusteringP2P (default) + revision: 385e3cb46b4cfa89021f56c4380204149d0efe33 + split: test + type: mteb/reddit-clustering-p2p + metrics: + - type: main_score + value: 59.2315677494836 + - type: v_measure + value: 59.2315677494836 + - type: v_measure_std + value: 11.779242220000999 + task: + type: Clustering + - dataset: + config: default + name: MTEB SCIDOCS (default) + revision: f8c2fcf00f625baaa80f62ec5bd9e1fff3b8ae88 + split: test + type: mteb/scidocs + metrics: + - type: main_score + value: 16.587 + - type: map_at_1 + value: 4.063 + - type: map_at_10 + value: 9.532 + - type: map_at_100 + value: 11.299 + - type: map_at_1000 + value: 11.558 + - type: map_at_20 + value: 10.42 + - type: map_at_3 + value: 6.865 + - type: map_at_5 + value: 8.161999999999999 + - type: mrr_at_1 + value: 19.900000000000002 + - type: mrr_at_10 + value: 29.43638888888887 + - type: mrr_at_100 + value: 30.527927192673722 + - type: mrr_at_1000 + value: 30.60031506511264 + - type: mrr_at_20 + value: 30.037529826554575 + - type: mrr_at_3 + value: 26.066666666666688 + - type: mrr_at_5 + value: 28.126666666666672 + - type: nauc_map_at_1000_diff1 + value: 15.224708729351386 + - type: nauc_map_at_1000_max + value: 18.993228897455452 + - type: nauc_map_at_1000_std + value: 16.1566347502784 + - type: nauc_map_at_100_diff1 + value: 15.222215926602361 + - type: nauc_map_at_100_max + value: 18.788592627965286 + - type: nauc_map_at_100_std + value: 15.81868227825019 + - type: nauc_map_at_10_diff1 + value: 15.065320177842503 + - type: nauc_map_at_10_max + value: 18.00293887596623 + - type: nauc_map_at_10_std + value: 11.554719951667101 + - type: nauc_map_at_1_diff1 + value: 22.731306984633864 + - type: nauc_map_at_1_max + value: 16.732450136016038 + - type: nauc_map_at_1_std + value: 6.243799036200585 + - type: nauc_map_at_20_diff1 + value: 14.88897763312719 + - type: nauc_map_at_20_max + value: 18.50397067974761 + - type: nauc_map_at_20_std + value: 14.0230354195294 + - type: nauc_map_at_3_diff1 + value: 17.004711972329133 + - type: nauc_map_at_3_max + value: 17.329227393348987 + - type: nauc_map_at_3_std + value: 7.168898045642296 + - type: nauc_map_at_5_diff1 + value: 16.343421828613344 + - type: nauc_map_at_5_max + value: 18.349396456495015 + - type: nauc_map_at_5_std + value: 9.218903337569738 + - type: nauc_mrr_at_1000_diff1 + value: 19.995142890037233 + - type: nauc_mrr_at_1000_max + value: 17.140401977270585 + - type: nauc_mrr_at_1000_std + value: 10.758272954526111 + - type: nauc_mrr_at_100_diff1 + value: 20.012275814149728 + - type: nauc_mrr_at_100_max + value: 17.143245269109613 + - type: nauc_mrr_at_100_std + value: 10.795772735666478 + - type: nauc_mrr_at_10_diff1 + value: 19.87714869402365 + - type: nauc_mrr_at_10_max + value: 16.969158476979743 + - type: nauc_mrr_at_10_std + value: 10.338406385049176 + - type: nauc_mrr_at_1_diff1 + value: 23.251309299389074 + - type: nauc_mrr_at_1_max + value: 17.459594477844437 + - type: nauc_mrr_at_1_std + value: 6.691812563114626 + - type: nauc_mrr_at_20_diff1 + value: 19.965416788960702 + - type: nauc_mrr_at_20_max + value: 17.15577552728726 + - type: nauc_mrr_at_20_std + value: 10.792284990205749 + - type: nauc_mrr_at_3_diff1 + value: 19.970106067665583 + - type: nauc_mrr_at_3_max + value: 16.276189003261493 + - type: nauc_mrr_at_3_std + value: 8.273713503756783 + - type: nauc_mrr_at_5_diff1 + value: 19.49236532733627 + - type: nauc_mrr_at_5_max + value: 16.548593440848105 + - type: nauc_mrr_at_5_std + value: 9.841948745125203 + - type: nauc_ndcg_at_1000_diff1 + value: 16.42963352199297 + - type: nauc_ndcg_at_1000_max + value: 21.46007498626136 + - type: nauc_ndcg_at_1000_std + value: 24.93505486916626 + - type: nauc_ndcg_at_100_diff1 + value: 16.548440641806003 + - type: nauc_ndcg_at_100_max + value: 19.76110620635574 + - type: nauc_ndcg_at_100_std + value: 22.919214672556258 + - type: nauc_ndcg_at_10_diff1 + value: 15.48853670082967 + - type: nauc_ndcg_at_10_max + value: 17.665457341212623 + - type: nauc_ndcg_at_10_std + value: 14.006969148938953 + - type: nauc_ndcg_at_1_diff1 + value: 23.251309299389074 + - type: nauc_ndcg_at_1_max + value: 17.459594477844437 + - type: nauc_ndcg_at_1_std + value: 6.691812563114626 + - type: nauc_ndcg_at_20_diff1 + value: 15.559225739122894 + - type: nauc_ndcg_at_20_max + value: 18.488951475704425 + - type: nauc_ndcg_at_20_std + value: 17.71628742537456 + - type: nauc_ndcg_at_3_diff1 + value: 17.72471697922835 + - type: nauc_ndcg_at_3_max + value: 16.944128543846666 + - type: nauc_ndcg_at_3_std + value: 8.3024474128589 + - type: nauc_ndcg_at_5_diff1 + value: 16.91525688512956 + - type: nauc_ndcg_at_5_max + value: 17.787716347748862 + - type: nauc_ndcg_at_5_std + value: 10.947063917850688 + - type: nauc_precision_at_1000_diff1 + value: 10.304319268584148 + - type: nauc_precision_at_1000_max + value: 23.43757587322904 + - type: nauc_precision_at_1000_std + value: 39.919860831992295 + - type: nauc_precision_at_100_diff1 + value: 13.030842771655799 + - type: nauc_precision_at_100_max + value: 19.55241538006816 + - type: nauc_precision_at_100_std + value: 34.65618147576725 + - type: nauc_precision_at_10_diff1 + value: 11.22839280596361 + - type: nauc_precision_at_10_max + value: 16.968576626532755 + - type: nauc_precision_at_10_std + value: 18.14370842767139 + - type: nauc_precision_at_1_diff1 + value: 23.251309299389074 + - type: nauc_precision_at_1_max + value: 17.459594477844437 + - type: nauc_precision_at_1_std + value: 6.691812563114626 + - type: nauc_precision_at_20_diff1 + value: 11.589143818033657 + - type: nauc_precision_at_20_max + value: 18.119338869446032 + - type: nauc_precision_at_20_std + value: 24.597721181518853 + - type: nauc_precision_at_3_diff1 + value: 15.33289261920458 + - type: nauc_precision_at_3_max + value: 16.655587432649895 + - type: nauc_precision_at_3_std + value: 9.149298969907441 + - type: nauc_precision_at_5_diff1 + value: 14.408517897116486 + - type: nauc_precision_at_5_max + value: 18.250676205105957 + - type: nauc_precision_at_5_std + value: 13.676165264085213 + - type: nauc_recall_at_1000_diff1 + value: 10.225901462029961 + - type: nauc_recall_at_1000_max + value: 22.889994273602174 + - type: nauc_recall_at_1000_std + value: 40.882321515860376 + - type: nauc_recall_at_100_diff1 + value: 12.57203864936356 + - type: nauc_recall_at_100_max + value: 18.923199845897976 + - type: nauc_recall_at_100_std + value: 34.60652900859492 + - type: nauc_recall_at_10_diff1 + value: 10.951620928723166 + - type: nauc_recall_at_10_max + value: 16.489530994203765 + - type: nauc_recall_at_10_std + value: 17.883112455067742 + - type: nauc_recall_at_1_diff1 + value: 22.731306984633864 + - type: nauc_recall_at_1_max + value: 16.732450136016038 + - type: nauc_recall_at_1_std + value: 6.243799036200585 + - type: nauc_recall_at_20_diff1 + value: 11.18720817914625 + - type: nauc_recall_at_20_max + value: 17.582627085681455 + - type: nauc_recall_at_20_std + value: 24.54606161002407 + - type: nauc_recall_at_3_diff1 + value: 14.754718730422384 + - type: nauc_recall_at_3_max + value: 16.02292066220047 + - type: nauc_recall_at_3_std + value: 8.74895886090382 + - type: nauc_recall_at_5_diff1 + value: 13.898773135499074 + - type: nauc_recall_at_5_max + value: 17.670636535187516 + - type: nauc_recall_at_5_std + value: 13.320309329809682 + - type: ndcg_at_1 + value: 19.900000000000002 + - type: ndcg_at_10 + value: 16.587 + - type: ndcg_at_100 + value: 23.785 + - type: ndcg_at_1000 + value: 28.939999999999998 + - type: ndcg_at_20 + value: 19.078 + - type: ndcg_at_3 + value: 15.531 + - type: ndcg_at_5 + value: 13.672 + - type: precision_at_1 + value: 19.900000000000002 + - type: precision_at_10 + value: 8.59 + - type: precision_at_100 + value: 1.9029999999999998 + - type: precision_at_1000 + value: 0.316 + - type: precision_at_20 + value: 5.74 + - type: precision_at_3 + value: 14.267 + - type: precision_at_5 + value: 11.88 + - type: recall_at_1 + value: 4.063 + - type: recall_at_10 + value: 17.432 + - type: recall_at_100 + value: 38.633 + - type: recall_at_1000 + value: 64.08500000000001 + - type: recall_at_20 + value: 23.283 + - type: recall_at_3 + value: 8.693 + - type: recall_at_5 + value: 12.078 + task: + type: Retrieval + - dataset: + config: default + name: MTEB SICK-R (default) + revision: 20a6d6f312dd54037fe07a32d58e5e168867909d + split: test + type: mteb/sickr-sts + metrics: + - type: cosine_pearson + value: 85.37258817504542 + - type: cosine_spearman + value: 82.50244918645149 + - type: euclidean_pearson + value: 82.52677722125374 + - type: euclidean_spearman + value: 82.50244558654384 + - type: main_score + value: 82.50244918645149 + - type: manhattan_pearson + value: 82.51133832632853 + - type: manhattan_spearman + value: 82.49662550471665 + - type: pearson + value: 85.37258800555489 + - type: spearman + value: 82.5024428672302 + task: + type: STS + - dataset: + config: default + name: MTEB STS12 (default) + revision: a0d554a64d88156834ff5ae9920b964011b16384 + split: test + type: mteb/sts12-sts + metrics: + - type: cosine_pearson + value: 83.86076828442609 + - type: cosine_spearman + value: 77.04219479271217 + - type: euclidean_pearson + value: 80.34163032062519 + - type: euclidean_spearman + value: 77.04219479271217 + - type: main_score + value: 77.04219479271217 + - type: manhattan_pearson + value: 80.35439040706335 + - type: manhattan_spearman + value: 77.0387845391881 + - type: pearson + value: 83.86076826699718 + - type: spearman + value: 77.04202306291008 + task: + type: STS + - dataset: + config: default + name: MTEB STS13 (default) + revision: 7e90230a92c190f1bf69ae9002b8cea547a64cca + split: test + type: mteb/sts13-sts + metrics: + - type: cosine_pearson + value: 83.7972617126119 + - type: cosine_spearman + value: 84.55897849397878 + - type: euclidean_pearson + value: 84.42295469680549 + - type: euclidean_spearman + value: 84.55897849397878 + - type: main_score + value: 84.55897849397878 + - type: manhattan_pearson + value: 84.44235285346014 + - type: manhattan_spearman + value: 84.58932270494438 + - type: pearson + value: 83.79726180782596 + - type: spearman + value: 84.55897841881523 + task: + type: STS + - dataset: + config: default + name: MTEB STS14 (default) + revision: 6031580fec1f6af667f0bd2da0a551cf4f0b2375 + split: test + type: mteb/sts14-sts + metrics: + - type: cosine_pearson + value: 83.00555258054544 + - type: cosine_spearman + value: 82.11575580351311 + - type: euclidean_pearson + value: 82.56990846716685 + - type: euclidean_spearman + value: 82.11574144976387 + - type: main_score + value: 82.11575580351311 + - type: manhattan_pearson + value: 82.6157565067795 + - type: manhattan_spearman + value: 82.17772298987957 + - type: pearson + value: 83.00555236868006 + - type: spearman + value: 82.11574276168872 + task: + type: STS + - dataset: + config: default + name: MTEB STS15 (default) + revision: ae752c7c21bf194d8b67fd573edf7ae58183cbe3 + split: test + type: mteb/sts15-sts + metrics: + - type: cosine_pearson + value: 87.69005965005643 + - type: cosine_spearman + value: 88.13883547463267 + - type: euclidean_pearson + value: 87.76689475304703 + - type: euclidean_spearman + value: 88.13883547463267 + - type: main_score + value: 88.13883547463267 + - type: manhattan_pearson + value: 87.76546363057692 + - type: manhattan_spearman + value: 88.13616648890255 + - type: pearson + value: 87.69005919220253 + - type: spearman + value: 88.13882616237342 + task: + type: STS + - dataset: + config: default + name: MTEB STS16 (default) + revision: 4d8694f8f0e0100860b497b999b3dbed754a0513 + split: test + type: mteb/sts16-sts + metrics: + - type: cosine_pearson + value: 84.62255514364988 + - type: cosine_spearman + value: 86.37608304813216 + - type: euclidean_pearson + value: 86.00302798829127 + - type: euclidean_spearman + value: 86.37608304813216 + - type: main_score + value: 86.37608304813216 + - type: manhattan_pearson + value: 85.9826031225404 + - type: manhattan_spearman + value: 86.3468720939325 + - type: pearson + value: 84.62255415297004 + - type: spearman + value: 86.37608242680406 + task: + type: STS + - dataset: + config: en-tr + name: MTEB STS17 (en-tr) + revision: faeb762787bd10488a50c8b5be4a3b82e411949c + split: test + type: mteb/sts17-crosslingual-sts + metrics: + - type: cosine_pearson + value: 38.13670940689118 + - type: cosine_spearman + value: 36.22780145157299 + - type: euclidean_pearson + value: 37.7571546803747 + - type: euclidean_spearman + value: 36.22780145157299 + - type: main_score + value: 36.22780145157299 + - type: manhattan_pearson + value: 37.52000840210691 + - type: manhattan_spearman + value: 36.27024703052964 + - type: pearson + value: 38.13672784070733 + - type: spearman + value: 36.22780145157299 + task: + type: STS + - dataset: + config: en-ar + name: MTEB STS17 (en-ar) + revision: faeb762787bd10488a50c8b5be4a3b82e411949c + split: test + type: mteb/sts17-crosslingual-sts + metrics: + - type: cosine_pearson + value: 36.491385057961466 + - type: cosine_spearman + value: 34.96834670169282 + - type: euclidean_pearson + value: 36.39474999881891 + - type: euclidean_spearman + value: 34.96834670169282 + - type: main_score + value: 34.96834670169282 + - type: manhattan_pearson + value: 36.56940423856487 + - type: manhattan_spearman + value: 35.10988238317917 + - type: pearson + value: 36.491387452032605 + - type: spearman + value: 34.96834670169282 + task: + type: STS + - dataset: + config: nl-en + name: MTEB STS17 (nl-en) + revision: faeb762787bd10488a50c8b5be4a3b82e411949c + split: test + type: mteb/sts17-crosslingual-sts + metrics: + - type: cosine_pearson + value: 78.44900368177643 + - type: cosine_spearman + value: 78.38602937602764 + - type: euclidean_pearson + value: 78.15471499279187 + - type: euclidean_spearman + value: 78.38602937602764 + - type: main_score + value: 78.38602937602764 + - type: manhattan_pearson + value: 78.2663671956205 + - type: manhattan_spearman + value: 78.4597181948685 + - type: pearson + value: 78.44900356554596 + - type: spearman + value: 78.38602937602764 + task: + type: STS + - dataset: + config: fr-en + name: MTEB STS17 (fr-en) + revision: faeb762787bd10488a50c8b5be4a3b82e411949c + split: test + type: mteb/sts17-crosslingual-sts + metrics: + - type: cosine_pearson + value: 81.90350869144417 + - type: cosine_spearman + value: 82.22188411953913 + - type: euclidean_pearson + value: 81.7405323006859 + - type: euclidean_spearman + value: 82.22188411953913 + - type: main_score + value: 82.22188411953913 + - type: manhattan_pearson + value: 81.7736513878681 + - type: manhattan_spearman + value: 82.21219732645207 + - type: pearson + value: 81.90350588377795 + - type: spearman + value: 82.22188411953913 + task: + type: STS + - dataset: + config: it-en + name: MTEB STS17 (it-en) + revision: faeb762787bd10488a50c8b5be4a3b82e411949c + split: test + type: mteb/sts17-crosslingual-sts + metrics: + - type: cosine_pearson + value: 81.30807376499745 + - type: cosine_spearman + value: 81.73327766346907 + - type: euclidean_pearson + value: 80.67143201419856 + - type: euclidean_spearman + value: 81.73327766346907 + - type: main_score + value: 81.73327766346907 + - type: manhattan_pearson + value: 80.62633905498645 + - type: manhattan_spearman + value: 81.7459243099994 + - type: pearson + value: 81.3080706227506 + - type: spearman + value: 81.73327766346907 + task: + type: STS + - dataset: + config: en-de + name: MTEB STS17 (en-de) + revision: faeb762787bd10488a50c8b5be4a3b82e411949c + split: test + type: mteb/sts17-crosslingual-sts + metrics: + - type: cosine_pearson + value: 82.52002675704622 + - type: cosine_spearman + value: 82.5229050586851 + - type: euclidean_pearson + value: 82.5466178353125 + - type: euclidean_spearman + value: 82.5229050586851 + - type: main_score + value: 82.5229050586851 + - type: manhattan_pearson + value: 82.61119115887921 + - type: manhattan_spearman + value: 82.62561581713207 + - type: pearson + value: 82.52002832656315 + - type: spearman + value: 82.5229050586851 + task: + type: STS + - dataset: + config: en-en + name: MTEB STS17 (en-en) + revision: faeb762787bd10488a50c8b5be4a3b82e411949c + split: test + type: mteb/sts17-crosslingual-sts + metrics: + - type: cosine_pearson + value: 91.12303081825787 + - type: cosine_spearman + value: 91.4879551814514 + - type: euclidean_pearson + value: 91.3598757815376 + - type: euclidean_spearman + value: 91.4879551814514 + - type: main_score + value: 91.4879551814514 + - type: manhattan_pearson + value: 91.39588782135479 + - type: manhattan_spearman + value: 91.58982026748599 + - type: pearson + value: 91.12302950571028 + - type: spearman + value: 91.4879551814514 + task: + type: STS + - dataset: + config: es-en + name: MTEB STS17 (es-en) + revision: faeb762787bd10488a50c8b5be4a3b82e411949c + split: test + type: mteb/sts17-crosslingual-sts + metrics: + - type: cosine_pearson + value: 81.85413285652889 + - type: cosine_spearman + value: 82.00141539229648 + - type: euclidean_pearson + value: 82.3535425292008 + - type: euclidean_spearman + value: 82.00141539229648 + - type: main_score + value: 82.00141539229648 + - type: manhattan_pearson + value: 82.33495932102197 + - type: manhattan_spearman + value: 81.99868592786007 + - type: pearson + value: 81.8541318915654 + - type: spearman + value: 82.00141539229648 + task: + type: STS + - dataset: + config: en + name: MTEB STS22 (en) + revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3 + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: cosine_pearson + value: 69.08206380486686 + - type: cosine_spearman + value: 68.45269476114063 + - type: euclidean_pearson + value: 69.6318276467813 + - type: euclidean_spearman + value: 68.45269476114063 + - type: main_score + value: 68.45269476114063 + - type: manhattan_pearson + value: 69.60934442056693 + - type: manhattan_spearman + value: 68.47442494856733 + - type: pearson + value: 69.08206389529616 + - type: spearman + value: 68.45269476114063 + task: + type: STS + - dataset: + config: pl-en + name: MTEB STS22 (pl-en) + revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3 + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: cosine_pearson + value: 70.60323219651768 + - type: cosine_spearman + value: 70.99268706934568 + - type: euclidean_pearson + value: 71.64084169577119 + - type: euclidean_spearman + value: 70.99268706934568 + - type: main_score + value: 70.99268706934568 + - type: manhattan_pearson + value: 71.5830369155138 + - type: manhattan_spearman + value: 70.95622179127955 + - type: pearson + value: 70.60324838804019 + - type: spearman + value: 70.99268706934568 + task: + type: STS + - dataset: + config: zh-en + name: MTEB STS22 (zh-en) + revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3 + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: cosine_pearson + value: 65.76137993670457 + - type: cosine_spearman + value: 66.90998671209492 + - type: euclidean_pearson + value: 66.32399281597182 + - type: euclidean_spearman + value: 66.90998671209492 + - type: main_score + value: 66.90998671209492 + - type: manhattan_pearson + value: 65.8797730629511 + - type: manhattan_spearman + value: 66.65399968531256 + - type: pearson + value: 65.76137914069382 + - type: spearman + value: 66.90998671209492 + task: + type: STS + - dataset: + config: es-en + name: MTEB STS22 (es-en) + revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3 + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: cosine_pearson + value: 75.42722764447008 + - type: cosine_spearman + value: 78.00649363768557 + - type: euclidean_pearson + value: 77.40814200686553 + - type: euclidean_spearman + value: 78.00649363768557 + - type: main_score + value: 78.00649363768557 + - type: manhattan_pearson + value: 77.49653043698983 + - type: manhattan_spearman + value: 78.12745373892952 + - type: pearson + value: 75.42722634870594 + - type: spearman + value: 78.00649363768557 + task: + type: STS + - dataset: + config: de-en + name: MTEB STS22 (de-en) + revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3 + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: cosine_pearson + value: 63.41020814512166 + - type: cosine_spearman + value: 58.85143370722147 + - type: euclidean_pearson + value: 66.30509221895342 + - type: euclidean_spearman + value: 58.85143370722147 + - type: main_score + value: 58.85143370722147 + - type: manhattan_pearson + value: 66.05644174196435 + - type: manhattan_spearman + value: 58.80931182369843 + - type: pearson + value: 63.410207932888675 + - type: spearman + value: 58.85143370722147 + task: + type: STS + - dataset: + config: default + name: MTEB STSBenchmark (default) + revision: b0fddb56ed78048fa8b90373c8a3cfc37b684831 + split: test + type: mteb/stsbenchmark-sts + metrics: + - type: cosine_pearson + value: 85.55843349363072 + - type: cosine_spearman + value: 86.50893238975354 + - type: euclidean_pearson + value: 86.45377913994999 + - type: euclidean_spearman + value: 86.50893238975354 + - type: main_score + value: 86.50893238975354 + - type: manhattan_pearson + value: 86.42194891960956 + - type: manhattan_spearman + value: 86.47423652611648 + - type: pearson + value: 85.55843296641845 + - type: spearman + value: 86.50893238975354 + task: + type: STS + - dataset: + config: default + name: MTEB SciDocsRR (default) + revision: d3c5e1fc0b855ab6097bf1cda04dd73947d7caab + split: test + type: mteb/scidocs-reranking + metrics: + - type: main_score + value: 77.21323436562753 + - type: map + value: 77.21323436562753 + - type: mrr + value: 93.0430462783404 + - type: nAUC_map_diff1 + value: 6.056824388247704 + - type: nAUC_map_max + value: 48.89047615875819 + - type: nAUC_map_std + value: 51.569488595258115 + - type: nAUC_mrr_diff1 + value: 43.66391377805882 + - type: nAUC_mrr_max + value: 71.70290781454408 + - type: nAUC_mrr_std + value: 55.41201318211277 + task: + type: Reranking + - dataset: + config: default + name: MTEB SciFact (default) + revision: 0228b52cf27578f30900b9e5271d331663a030d7 + split: test + type: mteb/scifact + metrics: + - type: main_score + value: 69.697 + - type: map_at_1 + value: 53.317 + - type: map_at_10 + value: 64.274 + - type: map_at_100 + value: 64.776 + - type: map_at_1000 + value: 64.802 + - type: map_at_20 + value: 64.575 + - type: map_at_3 + value: 61.352 + - type: map_at_5 + value: 62.735 + - type: mrr_at_1 + value: 56.00000000000001 + - type: mrr_at_10 + value: 65.48465608465608 + - type: mrr_at_100 + value: 65.8353737956075 + - type: mrr_at_1000 + value: 65.85697172702244 + - type: mrr_at_20 + value: 65.68153665506607 + - type: mrr_at_3 + value: 63.111111111111114 + - type: mrr_at_5 + value: 64.1611111111111 + - type: nauc_map_at_1000_diff1 + value: 65.70971281458692 + - type: nauc_map_at_1000_max + value: 52.77579746646612 + - type: nauc_map_at_1000_std + value: -14.911541839598094 + - type: nauc_map_at_100_diff1 + value: 65.67734726516375 + - type: nauc_map_at_100_max + value: 52.778410830493286 + - type: nauc_map_at_100_std + value: -14.931185127354155 + - type: nauc_map_at_10_diff1 + value: 65.68182931361449 + - type: nauc_map_at_10_max + value: 52.60659920767932 + - type: nauc_map_at_10_std + value: -15.206946412783036 + - type: nauc_map_at_1_diff1 + value: 70.21802988115712 + - type: nauc_map_at_1_max + value: 48.758330810557624 + - type: nauc_map_at_1_std + value: -15.708168988961143 + - type: nauc_map_at_20_diff1 + value: 65.61751128689156 + - type: nauc_map_at_20_max + value: 52.671200411997965 + - type: nauc_map_at_20_std + value: -15.01183827390203 + - type: nauc_map_at_3_diff1 + value: 66.38011913916482 + - type: nauc_map_at_3_max + value: 50.91046543627231 + - type: nauc_map_at_3_std + value: -18.832004190575258 + - type: nauc_map_at_5_diff1 + value: 65.63602285008251 + - type: nauc_map_at_5_max + value: 51.6836308304939 + - type: nauc_map_at_5_std + value: -16.451934981619168 + - type: nauc_mrr_at_1000_diff1 + value: 65.34401147188046 + - type: nauc_mrr_at_1000_max + value: 54.494706358826406 + - type: nauc_mrr_at_1000_std + value: -11.834008649333757 + - type: nauc_mrr_at_100_diff1 + value: 65.31625030575647 + - type: nauc_mrr_at_100_max + value: 54.48714557449185 + - type: nauc_mrr_at_100_std + value: -11.851001814641718 + - type: nauc_mrr_at_10_diff1 + value: 65.30725362559637 + - type: nauc_mrr_at_10_max + value: 54.57353395283738 + - type: nauc_mrr_at_10_std + value: -11.766814552703565 + - type: nauc_mrr_at_1_diff1 + value: 70.13636535695356 + - type: nauc_mrr_at_1_max + value: 54.15956923309864 + - type: nauc_mrr_at_1_std + value: -8.969736543265979 + - type: nauc_mrr_at_20_diff1 + value: 65.27845535464311 + - type: nauc_mrr_at_20_max + value: 54.493867387012116 + - type: nauc_mrr_at_20_std + value: -11.744643224007408 + - type: nauc_mrr_at_3_diff1 + value: 65.48053290333581 + - type: nauc_mrr_at_3_max + value: 54.31776368130934 + - type: nauc_mrr_at_3_std + value: -13.451689208387782 + - type: nauc_mrr_at_5_diff1 + value: 65.05726639486943 + - type: nauc_mrr_at_5_max + value: 53.960177751491166 + - type: nauc_mrr_at_5_std + value: -12.46500245175413 + - type: nauc_ndcg_at_1000_diff1 + value: 64.51055568204761 + - type: nauc_ndcg_at_1000_max + value: 53.738423378550074 + - type: nauc_ndcg_at_1000_std + value: -13.47627107407215 + - type: nauc_ndcg_at_100_diff1 + value: 63.697876163120746 + - type: nauc_ndcg_at_100_max + value: 53.68294016869199 + - type: nauc_ndcg_at_100_std + value: -13.862491000992627 + - type: nauc_ndcg_at_10_diff1 + value: 63.59533647641533 + - type: nauc_ndcg_at_10_max + value: 53.74201238476266 + - type: nauc_ndcg_at_10_std + value: -13.993868727112787 + - type: nauc_ndcg_at_1_diff1 + value: 70.13636535695356 + - type: nauc_ndcg_at_1_max + value: 54.15956923309864 + - type: nauc_ndcg_at_1_std + value: -8.969736543265979 + - type: nauc_ndcg_at_20_diff1 + value: 63.448204407530326 + - type: nauc_ndcg_at_20_max + value: 53.68472504721487 + - type: nauc_ndcg_at_20_std + value: -13.428013898376184 + - type: nauc_ndcg_at_3_diff1 + value: 63.9886436930677 + - type: nauc_ndcg_at_3_max + value: 51.99495754002938 + - type: nauc_ndcg_at_3_std + value: -18.72944359507384 + - type: nauc_ndcg_at_5_diff1 + value: 63.33872281777501 + - type: nauc_ndcg_at_5_max + value: 51.88564924442115 + - type: nauc_ndcg_at_5_std + value: -16.38051976568326 + - type: nauc_precision_at_1000_diff1 + value: -23.34317145156963 + - type: nauc_precision_at_1000_max + value: 14.969763265980163 + - type: nauc_precision_at_1000_std + value: 51.74093413964622 + - type: nauc_precision_at_100_diff1 + value: -14.585055356119412 + - type: nauc_precision_at_100_max + value: 23.232676702163026 + - type: nauc_precision_at_100_std + value: 39.58188783975387 + - type: nauc_precision_at_10_diff1 + value: 9.810418017644905 + - type: nauc_precision_at_10_max + value: 39.53048624318149 + - type: nauc_precision_at_10_std + value: 26.33058869047298 + - type: nauc_precision_at_1_diff1 + value: 70.13636535695356 + - type: nauc_precision_at_1_max + value: 54.15956923309864 + - type: nauc_precision_at_1_std + value: -8.969736543265979 + - type: nauc_precision_at_20_diff1 + value: 1.9060283687943103 + - type: nauc_precision_at_20_max + value: 33.20302486541906 + - type: nauc_precision_at_20_std + value: 31.66549175425098 + - type: nauc_precision_at_3_diff1 + value: 39.618029703358296 + - type: nauc_precision_at_3_max + value: 48.178807947019884 + - type: nauc_precision_at_3_std + value: -5.987205611505178 + - type: nauc_precision_at_5_diff1 + value: 27.20057015659778 + - type: nauc_precision_at_5_max + value: 45.785332135744106 + - type: nauc_precision_at_5_std + value: 8.16720037489748 + - type: nauc_recall_at_1000_diff1 + value: .nan + - type: nauc_recall_at_1000_max + value: .nan + - type: nauc_recall_at_1000_std + value: .nan + - type: nauc_recall_at_100_diff1 + value: 32.93906370334051 + - type: nauc_recall_at_100_max + value: 49.68206260606445 + - type: nauc_recall_at_100_std + value: -25.196112508263564 + - type: nauc_recall_at_10_diff1 + value: 51.48855826931585 + - type: nauc_recall_at_10_max + value: 54.77628165208075 + - type: nauc_recall_at_10_std + value: -13.846155538880303 + - type: nauc_recall_at_1_diff1 + value: 70.21802988115712 + - type: nauc_recall_at_1_max + value: 48.758330810557624 + - type: nauc_recall_at_1_std + value: -15.708168988961143 + - type: nauc_recall_at_20_diff1 + value: 48.453190748012105 + - type: nauc_recall_at_20_max + value: 54.56752316536468 + - type: nauc_recall_at_20_std + value: -10.167892009270668 + - type: nauc_recall_at_3_diff1 + value: 58.15468335222506 + - type: nauc_recall_at_3_max + value: 48.29413731959822 + - type: nauc_recall_at_3_std + value: -27.8291190494878 + - type: nauc_recall_at_5_diff1 + value: 54.41460221884665 + - type: nauc_recall_at_5_max + value: 48.25356272590511 + - type: nauc_recall_at_5_std + value: -21.19596753064982 + - type: ndcg_at_1 + value: 56.00000000000001 + - type: ndcg_at_10 + value: 69.697 + - type: ndcg_at_100 + value: 71.907 + - type: ndcg_at_1000 + value: 72.50999999999999 + - type: ndcg_at_20 + value: 70.595 + - type: ndcg_at_3 + value: 64.322 + - type: ndcg_at_5 + value: 66.327 + - type: precision_at_1 + value: 56.00000000000001 + - type: precision_at_10 + value: 9.567 + - type: precision_at_100 + value: 1.08 + - type: precision_at_1000 + value: 0.11299999999999999 + - type: precision_at_20 + value: 5.0 + - type: precision_at_3 + value: 25.556 + - type: precision_at_5 + value: 16.667 + - type: recall_at_1 + value: 53.317 + - type: recall_at_10 + value: 85.233 + - type: recall_at_100 + value: 95.433 + - type: recall_at_1000 + value: 100.0 + - type: recall_at_20 + value: 88.522 + - type: recall_at_3 + value: 70.3 + - type: recall_at_5 + value: 75.43900000000001 + task: + type: Retrieval + - dataset: + config: default + name: MTEB SprintDuplicateQuestions (default) + revision: d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46 + split: test + type: mteb/sprintduplicatequestions-pairclassification + metrics: + - type: cosine_accuracy + value: 99.83069306930693 + - type: cosine_accuracy_threshold + value: 80.45519590377808 + - type: cosine_ap + value: 95.97544448108026 + - type: cosine_f1 + value: 91.27105666156203 + - type: cosine_f1_threshold + value: 80.45519590377808 + - type: cosine_precision + value: 93.2221063607925 + - type: cosine_recall + value: 89.4 + - type: dot_accuracy + value: 99.83069306930693 + - type: dot_accuracy_threshold + value: 80.45518398284912 + - type: dot_ap + value: 95.97544448108026 + - type: dot_f1 + value: 91.27105666156203 + - type: dot_f1_threshold + value: 80.45518398284912 + - type: dot_precision + value: 93.2221063607925 + - type: dot_recall + value: 89.4 + - type: euclidean_accuracy + value: 99.83069306930693 + - type: euclidean_accuracy_threshold + value: 62.521690130233765 + - type: euclidean_ap + value: 95.97544448108025 + - type: euclidean_f1 + value: 91.27105666156203 + - type: euclidean_f1_threshold + value: 62.521690130233765 + - type: euclidean_precision + value: 93.2221063607925 + - type: euclidean_recall + value: 89.4 + - type: main_score + value: 95.98181196721418 + - type: manhattan_accuracy + value: 99.83168316831683 + - type: manhattan_accuracy_threshold + value: 2223.4453201293945 + - type: manhattan_ap + value: 95.98181196721418 + - type: manhattan_f1 + value: 91.29098360655738 + - type: manhattan_f1_threshold + value: 2223.4453201293945 + - type: manhattan_precision + value: 93.59243697478992 + - type: manhattan_recall + value: 89.1 + - type: max_accuracy + value: 99.83168316831683 + - type: max_ap + value: 95.98181196721418 + - type: max_f1 + value: 91.29098360655738 + - type: max_precision + value: 93.59243697478992 + - type: max_recall + value: 89.4 + - type: similarity_accuracy + value: 99.83069306930693 + - type: similarity_accuracy_threshold + value: 2050.0064849853516 + - type: similarity_ap + value: 95.97544836417856 + - type: similarity_f1 + value: 91.27105666156203 + - type: similarity_f1_threshold + value: 2050.0064849853516 + - type: similarity_precision + value: 93.2221063607925 + - type: similarity_recall + value: 89.4 + task: + type: PairClassification + - dataset: + config: default + name: MTEB StackExchangeClustering (default) + revision: 6cbc1f7b2bc0622f2e39d2c77fa502909748c259 + split: test + type: mteb/stackexchange-clustering + metrics: + - type: main_score + value: 62.05884434327007 + - type: v_measure + value: 62.05884434327007 + - type: v_measure_std + value: 4.604982141730471 + task: + type: Clustering + - dataset: + config: default + name: MTEB StackExchangeClusteringP2P (default) + revision: 815ca46b2622cec33ccafc3735d572c266efdb44 + split: test + type: mteb/stackexchange-clustering-p2p + metrics: + - type: main_score + value: 32.518141744754665 + - type: v_measure + value: 32.518141744754665 + - type: v_measure_std + value: 1.8836100532310243 + task: + type: Clustering + - dataset: + config: default + name: MTEB StackOverflowDupQuestions (default) + revision: e185fbe320c72810689fc5848eb6114e1ef5ec69 + split: test + type: mteb/stackoverflowdupquestions-reranking + metrics: + - type: main_score + value: 47.325668601708905 + - type: map + value: 47.325668601708905 + - type: mrr + value: 47.858747665732956 + - type: nAUC_map_diff1 + value: 33.097824240228164 + - type: nAUC_map_max + value: 10.523947929131404 + - type: nAUC_map_std + value: 5.900195335200314 + - type: nAUC_mrr_diff1 + value: 33.31284375160937 + - type: nAUC_mrr_max + value: 11.17638353192733 + - type: nAUC_mrr_std + value: 6.435958892093385 + task: + type: Reranking + - dataset: + config: default + name: MTEB SummEval (default) + revision: cda12ad7615edc362dbf25a00fdd61d3b1eaf93c + split: test + type: mteb/summeval + metrics: + - type: cosine_pearson + value: 31.212243301102593 + - type: cosine_spearman + value: 30.79283739754972 + - type: dot_pearson + value: 31.21224262522688 + - type: dot_spearman + value: 30.79283739754972 + - type: main_score + value: 30.79283739754972 + - type: pearson + value: 31.212240810238555 + - type: spearman + value: 30.79283739754972 + task: + type: Summarization + - dataset: + config: default + name: MTEB TRECCOVID (default) + revision: bb9466bac8153a0349341eb1b22e06409e78ef4e + split: test + type: mteb/trec-covid + metrics: + - type: main_score + value: 80.573 + - type: map_at_1 + value: 0.248 + - type: map_at_10 + value: 2.077 + - type: map_at_100 + value: 12.859000000000002 + - type: map_at_1000 + value: 30.965999999999998 + - type: map_at_20 + value: 3.723 + - type: map_at_3 + value: 0.698 + - type: map_at_5 + value: 1.126 + - type: mrr_at_1 + value: 92.0 + - type: mrr_at_10 + value: 94.75 + - type: mrr_at_100 + value: 94.75 + - type: mrr_at_1000 + value: 94.75 + - type: mrr_at_20 + value: 94.75 + - type: mrr_at_3 + value: 94.0 + - type: mrr_at_5 + value: 94.5 + - type: nauc_map_at_1000_diff1 + value: -28.228949028851215 + - type: nauc_map_at_1000_max + value: 27.945590832994643 + - type: nauc_map_at_1000_std + value: 74.19160423735646 + - type: nauc_map_at_100_diff1 + value: -5.524042560180618 + - type: nauc_map_at_100_max + value: 10.269168473582734 + - type: nauc_map_at_100_std + value: 54.784660902763264 + - type: nauc_map_at_10_diff1 + value: 37.73581568246975 + - type: nauc_map_at_10_max + value: -4.1527002424638875 + - type: nauc_map_at_10_std + value: 12.541351630702893 + - type: nauc_map_at_1_diff1 + value: 40.71477618834696 + - type: nauc_map_at_1_max + value: -26.6188251260968 + - type: nauc_map_at_1_std + value: -6.003848303860384 + - type: nauc_map_at_20_diff1 + value: 23.317949672301513 + - type: nauc_map_at_20_max + value: 1.1096418483517367 + - type: nauc_map_at_20_std + value: 25.906723413815826 + - type: nauc_map_at_3_diff1 + value: 41.91731511073812 + - type: nauc_map_at_3_max + value: -20.725999474667393 + - type: nauc_map_at_3_std + value: 1.875550773349184 + - type: nauc_map_at_5_diff1 + value: 42.82250487520229 + - type: nauc_map_at_5_max + value: -13.114106018776624 + - type: nauc_map_at_5_std + value: 4.892105221021451 + - type: nauc_mrr_at_1000_diff1 + value: -3.9415766306522424 + - type: nauc_mrr_at_1000_max + value: -42.70152505446607 + - type: nauc_mrr_at_1000_std + value: 42.07905384375971 + - type: nauc_mrr_at_100_diff1 + value: -3.9415766306522424 + - type: nauc_mrr_at_100_max + value: -42.70152505446607 + - type: nauc_mrr_at_100_std + value: 42.07905384375971 + - type: nauc_mrr_at_10_diff1 + value: -3.9415766306522424 + - type: nauc_mrr_at_10_max + value: -42.70152505446607 + - type: nauc_mrr_at_10_std + value: 42.07905384375971 + - type: nauc_mrr_at_1_diff1 + value: -1.1671335200746933 + - type: nauc_mrr_at_1_max + value: -53.0462184873951 + - type: nauc_mrr_at_1_std + value: 38.62044817927166 + - type: nauc_mrr_at_20_diff1 + value: -3.9415766306522424 + - type: nauc_mrr_at_20_max + value: -42.70152505446607 + - type: nauc_mrr_at_20_std + value: 42.07905384375971 + - type: nauc_mrr_at_3_diff1 + value: -10.737628384687072 + - type: nauc_mrr_at_3_max + value: -47.69685652038563 + - type: nauc_mrr_at_3_std + value: 39.55804544039857 + - type: nauc_mrr_at_5_diff1 + value: -1.2435277141157486 + - type: nauc_mrr_at_5_max + value: -36.21509209744452 + - type: nauc_mrr_at_5_std + value: 44.71182412358887 + - type: nauc_ndcg_at_1000_diff1 + value: -15.11796647617939 + - type: nauc_ndcg_at_1000_max + value: 24.09364743671451 + - type: nauc_ndcg_at_1000_std + value: 70.41240897846865 + - type: nauc_ndcg_at_100_diff1 + value: -36.295842218504205 + - type: nauc_ndcg_at_100_max + value: 22.525544625207868 + - type: nauc_ndcg_at_100_std + value: 70.11594715332592 + - type: nauc_ndcg_at_10_diff1 + value: -9.634096563874689 + - type: nauc_ndcg_at_10_max + value: 22.505775413925296 + - type: nauc_ndcg_at_10_std + value: 55.596784740944194 + - type: nauc_ndcg_at_1_diff1 + value: 14.618723340567252 + - type: nauc_ndcg_at_1_max + value: -46.53520915961991 + - type: nauc_ndcg_at_1_std + value: 24.754920302327367 + - type: nauc_ndcg_at_20_diff1 + value: -25.71794298372455 + - type: nauc_ndcg_at_20_max + value: 24.97448784722531 + - type: nauc_ndcg_at_20_std + value: 65.6534762319551 + - type: nauc_ndcg_at_3_diff1 + value: -9.653477084520638 + - type: nauc_ndcg_at_3_max + value: -3.9931415753084396 + - type: nauc_ndcg_at_3_std + value: 45.28927180902579 + - type: nauc_ndcg_at_5_diff1 + value: -5.994930539767907 + - type: nauc_ndcg_at_5_max + value: 13.719936630127322 + - type: nauc_ndcg_at_5_std + value: 50.31945673174059 + - type: nauc_precision_at_1000_diff1 + value: -39.66997117398775 + - type: nauc_precision_at_1000_max + value: 28.410108388980838 + - type: nauc_precision_at_1000_std + value: 37.53443232999052 + - type: nauc_precision_at_100_diff1 + value: -44.593126088665095 + - type: nauc_precision_at_100_max + value: 26.307872607418446 + - type: nauc_precision_at_100_std + value: 71.89759263033184 + - type: nauc_precision_at_10_diff1 + value: -4.948473948610629 + - type: nauc_precision_at_10_max + value: 41.54942403132545 + - type: nauc_precision_at_10_std + value: 64.07442820501136 + - type: nauc_precision_at_1_diff1 + value: -1.1671335200746933 + - type: nauc_precision_at_1_max + value: -53.0462184873951 + - type: nauc_precision_at_1_std + value: 38.62044817927166 + - type: nauc_precision_at_20_diff1 + value: -29.92166012748137 + - type: nauc_precision_at_20_max + value: 33.98138613345162 + - type: nauc_precision_at_20_std + value: 70.61277161515709 + - type: nauc_precision_at_3_diff1 + value: -2.7688195473429884 + - type: nauc_precision_at_3_max + value: 1.9030186815422518 + - type: nauc_precision_at_3_std + value: 58.63621836776197 + - type: nauc_precision_at_5_diff1 + value: 8.445690682875119 + - type: nauc_precision_at_5_max + value: 36.27617240626628 + - type: nauc_precision_at_5_std + value: 60.80393572543947 + - type: nauc_recall_at_1000_diff1 + value: -4.21772906001991 + - type: nauc_recall_at_1000_max + value: 26.891105142539974 + - type: nauc_recall_at_1000_std + value: 59.66813401085308 + - type: nauc_recall_at_100_diff1 + value: 11.709253688521393 + - type: nauc_recall_at_100_max + value: 4.267312216386427 + - type: nauc_recall_at_100_std + value: 38.9820261668782 + - type: nauc_recall_at_10_diff1 + value: 43.63597197451749 + - type: nauc_recall_at_10_max + value: -1.7519454756697899 + - type: nauc_recall_at_10_std + value: 7.612891498872076 + - type: nauc_recall_at_1_diff1 + value: 40.71477618834696 + - type: nauc_recall_at_1_max + value: -26.6188251260968 + - type: nauc_recall_at_1_std + value: -6.003848303860384 + - type: nauc_recall_at_20_diff1 + value: 30.545045845878498 + - type: nauc_recall_at_20_max + value: -0.5426561731846394 + - type: nauc_recall_at_20_std + value: 18.101611001943233 + - type: nauc_recall_at_3_diff1 + value: 42.249663037638754 + - type: nauc_recall_at_3_max + value: -19.91735278310182 + - type: nauc_recall_at_3_std + value: -0.10112748906611038 + - type: nauc_recall_at_5_diff1 + value: 46.70043551551114 + - type: nauc_recall_at_5_max + value: -9.115316595368457 + - type: nauc_recall_at_5_std + value: 1.9890725209896063 + - type: ndcg_at_1 + value: 87.0 + - type: ndcg_at_10 + value: 80.573 + - type: ndcg_at_100 + value: 63.56 + - type: ndcg_at_1000 + value: 56.882999999999996 + - type: ndcg_at_20 + value: 76.984 + - type: ndcg_at_3 + value: 83.939 + - type: ndcg_at_5 + value: 83.38600000000001 + - type: precision_at_1 + value: 92.0 + - type: precision_at_10 + value: 85.0 + - type: precision_at_100 + value: 65.84 + - type: precision_at_1000 + value: 25.282 + - type: precision_at_20 + value: 80.80000000000001 + - type: precision_at_3 + value: 88.667 + - type: precision_at_5 + value: 87.6 + - type: recall_at_1 + value: 0.248 + - type: recall_at_10 + value: 2.215 + - type: recall_at_100 + value: 15.933 + - type: recall_at_1000 + value: 53.859 + - type: recall_at_20 + value: 4.101 + - type: recall_at_3 + value: 0.711 + - type: recall_at_5 + value: 1.171 + task: + type: Retrieval + - dataset: + config: default + name: MTEB Touche2020 (default) + revision: a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f + split: test + type: mteb/touche2020 + metrics: + - type: main_score + value: 24.566 + - type: map_at_1 + value: 1.5630000000000002 + - type: map_at_10 + value: 9.753 + - type: map_at_100 + value: 15.922 + - type: map_at_1000 + value: 17.557000000000002 + - type: map_at_20 + value: 12.374 + - type: map_at_3 + value: 4.364 + - type: map_at_5 + value: 6.673 + - type: mrr_at_1 + value: 18.367346938775512 + - type: mrr_at_10 + value: 38.28150307742145 + - type: mrr_at_100 + value: 38.508861925469944 + - type: mrr_at_1000 + value: 38.53048249583922 + - type: mrr_at_20 + value: 38.43848894869303 + - type: mrr_at_3 + value: 30.95238095238096 + - type: mrr_at_5 + value: 33.911564625850346 + - type: nauc_map_at_1000_diff1 + value: 8.379088363612524 + - type: nauc_map_at_1000_max + value: 23.967307666416605 + - type: nauc_map_at_1000_std + value: 14.10536002826737 + - type: nauc_map_at_100_diff1 + value: 9.589942293293635 + - type: nauc_map_at_100_max + value: 21.73401718667119 + - type: nauc_map_at_100_std + value: 11.160396359086741 + - type: nauc_map_at_10_diff1 + value: 24.41178106915404 + - type: nauc_map_at_10_max + value: 19.35717490875409 + - type: nauc_map_at_10_std + value: -4.593906970395038 + - type: nauc_map_at_1_diff1 + value: 7.288351744555407 + - type: nauc_map_at_1_max + value: 15.475070675287576 + - type: nauc_map_at_1_std + value: 2.9602156673990057 + - type: nauc_map_at_20_diff1 + value: 17.270568644701402 + - type: nauc_map_at_20_max + value: 20.152393718131737 + - type: nauc_map_at_20_std + value: -2.5418243043714877 + - type: nauc_map_at_3_diff1 + value: 22.370701469608058 + - type: nauc_map_at_3_max + value: 30.484243753975377 + - type: nauc_map_at_3_std + value: -4.875955625626936 + - type: nauc_map_at_5_diff1 + value: 27.95765241966459 + - type: nauc_map_at_5_max + value: 21.47291742393793 + - type: nauc_map_at_5_std + value: -4.537266472919711 + - type: nauc_mrr_at_1000_diff1 + value: 23.453289947277312 + - type: nauc_mrr_at_1000_max + value: 4.282804898246936 + - type: nauc_mrr_at_1000_std + value: 8.078502129225559 + - type: nauc_mrr_at_100_diff1 + value: 23.458906272174882 + - type: nauc_mrr_at_100_max + value: 4.298306981713938 + - type: nauc_mrr_at_100_std + value: 8.152386487267894 + - type: nauc_mrr_at_10_diff1 + value: 23.698246972641098 + - type: nauc_mrr_at_10_max + value: 4.438754391100767 + - type: nauc_mrr_at_10_std + value: 7.4636801655429545 + - type: nauc_mrr_at_1_diff1 + value: 22.60058803478894 + - type: nauc_mrr_at_1_max + value: 1.0891874176490723 + - type: nauc_mrr_at_1_std + value: 5.9563087699088975 + - type: nauc_mrr_at_20_diff1 + value: 23.254098040390122 + - type: nauc_mrr_at_20_max + value: 4.143496279890667 + - type: nauc_mrr_at_20_std + value: 8.163013435501183 + - type: nauc_mrr_at_3_diff1 + value: 23.937401710193377 + - type: nauc_mrr_at_3_max + value: 3.20166483126013 + - type: nauc_mrr_at_3_std + value: 5.963929075019466 + - type: nauc_mrr_at_5_diff1 + value: 22.4803458212637 + - type: nauc_mrr_at_5_max + value: 3.856908742997917 + - type: nauc_mrr_at_5_std + value: 7.326616106166392 + - type: nauc_ndcg_at_1000_diff1 + value: 3.0239562561442224 + - type: nauc_ndcg_at_1000_max + value: 26.39026108631468 + - type: nauc_ndcg_at_1000_std + value: 38.64302894200628 + - type: nauc_ndcg_at_100_diff1 + value: 3.837868499368667 + - type: nauc_ndcg_at_100_max + value: 14.63612288189351 + - type: nauc_ndcg_at_100_std + value: 32.81619611109916 + - type: nauc_ndcg_at_10_diff1 + value: 21.414708148504168 + - type: nauc_ndcg_at_10_max + value: 12.311329261132933 + - type: nauc_ndcg_at_10_std + value: 6.266776292728888 + - type: nauc_ndcg_at_1_diff1 + value: 26.51646134257643 + - type: nauc_ndcg_at_1_max + value: -5.696453847280905 + - type: nauc_ndcg_at_1_std + value: 6.716200696424981 + - type: nauc_ndcg_at_20_diff1 + value: 15.588853076805243 + - type: nauc_ndcg_at_20_max + value: 11.711244748670161 + - type: nauc_ndcg_at_20_std + value: 6.53026883006202 + - type: nauc_ndcg_at_3_diff1 + value: 31.5453546627245 + - type: nauc_ndcg_at_3_max + value: 16.234521793047943 + - type: nauc_ndcg_at_3_std + value: 2.4015349385874534 + - type: nauc_ndcg_at_5_diff1 + value: 28.507408209745023 + - type: nauc_ndcg_at_5_max + value: 12.041126602710166 + - type: nauc_ndcg_at_5_std + value: 7.673586244565096 + - type: nauc_precision_at_1000_diff1 + value: -33.824323968656586 + - type: nauc_precision_at_1000_max + value: 38.694251245156664 + - type: nauc_precision_at_1000_std + value: 32.4513959531412 + - type: nauc_precision_at_100_diff1 + value: -32.29835929431547 + - type: nauc_precision_at_100_max + value: 21.962799652743108 + - type: nauc_precision_at_100_std + value: 71.13245115182096 + - type: nauc_precision_at_10_diff1 + value: 15.452594287336346 + - type: nauc_precision_at_10_max + value: 14.196190204253972 + - type: nauc_precision_at_10_std + value: 9.557142403906905 + - type: nauc_precision_at_1_diff1 + value: 22.60058803478894 + - type: nauc_precision_at_1_max + value: 1.0891874176490723 + - type: nauc_precision_at_1_std + value: 5.9563087699088975 + - type: nauc_precision_at_20_diff1 + value: 0.07120273343348586 + - type: nauc_precision_at_20_max + value: 18.435738515855423 + - type: nauc_precision_at_20_std + value: 22.024683052458528 + - type: nauc_precision_at_3_diff1 + value: 32.10278266411663 + - type: nauc_precision_at_3_max + value: 22.45358810490925 + - type: nauc_precision_at_3_std + value: -0.7601999094355234 + - type: nauc_precision_at_5_diff1 + value: 29.706505849509423 + - type: nauc_precision_at_5_max + value: 12.191511604928227 + - type: nauc_precision_at_5_std + value: 6.405562930479255 + - type: nauc_recall_at_1000_diff1 + value: -16.921121573612396 + - type: nauc_recall_at_1000_max + value: 33.94176680152185 + - type: nauc_recall_at_1000_std + value: 76.10929434508863 + - type: nauc_recall_at_100_diff1 + value: -11.003726211544379 + - type: nauc_recall_at_100_max + value: 9.533297653391818 + - type: nauc_recall_at_100_std + value: 45.39107593995866 + - type: nauc_recall_at_10_diff1 + value: 20.13480707903671 + - type: nauc_recall_at_10_max + value: 8.903392756077443 + - type: nauc_recall_at_10_std + value: -3.999031939065758 + - type: nauc_recall_at_1_diff1 + value: 7.288351744555407 + - type: nauc_recall_at_1_max + value: 15.475070675287576 + - type: nauc_recall_at_1_std + value: 2.9602156673990057 + - type: nauc_recall_at_20_diff1 + value: 8.612488297042658 + - type: nauc_recall_at_20_max + value: 7.786511726520547 + - type: nauc_recall_at_20_std + value: 1.2269438454181592 + - type: nauc_recall_at_3_diff1 + value: 25.45609125295747 + - type: nauc_recall_at_3_max + value: 22.59372983192127 + - type: nauc_recall_at_3_std + value: -6.391547945047826 + - type: nauc_recall_at_5_diff1 + value: 27.009628162494355 + - type: nauc_recall_at_5_max + value: 12.058022467629506 + - type: nauc_recall_at_5_std + value: -3.645620452231331 + - type: ndcg_at_1 + value: 17.347 + - type: ndcg_at_10 + value: 24.566 + - type: ndcg_at_100 + value: 36.268 + - type: ndcg_at_1000 + value: 48.146 + - type: ndcg_at_20 + value: 25.757 + - type: ndcg_at_3 + value: 21.73 + - type: ndcg_at_5 + value: 22.875999999999998 + - type: precision_at_1 + value: 18.367 + - type: precision_at_10 + value: 24.285999999999998 + - type: precision_at_100 + value: 7.857 + - type: precision_at_1000 + value: 1.58 + - type: precision_at_20 + value: 18.163 + - type: precision_at_3 + value: 24.490000000000002 + - type: precision_at_5 + value: 25.306 + - type: recall_at_1 + value: 1.5630000000000002 + - type: recall_at_10 + value: 17.328 + - type: recall_at_100 + value: 47.887 + - type: recall_at_1000 + value: 85.2 + - type: recall_at_20 + value: 25.567 + - type: recall_at_3 + value: 5.691 + - type: recall_at_5 + value: 9.457 + task: + type: Retrieval + - dataset: + config: default + name: MTEB ToxicConversationsClassification (default) + revision: edfaf9da55d3dd50d43143d90c1ac476895ae6de + split: test + type: mteb/toxic_conversations_50k + metrics: + - type: accuracy + value: 63.27636718750001 + - type: ap + value: 11.080705574160344 + - type: ap_weighted + value: 11.080705574160344 + - type: f1 + value: 48.55796674291463 + - type: f1_weighted + value: 71.34065889818143 + - type: main_score + value: 63.27636718750001 + task: + type: Classification + - dataset: + config: default + name: MTEB TweetSentimentExtractionClassification (default) + revision: d604517c81ca91fe16a244d1248fc021f9ecee7a + split: test + type: mteb/tweet_sentiment_extraction + metrics: + - type: accuracy + value: 63.00792303338992 + - type: f1 + value: 63.0511013735515 + - type: f1_weighted + value: 61.88669960760824 + - type: main_score + value: 63.00792303338992 + task: + type: Classification + - dataset: + config: default + name: MTEB TwentyNewsgroupsClustering (default) + revision: 6125ec4e24fa026cec8a478383ee943acfbd5449 + split: test + type: mteb/twentynewsgroups-clustering + metrics: + - type: main_score + value: 50.97015518322318 + - type: v_measure + value: 50.97015518322318 + - type: v_measure_std + value: 1.6097566268467516 + task: + type: Clustering + - dataset: + config: default + name: MTEB TwitterSemEval2015 (default) + revision: 70970daeab8776df92f5ea462b6173c0b46fd2d1 + split: test + type: mteb/twittersemeval2015-pairclassification + metrics: + - type: cosine_accuracy + value: 87.09542826488645 + - type: cosine_accuracy_threshold + value: 74.52629804611206 + - type: cosine_ap + value: 77.1339941017208 + - type: cosine_f1 + value: 69.99210318504869 + - type: cosine_f1_threshold + value: 71.03879451751709 + - type: cosine_precision + value: 69.8266806722689 + - type: cosine_recall + value: 70.15831134564644 + - type: dot_accuracy + value: 87.09542826488645 + - type: dot_accuracy_threshold + value: 74.52629804611206 + - type: dot_ap + value: 77.1339891310131 + - type: dot_f1 + value: 69.99210318504869 + - type: dot_f1_threshold + value: 71.03880643844604 + - type: dot_precision + value: 69.8266806722689 + - type: dot_recall + value: 70.15831134564644 + - type: euclidean_accuracy + value: 87.09542826488645 + - type: euclidean_accuracy_threshold + value: 71.3774561882019 + - type: euclidean_ap + value: 77.13398642478515 + - type: euclidean_f1 + value: 69.99210318504869 + - type: euclidean_f1_threshold + value: 76.10677480697632 + - type: euclidean_precision + value: 69.8266806722689 + - type: euclidean_recall + value: 70.15831134564644 + - type: main_score + value: 77.13402223085791 + - type: manhattan_accuracy + value: 87.09542826488645 + - type: manhattan_accuracy_threshold + value: 2514.6324157714844 + - type: manhattan_ap + value: 77.12113484524178 + - type: manhattan_f1 + value: 70.03146633102581 + - type: manhattan_f1_threshold + value: 2777.375030517578 + - type: manhattan_precision + value: 66.95547533092659 + - type: manhattan_recall + value: 73.40369393139842 + - type: max_accuracy + value: 87.09542826488645 + - type: max_ap + value: 77.13402223085791 + - type: max_f1 + value: 70.03146633102581 + - type: max_precision + value: 69.8266806722689 + - type: max_recall + value: 73.40369393139842 + - type: similarity_accuracy + value: 87.09542826488645 + - type: similarity_accuracy_threshold + value: 1898.9376068115234 + - type: similarity_ap + value: 77.13402223085791 + - type: similarity_f1 + value: 69.99210318504869 + - type: similarity_f1_threshold + value: 1810.0759506225586 + - type: similarity_precision + value: 69.8266806722689 + - type: similarity_recall + value: 70.15831134564644 + task: + type: PairClassification + - dataset: + config: default + name: MTEB TwitterURLCorpus (default) + revision: 8b6510b0b1fa4e4c4f879467980e9be563ec1cdf + split: test + type: mteb/twitterurlcorpus-pairclassification + metrics: + - type: cosine_accuracy + value: 89.42445763961656 + - type: cosine_accuracy_threshold + value: 71.79734706878662 + - type: cosine_ap + value: 86.71072588382813 + - type: cosine_f1 + value: 79.19670831603186 + - type: cosine_f1_threshold + value: 70.40935158729553 + - type: cosine_precision + value: 77.68644005035918 + - type: cosine_recall + value: 80.7668617185094 + - type: dot_accuracy + value: 89.42445763961656 + - type: dot_accuracy_threshold + value: 71.79734706878662 + - type: dot_ap + value: 86.71072523792228 + - type: dot_f1 + value: 79.19670831603186 + - type: dot_f1_threshold + value: 70.40935158729553 + - type: dot_precision + value: 77.68644005035918 + - type: dot_recall + value: 80.7668617185094 + - type: euclidean_accuracy + value: 89.42445763961656 + - type: euclidean_accuracy_threshold + value: 75.10347366333008 + - type: euclidean_ap + value: 86.71072372123429 + - type: euclidean_f1 + value: 79.19670831603186 + - type: euclidean_f1_threshold + value: 76.92937850952148 + - type: euclidean_precision + value: 77.68644005035918 + - type: euclidean_recall + value: 80.7668617185094 + - type: main_score + value: 86.71233198788845 + - type: manhattan_accuracy + value: 89.42057670664028 + - type: manhattan_accuracy_threshold + value: 2687.8875732421875 + - type: manhattan_ap + value: 86.71233198788845 + - type: manhattan_f1 + value: 79.15346310547758 + - type: manhattan_f1_threshold + value: 2746.4208602905273 + - type: manhattan_precision + value: 77.59615384615385 + - type: manhattan_recall + value: 80.77456113335386 + - type: max_accuracy + value: 89.42445763961656 + - type: max_ap + value: 86.71233198788845 + - type: max_f1 + value: 79.19670831603186 + - type: max_precision + value: 77.68644005035918 + - type: max_recall + value: 80.77456113335386 + - type: similarity_accuracy + value: 89.42445763961656 + - type: similarity_accuracy_threshold + value: 1829.4036865234375 + - type: similarity_ap + value: 86.7107185363961 + - type: similarity_f1 + value: 79.19670831603186 + - type: similarity_f1_threshold + value: 1794.0374374389648 + - type: similarity_precision + value: 77.68644005035918 + - type: similarity_recall + value: 80.7668617185094 + task: + type: PairClassification --- -# Model Card for Model ID - +# BinGE: TODO +TODO: 2 line summary and link to paper +## Usage +```python +import torch +from transformers import AutoTokenizer, AutoModel, AutoConfig +from peft.peft_model import PeftModel -## Model Details +if __name__ == "__main__": -### Model Description + # Loading base Sheared-Llama model, along with custom code that enables bidirectional connections in decoder-only LLMs. + tokenizer = AutoTokenizer.from_pretrained( + "McGill-NLP/LLM2Vec-Sheared-LLaMA-mntp" + ) + config = AutoConfig.from_pretrained( + "McGill-NLP/LLM2Vec-Sheared-LLaMA-mntp", trust_remote_code=True + ) + model = AutoModel.from_pretrained( + "McGill-NLP/LLM2Vec-Sheared-LLaMA-mntp", + trust_remote_code=True, + config=config, + torch_dtype=torch.bfloat16, + device_map="cuda" if torch.cuda.is_available() else "cpu", + ) - + # Load the MNTP LoRA weights + model = PeftModel.from_pretrained( + model, + "McGill-NLP/LLM2Vec-Sheared-LLaMA-mntp", + ) + # Merge the LoRA weights with the base model + model = model.merge_and_unload() # This can take several minutes on cpu -This is the model card of a 🤗 transformers model that has been pushed on the Hub. This model card has been automatically generated. + # Loading BinGE model. This loads the trained LoRA weights on top of MNTP model. Hence the final weights are -- Base model + MNTP (LoRA) + BinGE (LoRA). + model = PeftModel.from_pretrained( + model, "tsirif/BinGE-Sheared-LLaMA" + ) -- **Developed by:** [More Information Needed] -- **Funded by [optional]:** [More Information Needed] -- **Shared by [optional]:** [More Information Needed] -- **Model type:** [More Information Needed] -- **Language(s) (NLP):** [More Information Needed] -- **License:** [More Information Needed] -- **Finetuned from model [optional]:** [More Information Needed] +``` -### Model Sources [optional] - - - -- **Repository:** [More Information Needed] -- **Paper [optional]:** [More Information Needed] -- **Demo [optional]:** [More Information Needed] - -## Uses - - - -### Direct Use - - - -[More Information Needed] - -### Downstream Use [optional] - - - -[More Information Needed] - -### Out-of-Scope Use - - - -[More Information Needed] - -## Bias, Risks, and Limitations - - - -[More Information Needed] - -### Recommendations - - - -Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. - -## How to Get Started with the Model - -Use the code below to get started with the model. - -[More Information Needed] - -## Training Details - -### Training Data - - - -[More Information Needed] - -### Training Procedure - - - -#### Preprocessing [optional] - -[More Information Needed] - - -#### Training Hyperparameters - -- **Training regime:** [More Information Needed] - -#### Speeds, Sizes, Times [optional] - - - -[More Information Needed] - -## Evaluation - - - -### Testing Data, Factors & Metrics - -#### Testing Data - - - -[More Information Needed] - -#### Factors - - - -[More Information Needed] - -#### Metrics - - - -[More Information Needed] - -### Results - -[More Information Needed] - -#### Summary - - - -## Model Examination [optional] - - - -[More Information Needed] - -## Environmental Impact - - - -Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). - -- **Hardware Type:** [More Information Needed] -- **Hours used:** [More Information Needed] -- **Cloud Provider:** [More Information Needed] -- **Compute Region:** [More Information Needed] -- **Carbon Emitted:** [More Information Needed] - -## Technical Specifications [optional] - -### Model Architecture and Objective - -[More Information Needed] - -### Compute Infrastructure - -[More Information Needed] - -#### Hardware - -[More Information Needed] - -#### Software - -[More Information Needed] - -## Citation [optional] - - - -**BibTeX:** - -[More Information Needed] - -**APA:** - -[More Information Needed] - -## Glossary [optional] - - - -[More Information Needed] - -## More Information [optional] - -[More Information Needed] - -## Model Card Authors [optional] - -[More Information Needed] - -## Model Card Contact - -[More Information Needed] \ No newline at end of file +TODO: initialize wrapper, provide example to check loading happened properly - see https://huggingface.co/McGill-NLP/LLM2Vec-Sheared-LLaMA-mntp-unsup-simcse \ No newline at end of file