diff --git "a/README.md" "b/README.md" --- "a/README.md" +++ "b/README.md" @@ -6,2396 +6,4494 @@ tags: - sentence-similarity - mteb model-index: -- name: SGPT-1.3B-weightedmean-msmarco-specb-bitfit +- name: SGPT-125M-weightedmean-msmarco-specb-bitfit results: - task: type: Classification dataset: type: mteb/amazon_counterfactual name: MTEB AmazonCounterfactualClassification (en) + config: en + split: test metrics: - type: accuracy - value: 65.20895522388061 + value: 61.23880597014926 - type: ap - value: 29.59212705444778 + value: 25.854431650388644 - type: f1 - value: 59.97099864321921 + value: 55.751862762818604 + - task: + type: Classification + dataset: + type: mteb/amazon_counterfactual + name: MTEB AmazonCounterfactualClassification (de) + config: de + split: test + metrics: + - type: accuracy + value: 56.88436830835117 + - type: ap + value: 72.67279104379772 + - type: f1 + value: 54.449840243786404 + - task: + type: Classification + dataset: + type: mteb/amazon_counterfactual + name: MTEB AmazonCounterfactualClassification (en-ext) + config: en-ext + split: test + metrics: + - type: accuracy + value: 58.27586206896551 + - type: ap + value: 14.067357642500387 + - type: f1 + value: 48.172318518691334 + - task: + type: Classification + dataset: + type: mteb/amazon_counterfactual + name: MTEB AmazonCounterfactualClassification (ja) + config: ja + split: test + metrics: + - type: accuracy + value: 54.64668094218415 + - type: ap + value: 11.776694555054965 + - type: f1 + value: 44.526622834078765 - task: type: Classification dataset: type: mteb/amazon_polarity name: MTEB AmazonPolarityClassification + config: default + split: test metrics: - type: accuracy - value: 73.20565 + value: 65.401225 - type: ap - value: 67.36680643550963 + value: 60.22809958678552 - type: f1 - value: 72.90420520325125 + value: 65.0251824898292 - task: type: Classification dataset: type: mteb/amazon_reviews_multi name: MTEB AmazonReviewsClassification (en) + config: en + split: test + metrics: + - type: accuracy + value: 31.165999999999993 + - type: f1 + value: 30.908870050167437 + - task: + type: Classification + dataset: + type: mteb/amazon_reviews_multi + name: MTEB AmazonReviewsClassification (de) + config: de + split: test + metrics: + - type: accuracy + value: 24.79 + - type: f1 + value: 24.5833598854121 + - task: + type: Classification + dataset: + type: mteb/amazon_reviews_multi + name: MTEB AmazonReviewsClassification (es) + config: es + split: test + metrics: + - type: accuracy + value: 26.643999999999995 + - type: f1 + value: 26.39012792213563 + - task: + type: Classification + dataset: + type: mteb/amazon_reviews_multi + name: MTEB AmazonReviewsClassification (fr) + config: fr + split: test + metrics: + - type: accuracy + value: 26.386000000000003 + - type: f1 + value: 26.276867791454873 + - task: + type: Classification + dataset: + type: mteb/amazon_reviews_multi + name: MTEB AmazonReviewsClassification (ja) + config: ja + split: test + metrics: + - type: accuracy + value: 22.078000000000003 + - type: f1 + value: 21.797960290226843 + - task: + type: Classification + dataset: + type: mteb/amazon_reviews_multi + name: MTEB AmazonReviewsClassification (zh) + config: zh + split: test metrics: - type: accuracy - value: 34.955999999999996 + value: 24.274 - type: f1 - value: 34.719324437696955 + value: 23.887054434822627 - task: type: Retrieval dataset: type: arguana name: MTEB ArguAna + config: default + split: test metrics: - type: map_at_1 - value: 26.101999999999997 + value: 22.404 - type: map_at_10 - value: 40.958 + value: 36.845 - type: map_at_100 - value: 42.033 + value: 37.945 - type: map_at_1000 - value: 42.042 + value: 37.966 - type: map_at_3 - value: 36.332 + value: 31.78 - type: map_at_5 - value: 38.608 + value: 34.608 - type: mrr_at_1 - value: 26.387 + value: 22.902 - type: mrr_at_10 - value: 41.051 + value: 37.034 - type: mrr_at_100 - value: 42.118 + value: 38.134 - type: mrr_at_1000 - value: 42.126999999999995 + value: 38.155 - type: mrr_at_3 - value: 36.415 + value: 31.935000000000002 - type: mrr_at_5 - value: 38.72 + value: 34.812 - type: ndcg_at_1 - value: 26.101999999999997 + value: 22.404 - type: ndcg_at_10 - value: 49.68 + value: 45.425 - type: ndcg_at_100 - value: 54.257999999999996 + value: 50.354 - type: ndcg_at_1000 - value: 54.486000000000004 + value: 50.873999999999995 - type: ndcg_at_3 - value: 39.864 + value: 34.97 - type: ndcg_at_5 - value: 43.980000000000004 + value: 40.081 - type: precision_at_1 - value: 26.101999999999997 + value: 22.404 - type: precision_at_10 - value: 7.781000000000001 + value: 7.303999999999999 - type: precision_at_100 - value: 0.979 + value: 0.951 - type: precision_at_1000 - value: 0.1 + value: 0.099 - type: precision_at_3 - value: 16.714000000000002 + value: 14.746 - type: precision_at_5 - value: 12.034 + value: 11.337 - type: recall_at_1 - value: 26.101999999999997 + value: 22.404 - type: recall_at_10 - value: 77.809 + value: 73.044 - type: recall_at_100 - value: 97.866 + value: 95.092 - type: recall_at_1000 - value: 99.644 + value: 99.075 - type: recall_at_3 - value: 50.141999999999996 + value: 44.239 - type: recall_at_5 - value: 60.171 + value: 56.686 - task: type: Clustering dataset: type: mteb/arxiv-clustering-p2p name: MTEB ArxivClusteringP2P + config: default + split: test metrics: - type: v_measure - value: 43.384194916953774 + value: 39.70858340673288 - task: type: Clustering dataset: type: mteb/arxiv-clustering-s2s name: MTEB ArxivClusteringS2S + config: default + split: test metrics: - type: v_measure - value: 33.70962633433912 + value: 28.242847713721048 - task: type: Reranking dataset: type: mteb/askubuntudupquestions-reranking name: MTEB AskUbuntuDupQuestions + config: default + split: test metrics: - type: map - value: 58.133058996870076 + value: 55.83700395192393 - type: mrr - value: 72.10922041946972 + value: 70.3891307215407 - task: type: STS dataset: type: mteb/biosses-sts name: MTEB BIOSSES + config: default + split: test metrics: - type: cos_sim_pearson - value: 86.62153841660047 + value: 79.25366801756223 - type: cos_sim_spearman - value: 83.01514456843276 + value: 75.20954502580506 - type: euclidean_pearson - value: 86.00431518427241 + value: 78.79900722991617 - type: euclidean_spearman - value: 83.85552516285783 + value: 77.79996549607588 - type: manhattan_pearson - value: 85.83025803351181 + value: 78.18408109480399 - type: manhattan_spearman - value: 83.86636878343106 + value: 76.85958262303106 - task: type: Classification dataset: type: mteb/banking77 name: MTEB Banking77Classification + config: default + split: test metrics: - type: accuracy - value: 82.05844155844156 + value: 77.70454545454545 - type: f1 - value: 82.0185837884764 + value: 77.6929000113803 - task: type: Clustering dataset: type: mteb/biorxiv-clustering-p2p name: MTEB BiorxivClusteringP2P + config: default + split: test metrics: - type: v_measure - value: 35.05918333141837 + value: 33.63260395543984 - task: type: Clustering dataset: type: mteb/biorxiv-clustering-s2s name: MTEB BiorxivClusteringS2S + config: default + split: test metrics: - type: v_measure - value: 30.71055028830579 + value: 27.038042665369925 - task: type: Retrieval dataset: type: BeIR/cqadupstack name: MTEB CQADupstackAndroidRetrieval + config: default + split: test metrics: - type: map_at_1 - value: 26.519 + value: 22.139 - type: map_at_10 - value: 35.634 + value: 28.839 - type: map_at_100 - value: 36.961 + value: 30.023 - type: map_at_1000 - value: 37.088 + value: 30.153000000000002 - type: map_at_3 - value: 32.254 + value: 26.521 - type: map_at_5 - value: 34.22 + value: 27.775 - type: mrr_at_1 - value: 32.332 + value: 26.466 - type: mrr_at_10 - value: 41.168 + value: 33.495000000000005 - type: mrr_at_100 - value: 41.977 + value: 34.416999999999994 - type: mrr_at_1000 - value: 42.028999999999996 + value: 34.485 - type: mrr_at_3 - value: 38.196999999999996 + value: 31.402 - type: mrr_at_5 - value: 40.036 + value: 32.496 - type: ndcg_at_1 - value: 32.332 + value: 26.466 - type: ndcg_at_10 - value: 41.471000000000004 + value: 33.372 - type: ndcg_at_100 - value: 46.955999999999996 + value: 38.7 - type: ndcg_at_1000 - value: 49.262 + value: 41.696 - type: ndcg_at_3 - value: 35.937999999999995 + value: 29.443 - type: ndcg_at_5 - value: 38.702999999999996 + value: 31.121 - type: precision_at_1 - value: 32.332 + value: 26.466 - type: precision_at_10 - value: 7.7829999999999995 + value: 6.037 - type: precision_at_100 - value: 1.29 + value: 1.0670000000000002 - type: precision_at_1000 - value: 0.178 + value: 0.16199999999999998 - type: precision_at_3 - value: 16.834 + value: 13.782 - type: precision_at_5 - value: 12.418 + value: 9.757 - type: recall_at_1 - value: 26.519 + value: 22.139 - type: recall_at_10 - value: 53.190000000000005 + value: 42.39 - type: recall_at_100 - value: 76.56500000000001 + value: 65.427 - type: recall_at_1000 - value: 91.47800000000001 + value: 86.04899999999999 - type: recall_at_3 - value: 38.034 + value: 31.127 - type: recall_at_5 - value: 45.245999999999995 + value: 35.717999999999996 - task: type: Retrieval dataset: type: BeIR/cqadupstack name: MTEB CQADupstackEnglishRetrieval + config: default + split: test metrics: - type: map_at_1 - value: 25.356 + value: 20.652 - type: map_at_10 - value: 34.596 + value: 27.558 - type: map_at_100 - value: 35.714 + value: 28.473 - type: map_at_1000 - value: 35.839999999999996 + value: 28.577 - type: map_at_3 - value: 32.073 + value: 25.402 - type: map_at_5 - value: 33.475 + value: 26.68 - type: mrr_at_1 - value: 31.274 + value: 25.223000000000003 - type: mrr_at_10 - value: 39.592 + value: 31.966 - type: mrr_at_100 - value: 40.284 + value: 32.664 - type: mrr_at_1000 - value: 40.339999999999996 + value: 32.724 - type: mrr_at_3 - value: 37.378 + value: 30.074 - type: mrr_at_5 - value: 38.658 + value: 31.249 - type: ndcg_at_1 - value: 31.274 + value: 25.223000000000003 - type: ndcg_at_10 - value: 39.766 + value: 31.694 - type: ndcg_at_100 - value: 44.028 + value: 35.662 - type: ndcg_at_1000 - value: 46.445 + value: 38.092 - type: ndcg_at_3 - value: 35.934 + value: 28.294000000000004 - type: ndcg_at_5 - value: 37.751000000000005 + value: 30.049 - type: precision_at_1 - value: 31.274 + value: 25.223000000000003 - type: precision_at_10 - value: 7.452 + value: 5.777 - type: precision_at_100 - value: 1.217 + value: 0.9730000000000001 - type: precision_at_1000 - value: 0.16999999999999998 + value: 0.13999999999999999 - type: precision_at_3 - value: 17.431 + value: 13.397 - type: precision_at_5 - value: 12.306000000000001 + value: 9.605 - type: recall_at_1 - value: 25.356 + value: 20.652 - type: recall_at_10 - value: 49.344 + value: 39.367999999999995 - type: recall_at_100 - value: 67.497 + value: 56.485 - type: recall_at_1000 - value: 83.372 + value: 73.292 - type: recall_at_3 - value: 38.227 + value: 29.830000000000002 - type: recall_at_5 - value: 43.187999999999995 + value: 34.43 - task: type: Retrieval dataset: type: BeIR/cqadupstack name: MTEB CQADupstackGamingRetrieval + config: default + split: test metrics: - type: map_at_1 - value: 32.759 + value: 25.180000000000003 - type: map_at_10 - value: 43.937 + value: 34.579 - type: map_at_100 - value: 45.004 + value: 35.589999999999996 - type: map_at_1000 - value: 45.07 + value: 35.68 - type: map_at_3 - value: 40.805 + value: 31.735999999999997 - type: map_at_5 - value: 42.497 + value: 33.479 - type: mrr_at_1 - value: 37.367 + value: 29.467 - type: mrr_at_10 - value: 47.237 + value: 37.967 - type: mrr_at_100 - value: 47.973 + value: 38.800000000000004 - type: mrr_at_1000 - value: 48.010999999999996 + value: 38.858 - type: mrr_at_3 - value: 44.65 + value: 35.465 - type: mrr_at_5 - value: 46.050999999999995 + value: 37.057 - type: ndcg_at_1 - value: 37.367 + value: 29.467 - type: ndcg_at_10 - value: 49.659 + value: 39.796 - type: ndcg_at_100 - value: 54.069 + value: 44.531 - type: ndcg_at_1000 - value: 55.552 + value: 46.666000000000004 - type: ndcg_at_3 - value: 44.169000000000004 + value: 34.676 - type: ndcg_at_5 - value: 46.726 + value: 37.468 - type: precision_at_1 - value: 37.367 + value: 29.467 - type: precision_at_10 - value: 8.163 + value: 6.601999999999999 - type: precision_at_100 - value: 1.133 + value: 0.9900000000000001 - type: precision_at_1000 - value: 0.131 + value: 0.124 - type: precision_at_3 - value: 19.707 + value: 15.568999999999999 - type: precision_at_5 - value: 13.718 + value: 11.172 - type: recall_at_1 - value: 32.759 + value: 25.180000000000003 - type: recall_at_10 - value: 63.341 + value: 52.269 - type: recall_at_100 - value: 82.502 + value: 73.574 - type: recall_at_1000 - value: 93.259 + value: 89.141 - type: recall_at_3 - value: 48.796 + value: 38.522 - type: recall_at_5 - value: 54.921 + value: 45.323 - task: type: Retrieval dataset: type: BeIR/cqadupstack name: MTEB CQADupstackGisRetrieval + config: default + split: test metrics: - type: map_at_1 - value: 18.962 + value: 16.303 - type: map_at_10 - value: 25.863000000000003 + value: 21.629 - type: map_at_100 - value: 26.817999999999998 + value: 22.387999999999998 - type: map_at_1000 - value: 26.918 + value: 22.489 - type: map_at_3 - value: 23.043 + value: 19.608 - type: map_at_5 - value: 24.599 + value: 20.774 - type: mrr_at_1 - value: 20.452 + value: 17.740000000000002 - type: mrr_at_10 - value: 27.301 + value: 23.214000000000002 - type: mrr_at_100 - value: 28.233000000000004 + value: 23.97 - type: mrr_at_1000 - value: 28.310000000000002 + value: 24.054000000000002 - type: mrr_at_3 - value: 24.539 + value: 21.243000000000002 - type: mrr_at_5 - value: 26.108999999999998 + value: 22.322 - type: ndcg_at_1 - value: 20.452 + value: 17.740000000000002 - type: ndcg_at_10 - value: 30.354999999999997 + value: 25.113000000000003 - type: ndcg_at_100 - value: 35.336 + value: 29.287999999999997 - type: ndcg_at_1000 - value: 37.927 + value: 32.204 - type: ndcg_at_3 - value: 24.705 + value: 21.111 - type: ndcg_at_5 - value: 27.42 + value: 23.061999999999998 - type: precision_at_1 - value: 20.452 + value: 17.740000000000002 - type: precision_at_10 - value: 4.949 + value: 3.955 - type: precision_at_100 - value: 0.7799999999999999 + value: 0.644 - type: precision_at_1000 - value: 0.104 + value: 0.093 - type: precision_at_3 - value: 10.358 + value: 8.851 - type: precision_at_5 - value: 7.774 + value: 6.418 - type: recall_at_1 - value: 18.962 + value: 16.303 - type: recall_at_10 - value: 43.056 + value: 34.487 - type: recall_at_100 - value: 66.27300000000001 + value: 54.413999999999994 - type: recall_at_1000 - value: 85.96000000000001 + value: 77.158 - type: recall_at_3 - value: 27.776 + value: 23.733 - type: recall_at_5 - value: 34.287 + value: 28.381 - task: type: Retrieval dataset: type: BeIR/cqadupstack name: MTEB CQADupstackMathematicaRetrieval + config: default + split: test metrics: - type: map_at_1 - value: 11.24 + value: 10.133000000000001 - type: map_at_10 - value: 18.503 + value: 15.665999999999999 - type: map_at_100 - value: 19.553 + value: 16.592000000000002 - type: map_at_1000 - value: 19.689999999999998 + value: 16.733999999999998 - type: map_at_3 - value: 16.150000000000002 + value: 13.625000000000002 - type: map_at_5 - value: 17.254 + value: 14.721 - type: mrr_at_1 - value: 13.806 + value: 12.562000000000001 - type: mrr_at_10 - value: 21.939 + value: 18.487000000000002 - type: mrr_at_100 - value: 22.827 + value: 19.391 - type: mrr_at_1000 - value: 22.911 + value: 19.487 - type: mrr_at_3 - value: 19.32 + value: 16.418 - type: mrr_at_5 - value: 20.558 + value: 17.599999999999998 - type: ndcg_at_1 - value: 13.806 + value: 12.562000000000001 - type: ndcg_at_10 - value: 23.383000000000003 + value: 19.43 - type: ndcg_at_100 - value: 28.834 + value: 24.546 - type: ndcg_at_1000 - value: 32.175 + value: 28.193 - type: ndcg_at_3 - value: 18.651999999999997 + value: 15.509999999999998 - type: ndcg_at_5 - value: 20.505000000000003 + value: 17.322000000000003 - type: precision_at_1 - value: 13.806 + value: 12.562000000000001 - type: precision_at_10 - value: 4.714 + value: 3.794 - type: precision_at_100 - value: 0.864 + value: 0.74 - type: precision_at_1000 - value: 0.13 + value: 0.122 - type: precision_at_3 - value: 9.328 + value: 7.546 - type: precision_at_5 - value: 6.841 + value: 5.721 - type: recall_at_1 - value: 11.24 + value: 10.133000000000001 - type: recall_at_10 - value: 34.854 + value: 28.261999999999997 - type: recall_at_100 - value: 59.50299999999999 + value: 51.742999999999995 - type: recall_at_1000 - value: 83.25 + value: 78.075 - type: recall_at_3 - value: 22.02 + value: 17.634 - type: recall_at_5 - value: 26.715 + value: 22.128999999999998 - task: type: Retrieval dataset: type: BeIR/cqadupstack name: MTEB CQADupstackPhysicsRetrieval + config: default + split: test metrics: - type: map_at_1 - value: 23.012 + value: 19.991999999999997 - type: map_at_10 - value: 33.048 + value: 27.346999999999998 - type: map_at_100 - value: 34.371 + value: 28.582 - type: map_at_1000 - value: 34.489 + value: 28.716 - type: map_at_3 - value: 29.942999999999998 + value: 24.907 - type: map_at_5 - value: 31.602000000000004 + value: 26.1 - type: mrr_at_1 - value: 28.104000000000003 + value: 23.773 - type: mrr_at_10 - value: 37.99 + value: 31.647 - type: mrr_at_100 - value: 38.836 + value: 32.639 - type: mrr_at_1000 - value: 38.891 + value: 32.706 - type: mrr_at_3 - value: 35.226 + value: 29.195 - type: mrr_at_5 - value: 36.693999999999996 + value: 30.484 - type: ndcg_at_1 - value: 28.104000000000003 + value: 23.773 - type: ndcg_at_10 - value: 39.037 + value: 32.322 - type: ndcg_at_100 - value: 44.643 + value: 37.996 - type: ndcg_at_1000 - value: 46.939 + value: 40.819 - type: ndcg_at_3 - value: 33.784 + value: 27.876 - type: ndcg_at_5 - value: 36.126000000000005 + value: 29.664 - type: precision_at_1 - value: 28.104000000000003 + value: 23.773 - type: precision_at_10 - value: 7.2669999999999995 + value: 5.976999999999999 - type: precision_at_100 - value: 1.193 + value: 1.055 - type: precision_at_1000 - value: 0.159 + value: 0.15 - type: precision_at_3 - value: 16.298000000000002 + value: 13.122 - type: precision_at_5 - value: 11.684 + value: 9.451 - type: recall_at_1 - value: 23.012 + value: 19.991999999999997 - type: recall_at_10 - value: 52.054 + value: 43.106 - type: recall_at_100 - value: 75.622 + value: 67.264 - type: recall_at_1000 - value: 90.675 + value: 86.386 - type: recall_at_3 - value: 37.282 + value: 30.392000000000003 - type: recall_at_5 - value: 43.307 + value: 34.910999999999994 - task: type: Retrieval dataset: type: BeIR/cqadupstack name: MTEB CQADupstackProgrammersRetrieval + config: default + split: test metrics: - type: map_at_1 - value: 21.624 + value: 17.896 - type: map_at_10 - value: 30.209999999999997 + value: 24.644 - type: map_at_100 - value: 31.52 + value: 25.790000000000003 - type: map_at_1000 - value: 31.625999999999998 + value: 25.913999999999998 - type: map_at_3 - value: 26.951000000000004 + value: 22.694 - type: map_at_5 - value: 28.938999999999997 + value: 23.69 - type: mrr_at_1 - value: 26.941 + value: 21.346999999999998 - type: mrr_at_10 - value: 35.13 + value: 28.594 - type: mrr_at_100 - value: 36.15 + value: 29.543999999999997 - type: mrr_at_1000 - value: 36.204 + value: 29.621 - type: mrr_at_3 - value: 32.42 + value: 26.807 - type: mrr_at_5 - value: 34.155 + value: 27.669 - type: ndcg_at_1 - value: 26.941 + value: 21.346999999999998 - type: ndcg_at_10 - value: 35.726 + value: 28.833 - type: ndcg_at_100 - value: 41.725 + value: 34.272000000000006 - type: ndcg_at_1000 - value: 44.105 + value: 37.355 - type: ndcg_at_3 - value: 30.184 + value: 25.373 - type: ndcg_at_5 - value: 33.176 + value: 26.756 - type: precision_at_1 - value: 26.941 + value: 21.346999999999998 - type: precision_at_10 - value: 6.654999999999999 + value: 5.2170000000000005 - type: precision_at_100 - value: 1.1520000000000001 + value: 0.954 - type: precision_at_1000 - value: 0.152 + value: 0.13899999999999998 - type: precision_at_3 - value: 14.346 + value: 11.948 - type: precision_at_5 - value: 10.868 + value: 8.425 - type: recall_at_1 - value: 21.624 + value: 17.896 - type: recall_at_10 - value: 47.359 + value: 37.291000000000004 - type: recall_at_100 - value: 73.436 + value: 61.138000000000005 - type: recall_at_1000 - value: 89.988 + value: 83.212 - type: recall_at_3 - value: 32.34 + value: 27.705999999999996 - type: recall_at_5 - value: 39.856 + value: 31.234 - task: type: Retrieval dataset: type: BeIR/cqadupstack name: MTEB CQADupstackRetrieval + config: default + split: test metrics: - type: map_at_1 - value: 20.67566666666667 + value: 17.195166666666665 - type: map_at_10 - value: 28.479333333333333 + value: 23.329083333333333 - type: map_at_100 - value: 29.612249999999996 + value: 24.30308333333333 - type: map_at_1000 - value: 29.731166666666663 + value: 24.422416666666667 - type: map_at_3 - value: 25.884 + value: 21.327416666666664 - type: map_at_5 - value: 27.298916666666667 + value: 22.419999999999998 - type: mrr_at_1 - value: 24.402583333333332 + value: 19.999916666666667 - type: mrr_at_10 - value: 32.07041666666667 + value: 26.390166666666666 - type: mrr_at_100 - value: 32.95841666666667 + value: 27.230999999999998 - type: mrr_at_1000 - value: 33.025416666666665 + value: 27.308333333333334 - type: mrr_at_3 - value: 29.677749999999996 + value: 24.4675 - type: mrr_at_5 - value: 31.02391666666667 + value: 25.541083333333336 - type: ndcg_at_1 - value: 24.402583333333332 + value: 19.999916666666667 - type: ndcg_at_10 - value: 33.326166666666666 + value: 27.248666666666665 - type: ndcg_at_100 - value: 38.51566666666667 + value: 32.00258333333334 - type: ndcg_at_1000 - value: 41.13791666666667 + value: 34.9465 - type: ndcg_at_3 - value: 28.687749999999994 + value: 23.58566666666667 - type: ndcg_at_5 - value: 30.84766666666667 + value: 25.26341666666666 - type: precision_at_1 - value: 24.402583333333332 + value: 19.999916666666667 - type: precision_at_10 - value: 5.943749999999999 + value: 4.772166666666666 - type: precision_at_100 - value: 1.0098333333333334 + value: 0.847 - type: precision_at_1000 - value: 0.14183333333333334 + value: 0.12741666666666668 - type: precision_at_3 - value: 13.211500000000001 + value: 10.756166666666669 - type: precision_at_5 - value: 9.548416666666668 + value: 7.725416666666667 - type: recall_at_1 - value: 20.67566666666667 + value: 17.195166666666665 - type: recall_at_10 - value: 44.245583333333336 + value: 35.99083333333334 - type: recall_at_100 - value: 67.31116666666667 + value: 57.467999999999996 - type: recall_at_1000 - value: 85.87841666666665 + value: 78.82366666666667 - type: recall_at_3 - value: 31.49258333333333 + value: 25.898499999999995 - type: recall_at_5 - value: 36.93241666666667 + value: 30.084333333333333 - task: type: Retrieval dataset: type: BeIR/cqadupstack name: MTEB CQADupstackStatsRetrieval + config: default + split: test metrics: - type: map_at_1 - value: 18.34 + value: 16.779 - type: map_at_10 - value: 23.988 + value: 21.557000000000002 - type: map_at_100 - value: 24.895 + value: 22.338 - type: map_at_1000 - value: 24.992 + value: 22.421 - type: map_at_3 - value: 21.831 + value: 19.939 - type: map_at_5 - value: 23.0 + value: 20.903 - type: mrr_at_1 - value: 20.399 + value: 18.404999999999998 - type: mrr_at_10 - value: 26.186 + value: 23.435 - type: mrr_at_100 - value: 27.017999999999997 + value: 24.179000000000002 - type: mrr_at_1000 - value: 27.090999999999998 + value: 24.25 - type: mrr_at_3 - value: 24.08 + value: 21.907 - type: mrr_at_5 - value: 25.230000000000004 + value: 22.781000000000002 - type: ndcg_at_1 - value: 20.399 + value: 18.404999999999998 - type: ndcg_at_10 - value: 27.799000000000003 + value: 24.515 - type: ndcg_at_100 - value: 32.579 + value: 28.721000000000004 - type: ndcg_at_1000 - value: 35.209 + value: 31.259999999999998 - type: ndcg_at_3 - value: 23.684 + value: 21.508 - type: ndcg_at_5 - value: 25.521 + value: 23.01 - type: precision_at_1 - value: 20.399 + value: 18.404999999999998 - type: precision_at_10 - value: 4.585999999999999 + value: 3.834 - type: precision_at_100 - value: 0.755 + value: 0.641 - type: precision_at_1000 - value: 0.105 + value: 0.093 - type: precision_at_3 - value: 10.276 + value: 9.151 - type: precision_at_5 - value: 7.362 + value: 6.503 - type: recall_at_1 - value: 18.34 + value: 16.779 - type: recall_at_10 - value: 37.456 + value: 31.730000000000004 - type: recall_at_100 - value: 59.86 + value: 51.673 - type: recall_at_1000 - value: 79.703 + value: 71.17599999999999 - type: recall_at_3 - value: 26.163999999999998 + value: 23.518 - type: recall_at_5 - value: 30.652 + value: 27.230999999999998 - task: type: Retrieval dataset: type: BeIR/cqadupstack name: MTEB CQADupstackTexRetrieval + config: default + split: test metrics: - type: map_at_1 - value: 12.327 + value: 9.279 - type: map_at_10 - value: 17.572 + value: 13.822000000000001 - type: map_at_100 - value: 18.534 + value: 14.533 - type: map_at_1000 - value: 18.653 + value: 14.649999999999999 - type: map_at_3 - value: 15.703 + value: 12.396 - type: map_at_5 - value: 16.752 + value: 13.214 - type: mrr_at_1 - value: 15.038000000000002 + value: 11.149000000000001 - type: mrr_at_10 - value: 20.726 + value: 16.139 - type: mrr_at_100 - value: 21.61 + value: 16.872 - type: mrr_at_1000 - value: 21.695 + value: 16.964000000000002 - type: mrr_at_3 - value: 18.829 + value: 14.613000000000001 - type: mrr_at_5 - value: 19.885 + value: 15.486 - type: ndcg_at_1 - value: 15.038000000000002 + value: 11.149000000000001 - type: ndcg_at_10 - value: 21.241 + value: 16.82 - type: ndcg_at_100 - value: 26.179000000000002 + value: 20.73 - type: ndcg_at_1000 - value: 29.316 + value: 23.894000000000002 - type: ndcg_at_3 - value: 17.762 + value: 14.11 - type: ndcg_at_5 - value: 19.413 + value: 15.404000000000002 - type: precision_at_1 - value: 15.038000000000002 + value: 11.149000000000001 - type: precision_at_10 - value: 3.8920000000000003 + value: 3.063 - type: precision_at_100 - value: 0.75 + value: 0.587 - type: precision_at_1000 - value: 0.11800000000000001 + value: 0.1 - type: precision_at_3 - value: 8.351 + value: 6.699 - type: precision_at_5 - value: 6.187 + value: 4.928 - type: recall_at_1 - value: 12.327 + value: 9.279 - type: recall_at_10 - value: 29.342000000000002 + value: 23.745 - type: recall_at_100 - value: 51.854 + value: 41.873 - type: recall_at_1000 - value: 74.648 + value: 64.982 - type: recall_at_3 - value: 19.596 + value: 16.152 - type: recall_at_5 - value: 23.899 + value: 19.409000000000002 - task: type: Retrieval dataset: type: BeIR/cqadupstack name: MTEB CQADupstackUnixRetrieval + config: default + split: test metrics: - type: map_at_1 - value: 20.594 + value: 16.36 - type: map_at_10 - value: 27.878999999999998 + value: 21.927 - type: map_at_100 - value: 28.926000000000002 + value: 22.889 - type: map_at_1000 - value: 29.041 + value: 22.994 - type: map_at_3 - value: 25.668999999999997 + value: 20.433 - type: map_at_5 - value: 26.773999999999997 + value: 21.337 - type: mrr_at_1 - value: 23.694000000000003 + value: 18.75 - type: mrr_at_10 - value: 31.335 + value: 24.859 - type: mrr_at_100 - value: 32.218 + value: 25.746999999999996 - type: mrr_at_1000 - value: 32.298 + value: 25.829 - type: mrr_at_3 - value: 29.26 + value: 23.383000000000003 - type: mrr_at_5 - value: 30.328 + value: 24.297 - type: ndcg_at_1 - value: 23.694000000000003 + value: 18.75 - type: ndcg_at_10 - value: 32.456 + value: 25.372 - type: ndcg_at_100 - value: 37.667 + value: 30.342999999999996 - type: ndcg_at_1000 - value: 40.571 + value: 33.286 - type: ndcg_at_3 - value: 28.283 + value: 22.627 - type: ndcg_at_5 - value: 29.986 + value: 24.04 - type: precision_at_1 - value: 23.694000000000003 + value: 18.75 - type: precision_at_10 - value: 5.448 + value: 4.1419999999999995 - type: precision_at_100 - value: 0.9119999999999999 + value: 0.738 - type: precision_at_1000 - value: 0.127 + value: 0.11100000000000002 - type: precision_at_3 - value: 12.717999999999998 + value: 10.261000000000001 - type: precision_at_5 - value: 8.843 + value: 7.164 - type: recall_at_1 - value: 20.594 + value: 16.36 - type: recall_at_10 - value: 43.004999999999995 + value: 32.949 - type: recall_at_100 - value: 66.228 + value: 55.552 - type: recall_at_1000 - value: 87.17099999999999 + value: 77.09899999999999 - type: recall_at_3 - value: 31.554 + value: 25.538 - type: recall_at_5 - value: 35.838 + value: 29.008 - task: type: Retrieval dataset: type: BeIR/cqadupstack name: MTEB CQADupstackWebmastersRetrieval + config: default + split: test metrics: - type: map_at_1 - value: 20.855999999999998 + value: 17.39 - type: map_at_10 - value: 28.372000000000003 + value: 23.058 - type: map_at_100 - value: 29.87 + value: 24.445 - type: map_at_1000 - value: 30.075000000000003 + value: 24.637999999999998 - type: map_at_3 - value: 26.054 + value: 21.037 - type: map_at_5 - value: 27.128999999999998 + value: 21.966 - type: mrr_at_1 - value: 25.494 + value: 19.96 - type: mrr_at_10 - value: 32.735 + value: 26.301000000000002 - type: mrr_at_100 - value: 33.794000000000004 + value: 27.297 - type: mrr_at_1000 - value: 33.85 + value: 27.375 - type: mrr_at_3 - value: 30.731 + value: 24.340999999999998 - type: mrr_at_5 - value: 31.897 + value: 25.339 - type: ndcg_at_1 - value: 25.494 + value: 19.96 - type: ndcg_at_10 - value: 33.385 + value: 27.249000000000002 - type: ndcg_at_100 - value: 39.436 + value: 32.997 - type: ndcg_at_1000 - value: 42.313 + value: 36.359 - type: ndcg_at_3 - value: 29.612 + value: 23.519000000000002 - type: ndcg_at_5 - value: 31.186999999999998 + value: 24.915000000000003 - type: precision_at_1 - value: 25.494 + value: 19.96 - type: precision_at_10 - value: 6.422999999999999 + value: 5.356000000000001 - type: precision_at_100 - value: 1.383 + value: 1.198 - type: precision_at_1000 - value: 0.22399999999999998 + value: 0.20400000000000001 - type: precision_at_3 - value: 13.834 + value: 10.738 - type: precision_at_5 - value: 10.0 + value: 7.904999999999999 - type: recall_at_1 - value: 20.855999999999998 + value: 17.39 - type: recall_at_10 - value: 42.678 + value: 35.254999999999995 - type: recall_at_100 - value: 70.224 + value: 61.351 - type: recall_at_1000 - value: 89.369 + value: 84.395 - type: recall_at_3 - value: 31.957 + value: 25.194 - type: recall_at_5 - value: 36.026 + value: 28.546 - task: type: Retrieval dataset: type: BeIR/cqadupstack name: MTEB CQADupstackWordpressRetrieval + config: default + split: test metrics: - type: map_at_1 - value: 16.519000000000002 + value: 14.238999999999999 - type: map_at_10 - value: 22.15 + value: 19.323 - type: map_at_100 - value: 23.180999999999997 + value: 19.994 - type: map_at_1000 - value: 23.291999999999998 + value: 20.102999999999998 - type: map_at_3 - value: 20.132 + value: 17.631 - type: map_at_5 - value: 21.346 + value: 18.401 - type: mrr_at_1 - value: 17.93 + value: 15.157000000000002 - type: mrr_at_10 - value: 23.506 + value: 20.578 - type: mrr_at_100 - value: 24.581 + value: 21.252 - type: mrr_at_1000 - value: 24.675 + value: 21.346999999999998 - type: mrr_at_3 - value: 21.503 + value: 18.762 - type: mrr_at_5 - value: 22.686 + value: 19.713 - type: ndcg_at_1 - value: 17.93 + value: 15.157000000000002 - type: ndcg_at_10 - value: 25.636 + value: 22.468 - type: ndcg_at_100 - value: 30.736 + value: 26.245 - type: ndcg_at_1000 - value: 33.841 + value: 29.534 - type: ndcg_at_3 - value: 21.546000000000003 + value: 18.981 - type: ndcg_at_5 - value: 23.658 + value: 20.349999999999998 - type: precision_at_1 - value: 17.93 + value: 15.157000000000002 - type: precision_at_10 - value: 3.993 + value: 3.512 - type: precision_at_100 - value: 0.6890000000000001 + value: 0.577 - type: precision_at_1000 - value: 0.104 + value: 0.091 - type: precision_at_3 - value: 9.057 + value: 8.01 - type: precision_at_5 - value: 6.58 + value: 5.656 - type: recall_at_1 - value: 16.519000000000002 + value: 14.238999999999999 - type: recall_at_10 - value: 35.268 + value: 31.038 - type: recall_at_100 - value: 58.17 + value: 49.122 - type: recall_at_1000 - value: 81.66799999999999 + value: 74.919 - type: recall_at_3 - value: 24.165 + value: 21.436 - type: recall_at_5 - value: 29.254 + value: 24.692 - task: type: Retrieval dataset: type: climate-fever name: MTEB ClimateFEVER + config: default + split: test metrics: - type: map_at_1 - value: 10.363 + value: 8.828 - type: map_at_10 - value: 18.301000000000002 + value: 14.982000000000001 - type: map_at_100 - value: 20.019000000000002 + value: 16.495 - type: map_at_1000 - value: 20.207 + value: 16.658 - type: map_at_3 - value: 14.877 + value: 12.366000000000001 - type: map_at_5 - value: 16.544 + value: 13.655000000000001 - type: mrr_at_1 - value: 22.866 + value: 19.088 - type: mrr_at_10 - value: 34.935 + value: 29.29 - type: mrr_at_100 - value: 35.802 + value: 30.291 - type: mrr_at_1000 - value: 35.839999999999996 + value: 30.342000000000002 - type: mrr_at_3 - value: 30.965999999999998 + value: 25.907000000000004 - type: mrr_at_5 - value: 33.204 + value: 27.840999999999998 - type: ndcg_at_1 - value: 22.866 + value: 19.088 - type: ndcg_at_10 - value: 26.595000000000002 + value: 21.858 - type: ndcg_at_100 - value: 33.513999999999996 + value: 28.323999999999998 - type: ndcg_at_1000 - value: 36.872 + value: 31.561 - type: ndcg_at_3 - value: 20.666999999999998 + value: 17.175 - type: ndcg_at_5 - value: 22.728 + value: 18.869 - type: precision_at_1 - value: 22.866 + value: 19.088 - type: precision_at_10 - value: 8.632 + value: 6.9190000000000005 - type: precision_at_100 - value: 1.6119999999999999 + value: 1.376 - type: precision_at_1000 - value: 0.22399999999999998 + value: 0.197 - type: precision_at_3 - value: 15.504999999999999 + value: 12.703999999999999 - type: precision_at_5 - value: 12.404 + value: 9.993 - type: recall_at_1 - value: 10.363 + value: 8.828 - type: recall_at_10 - value: 33.494 + value: 27.381 - type: recall_at_100 - value: 57.593 + value: 50.0 - type: recall_at_1000 - value: 76.342 + value: 68.355 - type: recall_at_3 - value: 19.157 + value: 16.118 - type: recall_at_5 - value: 24.637999999999998 + value: 20.587 - task: type: Retrieval dataset: type: dbpedia-entity name: MTEB DBPedia + config: default + split: test metrics: - type: map_at_1 - value: 7.436 + value: 5.586 - type: map_at_10 - value: 14.760000000000002 + value: 10.040000000000001 - type: map_at_100 - value: 19.206 + value: 12.55 - type: map_at_1000 - value: 20.267 + value: 13.123999999999999 - type: map_at_3 - value: 10.894 + value: 7.75 - type: map_at_5 - value: 12.828999999999999 + value: 8.835999999999999 - type: mrr_at_1 - value: 54.25 + value: 42.25 - type: mrr_at_10 - value: 63.769 + value: 51.205999999999996 - type: mrr_at_100 - value: 64.193 + value: 51.818 - type: mrr_at_1000 - value: 64.211 + value: 51.855 - type: mrr_at_3 - value: 61.458 + value: 48.875 - type: mrr_at_5 - value: 63.096 + value: 50.488 - type: ndcg_at_1 - value: 42.875 + value: 32.25 - type: ndcg_at_10 - value: 31.507 + value: 22.718 - type: ndcg_at_100 - value: 34.559 + value: 24.359 - type: ndcg_at_1000 - value: 41.246 + value: 29.232000000000003 - type: ndcg_at_3 - value: 35.058 + value: 25.974000000000004 - type: ndcg_at_5 - value: 33.396 + value: 24.291999999999998 - type: precision_at_1 - value: 54.25 + value: 42.25 - type: precision_at_10 - value: 24.45 + value: 17.75 - type: precision_at_100 - value: 7.383000000000001 + value: 5.032 - type: precision_at_1000 - value: 1.582 + value: 1.117 - type: precision_at_3 - value: 38.083 + value: 28.833 - type: precision_at_5 - value: 32.6 + value: 24.25 - type: recall_at_1 - value: 7.436 + value: 5.586 - type: recall_at_10 - value: 19.862 + value: 14.16 - type: recall_at_100 - value: 38.981 + value: 28.051 - type: recall_at_1000 - value: 61.038000000000004 + value: 45.157000000000004 - type: recall_at_3 - value: 11.949 + value: 8.758000000000001 - type: recall_at_5 - value: 15.562000000000001 + value: 10.975999999999999 - task: type: Classification dataset: type: mteb/emotion name: MTEB EmotionClassification + config: default + split: test metrics: - type: accuracy - value: 46.39 + value: 39.075 - type: f1 - value: 42.26424885856703 + value: 35.01420354708222 - task: type: Retrieval dataset: type: fever name: MTEB FEVER + config: default + split: test metrics: - type: map_at_1 - value: 50.916 + value: 43.519999999999996 - type: map_at_10 - value: 62.258 + value: 54.368 - type: map_at_100 - value: 62.741 + value: 54.918 - type: map_at_1000 - value: 62.763000000000005 + value: 54.942 - type: map_at_3 - value: 60.01800000000001 + value: 51.712 - type: map_at_5 - value: 61.419999999999995 + value: 53.33599999999999 - type: mrr_at_1 - value: 54.964999999999996 + value: 46.955000000000005 - type: mrr_at_10 - value: 66.554 + value: 58.219 - type: mrr_at_100 - value: 66.96600000000001 + value: 58.73500000000001 - type: mrr_at_1000 - value: 66.97800000000001 + value: 58.753 - type: mrr_at_3 - value: 64.414 + value: 55.518 - type: mrr_at_5 - value: 65.77 + value: 57.191 - type: ndcg_at_1 - value: 54.964999999999996 + value: 46.955000000000005 - type: ndcg_at_10 - value: 68.12 + value: 60.45 - type: ndcg_at_100 - value: 70.282 + value: 63.047 - type: ndcg_at_1000 - value: 70.788 + value: 63.712999999999994 - type: ndcg_at_3 - value: 63.861999999999995 + value: 55.233 - type: ndcg_at_5 - value: 66.216 + value: 58.072 - type: precision_at_1 - value: 54.964999999999996 + value: 46.955000000000005 - type: precision_at_10 - value: 8.998000000000001 + value: 8.267 - type: precision_at_100 - value: 1.016 + value: 0.962 - type: precision_at_1000 - value: 0.107 + value: 0.10300000000000001 - type: precision_at_3 - value: 25.618000000000002 + value: 22.326999999999998 - type: precision_at_5 - value: 16.676 + value: 14.940999999999999 - type: recall_at_1 - value: 50.916 + value: 43.519999999999996 - type: recall_at_10 - value: 82.04 + value: 75.632 - type: recall_at_100 - value: 91.689 + value: 87.41600000000001 - type: recall_at_1000 - value: 95.34899999999999 + value: 92.557 - type: recall_at_3 - value: 70.512 + value: 61.597 - type: recall_at_5 - value: 76.29899999999999 + value: 68.518 - task: type: Retrieval dataset: type: fiqa name: MTEB FiQA2018 + config: default + split: test metrics: - type: map_at_1 - value: 13.568 + value: 9.549000000000001 - type: map_at_10 - value: 23.264000000000003 + value: 15.762 - type: map_at_100 - value: 24.823999999999998 + value: 17.142 - type: map_at_1000 - value: 25.013999999999996 + value: 17.329 - type: map_at_3 - value: 19.724 + value: 13.575000000000001 - type: map_at_5 - value: 21.772 + value: 14.754000000000001 - type: mrr_at_1 - value: 27.315 + value: 19.753 - type: mrr_at_10 - value: 35.935 + value: 26.568 - type: mrr_at_100 - value: 36.929 + value: 27.606 - type: mrr_at_1000 - value: 36.985 + value: 27.68 - type: mrr_at_3 - value: 33.591 + value: 24.203 - type: mrr_at_5 - value: 34.848 + value: 25.668999999999997 - type: ndcg_at_1 - value: 27.315 + value: 19.753 - type: ndcg_at_10 - value: 29.988 + value: 21.118000000000002 - type: ndcg_at_100 - value: 36.41 + value: 27.308 - type: ndcg_at_1000 - value: 40.184999999999995 + value: 31.304 - type: ndcg_at_3 - value: 26.342 + value: 18.319 - type: ndcg_at_5 - value: 27.68 + value: 19.414 - type: precision_at_1 - value: 27.315 + value: 19.753 - type: precision_at_10 - value: 8.565000000000001 + value: 6.08 - type: precision_at_100 - value: 1.508 + value: 1.204 - type: precision_at_1000 - value: 0.219 + value: 0.192 - type: precision_at_3 - value: 17.849999999999998 + value: 12.191 - type: precision_at_5 - value: 13.672999999999998 + value: 9.383 - type: recall_at_1 - value: 13.568 + value: 9.549000000000001 - type: recall_at_10 - value: 37.133 + value: 26.131 - type: recall_at_100 - value: 61.475 + value: 50.544999999999995 - type: recall_at_1000 - value: 84.372 + value: 74.968 - type: recall_at_3 - value: 24.112000000000002 + value: 16.951 - type: recall_at_5 - value: 29.507 + value: 20.95 - task: type: Retrieval dataset: type: hotpotqa name: MTEB HotpotQA + config: default + split: test metrics: - type: map_at_1 - value: 30.878 + value: 25.544 - type: map_at_10 - value: 40.868 + value: 32.62 - type: map_at_100 - value: 41.693999999999996 + value: 33.275 - type: map_at_1000 - value: 41.775 + value: 33.344 - type: map_at_3 - value: 38.56 + value: 30.851 - type: map_at_5 - value: 39.947 + value: 31.868999999999996 - type: mrr_at_1 - value: 61.756 + value: 51.087 - type: mrr_at_10 - value: 68.265 + value: 57.704 - type: mrr_at_100 - value: 68.671 + value: 58.175 - type: mrr_at_1000 - value: 68.694 + value: 58.207 - type: mrr_at_3 - value: 66.78399999999999 + value: 56.106 - type: mrr_at_5 - value: 67.704 + value: 57.074000000000005 - type: ndcg_at_1 - value: 61.756 + value: 51.087 - type: ndcg_at_10 - value: 49.931 + value: 40.876000000000005 - type: ndcg_at_100 - value: 53.179 + value: 43.762 - type: ndcg_at_1000 - value: 54.94799999999999 + value: 45.423 - type: ndcg_at_3 - value: 46.103 + value: 37.65 - type: ndcg_at_5 - value: 48.147 + value: 39.305 - type: precision_at_1 - value: 61.756 + value: 51.087 - type: precision_at_10 - value: 10.163 + value: 8.304 - type: precision_at_100 - value: 1.2710000000000001 + value: 1.059 - type: precision_at_1000 - value: 0.151 + value: 0.128 - type: precision_at_3 - value: 28.179 + value: 22.875999999999998 - type: precision_at_5 - value: 18.528 + value: 15.033 - type: recall_at_1 - value: 30.878 + value: 25.544 - type: recall_at_10 - value: 50.817 + value: 41.519 - type: recall_at_100 - value: 63.544999999999995 + value: 52.957 - type: recall_at_1000 - value: 75.361 + value: 64.132 - type: recall_at_3 - value: 42.269 + value: 34.315 - type: recall_at_5 - value: 46.32 + value: 37.583 - task: type: Classification dataset: type: mteb/imdb name: MTEB ImdbClassification + config: default + split: test metrics: - type: accuracy - value: 64.04799999999999 + value: 58.6696 - type: ap - value: 59.185251455339284 + value: 55.3644880984279 - type: f1 - value: 63.947123181349255 + value: 58.07942097405652 - task: type: Retrieval dataset: type: msmarco name: MTEB MSMARCO + config: default + split: validation metrics: - type: map_at_1 - value: 18.9 + value: 14.442 - type: map_at_10 - value: 29.748 + value: 22.932 - type: map_at_100 - value: 30.976 + value: 24.132 - type: map_at_1000 - value: 31.041 + value: 24.213 - type: map_at_3 - value: 26.112999999999996 + value: 20.002 - type: map_at_5 - value: 28.197 + value: 21.636 - type: mrr_at_1 - value: 19.413 + value: 14.841999999999999 - type: mrr_at_10 - value: 30.322 + value: 23.416 - type: mrr_at_100 - value: 31.497000000000003 + value: 24.593999999999998 - type: mrr_at_1000 - value: 31.555 + value: 24.669 - type: mrr_at_3 - value: 26.729000000000003 + value: 20.494 - type: mrr_at_5 - value: 28.788999999999998 + value: 22.14 - type: ndcg_at_1 - value: 19.413 + value: 14.841999999999999 - type: ndcg_at_10 - value: 36.048 + value: 27.975 - type: ndcg_at_100 - value: 42.152 + value: 34.143 - type: ndcg_at_1000 - value: 43.772 + value: 36.370000000000005 - type: ndcg_at_3 - value: 28.642 + value: 21.944 - type: ndcg_at_5 - value: 32.358 + value: 24.881 - type: precision_at_1 - value: 19.413 + value: 14.841999999999999 - type: precision_at_10 - value: 5.785 + value: 4.537 - type: precision_at_100 - value: 0.8869999999999999 + value: 0.767 - type: precision_at_1000 - value: 0.10300000000000001 + value: 0.096 - type: precision_at_3 - value: 12.192 + value: 9.322 - type: precision_at_5 - value: 9.189 + value: 7.074 - type: recall_at_1 - value: 18.9 + value: 14.442 - type: recall_at_10 - value: 55.457 + value: 43.557 - type: recall_at_100 - value: 84.09100000000001 + value: 72.904 - type: recall_at_1000 - value: 96.482 + value: 90.40700000000001 - type: recall_at_3 - value: 35.359 + value: 27.088 - type: recall_at_5 - value: 44.275 + value: 34.144000000000005 - task: type: Classification dataset: type: mteb/mtop_domain name: MTEB MTOPDomainClassification (en) + config: en + split: test metrics: - type: accuracy - value: 92.07706338349293 + value: 86.95622435020519 - type: f1 - value: 91.56680443236652 + value: 86.58363130708494 - task: type: Classification dataset: - type: mteb/mtop_intent - name: MTEB MTOPIntentClassification (en) + type: mteb/mtop_domain + name: MTEB MTOPDomainClassification (de) + config: de + split: test metrics: - type: accuracy - value: 71.18559051527589 + value: 62.73034657650043 - type: f1 - value: 52.42887061726789 + value: 60.78623915840713 - task: type: Classification dataset: - type: mteb/amazon_massive_intent - name: MTEB MassiveIntentClassification (en) + type: mteb/mtop_domain + name: MTEB MTOPDomainClassification (es) + config: es + split: test metrics: - type: accuracy - value: 68.64828513786148 + value: 67.54503002001334 - type: f1 - value: 66.54281381596097 + value: 65.34879794116112 - task: type: Classification dataset: - type: mteb/amazon_massive_scenario - name: MTEB MassiveScenarioClassification (en) + type: mteb/mtop_domain + name: MTEB MTOPDomainClassification (fr) + config: fr + split: test metrics: - type: accuracy - value: 76.04236718224612 + value: 65.35233322893829 - type: f1 - value: 75.89170458655639 + value: 62.994001882446646 - task: - type: Clustering + type: Classification dataset: - type: mteb/medrxiv-clustering-p2p - name: MTEB MedrxivClusteringP2P + type: mteb/mtop_domain + name: MTEB MTOPDomainClassification (hi) + config: hi + split: test metrics: - - type: v_measure - value: 32.0840369055247 + - type: accuracy + value: 45.37110075295806 + - type: f1 + value: 44.26285860740745 - task: - type: Clustering + type: Classification dataset: - type: mteb/medrxiv-clustering-s2s - name: MTEB MedrxivClusteringS2S + type: mteb/mtop_domain + name: MTEB MTOPDomainClassification (th) + config: th + split: test metrics: - - type: v_measure - value: 29.448729560244537 + - type: accuracy + value: 55.276672694394215 + - type: f1 + value: 53.28388179869587 - task: - type: Reranking + type: Classification dataset: - type: mteb/mind_small - name: MTEB MindSmallReranking + type: mteb/mtop_intent + name: MTEB MTOPIntentClassification (en) + config: en + split: test metrics: - - type: map - value: 31.340856463122375 - - type: mrr - value: 32.398547669840916 + - type: accuracy + value: 62.25262197902417 + - type: f1 + value: 43.44084037148853 - task: - type: Retrieval + type: Classification dataset: - type: nfcorpus - name: MTEB NFCorpus + type: mteb/mtop_intent + name: MTEB MTOPIntentClassification (de) + config: de + split: test metrics: - - type: map_at_1 - value: 5.526 - - type: map_at_10 - value: 11.745 - - type: map_at_100 - value: 14.831 - - type: map_at_1000 - value: 16.235 - - type: map_at_3 - value: 8.716 - - type: map_at_5 - value: 10.101 - - type: mrr_at_1 - value: 43.653 - - type: mrr_at_10 - value: 51.06699999999999 - - type: mrr_at_100 - value: 51.881 - - type: mrr_at_1000 - value: 51.912000000000006 - - type: mrr_at_3 - value: 49.02 - - type: mrr_at_5 - value: 50.288999999999994 - - type: ndcg_at_1 - value: 41.949999999999996 - - type: ndcg_at_10 - value: 32.083 - - type: ndcg_at_100 - value: 30.049999999999997 - - type: ndcg_at_1000 - value: 38.661 - - type: ndcg_at_3 - value: 37.940000000000005 - - type: ndcg_at_5 - value: 35.455999999999996 - - type: precision_at_1 - value: 43.344 - - type: precision_at_10 - value: 23.437 - - type: precision_at_100 - value: 7.829999999999999 - - type: precision_at_1000 - value: 2.053 - - type: precision_at_3 - value: 35.501 - - type: precision_at_5 - value: 30.464000000000002 - - type: recall_at_1 - value: 5.526 - - type: recall_at_10 - value: 15.445999999999998 - - type: recall_at_100 - value: 31.179000000000002 - - type: recall_at_1000 - value: 61.578 - - type: recall_at_3 - value: 9.71 - - type: recall_at_5 - value: 12.026 + - type: accuracy + value: 49.56043956043956 + - type: f1 + value: 32.86333673498598 - task: - type: Retrieval + type: Classification dataset: - type: nq - name: MTEB NQ + type: mteb/mtop_intent + name: MTEB MTOPIntentClassification (es) + config: es + split: test metrics: - - type: map_at_1 - value: 23.467 - - type: map_at_10 - value: 36.041000000000004 - - type: map_at_100 - value: 37.268 - - type: map_at_1000 - value: 37.322 - - type: map_at_3 - value: 32.09 - - type: map_at_5 - value: 34.414 - - type: mrr_at_1 - value: 26.738 - - type: mrr_at_10 - value: 38.665 - - type: mrr_at_100 - value: 39.64 - - type: mrr_at_1000 - value: 39.681 - - type: mrr_at_3 - value: 35.207 - - type: mrr_at_5 - value: 37.31 - - type: ndcg_at_1 - value: 26.709 - - type: ndcg_at_10 - value: 42.942 - - type: ndcg_at_100 - value: 48.296 - - type: ndcg_at_1000 - value: 49.651 - - type: ndcg_at_3 - value: 35.413 - - type: ndcg_at_5 - value: 39.367999999999995 - - type: precision_at_1 - value: 26.709 - - type: precision_at_10 - value: 7.306 - - type: precision_at_100 - value: 1.0290000000000001 - - type: precision_at_1000 - value: 0.116 - - type: precision_at_3 - value: 16.348 - - type: precision_at_5 - value: 12.068 - - type: recall_at_1 - value: 23.467 - - type: recall_at_10 - value: 61.492999999999995 - - type: recall_at_100 - value: 85.01100000000001 - - type: recall_at_1000 - value: 95.261 - - type: recall_at_3 - value: 41.952 - - type: recall_at_5 - value: 51.105999999999995 + - type: accuracy + value: 49.93995997331555 + - type: f1 + value: 34.726671876888126 - task: - type: Retrieval + type: Classification dataset: - type: quora - name: MTEB QuoraRetrieval + type: mteb/mtop_intent + name: MTEB MTOPIntentClassification (fr) + config: fr + split: test metrics: - - type: map_at_1 - value: 67.51700000000001 - - type: map_at_10 - value: 81.054 - - type: map_at_100 - value: 81.727 - - type: map_at_1000 - value: 81.75200000000001 - - type: map_at_3 - value: 78.018 - - type: map_at_5 - value: 79.879 - - type: mrr_at_1 - value: 77.52 - - type: mrr_at_10 - value: 84.429 - - type: mrr_at_100 - value: 84.58200000000001 - - type: mrr_at_1000 - value: 84.584 - - type: mrr_at_3 - value: 83.268 - - type: mrr_at_5 - value: 84.013 - - type: ndcg_at_1 - value: 77.53 - - type: ndcg_at_10 - value: 85.277 - - type: ndcg_at_100 - value: 86.80499999999999 - - type: ndcg_at_1000 - value: 87.01 - - type: ndcg_at_3 - value: 81.975 - - type: ndcg_at_5 - value: 83.723 - - type: precision_at_1 - value: 77.53 - - type: precision_at_10 - value: 12.961 - - type: precision_at_100 - value: 1.502 - - type: precision_at_1000 - value: 0.156 - - type: precision_at_3 - value: 35.713 - - type: precision_at_5 - value: 23.574 - - type: recall_at_1 - value: 67.51700000000001 - - type: recall_at_10 - value: 93.486 - - type: recall_at_100 - value: 98.9 - - type: recall_at_1000 - value: 99.92999999999999 - - type: recall_at_3 - value: 84.17999999999999 - - type: recall_at_5 - value: 88.97500000000001 + - type: accuracy + value: 46.32947071719386 + - type: f1 + value: 32.325273615982795 - task: - type: Clustering + type: Classification dataset: - type: mteb/reddit-clustering - name: MTEB RedditClustering + type: mteb/mtop_intent + name: MTEB MTOPIntentClassification (hi) + config: hi + split: test metrics: - - type: v_measure - value: 48.225994608749915 + - type: accuracy + value: 32.208676945141626 + - type: f1 + value: 21.32185122815139 - task: - type: Clustering + type: Classification dataset: - type: mteb/reddit-clustering-p2p - name: MTEB RedditClusteringP2P + type: mteb/mtop_intent + name: MTEB MTOPIntentClassification (th) + config: th + split: test metrics: - - type: v_measure - value: 53.17635557157765 + - type: accuracy + value: 43.627486437613015 + - type: f1 + value: 27.04872922347508 - task: - type: Retrieval + type: Classification dataset: - type: scidocs - name: MTEB SCIDOCS + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (af) + config: af + split: test metrics: - - type: map_at_1 - value: 3.988 - - type: map_at_10 - value: 9.4 - - type: map_at_100 - value: 10.968 - - type: map_at_1000 - value: 11.257 - - type: map_at_3 - value: 7.123 - - type: map_at_5 - value: 8.221 - - type: mrr_at_1 - value: 19.7 - - type: mrr_at_10 - value: 29.098000000000003 - - type: mrr_at_100 - value: 30.247 - - type: mrr_at_1000 - value: 30.318 - - type: mrr_at_3 - value: 26.55 - - type: mrr_at_5 - value: 27.915 - - type: ndcg_at_1 - value: 19.7 - - type: ndcg_at_10 - value: 16.176 + - type: accuracy + value: 40.548083389374575 + - type: f1 + value: 39.490307545239716 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (am) + config: am + split: test + metrics: + - type: accuracy + value: 24.18291862811029 + - type: f1 + value: 23.437620034727473 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (ar) + config: ar + split: test + metrics: + - type: accuracy + value: 30.134498991257562 + - type: f1 + value: 28.787175191531283 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (az) + config: az + split: test + metrics: + - type: accuracy + value: 35.88433086751849 + - type: f1 + value: 36.264500398782126 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (bn) + config: bn + split: test + metrics: + - type: accuracy + value: 29.17283120376597 + - type: f1 + value: 27.8101616531901 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (cy) + config: cy + split: test + metrics: + - type: accuracy + value: 41.788836583725626 + - type: f1 + value: 39.71413181054801 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (da) + config: da + split: test + metrics: + - type: accuracy + value: 44.176193678547406 + - type: f1 + value: 42.192499826552286 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (de) + config: de + split: test + metrics: + - type: accuracy + value: 42.07464694014795 + - type: f1 + value: 39.44188259183162 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (el) + config: el + split: test + metrics: + - type: accuracy + value: 36.254203093476804 + - type: f1 + value: 34.46592715936761 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (en) + config: en + split: test + metrics: + - type: accuracy + value: 61.40887693342301 + - type: f1 + value: 59.79854802683996 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (es) + config: es + split: test + metrics: + - type: accuracy + value: 42.679892400807 + - type: f1 + value: 42.04801248338172 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (fa) + config: fa + split: test + metrics: + - type: accuracy + value: 35.59179556153329 + - type: f1 + value: 34.045862930486166 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (fi) + config: fi + split: test + metrics: + - type: accuracy + value: 40.036987222595826 + - type: f1 + value: 38.117703439362785 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (fr) + config: fr + split: test + metrics: + - type: accuracy + value: 43.43981170141224 + - type: f1 + value: 42.7084388987865 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (he) + config: he + split: test + metrics: + - type: accuracy + value: 31.593813046402154 + - type: f1 + value: 29.98550522450782 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (hi) + config: hi + split: test + metrics: + - type: accuracy + value: 27.044384667114997 + - type: f1 + value: 27.313059184832667 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (hu) + config: hu + split: test + metrics: + - type: accuracy + value: 38.453261600538 + - type: f1 + value: 37.309189326110435 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (hy) + config: hy + split: test + metrics: + - type: accuracy + value: 27.979152656355076 + - type: f1 + value: 27.430939684346445 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (id) + config: id + split: test + metrics: + - type: accuracy + value: 43.97108271687963 + - type: f1 + value: 43.40585705688761 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (is) + config: is + split: test + metrics: + - type: accuracy + value: 40.302622730329524 + - type: f1 + value: 39.108052180520744 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (it) + config: it + split: test + metrics: + - type: accuracy + value: 45.474108944182916 + - type: f1 + value: 45.85950328241134 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (ja) + config: ja + split: test + metrics: + - type: accuracy + value: 45.60860793544048 + - type: f1 + value: 43.94920708216737 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (jv) + config: jv + split: test + metrics: + - type: accuracy + value: 38.668459986550104 + - type: f1 + value: 37.6990034018859 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (ka) + config: ka + split: test + metrics: + - type: accuracy + value: 25.6523201075992 + - type: f1 + value: 25.279084273189582 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (km) + config: km + split: test + metrics: + - type: accuracy + value: 28.295225285810353 + - type: f1 + value: 26.645825638771548 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (kn) + config: kn + split: test + metrics: + - type: accuracy + value: 23.480161398789505 + - type: f1 + value: 22.275241866506732 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (ko) + config: ko + split: test + metrics: + - type: accuracy + value: 36.55682582380632 + - type: f1 + value: 36.004753171063605 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (lv) + config: lv + split: test + metrics: + - type: accuracy + value: 41.84936112979153 + - type: f1 + value: 41.38932672359119 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (ml) + config: ml + split: test + metrics: + - type: accuracy + value: 24.90921318090114 + - type: f1 + value: 23.968687483768807 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (mn) + config: mn + split: test + metrics: + - type: accuracy + value: 29.86213853396099 + - type: f1 + value: 29.977152075255407 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (ms) + config: ms + split: test + metrics: + - type: accuracy + value: 42.42098184263618 + - type: f1 + value: 41.50877432664628 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (my) + config: my + split: test + metrics: + - type: accuracy + value: 25.131136516476126 + - type: f1 + value: 23.938932214086776 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (nb) + config: nb + split: test + metrics: + - type: accuracy + value: 39.81506388702084 + - type: f1 + value: 38.809586587791664 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (nl) + config: nl + split: test + metrics: + - type: accuracy + value: 43.62138533960995 + - type: f1 + value: 42.01386842914633 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (pl) + config: pl + split: test + metrics: + - type: accuracy + value: 42.19569603227976 + - type: f1 + value: 40.00556559825827 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (pt) + config: pt + split: test + metrics: + - type: accuracy + value: 45.20847343644923 + - type: f1 + value: 44.24115005029051 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (ro) + config: ro + split: test + metrics: + - type: accuracy + value: 41.80901143241426 + - type: f1 + value: 40.474074848670085 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (ru) + config: ru + split: test + metrics: + - type: accuracy + value: 35.96839273705447 + - type: f1 + value: 35.095456843621 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (sl) + config: sl + split: test + metrics: + - type: accuracy + value: 40.60524546065905 + - type: f1 + value: 39.302383051500136 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (sq) + config: sq + split: test + metrics: + - type: accuracy + value: 42.75722932078009 + - type: f1 + value: 41.53763931497389 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (sv) + config: sv + split: test + metrics: + - type: accuracy + value: 42.347007397444514 + - type: f1 + value: 41.04366017948627 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (sw) + config: sw + split: test + metrics: + - type: accuracy + value: 41.12306657700067 + - type: f1 + value: 39.712940473289024 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (ta) + config: ta + split: test + metrics: + - type: accuracy + value: 24.603227975790183 + - type: f1 + value: 23.969236788828606 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (te) + config: te + split: test + metrics: + - type: accuracy + value: 25.03698722259583 + - type: f1 + value: 24.37196123281459 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (th) + config: th + split: test + metrics: + - type: accuracy + value: 35.40013449899126 + - type: f1 + value: 35.063600413688036 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (tl) + config: tl + split: test + metrics: + - type: accuracy + value: 41.19031607262945 + - type: f1 + value: 40.240432304273014 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (tr) + config: tr + split: test + metrics: + - type: accuracy + value: 36.405514458641555 + - type: f1 + value: 36.03844992856558 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (ur) + config: ur + split: test + metrics: + - type: accuracy + value: 25.934767989240076 + - type: f1 + value: 25.2074457023531 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (vi) + config: vi + split: test + metrics: + - type: accuracy + value: 38.79959650302622 + - type: f1 + value: 37.160233794673125 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (zh-CN) + config: zh-CN + split: test + metrics: + - type: accuracy + value: 46.244115669132476 + - type: f1 + value: 44.367480561291906 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (zh-TW) + config: zh-TW + split: test + metrics: + - type: accuracy + value: 42.30665770006724 + - type: f1 + value: 41.9642223283514 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (af) + config: af + split: test + metrics: + - type: accuracy + value: 43.2481506388702 + - type: f1 + value: 40.924230769590785 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (am) + config: am + split: test + metrics: + - type: accuracy + value: 25.30262273032952 + - type: f1 + value: 24.937105830264066 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (ar) + config: ar + split: test + metrics: + - type: accuracy + value: 32.07128446536651 + - type: f1 + value: 31.80245816594883 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (az) + config: az + split: test + metrics: + - type: accuracy + value: 36.681237390719566 + - type: f1 + value: 36.37219042508338 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (bn) + config: bn + split: test + metrics: + - type: accuracy + value: 29.56624075319435 + - type: f1 + value: 28.386042056362758 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (cy) + config: cy + split: test + metrics: + - type: accuracy + value: 42.1049092131809 + - type: f1 + value: 38.926150886991294 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (da) + config: da + split: test + metrics: + - type: accuracy + value: 45.44384667114997 + - type: f1 + value: 42.578252395460005 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (de) + config: de + split: test + metrics: + - type: accuracy + value: 43.211163416274374 + - type: f1 + value: 41.04465858304789 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (el) + config: el + split: test + metrics: + - type: accuracy + value: 36.503026227303295 + - type: f1 + value: 34.49785095312759 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (en) + config: en + split: test + metrics: + - type: accuracy + value: 69.73772696704773 + - type: f1 + value: 69.21759502909043 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (es) + config: es + split: test + metrics: + - type: accuracy + value: 44.078681909885674 + - type: f1 + value: 43.05914426901129 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (fa) + config: fa + split: test + metrics: + - type: accuracy + value: 32.61264290517821 + - type: f1 + value: 32.02463177462754 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (fi) + config: fi + split: test + metrics: + - type: accuracy + value: 40.35642232683255 + - type: f1 + value: 38.13642481807678 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (fr) + config: fr + split: test + metrics: + - type: accuracy + value: 45.06724949562878 + - type: f1 + value: 43.19827608343738 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (he) + config: he + split: test + metrics: + - type: accuracy + value: 32.178883658372555 + - type: f1 + value: 29.979761884698775 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (hi) + config: hi + split: test + metrics: + - type: accuracy + value: 26.903160726294555 + - type: f1 + value: 25.833010434083363 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (hu) + config: hu + split: test + metrics: + - type: accuracy + value: 40.379959650302624 + - type: f1 + value: 37.93134355292882 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (hy) + config: hy + split: test + metrics: + - type: accuracy + value: 28.375924680564896 + - type: f1 + value: 26.96255693013172 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (id) + config: id + split: test + metrics: + - type: accuracy + value: 44.361129791526565 + - type: f1 + value: 43.54445012295126 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (is) + config: is + split: test + metrics: + - type: accuracy + value: 39.290517821116346 + - type: f1 + value: 37.26982052174147 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (it) + config: it + split: test + metrics: + - type: accuracy + value: 46.4694014794889 + - type: f1 + value: 44.060986162841566 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (ja) + config: ja + split: test + metrics: + - type: accuracy + value: 46.25756556825824 + - type: f1 + value: 45.625139456758816 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (jv) + config: jv + split: test + metrics: + - type: accuracy + value: 41.12642905178212 + - type: f1 + value: 39.54392378396527 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (ka) + config: ka + split: test + metrics: + - type: accuracy + value: 24.72763954270343 + - type: f1 + value: 23.337743140804484 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (km) + config: km + split: test + metrics: + - type: accuracy + value: 29.741089441829182 + - type: f1 + value: 27.570876190083748 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (kn) + config: kn + split: test + metrics: + - type: accuracy + value: 23.850033624747816 + - type: f1 + value: 22.86733484540032 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (ko) + config: ko + split: test + metrics: + - type: accuracy + value: 36.56691324815064 + - type: f1 + value: 35.504081677134565 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (lv) + config: lv + split: test + metrics: + - type: accuracy + value: 40.928043039677206 + - type: f1 + value: 39.108589131211254 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (ml) + config: ml + split: test + metrics: + - type: accuracy + value: 25.527908540685946 + - type: f1 + value: 25.333391622280477 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (mn) + config: mn + split: test + metrics: + - type: accuracy + value: 29.105581708137183 + - type: f1 + value: 28.478235012692814 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (ms) + config: ms + split: test + metrics: + - type: accuracy + value: 43.78614660390047 + - type: f1 + value: 41.9640143926267 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (my) + config: my + split: test + metrics: + - type: accuracy + value: 27.269670477471415 + - type: f1 + value: 26.228386764141852 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (nb) + config: nb + split: test + metrics: + - type: accuracy + value: 39.018157363819775 + - type: f1 + value: 37.641949339321854 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (nl) + config: nl + split: test + metrics: + - type: accuracy + value: 45.35978480161399 + - type: f1 + value: 42.6851176096831 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (pl) + config: pl + split: test + metrics: + - type: accuracy + value: 41.89307330195023 + - type: f1 + value: 40.888710642615024 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (pt) + config: pt + split: test + metrics: + - type: accuracy + value: 45.901143241425686 + - type: f1 + value: 44.496942353920545 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (ro) + config: ro + split: test + metrics: + - type: accuracy + value: 44.11566913248151 + - type: f1 + value: 41.953945105870616 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (ru) + config: ru + split: test + metrics: + - type: accuracy + value: 32.76395427034297 + - type: f1 + value: 31.436372571600934 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (sl) + config: sl + split: test + metrics: + - type: accuracy + value: 40.504371217215876 + - type: f1 + value: 39.322752749628165 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (sq) + config: sq + split: test + metrics: + - type: accuracy + value: 42.51849361129792 + - type: f1 + value: 41.4139297118463 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (sv) + config: sv + split: test + metrics: + - type: accuracy + value: 42.293207800941495 + - type: f1 + value: 40.50409536806683 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (sw) + config: sw + split: test + metrics: + - type: accuracy + value: 42.9993275050437 + - type: f1 + value: 41.045416224973266 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (ta) + config: ta + split: test + metrics: + - type: accuracy + value: 28.32548755884331 + - type: f1 + value: 27.276841995561867 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (te) + config: te + split: test + metrics: + - type: accuracy + value: 26.593813046402154 + - type: f1 + value: 25.483878616197586 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (th) + config: th + split: test + metrics: + - type: accuracy + value: 36.788836583725626 + - type: f1 + value: 34.603932909177686 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (tl) + config: tl + split: test + metrics: + - type: accuracy + value: 42.5689307330195 + - type: f1 + value: 40.924469309079825 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (tr) + config: tr + split: test + metrics: + - type: accuracy + value: 37.09482178883658 + - type: f1 + value: 37.949628822857164 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (ur) + config: ur + split: test + metrics: + - type: accuracy + value: 28.836583725622063 + - type: f1 + value: 27.806558655512344 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (vi) + config: vi + split: test + metrics: + - type: accuracy + value: 37.357094821788834 + - type: f1 + value: 37.507918961038165 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (zh-CN) + config: zh-CN + split: test + metrics: + - type: accuracy + value: 49.37794216543375 + - type: f1 + value: 47.20421153697707 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (zh-TW) + config: zh-TW + split: test + metrics: + - type: accuracy + value: 44.42165433759248 + - type: f1 + value: 44.34741861198931 + - task: + type: Clustering + dataset: + type: mteb/medrxiv-clustering-p2p + name: MTEB MedrxivClusteringP2P + config: default + split: test + metrics: + - type: v_measure + value: 31.374938993074252 + - task: + type: Clustering + dataset: + type: mteb/medrxiv-clustering-s2s + name: MTEB MedrxivClusteringS2S + config: default + split: test + metrics: + - type: v_measure + value: 26.871455379644093 + - task: + type: Reranking + dataset: + type: mteb/mind_small + name: MTEB MindSmallReranking + config: default + split: test + metrics: + - type: map + value: 30.402396942935333 + - type: mrr + value: 31.42600938803256 + - task: + type: Retrieval + dataset: + type: nfcorpus + name: MTEB NFCorpus + config: default + split: test + metrics: + - type: map_at_1 + value: 3.7740000000000005 + - type: map_at_10 + value: 7.614999999999999 + - type: map_at_100 + value: 9.574 + - type: map_at_1000 + value: 10.711 + - type: map_at_3 + value: 5.7540000000000004 + - type: map_at_5 + value: 6.6659999999999995 + - type: mrr_at_1 + value: 33.127 + - type: mrr_at_10 + value: 40.351 + - type: mrr_at_100 + value: 41.144 + - type: mrr_at_1000 + value: 41.202 + - type: mrr_at_3 + value: 38.029 + - type: mrr_at_5 + value: 39.190000000000005 + - type: ndcg_at_1 + value: 31.579 + - type: ndcg_at_10 + value: 22.792 + - type: ndcg_at_100 + value: 21.698999999999998 + - type: ndcg_at_1000 + value: 30.892999999999997 + - type: ndcg_at_3 + value: 26.828999999999997 + - type: ndcg_at_5 + value: 25.119000000000003 + - type: precision_at_1 + value: 33.127 + - type: precision_at_10 + value: 16.718 + - type: precision_at_100 + value: 5.7090000000000005 + - type: precision_at_1000 + value: 1.836 + - type: precision_at_3 + value: 24.768 + - type: precision_at_5 + value: 21.3 + - type: recall_at_1 + value: 3.7740000000000005 + - type: recall_at_10 + value: 10.302999999999999 + - type: recall_at_100 + value: 23.013 + - type: recall_at_1000 + value: 54.864999999999995 + - type: recall_at_3 + value: 6.554 + - type: recall_at_5 + value: 8.087 + - task: + type: Retrieval + dataset: + type: nq + name: MTEB NQ + config: default + split: test + metrics: + - type: map_at_1 + value: 15.620999999999999 + - type: map_at_10 + value: 24.519 + - type: map_at_100 + value: 25.586 + - type: map_at_1000 + value: 25.662000000000003 + - type: map_at_3 + value: 21.619 + - type: map_at_5 + value: 23.232 + - type: mrr_at_1 + value: 17.497 + - type: mrr_at_10 + value: 26.301000000000002 + - type: mrr_at_100 + value: 27.235 + - type: mrr_at_1000 + value: 27.297 + - type: mrr_at_3 + value: 23.561 + - type: mrr_at_5 + value: 25.111 + - type: ndcg_at_1 + value: 17.497 + - type: ndcg_at_10 + value: 29.725 + - type: ndcg_at_100 + value: 34.824 + - type: ndcg_at_1000 + value: 36.907000000000004 + - type: ndcg_at_3 + value: 23.946 + - type: ndcg_at_5 + value: 26.739 + - type: precision_at_1 + value: 17.497 + - type: precision_at_10 + value: 5.2170000000000005 + - type: precision_at_100 + value: 0.8099999999999999 + - type: precision_at_1000 + value: 0.101 + - type: precision_at_3 + value: 11.114 + - type: precision_at_5 + value: 8.285 + - type: recall_at_1 + value: 15.620999999999999 + - type: recall_at_10 + value: 43.999 + - type: recall_at_100 + value: 67.183 + - type: recall_at_1000 + value: 83.174 + - type: recall_at_3 + value: 28.720000000000002 + - type: recall_at_5 + value: 35.154 + - task: + type: Retrieval + dataset: + type: quora + name: MTEB QuoraRetrieval + config: default + split: test + metrics: + - type: map_at_1 + value: 54.717000000000006 + - type: map_at_10 + value: 67.514 + - type: map_at_100 + value: 68.484 + - type: map_at_1000 + value: 68.523 + - type: map_at_3 + value: 64.169 + - type: map_at_5 + value: 66.054 + - type: mrr_at_1 + value: 62.46000000000001 + - type: mrr_at_10 + value: 71.503 + - type: mrr_at_100 + value: 71.91499999999999 + - type: mrr_at_1000 + value: 71.923 + - type: mrr_at_3 + value: 69.46799999999999 + - type: mrr_at_5 + value: 70.677 + - type: ndcg_at_1 + value: 62.480000000000004 + - type: ndcg_at_10 + value: 72.98 + - type: ndcg_at_100 + value: 76.023 + - type: ndcg_at_1000 + value: 76.512 + - type: ndcg_at_3 + value: 68.138 + - type: ndcg_at_5 + value: 70.458 + - type: precision_at_1 + value: 62.480000000000004 + - type: precision_at_10 + value: 11.373 + - type: precision_at_100 + value: 1.437 + - type: precision_at_1000 + value: 0.154 + - type: precision_at_3 + value: 29.622999999999998 + - type: precision_at_5 + value: 19.918 + - type: recall_at_1 + value: 54.717000000000006 + - type: recall_at_10 + value: 84.745 + - type: recall_at_100 + value: 96.528 + - type: recall_at_1000 + value: 99.39 + - type: recall_at_3 + value: 71.60600000000001 + - type: recall_at_5 + value: 77.511 + - task: + type: Clustering + dataset: + type: mteb/reddit-clustering + name: MTEB RedditClustering + config: default + split: test + metrics: + - type: v_measure + value: 40.23390747226228 + - task: + type: Clustering + dataset: + type: mteb/reddit-clustering-p2p + name: MTEB RedditClusteringP2P + config: default + split: test + metrics: + - type: v_measure + value: 49.090518272935626 + - task: + type: Retrieval + dataset: + type: scidocs + name: MTEB SCIDOCS + config: default + split: test + metrics: + - type: map_at_1 + value: 3.028 + - type: map_at_10 + value: 6.968000000000001 + - type: map_at_100 + value: 8.200000000000001 + - type: map_at_1000 + value: 8.432 + - type: map_at_3 + value: 5.3069999999999995 + - type: map_at_5 + value: 6.099 + - type: mrr_at_1 + value: 14.799999999999999 + - type: mrr_at_10 + value: 22.425 + - type: mrr_at_100 + value: 23.577 + - type: mrr_at_1000 + value: 23.669999999999998 + - type: mrr_at_3 + value: 20.233 + - type: mrr_at_5 + value: 21.318 + - type: ndcg_at_1 + value: 14.799999999999999 + - type: ndcg_at_10 + value: 12.206 - type: ndcg_at_100 - value: 22.931 + value: 17.799 - type: ndcg_at_1000 - value: 28.301 + value: 22.891000000000002 - type: ndcg_at_3 - value: 16.142 + value: 12.128 - type: ndcg_at_5 - value: 13.633999999999999 + value: 10.212 - type: precision_at_1 - value: 19.7 + value: 14.799999999999999 - type: precision_at_10 - value: 8.18 + value: 6.17 - type: precision_at_100 - value: 1.8010000000000002 + value: 1.428 - type: precision_at_1000 - value: 0.309 + value: 0.266 - type: precision_at_3 - value: 15.1 + value: 11.333 - type: precision_at_5 - value: 11.74 + value: 8.74 - type: recall_at_1 - value: 3.988 + value: 3.028 - type: recall_at_10 - value: 16.625 + value: 12.522 - type: recall_at_100 - value: 36.61 + value: 28.975 - type: recall_at_1000 - value: 62.805 + value: 54.038 - type: recall_at_3 - value: 9.168 + value: 6.912999999999999 - type: recall_at_5 - value: 11.902 + value: 8.883000000000001 + - task: + type: STS + dataset: + type: mteb/sickr-sts + name: MTEB SICK-R + config: default + split: test + metrics: + - type: cos_sim_pearson + value: 76.62983928119752 + - type: cos_sim_spearman + value: 65.92910683118656 + - type: euclidean_pearson + value: 71.10290039690963 + - type: euclidean_spearman + value: 64.80076622426652 + - type: manhattan_pearson + value: 70.8944726230188 + - type: manhattan_spearman + value: 64.75082576033986 + - task: + type: STS + dataset: + type: mteb/sts12-sts + name: MTEB STS12 + config: default + split: test + metrics: + - type: cos_sim_pearson + value: 74.42679147085553 + - type: cos_sim_spearman + value: 66.52980061546658 + - type: euclidean_pearson + value: 74.87039477408763 + - type: euclidean_spearman + value: 70.63397666902786 + - type: manhattan_pearson + value: 74.97015137513088 + - type: manhattan_spearman + value: 70.75951355434326 + - task: + type: STS + dataset: + type: mteb/sts13-sts + name: MTEB STS13 + config: default + split: test + metrics: + - type: cos_sim_pearson + value: 75.62472426599543 + - type: cos_sim_spearman + value: 76.1662886374236 + - type: euclidean_pearson + value: 76.3297128081315 + - type: euclidean_spearman + value: 77.19385151966563 + - type: manhattan_pearson + value: 76.50363291423257 + - type: manhattan_spearman + value: 77.37081896355399 + - task: + type: STS + dataset: + type: mteb/sts14-sts + name: MTEB STS14 + config: default + split: test + metrics: + - type: cos_sim_pearson + value: 74.48227705407035 + - type: cos_sim_spearman + value: 69.04572664009687 + - type: euclidean_pearson + value: 71.76138185714849 + - type: euclidean_spearman + value: 68.93415452043307 + - type: manhattan_pearson + value: 71.68010915543306 + - type: manhattan_spearman + value: 68.99176321262806 + - task: + type: STS + dataset: + type: mteb/sts15-sts + name: MTEB STS15 + config: default + split: test + metrics: + - type: cos_sim_pearson + value: 78.1566527175902 + - type: cos_sim_spearman + value: 79.23677712825851 + - type: euclidean_pearson + value: 76.29138438696417 + - type: euclidean_spearman + value: 77.20108266215374 + - type: manhattan_pearson + value: 76.27464935799118 + - type: manhattan_spearman + value: 77.15286174478099 + - task: + type: STS + dataset: + type: mteb/sts16-sts + name: MTEB STS16 + config: default + split: test + metrics: + - type: cos_sim_pearson + value: 75.068454465977 + - type: cos_sim_spearman + value: 76.06792422441929 + - type: euclidean_pearson + value: 70.64605440627699 + - type: euclidean_spearman + value: 70.21776051117844 + - type: manhattan_pearson + value: 70.32479295054918 + - type: manhattan_spearman + value: 69.89782458638528 - task: type: STS dataset: - type: mteb/sickr-sts - name: MTEB SICK-R + type: mteb/sts17-crosslingual-sts + name: MTEB STS17 (ko-ko) + config: ko-ko + split: test metrics: - type: cos_sim_pearson - value: 77.29330379162072 + value: 39.43327289939437 - type: cos_sim_spearman - value: 67.22953551111448 + value: 52.386010275505654 - type: euclidean_pearson - value: 71.44682700059415 + value: 46.40999904885745 - type: euclidean_spearman - value: 66.33178012153247 + value: 51.00333465175934 - type: manhattan_pearson - value: 71.46941734657887 + value: 46.55753533133655 - type: manhattan_spearman - value: 66.43234359835814 + value: 51.07550440519388 - task: type: STS dataset: - type: mteb/sts12-sts - name: MTEB STS12 + type: mteb/sts17-crosslingual-sts + name: MTEB STS17 (ar-ar) + config: ar-ar + split: test metrics: - type: cos_sim_pearson - value: 75.40943196466576 + value: 55.54431928210687 - type: cos_sim_spearman - value: 66.59241013465915 + value: 55.61674586076298 - type: euclidean_pearson - value: 71.32500540796616 + value: 58.07442713714088 - type: euclidean_spearman - value: 67.86667467202591 + value: 55.74066216931719 - type: manhattan_pearson - value: 71.48209832089134 + value: 57.84021675638542 - type: manhattan_spearman - value: 67.94511626964879 + value: 55.20365812536853 - task: type: STS dataset: - type: mteb/sts13-sts - name: MTEB STS13 + type: mteb/sts17-crosslingual-sts + name: MTEB STS17 (en-ar) + config: en-ar + split: test metrics: - type: cos_sim_pearson - value: 77.08302398877518 + value: 11.378463868809098 - type: cos_sim_spearman - value: 77.33151317062642 + value: 8.209569244801065 - type: euclidean_pearson - value: 76.77020279715008 + value: 1.07041700730406 - type: euclidean_spearman - value: 77.13893776083225 + value: 2.2052197108931892 - type: manhattan_pearson - value: 76.76732290707477 + value: 0.7671300251104268 - type: manhattan_spearman - value: 77.14500877396631 + value: 3.430645020535567 - task: type: STS dataset: - type: mteb/sts14-sts - name: MTEB STS14 + type: mteb/sts17-crosslingual-sts + name: MTEB STS17 (en-de) + config: en-de + split: test metrics: - type: cos_sim_pearson - value: 77.46886184932168 + value: 32.71403560929013 - type: cos_sim_spearman - value: 71.82815265534886 + value: 30.18181775929109 - type: euclidean_pearson - value: 75.19783284299076 + value: 25.57368595910298 - type: euclidean_spearman - value: 71.36479611710412 + value: 23.316649115731376 - type: manhattan_pearson - value: 75.30375233959337 + value: 24.144200325329614 - type: manhattan_spearman - value: 71.46280266488021 + value: 21.64621546338457 - task: type: STS dataset: - type: mteb/sts15-sts - name: MTEB STS15 + type: mteb/sts17-crosslingual-sts + name: MTEB STS17 (en-en) + config: en-en + split: test metrics: - type: cos_sim_pearson - value: 80.093017609484 + value: 83.36340470799158 - type: cos_sim_spearman - value: 80.65931167868882 + value: 84.95398260629699 - type: euclidean_pearson - value: 80.36786337117047 + value: 80.69876969911644 - type: euclidean_spearman - value: 81.30521389642827 + value: 80.97451731130427 - type: manhattan_pearson - value: 80.37922433220973 + value: 80.65869354146945 - type: manhattan_spearman - value: 81.30496664496285 + value: 80.8540858718528 - task: type: STS dataset: - type: mteb/sts16-sts - name: MTEB STS16 + type: mteb/sts17-crosslingual-sts + name: MTEB STS17 (en-tr) + config: en-tr + split: test metrics: - type: cos_sim_pearson - value: 77.98998347238742 + value: 1.9200044163754912 - type: cos_sim_spearman - value: 78.91151365939403 + value: 1.0393399782021342 - type: euclidean_pearson - value: 76.40510899217841 + value: 1.1376003191297994 - type: euclidean_spearman - value: 76.8551459824213 + value: 1.8947106671763914 - type: manhattan_pearson - value: 76.3986079603294 + value: 3.8362564474484335 - type: manhattan_spearman - value: 76.8848053254288 + value: 4.242750882792888 - task: type: STS dataset: type: mteb/sts17-crosslingual-sts - name: MTEB STS17 (en-en) + name: MTEB STS17 (es-en) + config: es-en + split: test + metrics: + - type: cos_sim_pearson + value: 26.561262451099577 + - type: cos_sim_spearman + value: 28.776666666659906 + - type: euclidean_pearson + value: 14.640410196999088 + - type: euclidean_spearman + value: 16.10557011701786 + - type: manhattan_pearson + value: 15.019405495911272 + - type: manhattan_spearman + value: 15.37192083104197 + - task: + type: STS + dataset: + type: mteb/sts17-crosslingual-sts + name: MTEB STS17 (es-es) + config: es-es + split: test + metrics: + - type: cos_sim_pearson + value: 69.7544202001433 + - type: cos_sim_spearman + value: 71.88444295144646 + - type: euclidean_pearson + value: 73.84934185952773 + - type: euclidean_spearman + value: 73.26911108021089 + - type: manhattan_pearson + value: 74.04354196954574 + - type: manhattan_spearman + value: 73.37650787943872 + - task: + type: STS + dataset: + type: mteb/sts17-crosslingual-sts + name: MTEB STS17 (fr-en) + config: fr-en + split: test + metrics: + - type: cos_sim_pearson + value: 27.70511842301491 + - type: cos_sim_spearman + value: 26.339466714066447 + - type: euclidean_pearson + value: 9.323158236506385 + - type: euclidean_spearman + value: 7.32083231520273 + - type: manhattan_pearson + value: 7.807399527573071 + - type: manhattan_spearman + value: 5.525546663067113 + - task: + type: STS + dataset: + type: mteb/sts17-crosslingual-sts + name: MTEB STS17 (it-en) + config: it-en + split: test + metrics: + - type: cos_sim_pearson + value: 24.226521799447692 + - type: cos_sim_spearman + value: 20.72992940458968 + - type: euclidean_pearson + value: 6.753378617205011 + - type: euclidean_spearman + value: 6.281654679029505 + - type: manhattan_pearson + value: 7.087180250449323 + - type: manhattan_spearman + value: 6.41611659259516 + - task: + type: STS + dataset: + type: mteb/sts17-crosslingual-sts + name: MTEB STS17 (nl-en) + config: nl-en + split: test metrics: - type: cos_sim_pearson - value: 85.63510653472044 + value: 29.131412364061234 - type: cos_sim_spearman - value: 86.98674844768605 + value: 25.053429612793547 - type: euclidean_pearson - value: 85.205080538809 + value: 10.657141303962 - type: euclidean_spearman - value: 85.53630494151886 + value: 9.712124819778452 - type: manhattan_pearson - value: 85.48612469885626 + value: 12.481782693315688 - type: manhattan_spearman - value: 85.81741413931921 + value: 11.287958480905973 - task: type: STS dataset: type: mteb/sts22-crosslingual-sts name: MTEB STS22 (en) + config: en + split: test + metrics: + - type: cos_sim_pearson + value: 64.04750650962879 + - type: cos_sim_spearman + value: 65.66183708171826 + - type: euclidean_pearson + value: 66.90887604405887 + - type: euclidean_spearman + value: 66.89814072484552 + - type: manhattan_pearson + value: 67.31627110509089 + - type: manhattan_spearman + value: 67.01048176165322 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (de) + config: de + split: test + metrics: + - type: cos_sim_pearson + value: 19.26519187000913 + - type: cos_sim_spearman + value: 21.987647321429005 + - type: euclidean_pearson + value: 17.850618752342946 + - type: euclidean_spearman + value: 22.86669392885474 + - type: manhattan_pearson + value: 18.16183594260708 + - type: manhattan_spearman + value: 23.637510352837907 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (es) + config: es + split: test + metrics: + - type: cos_sim_pearson + value: 34.221261828226936 + - type: cos_sim_spearman + value: 49.811823238907664 + - type: euclidean_pearson + value: 44.50394399762147 + - type: euclidean_spearman + value: 50.959184495072876 + - type: manhattan_pearson + value: 45.83191034038624 + - type: manhattan_spearman + value: 50.190409866117946 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (pl) + config: pl + split: test + metrics: + - type: cos_sim_pearson + value: 3.620381732096531 + - type: cos_sim_spearman + value: 23.30843951799194 + - type: euclidean_pearson + value: 0.965453312113125 + - type: euclidean_spearman + value: 24.235967620790316 + - type: manhattan_pearson + value: 1.4408922275701606 + - type: manhattan_spearman + value: 25.161920137046096 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (tr) + config: tr + split: test + metrics: + - type: cos_sim_pearson + value: 16.69489628726267 + - type: cos_sim_spearman + value: 34.66348380997687 + - type: euclidean_pearson + value: 29.415825529188606 + - type: euclidean_spearman + value: 38.33011033170646 + - type: manhattan_pearson + value: 31.23273195263394 + - type: manhattan_spearman + value: 39.10055785755795 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (ar) + config: ar + split: test + metrics: + - type: cos_sim_pearson + value: 9.134927430889528 + - type: cos_sim_spearman + value: 28.18922448944151 + - type: euclidean_pearson + value: 19.86814169549051 + - type: euclidean_spearman + value: 27.519588644948627 + - type: manhattan_pearson + value: 21.80949221238945 + - type: manhattan_spearman + value: 28.25217200494078 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (ru) + config: ru + split: test + metrics: + - type: cos_sim_pearson + value: 3.6386482942352085 + - type: cos_sim_spearman + value: 9.068119621940966 + - type: euclidean_pearson + value: 0.8123129118737714 + - type: euclidean_spearman + value: 9.173672890166147 + - type: manhattan_pearson + value: 0.754518899822658 + - type: manhattan_spearman + value: 8.431719541986524 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (zh) + config: zh + split: test + metrics: + - type: cos_sim_pearson + value: 2.972091574908432 + - type: cos_sim_spearman + value: 25.48511383289232 + - type: euclidean_pearson + value: 12.751569670148918 + - type: euclidean_spearman + value: 24.940721642439286 + - type: manhattan_pearson + value: 14.310238482989826 + - type: manhattan_spearman + value: 24.69821216148647 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (fr) + config: fr + split: test + metrics: + - type: cos_sim_pearson + value: 54.4745185734135 + - type: cos_sim_spearman + value: 67.66493409568727 + - type: euclidean_pearson + value: 60.13580336797049 + - type: euclidean_spearman + value: 66.12319300814538 + - type: manhattan_pearson + value: 60.816210368708155 + - type: manhattan_spearman + value: 65.70010026716766 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (de-en) + config: de-en + split: test + metrics: + - type: cos_sim_pearson + value: 49.37865412588201 + - type: cos_sim_spearman + value: 53.07135629778897 + - type: euclidean_pearson + value: 49.29201416711091 + - type: euclidean_spearman + value: 50.54523702399645 + - type: manhattan_pearson + value: 51.265764141268534 + - type: manhattan_spearman + value: 51.979086403193605 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (es-en) + config: es-en + split: test + metrics: + - type: cos_sim_pearson + value: 44.925652392562135 + - type: cos_sim_spearman + value: 49.51253904767726 + - type: euclidean_pearson + value: 48.79346518897415 + - type: euclidean_spearman + value: 51.47957870101565 + - type: manhattan_pearson + value: 49.51314553898044 + - type: manhattan_spearman + value: 51.895207893189166 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (it) + config: it + split: test + metrics: + - type: cos_sim_pearson + value: 45.241690321111875 + - type: cos_sim_spearman + value: 48.24795739512037 + - type: euclidean_pearson + value: 49.22719494399897 + - type: euclidean_spearman + value: 49.64102442042809 + - type: manhattan_pearson + value: 49.497887732970256 + - type: manhattan_spearman + value: 49.940515338096304 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (pl-en) + config: pl-en + split: test + metrics: + - type: cos_sim_pearson + value: 36.42138324083909 + - type: cos_sim_spearman + value: 36.79867489417801 + - type: euclidean_pearson + value: 27.760612942610084 + - type: euclidean_spearman + value: 29.140966500287625 + - type: manhattan_pearson + value: 28.456674031350115 + - type: manhattan_spearman + value: 27.46356370924497 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (zh-en) + config: zh-en + split: test + metrics: + - type: cos_sim_pearson + value: 26.55350664089358 + - type: cos_sim_spearman + value: 28.681707196975008 + - type: euclidean_pearson + value: 12.613577889195138 + - type: euclidean_spearman + value: 13.589493311702933 + - type: manhattan_pearson + value: 11.640157427420958 + - type: manhattan_spearman + value: 10.345223941212415 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (es-it) + config: es-it + split: test + metrics: + - type: cos_sim_pearson + value: 38.54682179114309 + - type: cos_sim_spearman + value: 45.782560880405704 + - type: euclidean_pearson + value: 46.496857002368486 + - type: euclidean_spearman + value: 48.21270426410012 + - type: manhattan_pearson + value: 46.871839119374044 + - type: manhattan_spearman + value: 47.556987773851525 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (de-fr) + config: de-fr + split: test + metrics: + - type: cos_sim_pearson + value: 35.12956772546032 + - type: cos_sim_spearman + value: 32.96920218281008 + - type: euclidean_pearson + value: 34.23140384382136 + - type: euclidean_spearman + value: 32.19303153191447 + - type: manhattan_pearson + value: 34.189468276600635 + - type: manhattan_spearman + value: 34.887065709732376 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (de-pl) + config: de-pl + split: test + metrics: + - type: cos_sim_pearson + value: 30.507667380509634 + - type: cos_sim_spearman + value: 20.447284723752716 + - type: euclidean_pearson + value: 29.662041381794474 + - type: euclidean_spearman + value: 20.939990379746757 + - type: manhattan_pearson + value: 32.5112080506328 + - type: manhattan_spearman + value: 23.773047901712495 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (fr-pl) + config: fr-pl + split: test metrics: - type: cos_sim_pearson - value: 66.7257987615171 + value: 71.10820459712156 - type: cos_sim_spearman - value: 67.30387805090024 + value: 61.97797868009122 - type: euclidean_pearson - value: 69.46877227885867 + value: 60.30910689156633 - type: euclidean_spearman - value: 69.33161798704344 + value: 61.97797868009122 - type: manhattan_pearson - value: 69.82773311626424 + value: 66.3405176964038 - type: manhattan_spearman - value: 69.57199940498796 + value: 61.97797868009122 - task: type: STS dataset: type: mteb/stsbenchmark-sts name: MTEB STSBenchmark + config: default + split: test metrics: - type: cos_sim_pearson - value: 79.37322139418472 + value: 76.53032504460737 - type: cos_sim_spearman - value: 77.5887175717799 + value: 75.33716094627373 - type: euclidean_pearson - value: 78.23006410562164 + value: 69.64662673290599 - type: euclidean_spearman - value: 77.18470385673044 + value: 67.30188896368857 - type: manhattan_pearson - value: 78.40868369362455 + value: 69.45096082050807 - type: manhattan_spearman - value: 77.36675823897656 + value: 67.0718727259371 - task: type: Reranking dataset: type: mteb/scidocs-reranking name: MTEB SciDocsRR + config: default + split: test metrics: - type: map - value: 77.21233007730808 + value: 71.33941904192648 - type: mrr - value: 93.0502386139641 + value: 89.73766429648782 - task: type: Retrieval dataset: type: scifact name: MTEB SciFact + config: default + split: test metrics: - type: map_at_1 - value: 54.567 + value: 43.333 - type: map_at_10 - value: 63.653000000000006 + value: 52.364 - type: map_at_100 - value: 64.282 + value: 53.184 - type: map_at_1000 - value: 64.31099999999999 + value: 53.234 - type: map_at_3 - value: 60.478 + value: 49.832 - type: map_at_5 - value: 62.322 + value: 51.244 - type: mrr_at_1 - value: 56.99999999999999 + value: 45.333 - type: mrr_at_10 - value: 64.759 + value: 53.455 - type: mrr_at_100 - value: 65.274 + value: 54.191 - type: mrr_at_1000 - value: 65.301 + value: 54.235 - type: mrr_at_3 - value: 62.333000000000006 + value: 51.556000000000004 - type: mrr_at_5 - value: 63.817 + value: 52.622 - type: ndcg_at_1 - value: 56.99999999999999 + value: 45.333 - type: ndcg_at_10 - value: 68.28699999999999 + value: 56.899 - type: ndcg_at_100 - value: 70.98400000000001 + value: 60.702 - type: ndcg_at_1000 - value: 71.695 + value: 62.046 - type: ndcg_at_3 - value: 62.656 + value: 52.451 - type: ndcg_at_5 - value: 65.523 + value: 54.534000000000006 - type: precision_at_1 - value: 56.99999999999999 + value: 45.333 - type: precision_at_10 - value: 9.232999999999999 + value: 7.8 - type: precision_at_100 - value: 1.0630000000000002 + value: 0.987 - type: precision_at_1000 - value: 0.11199999999999999 + value: 0.11 - type: precision_at_3 - value: 24.221999999999998 + value: 20.778 - type: precision_at_5 - value: 16.333000000000002 + value: 13.866999999999999 - type: recall_at_1 - value: 54.567 + value: 43.333 - type: recall_at_10 - value: 81.45599999999999 + value: 69.69999999999999 - type: recall_at_100 - value: 93.5 + value: 86.9 - type: recall_at_1000 - value: 99.0 + value: 97.6 - type: recall_at_3 - value: 66.228 + value: 57.81699999999999 - type: recall_at_5 - value: 73.489 + value: 62.827999999999996 - task: type: PairClassification dataset: type: mteb/sprintduplicatequestions-pairclassification name: MTEB SprintDuplicateQuestions + config: default + split: test metrics: - type: cos_sim_accuracy - value: 99.74455445544554 + value: 99.7 - type: cos_sim_ap - value: 92.57836032673468 + value: 89.88577913120001 - type: cos_sim_f1 - value: 87.0471464019851 + value: 84.62694041061593 - type: cos_sim_precision - value: 86.4039408866995 + value: 84.7542627883651 - type: cos_sim_recall - value: 87.7 + value: 84.5 - type: dot_accuracy - value: 99.56039603960396 + value: 99.24752475247524 - type: dot_ap - value: 82.47233353407186 + value: 56.81855467290009 - type: dot_f1 - value: 76.78207739307537 + value: 56.084126189283936 - type: dot_precision - value: 78.21576763485477 + value: 56.16850551654965 - type: dot_recall - value: 75.4 + value: 56.00000000000001 - type: euclidean_accuracy - value: 99.73069306930694 + value: 99.7059405940594 - type: euclidean_ap - value: 91.70507666665775 + value: 90.12451226491524 - type: euclidean_f1 - value: 86.26262626262626 + value: 84.44211629125196 - type: euclidean_precision - value: 87.14285714285714 + value: 88.66886688668868 - type: euclidean_recall - value: 85.39999999999999 + value: 80.60000000000001 - type: manhattan_accuracy - value: 99.73861386138614 + value: 99.7128712871287 - type: manhattan_ap - value: 91.96809459281754 + value: 90.67590584183216 - type: manhattan_f1 - value: 86.6 + value: 84.85436893203884 - type: manhattan_precision - value: 86.6 + value: 82.45283018867924 - type: manhattan_recall - value: 86.6 + value: 87.4 - type: max_accuracy - value: 99.74455445544554 + value: 99.7128712871287 - type: max_ap - value: 92.57836032673468 + value: 90.67590584183216 - type: max_f1 - value: 87.0471464019851 + value: 84.85436893203884 - task: type: Clustering dataset: type: mteb/stackexchange-clustering name: MTEB StackExchangeClustering + config: default + split: test metrics: - type: v_measure - value: 60.85593925770172 + value: 52.74481093815175 - task: type: Clustering dataset: type: mteb/stackexchange-clustering-p2p name: MTEB StackExchangeClusteringP2P + config: default + split: test metrics: - type: v_measure - value: 32.356772998237496 + value: 32.65999453562101 - task: type: Reranking dataset: type: mteb/stackoverflowdupquestions-reranking name: MTEB StackOverflowDupQuestions + config: default + split: test metrics: - type: map - value: 49.320607035290735 + value: 44.74498464555465 - type: mrr - value: 50.09196481622952 + value: 45.333879764026825 - task: type: Summarization dataset: type: mteb/summeval name: MTEB SummEval + config: default + split: test metrics: - type: cos_sim_pearson - value: 25.57602918901377 + value: 29.5961822471627 - type: cos_sim_spearman - value: 25.440272876996694 + value: 28.901450309119646 - type: dot_pearson - value: 24.909680980895065 + value: 29.174743399629012 - type: dot_spearman - value: 24.032627570006824 + value: 27.362975970813956 - task: type: Retrieval dataset: type: trec-covid name: MTEB TRECCOVID + config: default + split: test metrics: - type: map_at_1 - value: 0.22100000000000003 + value: 0.241 - type: map_at_10 - value: 1.7229999999999999 + value: 1.672 - type: map_at_100 - value: 9.195 + value: 7.858999999999999 - type: map_at_1000 - value: 21.999 + value: 17.616 - type: map_at_3 - value: 0.6479999999999999 + value: 0.631 - type: map_at_5 - value: 0.964 + value: 0.968 - type: mrr_at_1 - value: 86.0 + value: 90.0 - type: mrr_at_10 - value: 90.667 + value: 92.952 - type: mrr_at_100 - value: 90.858 + value: 93.036 - type: mrr_at_1000 - value: 90.858 + value: 93.036 - type: mrr_at_3 - value: 90.667 + value: 92.667 - type: mrr_at_5 - value: 90.667 + value: 92.667 - type: ndcg_at_1 - value: 82.0 + value: 83.0 - type: ndcg_at_10 - value: 72.98 + value: 70.30199999999999 - type: ndcg_at_100 - value: 52.868 + value: 48.149 - type: ndcg_at_1000 - value: 46.541 + value: 40.709 - type: ndcg_at_3 - value: 80.39699999999999 + value: 79.173 - type: ndcg_at_5 - value: 76.303 + value: 75.347 - type: precision_at_1 - value: 86.0 + value: 90.0 - type: precision_at_10 - value: 75.8 + value: 72.6 - type: precision_at_100 - value: 53.5 + value: 48.46 - type: precision_at_1000 - value: 20.946 + value: 18.093999999999998 - type: precision_at_3 - value: 85.333 + value: 84.0 - type: precision_at_5 - value: 79.2 + value: 78.8 - type: recall_at_1 - value: 0.22100000000000003 + value: 0.241 - type: recall_at_10 - value: 1.9109999999999998 + value: 1.814 - type: recall_at_100 - value: 12.437 + value: 11.141 - type: recall_at_1000 - value: 43.606 + value: 37.708999999999996 - type: recall_at_3 - value: 0.681 + value: 0.647 - type: recall_at_5 - value: 1.023 + value: 1.015 - task: type: Retrieval dataset: type: webis-touche2020 name: MTEB Touche2020 + config: default + split: test metrics: - type: map_at_1 - value: 2.5 + value: 2.782 - type: map_at_10 - value: 9.568999999999999 + value: 9.06 - type: map_at_100 - value: 15.653 + value: 14.571000000000002 - type: map_at_1000 - value: 17.188 + value: 16.006999999999998 - type: map_at_3 - value: 5.335999999999999 + value: 5.037 - type: map_at_5 - value: 6.522 + value: 6.63 - type: mrr_at_1 value: 34.694 - type: mrr_at_10 - value: 49.184 + value: 48.243 - type: mrr_at_100 - value: 50.512 + value: 49.065 - type: mrr_at_1000 - value: 50.512 + value: 49.065 - type: mrr_at_3 - value: 46.259 + value: 44.897999999999996 - type: mrr_at_5 - value: 48.299 + value: 46.428999999999995 - type: ndcg_at_1 - value: 30.612000000000002 + value: 31.633 - type: ndcg_at_10 - value: 24.45 + value: 22.972 - type: ndcg_at_100 - value: 35.870999999999995 + value: 34.777 - type: ndcg_at_1000 - value: 47.272999999999996 + value: 45.639 - type: ndcg_at_3 - value: 28.528 + value: 26.398 - type: ndcg_at_5 - value: 25.768 + value: 24.418 - type: precision_at_1 value: 34.694 - type: precision_at_10 - value: 21.429000000000002 + value: 19.796 - type: precision_at_100 - value: 7.265000000000001 + value: 7.224 - type: precision_at_1000 - value: 1.504 + value: 1.4449999999999998 - type: precision_at_3 - value: 29.252 + value: 26.531 - type: precision_at_5 - value: 24.898 + value: 23.265 - type: recall_at_1 - value: 2.5 + value: 2.782 - type: recall_at_10 - value: 15.844 + value: 14.841 - type: recall_at_100 - value: 45.469 + value: 44.86 - type: recall_at_1000 - value: 81.148 + value: 78.227 - type: recall_at_3 - value: 6.496 + value: 5.959 - type: recall_at_5 - value: 8.790000000000001 + value: 8.969000000000001 - task: type: Classification dataset: type: mteb/toxic_conversations_50k name: MTEB ToxicConversationsClassification + config: default + split: test metrics: - type: accuracy - value: 68.7272 + value: 62.657999999999994 - type: ap - value: 13.156450706152686 + value: 10.96353161716344 - type: f1 - value: 52.814703437064395 + value: 48.294226423442645 - task: type: Classification dataset: type: mteb/tweet_sentiment_extraction name: MTEB TweetSentimentExtractionClassification + config: default + split: test metrics: - type: accuracy - value: 55.6677985285795 + value: 52.40803621958121 - type: f1 - value: 55.9373937514999 + value: 52.61009636022186 - task: type: Clustering dataset: type: mteb/twentynewsgroups-clustering name: MTEB TwentyNewsgroupsClustering + config: default + split: test metrics: - type: v_measure - value: 40.05809562275603 + value: 32.12697126747911 - task: type: PairClassification dataset: type: mteb/twittersemeval2015-pairclassification name: MTEB TwitterSemEval2015 + config: default + split: test metrics: - type: cos_sim_accuracy - value: 82.76807534124099 + value: 80.69976753889253 - type: cos_sim_ap - value: 62.37052608803734 + value: 54.74680676121268 - type: cos_sim_f1 - value: 59.077414934916646 + value: 53.18923998590391 - type: cos_sim_precision - value: 52.07326892109501 + value: 47.93563413084904 - type: cos_sim_recall - value: 68.25857519788919 + value: 59.73614775725594 - type: dot_accuracy - value: 80.56267509089825 + value: 79.3348036001669 - type: dot_ap - value: 54.75349561321037 + value: 48.46902128933627 - type: dot_f1 - value: 54.75483794372552 + value: 50.480109739369006 - type: dot_precision - value: 49.77336499028707 + value: 42.06084051345173 - type: dot_recall - value: 60.844327176781 + value: 63.113456464379944 - type: euclidean_accuracy - value: 82.476008821601 + value: 79.78780473266973 - type: euclidean_ap - value: 61.17417554210511 + value: 50.258327255164815 - type: euclidean_f1 - value: 57.80318696022382 + value: 49.655838666827684 - type: euclidean_precision - value: 53.622207176709544 + value: 45.78044978846582 - type: euclidean_recall - value: 62.69129287598945 + value: 54.24802110817942 - type: manhattan_accuracy - value: 82.48792990403528 + value: 79.76992310901831 - type: manhattan_ap - value: 61.044816292966544 + value: 49.89892485714363 - type: manhattan_f1 - value: 58.03033951360462 + value: 49.330433787341185 - type: manhattan_precision - value: 53.36581045172719 + value: 43.56175459874672 - type: manhattan_recall - value: 63.58839050131926 + value: 56.86015831134564 - type: max_accuracy - value: 82.76807534124099 + value: 80.69976753889253 - type: max_ap - value: 62.37052608803734 + value: 54.74680676121268 - type: max_f1 - value: 59.077414934916646 + value: 53.18923998590391 - task: type: PairClassification dataset: type: mteb/twitterurlcorpus-pairclassification name: MTEB TwitterURLCorpus + config: default + split: test metrics: - type: cos_sim_accuracy - value: 87.97881010594946 + value: 86.90573213800597 - type: cos_sim_ap - value: 83.78748636891035 + value: 81.05760818661524 - type: cos_sim_f1 - value: 75.94113995691386 + value: 73.64688856729379 - type: cos_sim_precision - value: 72.22029307590805 + value: 69.46491946491946 - type: cos_sim_recall - value: 80.06621496766245 + value: 78.3646442870342 - type: dot_accuracy - value: 85.69294058291614 + value: 83.80680715644041 - type: dot_ap - value: 78.15363722278026 + value: 72.49774005947461 - type: dot_f1 - value: 72.08894926888564 + value: 68.68460650173216 - type: dot_precision - value: 67.28959487419075 + value: 62.954647507858105 - type: dot_recall - value: 77.62550046196489 + value: 75.56205728364644 - type: euclidean_accuracy - value: 87.73625179493149 + value: 85.97430822369697 - type: euclidean_ap - value: 83.19012184470559 + value: 78.86101740829326 - type: euclidean_f1 - value: 75.5148064623461 + value: 71.07960824663695 - type: euclidean_precision - value: 72.63352535381551 + value: 70.36897306270279 - type: euclidean_recall - value: 78.6341238065907 + value: 71.8047428395442 - type: manhattan_accuracy - value: 87.74013272790779 + value: 85.94132029339853 - type: manhattan_ap - value: 83.23305405113403 + value: 78.77876711171923 - type: manhattan_f1 - value: 75.63960775639607 + value: 71.07869075515912 - type: manhattan_precision - value: 72.563304569246 + value: 69.80697847067557 - type: manhattan_recall - value: 78.9882968894364 + value: 72.39759778256852 - type: max_accuracy - value: 87.97881010594946 + value: 86.90573213800597 - type: max_ap - value: 83.78748636891035 + value: 81.05760818661524 - type: max_f1 - value: 75.94113995691386 + value: 73.64688856729379 --- # SGPT-1.3B-weightedmean-msmarco-specb-bitfit