--- base_model: - Alibaba-NLP/gte-Qwen2-7B-instruct language: - en - zh license: apache-2.0 model-index: - name: infly/inf-retriever-v1 results: - dataset: config: default name: MTEB CmedqaRetrieval (default) revision: cd540c506dae1cf9e9a59c3e06f42030d54e7301 split: dev type: C-MTEB/CmedqaRetrieval metrics: - type: ndcg_at_1 value: 38.185 - type: ndcg_at_3 value: 38.438 - type: ndcg_at_5 value: 40.445 - type: ndcg_at_10 value: 43.308 - type: ndcg_at_20 value: 46.177 - type: ndcg_at_100 value: 50.644999999999996 - type: ndcg_at_1000 value: 52.819 - type: recall_at_1 value: 25.14 - type: recall_at_3 value: 38.253 - type: recall_at_5 value: 44.507999999999996 - type: recall_at_10 value: 53.025 - type: recall_at_20 value: 62.89 - type: recall_at_100 value: 83.487 - type: recall_at_1000 value: 98.059 - type: main_score value: 43.308 task: type: Retrieval - dataset: config: default name: MTEB CovidRetrieval (default) revision: 1271c7809071a13532e05f25fb53511ffce77117 split: dev type: C-MTEB/CovidRetrieval metrics: - type: ndcg_at_1 value: 77.97699999999999 - type: ndcg_at_3 value: 85.24199999999999 - type: ndcg_at_5 value: 86.901 - type: ndcg_at_10 value: 87.77000000000001 - type: ndcg_at_20 value: 88.295 - type: ndcg_at_100 value: 88.479 - type: ndcg_at_1000 value: 88.527 - type: recall_at_1 value: 77.819 - type: recall_at_3 value: 89.96300000000001 - type: recall_at_5 value: 93.941 - type: recall_at_10 value: 96.575 - type: recall_at_20 value: 98.63 - type: recall_at_100 value: 99.579 - type: recall_at_1000 value: 100.0 - type: main_score value: 87.77000000000001 task: type: Retrieval - dataset: config: default name: MTEB DuRetrieval (default) revision: a1a333e290fe30b10f3f56498e3a0d911a693ced split: dev type: C-MTEB/DuRetrieval metrics: - type: ndcg_at_1 value: 91.45 - type: ndcg_at_3 value: 89.249 - type: ndcg_at_5 value: 88.506 - type: ndcg_at_10 value: 90.66 - type: ndcg_at_20 value: 91.886 - type: ndcg_at_100 value: 92.78699999999999 - type: ndcg_at_1000 value: 92.944 - type: recall_at_1 value: 27.332 - type: recall_at_3 value: 61.07599999999999 - type: recall_at_5 value: 78.49199999999999 - type: recall_at_10 value: 92.002 - type: recall_at_20 value: 96.116 - type: recall_at_100 value: 99.009 - type: recall_at_1000 value: 99.844 - type: main_score value: 90.66 task: type: Retrieval - dataset: config: default name: MTEB EcomRetrieval (default) revision: 687de13dc7294d6fd9be10c6945f9e8fec8166b9 split: dev type: C-MTEB/EcomRetrieval metrics: - type: ndcg_at_1 value: 55.900000000000006 - type: ndcg_at_3 value: 66.019 - type: ndcg_at_5 value: 68.47999999999999 - type: ndcg_at_10 value: 70.678 - type: ndcg_at_20 value: 72.024 - type: ndcg_at_100 value: 72.933 - type: ndcg_at_1000 value: 73.20400000000001 - type: recall_at_1 value: 55.900000000000006 - type: recall_at_3 value: 73.1 - type: recall_at_5 value: 79.10000000000001 - type: recall_at_10 value: 85.9 - type: recall_at_20 value: 91.2 - type: recall_at_100 value: 96.1 - type: recall_at_1000 value: 98.3 - type: main_score value: 70.678 task: type: Retrieval - dataset: config: default name: MTEB MMarcoRetrieval (default) revision: 539bbde593d947e2a124ba72651aafc09eb33fc2 split: dev type: C-MTEB/MMarcoRetrieval metrics: - type: ndcg_at_1 value: 75.74499999999999 - type: ndcg_at_3 value: 82.188 - type: ndcg_at_5 value: 83.869 - type: ndcg_at_10 value: 85.119 - type: ndcg_at_20 value: 85.624 - type: ndcg_at_100 value: 86.051 - type: ndcg_at_1000 value: 86.177 - type: recall_at_1 value: 73.33 - type: recall_at_3 value: 86.823 - type: recall_at_5 value: 90.814 - type: recall_at_10 value: 94.509 - type: recall_at_20 value: 96.422 - type: recall_at_100 value: 98.6 - type: recall_at_1000 value: 99.599 - type: main_score value: 85.119 task: type: Retrieval - dataset: config: default name: MTEB MedicalRetrieval (default) revision: 2039188fb5800a9803ba5048df7b76e6fb151fc6 split: dev type: C-MTEB/MedicalRetrieval metrics: - type: ndcg_at_1 value: 55.00000000000001 - type: ndcg_at_3 value: 61.334 - type: ndcg_at_5 value: 62.590999999999994 - type: ndcg_at_10 value: 63.913 - type: ndcg_at_20 value: 64.748 - type: ndcg_at_100 value: 66.675 - type: ndcg_at_1000 value: 67.894 - type: recall_at_1 value: 55.00000000000001 - type: recall_at_3 value: 65.60000000000001 - type: recall_at_5 value: 68.60000000000001 - type: recall_at_10 value: 72.7 - type: recall_at_20 value: 76.0 - type: recall_at_100 value: 86.6 - type: recall_at_1000 value: 96.3 - type: main_score value: 63.913 task: type: Retrieval - dataset: config: default name: MTEB T2Retrieval (default) revision: 8731a845f1bf500a4f111cf1070785c793d10e64 split: dev type: C-MTEB/T2Retrieval metrics: - type: ndcg_at_1 value: 91.526 - type: ndcg_at_3 value: 88.35499999999999 - type: ndcg_at_5 value: 87.408 - type: ndcg_at_10 value: 87.641 - type: ndcg_at_20 value: 89.265 - type: ndcg_at_100 value: 90.693 - type: ndcg_at_1000 value: 91.105 - type: recall_at_1 value: 28.359 - type: recall_at_3 value: 58.101 - type: recall_at_5 value: 72.99 - type: recall_at_10 value: 86.921 - type: recall_at_20 value: 92.497 - type: recall_at_100 value: 96.978 - type: recall_at_1000 value: 99.075 - type: main_score value: 87.641 task: type: Retrieval - dataset: config: default name: MTEB VideoRetrieval (default) revision: 58c2597a5943a2ba48f4668c3b90d796283c5639 split: dev type: C-MTEB/VideoRetrieval metrics: - type: ndcg_at_1 value: 66.0 - type: ndcg_at_3 value: 75.495 - type: ndcg_at_5 value: 77.027 - type: ndcg_at_10 value: 78.606 - type: ndcg_at_20 value: 79.54599999999999 - type: ndcg_at_100 value: 80.326 - type: ndcg_at_1000 value: 80.516 - type: recall_at_1 value: 66.0 - type: recall_at_3 value: 81.89999999999999 - type: recall_at_5 value: 85.6 - type: recall_at_10 value: 90.4 - type: recall_at_20 value: 94.1 - type: recall_at_100 value: 98.2 - type: recall_at_1000 value: 99.7 - type: main_score value: 78.606 task: type: Retrieval - dataset: config: default name: MTEB AILACasedocs (default) revision: 4106e6bcc72e0698d714ea8b101355e3e238431a split: test type: mteb/AILA_casedocs metrics: - type: ndcg_at_1 value: 40.0 - type: ndcg_at_3 value: 37.37 - type: ndcg_at_5 value: 37.913999999999994 - type: ndcg_at_10 value: 41.162 - type: ndcg_at_20 value: 45.72 - type: ndcg_at_100 value: 54.126 - type: ndcg_at_1000 value: 55.907 - type: recall_at_1 value: 15.406 - type: recall_at_3 value: 26.56 - type: recall_at_5 value: 33.084 - type: recall_at_10 value: 45.972 - type: recall_at_20 value: 60.775 - type: recall_at_100 value: 91.105 - type: recall_at_1000 value: 100.0 - type: main_score value: 41.162 task: type: Retrieval - dataset: config: default name: MTEB AILAStatutes (default) revision: ebfcd844eadd3d667efa3c57fc5c8c87f5c2867e split: test type: mteb/AILA_statutes metrics: - type: ndcg_at_1 value: 36.0 - type: ndcg_at_3 value: 32.427 - type: ndcg_at_5 value: 31.512 - type: ndcg_at_10 value: 37.727 - type: ndcg_at_20 value: 43.808 - type: ndcg_at_100 value: 56.445 - type: ndcg_at_1000 value: 56.445 - type: recall_at_1 value: 8.1 - type: recall_at_3 value: 20.599999999999998 - type: recall_at_5 value: 30.733 - type: recall_at_10 value: 42.733 - type: recall_at_20 value: 57.733000000000004 - type: recall_at_100 value: 100.0 - type: recall_at_1000 value: 100.0 - type: main_score value: 37.727 task: type: Retrieval - dataset: config: default name: MTEB AlloprofRetrieval (default) revision: fcf295ea64c750f41fadbaa37b9b861558e1bfbd split: test type: lyon-nlp/alloprof metrics: - type: ndcg_at_1 value: 45.509 - type: ndcg_at_3 value: 57.912 - type: ndcg_at_5 value: 60.885 - type: ndcg_at_10 value: 63.611 - type: ndcg_at_20 value: 64.976 - type: ndcg_at_100 value: 66.507 - type: ndcg_at_1000 value: 66.998 - type: recall_at_1 value: 45.509 - type: recall_at_3 value: 66.537 - type: recall_at_5 value: 73.748 - type: recall_at_10 value: 82.16799999999999 - type: recall_at_20 value: 87.522 - type: recall_at_100 value: 95.72500000000001 - type: recall_at_1000 value: 99.655 - type: main_score value: 63.611 task: type: Retrieval - dataset: config: default name: MTEB AppsRetrieval (default) revision: f22508f96b7a36c2415181ed8bb76f76e04ae2d5 split: test type: CoIR-Retrieval/apps metrics: - type: ndcg_at_1 value: 35.405 - type: ndcg_at_3 value: 42.945 - type: ndcg_at_5 value: 44.984 - type: ndcg_at_10 value: 47.369 - type: ndcg_at_20 value: 49.095 - type: ndcg_at_100 value: 51.821 - type: ndcg_at_1000 value: 53.581 - type: recall_at_1 value: 35.405 - type: recall_at_3 value: 48.287 - type: recall_at_5 value: 53.227000000000004 - type: recall_at_10 value: 60.611000000000004 - type: recall_at_20 value: 67.437 - type: recall_at_100 value: 82.231 - type: recall_at_1000 value: 96.38799999999999 - type: main_score value: 47.369 task: type: Retrieval - dataset: config: default name: MTEB ArguAna (default) revision: c22ab2a51041ffd869aaddef7af8d8215647e41a split: test type: mteb/arguana metrics: - type: ndcg_at_1 value: 69.132 - type: ndcg_at_3 value: 81.661 - type: ndcg_at_5 value: 83.773 - type: ndcg_at_10 value: 84.855 - type: ndcg_at_20 value: 85.073 - type: ndcg_at_100 value: 85.134 - type: ndcg_at_1000 value: 85.134 - type: recall_at_1 value: 69.132 - type: recall_at_3 value: 90.185 - type: recall_at_5 value: 95.235 - type: recall_at_10 value: 98.506 - type: recall_at_20 value: 99.36 - type: recall_at_100 value: 99.644 - type: recall_at_1000 value: 99.644 - type: main_score value: 84.855 task: type: Retrieval - dataset: config: default name: MTEB ArguAna-PL (default) revision: 63fc86750af76253e8c760fc9e534bbf24d260a2 split: test type: clarin-knext/arguana-pl metrics: - type: ndcg_at_1 value: 46.657 - type: ndcg_at_3 value: 63.388999999999996 - type: ndcg_at_5 value: 67.931 - type: ndcg_at_10 value: 70.745 - type: ndcg_at_20 value: 71.60300000000001 - type: ndcg_at_100 value: 71.941 - type: ndcg_at_1000 value: 71.961 - type: recall_at_1 value: 46.657 - type: recall_at_3 value: 75.036 - type: recall_at_5 value: 85.989 - type: recall_at_10 value: 94.523 - type: recall_at_20 value: 97.795 - type: recall_at_100 value: 99.502 - type: recall_at_1000 value: 99.644 - type: main_score value: 70.745 task: type: Retrieval - dataset: config: default name: MTEB CQADupstackAndroidRetrieval (default) revision: f46a197baaae43b4f621051089b82a364682dfeb split: test type: mteb/cqadupstack-android metrics: - type: ndcg_at_1 value: 45.494 - type: ndcg_at_3 value: 51.53 - type: ndcg_at_5 value: 54.062 - type: ndcg_at_10 value: 56.599 - type: ndcg_at_20 value: 58.663 - type: ndcg_at_100 value: 61.36200000000001 - type: ndcg_at_1000 value: 62.824000000000005 - type: recall_at_1 value: 37.078 - type: recall_at_3 value: 53.529 - type: recall_at_5 value: 60.772999999999996 - type: recall_at_10 value: 68.65299999999999 - type: recall_at_20 value: 75.92999999999999 - type: recall_at_100 value: 88.127 - type: recall_at_1000 value: 97.059 - type: main_score value: 56.599 task: type: Retrieval - dataset: config: default name: MTEB CQADupstackEnglishRetrieval (default) revision: ad9991cb51e31e31e430383c75ffb2885547b5f0 split: test type: mteb/cqadupstack-english metrics: - type: ndcg_at_1 value: 47.134 - type: ndcg_at_3 value: 52.186 - type: ndcg_at_5 value: 53.94 - type: ndcg_at_10 value: 55.96 - type: ndcg_at_20 value: 57.521 - type: ndcg_at_100 value: 59.865 - type: ndcg_at_1000 value: 61.611000000000004 - type: recall_at_1 value: 37.405 - type: recall_at_3 value: 53.869 - type: recall_at_5 value: 59.18600000000001 - type: recall_at_10 value: 65.786 - type: recall_at_20 value: 71.56099999999999 - type: recall_at_100 value: 82.062 - type: recall_at_1000 value: 92.863 - type: main_score value: 55.96 task: type: Retrieval - dataset: config: default name: MTEB CQADupstackGamingRetrieval (default) revision: 4885aa143210c98657558c04aaf3dc47cfb54340 split: test type: mteb/cqadupstack-gaming metrics: - type: ndcg_at_1 value: 52.22599999999999 - type: ndcg_at_3 value: 59.797999999999995 - type: ndcg_at_5 value: 62.260000000000005 - type: ndcg_at_10 value: 64.85300000000001 - type: ndcg_at_20 value: 66.398 - type: ndcg_at_100 value: 68.298 - type: ndcg_at_1000 value: 69.003 - type: recall_at_1 value: 45.789 - type: recall_at_3 value: 64.9 - type: recall_at_5 value: 70.902 - type: recall_at_10 value: 78.388 - type: recall_at_20 value: 84.086 - type: recall_at_100 value: 93.006 - type: recall_at_1000 value: 97.928 - type: main_score value: 64.85300000000001 task: type: Retrieval - dataset: config: default name: MTEB CQADupstackGisRetrieval (default) revision: 5003b3064772da1887988e05400cf3806fe491f2 split: test type: mteb/cqadupstack-gis metrics: - type: ndcg_at_1 value: 32.09 - type: ndcg_at_3 value: 38.339 - type: ndcg_at_5 value: 41.427 - type: ndcg_at_10 value: 43.606 - type: ndcg_at_20 value: 45.784000000000006 - type: ndcg_at_100 value: 48.908 - type: ndcg_at_1000 value: 50.585 - type: recall_at_1 value: 29.146 - type: recall_at_3 value: 43.168 - type: recall_at_5 value: 50.717 - type: recall_at_10 value: 57.120000000000005 - type: recall_at_20 value: 65.254 - type: recall_at_100 value: 81.04599999999999 - type: recall_at_1000 value: 93.487 - type: main_score value: 43.606 task: type: Retrieval - dataset: config: default name: MTEB CQADupstackMathematicaRetrieval (default) revision: 90fceea13679c63fe563ded68f3b6f06e50061de split: test type: mteb/cqadupstack-mathematica metrics: - type: ndcg_at_1 value: 24.876 - type: ndcg_at_3 value: 29.663 - type: ndcg_at_5 value: 32.193 - type: ndcg_at_10 value: 34.694 - type: ndcg_at_20 value: 37.075 - type: ndcg_at_100 value: 40.615 - type: ndcg_at_1000 value: 43.317 - type: recall_at_1 value: 20.395 - type: recall_at_3 value: 32.521 - type: recall_at_5 value: 38.887 - type: recall_at_10 value: 46.388 - type: recall_at_20 value: 54.885 - type: recall_at_100 value: 71.597 - type: recall_at_1000 value: 90.75 - type: main_score value: 34.694 task: type: Retrieval - dataset: config: default name: MTEB CQADupstackPhysicsRetrieval (default) revision: 79531abbd1fb92d06c6d6315a0cbbbf5bb247ea4 split: test type: mteb/cqadupstack-physics metrics: - type: ndcg_at_1 value: 42.733 - type: ndcg_at_3 value: 47.236 - type: ndcg_at_5 value: 49.327 - type: ndcg_at_10 value: 52.346000000000004 - type: ndcg_at_20 value: 54.446000000000005 - type: ndcg_at_100 value: 57.736 - type: ndcg_at_1000 value: 59.245000000000005 - type: recall_at_1 value: 34.414 - type: recall_at_3 value: 50.233000000000004 - type: recall_at_5 value: 55.967 - type: recall_at_10 value: 65.173 - type: recall_at_20 value: 72.27799999999999 - type: recall_at_100 value: 87.163 - type: recall_at_1000 value: 96.64 - type: main_score value: 52.346000000000004 task: type: Retrieval - dataset: config: default name: MTEB CQADupstackProgrammersRetrieval (default) revision: 6184bc1440d2dbc7612be22b50686b8826d22b32 split: test type: mteb/cqadupstack-programmers metrics: - type: ndcg_at_1 value: 37.329 - type: ndcg_at_3 value: 41.319 - type: ndcg_at_5 value: 43.444 - type: ndcg_at_10 value: 46.643 - type: ndcg_at_20 value: 49.257 - type: ndcg_at_100 value: 52.524 - type: ndcg_at_1000 value: 54.478 - type: recall_at_1 value: 30.278 - type: recall_at_3 value: 43.464999999999996 - type: recall_at_5 value: 49.419999999999995 - type: recall_at_10 value: 58.650999999999996 - type: recall_at_20 value: 67.90899999999999 - type: recall_at_100 value: 83.276 - type: recall_at_1000 value: 96.114 - type: main_score value: 46.643 task: type: Retrieval - dataset: config: default name: MTEB CQADupstackRetrieval (default) revision: CQADupstackRetrieval_is_a_combined_dataset split: test type: CQADupstackRetrieval_is_a_combined_dataset metrics: - type: main_score value: 46.644083333333334 - type: ndcg_at_10 value: 46.644083333333334 task: type: Retrieval - dataset: config: default name: MTEB CQADupstackStatsRetrieval (default) revision: 65ac3a16b8e91f9cee4c9828cc7c335575432a2a split: test type: mteb/cqadupstack-stats metrics: - type: ndcg_at_1 value: 30.368000000000002 - type: ndcg_at_3 value: 35.004000000000005 - type: ndcg_at_5 value: 37.125 - type: ndcg_at_10 value: 39.831 - type: ndcg_at_20 value: 42.099 - type: ndcg_at_100 value: 45.032 - type: ndcg_at_1000 value: 47.016999999999996 - type: recall_at_1 value: 27.151999999999997 - type: recall_at_3 value: 38.2 - type: recall_at_5 value: 43.349 - type: recall_at_10 value: 51.50599999999999 - type: recall_at_20 value: 60.035000000000004 - type: recall_at_100 value: 74.869 - type: recall_at_1000 value: 89.159 - type: main_score value: 39.831 task: type: Retrieval - dataset: config: default name: MTEB CQADupstackTexRetrieval (default) revision: 46989137a86843e03a6195de44b09deda022eec7 split: test type: mteb/cqadupstack-tex metrics: - type: ndcg_at_1 value: 26.222 - type: ndcg_at_3 value: 30.085 - type: ndcg_at_5 value: 31.977 - type: ndcg_at_10 value: 34.107 - type: ndcg_at_20 value: 35.939 - type: ndcg_at_100 value: 39.054 - type: ndcg_at_1000 value: 41.899 - type: recall_at_1 value: 21.552 - type: recall_at_3 value: 32.66 - type: recall_at_5 value: 37.785000000000004 - type: recall_at_10 value: 44.143 - type: recall_at_20 value: 50.968999999999994 - type: recall_at_100 value: 66.392 - type: recall_at_1000 value: 86.601 - type: main_score value: 34.107 task: type: Retrieval - dataset: config: default name: MTEB CQADupstackUnixRetrieval (default) revision: 6c6430d3a6d36f8d2a829195bc5dc94d7e063e53 split: test type: mteb/cqadupstack-unix metrics: - type: ndcg_at_1 value: 36.287000000000006 - type: ndcg_at_3 value: 41.15 - type: ndcg_at_5 value: 43.283 - type: ndcg_at_10 value: 45.698 - type: ndcg_at_20 value: 47.754000000000005 - type: ndcg_at_100 value: 50.800999999999995 - type: ndcg_at_1000 value: 53.024 - type: recall_at_1 value: 30.791 - type: recall_at_3 value: 44.802 - type: recall_at_5 value: 50.434999999999995 - type: recall_at_10 value: 57.424 - type: recall_at_20 value: 64.702 - type: recall_at_100 value: 79.216 - type: recall_at_1000 value: 94.602 - type: main_score value: 45.698 task: type: Retrieval - dataset: config: default name: MTEB CQADupstackWebmastersRetrieval (default) revision: 160c094312a0e1facb97e55eeddb698c0abe3571 split: test type: mteb/cqadupstack-webmasters metrics: - type: ndcg_at_1 value: 37.352000000000004 - type: ndcg_at_3 value: 43.029 - type: ndcg_at_5 value: 44.811 - type: ndcg_at_10 value: 47.493 - type: ndcg_at_20 value: 49.76 - type: ndcg_at_100 value: 52.925 - type: ndcg_at_1000 value: 55.117000000000004 - type: recall_at_1 value: 31.719 - type: recall_at_3 value: 45.466 - type: recall_at_5 value: 50.087 - type: recall_at_10 value: 57.86 - type: recall_at_20 value: 66.27 - type: recall_at_100 value: 81.437 - type: recall_at_1000 value: 95.162 - type: main_score value: 47.493 task: type: Retrieval - dataset: config: default name: MTEB CQADupstackWordpressRetrieval (default) revision: 4ffe81d471b1924886b33c7567bfb200e9eec5c4 split: test type: mteb/cqadupstack-wordpress metrics: - type: ndcg_at_1 value: 29.020000000000003 - type: ndcg_at_3 value: 33.715 - type: ndcg_at_5 value: 35.266 - type: ndcg_at_10 value: 37.899 - type: ndcg_at_20 value: 39.812999999999995 - type: ndcg_at_100 value: 42.998999999999995 - type: ndcg_at_1000 value: 45.257 - type: recall_at_1 value: 26.784000000000002 - type: recall_at_3 value: 37.049 - type: recall_at_5 value: 40.638000000000005 - type: recall_at_10 value: 48.204 - type: recall_at_20 value: 55.496 - type: recall_at_100 value: 71.749 - type: recall_at_1000 value: 88.22 - type: main_score value: 37.899 task: type: Retrieval - dataset: config: default name: MTEB CodeFeedbackMT (default) revision: b0f12fa0c0dd67f59c95a5c33d02aeeb4c398c5f split: test type: CoIR-Retrieval/codefeedback-mt metrics: - type: ndcg_at_1 value: 67.214 - type: ndcg_at_3 value: 74.774 - type: ndcg_at_5 value: 76.297 - type: ndcg_at_10 value: 77.644 - type: ndcg_at_20 value: 78.41 - type: ndcg_at_100 value: 79.374 - type: ndcg_at_1000 value: 79.77 - type: recall_at_1 value: 67.214 - type: recall_at_3 value: 79.95 - type: recall_at_5 value: 83.65599999999999 - type: recall_at_10 value: 87.776 - type: recall_at_20 value: 90.781 - type: recall_at_100 value: 95.993 - type: recall_at_1000 value: 99.104 - type: main_score value: 77.644 task: type: Retrieval - dataset: config: default name: MTEB CodeFeedbackST (default) revision: d213819e87aab9010628da8b73ab4eb337c89340 split: test type: CoIR-Retrieval/codefeedback-st metrics: - type: ndcg_at_1 value: 74.05000000000001 - type: ndcg_at_3 value: 84.59 - type: ndcg_at_5 value: 85.949 - type: ndcg_at_10 value: 86.627 - type: ndcg_at_20 value: 86.907 - type: ndcg_at_100 value: 87.149 - type: ndcg_at_1000 value: 87.21799999999999 - type: recall_at_1 value: 74.05000000000001 - type: recall_at_3 value: 91.685 - type: recall_at_5 value: 94.959 - type: recall_at_10 value: 97.017 - type: recall_at_20 value: 98.10900000000001 - type: recall_at_100 value: 99.396 - type: recall_at_1000 value: 99.92699999999999 - type: main_score value: 86.627 task: type: Retrieval - dataset: config: python name: MTEB CodeSearchNetCCRetrieval (python) revision: 6e1effa2c03723c5fde48ee912b5ee08d4f211e8 split: test type: CoIR-Retrieval/CodeSearchNet-ccr metrics: - type: ndcg_at_1 value: 69.875 - type: ndcg_at_3 value: 79.45100000000001 - type: ndcg_at_5 value: 80.95400000000001 - type: ndcg_at_10 value: 82.025 - type: ndcg_at_20 value: 82.526 - type: ndcg_at_100 value: 83.07 - type: ndcg_at_1000 value: 83.28999999999999 - type: recall_at_1 value: 69.875 - type: recall_at_3 value: 85.957 - type: recall_at_5 value: 89.59 - type: recall_at_10 value: 92.874 - type: recall_at_20 value: 94.838 - type: recall_at_100 value: 97.748 - type: recall_at_1000 value: 99.47 - type: main_score value: 82.025 task: type: Retrieval - dataset: config: javascript name: MTEB CodeSearchNetCCRetrieval (javascript) revision: 6e1effa2c03723c5fde48ee912b5ee08d4f211e8 split: test type: CoIR-Retrieval/CodeSearchNet-ccr metrics: - type: ndcg_at_1 value: 66.18 - type: ndcg_at_3 value: 76.294 - type: ndcg_at_5 value: 77.849 - type: ndcg_at_10 value: 78.95400000000001 - type: ndcg_at_20 value: 79.71000000000001 - type: ndcg_at_100 value: 80.402 - type: ndcg_at_1000 value: 80.694 - type: recall_at_1 value: 66.18 - type: recall_at_3 value: 83.10499999999999 - type: recall_at_5 value: 86.873 - type: recall_at_10 value: 90.277 - type: recall_at_20 value: 93.22399999999999 - type: recall_at_100 value: 96.87 - type: recall_at_1000 value: 99.21 - type: main_score value: 78.95400000000001 task: type: Retrieval - dataset: config: go name: MTEB CodeSearchNetCCRetrieval (go) revision: 6e1effa2c03723c5fde48ee912b5ee08d4f211e8 split: test type: CoIR-Retrieval/CodeSearchNet-ccr metrics: - type: ndcg_at_1 value: 57.24 - type: ndcg_at_3 value: 67.84700000000001 - type: ndcg_at_5 value: 70.126 - type: ndcg_at_10 value: 71.839 - type: ndcg_at_20 value: 72.89 - type: ndcg_at_100 value: 73.904 - type: ndcg_at_1000 value: 74.343 - type: recall_at_1 value: 57.24 - type: recall_at_3 value: 75.179 - type: recall_at_5 value: 80.67 - type: recall_at_10 value: 85.939 - type: recall_at_20 value: 90.076 - type: recall_at_100 value: 95.48100000000001 - type: recall_at_1000 value: 98.929 - type: main_score value: 71.839 task: type: Retrieval - dataset: config: ruby name: MTEB CodeSearchNetCCRetrieval (ruby) revision: 6e1effa2c03723c5fde48ee912b5ee08d4f211e8 split: test type: CoIR-Retrieval/CodeSearchNet-ccr metrics: - type: ndcg_at_1 value: 64.235 - type: ndcg_at_3 value: 73.451 - type: ndcg_at_5 value: 75.233 - type: ndcg_at_10 value: 76.53 - type: ndcg_at_20 value: 77.35 - type: ndcg_at_100 value: 78.13799999999999 - type: ndcg_at_1000 value: 78.57 - type: recall_at_1 value: 64.235 - type: recall_at_3 value: 79.699 - type: recall_at_5 value: 83.981 - type: recall_at_10 value: 88.02499999999999 - type: recall_at_20 value: 91.277 - type: recall_at_100 value: 95.638 - type: recall_at_1000 value: 99.048 - type: main_score value: 76.53 task: type: Retrieval - dataset: config: java name: MTEB CodeSearchNetCCRetrieval (java) revision: 6e1effa2c03723c5fde48ee912b5ee08d4f211e8 split: test type: CoIR-Retrieval/CodeSearchNet-ccr metrics: - type: ndcg_at_1 value: 65.468 - type: ndcg_at_3 value: 75.064 - type: ndcg_at_5 value: 76.786 - type: ndcg_at_10 value: 77.929 - type: ndcg_at_20 value: 78.596 - type: ndcg_at_100 value: 79.28699999999999 - type: ndcg_at_1000 value: 79.625 - type: recall_at_1 value: 65.468 - type: recall_at_3 value: 81.56099999999999 - type: recall_at_5 value: 85.714 - type: recall_at_10 value: 89.229 - type: recall_at_20 value: 91.83 - type: recall_at_100 value: 95.509 - type: recall_at_1000 value: 98.17399999999999 - type: main_score value: 77.929 task: type: Retrieval - dataset: config: php name: MTEB CodeSearchNetCCRetrieval (php) revision: 6e1effa2c03723c5fde48ee912b5ee08d4f211e8 split: test type: CoIR-Retrieval/CodeSearchNet-ccr metrics: - type: ndcg_at_1 value: 52.71900000000001 - type: ndcg_at_3 value: 63.025 - type: ndcg_at_5 value: 65.17399999999999 - type: ndcg_at_10 value: 66.982 - type: ndcg_at_20 value: 68.113 - type: ndcg_at_100 value: 69.443 - type: ndcg_at_1000 value: 70.111 - type: recall_at_1 value: 52.71900000000001 - type: recall_at_3 value: 70.158 - type: recall_at_5 value: 75.35300000000001 - type: recall_at_10 value: 80.919 - type: recall_at_20 value: 85.36500000000001 - type: recall_at_100 value: 92.486 - type: recall_at_1000 value: 97.788 - type: main_score value: 66.982 task: type: Retrieval - dataset: config: python name: MTEB CodeSearchNetRetrieval (python) revision: fdc6a9e39575768c27eb8a2a5f702bf846eb4759 split: test type: code-search-net/code_search_net metrics: - type: ndcg_at_1 value: 86.9 - type: ndcg_at_3 value: 92.012 - type: ndcg_at_5 value: 93.002 - type: ndcg_at_10 value: 93.304 - type: ndcg_at_20 value: 93.432 - type: ndcg_at_100 value: 93.50500000000001 - type: ndcg_at_1000 value: 93.54 - type: recall_at_1 value: 86.9 - type: recall_at_3 value: 95.5 - type: recall_at_5 value: 97.89999999999999 - type: recall_at_10 value: 98.8 - type: recall_at_20 value: 99.3 - type: recall_at_100 value: 99.7 - type: recall_at_1000 value: 100.0 - type: main_score value: 93.304 task: type: Retrieval - dataset: config: javascript name: MTEB CodeSearchNetRetrieval (javascript) revision: fdc6a9e39575768c27eb8a2a5f702bf846eb4759 split: test type: code-search-net/code_search_net metrics: - type: ndcg_at_1 value: 73.9 - type: ndcg_at_3 value: 80.297 - type: ndcg_at_5 value: 81.162 - type: ndcg_at_10 value: 82.075 - type: ndcg_at_20 value: 82.432 - type: ndcg_at_100 value: 82.948 - type: ndcg_at_1000 value: 83.722 - type: recall_at_1 value: 73.9 - type: recall_at_3 value: 84.6 - type: recall_at_5 value: 86.7 - type: recall_at_10 value: 89.5 - type: recall_at_20 value: 90.9 - type: recall_at_100 value: 93.7 - type: recall_at_1000 value: 100.0 - type: main_score value: 82.075 task: type: Retrieval - dataset: config: go name: MTEB CodeSearchNetRetrieval (go) revision: fdc6a9e39575768c27eb8a2a5f702bf846eb4759 split: test type: code-search-net/code_search_net metrics: - type: ndcg_at_1 value: 86.9 - type: ndcg_at_3 value: 92.961 - type: ndcg_at_5 value: 93.632 - type: ndcg_at_10 value: 93.865 - type: ndcg_at_20 value: 93.917 - type: ndcg_at_100 value: 93.994 - type: ndcg_at_1000 value: 94.02199999999999 - type: recall_at_1 value: 86.9 - type: recall_at_3 value: 96.89999999999999 - type: recall_at_5 value: 98.5 - type: recall_at_10 value: 99.2 - type: recall_at_20 value: 99.4 - type: recall_at_100 value: 99.8 - type: recall_at_1000 value: 100.0 - type: main_score value: 93.865 task: type: Retrieval - dataset: config: ruby name: MTEB CodeSearchNetRetrieval (ruby) revision: fdc6a9e39575768c27eb8a2a5f702bf846eb4759 split: test type: code-search-net/code_search_net metrics: - type: ndcg_at_1 value: 79.10000000000001 - type: ndcg_at_3 value: 85.626 - type: ndcg_at_5 value: 86.629 - type: ndcg_at_10 value: 87.16000000000001 - type: ndcg_at_20 value: 87.414 - type: ndcg_at_100 value: 87.7 - type: ndcg_at_1000 value: 88.115 - type: recall_at_1 value: 79.10000000000001 - type: recall_at_3 value: 89.9 - type: recall_at_5 value: 92.30000000000001 - type: recall_at_10 value: 93.89999999999999 - type: recall_at_20 value: 94.89999999999999 - type: recall_at_100 value: 96.39999999999999 - type: recall_at_1000 value: 100.0 - type: main_score value: 87.16000000000001 task: type: Retrieval - dataset: config: java name: MTEB CodeSearchNetRetrieval (java) revision: fdc6a9e39575768c27eb8a2a5f702bf846eb4759 split: test type: code-search-net/code_search_net metrics: - type: ndcg_at_1 value: 82.0 - type: ndcg_at_3 value: 89.205 - type: ndcg_at_5 value: 89.86699999999999 - type: ndcg_at_10 value: 90.269 - type: ndcg_at_20 value: 90.32 - type: ndcg_at_100 value: 90.36999999999999 - type: ndcg_at_1000 value: 90.691 - type: recall_at_1 value: 82.0 - type: recall_at_3 value: 94.0 - type: recall_at_5 value: 95.6 - type: recall_at_10 value: 96.8 - type: recall_at_20 value: 97.0 - type: recall_at_100 value: 97.3 - type: recall_at_1000 value: 100.0 - type: main_score value: 90.269 task: type: Retrieval - dataset: config: php name: MTEB CodeSearchNetRetrieval (php) revision: fdc6a9e39575768c27eb8a2a5f702bf846eb4759 split: test type: code-search-net/code_search_net metrics: - type: ndcg_at_1 value: 76.1 - type: ndcg_at_3 value: 83.97 - type: ndcg_at_5 value: 85.128 - type: ndcg_at_10 value: 85.922 - type: ndcg_at_20 value: 86.279 - type: ndcg_at_100 value: 86.53 - type: ndcg_at_1000 value: 86.846 - type: recall_at_1 value: 76.1 - type: recall_at_3 value: 89.3 - type: recall_at_5 value: 92.10000000000001 - type: recall_at_10 value: 94.5 - type: recall_at_20 value: 95.89999999999999 - type: recall_at_100 value: 97.3 - type: recall_at_1000 value: 100.0 - type: main_score value: 85.922 task: type: Retrieval - dataset: config: default name: MTEB CodeTransOceanContest (default) revision: 20da4eb20a4b17300c0986ee148c90867a7f2a4d split: test type: CoIR-Retrieval/codetrans-contest metrics: - type: ndcg_at_1 value: 82.353 - type: ndcg_at_3 value: 86.792 - type: ndcg_at_5 value: 88.116 - type: ndcg_at_10 value: 89.164 - type: ndcg_at_20 value: 89.627 - type: ndcg_at_100 value: 89.816 - type: ndcg_at_1000 value: 89.929 - type: recall_at_1 value: 82.353 - type: recall_at_3 value: 90.045 - type: recall_at_5 value: 93.21300000000001 - type: recall_at_10 value: 96.38 - type: recall_at_20 value: 98.19 - type: recall_at_100 value: 99.095 - type: recall_at_1000 value: 100.0 - type: main_score value: 89.164 task: type: Retrieval - dataset: config: default name: MTEB CodeTransOceanDL (default) revision: 281562cb8a1265ab5c0824bfa6ddcd9b0a15618f split: test type: CoIR-Retrieval/codetrans-dl metrics: - type: ndcg_at_1 value: 9.443999999999999 - type: ndcg_at_3 value: 13.141 - type: ndcg_at_5 value: 20.149 - type: ndcg_at_10 value: 35.181000000000004 - type: ndcg_at_20 value: 39.898 - type: ndcg_at_100 value: 40.337 - type: ndcg_at_1000 value: 40.337 - type: recall_at_1 value: 9.443999999999999 - type: recall_at_3 value: 16.111 - type: recall_at_5 value: 32.778 - type: recall_at_10 value: 80.55600000000001 - type: recall_at_20 value: 97.77799999999999 - type: recall_at_100 value: 100.0 - type: recall_at_1000 value: 100.0 - type: main_score value: 35.181000000000004 task: type: Retrieval - dataset: config: default name: MTEB CosQA (default) revision: bc5efb7e9d437246ce393ed19d772e08e4a79535 split: test type: CoIR-Retrieval/cosqa metrics: - type: ndcg_at_1 value: 14.2 - type: ndcg_at_3 value: 23.647000000000002 - type: ndcg_at_5 value: 28.655 - type: ndcg_at_10 value: 34.175 - type: ndcg_at_20 value: 37.04 - type: ndcg_at_100 value: 41.074 - type: ndcg_at_1000 value: 41.917 - type: recall_at_1 value: 14.2 - type: recall_at_3 value: 31.0 - type: recall_at_5 value: 43.4 - type: recall_at_10 value: 60.4 - type: recall_at_20 value: 71.8 - type: recall_at_100 value: 93.0 - type: recall_at_1000 value: 99.2 - type: main_score value: 34.175 task: type: Retrieval - dataset: config: default name: MTEB DBPedia (default) revision: c0f706b76e590d620bd6618b3ca8efdd34e2d659 split: test type: mteb/dbpedia metrics: - type: ndcg_at_1 value: 61.5 - type: ndcg_at_3 value: 53.476 - type: ndcg_at_5 value: 51.601 - type: ndcg_at_10 value: 50.391 - type: ndcg_at_20 value: 49.342000000000006 - type: ndcg_at_100 value: 55.37800000000001 - type: ndcg_at_1000 value: 62.470000000000006 - type: recall_at_1 value: 9.757 - type: recall_at_3 value: 17.203 - type: recall_at_5 value: 21.878 - type: recall_at_10 value: 30.425 - type: recall_at_20 value: 39.137 - type: recall_at_100 value: 62.885000000000005 - type: recall_at_1000 value: 85.795 - type: main_score value: 50.391 task: type: Retrieval - dataset: config: default name: MTEB FiQA-PL (default) revision: 2e535829717f8bf9dc829b7f911cc5bbd4e6608e split: test type: clarin-knext/fiqa-pl metrics: - type: ndcg_at_1 value: 46.296 - type: ndcg_at_3 value: 43.682 - type: ndcg_at_5 value: 44.818999999999996 - type: ndcg_at_10 value: 47.137 - type: ndcg_at_20 value: 49.957 - type: ndcg_at_100 value: 53.998999999999995 - type: ndcg_at_1000 value: 56.547000000000004 - type: recall_at_1 value: 23.116999999999997 - type: recall_at_3 value: 39.967000000000006 - type: recall_at_5 value: 46.745 - type: recall_at_10 value: 54.202 - type: recall_at_20 value: 62.61600000000001 - type: recall_at_100 value: 79.322 - type: recall_at_1000 value: 94.114 - type: main_score value: 47.137 task: type: Retrieval - dataset: config: default name: MTEB FiQA2018 (default) revision: 27a168819829fe9bcd655c2df245fb19452e8e06 split: test type: mteb/fiqa metrics: - type: ndcg_at_1 value: 63.117000000000004 - type: ndcg_at_3 value: 58.538999999999994 - type: ndcg_at_5 value: 59.147000000000006 - type: ndcg_at_10 value: 62.35000000000001 - type: ndcg_at_20 value: 65.36800000000001 - type: ndcg_at_100 value: 68.801 - type: ndcg_at_1000 value: 70.06599999999999 - type: recall_at_1 value: 33.377 - type: recall_at_3 value: 52.817 - type: recall_at_5 value: 59.03699999999999 - type: recall_at_10 value: 69.116 - type: recall_at_20 value: 78.30799999999999 - type: recall_at_100 value: 91.715 - type: recall_at_1000 value: 98.783 - type: main_score value: 62.35000000000001 task: type: Retrieval - dataset: config: default name: MTEB GerDaLIRSmall (default) revision: 48327de6ee192e9610f3069789719788957c7abd split: test type: mteb/GerDaLIRSmall metrics: - type: ndcg_at_1 value: 30.047 - type: ndcg_at_3 value: 36.635 - type: ndcg_at_5 value: 39.237 - type: ndcg_at_10 value: 41.752 - type: ndcg_at_20 value: 43.467 - type: ndcg_at_100 value: 45.793 - type: ndcg_at_1000 value: 47.404 - type: recall_at_1 value: 27.272999999999996 - type: recall_at_3 value: 41.534 - type: recall_at_5 value: 47.678 - type: recall_at_10 value: 55.131 - type: recall_at_20 value: 61.592 - type: recall_at_100 value: 73.604 - type: recall_at_1000 value: 86.146 - type: main_score value: 41.752 task: type: Retrieval - dataset: config: default name: MTEB HotpotQA (default) revision: ab518f4d6fcca38d87c25209f94beba119d02014 split: test type: mteb/hotpotqa metrics: - type: ndcg_at_1 value: 88.062 - type: ndcg_at_3 value: 77.443 - type: ndcg_at_5 value: 80.05600000000001 - type: ndcg_at_10 value: 81.979 - type: ndcg_at_20 value: 83.033 - type: ndcg_at_100 value: 84.232 - type: ndcg_at_1000 value: 84.827 - type: recall_at_1 value: 44.031 - type: recall_at_3 value: 75.71900000000001 - type: recall_at_5 value: 80.851 - type: recall_at_10 value: 85.652 - type: recall_at_20 value: 89.021 - type: recall_at_100 value: 94.267 - type: recall_at_1000 value: 98.136 - type: main_score value: 81.979 task: type: Retrieval - dataset: config: default name: MTEB LEMBNarrativeQARetrieval (default) revision: 6e346642246bfb4928c560ee08640dc84d074e8c split: test type: dwzhu/LongEmbed metrics: - type: ndcg_at_1 value: 49.009 - type: ndcg_at_3 value: 56.69 - type: ndcg_at_5 value: 58.572 - type: ndcg_at_10 value: 60.702 - type: ndcg_at_20 value: 62.160000000000004 - type: ndcg_at_100 value: 64.461 - type: ndcg_at_1000 value: 65.604 - type: recall_at_1 value: 49.009 - type: recall_at_3 value: 62.073 - type: recall_at_5 value: 66.648 - type: recall_at_10 value: 73.222 - type: recall_at_20 value: 78.974 - type: recall_at_100 value: 91.444 - type: recall_at_1000 value: 100.0 - type: main_score value: 60.702 task: type: Retrieval - dataset: config: default name: MTEB LEMBNeedleRetrieval (default) revision: 6e346642246bfb4928c560ee08640dc84d074e8c split: test_256 type: dwzhu/LongEmbed metrics: - type: ndcg_at_1 value: 8.0 - type: ndcg_at_3 value: 12.786 - type: ndcg_at_5 value: 15.282000000000002 - type: ndcg_at_10 value: 20.096 - type: ndcg_at_20 value: 22.631 - type: ndcg_at_100 value: 32.174 - type: ndcg_at_1000 value: 32.174 - type: recall_at_1 value: 8.0 - type: recall_at_3 value: 16.0 - type: recall_at_5 value: 22.0 - type: recall_at_10 value: 36.0 - type: recall_at_20 value: 46.0 - type: recall_at_100 value: 100.0 - type: recall_at_1000 value: 100.0 - type: main_score value: 8.0 task: type: Retrieval - dataset: config: default name: MTEB LEMBPasskeyRetrieval (default) revision: 6e346642246bfb4928c560ee08640dc84d074e8c split: test_256 type: dwzhu/LongEmbed metrics: - type: ndcg_at_1 value: 10.0 - type: ndcg_at_3 value: 12.0 - type: ndcg_at_5 value: 12.0 - type: ndcg_at_10 value: 12.631 - type: ndcg_at_20 value: 14.982000000000001 - type: ndcg_at_100 value: 28.534 - type: ndcg_at_1000 value: 28.534 - type: recall_at_1 value: 10.0 - type: recall_at_3 value: 14.000000000000002 - type: recall_at_5 value: 14.000000000000002 - type: recall_at_10 value: 16.0 - type: recall_at_20 value: 26.0 - type: recall_at_100 value: 100.0 - type: recall_at_1000 value: 100.0 - type: main_score value: 10.0 task: type: Retrieval - dataset: config: default name: MTEB LEMBQMSumRetrieval (default) revision: 6e346642246bfb4928c560ee08640dc84d074e8c split: test type: dwzhu/LongEmbed metrics: - type: ndcg_at_1 value: 37.263000000000005 - type: ndcg_at_3 value: 48.207 - type: ndcg_at_5 value: 51.464 - type: ndcg_at_10 value: 55.071999999999996 - type: ndcg_at_20 value: 57.364000000000004 - type: ndcg_at_100 value: 60.236999999999995 - type: ndcg_at_1000 value: 60.352 - type: recall_at_1 value: 37.263000000000005 - type: recall_at_3 value: 55.92700000000001 - type: recall_at_5 value: 63.851 - type: recall_at_10 value: 74.91799999999999 - type: recall_at_20 value: 83.955 - type: recall_at_100 value: 99.214 - type: recall_at_1000 value: 100.0 - type: main_score value: 55.071999999999996 task: type: Retrieval - dataset: config: default name: MTEB LEMBSummScreenFDRetrieval (default) revision: 6e346642246bfb4928c560ee08640dc84d074e8c split: validation type: dwzhu/LongEmbed metrics: - type: ndcg_at_1 value: 94.345 - type: ndcg_at_3 value: 96.66900000000001 - type: ndcg_at_5 value: 97.297 - type: ndcg_at_10 value: 97.387 - type: ndcg_at_20 value: 97.387 - type: ndcg_at_100 value: 97.387 - type: ndcg_at_1000 value: 97.387 - type: recall_at_1 value: 94.345 - type: recall_at_3 value: 98.214 - type: recall_at_5 value: 99.702 - type: recall_at_10 value: 100.0 - type: recall_at_20 value: 100.0 - type: recall_at_100 value: 100.0 - type: recall_at_1000 value: 100.0 - type: main_score value: 97.387 task: type: Retrieval - dataset: config: default name: MTEB LEMBWikimQARetrieval (default) revision: 6e346642246bfb4928c560ee08640dc84d074e8c split: test type: dwzhu/LongEmbed metrics: - type: ndcg_at_1 value: 80.0 - type: ndcg_at_3 value: 84.024 - type: ndcg_at_5 value: 84.985 - type: ndcg_at_10 value: 85.751 - type: ndcg_at_20 value: 86.634 - type: ndcg_at_100 value: 87.348 - type: ndcg_at_1000 value: 87.48599999999999 - type: recall_at_1 value: 80.0 - type: recall_at_3 value: 87.0 - type: recall_at_5 value: 89.333 - type: recall_at_10 value: 91.667 - type: recall_at_20 value: 95.0 - type: recall_at_100 value: 99.0 - type: recall_at_1000 value: 100.0 - type: main_score value: 85.751 task: type: Retrieval - dataset: config: default name: MTEB LeCaRDv2 (default) revision: b78e18688c3d012a33dc3676597c1d1b2243ce1c split: test type: mteb/LeCaRDv2 metrics: - type: ndcg_at_1 value: 87.421 - type: ndcg_at_3 value: 83.159 - type: ndcg_at_5 value: 79.818 - type: ndcg_at_10 value: 74.168 - type: ndcg_at_20 value: 67.81 - type: ndcg_at_100 value: 80.432 - type: ndcg_at_1000 value: 84.423 - type: recall_at_1 value: 4.1450000000000005 - type: recall_at_3 value: 10.988000000000001 - type: recall_at_5 value: 16.808999999999997 - type: recall_at_10 value: 29.329 - type: recall_at_20 value: 48.425000000000004 - type: recall_at_100 value: 89.63600000000001 - type: recall_at_1000 value: 99.823 - type: main_score value: 74.168 task: type: Retrieval - dataset: config: default name: MTEB LegalBenchConsumerContractsQA (default) revision: b23590301ec94e8087e2850b21d43d4956b1cca9 split: test type: mteb/legalbench_consumer_contracts_qa metrics: - type: ndcg_at_1 value: 73.485 - type: ndcg_at_3 value: 81.977 - type: ndcg_at_5 value: 84.63000000000001 - type: ndcg_at_10 value: 85.444 - type: ndcg_at_20 value: 86.008 - type: ndcg_at_100 value: 86.262 - type: ndcg_at_1000 value: 86.262 - type: recall_at_1 value: 73.485 - type: recall_at_3 value: 87.626 - type: recall_at_5 value: 93.939 - type: recall_at_10 value: 96.465 - type: recall_at_20 value: 98.737 - type: recall_at_100 value: 100.0 - type: recall_at_1000 value: 100.0 - type: main_score value: 85.444 task: type: Retrieval - dataset: config: default name: MTEB LegalBenchCorporateLobbying (default) revision: f69691c650464e62546d7f2a4536f8f87c891e38 split: test type: mteb/legalbench_corporate_lobbying metrics: - type: ndcg_at_1 value: 91.471 - type: ndcg_at_3 value: 95.84700000000001 - type: ndcg_at_5 value: 96.088 - type: ndcg_at_10 value: 96.17999999999999 - type: ndcg_at_20 value: 96.17999999999999 - type: ndcg_at_100 value: 96.17999999999999 - type: ndcg_at_1000 value: 96.259 - type: recall_at_1 value: 91.471 - type: recall_at_3 value: 98.529 - type: recall_at_5 value: 99.118 - type: recall_at_10 value: 99.412 - type: recall_at_20 value: 99.412 - type: recall_at_100 value: 99.412 - type: recall_at_1000 value: 100.0 - type: main_score value: 96.17999999999999 task: type: Retrieval - dataset: config: default name: MTEB LegalQuAD (default) revision: 37aa6cfb01d48960b0f8e3f17d6e3d99bf1ebc3e split: test type: mteb/LegalQuAD metrics: - type: ndcg_at_1 value: 48.0 - type: ndcg_at_3 value: 59.397999999999996 - type: ndcg_at_5 value: 61.05500000000001 - type: ndcg_at_10 value: 63.219 - type: ndcg_at_20 value: 65.102 - type: ndcg_at_100 value: 67.254 - type: ndcg_at_1000 value: 67.746 - type: recall_at_1 value: 48.0 - type: recall_at_3 value: 67.0 - type: recall_at_5 value: 71.0 - type: recall_at_10 value: 77.5 - type: recall_at_20 value: 85.0 - type: recall_at_100 value: 96.5 - type: recall_at_1000 value: 100.0 - type: main_score value: 63.219 task: type: Retrieval - dataset: config: default name: MTEB LegalSummarization (default) revision: 3bb1a05c66872889662af04c5691c14489cebd72 split: test type: mteb/legal_summarization metrics: - type: ndcg_at_1 value: 58.451 - type: ndcg_at_3 value: 63.70099999999999 - type: ndcg_at_5 value: 66.792 - type: ndcg_at_10 value: 69.76 - type: ndcg_at_20 value: 71.487 - type: ndcg_at_100 value: 73.6 - type: ndcg_at_1000 value: 74.05000000000001 - type: recall_at_1 value: 52.028 - type: recall_at_3 value: 66.7 - type: recall_at_5 value: 74.119 - type: recall_at_10 value: 82.595 - type: recall_at_20 value: 88.209 - type: recall_at_100 value: 97.24000000000001 - type: recall_at_1000 value: 100.0 - type: main_score value: 69.76 task: type: Retrieval - dataset: config: ar name: MTEB MintakaRetrieval (ar) revision: efa78cc2f74bbcd21eff2261f9e13aebe40b814e split: test type: jinaai/mintakaqa metrics: - type: ndcg_at_1 value: 19.791 - type: ndcg_at_3 value: 29.751 - type: ndcg_at_5 value: 32.83 - type: ndcg_at_10 value: 35.553000000000004 - type: ndcg_at_20 value: 37.528 - type: ndcg_at_100 value: 40.025 - type: ndcg_at_1000 value: 42.693 - type: recall_at_1 value: 19.791 - type: recall_at_3 value: 36.632 - type: recall_at_5 value: 44.076 - type: recall_at_10 value: 52.474 - type: recall_at_20 value: 60.281 - type: recall_at_100 value: 73.94500000000001 - type: recall_at_1000 value: 96.096 - type: main_score value: 35.553000000000004 task: type: Retrieval - dataset: config: de name: MTEB MintakaRetrieval (de) revision: efa78cc2f74bbcd21eff2261f9e13aebe40b814e split: test type: jinaai/mintakaqa metrics: - type: ndcg_at_1 value: 34.288000000000004 - type: ndcg_at_3 value: 47.29 - type: ndcg_at_5 value: 50.622 - type: ndcg_at_10 value: 53.291999999999994 - type: ndcg_at_20 value: 55.062999999999995 - type: ndcg_at_100 value: 56.987 - type: ndcg_at_1000 value: 58.084 - type: recall_at_1 value: 34.288000000000004 - type: recall_at_3 value: 56.486999999999995 - type: recall_at_5 value: 64.532 - type: recall_at_10 value: 72.746 - type: recall_at_20 value: 79.697 - type: recall_at_100 value: 90.185 - type: recall_at_1000 value: 98.989 - type: main_score value: 53.291999999999994 task: type: Retrieval - dataset: config: es name: MTEB MintakaRetrieval (es) revision: efa78cc2f74bbcd21eff2261f9e13aebe40b814e split: test type: jinaai/mintakaqa metrics: - type: ndcg_at_1 value: 31.889 - type: ndcg_at_3 value: 45.182 - type: ndcg_at_5 value: 48.475 - type: ndcg_at_10 value: 51.402 - type: ndcg_at_20 value: 53.089 - type: ndcg_at_100 value: 55.116 - type: ndcg_at_1000 value: 56.333999999999996 - type: recall_at_1 value: 31.889 - type: recall_at_3 value: 54.455 - type: recall_at_5 value: 62.417 - type: recall_at_10 value: 71.328 - type: recall_at_20 value: 77.97 - type: recall_at_100 value: 88.944 - type: recall_at_1000 value: 98.639 - type: main_score value: 51.402 task: type: Retrieval - dataset: config: fr name: MTEB MintakaRetrieval (fr) revision: efa78cc2f74bbcd21eff2261f9e13aebe40b814e split: test type: jinaai/mintakaqa metrics: - type: ndcg_at_1 value: 32.555 - type: ndcg_at_3 value: 45.278 - type: ndcg_at_5 value: 48.559000000000005 - type: ndcg_at_10 value: 51.485 - type: ndcg_at_20 value: 53.263000000000005 - type: ndcg_at_100 value: 55.221 - type: ndcg_at_1000 value: 56.501999999999995 - type: recall_at_1 value: 32.555 - type: recall_at_3 value: 54.054 - type: recall_at_5 value: 62.039 - type: recall_at_10 value: 70.966 - type: recall_at_20 value: 77.969 - type: recall_at_100 value: 88.411 - type: recall_at_1000 value: 98.69 - type: main_score value: 51.485 task: type: Retrieval - dataset: config: hi name: MTEB MintakaRetrieval (hi) revision: efa78cc2f74bbcd21eff2261f9e13aebe40b814e split: test type: jinaai/mintakaqa metrics: - type: ndcg_at_1 value: 24.757 - type: ndcg_at_3 value: 35.427 - type: ndcg_at_5 value: 38.431 - type: ndcg_at_10 value: 41.459 - type: ndcg_at_20 value: 44.137 - type: ndcg_at_100 value: 47.174 - type: ndcg_at_1000 value: 48.907000000000004 - type: recall_at_1 value: 24.757 - type: recall_at_3 value: 43.082 - type: recall_at_5 value: 50.336999999999996 - type: recall_at_10 value: 59.611000000000004 - type: recall_at_20 value: 70.157 - type: recall_at_100 value: 86.387 - type: recall_at_1000 value: 100.0 - type: main_score value: 41.459 task: type: Retrieval - dataset: config: it name: MTEB MintakaRetrieval (it) revision: efa78cc2f74bbcd21eff2261f9e13aebe40b814e split: test type: jinaai/mintakaqa metrics: - type: ndcg_at_1 value: 32.818000000000005 - type: ndcg_at_3 value: 46.503 - type: ndcg_at_5 value: 49.68 - type: ndcg_at_10 value: 52.510999999999996 - type: ndcg_at_20 value: 54.269999999999996 - type: ndcg_at_100 value: 56.17100000000001 - type: ndcg_at_1000 value: 57.38100000000001 - type: recall_at_1 value: 32.818000000000005 - type: recall_at_3 value: 56.033 - type: recall_at_5 value: 63.715999999999994 - type: recall_at_10 value: 72.48400000000001 - type: recall_at_20 value: 79.374 - type: recall_at_100 value: 89.436 - type: recall_at_1000 value: 98.914 - type: main_score value: 52.510999999999996 task: type: Retrieval - dataset: config: ja name: MTEB MintakaRetrieval (ja) revision: efa78cc2f74bbcd21eff2261f9e13aebe40b814e split: test type: jinaai/mintakaqa metrics: - type: ndcg_at_1 value: 25.216 - type: ndcg_at_3 value: 35.982 - type: ndcg_at_5 value: 38.694 - type: ndcg_at_10 value: 41.585 - type: ndcg_at_20 value: 43.334 - type: ndcg_at_100 value: 45.831 - type: ndcg_at_1000 value: 48.06 - type: recall_at_1 value: 25.216 - type: recall_at_3 value: 43.599 - type: recall_at_5 value: 50.173 - type: recall_at_10 value: 59.083 - type: recall_at_20 value: 65.96 - type: recall_at_100 value: 79.542 - type: recall_at_1000 value: 97.794 - type: main_score value: 41.585 task: type: Retrieval - dataset: config: pt name: MTEB MintakaRetrieval (pt) revision: efa78cc2f74bbcd21eff2261f9e13aebe40b814e split: test type: jinaai/mintakaqa metrics: - type: ndcg_at_1 value: 33.517 - type: ndcg_at_3 value: 46.955999999999996 - type: ndcg_at_5 value: 50.441 - type: ndcg_at_10 value: 53.256 - type: ndcg_at_20 value: 55.086 - type: ndcg_at_100 value: 57.104 - type: ndcg_at_1000 value: 58.07600000000001 - type: recall_at_1 value: 33.517 - type: recall_at_3 value: 56.245 - type: recall_at_5 value: 64.63499999999999 - type: recall_at_10 value: 73.258 - type: recall_at_20 value: 80.47999999999999 - type: recall_at_100 value: 91.27 - type: recall_at_1000 value: 99.10799999999999 - type: main_score value: 53.256 task: type: Retrieval - dataset: config: default name: MTEB NFCorpus (default) revision: ec0fa4fe99da2ff19ca1214b7966684033a58814 split: test type: mteb/nfcorpus metrics: - type: ndcg_at_1 value: 51.702999999999996 - type: ndcg_at_3 value: 48.064 - type: ndcg_at_5 value: 46.379 - type: ndcg_at_10 value: 43.663999999999994 - type: ndcg_at_20 value: 41.407 - type: ndcg_at_100 value: 42.083 - type: ndcg_at_1000 value: 52.335 - type: recall_at_1 value: 6.241 - type: recall_at_3 value: 12.214 - type: recall_at_5 value: 16.473 - type: recall_at_10 value: 21.84 - type: recall_at_20 value: 27.474999999999998 - type: recall_at_100 value: 45.01 - type: recall_at_1000 value: 80.71300000000001 - type: main_score value: 43.663999999999994 task: type: Retrieval - dataset: config: default name: MTEB NFCorpus-PL (default) revision: 9a6f9567fda928260afed2de480d79c98bf0bec0 split: test type: clarin-knext/nfcorpus-pl metrics: - type: ndcg_at_1 value: 41.641 - type: ndcg_at_3 value: 37.617 - type: ndcg_at_5 value: 36.024 - type: ndcg_at_10 value: 33.51 - type: ndcg_at_20 value: 31.575999999999997 - type: ndcg_at_100 value: 31.601000000000003 - type: ndcg_at_1000 value: 41.099000000000004 - type: recall_at_1 value: 4.61 - type: recall_at_3 value: 9.366 - type: recall_at_5 value: 11.793 - type: recall_at_10 value: 16.255 - type: recall_at_20 value: 20.713 - type: recall_at_100 value: 33.396 - type: recall_at_1000 value: 65.532 - type: main_score value: 33.51 task: type: Retrieval - dataset: config: default name: MTEB NQ (default) revision: b774495ed302d8c44a3a7ea25c90dbce03968f31 split: test type: mteb/nq metrics: - type: ndcg_at_1 value: 50.753 - type: ndcg_at_3 value: 62.541000000000004 - type: ndcg_at_5 value: 66.46600000000001 - type: ndcg_at_10 value: 69.65400000000001 - type: ndcg_at_20 value: 70.91499999999999 - type: ndcg_at_100 value: 71.908 - type: ndcg_at_1000 value: 72.08200000000001 - type: recall_at_1 value: 45.293 - type: recall_at_3 value: 71.089 - type: recall_at_5 value: 79.93 - type: recall_at_10 value: 89.01599999999999 - type: recall_at_20 value: 93.60300000000001 - type: recall_at_100 value: 98.501 - type: recall_at_1000 value: 99.768 - type: main_score value: 69.65400000000001 task: type: Retrieval - dataset: config: default name: MTEB NQ-PL (default) revision: f171245712cf85dd4700b06bef18001578d0ca8d split: test type: clarin-knext/nq-pl metrics: - type: ndcg_at_1 value: 34.791 - type: ndcg_at_3 value: 45.418 - type: ndcg_at_5 value: 49.486000000000004 - type: ndcg_at_10 value: 53.141000000000005 - type: ndcg_at_20 value: 55.230999999999995 - type: ndcg_at_100 value: 57.358 - type: ndcg_at_1000 value: 58.166 - type: recall_at_1 value: 31.04 - type: recall_at_3 value: 53.179 - type: recall_at_5 value: 62.539 - type: recall_at_10 value: 73.08099999999999 - type: recall_at_20 value: 80.83500000000001 - type: recall_at_100 value: 91.503 - type: recall_at_1000 value: 97.429 - type: main_score value: 53.141000000000005 task: type: Retrieval - dataset: config: default name: MTEB Quora-PL (default) revision: 0be27e93455051e531182b85e85e425aba12e9d4 split: validation type: clarin-knext/quora-pl metrics: - type: ndcg_at_1 value: 76.99000000000001 - type: ndcg_at_3 value: 81.781 - type: ndcg_at_5 value: 83.627 - type: ndcg_at_10 value: 85.146 - type: ndcg_at_20 value: 86.015 - type: ndcg_at_100 value: 86.745 - type: ndcg_at_1000 value: 86.882 - type: recall_at_1 value: 66.806 - type: recall_at_3 value: 84.09400000000001 - type: recall_at_5 value: 89.09899999999999 - type: recall_at_10 value: 93.512 - type: recall_at_20 value: 96.365 - type: recall_at_100 value: 99.22 - type: recall_at_1000 value: 99.937 - type: main_score value: 85.146 task: type: Retrieval - dataset: config: default name: MTEB QuoraRetrieval (default) revision: e4e08e0b7dbe3c8700f0daef558ff32256715259 split: test type: mteb/quora metrics: - type: ndcg_at_1 value: 83.66 - type: ndcg_at_3 value: 87.863 - type: ndcg_at_5 value: 89.279 - type: ndcg_at_10 value: 90.372 - type: ndcg_at_20 value: 90.955 - type: ndcg_at_100 value: 91.352 - type: ndcg_at_1000 value: 91.39500000000001 - type: recall_at_1 value: 72.75399999999999 - type: recall_at_3 value: 89.41799999999999 - type: recall_at_5 value: 93.509 - type: recall_at_10 value: 96.679 - type: recall_at_20 value: 98.519 - type: recall_at_100 value: 99.845 - type: recall_at_1000 value: 99.998 - type: main_score value: 90.372 task: type: Retrieval - dataset: config: default name: MTEB RiaNewsRetrieval (default) revision: 82374b0bbacda6114f39ff9c5b925fa1512ca5d7 split: test type: ai-forever/ria-news-retrieval metrics: - type: ndcg_at_1 value: 75.41 - type: ndcg_at_3 value: 83.13000000000001 - type: ndcg_at_5 value: 84.313 - type: ndcg_at_10 value: 85.009 - type: ndcg_at_20 value: 85.436 - type: ndcg_at_100 value: 85.875 - type: ndcg_at_1000 value: 86.048 - type: recall_at_1 value: 75.41 - type: recall_at_3 value: 88.38000000000001 - type: recall_at_5 value: 91.23 - type: recall_at_10 value: 93.34 - type: recall_at_20 value: 95.02000000000001 - type: recall_at_100 value: 97.37 - type: recall_at_1000 value: 98.78 - type: main_score value: 85.009 task: type: Retrieval - dataset: config: default name: MTEB RuBQRetrieval (default) revision: e19b6ffa60b3bc248e0b41f4cc37c26a55c2a67b split: test type: ai-forever/rubq-retrieval metrics: - type: ndcg_at_1 value: 63.652 - type: ndcg_at_3 value: 67.829 - type: ndcg_at_5 value: 72.141 - type: ndcg_at_10 value: 75.551 - type: ndcg_at_20 value: 76.925 - type: ndcg_at_100 value: 77.813 - type: ndcg_at_1000 value: 77.994 - type: recall_at_1 value: 45.09 - type: recall_at_3 value: 71.562 - type: recall_at_5 value: 81.474 - type: recall_at_10 value: 90.237 - type: recall_at_20 value: 94.679 - type: recall_at_100 value: 98.752 - type: recall_at_1000 value: 99.83999999999999 - type: main_score value: 75.551 task: type: Retrieval - dataset: config: default name: MTEB SCIDOCS (default) revision: f8c2fcf00f625baaa80f62ec5bd9e1fff3b8ae88 split: test type: mteb/scidocs metrics: - type: ndcg_at_1 value: 33.7 - type: ndcg_at_3 value: 28.360999999999997 - type: ndcg_at_5 value: 25.259999999999998 - type: ndcg_at_10 value: 30.775999999999996 - type: ndcg_at_20 value: 34.782000000000004 - type: ndcg_at_100 value: 41.753 - type: ndcg_at_1000 value: 46.887 - type: recall_at_1 value: 6.843000000000001 - type: recall_at_3 value: 16.228 - type: recall_at_5 value: 22.828 - type: recall_at_10 value: 33.007 - type: recall_at_20 value: 42.433 - type: recall_at_100 value: 64.967 - type: recall_at_1000 value: 89.587 - type: main_score value: 30.775999999999996 task: type: Retrieval - dataset: config: default name: MTEB SCIDOCS-PL (default) revision: 45452b03f05560207ef19149545f168e596c9337 split: test type: clarin-knext/scidocs-pl metrics: - type: ndcg_at_1 value: 26.5 - type: ndcg_at_3 value: 21.079 - type: ndcg_at_5 value: 18.63 - type: ndcg_at_10 value: 22.483 - type: ndcg_at_20 value: 25.552999999999997 - type: ndcg_at_100 value: 31.572 - type: ndcg_at_1000 value: 37.147000000000006 - type: recall_at_1 value: 5.367999999999999 - type: recall_at_3 value: 11.907 - type: recall_at_5 value: 16.631999999999998 - type: recall_at_10 value: 23.647000000000002 - type: recall_at_20 value: 30.857 - type: recall_at_100 value: 50.236999999999995 - type: recall_at_1000 value: 77.445 - type: main_score value: 22.483 task: type: Retrieval - dataset: config: default name: MTEB SciFact (default) revision: 0228b52cf27578f30900b9e5271d331663a030d7 split: test type: mteb/scifact metrics: - type: ndcg_at_1 value: 74.333 - type: ndcg_at_3 value: 82.071 - type: ndcg_at_5 value: 83.83800000000001 - type: ndcg_at_10 value: 85.399 - type: ndcg_at_20 value: 85.57900000000001 - type: ndcg_at_100 value: 86.075 - type: ndcg_at_1000 value: 86.164 - type: recall_at_1 value: 70.994 - type: recall_at_3 value: 87.417 - type: recall_at_5 value: 91.89399999999999 - type: recall_at_10 value: 96.167 - type: recall_at_20 value: 96.833 - type: recall_at_100 value: 99.333 - type: recall_at_1000 value: 100.0 - type: main_score value: 85.399 task: type: Retrieval - dataset: config: default name: MTEB SciFact-PL (default) revision: 47932a35f045ef8ed01ba82bf9ff67f6e109207e split: test type: clarin-knext/scifact-pl metrics: - type: ndcg_at_1 value: 65.333 - type: ndcg_at_3 value: 73.291 - type: ndcg_at_5 value: 75.149 - type: ndcg_at_10 value: 77.633 - type: ndcg_at_20 value: 78.236 - type: ndcg_at_100 value: 79.182 - type: ndcg_at_1000 value: 79.431 - type: recall_at_1 value: 61.99400000000001 - type: recall_at_3 value: 79.01700000000001 - type: recall_at_5 value: 83.72800000000001 - type: recall_at_10 value: 90.72200000000001 - type: recall_at_20 value: 93.0 - type: recall_at_100 value: 98.0 - type: recall_at_1000 value: 100.0 - type: main_score value: 77.633 task: type: Retrieval - dataset: config: default name: MTEB StackOverflowQA (default) revision: db8f169f3894c14a00251061f957b2063eef2bd5 split: test type: CoIR-Retrieval/stackoverflow-qa metrics: - type: ndcg_at_1 value: 90.07 - type: ndcg_at_3 value: 93.30199999999999 - type: ndcg_at_5 value: 93.812 - type: ndcg_at_10 value: 94.219 - type: ndcg_at_20 value: 94.46799999999999 - type: ndcg_at_100 value: 94.581 - type: ndcg_at_1000 value: 94.626 - type: recall_at_1 value: 90.07 - type: recall_at_3 value: 95.537 - type: recall_at_5 value: 96.78999999999999 - type: recall_at_10 value: 98.044 - type: recall_at_20 value: 99.047 - type: recall_at_100 value: 99.649 - type: recall_at_1000 value: 100.0 - type: main_score value: 94.219 task: type: Retrieval - dataset: config: default name: MTEB SyntecRetrieval (default) revision: 19661ccdca4dfc2d15122d776b61685f48c68ca9 split: test type: lyon-nlp/mteb-fr-retrieval-syntec-s2p metrics: - type: ndcg_at_1 value: 83.0 - type: ndcg_at_3 value: 90.809 - type: ndcg_at_5 value: 91.583 - type: ndcg_at_10 value: 92.199 - type: ndcg_at_20 value: 92.199 - type: ndcg_at_100 value: 92.199 - type: ndcg_at_1000 value: 92.199 - type: recall_at_1 value: 83.0 - type: recall_at_3 value: 96.0 - type: recall_at_5 value: 98.0 - type: recall_at_10 value: 100.0 - type: recall_at_20 value: 100.0 - type: recall_at_100 value: 100.0 - type: recall_at_1000 value: 100.0 - type: main_score value: 92.199 task: type: Retrieval - dataset: config: default name: MTEB SyntheticText2SQL (default) revision: 686b87296c3a0191b5d9415a00526c62db9fce09 split: test type: CoIR-Retrieval/synthetic-text2sql metrics: - type: ndcg_at_1 value: 20.526 - type: ndcg_at_3 value: 60.12 - type: ndcg_at_5 value: 62.134 - type: ndcg_at_10 value: 63.50599999999999 - type: ndcg_at_20 value: 64.167 - type: ndcg_at_100 value: 64.687 - type: ndcg_at_1000 value: 64.801 - type: recall_at_1 value: 20.526 - type: recall_at_3 value: 84.721 - type: recall_at_5 value: 89.574 - type: recall_at_10 value: 93.762 - type: recall_at_20 value: 96.36 - type: recall_at_100 value: 99.09400000000001 - type: recall_at_1000 value: 99.966 - type: main_score value: 63.50599999999999 task: type: Retrieval - dataset: config: default name: MTEB TRECCOVID (default) revision: bb9466bac8153a0349341eb1b22e06409e78ef4e split: test type: mteb/trec-covid metrics: - type: ndcg_at_1 value: 76.0 - type: ndcg_at_3 value: 78.899 - type: ndcg_at_5 value: 78.212 - type: ndcg_at_10 value: 75.09700000000001 - type: ndcg_at_20 value: 72.158 - type: ndcg_at_100 value: 58.465999999999994 - type: ndcg_at_1000 value: 53.702000000000005 - type: recall_at_1 value: 0.231 - type: recall_at_3 value: 0.7000000000000001 - type: recall_at_5 value: 1.146 - type: recall_at_10 value: 2.174 - type: recall_at_20 value: 4.031 - type: recall_at_100 value: 14.713999999999999 - type: recall_at_1000 value: 50.8 - type: main_score value: 75.09700000000001 task: type: Retrieval - dataset: config: default name: MTEB TRECCOVID-PL (default) revision: 81bcb408f33366c2a20ac54adafad1ae7e877fdd split: test type: clarin-knext/trec-covid-pl metrics: - type: ndcg_at_1 value: 75.0 - type: ndcg_at_3 value: 75.531 - type: ndcg_at_5 value: 75.327 - type: ndcg_at_10 value: 74.28 - type: ndcg_at_20 value: 71.5 - type: ndcg_at_100 value: 58.412 - type: ndcg_at_1000 value: 52.580000000000005 - type: recall_at_1 value: 0.214 - type: recall_at_3 value: 0.647 - type: recall_at_5 value: 1.083 - type: recall_at_10 value: 2.141 - type: recall_at_20 value: 3.9309999999999996 - type: recall_at_100 value: 14.738999999999999 - type: recall_at_1000 value: 49.494 - type: main_score value: 74.28 task: type: Retrieval - dataset: config: default name: MTEB Touche2020 (default) revision: a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f split: test type: mteb/touche2020 metrics: - type: ndcg_at_1 value: 20.408 - type: ndcg_at_3 value: 23.368 - type: ndcg_at_5 value: 24.795 - type: ndcg_at_10 value: 24.442 - type: ndcg_at_20 value: 26.712000000000003 - type: ndcg_at_100 value: 38.218999999999994 - type: ndcg_at_1000 value: 50.395 - type: recall_at_1 value: 2.414 - type: recall_at_3 value: 6.3549999999999995 - type: recall_at_5 value: 9.888 - type: recall_at_10 value: 16.31 - type: recall_at_20 value: 25.369000000000003 - type: recall_at_100 value: 51.449999999999996 - type: recall_at_1000 value: 88.532 - type: main_score value: 24.442 task: type: Retrieval - dataset: config: default name: MTEB ARCChallenge (default) revision: c481e0da3dcbbad8bce7721dea9085b74320a0a3 split: test type: RAR-b/ARC-Challenge metrics: - type: ndcg_at_1 value: 8.959 - type: ndcg_at_3 value: 16.238 - type: ndcg_at_5 value: 18.841 - type: ndcg_at_10 value: 21.606 - type: ndcg_at_20 value: 24.326 - type: ndcg_at_100 value: 28.410999999999998 - type: ndcg_at_1000 value: 31.279 - type: recall_at_1 value: 8.959 - type: recall_at_3 value: 21.416 - type: recall_at_5 value: 27.73 - type: recall_at_10 value: 36.348 - type: recall_at_20 value: 47.184 - type: recall_at_100 value: 69.539 - type: recall_at_1000 value: 92.747 - type: main_score value: 21.606 task: type: Retrieval - dataset: config: default name: MTEB AlphaNLI (default) revision: 303f40ef3d50918d3dc43577d33f2f7344ad72c1 split: test type: RAR-b/alphanli metrics: - type: ndcg_at_1 value: 29.047 - type: ndcg_at_3 value: 37.782 - type: ndcg_at_5 value: 39.989999999999995 - type: ndcg_at_10 value: 41.926 - type: ndcg_at_20 value: 43.573 - type: ndcg_at_100 value: 45.957 - type: ndcg_at_1000 value: 47.799 - type: recall_at_1 value: 29.047 - type: recall_at_3 value: 43.799 - type: recall_at_5 value: 49.151 - type: recall_at_10 value: 55.222 - type: recall_at_20 value: 61.748999999999995 - type: recall_at_100 value: 74.543 - type: recall_at_1000 value: 89.491 - type: main_score value: 41.926 task: type: Retrieval - dataset: config: default name: MTEB BSARDRetrieval (default) revision: 5effa1b9b5fa3b0f9e12523e6e43e5f86a6e6d59 split: test type: maastrichtlawtech/bsard metrics: - type: ndcg_at_1 value: 15.315000000000001 - type: ndcg_at_3 value: 22.742 - type: ndcg_at_5 value: 25.146 - type: ndcg_at_10 value: 28.993000000000002 - type: ndcg_at_20 value: 30.797 - type: ndcg_at_100 value: 34.189 - type: ndcg_at_1000 value: 36.507 - type: recall_at_1 value: 15.315000000000001 - type: recall_at_3 value: 27.927999999999997 - type: recall_at_5 value: 33.784 - type: recall_at_10 value: 45.495000000000005 - type: recall_at_20 value: 52.703 - type: recall_at_100 value: 71.622 - type: recall_at_1000 value: 90.54100000000001 - type: main_score value: 71.622 task: type: Retrieval - dataset: config: default name: MTEB ClimateFEVER (default) revision: 47f2ac6acb640fc46020b02a5b59fdda04d39380 split: test type: mteb/climate-fever metrics: - type: ndcg_at_1 value: 38.111 - type: ndcg_at_3 value: 34.489999999999995 - type: ndcg_at_5 value: 36.986999999999995 - type: ndcg_at_10 value: 41.825 - type: ndcg_at_20 value: 45.326 - type: ndcg_at_100 value: 50.207 - type: ndcg_at_1000 value: 52.686 - type: recall_at_1 value: 16.898 - type: recall_at_3 value: 31.636999999999997 - type: recall_at_5 value: 39.147 - type: recall_at_10 value: 49.787 - type: recall_at_20 value: 59.41499999999999 - type: recall_at_100 value: 77.506 - type: recall_at_1000 value: 90.803 - type: main_score value: 41.825 task: type: Retrieval - dataset: config: default name: MTEB DBPedia-PL (default) revision: 76afe41d9af165cc40999fcaa92312b8b012064a split: test type: clarin-knext/dbpedia-pl metrics: - type: ndcg_at_1 value: 50.875 - type: ndcg_at_3 value: 43.745 - type: ndcg_at_5 value: 42.186 - type: ndcg_at_10 value: 40.506 - type: ndcg_at_20 value: 40.372 - type: ndcg_at_100 value: 45.967 - type: ndcg_at_1000 value: 53.247 - type: recall_at_1 value: 8.14 - type: recall_at_3 value: 14.038 - type: recall_at_5 value: 18.394 - type: recall_at_10 value: 24.476 - type: recall_at_20 value: 32.141999999999996 - type: recall_at_100 value: 53.027 - type: recall_at_1000 value: 76.108 - type: main_score value: 40.506 task: type: Retrieval - dataset: config: default name: MTEB FEVER (default) revision: bea83ef9e8fb933d90a2f1d5515737465d613e12 split: test type: mteb/fever metrics: - type: ndcg_at_1 value: 91.899 - type: ndcg_at_3 value: 93.267 - type: ndcg_at_5 value: 93.757 - type: ndcg_at_10 value: 94.146 - type: ndcg_at_20 value: 94.42399999999999 - type: ndcg_at_100 value: 94.647 - type: ndcg_at_1000 value: 94.765 - type: recall_at_1 value: 85.329 - type: recall_at_3 value: 94.89 - type: recall_at_5 value: 96.185 - type: recall_at_10 value: 97.234 - type: recall_at_20 value: 98.059 - type: recall_at_100 value: 98.946 - type: recall_at_1000 value: 99.605 - type: main_score value: 94.146 task: type: Retrieval - dataset: config: default name: MTEB GermanDPR (default) revision: 5129d02422a66be600ac89cd3e8531b4f97d347d split: test type: deepset/germandpr metrics: - type: ndcg_at_1 value: 67.415 - type: ndcg_at_3 value: 81.684 - type: ndcg_at_5 value: 83.829 - type: ndcg_at_10 value: 84.624 - type: ndcg_at_20 value: 84.77900000000001 - type: ndcg_at_100 value: 84.832 - type: ndcg_at_1000 value: 84.832 - type: recall_at_1 value: 67.415 - type: recall_at_3 value: 91.61 - type: recall_at_5 value: 96.78 - type: recall_at_10 value: 99.122 - type: recall_at_20 value: 99.70700000000001 - type: recall_at_100 value: 100.0 - type: recall_at_1000 value: 100.0 - type: main_score value: 84.624 task: type: Retrieval - dataset: config: default name: MTEB GermanQuAD-Retrieval (default) revision: f5c87ae5a2e7a5106606314eef45255f03151bb3 split: test type: mteb/germanquad-retrieval metrics: - type: ndcg_at_1 value: 92.967 - type: ndcg_at_3 value: 96.289 - type: ndcg_at_5 value: 96.626 - type: ndcg_at_10 value: 96.68900000000001 - type: ndcg_at_20 value: 96.767 - type: ndcg_at_100 value: 96.812 - type: ndcg_at_1000 value: 96.812 - type: recall_at_1 value: 92.967 - type: recall_at_3 value: 98.457 - type: recall_at_5 value: 99.274 - type: recall_at_10 value: 99.456 - type: recall_at_20 value: 99.773 - type: recall_at_100 value: 100.0 - type: recall_at_1000 value: 100.0 - type: main_score value: 95.7191 task: type: Retrieval - dataset: config: default name: MTEB HellaSwag (default) revision: a5c990205e017d10761197ccab3000936689c3ae split: test type: RAR-b/hellaswag metrics: - type: ndcg_at_1 value: 24.139 - type: ndcg_at_3 value: 34.455999999999996 - type: ndcg_at_5 value: 37.217 - type: ndcg_at_10 value: 39.655 - type: ndcg_at_20 value: 41.177 - type: ndcg_at_100 value: 43.695 - type: ndcg_at_1000 value: 45.528 - type: recall_at_1 value: 24.139 - type: recall_at_3 value: 41.894 - type: recall_at_5 value: 48.565999999999995 - type: recall_at_10 value: 56.065 - type: recall_at_20 value: 62.07899999999999 - type: recall_at_100 value: 75.812 - type: recall_at_1000 value: 90.5 - type: main_score value: 39.655 task: type: Retrieval - dataset: config: default name: MTEB HotpotQA-PL (default) revision: a0bd479ac97b4ccb5bd6ce320c415d0bb4beb907 split: test type: clarin-knext/hotpotqa-pl metrics: - type: ndcg_at_1 value: 81.796 - type: ndcg_at_3 value: 68.66499999999999 - type: ndcg_at_5 value: 71.364 - type: ndcg_at_10 value: 73.414 - type: ndcg_at_20 value: 74.634 - type: ndcg_at_100 value: 76.276 - type: ndcg_at_1000 value: 77.34299999999999 - type: recall_at_1 value: 40.898 - type: recall_at_3 value: 66.009 - type: recall_at_5 value: 71.317 - type: recall_at_10 value: 76.435 - type: recall_at_20 value: 80.35799999999999 - type: recall_at_100 value: 87.54899999999999 - type: recall_at_1000 value: 94.537 - type: main_score value: 73.414 task: type: Retrieval - dataset: config: default name: MTEB MSMARCO (default) revision: c5a29a104738b98a9e76336939199e264163d4a0 split: dev type: mteb/msmarco metrics: - type: ndcg_at_1 value: 23.854 - type: ndcg_at_3 value: 35.573 - type: ndcg_at_5 value: 39.96 - type: ndcg_at_10 value: 44.064 - type: ndcg_at_20 value: 46.572 - type: ndcg_at_100 value: 49.492000000000004 - type: ndcg_at_1000 value: 50.43 - type: recall_at_1 value: 23.202 - type: recall_at_3 value: 44.092999999999996 - type: recall_at_5 value: 54.6 - type: recall_at_10 value: 67.11399999999999 - type: recall_at_20 value: 76.79899999999999 - type: recall_at_100 value: 92.085 - type: recall_at_1000 value: 99.122 - type: main_score value: 44.064 task: type: Retrieval - dataset: config: default name: MTEB PIQA (default) revision: bb30be7e9184e6b6b1d99bbfe1bb90a3a81842e6 split: test type: RAR-b/piqa metrics: - type: ndcg_at_1 value: 26.387 - type: ndcg_at_3 value: 36.972 - type: ndcg_at_5 value: 39.534000000000006 - type: ndcg_at_10 value: 42.443 - type: ndcg_at_20 value: 44.36 - type: ndcg_at_100 value: 46.575 - type: ndcg_at_1000 value: 48.024 - type: recall_at_1 value: 26.387 - type: recall_at_3 value: 44.45 - type: recall_at_5 value: 50.598 - type: recall_at_10 value: 59.57599999999999 - type: recall_at_20 value: 67.13799999999999 - type: recall_at_100 value: 79.217 - type: recall_at_1000 value: 91.023 - type: main_score value: 42.443 task: type: Retrieval - dataset: config: default name: MTEB Quail (default) revision: 1851bc536f8bdab29e03e29191c4586b1d8d7c5a split: test type: RAR-b/quail metrics: - type: ndcg_at_1 value: 7.242999999999999 - type: ndcg_at_3 value: 11.727 - type: ndcg_at_5 value: 13.69 - type: ndcg_at_10 value: 16.186 - type: ndcg_at_20 value: 17.988 - type: ndcg_at_100 value: 20.926000000000002 - type: ndcg_at_1000 value: 23.980999999999998 - type: recall_at_1 value: 7.242999999999999 - type: recall_at_3 value: 15.037 - type: recall_at_5 value: 19.853 - type: recall_at_10 value: 27.573999999999998 - type: recall_at_20 value: 34.669 - type: recall_at_100 value: 50.662 - type: recall_at_1000 value: 75.735 - type: main_score value: 16.186 task: type: Retrieval - dataset: config: default name: MTEB RARbCode (default) revision: 25f7d11a7ac12dcbb8d3836eb2de682b98c825e4 split: test type: RAR-b/humanevalpack-mbpp-pooled metrics: - type: ndcg_at_1 value: 75.40400000000001 - type: ndcg_at_3 value: 84.796 - type: ndcg_at_5 value: 86.68599999999999 - type: ndcg_at_10 value: 87.63499999999999 - type: ndcg_at_20 value: 87.813 - type: ndcg_at_100 value: 87.912 - type: ndcg_at_1000 value: 87.938 - type: recall_at_1 value: 75.40400000000001 - type: recall_at_3 value: 91.24 - type: recall_at_5 value: 95.822 - type: recall_at_10 value: 98.585 - type: recall_at_20 value: 99.259 - type: recall_at_100 value: 99.798 - type: recall_at_1000 value: 100.0 - type: main_score value: 87.63499999999999 task: type: Retrieval - dataset: config: default name: MTEB RARbMath (default) revision: 2393603c0221ff52f448d12dd75f0856103c6cca split: test type: RAR-b/math-pooled metrics: - type: ndcg_at_1 value: 90.869 - type: ndcg_at_3 value: 92.971 - type: ndcg_at_5 value: 93.365 - type: ndcg_at_10 value: 93.75099999999999 - type: ndcg_at_20 value: 94.05799999999999 - type: ndcg_at_100 value: 94.426 - type: ndcg_at_1000 value: 94.46600000000001 - type: recall_at_1 value: 90.869 - type: recall_at_3 value: 94.414 - type: recall_at_5 value: 95.363 - type: recall_at_10 value: 96.55 - type: recall_at_20 value: 97.753 - type: recall_at_100 value: 99.699 - type: recall_at_1000 value: 100.0 - type: main_score value: 93.75099999999999 task: type: Retrieval - dataset: config: default name: MTEB SIQA (default) revision: 4ed8415e9dc24060deefc84be59e2db0aacbadcc split: test type: RAR-b/siqa metrics: - type: ndcg_at_1 value: 2.661 - type: ndcg_at_3 value: 4.207000000000001 - type: ndcg_at_5 value: 4.577 - type: ndcg_at_10 value: 5.219 - type: ndcg_at_20 value: 5.917 - type: ndcg_at_100 value: 7.9670000000000005 - type: ndcg_at_1000 value: 11.527999999999999 - type: recall_at_1 value: 2.661 - type: recall_at_3 value: 5.271 - type: recall_at_5 value: 6.192 - type: recall_at_10 value: 8.187999999999999 - type: recall_at_20 value: 10.952 - type: recall_at_100 value: 22.262 - type: recall_at_1000 value: 52.098 - type: main_score value: 5.219 task: type: Retrieval - dataset: config: default name: MTEB SpartQA (default) revision: 9ab3ca3ccdd0d43f9cd6d346a363935d127f4f45 split: test type: RAR-b/spartqa metrics: - type: ndcg_at_1 value: 1.252 - type: ndcg_at_3 value: 3.644 - type: ndcg_at_5 value: 5.27 - type: ndcg_at_10 value: 7.768 - type: ndcg_at_20 value: 10.181 - type: ndcg_at_100 value: 14.29 - type: ndcg_at_1000 value: 18.417 - type: recall_at_1 value: 0.788 - type: recall_at_3 value: 5.157 - type: recall_at_5 value: 8.728 - type: recall_at_10 value: 15.786 - type: recall_at_20 value: 24.365000000000002 - type: recall_at_100 value: 43.553999999999995 - type: recall_at_1000 value: 73.66 - type: main_score value: 7.768 task: type: Retrieval - dataset: config: default name: MTEB TempReasonL1 (default) revision: 9097e99aa8c9d827189c65f2e11bfe756af439f6 split: test type: RAR-b/TempReason-l1 metrics: - type: ndcg_at_1 value: 0.1 - type: ndcg_at_3 value: 0.716 - type: ndcg_at_5 value: 1.095 - type: ndcg_at_10 value: 1.6889999999999998 - type: ndcg_at_20 value: 2.374 - type: ndcg_at_100 value: 4.125 - type: ndcg_at_1000 value: 9.126 - type: recall_at_1 value: 0.1 - type: recall_at_3 value: 1.175 - type: recall_at_5 value: 2.1 - type: recall_at_10 value: 3.975 - type: recall_at_20 value: 6.675000000000001 - type: recall_at_100 value: 16.575 - type: recall_at_1000 value: 59.275 - type: main_score value: 1.6889999999999998 task: type: Retrieval - dataset: config: default name: MTEB TempReasonL2Fact (default) revision: 13758bcf978613b249d0de4d0840f57815122bdf split: test type: RAR-b/TempReason-l2-fact metrics: - type: ndcg_at_1 value: 28.942 - type: ndcg_at_3 value: 45.412 - type: ndcg_at_5 value: 50.43299999999999 - type: ndcg_at_10 value: 53.976 - type: ndcg_at_20 value: 55.703 - type: ndcg_at_100 value: 57.445 - type: ndcg_at_1000 value: 57.838 - type: recall_at_1 value: 28.942 - type: recall_at_3 value: 57.495 - type: recall_at_5 value: 69.631 - type: recall_at_10 value: 80.452 - type: recall_at_20 value: 87.252 - type: recall_at_100 value: 96.44200000000001 - type: recall_at_1000 value: 99.518 - type: main_score value: 53.976 task: type: Retrieval - dataset: config: default name: MTEB TempReasonL2Pure (default) revision: 27668949b97bfb178901e0cf047cbee805305dc1 split: test type: RAR-b/TempReason-l2-pure metrics: - type: ndcg_at_1 value: 2.001 - type: ndcg_at_3 value: 3.746 - type: ndcg_at_5 value: 4.665 - type: ndcg_at_10 value: 5.972 - type: ndcg_at_20 value: 7.321999999999999 - type: ndcg_at_100 value: 11.068 - type: ndcg_at_1000 value: 15.675 - type: recall_at_1 value: 2.001 - type: recall_at_3 value: 5.04 - type: recall_at_5 value: 7.3 - type: recall_at_10 value: 11.34 - type: recall_at_20 value: 16.713 - type: recall_at_100 value: 37.576 - type: recall_at_1000 value: 75.394 - type: main_score value: 5.972 task: type: Retrieval - dataset: config: default name: MTEB TempReasonL3Fact (default) revision: 4b70e90197901da24f3cfcd51d27111292878680 split: test type: RAR-b/TempReason-l3-fact metrics: - type: ndcg_at_1 value: 19.114 - type: ndcg_at_3 value: 34.72 - type: ndcg_at_5 value: 40.509 - type: ndcg_at_10 value: 44.894 - type: ndcg_at_20 value: 47.021 - type: ndcg_at_100 value: 49.162 - type: ndcg_at_1000 value: 49.833 - type: recall_at_1 value: 19.114 - type: recall_at_3 value: 46.385 - type: recall_at_5 value: 60.438 - type: recall_at_10 value: 73.882 - type: recall_at_20 value: 82.219 - type: recall_at_100 value: 93.47 - type: recall_at_1000 value: 98.735 - type: main_score value: 44.894 task: type: Retrieval - dataset: config: default name: MTEB TempReasonL3Pure (default) revision: 68fba138e7e63daccecfbdad0a9d2714e56e34ff split: test type: RAR-b/TempReason-l3-pure metrics: - type: ndcg_at_1 value: 0.836 - type: ndcg_at_3 value: 5.319 - type: ndcg_at_5 value: 7.468 - type: ndcg_at_10 value: 10.282 - type: ndcg_at_20 value: 12.457 - type: ndcg_at_100 value: 16.384 - type: ndcg_at_1000 value: 20.081 - type: recall_at_1 value: 0.836 - type: recall_at_3 value: 8.744 - type: recall_at_5 value: 13.963000000000001 - type: recall_at_10 value: 22.729 - type: recall_at_20 value: 31.338 - type: recall_at_100 value: 52.824000000000005 - type: recall_at_1000 value: 82.784 - type: main_score value: 10.282 task: type: Retrieval - dataset: config: default name: MTEB WinoGrande (default) revision: f74c094f321077cf909ddfb8bccc1b5912a4ac28 split: test type: RAR-b/winogrande metrics: - type: ndcg_at_1 value: 47.908 - type: ndcg_at_3 value: 71.58200000000001 - type: ndcg_at_5 value: 74.265 - type: ndcg_at_10 value: 75.61099999999999 - type: ndcg_at_20 value: 76.07300000000001 - type: ndcg_at_100 value: 76.249 - type: ndcg_at_1000 value: 76.249 - type: recall_at_1 value: 47.908 - type: recall_at_3 value: 86.74 - type: recall_at_5 value: 93.21199999999999 - type: recall_at_10 value: 97.316 - type: recall_at_20 value: 99.132 - type: recall_at_100 value: 100.0 - type: recall_at_1000 value: 100.0 - type: main_score value: 75.61099999999999 task: type: Retrieval - dataset: config: de name: MTEB XMarket (de) revision: dfe57acff5b62c23732a7b7d3e3fb84ff501708b split: test type: jinaai/xmarket_ml metrics: - type: ndcg_at_1 value: 30.394 - type: ndcg_at_3 value: 30.701 - type: ndcg_at_5 value: 31.574 - type: ndcg_at_10 value: 32.961 - type: ndcg_at_20 value: 34.765 - type: ndcg_at_100 value: 38.772 - type: ndcg_at_1000 value: 43.317 - type: recall_at_1 value: 10.193000000000001 - type: recall_at_3 value: 19.141 - type: recall_at_5 value: 24.362000000000002 - type: recall_at_10 value: 31.995 - type: recall_at_20 value: 40.047 - type: recall_at_100 value: 56.769000000000005 - type: recall_at_1000 value: 76.318 - type: main_score value: 32.961 task: type: Retrieval - dataset: config: en name: MTEB XMarket (en) revision: dfe57acff5b62c23732a7b7d3e3fb84ff501708b split: test type: jinaai/xmarket_ml metrics: - type: ndcg_at_1 value: 37.652 - type: ndcg_at_3 value: 38.444 - type: ndcg_at_5 value: 39.163 - type: ndcg_at_10 value: 40.557 - type: ndcg_at_20 value: 42.224000000000004 - type: ndcg_at_100 value: 46.817 - type: ndcg_at_1000 value: 51.939 - type: recall_at_1 value: 8.909 - type: recall_at_3 value: 18.673000000000002 - type: recall_at_5 value: 24.364 - type: recall_at_10 value: 32.919 - type: recall_at_20 value: 41.908 - type: recall_at_100 value: 61.663999999999994 - type: recall_at_1000 value: 80.619 - type: main_score value: 40.557 task: type: Retrieval - dataset: config: es name: MTEB XMarket (es) revision: dfe57acff5b62c23732a7b7d3e3fb84ff501708b split: test type: jinaai/xmarket_ml metrics: - type: ndcg_at_1 value: 32.168 - type: ndcg_at_3 value: 32.389 - type: ndcg_at_5 value: 33.054 - type: ndcg_at_10 value: 34.549 - type: ndcg_at_20 value: 36.34 - type: ndcg_at_100 value: 40.324 - type: ndcg_at_1000 value: 44.784 - type: recall_at_1 value: 10.845 - type: recall_at_3 value: 21.058 - type: recall_at_5 value: 26.327 - type: recall_at_10 value: 34.306 - type: recall_at_20 value: 42.46 - type: recall_at_100 value: 59.156 - type: recall_at_1000 value: 78.249 - type: main_score value: 34.549 task: type: Retrieval - dataset: config: ara-ara name: MTEB XPQARetrieval (ara-ara) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 42.0 - type: ndcg_at_3 value: 43.802 - type: ndcg_at_5 value: 46.1 - type: ndcg_at_10 value: 50.858000000000004 - type: ndcg_at_20 value: 54.303999999999995 - type: ndcg_at_100 value: 57.692 - type: ndcg_at_1000 value: 58.97599999999999 - type: recall_at_1 value: 23.989 - type: recall_at_3 value: 42.753 - type: recall_at_5 value: 51.56699999999999 - type: recall_at_10 value: 63.92400000000001 - type: recall_at_20 value: 75.249 - type: recall_at_100 value: 90.851 - type: recall_at_1000 value: 99.733 - type: main_score value: 50.858000000000004 task: type: Retrieval - dataset: config: eng-ara name: MTEB XPQARetrieval (eng-ara) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 31.2 - type: ndcg_at_3 value: 33.296 - type: ndcg_at_5 value: 35.727 - type: ndcg_at_10 value: 39.837 - type: ndcg_at_20 value: 43.354 - type: ndcg_at_100 value: 47.908 - type: ndcg_at_1000 value: 50.187000000000005 - type: recall_at_1 value: 18.007 - type: recall_at_3 value: 32.5 - type: recall_at_5 value: 41.422 - type: recall_at_10 value: 51.673 - type: recall_at_20 value: 63.144 - type: recall_at_100 value: 83.733 - type: recall_at_1000 value: 99.10900000000001 - type: main_score value: 39.837 task: type: Retrieval - dataset: config: ara-eng name: MTEB XPQARetrieval (ara-eng) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 40.431 - type: ndcg_at_3 value: 41.419 - type: ndcg_at_5 value: 44.051 - type: ndcg_at_10 value: 48.94 - type: ndcg_at_20 value: 52.532999999999994 - type: ndcg_at_100 value: 56.203 - type: ndcg_at_1000 value: 57.467999999999996 - type: recall_at_1 value: 22.534000000000002 - type: recall_at_3 value: 40.119 - type: recall_at_5 value: 49.569 - type: recall_at_10 value: 62.156 - type: recall_at_20 value: 74.191 - type: recall_at_100 value: 90.973 - type: recall_at_1000 value: 99.72999999999999 - type: main_score value: 48.94 task: type: Retrieval - dataset: config: deu-deu name: MTEB XPQARetrieval (deu-deu) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 76.50099999999999 - type: ndcg_at_3 value: 79.38199999999999 - type: ndcg_at_5 value: 81.00500000000001 - type: ndcg_at_10 value: 82.786 - type: ndcg_at_20 value: 83.844 - type: ndcg_at_100 value: 84.708 - type: ndcg_at_1000 value: 84.956 - type: recall_at_1 value: 58.464000000000006 - type: recall_at_3 value: 79.963 - type: recall_at_5 value: 85.757 - type: recall_at_10 value: 90.372 - type: recall_at_20 value: 94.13 - type: recall_at_100 value: 98.24000000000001 - type: recall_at_1000 value: 100.0 - type: main_score value: 82.786 task: type: Retrieval - dataset: config: eng-deu name: MTEB XPQARetrieval (eng-deu) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 52.611 - type: ndcg_at_3 value: 55.35099999999999 - type: ndcg_at_5 value: 57.452999999999996 - type: ndcg_at_10 value: 61.553999999999995 - type: ndcg_at_20 value: 63.919000000000004 - type: ndcg_at_100 value: 66.90700000000001 - type: ndcg_at_1000 value: 67.685 - type: recall_at_1 value: 33.47 - type: recall_at_3 value: 55.174 - type: recall_at_5 value: 63.512 - type: recall_at_10 value: 73.934 - type: recall_at_20 value: 81.26400000000001 - type: recall_at_100 value: 94.606 - type: recall_at_1000 value: 100.0 - type: main_score value: 61.553999999999995 task: type: Retrieval - dataset: config: deu-eng name: MTEB XPQARetrieval (deu-eng) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 70.235 - type: ndcg_at_3 value: 74.824 - type: ndcg_at_5 value: 76.47699999999999 - type: ndcg_at_10 value: 78.803 - type: ndcg_at_20 value: 80.19 - type: ndcg_at_100 value: 81.07799999999999 - type: ndcg_at_1000 value: 81.40899999999999 - type: recall_at_1 value: 52.818 - type: recall_at_3 value: 76.754 - type: recall_at_5 value: 82.637 - type: recall_at_10 value: 88.655 - type: recall_at_20 value: 93.61 - type: recall_at_100 value: 97.731 - type: recall_at_1000 value: 100.0 - type: main_score value: 78.803 task: type: Retrieval - dataset: config: spa-spa name: MTEB XPQARetrieval (spa-spa) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 64.18700000000001 - type: ndcg_at_3 value: 62.714999999999996 - type: ndcg_at_5 value: 64.134 - type: ndcg_at_10 value: 68.143 - type: ndcg_at_20 value: 70.625 - type: ndcg_at_100 value: 73.333 - type: ndcg_at_1000 value: 74.02300000000001 - type: recall_at_1 value: 34.400999999999996 - type: recall_at_3 value: 57.654 - type: recall_at_5 value: 67.167 - type: recall_at_10 value: 76.31599999999999 - type: recall_at_20 value: 83.731 - type: recall_at_100 value: 95.502 - type: recall_at_1000 value: 99.58 - type: main_score value: 68.143 task: type: Retrieval - dataset: config: eng-spa name: MTEB XPQARetrieval (eng-spa) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 47.667 - type: ndcg_at_3 value: 46.35 - type: ndcg_at_5 value: 47.879 - type: ndcg_at_10 value: 52.733 - type: ndcg_at_20 value: 55.620000000000005 - type: ndcg_at_100 value: 59.70100000000001 - type: ndcg_at_1000 value: 61.417 - type: recall_at_1 value: 23.394000000000002 - type: recall_at_3 value: 42.264 - type: recall_at_5 value: 51.144999999999996 - type: recall_at_10 value: 62.556 - type: recall_at_20 value: 71.269 - type: recall_at_100 value: 88.668 - type: recall_at_1000 value: 99.466 - type: main_score value: 52.733 task: type: Retrieval - dataset: config: spa-eng name: MTEB XPQARetrieval (spa-eng) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 61.285999999999994 - type: ndcg_at_3 value: 60.303 - type: ndcg_at_5 value: 62.062 - type: ndcg_at_10 value: 66.042 - type: ndcg_at_20 value: 68.509 - type: ndcg_at_100 value: 71.539 - type: ndcg_at_1000 value: 72.258 - type: recall_at_1 value: 32.224000000000004 - type: recall_at_3 value: 55.443 - type: recall_at_5 value: 65.67699999999999 - type: recall_at_10 value: 74.607 - type: recall_at_20 value: 82.234 - type: recall_at_100 value: 95.275 - type: recall_at_1000 value: 99.723 - type: main_score value: 66.042 task: type: Retrieval - dataset: config: fra-fra name: MTEB XPQARetrieval (fr) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 71.429 - type: ndcg_at_3 value: 71.13000000000001 - type: ndcg_at_5 value: 72.709 - type: ndcg_at_10 value: 76.236 - type: ndcg_at_20 value: 77.78500000000001 - type: ndcg_at_100 value: 79.634 - type: ndcg_at_1000 value: 79.953 - type: recall_at_1 value: 45.943 - type: recall_at_3 value: 68.293 - type: recall_at_5 value: 76.5 - type: recall_at_10 value: 85.11999999999999 - type: recall_at_20 value: 90.069 - type: recall_at_100 value: 97.82600000000001 - type: recall_at_1000 value: 99.866 - type: main_score value: 76.236 task: type: Retrieval - dataset: config: eng-fra name: MTEB XPQARetrieval (eng-fra) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 47.797 - type: ndcg_at_3 value: 49.514 - type: ndcg_at_5 value: 51.782 - type: ndcg_at_10 value: 55.891000000000005 - type: ndcg_at_20 value: 59.226 - type: ndcg_at_100 value: 62.612 - type: ndcg_at_1000 value: 63.749 - type: recall_at_1 value: 26.689 - type: recall_at_3 value: 47.408 - type: recall_at_5 value: 57.399 - type: recall_at_10 value: 67.147 - type: recall_at_20 value: 77.837 - type: recall_at_100 value: 92.494 - type: recall_at_1000 value: 99.74 - type: main_score value: 55.891000000000005 task: type: Retrieval - dataset: config: fra-eng name: MTEB XPQARetrieval (fra-eng) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 68.625 - type: ndcg_at_3 value: 68.239 - type: ndcg_at_5 value: 70.175 - type: ndcg_at_10 value: 73.452 - type: ndcg_at_20 value: 75.66000000000001 - type: ndcg_at_100 value: 77.506 - type: ndcg_at_1000 value: 77.936 - type: recall_at_1 value: 44.035999999999994 - type: recall_at_3 value: 65.291 - type: recall_at_5 value: 74.37899999999999 - type: recall_at_10 value: 82.15 - type: recall_at_20 value: 89.457 - type: recall_at_100 value: 97.194 - type: recall_at_1000 value: 99.933 - type: main_score value: 73.452 task: type: Retrieval - dataset: config: hin-hin name: MTEB XPQARetrieval (hin-hin) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 66.703 - type: ndcg_at_3 value: 72.993 - type: ndcg_at_5 value: 75.138 - type: ndcg_at_10 value: 77.371 - type: ndcg_at_20 value: 78.389 - type: ndcg_at_100 value: 79.623 - type: ndcg_at_1000 value: 79.975 - type: recall_at_1 value: 57.094 - type: recall_at_3 value: 77.2 - type: recall_at_5 value: 82.50800000000001 - type: recall_at_10 value: 88.486 - type: recall_at_20 value: 91.863 - type: recall_at_100 value: 97.359 - type: recall_at_1000 value: 99.892 - type: main_score value: 77.371 task: type: Retrieval - dataset: config: eng-hin name: MTEB XPQARetrieval (eng-hin) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 29.837999999999997 - type: ndcg_at_3 value: 34.187 - type: ndcg_at_5 value: 37.132 - type: ndcg_at_10 value: 41.357 - type: ndcg_at_20 value: 44.522 - type: ndcg_at_100 value: 49.486999999999995 - type: ndcg_at_1000 value: 51.458000000000006 - type: recall_at_1 value: 24.959999999999997 - type: recall_at_3 value: 36.472 - type: recall_at_5 value: 44.175 - type: recall_at_10 value: 55.371 - type: recall_at_20 value: 65.506 - type: recall_at_100 value: 87.252 - type: recall_at_1000 value: 99.78399999999999 - type: main_score value: 41.357 task: type: Retrieval - dataset: config: hin-eng name: MTEB XPQARetrieval (hin-eng) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 62.829 - type: ndcg_at_3 value: 68.886 - type: ndcg_at_5 value: 71.812 - type: ndcg_at_10 value: 74.405 - type: ndcg_at_20 value: 75.702 - type: ndcg_at_100 value: 77.08500000000001 - type: ndcg_at_1000 value: 77.377 - type: recall_at_1 value: 53.568000000000005 - type: recall_at_3 value: 73.095 - type: recall_at_5 value: 80.211 - type: recall_at_10 value: 87.229 - type: recall_at_20 value: 91.625 - type: recall_at_100 value: 97.844 - type: recall_at_1000 value: 100.0 - type: main_score value: 74.405 task: type: Retrieval - dataset: config: ita-ita name: MTEB XPQARetrieval (ita-ita) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 73.303 - type: ndcg_at_3 value: 74.51299999999999 - type: ndcg_at_5 value: 76.383 - type: ndcg_at_10 value: 78.968 - type: ndcg_at_20 value: 80.331 - type: ndcg_at_100 value: 81.65599999999999 - type: ndcg_at_1000 value: 82.075 - type: recall_at_1 value: 50.68899999999999 - type: recall_at_3 value: 72.763 - type: recall_at_5 value: 80.85 - type: recall_at_10 value: 87.071 - type: recall_at_20 value: 91.62599999999999 - type: recall_at_100 value: 97.333 - type: recall_at_1000 value: 100.0 - type: main_score value: 78.968 task: type: Retrieval - dataset: config: eng-ita name: MTEB XPQARetrieval (eng-ita) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 42.232 - type: ndcg_at_3 value: 46.231 - type: ndcg_at_5 value: 48.197 - type: ndcg_at_10 value: 52.217 - type: ndcg_at_20 value: 55.472 - type: ndcg_at_100 value: 58.803000000000004 - type: ndcg_at_1000 value: 60.321000000000005 - type: recall_at_1 value: 26.368000000000002 - type: recall_at_3 value: 46.709 - type: recall_at_5 value: 54.721 - type: recall_at_10 value: 64.46 - type: recall_at_20 value: 74.997 - type: recall_at_100 value: 89.527 - type: recall_at_1000 value: 99.698 - type: main_score value: 52.217 task: type: Retrieval - dataset: config: ita-eng name: MTEB XPQARetrieval (ita-eng) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 68.326 - type: ndcg_at_3 value: 70.71499999999999 - type: ndcg_at_5 value: 72.748 - type: ndcg_at_10 value: 75.31 - type: ndcg_at_20 value: 76.958 - type: ndcg_at_100 value: 78.66300000000001 - type: ndcg_at_1000 value: 79.089 - type: recall_at_1 value: 46.583999999999996 - type: recall_at_3 value: 69.887 - type: recall_at_5 value: 78.10000000000001 - type: recall_at_10 value: 84.329 - type: recall_at_20 value: 89.51 - type: recall_at_100 value: 97.235 - type: recall_at_1000 value: 100.0 - type: main_score value: 75.31 task: type: Retrieval - dataset: config: jpn-jpn name: MTEB XPQARetrieval (jpn-jpn) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 72.0 - type: ndcg_at_3 value: 74.005 - type: ndcg_at_5 value: 75.411 - type: ndcg_at_10 value: 77.12 - type: ndcg_at_20 value: 78.625 - type: ndcg_at_100 value: 80.281 - type: ndcg_at_1000 value: 80.682 - type: recall_at_1 value: 46.988 - type: recall_at_3 value: 72.36200000000001 - type: recall_at_5 value: 79.501 - type: recall_at_10 value: 83.83 - type: recall_at_20 value: 88.907 - type: recall_at_100 value: 96.739 - type: recall_at_1000 value: 99.636 - type: main_score value: 77.12 task: type: Retrieval - dataset: config: eng-jpn name: MTEB XPQARetrieval (eng-jpn) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 43.758 - type: ndcg_at_3 value: 45.513999999999996 - type: ndcg_at_5 value: 47.93 - type: ndcg_at_10 value: 51.983999999999995 - type: ndcg_at_20 value: 54.544000000000004 - type: ndcg_at_100 value: 58.022 - type: ndcg_at_1000 value: 59.843 - type: recall_at_1 value: 25.543 - type: recall_at_3 value: 44.374 - type: recall_at_5 value: 53.86300000000001 - type: recall_at_10 value: 63.756 - type: recall_at_20 value: 72.14699999999999 - type: recall_at_100 value: 87.58200000000001 - type: recall_at_1000 value: 99.295 - type: main_score value: 51.983999999999995 task: type: Retrieval - dataset: config: jpn-eng name: MTEB XPQARetrieval (jpn-eng) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 68.978 - type: ndcg_at_3 value: 71.019 - type: ndcg_at_5 value: 72.697 - type: ndcg_at_10 value: 75.267 - type: ndcg_at_20 value: 76.655 - type: ndcg_at_100 value: 78.388 - type: ndcg_at_1000 value: 78.899 - type: recall_at_1 value: 44.958999999999996 - type: recall_at_3 value: 69.56400000000001 - type: recall_at_5 value: 77.082 - type: recall_at_10 value: 83.646 - type: recall_at_20 value: 88.238 - type: recall_at_100 value: 96.194 - type: recall_at_1000 value: 99.818 - type: main_score value: 75.267 task: type: Retrieval - dataset: config: kor-kor name: MTEB XPQARetrieval (kor-kor) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 33.18 - type: ndcg_at_3 value: 35.311 - type: ndcg_at_5 value: 38.366 - type: ndcg_at_10 value: 41.654 - type: ndcg_at_20 value: 44.244 - type: ndcg_at_100 value: 49.001 - type: ndcg_at_1000 value: 51.01 - type: recall_at_1 value: 23.201 - type: recall_at_3 value: 37.011 - type: recall_at_5 value: 44.493 - type: recall_at_10 value: 53.489 - type: recall_at_20 value: 62.548 - type: recall_at_100 value: 85.55 - type: recall_at_1000 value: 100.0 - type: main_score value: 41.654 task: type: Retrieval - dataset: config: eng-kor name: MTEB XPQARetrieval (eng-kor) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 34.404 - type: ndcg_at_3 value: 35.821 - type: ndcg_at_5 value: 37.268 - type: ndcg_at_10 value: 40.967 - type: ndcg_at_20 value: 43.509 - type: ndcg_at_100 value: 49.326 - type: ndcg_at_1000 value: 51.410999999999994 - type: recall_at_1 value: 20.363999999999997 - type: recall_at_3 value: 35.293 - type: recall_at_5 value: 41.251 - type: recall_at_10 value: 50.766999999999996 - type: recall_at_20 value: 59.274 - type: recall_at_100 value: 86.669 - type: recall_at_1000 value: 100.0 - type: main_score value: 40.967 task: type: Retrieval - dataset: config: kor-eng name: MTEB XPQARetrieval (kor-eng) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 33.062000000000005 - type: ndcg_at_3 value: 35.619 - type: ndcg_at_5 value: 37.684 - type: ndcg_at_10 value: 40.986 - type: ndcg_at_20 value: 43.736999999999995 - type: ndcg_at_100 value: 48.632999999999996 - type: ndcg_at_1000 value: 50.78 - type: recall_at_1 value: 23.18 - type: recall_at_3 value: 37.235 - type: recall_at_5 value: 42.448 - type: recall_at_10 value: 51.395 - type: recall_at_20 value: 61.01 - type: recall_at_100 value: 84.382 - type: recall_at_1000 value: 100.0 - type: main_score value: 40.986 task: type: Retrieval - dataset: config: pol-pol name: MTEB XPQARetrieval (pol-pol) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 46.115 - type: ndcg_at_3 value: 45.966 - type: ndcg_at_5 value: 48.119 - type: ndcg_at_10 value: 51.53 - type: ndcg_at_20 value: 54.447 - type: ndcg_at_100 value: 58.939 - type: ndcg_at_1000 value: 60.428000000000004 - type: recall_at_1 value: 27.641 - type: recall_at_3 value: 45.021 - type: recall_at_5 value: 52.580000000000005 - type: recall_at_10 value: 61.141999999999996 - type: recall_at_20 value: 70.588 - type: recall_at_100 value: 90.29700000000001 - type: recall_at_1000 value: 99.851 - type: main_score value: 51.53 task: type: Retrieval - dataset: config: eng-pol name: MTEB XPQARetrieval (eng-pol) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 32.357 - type: ndcg_at_3 value: 31.573 - type: ndcg_at_5 value: 33.046 - type: ndcg_at_10 value: 37.364999999999995 - type: ndcg_at_20 value: 40.407 - type: ndcg_at_100 value: 45.965 - type: ndcg_at_1000 value: 48.982 - type: recall_at_1 value: 14.865999999999998 - type: recall_at_3 value: 28.51 - type: recall_at_5 value: 35.827999999999996 - type: recall_at_10 value: 46.11 - type: recall_at_20 value: 55.498999999999995 - type: recall_at_100 value: 79.73 - type: recall_at_1000 value: 99.236 - type: main_score value: 37.364999999999995 task: type: Retrieval - dataset: config: pol-eng name: MTEB XPQARetrieval (pol-eng) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 43.114999999999995 - type: ndcg_at_3 value: 42.306 - type: ndcg_at_5 value: 44.484 - type: ndcg_at_10 value: 48.374 - type: ndcg_at_20 value: 51.347 - type: ndcg_at_100 value: 56.223 - type: ndcg_at_1000 value: 57.93899999999999 - type: recall_at_1 value: 25.746000000000002 - type: recall_at_3 value: 41.160000000000004 - type: recall_at_5 value: 48.256 - type: recall_at_10 value: 58.038999999999994 - type: recall_at_20 value: 67.499 - type: recall_at_100 value: 88.912 - type: recall_at_1000 value: 99.85000000000001 - type: main_score value: 48.374 task: type: Retrieval - dataset: config: por-por name: MTEB XPQARetrieval (por-por) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 47.25 - type: ndcg_at_3 value: 46.225 - type: ndcg_at_5 value: 47.813 - type: ndcg_at_10 value: 51.383 - type: ndcg_at_20 value: 54.291 - type: ndcg_at_100 value: 58.434 - type: ndcg_at_1000 value: 60.07 - type: recall_at_1 value: 25.394 - type: recall_at_3 value: 43.446 - type: recall_at_5 value: 51.037 - type: recall_at_10 value: 59.61 - type: recall_at_20 value: 68.925 - type: recall_at_100 value: 88.277 - type: recall_at_1000 value: 99.44800000000001 - type: main_score value: 51.383 task: type: Retrieval - dataset: config: eng-por name: MTEB XPQARetrieval (eng-por) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 29.5 - type: ndcg_at_3 value: 29.971999999999998 - type: ndcg_at_5 value: 31.513999999999996 - type: ndcg_at_10 value: 35.449999999999996 - type: ndcg_at_20 value: 38.912 - type: ndcg_at_100 value: 44.695 - type: ndcg_at_1000 value: 47.309 - type: recall_at_1 value: 14.335 - type: recall_at_3 value: 27.839999999999996 - type: recall_at_5 value: 34.737 - type: recall_at_10 value: 44.358 - type: recall_at_20 value: 55.65 - type: recall_at_100 value: 82.077 - type: recall_at_1000 value: 99.44800000000001 - type: main_score value: 35.449999999999996 task: type: Retrieval - dataset: config: por-eng name: MTEB XPQARetrieval (por-eng) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 46.048 - type: ndcg_at_3 value: 45.519 - type: ndcg_at_5 value: 47.693999999999996 - type: ndcg_at_10 value: 51.535 - type: ndcg_at_20 value: 54.179 - type: ndcg_at_100 value: 58.416999999999994 - type: ndcg_at_1000 value: 59.955000000000005 - type: recall_at_1 value: 25.325999999999997 - type: recall_at_3 value: 42.779 - type: recall_at_5 value: 51.453 - type: recall_at_10 value: 60.876 - type: recall_at_20 value: 69.184 - type: recall_at_100 value: 88.97699999999999 - type: recall_at_1000 value: 99.58200000000001 - type: main_score value: 51.535 task: type: Retrieval - dataset: config: tam-tam name: MTEB XPQARetrieval (tam-tam) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 31.968999999999998 - type: ndcg_at_3 value: 34.555 - type: ndcg_at_5 value: 36.504999999999995 - type: ndcg_at_10 value: 38.958 - type: ndcg_at_20 value: 40.77 - type: ndcg_at_100 value: 43.779 - type: ndcg_at_1000 value: 47.388999999999996 - type: recall_at_1 value: 21.13 - type: recall_at_3 value: 35.838 - type: recall_at_5 value: 41.535 - type: recall_at_10 value: 48.075 - type: recall_at_20 value: 54.290000000000006 - type: recall_at_100 value: 68.325 - type: recall_at_1000 value: 95.62 - type: main_score value: 38.958 task: type: Retrieval - dataset: config: eng-tam name: MTEB XPQARetrieval (eng-tam) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 12.531999999999998 - type: ndcg_at_3 value: 12.849 - type: ndcg_at_5 value: 13.979 - type: ndcg_at_10 value: 16.573 - type: ndcg_at_20 value: 18.861 - type: ndcg_at_100 value: 23.779 - type: ndcg_at_1000 value: 29.859 - type: recall_at_1 value: 7.388999999999999 - type: recall_at_3 value: 12.531999999999998 - type: recall_at_5 value: 16.279 - type: recall_at_10 value: 23.099 - type: recall_at_20 value: 30.697000000000003 - type: recall_at_100 value: 53.608 - type: recall_at_1000 value: 94.719 - type: main_score value: 16.573 task: type: Retrieval - dataset: config: tam-eng name: MTEB XPQARetrieval (tam-eng) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 21.066 - type: ndcg_at_3 value: 23.677999999999997 - type: ndcg_at_5 value: 25.851000000000003 - type: ndcg_at_10 value: 28.615000000000002 - type: ndcg_at_20 value: 30.817 - type: ndcg_at_100 value: 34.874 - type: ndcg_at_1000 value: 39.24 - type: recall_at_1 value: 15.037 - type: recall_at_3 value: 25.285999999999998 - type: recall_at_5 value: 30.717 - type: recall_at_10 value: 37.722 - type: recall_at_20 value: 44.927 - type: recall_at_100 value: 63.917 - type: recall_at_1000 value: 96.145 - type: main_score value: 28.615000000000002 task: type: Retrieval - dataset: config: cmn-cmn name: MTEB XPQARetrieval (cmn-cmn) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 65.049 - type: ndcg_at_3 value: 65.534 - type: ndcg_at_5 value: 67.498 - type: ndcg_at_10 value: 70.812 - type: ndcg_at_20 value: 73.026 - type: ndcg_at_100 value: 75.316 - type: ndcg_at_1000 value: 75.882 - type: recall_at_1 value: 41.357 - type: recall_at_3 value: 63.176 - type: recall_at_5 value: 71.381 - type: recall_at_10 value: 79.47 - type: recall_at_20 value: 86.616 - type: recall_at_100 value: 96.36099999999999 - type: recall_at_1000 value: 100.0 - type: main_score value: 70.812 task: type: Retrieval - dataset: config: eng-cmn name: MTEB XPQARetrieval (eng-cmn) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 35.073 - type: ndcg_at_3 value: 35.782000000000004 - type: ndcg_at_5 value: 36.99 - type: ndcg_at_10 value: 40.974 - type: ndcg_at_20 value: 43.971 - type: ndcg_at_100 value: 49.165 - type: ndcg_at_1000 value: 51.93 - type: recall_at_1 value: 20.057 - type: recall_at_3 value: 34.064 - type: recall_at_5 value: 40.831 - type: recall_at_10 value: 50.33 - type: recall_at_20 value: 59.306000000000004 - type: recall_at_100 value: 82.231 - type: recall_at_1000 value: 99.759 - type: main_score value: 40.974 task: type: Retrieval - dataset: config: cmn-eng name: MTEB XPQARetrieval (cmn-eng) revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f split: test type: jinaai/xpqa metrics: - type: ndcg_at_1 value: 57.68299999999999 - type: ndcg_at_3 value: 60.089000000000006 - type: ndcg_at_5 value: 62.217999999999996 - type: ndcg_at_10 value: 65.81700000000001 - type: ndcg_at_20 value: 67.886 - type: ndcg_at_100 value: 70.804 - type: ndcg_at_1000 value: 71.54 - type: recall_at_1 value: 36.146 - type: recall_at_3 value: 59.035000000000004 - type: recall_at_5 value: 67.376 - type: recall_at_10 value: 76.213 - type: recall_at_20 value: 82.756 - type: recall_at_100 value: 95.341 - type: recall_at_1000 value: 100.0 - type: main_score value: 65.81700000000001 task: type: Retrieval tags: - sentence-transformers - transformers - sentence-similarity - mteb --- # INF-Retriever-v1 ## Model Overview - **INF-Retriever-v1** is an LLM-based dense retrieval model developed by [INF TECH](https://www.infly.cn/en). It is built upon the [gte-Qwen2-7B-instruct](https://huggingface.co/Alibaba-NLP/gte-Qwen2-7B-instruct) model and specifically fine-tuned to excel in retrieval tasks, particularly for Chinese and English data. - As of January 23, 2025, **INF-Retriever-v1** ranks both **No.1** on the Automated Heterogeneous Information Retrieval Benchmark of version 24.04 & 24.05([AIR-Bench](https://huggingface.co/spaces/AIR-Bench/leaderboard)), showcasing its cutting-edge performance in heterogeneous information retrieval tasks. ## Key Features - **Optimized for Chinese and English retrieval**: The model has been specifically fine-tuned with retrieval-focused datasets in both languages, significantly improving its accuracy and efficiency for a variety of retrieval scenarios. - **Top-tier performance**: **INF-Retriever-v1** has achieved outstanding results on the AIR-Bench leaderboard, making it a top choice for heterogeneous information retrieval tasks across various domains. ## Model Details - Model Size: 7B - Embedding Dimension: 3584 - Max Input Tokens: 32768 ## Usage ### Sentence Transformers ```python from sentence_transformers import SentenceTransformer model = SentenceTransformer("infly/inf-retriever-v1", trust_remote_code=True) # In case you want to reduce the maximum length: model.max_seq_length = 8192 queries = [ "how much protein should a female eat", "summit define", ] documents = [ "As a general guideline, the CDC's average requirement of protein for women ages 19 to 70 is 46 grams per day. But, as you can see from this chart, you'll need to increase that if you're expecting or training for a marathon. Check out the chart below to see how much protein you should be eating each day.", "Definition of summit for English Language Learners. : 1 the highest point of a mountain : the top of a mountain. : 2 the highest level. : 3 a meeting or series of meetings between the leaders of two or more governments.", ] query_embeddings = model.encode(queries, prompt_name="query") document_embeddings = model.encode(documents) scores = (query_embeddings @ document_embeddings.T) * 100 print(scores.tolist()) # [[86.8702392578125, 67.82364654541016], [59.51014709472656, 82.33668518066406]] ``` ### Transformers ```python import torch import torch.nn.functional as F from torch import Tensor from transformers import AutoTokenizer, AutoModel def last_token_pool(last_hidden_states: Tensor, attention_mask: Tensor) -> Tensor: left_padding = (attention_mask[:, -1].sum() == attention_mask.shape[0]) if left_padding: return last_hidden_states[:, -1] else: sequence_lengths = attention_mask.sum(dim=1) - 1 batch_size = last_hidden_states.shape[0] return last_hidden_states[torch.arange(batch_size, device=last_hidden_states.device), sequence_lengths] def get_detailed_instruct(task_description: str, query: str) -> str: return f'Instruct: {task_description}\nQuery: {query}' # Each query must come with a one-sentence instruction that describes the task task = 'Given a web search query, retrieve relevant passages that answer the query' queries = [ get_detailed_instruct(task, 'how much protein should a female eat'), get_detailed_instruct(task, 'summit define') ] # No need to add instruction for retrieval documents documents = [ "As a general guideline, the CDC's average requirement of protein for women ages 19 to 70 is 46 grams per day. But, as you can see from this chart, you'll need to increase that if you're expecting or training for a marathon. Check out the chart below to see how much protein you should be eating each day.", "Definition of summit for English Language Learners. : 1 the highest point of a mountain : the top of a mountain. : 2 the highest level. : 3 a meeting or series of meetings between the leaders of two or more governments." ] input_texts = queries + documents tokenizer = AutoTokenizer.from_pretrained('infly/inf-retriever-v1', trust_remote_code=True) model = AutoModel.from_pretrained('infly/inf-retriever-v1', trust_remote_code=True) max_length = 8192 # Tokenize the input texts batch_dict = tokenizer(input_texts, max_length=max_length, padding=True, truncation=True, return_tensors='pt') outputs = model(**batch_dict) embeddings = last_token_pool(outputs.last_hidden_state, batch_dict['attention_mask']) # normalize embeddings embeddings = F.normalize(embeddings, p=2, dim=1) scores = (embeddings[:2] @ embeddings[2:].T) * 100 print(scores.tolist()) # [[86.87025451660156, 67.82366180419922], [59.510135650634766, 82.33667755126953]] ``` ## Evaluation ### AIR-Bench **INF-Retriever-v1** has demonstrated superior retrieval capabilities across multiple domains and languages. The results from the Automated Heterogeneous Information Retrieval Benchmark ([AIR-Bench](https://huggingface.co/spaces/AIR-Bench/leaderboard)) as of January 23, 2025, are as follows: #### AIR-Bench_24.04 (Bilingual, EN & ZH) | Model Name | Average⬆️ | wiki_en | wiki_zh | web_en | web_zh | healthcare_en | healthcare_zh | law_en | arxiv_en | news_en | news_zh | finance_en | finance_zh | msmarco_en | |-----------------------------------------------------------------------------------|-----------|-----------|-----------|-----------|----------|---------------|---------------|-----------|-----------|-----------|-----------|------------|------------|------------| | [E5-mistral-7b-instruct](https://huggingface.co/intfloat/e5-mistral-7b-instruct) | 45.26 | 61.67 | 55.97 | 44.41 | 45.96 | 56.32 | 35.79 | 19.32 | 44.78 | 48.18 | 35.99 | 54.79 | 26.11 | 59.03 | | [BGE-M3](https://huggingface.co/BAAI/bge-m3) | 46.65 | 60.49 | 62.36 | 47.35 | 50.38 | 49.1 | **42.38** | 26.68 | 40.76 | 48.04 | 40.75 | 51.52 | 32.18 | 54.4 | | [BGE-Multilingual-Gemma2](https://huggingface.co/BAAI/bge-multilingual-gemma2) | 46.83 | 63.71 | 67.3 | 50.38 | 53.24 | 47.24 | 42.13 | 22.58 | 23.28 | 50.91 | 44.02 | 49.3 | 31.6 | **63.14** | | [GTE-Qwen2-7B-instruct](https://huggingface.co/Alibaba-NLP/gte-Qwen2-7B-instruct) | 48.38 | 63.46 | 66.44 | 51.2 | 51.98 | 54.2 | 38.82 | 22.31 | 40.27 | **54.07** | 43.03 | 58.2 | 26.63 | 58.39 | | **INF-Retriever-v1** | **52.56** | **65.25** | **68.44** | **52.13** | **56.6** | **56.96** | 42.03 | **34.51** | **50.62** | 53.32 | **50.02** | **58.34** | **35.42** | 59.64 | #### AIR-Bench_24.05 (Multilingual, 13 languages) Although INF-Retriever-v1 has been fine-tuned exclusively on English and Chinese, it continues to perform exceptionally well across other languages, securing the No. 1 position on this multilingual benchmark. | Model Name | Average⬆️ | wiki_en | wiki_zh | wiki_ar | wiki_bn | wiki_de | wiki_es | wiki_fa | wiki_fr | wiki_hi | wiki_id | wiki_ja | wiki_ko | wiki_ru | web_en | web_zh | web_ar | web_bn | web_de | web_es | web_fa | web_fr | web_hi | web_id | web_ja | web_ko | web_ru | healthcare_en | healthcare_zh | healthcare_de | healthcare_es | healthcare_fr | law_en | law_de | law_fr | arxiv_en | science_ru | news_en | news_zh | news_ar | news_bn | news_de | news_es | news_fa | news_fr | news_hi | news_id | news_ja | news_ko | news_ru | finance_en | finance_zh | finance_ar | finance_fr | |--------------------------------------------------------------------------------------------------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|-----------|----------|-----------|-----------|----------|--------|-----------|-----------|-----------|---------------|---------------|---------------|---------------|---------------|-----------|-----------|-----------|-----------|------------|-----------|-----------|-----------|-----------|-----------|----------|-----------|----------|-----------|-----------|-----------|-----------|-----------|------------|------------|------------|------------| | [GTE-Qwen2-7B-instruct](https://huggingface.co/Alibaba-NLP/gte-Qwen2-7B-instruct) | 50.05 | **73.59** | 67.5 | 59.44 | 58.17 | 63.96 | 67.62 | 57.05 | 70.32 | 60.54 | 61.81 | 62.88 | 59.17 | 62.95 | **58.99** | 51.66 | 55.56 | 51.45 | 48.62 | 54.11 | 49.54 | 55.16 | 53.06 | 55.51 | 57.27 | 57.54 | 55.88 | 54.46 | 38.66 | 53.92 | 53.78 | 30.29 | 22.75 | 13.18 | 13.15 | 41.32 | 45.21 | **52.74** | 43.17 | 37.63 | **61.31** | 44.89 | 45.21 | 30.1 | 49.76 | 30.28 | 46.44 | 44.13 | 47.19 | 46.55 | 59.23 | 34.61 | 43.56 | 39.57 | | [Multilingual-E5-large-instruct](https://huggingface.co/intfloat/multilingual-e5-large-instruct) | 51.11 | 68.62 | 62.82 | 63.21 | 64.45 | 65.81 | 68.1 | 64.2 | 69.72 | 71.81 | 66.36 | 64.12 | 64.79 | 62.57 | 41.58 | 47.06 | 56.4 | 56.17 | 50.87 | 52.24 | 58.68 | 50.2 | 56.32 | 54.49 | 54.89 | 55.81 | 54.97 | 54.02 | 39.76 | 52.06 | 51.74 | 36.64 | 16.9 | 15.59 | 15.12 | 39.52 | 56.86 | 44.28 | 35.46 | 48.2 | 49.31 | 47.84 | 45.99 | **45.59** | 50.58 | 39.66 | 48.59 | 47.6 | 50.52 | 48.81 | 52.79 | 37.72 | 48.95 | 42.74 | | [BGE-M3](https://huggingface.co/BAAI/bge-m3) | 51.31 | 69.7 | 63.52 | 59.65 | 64.33 | 64.68 | 65.4 | 61.14 | 66.04 | 69.02 | 66.3 | 60.86 | 62.36 | 60.18 | 53.88 | 50.2 | 52.53 | 55.53 | 51.89 | 51.78 | 55.81 | 51.46 | 57.06 | 53.14 | 54.75 | 55.28 | 54.53 | 49.05 | 42.31 | 49 | 53.05 | 39.29 | 26.95 | 20.11 | 20.2 | 41.64 | 55.18 | 47.34 | 41 | 44.93 | 59.03 | 47.87 | 44.7 | 43.81 | 49.52 | 42.12 | 47.45 | 47.09 | 48.14 | 48.31 | 52.92 | 40.23 | 45.76 | 41.44 | | [BGE-Multilingual-Gemma2](https://huggingface.co/BAAI/bge-multilingual-gemma2) | 54.46 | 72.8 | 68.64 | **63.42** | **69.48** | **67.91** | **71.79** | **67.57** | **71.28** | **75.39** | **68.91** | **68.29** | **66.78** | **64.15** | 56.48 | 53.04 | **59.97** | **59.68** | **57.72** | **58.2** | **62.43** | **59.54** | **64.5** | **60** | **60.26** | 59.64 | **60.12** | 47.48 | **42.35** | 55.4 | **63.13** | **45.13** | 22.6 | 15.75 | 14.29 | 24 | 44.13 | 50.29 | 43.42 | 48.41 | 58.77 | **52.05** | **49.9** | 43.4 | **56.8** | **44.89** | 50.65 | **51.51** | 51.64 | 51.48 | 50.08 | 39.23 | 50.25 | **51.1** | | **INF-Retriever-v1** | **54.47** | 73.52 | **69.45** | 63.13 | 61.58 | 66.8 | 69.29 | 63.03 | 69.74 | 69.02 | 68.63 | 63.45 | 64.44 | 62.74 | 57.6 | **56.46** | 58.48 | 53.7 | 55.2 | 57.08 | 53.27 | 57.35 | 55.64 | 58.85 | 59.52 | **60.01** | 58.79 | **57.03** | 41.82 | **55.46** | 57.6 | 43.25 | **34.76** | **21.75** | **21.87** | **51.38** | **59.72** | 52.7 | **49.78** | **49.11** | 43.62 | 51.47 | 49.52 | 40.43 | 54.54 | 38.57 | **51.06** | 51.12 | **53.15** | **51.88** | **59.44** | **44.13** | **50.71** | 44.2 | ## Contributors ### Supervisors Wei Chu • Yinghui Xu • Yuan Qi ### INF memory team Junhan Yang (junhanyang@inftech.ai) • Jiahe Wan • Yichen Yao (eason.yyc@inftech.ai) ## Citation If you find our model useful, please consider citing: ``` @misc {infly-ai_2025, author = { {infly-ai} }, title = { inf-retriever-v1 (Revision 5f469d7) }, year = 2025, url = { https://huggingface.co/infly/inf-retriever-v1 }, doi = { 10.57967/hf/4262 }, publisher = { Hugging Face } } ```