|
--- |
|
model-index: |
|
- name: Quark-Emb-8B |
|
results: |
|
- dataset: |
|
config: default |
|
name: MTEB AFQMC (default) |
|
revision: latest2023 |
|
split: validation |
|
type: C-MTEB/AFQMC |
|
metrics: |
|
- type: cosine_pearson |
|
value: 52.87704664791064 |
|
- type: cosine_spearman |
|
value: 53.567003436521375 |
|
- type: manhattan_pearson |
|
value: 52.07472780799189 |
|
- type: manhattan_spearman |
|
value: 53.5368469974003 |
|
- type: euclidean_pearson |
|
value: 52.074186684368016 |
|
- type: euclidean_spearman |
|
value: 53.515536447088074 |
|
- type: main_score |
|
value: 53.567003436521375 |
|
task: |
|
type: STS |
|
- dataset: |
|
config: default |
|
name: MTEB ATEC (default) |
|
revision: latest2023 |
|
split: test |
|
type: C-MTEB/ATEC |
|
metrics: |
|
- type: cosine_pearson |
|
value: 59.13301114775821 |
|
- type: cosine_spearman |
|
value: 53.42152760117668 |
|
- type: manhattan_pearson |
|
value: 60.05185745744783 |
|
- type: manhattan_spearman |
|
value: 53.36914545708813 |
|
- type: euclidean_pearson |
|
value: 60.17725014927802 |
|
- type: euclidean_spearman |
|
value: 53.431110991334485 |
|
- type: main_score |
|
value: 53.42152760117668 |
|
task: |
|
type: STS |
|
- dataset: |
|
config: zh |
|
name: MTEB AmazonReviewsClassification (zh) |
|
revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
|
split: test |
|
type: mteb/amazon_reviews_multi |
|
metrics: |
|
- type: accuracy |
|
value: 54.74600000000001 |
|
- type: accuracy_stderr |
|
value: 1.060492338491892 |
|
- type: f1 |
|
value: 53.49846112279175 |
|
- type: f1_stderr |
|
value: 1.729174511160517 |
|
- type: main_score |
|
value: 54.74600000000001 |
|
task: |
|
type: Classification |
|
- dataset: |
|
config: zh |
|
name: MTEB AmazonReviewsClassification (zh) |
|
revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
|
split: validation |
|
type: mteb/amazon_reviews_multi |
|
metrics: |
|
- type: accuracy |
|
value: 53.468 |
|
- type: accuracy_stderr |
|
value: 1.0129639677698308 |
|
- type: f1 |
|
value: 52.21987651679265 |
|
- type: f1_stderr |
|
value: 1.7016822177116173 |
|
- type: main_score |
|
value: 53.468 |
|
task: |
|
type: Classification |
|
- dataset: |
|
config: default |
|
name: MTEB BQ (default) |
|
revision: latest2023 |
|
split: test |
|
type: C-MTEB/BQ |
|
metrics: |
|
- type: cosine_pearson |
|
value: 69.640366232364 |
|
- type: cosine_spearman |
|
value: 70.65881213017273 |
|
- type: manhattan_pearson |
|
value: 67.76837799100343 |
|
- type: manhattan_spearman |
|
value: 70.5046111101055 |
|
- type: euclidean_pearson |
|
value: 67.83004194158737 |
|
- type: euclidean_spearman |
|
value: 70.60927547682859 |
|
- type: main_score |
|
value: 70.65881213017273 |
|
task: |
|
type: STS |
|
- dataset: |
|
config: default |
|
name: MTEB CLSClusteringP2P (default) |
|
revision: latest2023 |
|
split: test |
|
type: C-MTEB/CLSClusteringP2P |
|
metrics: |
|
- type: v_measure |
|
value: 62.32714079793593 |
|
- type: v_measure_std |
|
value: 1.5782386182731478 |
|
- type: main_score |
|
value: 62.32714079793593 |
|
task: |
|
type: Clustering |
|
- dataset: |
|
config: default |
|
name: MTEB CLSClusteringS2S (default) |
|
revision: latest2023 |
|
split: test |
|
type: C-MTEB/CLSClusteringS2S |
|
metrics: |
|
- type: v_measure |
|
value: 59.29532340833129 |
|
- type: v_measure_std |
|
value: 1.5258658358346424 |
|
- type: main_score |
|
value: 59.29532340833129 |
|
task: |
|
type: Clustering |
|
- dataset: |
|
config: default |
|
name: MTEB CMedQAv1 |
|
revision: latest2023 |
|
split: test |
|
type: C-MTEB/CMedQAv1-reranking |
|
metrics: |
|
- type: map |
|
value: 88.02263756085355 |
|
- type: mrr |
|
value: 90.18928571428572 |
|
- type: main_score |
|
value: 88.02263756085355 |
|
task: |
|
type: Reranking |
|
- dataset: |
|
config: default |
|
name: MTEB CMedQAv2 |
|
revision: latest2023 |
|
split: test |
|
type: C-MTEB/CMedQAv2-reranking |
|
metrics: |
|
- type: map |
|
value: 88.81199829110464 |
|
- type: mrr |
|
value: 90.81817460317461 |
|
- type: main_score |
|
value: 88.81199829110464 |
|
task: |
|
type: Reranking |
|
- dataset: |
|
config: default |
|
name: MTEB CmedqaRetrieval (default) |
|
revision: latest2023 |
|
split: dev |
|
type: C-MTEB/CmedqaRetrieval |
|
metrics: |
|
- type: map_at_1 |
|
value: 27.448 |
|
- type: map_at_10 |
|
value: 40.794000000000004 |
|
- type: map_at_100 |
|
value: 42.606 |
|
- type: map_at_1000 |
|
value: 42.711 |
|
- type: map_at_20 |
|
value: 41.778 |
|
- type: map_at_3 |
|
value: 36.429 |
|
- type: map_at_5 |
|
value: 38.841 |
|
- type: mrr_at_1 |
|
value: 41.510000000000005 |
|
- type: mrr_at_10 |
|
value: 49.986999999999995 |
|
- type: mrr_at_100 |
|
value: 50.908 |
|
- type: mrr_at_1000 |
|
value: 50.946000000000005 |
|
- type: mrr_at_20 |
|
value: 50.531000000000006 |
|
- type: mrr_at_3 |
|
value: 47.562 |
|
- type: mrr_at_5 |
|
value: 48.882 |
|
- type: ndcg_at_1 |
|
value: 41.510000000000005 |
|
- type: ndcg_at_10 |
|
value: 47.620000000000005 |
|
- type: ndcg_at_100 |
|
value: 54.586999999999996 |
|
- type: ndcg_at_1000 |
|
value: 56.324 |
|
- type: ndcg_at_20 |
|
value: 50.332 |
|
- type: ndcg_at_3 |
|
value: 42.27 |
|
- type: ndcg_at_5 |
|
value: 44.421 |
|
- type: precision_at_1 |
|
value: 41.510000000000005 |
|
- type: precision_at_10 |
|
value: 10.45 |
|
- type: precision_at_100 |
|
value: 1.6179999999999999 |
|
- type: precision_at_1000 |
|
value: 0.184 |
|
- type: precision_at_20 |
|
value: 6.1339999999999995 |
|
- type: precision_at_3 |
|
value: 23.823 |
|
- type: precision_at_5 |
|
value: 17.089 |
|
- type: recall_at_1 |
|
value: 27.448 |
|
- type: recall_at_10 |
|
value: 58.629 |
|
- type: recall_at_100 |
|
value: 87.26899999999999 |
|
- type: recall_at_1000 |
|
value: 98.713 |
|
- type: recall_at_20 |
|
value: 67.929 |
|
- type: recall_at_3 |
|
value: 42.331 |
|
- type: recall_at_5 |
|
value: 49.193999999999996 |
|
- type: main_score |
|
value: 47.620000000000005 |
|
task: |
|
type: Retrieval |
|
- dataset: |
|
config: default |
|
name: MTEB Cmnli (default) |
|
revision: latest2023 |
|
split: validation |
|
type: C-MTEB/CMNLI |
|
metrics: |
|
- type: cos_sim_accuracy |
|
value: 91.040288634997 |
|
- type: cos_sim_accuracy_threshold |
|
value: 96.31753556207411 |
|
- type: cos_sim_ap |
|
value: 95.94857244353375 |
|
- type: cos_sim_f1 |
|
value: 91.47971901200997 |
|
- type: cos_sim_f1_threshold |
|
value: 96.01236110428391 |
|
- type: cos_sim_precision |
|
value: 88.74477907232358 |
|
- type: cos_sim_recall |
|
value: 94.388590133271 |
|
- type: dot_accuracy |
|
value: 80.7696933253157 |
|
- type: dot_accuracy_threshold |
|
value: 58.4022485296251 |
|
- type: dot_ap |
|
value: 89.17817373664943 |
|
- type: dot_f1 |
|
value: 81.62572172534811 |
|
- type: dot_f1_threshold |
|
value: 58.18378730482039 |
|
- type: dot_precision |
|
value: 79.12642669007901 |
|
- type: dot_recall |
|
value: 84.28805237315876 |
|
- type: euclidean_accuracy |
|
value: 90.92002405291642 |
|
- type: euclidean_accuracy_threshold |
|
value: 21.553298629922512 |
|
- type: euclidean_ap |
|
value: 95.90941014786691 |
|
- type: euclidean_f1 |
|
value: 91.45241317095173 |
|
- type: euclidean_f1_threshold |
|
value: 22.109074422645463 |
|
- type: euclidean_precision |
|
value: 88.32498366368982 |
|
- type: euclidean_recall |
|
value: 94.80944587327565 |
|
- type: manhattan_accuracy |
|
value: 90.94407696933253 |
|
- type: manhattan_accuracy_threshold |
|
value: 524.1466620016906 |
|
- type: manhattan_ap |
|
value: 95.89310684813798 |
|
- type: manhattan_f1 |
|
value: 91.50400541577343 |
|
- type: manhattan_f1_threshold |
|
value: 525.5181014869215 |
|
- type: manhattan_precision |
|
value: 88.4212821631051 |
|
- type: manhattan_recall |
|
value: 94.80944587327565 |
|
- type: max_accuracy |
|
value: 91.040288634997 |
|
- type: max_ap |
|
value: 95.94857244353375 |
|
- type: max_f1 |
|
value: 91.50400541577343 |
|
task: |
|
type: PairClassification |
|
- dataset: |
|
config: default |
|
name: MTEB CovidRetrieval (default) |
|
revision: latest2023 |
|
split: dev |
|
type: C-MTEB/CovidRetrieval |
|
metrics: |
|
- type: map_at_1 |
|
value: 78.82000000000001 |
|
- type: map_at_10 |
|
value: 85.51100000000001 |
|
- type: map_at_100 |
|
value: 85.67099999999999 |
|
- type: map_at_1000 |
|
value: 85.672 |
|
- type: map_at_20 |
|
value: 85.641 |
|
- type: map_at_3 |
|
value: 84.321 |
|
- type: map_at_5 |
|
value: 85.048 |
|
- type: mrr_at_1 |
|
value: 78.925 |
|
- type: mrr_at_10 |
|
value: 85.548 |
|
- type: mrr_at_100 |
|
value: 85.698 |
|
- type: mrr_at_1000 |
|
value: 85.699 |
|
- type: mrr_at_20 |
|
value: 85.669 |
|
- type: mrr_at_3 |
|
value: 84.45700000000001 |
|
- type: mrr_at_5 |
|
value: 85.12100000000001 |
|
- type: ndcg_at_1 |
|
value: 78.925 |
|
- type: ndcg_at_10 |
|
value: 88.359 |
|
- type: ndcg_at_100 |
|
value: 88.98899999999999 |
|
- type: ndcg_at_1000 |
|
value: 89.017 |
|
- type: ndcg_at_20 |
|
value: 88.776 |
|
- type: ndcg_at_3 |
|
value: 86.086 |
|
- type: ndcg_at_5 |
|
value: 87.336 |
|
- type: precision_at_1 |
|
value: 78.925 |
|
- type: precision_at_10 |
|
value: 9.789 |
|
- type: precision_at_100 |
|
value: 1.0070000000000001 |
|
- type: precision_at_1000 |
|
value: 0.101 |
|
- type: precision_at_20 |
|
value: 4.979 |
|
- type: precision_at_3 |
|
value: 30.488 |
|
- type: precision_at_5 |
|
value: 18.925 |
|
- type: recall_at_1 |
|
value: 78.82000000000001 |
|
- type: recall_at_10 |
|
value: 96.997 |
|
- type: recall_at_100 |
|
value: 99.684 |
|
- type: recall_at_1000 |
|
value: 99.895 |
|
- type: recall_at_20 |
|
value: 98.52499999999999 |
|
- type: recall_at_3 |
|
value: 91.01700000000001 |
|
- type: recall_at_5 |
|
value: 93.994 |
|
- type: main_score |
|
value: 88.359 |
|
task: |
|
type: Retrieval |
|
- dataset: |
|
config: default |
|
name: MTEB DuRetrieval (default) |
|
revision: latest2023 |
|
split: dev |
|
type: C-MTEB/DuRetrieval |
|
metrics: |
|
- type: map_at_1 |
|
value: 28.03 |
|
- type: map_at_10 |
|
value: 85.60600000000001 |
|
- type: map_at_100 |
|
value: 88.14800000000001 |
|
- type: map_at_1000 |
|
value: 88.169 |
|
- type: map_at_20 |
|
value: 87.591 |
|
- type: map_at_3 |
|
value: 60.06 |
|
- type: map_at_5 |
|
value: 75.564 |
|
- type: mrr_at_1 |
|
value: 94.05 |
|
- type: mrr_at_10 |
|
value: 96.043 |
|
- type: mrr_at_100 |
|
value: 96.075 |
|
- type: mrr_at_1000 |
|
value: 96.077 |
|
- type: mrr_at_20 |
|
value: 96.06099999999999 |
|
- type: mrr_at_3 |
|
value: 95.875 |
|
- type: mrr_at_5 |
|
value: 96.017 |
|
- type: ndcg_at_1 |
|
value: 94.05 |
|
- type: ndcg_at_10 |
|
value: 91.58800000000001 |
|
- type: ndcg_at_100 |
|
value: 93.536 |
|
- type: ndcg_at_1000 |
|
value: 93.726 |
|
- type: ndcg_at_20 |
|
value: 92.64099999999999 |
|
- type: ndcg_at_3 |
|
value: 90.865 |
|
- type: ndcg_at_5 |
|
value: 89.972 |
|
- type: precision_at_1 |
|
value: 94.05 |
|
- type: precision_at_10 |
|
value: 43.19 |
|
- type: precision_at_100 |
|
value: 4.859 |
|
- type: precision_at_1000 |
|
value: 0.49 |
|
- type: precision_at_20 |
|
value: 23.3 |
|
- type: precision_at_3 |
|
value: 81.0 |
|
- type: precision_at_5 |
|
value: 68.36 |
|
- type: recall_at_1 |
|
value: 28.03 |
|
- type: recall_at_10 |
|
value: 92.095 |
|
- type: recall_at_100 |
|
value: 98.764 |
|
- type: recall_at_1000 |
|
value: 99.71 |
|
- type: recall_at_20 |
|
value: 95.87 |
|
- type: recall_at_3 |
|
value: 61.949 |
|
- type: recall_at_5 |
|
value: 79.41 |
|
- type: main_score |
|
value: 91.58800000000001 |
|
task: |
|
type: Retrieval |
|
- dataset: |
|
config: default |
|
name: MTEB EcomRetrieval (default) |
|
revision: latest2023 |
|
split: dev |
|
type: C-MTEB/EcomRetrieval |
|
metrics: |
|
- type: map_at_1 |
|
value: 58.599999999999994 |
|
- type: map_at_10 |
|
value: 68.88499999999999 |
|
- type: map_at_100 |
|
value: 69.269 |
|
- type: map_at_1000 |
|
value: 69.274 |
|
- type: map_at_20 |
|
value: 69.17699999999999 |
|
- type: map_at_3 |
|
value: 66.167 |
|
- type: map_at_5 |
|
value: 68.082 |
|
- type: mrr_at_1 |
|
value: 58.599999999999994 |
|
- type: mrr_at_10 |
|
value: 68.88499999999999 |
|
- type: mrr_at_100 |
|
value: 69.269 |
|
- type: mrr_at_1000 |
|
value: 69.274 |
|
- type: mrr_at_20 |
|
value: 69.17699999999999 |
|
- type: mrr_at_3 |
|
value: 66.167 |
|
- type: mrr_at_5 |
|
value: 68.082 |
|
- type: ndcg_at_1 |
|
value: 58.599999999999994 |
|
- type: ndcg_at_10 |
|
value: 74.018 |
|
- type: ndcg_at_100 |
|
value: 75.72 |
|
- type: ndcg_at_1000 |
|
value: 75.851 |
|
- type: ndcg_at_20 |
|
value: 75.08 |
|
- type: ndcg_at_3 |
|
value: 68.64 |
|
- type: ndcg_at_5 |
|
value: 72.075 |
|
- type: precision_at_1 |
|
value: 58.599999999999994 |
|
- type: precision_at_10 |
|
value: 9.01 |
|
- type: precision_at_100 |
|
value: 0.9769999999999999 |
|
- type: precision_at_1000 |
|
value: 0.099 |
|
- type: precision_at_20 |
|
value: 4.715 |
|
- type: precision_at_3 |
|
value: 25.267 |
|
- type: precision_at_5 |
|
value: 16.82 |
|
- type: recall_at_1 |
|
value: 58.599999999999994 |
|
- type: recall_at_10 |
|
value: 90.10000000000001 |
|
- type: recall_at_100 |
|
value: 97.7 |
|
- type: recall_at_1000 |
|
value: 98.7 |
|
- type: recall_at_20 |
|
value: 94.3 |
|
- type: recall_at_3 |
|
value: 75.8 |
|
- type: recall_at_5 |
|
value: 84.1 |
|
- type: main_score |
|
value: 74.018 |
|
task: |
|
type: Retrieval |
|
- dataset: |
|
config: default |
|
name: MTEB IFlyTek (default) |
|
revision: latest2023 |
|
split: validation |
|
type: C-MTEB/IFlyTek-classification |
|
metrics: |
|
- type: accuracy |
|
value: 55.79838399384378 |
|
- type: accuracy_stderr |
|
value: 0.273588131352537 |
|
- type: f1 |
|
value: 42.23811666656058 |
|
- type: f1_stderr |
|
value: 0.2317340030986553 |
|
- type: main_score |
|
value: 55.79838399384378 |
|
task: |
|
type: Classification |
|
- dataset: |
|
config: default |
|
name: MTEB JDReview (default) |
|
revision: latest2023 |
|
split: test |
|
type: C-MTEB/JDReview-classification |
|
metrics: |
|
- type: accuracy |
|
value: 89.11819887429644 |
|
- type: accuracy_stderr |
|
value: 1.5149440328845287 |
|
- type: ap |
|
value: 60.17445086411222 |
|
- type: ap_stderr |
|
value: 3.4864563160430384 |
|
- type: f1 |
|
value: 84.14324891240739 |
|
- type: f1_stderr |
|
value: 1.804154595730216 |
|
- type: main_score |
|
value: 89.11819887429644 |
|
task: |
|
type: Classification |
|
- dataset: |
|
config: default |
|
name: MTEB LCQMC (default) |
|
revision: latest2023 |
|
split: test |
|
type: C-MTEB/LCQMC |
|
metrics: |
|
- type: cosine_pearson |
|
value: 80.86541640109346 |
|
- type: cosine_spearman |
|
value: 79.60409318173409 |
|
- type: manhattan_pearson |
|
value: 81.12725142112909 |
|
- type: manhattan_spearman |
|
value: 79.61120096401483 |
|
- type: euclidean_pearson |
|
value: 81.1558178459699 |
|
- type: euclidean_spearman |
|
value: 79.63206760369867 |
|
- type: main_score |
|
value: 79.60409318173409 |
|
task: |
|
type: STS |
|
- dataset: |
|
config: default |
|
name: MTEB MMarcoReranking (default) |
|
revision: latest2023 |
|
split: dev |
|
type: C-MTEB/Mmarco-reranking |
|
metrics: |
|
- type: map |
|
value: 30.290346963620866 |
|
- type: mrr |
|
value: 29.661507936507935 |
|
- type: main_score |
|
value: 30.290346963620866 |
|
task: |
|
type: Reranking |
|
- dataset: |
|
config: default |
|
name: MTEB MMarcoRetrieval (default) |
|
revision: latest2023 |
|
split: dev |
|
type: C-MTEB/MMarcoRetrieval |
|
metrics: |
|
- type: map_at_1 |
|
value: 67.801 |
|
- type: map_at_10 |
|
value: 76.771 |
|
- type: map_at_100 |
|
value: 77.08 |
|
- type: map_at_1000 |
|
value: 77.091 |
|
- type: map_at_20 |
|
value: 76.982 |
|
- type: map_at_3 |
|
value: 75.035 |
|
- type: map_at_5 |
|
value: 76.171 |
|
- type: mrr_at_1 |
|
value: 70.057 |
|
- type: mrr_at_10 |
|
value: 77.387 |
|
- type: mrr_at_100 |
|
value: 77.65100000000001 |
|
- type: mrr_at_1000 |
|
value: 77.661 |
|
- type: mrr_at_20 |
|
value: 77.566 |
|
- type: mrr_at_3 |
|
value: 75.90299999999999 |
|
- type: mrr_at_5 |
|
value: 76.848 |
|
- type: ndcg_at_1 |
|
value: 70.057 |
|
- type: ndcg_at_10 |
|
value: 80.37100000000001 |
|
- type: ndcg_at_100 |
|
value: 81.71300000000001 |
|
- type: ndcg_at_1000 |
|
value: 81.982 |
|
- type: ndcg_at_20 |
|
value: 81.074 |
|
- type: ndcg_at_3 |
|
value: 77.12 |
|
- type: ndcg_at_5 |
|
value: 79.00500000000001 |
|
- type: precision_at_1 |
|
value: 70.057 |
|
- type: precision_at_10 |
|
value: 9.643 |
|
- type: precision_at_100 |
|
value: 1.031 |
|
- type: precision_at_1000 |
|
value: 0.105 |
|
- type: precision_at_20 |
|
value: 4.973000000000001 |
|
- type: precision_at_3 |
|
value: 28.959000000000003 |
|
- type: precision_at_5 |
|
value: 18.384 |
|
- type: recall_at_1 |
|
value: 67.801 |
|
- type: recall_at_10 |
|
value: 90.821 |
|
- type: recall_at_100 |
|
value: 96.809 |
|
- type: recall_at_1000 |
|
value: 98.87899999999999 |
|
- type: recall_at_20 |
|
value: 93.49300000000001 |
|
- type: recall_at_3 |
|
value: 82.26 |
|
- type: recall_at_5 |
|
value: 86.725 |
|
- type: main_score |
|
value: 80.37100000000001 |
|
task: |
|
type: Retrieval |
|
- dataset: |
|
config: zh-CN |
|
name: MTEB MassiveIntentClassification (zh-CN) |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
split: test |
|
type: mteb/amazon_massive_intent |
|
metrics: |
|
- type: accuracy |
|
value: 81.4862138533961 |
|
- type: accuracy_stderr |
|
value: 1.024666951162929 |
|
- type: f1 |
|
value: 78.57865898474617 |
|
- type: f1_stderr |
|
value: 1.1662766217911715 |
|
- type: main_score |
|
value: 81.4862138533961 |
|
task: |
|
type: Classification |
|
- dataset: |
|
config: zh-CN |
|
name: MTEB MassiveScenarioClassification (zh-CN) |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
split: test |
|
type: mteb/amazon_massive_scenario |
|
metrics: |
|
- type: accuracy |
|
value: 86.43913920645595 |
|
- type: accuracy_stderr |
|
value: 0.6198364524624383 |
|
- type: f1 |
|
value: 85.45514450914429 |
|
- type: f1_stderr |
|
value: 0.775295463718716 |
|
- type: main_score |
|
value: 86.43913920645595 |
|
task: |
|
type: Classification |
|
- dataset: |
|
config: default |
|
name: MTEB MedicalRetrieval (default) |
|
revision: latest2023 |
|
split: dev |
|
type: C-MTEB/MedicalRetrieval |
|
metrics: |
|
- type: map_at_1 |
|
value: 60.0 |
|
- type: map_at_10 |
|
value: 66.673 |
|
- type: map_at_100 |
|
value: 67.239 |
|
- type: map_at_1000 |
|
value: 67.26299999999999 |
|
- type: map_at_20 |
|
value: 67.01899999999999 |
|
- type: map_at_3 |
|
value: 65.333 |
|
- type: map_at_5 |
|
value: 66.063 |
|
- type: mrr_at_1 |
|
value: 60.099999999999994 |
|
- type: mrr_at_10 |
|
value: 66.739 |
|
- type: mrr_at_100 |
|
value: 67.306 |
|
- type: mrr_at_1000 |
|
value: 67.33 |
|
- type: mrr_at_20 |
|
value: 67.086 |
|
- type: mrr_at_3 |
|
value: 65.4 |
|
- type: mrr_at_5 |
|
value: 66.13 |
|
- type: ndcg_at_1 |
|
value: 60.0 |
|
- type: ndcg_at_10 |
|
value: 69.786 |
|
- type: ndcg_at_100 |
|
value: 72.693 |
|
- type: ndcg_at_1000 |
|
value: 73.373 |
|
- type: ndcg_at_20 |
|
value: 71.032 |
|
- type: ndcg_at_3 |
|
value: 67.024 |
|
- type: ndcg_at_5 |
|
value: 68.34 |
|
- type: precision_at_1 |
|
value: 60.0 |
|
- type: precision_at_10 |
|
value: 7.95 |
|
- type: precision_at_100 |
|
value: 0.935 |
|
- type: precision_at_1000 |
|
value: 0.099 |
|
- type: precision_at_20 |
|
value: 4.22 |
|
- type: precision_at_3 |
|
value: 23.967 |
|
- type: precision_at_5 |
|
value: 15.02 |
|
- type: recall_at_1 |
|
value: 60.0 |
|
- type: recall_at_10 |
|
value: 79.5 |
|
- type: recall_at_100 |
|
value: 93.5 |
|
- type: recall_at_1000 |
|
value: 98.9 |
|
- type: recall_at_20 |
|
value: 84.39999999999999 |
|
- type: recall_at_3 |
|
value: 71.89999999999999 |
|
- type: recall_at_5 |
|
value: 75.1 |
|
- type: main_score |
|
value: 69.786 |
|
task: |
|
type: Retrieval |
|
- dataset: |
|
config: default |
|
name: MTEB MultilingualSentiment (default) |
|
revision: latest2023 |
|
split: validation |
|
type: C-MTEB/MultilingualSentiment-classification |
|
metrics: |
|
- type: accuracy |
|
value: 81.19666666666667 |
|
- type: accuracy_stderr |
|
value: 0.6507175526550155 |
|
- type: f1 |
|
value: 81.3717120301294 |
|
- type: f1_stderr |
|
value: 0.629161893845245 |
|
- type: main_score |
|
value: 81.19666666666667 |
|
task: |
|
type: Classification |
|
- dataset: |
|
config: default |
|
name: MTEB Ocnli (default) |
|
revision: latest2023 |
|
split: validation |
|
type: C-MTEB/OCNLI |
|
metrics: |
|
- type: cos_sim_accuracy |
|
value: 89.92961559285327 |
|
- type: cos_sim_accuracy_threshold |
|
value: 95.72410743295985 |
|
- type: cos_sim_ap |
|
value: 94.04072585697942 |
|
- type: cos_sim_f1 |
|
value: 90.6060606060606 |
|
- type: cos_sim_f1_threshold |
|
value: 95.71085030120679 |
|
- type: cos_sim_precision |
|
value: 86.83446272991287 |
|
- type: cos_sim_recall |
|
value: 94.72016895459345 |
|
- type: dot_accuracy |
|
value: 84.29886302111532 |
|
- type: dot_accuracy_threshold |
|
value: 58.39834018444028 |
|
- type: dot_ap |
|
value: 90.71149047430606 |
|
- type: dot_f1 |
|
value: 84.83263598326361 |
|
- type: dot_f1_threshold |
|
value: 58.39834018444028 |
|
- type: dot_precision |
|
value: 84.04145077720207 |
|
- type: dot_recall |
|
value: 85.6388595564942 |
|
- type: euclidean_accuracy |
|
value: 89.87547374120194 |
|
- type: euclidean_accuracy_threshold |
|
value: 22.827705768877962 |
|
- type: euclidean_ap |
|
value: 93.87312138426815 |
|
- type: euclidean_f1 |
|
value: 90.5982905982906 |
|
- type: euclidean_f1_threshold |
|
value: 22.940558234199905 |
|
- type: euclidean_precision |
|
value: 86.468330134357 |
|
- type: euclidean_recall |
|
value: 95.14255543822597 |
|
- type: manhattan_accuracy |
|
value: 89.44233892799134 |
|
- type: manhattan_accuracy_threshold |
|
value: 544.6256417358975 |
|
- type: manhattan_ap |
|
value: 93.8313800715528 |
|
- type: manhattan_f1 |
|
value: 90.16641452344932 |
|
- type: manhattan_f1_threshold |
|
value: 544.6256417358975 |
|
- type: manhattan_precision |
|
value: 86.2934362934363 |
|
- type: manhattan_recall |
|
value: 94.40337909186906 |
|
- type: max_accuracy |
|
value: 89.92961559285327 |
|
- type: max_ap |
|
value: 94.04072585697942 |
|
- type: max_f1 |
|
value: 90.6060606060606 |
|
task: |
|
type: PairClassification |
|
- dataset: |
|
config: default |
|
name: MTEB OnlineShopping (default) |
|
revision: latest2023 |
|
split: test |
|
type: C-MTEB/OnlineShopping-classification |
|
metrics: |
|
- type: accuracy |
|
value: 93.83999999999999 |
|
- type: accuracy_stderr |
|
value: 0.4521061822182879 |
|
- type: ap |
|
value: 92.19373645628713 |
|
- type: ap_stderr |
|
value: 0.2927396159644918 |
|
- type: f1 |
|
value: 93.83158946571204 |
|
- type: f1_stderr |
|
value: 0.4472553159438725 |
|
- type: main_score |
|
value: 93.83999999999999 |
|
task: |
|
type: Classification |
|
- dataset: |
|
config: default |
|
name: MTEB PAWSX (default) |
|
revision: latest2023 |
|
split: test |
|
type: C-MTEB/PAWSX |
|
metrics: |
|
- type: cosine_pearson |
|
value: 50.30680596662101 |
|
- type: cosine_spearman |
|
value: 52.41534063346883 |
|
- type: manhattan_pearson |
|
value: 51.81137421589127 |
|
- type: manhattan_spearman |
|
value: 52.40332176267904 |
|
- type: euclidean_pearson |
|
value: 51.842454511431235 |
|
- type: euclidean_spearman |
|
value: 52.4062829337432 |
|
- type: main_score |
|
value: 52.41534063346883 |
|
task: |
|
type: STS |
|
- dataset: |
|
config: default |
|
name: MTEB QBQTC (default) |
|
revision: latest2023 |
|
split: test |
|
type: C-MTEB/QBQTC |
|
metrics: |
|
- type: cosine_pearson |
|
value: 58.31070289198933 |
|
- type: cosine_spearman |
|
value: 57.966010447080684 |
|
- type: manhattan_pearson |
|
value: 54.99874211888254 |
|
- type: manhattan_spearman |
|
value: 57.796012247889195 |
|
- type: euclidean_pearson |
|
value: 55.138798573277455 |
|
- type: euclidean_spearman |
|
value: 57.95150876116391 |
|
- type: main_score |
|
value: 57.966010447080684 |
|
task: |
|
type: STS |
|
- dataset: |
|
config: zh |
|
name: MTEB STS22 (zh) |
|
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 |
|
split: test |
|
type: mteb/sts22-crosslingual-sts |
|
metrics: |
|
- type: cosine_pearson |
|
value: 75.26028549682404 |
|
- type: cosine_spearman |
|
value: 73.9002967678025 |
|
- type: manhattan_pearson |
|
value: 73.47220514464013 |
|
- type: manhattan_spearman |
|
value: 73.74093326288234 |
|
- type: euclidean_pearson |
|
value: 73.59040445366989 |
|
- type: euclidean_spearman |
|
value: 73.9002967678025 |
|
- type: main_score |
|
value: 73.9002967678025 |
|
task: |
|
type: STS |
|
- dataset: |
|
config: default |
|
name: MTEB STSB (default) |
|
revision: latest2023 |
|
split: test |
|
type: C-MTEB/STSB |
|
metrics: |
|
- type: cosine_pearson |
|
value: 78.17081373176123 |
|
- type: cosine_spearman |
|
value: 78.61566426272397 |
|
- type: manhattan_pearson |
|
value: 77.66643088697434 |
|
- type: manhattan_spearman |
|
value: 78.94692354474782 |
|
- type: euclidean_pearson |
|
value: 77.69471041307843 |
|
- type: euclidean_spearman |
|
value: 78.92513847741967 |
|
- type: main_score |
|
value: 78.61566426272397 |
|
task: |
|
type: STS |
|
- dataset: |
|
config: default |
|
name: MTEB T2Reranking (default) |
|
revision: latest2023 |
|
split: dev |
|
type: C-MTEB/T2Reranking |
|
metrics: |
|
- type: map |
|
value: 68.13018101639273 |
|
- type: mrr |
|
value: 79.13973922902494 |
|
- type: main_score |
|
value: 68.13018101639273 |
|
task: |
|
type: Reranking |
|
- dataset: |
|
config: default |
|
name: MTEB T2Retrieval (default) |
|
revision: latest2023 |
|
split: dev |
|
type: C-MTEB/T2Retrieval |
|
metrics: |
|
- type: map_at_1 |
|
value: 28.591 |
|
- type: map_at_10 |
|
value: 80.979 |
|
- type: map_at_100 |
|
value: 84.411 |
|
- type: map_at_1000 |
|
value: 84.458 |
|
- type: map_at_20 |
|
value: 83.68100000000001 |
|
- type: map_at_3 |
|
value: 56.967999999999996 |
|
- type: map_at_5 |
|
value: 70.098 |
|
- type: mrr_at_1 |
|
value: 92.12700000000001 |
|
- type: mrr_at_10 |
|
value: 94.094 |
|
- type: mrr_at_100 |
|
value: 94.161 |
|
- type: mrr_at_1000 |
|
value: 94.164 |
|
- type: mrr_at_20 |
|
value: 94.14 |
|
- type: mrr_at_3 |
|
value: 93.753 |
|
- type: mrr_at_5 |
|
value: 93.98100000000001 |
|
- type: ndcg_at_1 |
|
value: 92.12700000000001 |
|
- type: ndcg_at_10 |
|
value: 87.586 |
|
- type: ndcg_at_100 |
|
value: 90.58500000000001 |
|
- type: ndcg_at_1000 |
|
value: 91.05 |
|
- type: ndcg_at_20 |
|
value: 89.132 |
|
- type: ndcg_at_3 |
|
value: 88.86800000000001 |
|
- type: ndcg_at_5 |
|
value: 87.673 |
|
- type: precision_at_1 |
|
value: 92.12700000000001 |
|
- type: precision_at_10 |
|
value: 43.35 |
|
- type: precision_at_100 |
|
value: 5.06 |
|
- type: precision_at_1000 |
|
value: 0.517 |
|
- type: precision_at_20 |
|
value: 23.895 |
|
- type: precision_at_3 |
|
value: 77.664 |
|
- type: precision_at_5 |
|
value: 65.231 |
|
- type: recall_at_1 |
|
value: 28.591 |
|
- type: recall_at_10 |
|
value: 86.342 |
|
- type: recall_at_100 |
|
value: 96.274 |
|
- type: recall_at_1000 |
|
value: 98.666 |
|
- type: recall_at_20 |
|
value: 91.741 |
|
- type: recall_at_3 |
|
value: 58.386 |
|
- type: recall_at_5 |
|
value: 72.942 |
|
- type: main_score |
|
value: 87.586 |
|
task: |
|
type: Retrieval |
|
- dataset: |
|
config: default |
|
name: MTEB TNews (default) |
|
revision: latest2023 |
|
split: validation |
|
type: C-MTEB/TNews-classification |
|
metrics: |
|
- type: accuracy |
|
value: 58.057 |
|
- type: accuracy_stderr |
|
value: 0.4056365368159032 |
|
- type: f1 |
|
value: 56.16542257610506 |
|
- type: f1_stderr |
|
value: 0.49560443919264746 |
|
- type: main_score |
|
value: 58.057 |
|
task: |
|
type: Classification |
|
- dataset: |
|
config: default |
|
name: MTEB ThuNewsClusteringP2P (default) |
|
revision: latest2023 |
|
split: test |
|
type: C-MTEB/ThuNewsClusteringP2P |
|
metrics: |
|
- type: v_measure |
|
value: 83.43086890900754 |
|
- type: v_measure_std |
|
value: 1.3242733220406704 |
|
- type: main_score |
|
value: 83.43086890900754 |
|
task: |
|
type: Clustering |
|
- dataset: |
|
config: default |
|
name: MTEB ThuNewsClusteringS2S (default) |
|
revision: latest2023 |
|
split: test |
|
type: C-MTEB/ThuNewsClusteringS2S |
|
metrics: |
|
- type: v_measure |
|
value: 80.17922689954183 |
|
- type: v_measure_std |
|
value: 2.1732975942130612 |
|
- type: main_score |
|
value: 80.17922689954183 |
|
task: |
|
type: Clustering |
|
- dataset: |
|
config: default |
|
name: MTEB VideoRetrieval (default) |
|
revision: latest2023 |
|
split: dev |
|
type: C-MTEB/VideoRetrieval |
|
metrics: |
|
- type: map_at_1 |
|
value: 68.60000000000001 |
|
- type: map_at_10 |
|
value: 77.518 |
|
- type: map_at_100 |
|
value: 77.815 |
|
- type: map_at_1000 |
|
value: 77.82 |
|
- type: map_at_20 |
|
value: 77.73299999999999 |
|
- type: map_at_3 |
|
value: 76.167 |
|
- type: map_at_5 |
|
value: 76.932 |
|
- type: mrr_at_1 |
|
value: 68.60000000000001 |
|
- type: mrr_at_10 |
|
value: 77.518 |
|
- type: mrr_at_100 |
|
value: 77.815 |
|
- type: mrr_at_1000 |
|
value: 77.82 |
|
- type: mrr_at_20 |
|
value: 77.73299999999999 |
|
- type: mrr_at_3 |
|
value: 76.167 |
|
- type: mrr_at_5 |
|
value: 76.932 |
|
- type: ndcg_at_1 |
|
value: 68.60000000000001 |
|
- type: ndcg_at_10 |
|
value: 81.339 |
|
- type: ndcg_at_100 |
|
value: 82.646 |
|
- type: ndcg_at_1000 |
|
value: 82.76599999999999 |
|
- type: ndcg_at_20 |
|
value: 82.107 |
|
- type: ndcg_at_3 |
|
value: 78.569 |
|
- type: ndcg_at_5 |
|
value: 79.937 |
|
- type: precision_at_1 |
|
value: 68.60000000000001 |
|
- type: precision_at_10 |
|
value: 9.31 |
|
- type: precision_at_100 |
|
value: 0.989 |
|
- type: precision_at_1000 |
|
value: 0.1 |
|
- type: precision_at_20 |
|
value: 4.805000000000001 |
|
- type: precision_at_3 |
|
value: 28.499999999999996 |
|
- type: precision_at_5 |
|
value: 17.76 |
|
- type: recall_at_1 |
|
value: 68.60000000000001 |
|
- type: recall_at_10 |
|
value: 93.10000000000001 |
|
- type: recall_at_100 |
|
value: 98.9 |
|
- type: recall_at_1000 |
|
value: 99.8 |
|
- type: recall_at_20 |
|
value: 96.1 |
|
- type: recall_at_3 |
|
value: 85.5 |
|
- type: recall_at_5 |
|
value: 88.8 |
|
- type: main_score |
|
value: 81.339 |
|
task: |
|
type: Retrieval |
|
- dataset: |
|
config: default |
|
name: MTEB Waimai (default) |
|
revision: latest2023 |
|
split: test |
|
type: C-MTEB/waimai-classification |
|
metrics: |
|
- type: accuracy |
|
value: 90.63000000000001 |
|
- type: accuracy_stderr |
|
value: 0.49203658400570216 |
|
- type: ap |
|
value: 77.93466200571231 |
|
- type: ap_stderr |
|
value: 1.2006502477223735 |
|
- type: f1 |
|
value: 89.36361097500829 |
|
- type: f1_stderr |
|
value: 0.43660966359249054 |
|
- type: main_score |
|
value: 90.63000000000001 |
|
task: |
|
type: Classification |
|
tags: |
|
- mteb |
|
--- |
|
|
|
# quark-llm-embedding-8B |
|
|
|
- Chinese Text Embedding Model developed by Alibaba Quark-LLM Team. Details will be published later. |
