metadata
tags:
- mteb
model-index:
- name: mist-zh
results:
- task:
type: STS
dataset:
type: C-MTEB/AFQMC
name: MTEB AFQMC
config: default
split: validation
revision: None
metrics:
- type: cos_sim_pearson
value: 44.80910972039708
- type: cos_sim_spearman
value: 46.97947004057185
- type: euclidean_pearson
value: 45.36774158404125
- type: euclidean_spearman
value: 46.97947004232487
- type: manhattan_pearson
value: 45.23486628014998
- type: manhattan_spearman
value: 46.87721960765866
- task:
type: STS
dataset:
type: C-MTEB/ATEC
name: MTEB ATEC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 49.5294624928126
- type: cos_sim_spearman
value: 51.34771777448503
- type: euclidean_pearson
value: 53.56859824288157
- type: euclidean_spearman
value: 51.34771439634126
- type: manhattan_pearson
value: 53.581640877132685
- type: manhattan_spearman
value: 51.349656519071274
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (zh)
config: zh
split: test
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
metrics:
- type: accuracy
value: 39.318
- type: f1
value: 37.37720144558489
- task:
type: STS
dataset:
type: C-MTEB/BQ
name: MTEB BQ
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 62.12016334764962
- type: cos_sim_spearman
value: 65.08208654969742
- type: euclidean_pearson
value: 63.53078822303454
- type: euclidean_spearman
value: 65.0820865487212
- type: manhattan_pearson
value: 63.510532363654725
- type: manhattan_spearman
value: 65.06622789125241
- task:
type: Clustering
dataset:
type: C-MTEB/CLSClusteringP2P
name: MTEB CLSClusteringP2P
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 39.5071157612481
- task:
type: Clustering
dataset:
type: C-MTEB/CLSClusteringS2S
name: MTEB CLSClusteringS2S
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 37.99964332311132
- task:
type: Reranking
dataset:
type: C-MTEB/CMedQAv1-reranking
name: MTEB CMedQAv1
config: default
split: test
revision: None
metrics:
- type: map
value: 84.67010533089491
- type: mrr
value: 86.99488095238095
- task:
type: Reranking
dataset:
type: C-MTEB/CMedQAv2-reranking
name: MTEB CMedQAv2
config: default
split: test
revision: None
metrics:
- type: map
value: 85.27288868896477
- type: mrr
value: 87.5929761904762
- task:
type: Retrieval
dataset:
type: C-MTEB/CmedqaRetrieval
name: MTEB CmedqaRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 23.949
- type: map_at_10
value: 35.394
- type: map_at_100
value: 37.235
- type: map_at_1000
value: 37.364999999999995
- type: map_at_3
value: 31.433
- type: map_at_5
value: 33.668
- type: mrr_at_1
value: 36.834
- type: mrr_at_10
value: 44.451
- type: mrr_at_100
value: 45.445
- type: mrr_at_1000
value: 45.501000000000005
- type: mrr_at_3
value: 42.010999999999996
- type: mrr_at_5
value: 43.34
- type: ndcg_at_1
value: 36.834
- type: ndcg_at_10
value: 41.803000000000004
- type: ndcg_at_100
value: 49.091
- type: ndcg_at_1000
value: 51.474
- type: ndcg_at_3
value: 36.736000000000004
- type: ndcg_at_5
value: 38.868
- type: precision_at_1
value: 36.834
- type: precision_at_10
value: 9.354999999999999
- type: precision_at_100
value: 1.5310000000000001
- type: precision_at_1000
value: 0.183
- type: precision_at_3
value: 20.78
- type: precision_at_5
value: 15.238999999999999
- type: recall_at_1
value: 23.949
- type: recall_at_10
value: 51.68000000000001
- type: recall_at_100
value: 81.938
- type: recall_at_1000
value: 98.091
- type: recall_at_3
value: 36.408
- type: recall_at_5
value: 42.952
- task:
type: PairClassification
dataset:
type: C-MTEB/CMNLI
name: MTEB Cmnli
config: default
split: validation
revision: None
metrics:
- type: cos_sim_accuracy
value: 76.24774503908598
- type: cos_sim_ap
value: 84.76081551540754
- type: cos_sim_f1
value: 77.76321537789427
- type: cos_sim_precision
value: 72.96577167452347
- type: cos_sim_recall
value: 83.23591302314706
- type: dot_accuracy
value: 76.24774503908598
- type: dot_ap
value: 84.75968761251127
- type: dot_f1
value: 77.76321537789427
- type: dot_precision
value: 72.96577167452347
- type: dot_recall
value: 83.23591302314706
- type: euclidean_accuracy
value: 76.24774503908598
- type: euclidean_ap
value: 84.7608250840413
- type: euclidean_f1
value: 77.76321537789427
- type: euclidean_precision
value: 72.96577167452347
- type: euclidean_recall
value: 83.23591302314706
- type: manhattan_accuracy
value: 76.19963920625375
- type: manhattan_ap
value: 84.76313920535411
- type: manhattan_f1
value: 77.74253527288636
- type: manhattan_precision
value: 73.0374023838882
- type: manhattan_recall
value: 83.09562777647884
- type: max_accuracy
value: 76.24774503908598
- type: max_ap
value: 84.76313920535411
- type: max_f1
value: 77.76321537789427
- task:
type: Retrieval
dataset:
type: C-MTEB/CovidRetrieval
name: MTEB CovidRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 66.149
- type: map_at_10
value: 75.22999999999999
- type: map_at_100
value: 75.536
- type: map_at_1000
value: 75.542
- type: map_at_3
value: 73.384
- type: map_at_5
value: 74.459
- type: mrr_at_1
value: 66.28
- type: mrr_at_10
value: 75.232
- type: mrr_at_100
value: 75.52799999999999
- type: mrr_at_1000
value: 75.534
- type: mrr_at_3
value: 73.446
- type: mrr_at_5
value: 74.473
- type: ndcg_at_1
value: 66.386
- type: ndcg_at_10
value: 79.295
- type: ndcg_at_100
value: 80.741
- type: ndcg_at_1000
value: 80.891
- type: ndcg_at_3
value: 75.613
- type: ndcg_at_5
value: 77.46300000000001
- type: precision_at_1
value: 66.386
- type: precision_at_10
value: 9.283
- type: precision_at_100
value: 0.996
- type: precision_at_1000
value: 0.101
- type: precision_at_3
value: 27.503
- type: precision_at_5
value: 17.408
- type: recall_at_1
value: 66.149
- type: recall_at_10
value: 91.886
- type: recall_at_100
value: 98.52499999999999
- type: recall_at_1000
value: 99.684
- type: recall_at_3
value: 81.849
- type: recall_at_5
value: 86.275
- task:
type: Retrieval
dataset:
type: C-MTEB/DuRetrieval
name: MTEB DuRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 25.166
- type: map_at_10
value: 78.805
- type: map_at_100
value: 81.782
- type: map_at_1000
value: 81.818
- type: map_at_3
value: 54.226
- type: map_at_5
value: 68.783
- type: mrr_at_1
value: 88.6
- type: mrr_at_10
value: 92.244
- type: mrr_at_100
value: 92.31899999999999
- type: mrr_at_1000
value: 92.321
- type: mrr_at_3
value: 91.867
- type: mrr_at_5
value: 92.119
- type: ndcg_at_1
value: 88.6
- type: ndcg_at_10
value: 86.432
- type: ndcg_at_100
value: 89.357
- type: ndcg_at_1000
value: 89.688
- type: ndcg_at_3
value: 84.90299999999999
- type: ndcg_at_5
value: 84.137
- type: precision_at_1
value: 88.6
- type: precision_at_10
value: 41.685
- type: precision_at_100
value: 4.811
- type: precision_at_1000
value: 0.48900000000000005
- type: precision_at_3
value: 76.44999999999999
- type: precision_at_5
value: 64.87
- type: recall_at_1
value: 25.166
- type: recall_at_10
value: 88.227
- type: recall_at_100
value: 97.597
- type: recall_at_1000
value: 99.359
- type: recall_at_3
value: 56.946
- type: recall_at_5
value: 74.261
- task:
type: Retrieval
dataset:
type: C-MTEB/EcomRetrieval
name: MTEB EcomRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 48.3
- type: map_at_10
value: 57.635999999999996
- type: map_at_100
value: 58.306000000000004
- type: map_at_1000
value: 58.326
- type: map_at_3
value: 54.900000000000006
- type: map_at_5
value: 56.620000000000005
- type: mrr_at_1
value: 48.3
- type: mrr_at_10
value: 57.635999999999996
- type: mrr_at_100
value: 58.306000000000004
- type: mrr_at_1000
value: 58.326
- type: mrr_at_3
value: 54.900000000000006
- type: mrr_at_5
value: 56.620000000000005
- type: ndcg_at_1
value: 48.3
- type: ndcg_at_10
value: 62.638000000000005
- type: ndcg_at_100
value: 65.726
- type: ndcg_at_1000
value: 66.253
- type: ndcg_at_3
value: 57.081
- type: ndcg_at_5
value: 60.217
- type: precision_at_1
value: 48.3
- type: precision_at_10
value: 7.85
- type: precision_at_100
value: 0.9249999999999999
- type: precision_at_1000
value: 0.097
- type: precision_at_3
value: 21.133
- type: precision_at_5
value: 14.219999999999999
- type: recall_at_1
value: 48.3
- type: recall_at_10
value: 78.5
- type: recall_at_100
value: 92.5
- type: recall_at_1000
value: 96.6
- type: recall_at_3
value: 63.4
- type: recall_at_5
value: 71.1
- task:
type: Classification
dataset:
type: C-MTEB/IFlyTek-classification
name: MTEB IFlyTek
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 47.9646017699115
- type: f1
value: 35.03552351349023
- task:
type: Classification
dataset:
type: C-MTEB/JDReview-classification
name: MTEB JDReview
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 84.8968105065666
- type: ap
value: 52.564605306946774
- type: f1
value: 79.59880155481291
- task:
type: STS
dataset:
type: C-MTEB/LCQMC
name: MTEB LCQMC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 70.03662039861051
- type: cos_sim_spearman
value: 76.9642260444222
- type: euclidean_pearson
value: 75.47376966815843
- type: euclidean_spearman
value: 76.9642282583736
- type: manhattan_pearson
value: 75.45535385433548
- type: manhattan_spearman
value: 76.94609742735338
- task:
type: Retrieval
dataset:
type: C-MTEB/MMarcoRetrieval
name: MTEB MMarcoRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 65.604
- type: map_at_10
value: 74.522
- type: map_at_100
value: 74.878
- type: map_at_1000
value: 74.889
- type: map_at_3
value: 72.61
- type: map_at_5
value: 73.882
- type: mrr_at_1
value: 67.75099999999999
- type: mrr_at_10
value: 75.08399999999999
- type: mrr_at_100
value: 75.402
- type: mrr_at_1000
value: 75.412
- type: mrr_at_3
value: 73.446
- type: mrr_at_5
value: 74.531
- type: ndcg_at_1
value: 67.75099999999999
- type: ndcg_at_10
value: 78.172
- type: ndcg_at_100
value: 79.753
- type: ndcg_at_1000
value: 80.06400000000001
- type: ndcg_at_3
value: 74.607
- type: ndcg_at_5
value: 76.728
- type: precision_at_1
value: 67.75099999999999
- type: precision_at_10
value: 9.443999999999999
- type: precision_at_100
value: 1.023
- type: precision_at_1000
value: 0.105
- type: precision_at_3
value: 28.009
- type: precision_at_5
value: 17.934
- type: recall_at_1
value: 65.604
- type: recall_at_10
value: 88.84100000000001
- type: recall_at_100
value: 95.954
- type: recall_at_1000
value: 98.425
- type: recall_at_3
value: 79.497
- type: recall_at_5
value: 84.515
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (zh-CN)
config: zh-CN
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 67.64963012777405
- type: f1
value: 65.01092085388518
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (zh-CN)
config: zh-CN
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 72.9724277067922
- type: f1
value: 72.48003852874602
- task:
type: Retrieval
dataset:
type: C-MTEB/MedicalRetrieval
name: MTEB MedicalRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 48.9
- type: map_at_10
value: 55.189
- type: map_at_100
value: 55.687
- type: map_at_1000
value: 55.74400000000001
- type: map_at_3
value: 53.75
- type: map_at_5
value: 54.555
- type: mrr_at_1
value: 49.1
- type: mrr_at_10
value: 55.289
- type: mrr_at_100
value: 55.788000000000004
- type: mrr_at_1000
value: 55.845
- type: mrr_at_3
value: 53.849999999999994
- type: mrr_at_5
value: 54.655
- type: ndcg_at_1
value: 48.9
- type: ndcg_at_10
value: 58.275
- type: ndcg_at_100
value: 60.980000000000004
- type: ndcg_at_1000
value: 62.672000000000004
- type: ndcg_at_3
value: 55.282
- type: ndcg_at_5
value: 56.749
- type: precision_at_1
value: 48.9
- type: precision_at_10
value: 6.800000000000001
- type: precision_at_100
value: 0.8130000000000001
- type: precision_at_1000
value: 0.095
- type: precision_at_3
value: 19.900000000000002
- type: precision_at_5
value: 12.659999999999998
- type: recall_at_1
value: 48.9
- type: recall_at_10
value: 68
- type: recall_at_100
value: 81.3
- type: recall_at_1000
value: 95
- type: recall_at_3
value: 59.699999999999996
- type: recall_at_5
value: 63.3
- task:
type: Classification
dataset:
type: C-MTEB/MultilingualSentiment-classification
name: MTEB MultilingualSentiment
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 71.53666666666668
- type: f1
value: 70.74267338218574
- task:
type: PairClassification
dataset:
type: C-MTEB/OCNLI
name: MTEB Ocnli
config: default
split: validation
revision: None
metrics:
- type: cos_sim_accuracy
value: 70.43854899837575
- type: cos_sim_ap
value: 75.25713109733296
- type: cos_sim_f1
value: 73.18777292576418
- type: cos_sim_precision
value: 62.397617274758
- type: cos_sim_recall
value: 88.48996832101372
- type: dot_accuracy
value: 70.43854899837575
- type: dot_ap
value: 75.25713109733296
- type: dot_f1
value: 73.18777292576418
- type: dot_precision
value: 62.397617274758
- type: dot_recall
value: 88.48996832101372
- type: euclidean_accuracy
value: 70.43854899837575
- type: euclidean_ap
value: 75.25713109733296
- type: euclidean_f1
value: 73.18777292576418
- type: euclidean_precision
value: 62.397617274758
- type: euclidean_recall
value: 88.48996832101372
- type: manhattan_accuracy
value: 70.60097455332972
- type: manhattan_ap
value: 75.22177995740668
- type: manhattan_f1
value: 73.13750532141337
- type: manhattan_precision
value: 61.26961483594865
- type: manhattan_recall
value: 90.70749736008447
- type: max_accuracy
value: 70.60097455332972
- type: max_ap
value: 75.25713109733296
- type: max_f1
value: 73.18777292576418
- task:
type: Classification
dataset:
type: C-MTEB/OnlineShopping-classification
name: MTEB OnlineShopping
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 91.3
- type: ap
value: 89.03601366589187
- type: f1
value: 91.28612226957141
- task:
type: STS
dataset:
type: C-MTEB/PAWSX
name: MTEB PAWSX
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 24.254041798082984
- type: cos_sim_spearman
value: 30.029755057178846
- type: euclidean_pearson
value: 30.394005237465905
- type: euclidean_spearman
value: 30.029751825186153
- type: manhattan_pearson
value: 30.400683181995863
- type: manhattan_spearman
value: 29.981240616043326
- task:
type: STS
dataset:
type: C-MTEB/QBQTC
name: MTEB QBQTC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 35.09911024323138
- type: cos_sim_spearman
value: 37.49790006053554
- type: euclidean_pearson
value: 35.65689785105493
- type: euclidean_spearman
value: 37.498032509597344
- type: manhattan_pearson
value: 35.68350134483341
- type: manhattan_spearman
value: 37.54046578100128
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (zh)
config: zh
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 68.26707578158273
- type: cos_sim_spearman
value: 69.19741429899995
- type: euclidean_pearson
value: 68.53026048034656
- type: euclidean_spearman
value: 69.1974135636389
- type: manhattan_pearson
value: 70.02306646353263
- type: manhattan_spearman
value: 70.46158498712836
- task:
type: STS
dataset:
type: C-MTEB/STSB
name: MTEB STSB
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 78.88749955421177
- type: cos_sim_spearman
value: 79.56695106617856
- type: euclidean_pearson
value: 79.13787024514338
- type: euclidean_spearman
value: 79.56690827015423
- type: manhattan_pearson
value: 79.08154812411563
- type: manhattan_spearman
value: 79.52391077945943
- task:
type: Reranking
dataset:
type: C-MTEB/T2Reranking
name: MTEB T2Reranking
config: default
split: dev
revision: None
metrics:
- type: map
value: 65.78663254562939
- type: mrr
value: 74.9786877626248
- task:
type: Retrieval
dataset:
type: C-MTEB/T2Retrieval
name: MTEB T2Retrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 26.169999999999998
- type: map_at_10
value: 74.009
- type: map_at_100
value: 77.788
- type: map_at_1000
value: 77.866
- type: map_at_3
value: 51.861000000000004
- type: map_at_5
value: 63.775000000000006
- type: mrr_at_1
value: 87.748
- type: mrr_at_10
value: 90.737
- type: mrr_at_100
value: 90.84400000000001
- type: mrr_at_1000
value: 90.849
- type: mrr_at_3
value: 90.257
- type: mrr_at_5
value: 90.54299999999999
- type: ndcg_at_1
value: 87.748
- type: ndcg_at_10
value: 82.114
- type: ndcg_at_100
value: 86.148
- type: ndcg_at_1000
value: 86.913
- type: ndcg_at_3
value: 83.54599999999999
- type: ndcg_at_5
value: 81.987
- type: precision_at_1
value: 87.748
- type: precision_at_10
value: 41.076
- type: precision_at_100
value: 4.976
- type: precision_at_1000
value: 0.515
- type: precision_at_3
value: 73.282
- type: precision_at_5
value: 61.351
- type: recall_at_1
value: 26.169999999999998
- type: recall_at_10
value: 81.292
- type: recall_at_100
value: 94.285
- type: recall_at_1000
value: 98.221
- type: recall_at_3
value: 53.824000000000005
- type: recall_at_5
value: 67.547
- task:
type: Classification
dataset:
type: C-MTEB/TNews-classification
name: MTEB TNews
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 51.564
- type: f1
value: 49.711462885083286
- task:
type: Clustering
dataset:
type: C-MTEB/ThuNewsClusteringP2P
name: MTEB ThuNewsClusteringP2P
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 62.57078038998942
- task:
type: Clustering
dataset:
type: C-MTEB/ThuNewsClusteringS2S
name: MTEB ThuNewsClusteringS2S
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 57.842602165392144
- task:
type: Retrieval
dataset:
type: C-MTEB/VideoRetrieval
name: MTEB VideoRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 52
- type: map_at_10
value: 62.932
- type: map_at_100
value: 63.471999999999994
- type: map_at_1000
value: 63.483999999999995
- type: map_at_3
value: 60.516999999999996
- type: map_at_5
value: 62.097
- type: mrr_at_1
value: 52
- type: mrr_at_10
value: 62.932
- type: mrr_at_100
value: 63.471999999999994
- type: mrr_at_1000
value: 63.483999999999995
- type: mrr_at_3
value: 60.516999999999996
- type: mrr_at_5
value: 62.097
- type: ndcg_at_1
value: 52
- type: ndcg_at_10
value: 67.963
- type: ndcg_at_100
value: 70.598
- type: ndcg_at_1000
value: 70.896
- type: ndcg_at_3
value: 63.144
- type: ndcg_at_5
value: 65.988
- type: precision_at_1
value: 52
- type: precision_at_10
value: 8.36
- type: precision_at_100
value: 0.959
- type: precision_at_1000
value: 0.098
- type: precision_at_3
value: 23.567
- type: precision_at_5
value: 15.52
- type: recall_at_1
value: 52
- type: recall_at_10
value: 83.6
- type: recall_at_100
value: 95.89999999999999
- type: recall_at_1000
value: 98.2
- type: recall_at_3
value: 70.7
- type: recall_at_5
value: 77.60000000000001
- task:
type: Classification
dataset:
type: C-MTEB/waimai-classification
name: MTEB Waimai
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 86.65999999999998
- type: ap
value: 69.91988858863054
- type: f1
value: 84.92982698422784
- task:
type: Reranking
dataset:
type: C-MTEB/Mmarco-reranking
name: MTEB MMarcoReranking
config: default
split: dev
revision: None
metrics:
- type: map
value: 27.838972963193314
- type: mrr
value: 26.65238095238095