metadata
tags:
- mteb
model-index:
- name: piccolo-embedding_mixed2
results:
- task:
type: STS
dataset:
type: C-MTEB/AFQMC
name: MTEB AFQMC
config: default
split: validation
revision: None
metrics:
- type: cos_sim_pearson
value: 56.918538280469875
- type: cos_sim_spearman
value: 60.95597435855258
- type: euclidean_pearson
value: 59.73821610051437
- type: euclidean_spearman
value: 60.956778530262454
- type: manhattan_pearson
value: 59.739675774225475
- type: manhattan_spearman
value: 60.95243600302903
- task:
type: STS
dataset:
type: C-MTEB/ATEC
name: MTEB ATEC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 56.79417977023184
- type: cos_sim_spearman
value: 58.80984726256814
- type: euclidean_pearson
value: 63.42225182281334
- type: euclidean_spearman
value: 58.80957930593542
- type: manhattan_pearson
value: 63.41128425333986
- type: manhattan_spearman
value: 58.80784321716389
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (zh)
config: zh
split: test
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
metrics:
- type: accuracy
value: 50.074000000000005
- type: f1
value: 47.11468271375511
- task:
type: STS
dataset:
type: C-MTEB/BQ
name: MTEB BQ
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 73.3412976021806
- type: cos_sim_spearman
value: 75.0799965464816
- type: euclidean_pearson
value: 73.7874729086686
- type: euclidean_spearman
value: 75.07910973646369
- type: manhattan_pearson
value: 73.7716616949607
- type: manhattan_spearman
value: 75.06089549008017
- task:
type: Clustering
dataset:
type: C-MTEB/CLSClusteringP2P
name: MTEB CLSClusteringP2P
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 60.4206935177474
- task:
type: Clustering
dataset:
type: C-MTEB/CLSClusteringS2S
name: MTEB CLSClusteringS2S
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 49.53654617222264
- task:
type: Reranking
dataset:
type: C-MTEB/CMedQAv1-reranking
name: MTEB CMedQAv1
config: default
split: test
revision: None
metrics:
- type: map
value: 90.96386786978509
- type: mrr
value: 92.8897619047619
- task:
type: Reranking
dataset:
type: C-MTEB/CMedQAv2-reranking
name: MTEB CMedQAv2
config: default
split: test
revision: None
metrics:
- type: map
value: 90.41014127763198
- type: mrr
value: 92.45039682539682
- task:
type: Retrieval
dataset:
type: C-MTEB/CmedqaRetrieval
name: MTEB CmedqaRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 26.901999999999997
- type: map_at_10
value: 40.321
- type: map_at_100
value: 42.176
- type: map_at_1000
value: 42.282
- type: map_at_3
value: 35.882
- type: map_at_5
value: 38.433
- type: mrr_at_1
value: 40.910000000000004
- type: mrr_at_10
value: 49.309999999999995
- type: mrr_at_100
value: 50.239
- type: mrr_at_1000
value: 50.278
- type: mrr_at_3
value: 46.803
- type: mrr_at_5
value: 48.137
- type: ndcg_at_1
value: 40.785
- type: ndcg_at_10
value: 47.14
- type: ndcg_at_100
value: 54.156000000000006
- type: ndcg_at_1000
value: 55.913999999999994
- type: ndcg_at_3
value: 41.669
- type: ndcg_at_5
value: 43.99
- type: precision_at_1
value: 40.785
- type: precision_at_10
value: 10.493
- type: precision_at_100
value: 1.616
- type: precision_at_1000
value: 0.184
- type: precision_at_3
value: 23.723
- type: precision_at_5
value: 17.249
- type: recall_at_1
value: 26.901999999999997
- type: recall_at_10
value: 58.25
- type: recall_at_100
value: 87.10900000000001
- type: recall_at_1000
value: 98.804
- type: recall_at_3
value: 41.804
- type: recall_at_5
value: 48.884
- task:
type: PairClassification
dataset:
type: C-MTEB/CMNLI
name: MTEB Cmnli
config: default
split: validation
revision: None
metrics:
- type: cos_sim_accuracy
value: 86.42212868310283
- type: cos_sim_ap
value: 92.83788702972741
- type: cos_sim_f1
value: 87.08912233141307
- type: cos_sim_precision
value: 84.24388111888112
- type: cos_sim_recall
value: 90.13327098433481
- type: dot_accuracy
value: 86.44618159951895
- type: dot_ap
value: 92.81146275060858
- type: dot_f1
value: 87.06857911250562
- type: dot_precision
value: 83.60232408005164
- type: dot_recall
value: 90.83469721767594
- type: euclidean_accuracy
value: 86.42212868310283
- type: euclidean_ap
value: 92.83805700492603
- type: euclidean_f1
value: 87.08803611738148
- type: euclidean_precision
value: 84.18066768492254
- type: euclidean_recall
value: 90.20341360766892
- type: manhattan_accuracy
value: 86.28983764281419
- type: manhattan_ap
value: 92.82818970981005
- type: manhattan_f1
value: 87.12625521832335
- type: manhattan_precision
value: 84.19101613606628
- type: manhattan_recall
value: 90.27355623100304
- type: max_accuracy
value: 86.44618159951895
- type: max_ap
value: 92.83805700492603
- type: max_f1
value: 87.12625521832335
- task:
type: Retrieval
dataset:
type: C-MTEB/CovidRetrieval
name: MTEB CovidRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 79.215
- type: map_at_10
value: 86.516
- type: map_at_100
value: 86.6
- type: map_at_1000
value: 86.602
- type: map_at_3
value: 85.52
- type: map_at_5
value: 86.136
- type: mrr_at_1
value: 79.663
- type: mrr_at_10
value: 86.541
- type: mrr_at_100
value: 86.625
- type: mrr_at_1000
value: 86.627
- type: mrr_at_3
value: 85.564
- type: mrr_at_5
value: 86.15899999999999
- type: ndcg_at_1
value: 79.663
- type: ndcg_at_10
value: 89.399
- type: ndcg_at_100
value: 89.727
- type: ndcg_at_1000
value: 89.781
- type: ndcg_at_3
value: 87.402
- type: ndcg_at_5
value: 88.479
- type: precision_at_1
value: 79.663
- type: precision_at_10
value: 9.926
- type: precision_at_100
value: 1.006
- type: precision_at_1000
value: 0.101
- type: precision_at_3
value: 31.226
- type: precision_at_5
value: 19.283
- type: recall_at_1
value: 79.215
- type: recall_at_10
value: 98.209
- type: recall_at_100
value: 99.579
- type: recall_at_1000
value: 100
- type: recall_at_3
value: 92.703
- type: recall_at_5
value: 95.364
- task:
type: Retrieval
dataset:
type: C-MTEB/DuRetrieval
name: MTEB DuRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 27.391
- type: map_at_10
value: 82.82000000000001
- type: map_at_100
value: 85.5
- type: map_at_1000
value: 85.533
- type: map_at_3
value: 57.802
- type: map_at_5
value: 72.82600000000001
- type: mrr_at_1
value: 92.80000000000001
- type: mrr_at_10
value: 94.83500000000001
- type: mrr_at_100
value: 94.883
- type: mrr_at_1000
value: 94.884
- type: mrr_at_3
value: 94.542
- type: mrr_at_5
value: 94.729
- type: ndcg_at_1
value: 92.7
- type: ndcg_at_10
value: 89.435
- type: ndcg_at_100
value: 91.78699999999999
- type: ndcg_at_1000
value: 92.083
- type: ndcg_at_3
value: 88.595
- type: ndcg_at_5
value: 87.53
- type: precision_at_1
value: 92.7
- type: precision_at_10
value: 42.4
- type: precision_at_100
value: 4.823
- type: precision_at_1000
value: 0.48900000000000005
- type: precision_at_3
value: 79.133
- type: precision_at_5
value: 66.8
- type: recall_at_1
value: 27.391
- type: recall_at_10
value: 90.069
- type: recall_at_100
value: 97.875
- type: recall_at_1000
value: 99.436
- type: recall_at_3
value: 59.367999999999995
- type: recall_at_5
value: 76.537
- task:
type: Retrieval
dataset:
type: C-MTEB/EcomRetrieval
name: MTEB EcomRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 54.800000000000004
- type: map_at_10
value: 65.289
- type: map_at_100
value: 65.845
- type: map_at_1000
value: 65.853
- type: map_at_3
value: 62.766999999999996
- type: map_at_5
value: 64.252
- type: mrr_at_1
value: 54.800000000000004
- type: mrr_at_10
value: 65.255
- type: mrr_at_100
value: 65.81700000000001
- type: mrr_at_1000
value: 65.824
- type: mrr_at_3
value: 62.683
- type: mrr_at_5
value: 64.248
- type: ndcg_at_1
value: 54.800000000000004
- type: ndcg_at_10
value: 70.498
- type: ndcg_at_100
value: 72.82300000000001
- type: ndcg_at_1000
value: 73.053
- type: ndcg_at_3
value: 65.321
- type: ndcg_at_5
value: 67.998
- type: precision_at_1
value: 54.800000000000004
- type: precision_at_10
value: 8.690000000000001
- type: precision_at_100
value: 0.97
- type: precision_at_1000
value: 0.099
- type: precision_at_3
value: 24.233
- type: precision_at_5
value: 15.840000000000002
- type: recall_at_1
value: 54.800000000000004
- type: recall_at_10
value: 86.9
- type: recall_at_100
value: 97
- type: recall_at_1000
value: 98.9
- type: recall_at_3
value: 72.7
- type: recall_at_5
value: 79.2
- task:
type: Classification
dataset:
type: C-MTEB/IFlyTek-classification
name: MTEB IFlyTek
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 51.758368603308966
- type: f1
value: 40.249503783871596
- task:
type: Classification
dataset:
type: C-MTEB/JDReview-classification
name: MTEB JDReview
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 89.08067542213884
- type: ap
value: 60.31281895139249
- type: f1
value: 84.20883153932607
- task:
type: STS
dataset:
type: C-MTEB/LCQMC
name: MTEB LCQMC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 74.04193577551248
- type: cos_sim_spearman
value: 79.81875884845549
- type: euclidean_pearson
value: 80.02581187503708
- type: euclidean_spearman
value: 79.81877215060574
- type: manhattan_pearson
value: 80.01767830530258
- type: manhattan_spearman
value: 79.81178852172727
- task:
type: Reranking
dataset:
type: C-MTEB/Mmarco-reranking
name: MTEB MMarcoReranking
config: default
split: dev
revision: None
metrics:
- type: map
value: 39.90939429947956
- type: mrr
value: 39.71071428571429
- task:
type: Retrieval
dataset:
type: C-MTEB/MMarcoRetrieval
name: MTEB MMarcoRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 68.485
- type: map_at_10
value: 78.27199999999999
- type: map_at_100
value: 78.54100000000001
- type: map_at_1000
value: 78.546
- type: map_at_3
value: 76.339
- type: map_at_5
value: 77.61099999999999
- type: mrr_at_1
value: 70.80199999999999
- type: mrr_at_10
value: 78.901
- type: mrr_at_100
value: 79.12400000000001
- type: mrr_at_1000
value: 79.128
- type: mrr_at_3
value: 77.237
- type: mrr_at_5
value: 78.323
- type: ndcg_at_1
value: 70.759
- type: ndcg_at_10
value: 82.191
- type: ndcg_at_100
value: 83.295
- type: ndcg_at_1000
value: 83.434
- type: ndcg_at_3
value: 78.57600000000001
- type: ndcg_at_5
value: 80.715
- type: precision_at_1
value: 70.759
- type: precision_at_10
value: 9.951
- type: precision_at_100
value: 1.049
- type: precision_at_1000
value: 0.106
- type: precision_at_3
value: 29.660999999999998
- type: precision_at_5
value: 18.94
- type: recall_at_1
value: 68.485
- type: recall_at_10
value: 93.65
- type: recall_at_100
value: 98.434
- type: recall_at_1000
value: 99.522
- type: recall_at_3
value: 84.20100000000001
- type: recall_at_5
value: 89.261
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (zh-CN)
config: zh-CN
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 77.45460659045055
- type: f1
value: 73.84987702455533
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (zh-CN)
config: zh-CN
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 85.29926025554808
- type: f1
value: 84.40636286569843
- task:
type: Retrieval
dataset:
type: C-MTEB/MedicalRetrieval
name: MTEB MedicalRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 57.599999999999994
- type: map_at_10
value: 64.691
- type: map_at_100
value: 65.237
- type: map_at_1000
value: 65.27
- type: map_at_3
value: 62.733000000000004
- type: map_at_5
value: 63.968
- type: mrr_at_1
value: 58.099999999999994
- type: mrr_at_10
value: 64.952
- type: mrr_at_100
value: 65.513
- type: mrr_at_1000
value: 65.548
- type: mrr_at_3
value: 63
- type: mrr_at_5
value: 64.235
- type: ndcg_at_1
value: 57.599999999999994
- type: ndcg_at_10
value: 68.19
- type: ndcg_at_100
value: 70.98400000000001
- type: ndcg_at_1000
value: 71.811
- type: ndcg_at_3
value: 64.276
- type: ndcg_at_5
value: 66.47999999999999
- type: precision_at_1
value: 57.599999999999994
- type: precision_at_10
value: 7.920000000000001
- type: precision_at_100
value: 0.9259999999999999
- type: precision_at_1000
value: 0.099
- type: precision_at_3
value: 22.900000000000002
- type: precision_at_5
value: 14.799999999999999
- type: recall_at_1
value: 57.599999999999994
- type: recall_at_10
value: 79.2
- type: recall_at_100
value: 92.60000000000001
- type: recall_at_1000
value: 99
- type: recall_at_3
value: 68.7
- type: recall_at_5
value: 74
- task:
type: Classification
dataset:
type: C-MTEB/MultilingualSentiment-classification
name: MTEB MultilingualSentiment
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 79.45
- type: f1
value: 79.25610578280538
- task:
type: PairClassification
dataset:
type: C-MTEB/OCNLI
name: MTEB Ocnli
config: default
split: validation
revision: None
metrics:
- type: cos_sim_accuracy
value: 85.43584190579317
- type: cos_sim_ap
value: 90.89979725191012
- type: cos_sim_f1
value: 86.48383937316358
- type: cos_sim_precision
value: 80.6392694063927
- type: cos_sim_recall
value: 93.24181626187962
- type: dot_accuracy
value: 85.38170005414185
- type: dot_ap
value: 90.87532457866699
- type: dot_f1
value: 86.48383937316358
- type: dot_precision
value: 80.6392694063927
- type: dot_recall
value: 93.24181626187962
- type: euclidean_accuracy
value: 85.43584190579317
- type: euclidean_ap
value: 90.90126652086121
- type: euclidean_f1
value: 86.48383937316358
- type: euclidean_precision
value: 80.6392694063927
- type: euclidean_recall
value: 93.24181626187962
- type: manhattan_accuracy
value: 85.43584190579317
- type: manhattan_ap
value: 90.87896997853466
- type: manhattan_f1
value: 86.47581441263573
- type: manhattan_precision
value: 81.18628359592215
- type: manhattan_recall
value: 92.5026399155227
- type: max_accuracy
value: 85.43584190579317
- type: max_ap
value: 90.90126652086121
- type: max_f1
value: 86.48383937316358
- task:
type: Classification
dataset:
type: C-MTEB/OnlineShopping-classification
name: MTEB OnlineShopping
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 94.9
- type: ap
value: 93.1468223150745
- type: f1
value: 94.88918689508299
- task:
type: STS
dataset:
type: C-MTEB/PAWSX
name: MTEB PAWSX
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 40.4831743182905
- type: cos_sim_spearman
value: 47.4163675550491
- type: euclidean_pearson
value: 46.456319899274924
- type: euclidean_spearman
value: 47.41567079730661
- type: manhattan_pearson
value: 46.48561639930895
- type: manhattan_spearman
value: 47.447721653461215
- task:
type: STS
dataset:
type: C-MTEB/QBQTC
name: MTEB QBQTC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 42.96423587663398
- type: cos_sim_spearman
value: 45.13742225167858
- type: euclidean_pearson
value: 39.275452114075435
- type: euclidean_spearman
value: 45.137763540967406
- type: manhattan_pearson
value: 39.24797626417764
- type: manhattan_spearman
value: 45.13817773119268
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (zh)
config: zh
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 66.26687809086202
- type: cos_sim_spearman
value: 66.9569145816897
- type: euclidean_pearson
value: 65.72390780809788
- type: euclidean_spearman
value: 66.95406938095539
- type: manhattan_pearson
value: 65.6220809000381
- type: manhattan_spearman
value: 66.88531036320953
- task:
type: STS
dataset:
type: C-MTEB/STSB
name: MTEB STSB
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 80.30831700726195
- type: cos_sim_spearman
value: 82.05184068558792
- type: euclidean_pearson
value: 81.73198597791563
- type: euclidean_spearman
value: 82.05326103582206
- type: manhattan_pearson
value: 81.70886400949136
- type: manhattan_spearman
value: 82.03473274756037
- task:
type: Reranking
dataset:
type: C-MTEB/T2Reranking
name: MTEB T2Reranking
config: default
split: dev
revision: None
metrics:
- type: map
value: 69.03398835347575
- type: mrr
value: 79.9212528613341
- task:
type: Retrieval
dataset:
type: C-MTEB/T2Retrieval
name: MTEB T2Retrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 27.515
- type: map_at_10
value: 77.40599999999999
- type: map_at_100
value: 81.087
- type: map_at_1000
value: 81.148
- type: map_at_3
value: 54.327000000000005
- type: map_at_5
value: 66.813
- type: mrr_at_1
value: 89.764
- type: mrr_at_10
value: 92.58
- type: mrr_at_100
value: 92.663
- type: mrr_at_1000
value: 92.666
- type: mrr_at_3
value: 92.15299999999999
- type: mrr_at_5
value: 92.431
- type: ndcg_at_1
value: 89.777
- type: ndcg_at_10
value: 85.013
- type: ndcg_at_100
value: 88.62100000000001
- type: ndcg_at_1000
value: 89.184
- type: ndcg_at_3
value: 86.19200000000001
- type: ndcg_at_5
value: 84.909
- type: precision_at_1
value: 89.777
- type: precision_at_10
value: 42.218
- type: precision_at_100
value: 5.032
- type: precision_at_1000
value: 0.517
- type: precision_at_3
value: 75.335
- type: precision_at_5
value: 63.199000000000005
- type: recall_at_1
value: 27.515
- type: recall_at_10
value: 84.258
- type: recall_at_100
value: 95.908
- type: recall_at_1000
value: 98.709
- type: recall_at_3
value: 56.189
- type: recall_at_5
value: 70.50800000000001
- task:
type: Classification
dataset:
type: C-MTEB/TNews-classification
name: MTEB TNews
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 54.635999999999996
- type: f1
value: 52.63073912739558
- task:
type: Clustering
dataset:
type: C-MTEB/ThuNewsClusteringP2P
name: MTEB ThuNewsClusteringP2P
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 78.75676284855221
- task:
type: Clustering
dataset:
type: C-MTEB/ThuNewsClusteringS2S
name: MTEB ThuNewsClusteringS2S
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 71.95583733802839
- task:
type: Retrieval
dataset:
type: C-MTEB/VideoRetrieval
name: MTEB VideoRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 64.9
- type: map_at_10
value: 75.622
- type: map_at_100
value: 75.93900000000001
- type: map_at_1000
value: 75.93900000000001
- type: map_at_3
value: 73.933
- type: map_at_5
value: 74.973
- type: mrr_at_1
value: 65
- type: mrr_at_10
value: 75.676
- type: mrr_at_100
value: 75.994
- type: mrr_at_1000
value: 75.994
- type: mrr_at_3
value: 74.05000000000001
- type: mrr_at_5
value: 75.03999999999999
- type: ndcg_at_1
value: 64.9
- type: ndcg_at_10
value: 80.08999999999999
- type: ndcg_at_100
value: 81.44500000000001
- type: ndcg_at_1000
value: 81.45599999999999
- type: ndcg_at_3
value: 76.688
- type: ndcg_at_5
value: 78.53
- type: precision_at_1
value: 64.9
- type: precision_at_10
value: 9.379999999999999
- type: precision_at_100
value: 0.997
- type: precision_at_1000
value: 0.1
- type: precision_at_3
value: 28.199999999999996
- type: precision_at_5
value: 17.8
- type: recall_at_1
value: 64.9
- type: recall_at_10
value: 93.8
- type: recall_at_100
value: 99.7
- type: recall_at_1000
value: 99.8
- type: recall_at_3
value: 84.6
- type: recall_at_5
value: 89
- task:
type: Classification
dataset:
type: C-MTEB/waimai-classification
name: MTEB Waimai
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 89.34
- type: ap
value: 75.20638024616892
- type: f1
value: 87.88648489072128
xiaobu-embedding-v2
基于piccolo-embedding[1]，主要改动如下：
- 合成数据替换为xiaobu-embedding-v1[2]所积累数据
- 在circle_loss[3]视角下统一处理CMTEB的6类问题，最大优势是可充分利用原始数据集中的多个正例，其次是可一定程度上避免考虑多个不同loss之间的权重问题
Usage (Sentence-Transformers)
pip install -U sentence-transformers
相似度计算：
from sentence_transformers import SentenceTransformer
sentences_1 = ["样例数据-1", "样例数据-2"]
sentences_2 = ["样例数据-3", "样例数据-4"]
model = SentenceTransformer('lier007/xiaobu-embedding-v2')
embeddings_1 = model.encode(sentences_1, normalize_embeddings=True)
embeddings_2 = model.encode(sentences_2, normalize_embeddings=True)
similarity = embeddings_1 @ embeddings_2.T
print(similarity)