diff --git a/results/all-MiniLM-L6-v2-self-optimum-q8/ArxivClusteringP2P.json b/results/all-MiniLM-L6-v2-self-optimum-q8/ArxivClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..87ddfa8e92ff760095f15bc4696cbc9d53b20837 --- /dev/null +++ b/results/all-MiniLM-L6-v2-self-optimum-q8/ArxivClusteringP2P.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "a122ad7f3f0291bf49cc6f4d32aa80929df69d5d", + "mteb_dataset_name": "ArxivClusteringP2P", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 14373.08, + "v_measure": 0.4618034932273851, + "v_measure_std": 0.14005599148945322 + } +} \ No newline at end of file diff --git a/results/all-MiniLM-L6-v2-self-optimum-q8/ArxivClusteringS2S.json b/results/all-MiniLM-L6-v2-self-optimum-q8/ArxivClusteringS2S.json new file mode 100644 index 0000000000000000000000000000000000000000..4e0719c8409757bcdbfbd7837e3d884278a42fb4 --- /dev/null +++ b/results/all-MiniLM-L6-v2-self-optimum-q8/ArxivClusteringS2S.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "f910caf1a6075f7329cdf8c1a6135696f37dbd53", + "mteb_dataset_name": "ArxivClusteringS2S", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 1068.65, + "v_measure": 0.3661110691976883, + "v_measure_std": 0.1428983905125625 + } +} \ No newline at end of file diff --git a/results/all-MiniLM-L6-v2-self-optimum-q8/BiorxivClusteringP2P.json b/results/all-MiniLM-L6-v2-self-optimum-q8/BiorxivClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..17bf9c92aa718a5d54de655ce0a17e8be9f9e251 --- /dev/null +++ b/results/all-MiniLM-L6-v2-self-optimum-q8/BiorxivClusteringP2P.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "65b79d1d13f80053f67aca9498d9402c2d9f1f40", + "mteb_dataset_name": "BiorxivClusteringP2P", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 2027.45, + "v_measure": 0.3826666857854354, + "v_measure_std": 0.009760375786673237 + } +} \ No newline at end of file diff --git a/results/all-MiniLM-L6-v2-self-optimum-q8/BiorxivClusteringS2S.json b/results/all-MiniLM-L6-v2-self-optimum-q8/BiorxivClusteringS2S.json new file mode 100644 index 0000000000000000000000000000000000000000..1a06697b7d5ca507176afd8d5fa10afa670dbbd3 --- /dev/null +++ b/results/all-MiniLM-L6-v2-self-optimum-q8/BiorxivClusteringS2S.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "258694dd0231531bc1fd9de6ceb52a0853c6d908", + "mteb_dataset_name": "BiorxivClusteringS2S", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 128.18, + "v_measure": 0.32225968859400944, + "v_measure_std": 0.010394710148885584 + } +} \ No newline at end of file diff --git a/results/all-MiniLM-L6-v2-self-optimum-q8/MedrxivClusteringP2P.json b/results/all-MiniLM-L6-v2-self-optimum-q8/MedrxivClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..32eccf41c33a97486d2eb5bf56da3df32afc0cc4 --- /dev/null +++ b/results/all-MiniLM-L6-v2-self-optimum-q8/MedrxivClusteringP2P.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "e7a26af6f3ae46b30dde8737f02c07b1505bcc73", + "mteb_dataset_name": "MedrxivClusteringP2P", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 1258.68, + "v_measure": 0.3214793378259306, + "v_measure_std": 0.010193796252019919 + } +} \ No newline at end of file diff --git a/results/all-MiniLM-L6-v2-self-optimum-q8/MedrxivClusteringS2S.json b/results/all-MiniLM-L6-v2-self-optimum-q8/MedrxivClusteringS2S.json new file mode 100644 index 0000000000000000000000000000000000000000..531906f9c2a7e018413b18f080f7033ec3dc5662 --- /dev/null +++ b/results/all-MiniLM-L6-v2-self-optimum-q8/MedrxivClusteringS2S.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "35191c8c0dca72d8ff3efcd72aa802307d469663", + "mteb_dataset_name": "MedrxivClusteringS2S", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 71.4, + "v_measure": 0.29467392686663063, + "v_measure_std": 0.009493099815860577 + } +} \ No newline at end of file diff --git a/results/all-MiniLM-L6-v2-self-optimum-q8/RedditClustering.json b/results/all-MiniLM-L6-v2-self-optimum-q8/RedditClustering.json new file mode 100644 index 0000000000000000000000000000000000000000..fbe7c6c751b915635373b3b75142d2d54559aa8b --- /dev/null +++ b/results/all-MiniLM-L6-v2-self-optimum-q8/RedditClustering.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "24640382cdbf8abc73003fb0fa6d111a705499eb", + "mteb_dataset_name": "RedditClustering", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 802.74, + "v_measure": 0.45130152555271275, + "v_measure_std": 0.050225232137712884 + } +} \ No newline at end of file diff --git a/results/all-MiniLM-L6-v2-self-optimum-q8/RedditClusteringP2P.json b/results/all-MiniLM-L6-v2-self-optimum-q8/RedditClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..d25b396e1768ed6e3c9ef14639082e3feb738a49 --- /dev/null +++ b/results/all-MiniLM-L6-v2-self-optimum-q8/RedditClusteringP2P.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "282350215ef01743dc01b456c7f5241fa8937f16", + "mteb_dataset_name": "RedditClusteringP2P", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 5975.57, + "v_measure": 0.5225768488129836, + "v_measure_std": 0.11421440945199224 + } +} \ No newline at end of file diff --git a/results/all-MiniLM-L6-v2-self-optimum-q8/SprintDuplicateQuestions.json b/results/all-MiniLM-L6-v2-self-optimum-q8/SprintDuplicateQuestions.json new file mode 100644 index 0000000000000000000000000000000000000000..f18d6c71a26d4e1b49086cbe77e4845f6e578c60 --- /dev/null +++ b/results/all-MiniLM-L6-v2-self-optimum-q8/SprintDuplicateQuestions.json @@ -0,0 +1,49 @@ +{ + "dataset_revision": "d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46", + "mteb_dataset_name": "SprintDuplicateQuestions", + "mteb_version": "1.1.0", + "test": { + "cos_sim": { + "accuracy": 0.9978613861386139, + "accuracy_threshold": 0.7232062816619873, + "ap": 0.9452324861684893, + "f1": 0.8893421723610403, + "f1_threshold": 0.7091838121414185, + "precision": 0.9073881373569199, + "recall": 0.872 + }, + "dot": { + "accuracy": 0.9949306930693069, + "accuracy_threshold": 22.577022552490234, + "ap": 0.8059011556474294, + "f1": 0.7370959961408587, + "f1_threshold": 20.937259674072266, + "precision": 0.712022367194781, + "recall": 0.764 + }, + "euclidean": { + "accuracy": 0.9974257425742574, + "accuracy_threshold": 4.111460208892822, + "ap": 0.9195496585306108, + "f1": 0.8673469387755102, + "f1_threshold": 4.147090435028076, + "precision": 0.8854166666666666, + "recall": 0.85 + }, + "evaluation_time": 11.13, + "manhattan": { + "accuracy": 0.9974455445544554, + "accuracy_threshold": 63.65602111816406, + "ap": 0.9200343282163483, + "f1": 0.8664987405541562, + "f1_threshold": 65.03108978271484, + "precision": 0.8730964467005076, + "recall": 0.86 + }, + "max": { + "accuracy": 0.9978613861386139, + "ap": 0.9452324861684893, + "f1": 0.8893421723610403 + } + } +} \ No newline at end of file diff --git a/results/all-MiniLM-L6-v2-self-optimum-q8/StackExchangeClustering.json b/results/all-MiniLM-L6-v2-self-optimum-q8/StackExchangeClustering.json new file mode 100644 index 0000000000000000000000000000000000000000..d77fbf4ed42d652352f0af54d7cf55fb03dd5692 --- /dev/null +++ b/results/all-MiniLM-L6-v2-self-optimum-q8/StackExchangeClustering.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "6cbc1f7b2bc0622f2e39d2c77fa502909748c259", + "mteb_dataset_name": "StackExchangeClustering", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 610.37, + "v_measure": 0.4670523695116733, + "v_measure_std": 0.06592905155287662 + } +} \ No newline at end of file diff --git a/results/all-MiniLM-L6-v2-self-optimum-q8/StackExchangeClusteringP2P.json b/results/all-MiniLM-L6-v2-self-optimum-q8/StackExchangeClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..6c9add3ef67aa1ea189dc7a8425e9cb48b436f2d --- /dev/null +++ b/results/all-MiniLM-L6-v2-self-optimum-q8/StackExchangeClusteringP2P.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "815ca46b2622cec33ccafc3735d572c266efdb44", + "mteb_dataset_name": "StackExchangeClusteringP2P", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 1492.99, + "v_measure": 0.3444038063243484, + "v_measure_std": 0.013884513241399048 + } +} \ No newline at end of file diff --git a/results/all-MiniLM-L6-v2-self-optimum-q8/TwentyNewsgroupsClustering.json b/results/all-MiniLM-L6-v2-self-optimum-q8/TwentyNewsgroupsClustering.json new file mode 100644 index 0000000000000000000000000000000000000000..2dd6879e127e8040c2b81b5ae4657cf27cbd82ad --- /dev/null +++ b/results/all-MiniLM-L6-v2-self-optimum-q8/TwentyNewsgroupsClustering.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "6125ec4e24fa026cec8a478383ee943acfbd5449", + "mteb_dataset_name": "TwentyNewsgroupsClustering", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 54.67, + "v_measure": 0.4193178041990425, + "v_measure_std": 0.011626728303010321 + } +} \ No newline at end of file diff --git a/results/all-MiniLM-L6-v2-self-optimum-q8/TwitterSemEval2015.json b/results/all-MiniLM-L6-v2-self-optimum-q8/TwitterSemEval2015.json new file mode 100644 index 0000000000000000000000000000000000000000..f62a5f0b36239343ec9ab15e2c370ffe05fc5624 --- /dev/null +++ b/results/all-MiniLM-L6-v2-self-optimum-q8/TwitterSemEval2015.json @@ -0,0 +1,49 @@ +{ + "dataset_revision": "70970daeab8776df92f5ea462b6173c0b46fd2d1", + "mteb_dataset_name": "TwitterSemEval2015", + "mteb_version": "1.1.0", + "test": { + "cos_sim": { + "accuracy": 0.8402574953805806, + "accuracy_threshold": 0.7548823356628418, + "ap": 0.6782973844761395, + "f1": 0.6368255504476167, + "f1_threshold": 0.6944434642791748, + "precision": 0.5880250223413762, + "recall": 0.6944591029023747 + }, + "dot": { + "accuracy": 0.7901889491565834, + "accuracy_threshold": 30.103286743164062, + "ap": 0.482856775311342, + "f1": 0.5083665338645418, + "f1_threshold": 20.793228149414062, + "precision": 0.40832, + "recall": 0.6733509234828496 + }, + "euclidean": { + "accuracy": 0.8328068188591524, + "accuracy_threshold": 3.928983211517334, + "ap": 0.6539558962841057, + "f1": 0.6220163876024225, + "f1_threshold": 4.456747055053711, + "precision": 0.5655366011660549, + "recall": 0.6910290237467018 + }, + "evaluation_time": 15.28, + "manhattan": { + "accuracy": 0.8323895809739524, + "accuracy_threshold": 60.54985046386719, + "ap": 0.652800674337047, + "f1": 0.6223784701175901, + "f1_threshold": 68.62921142578125, + "precision": 0.5756896165059431, + "recall": 0.6773087071240106 + }, + "max": { + "accuracy": 0.8402574953805806, + "ap": 0.6782973844761395, + "f1": 0.6368255504476167 + } + } +} \ No newline at end of file diff --git a/results/all-MiniLM-L6-v2-self-optimum-q8/TwitterURLCorpus.json b/results/all-MiniLM-L6-v2-self-optimum-q8/TwitterURLCorpus.json new file mode 100644 index 0000000000000000000000000000000000000000..6fc987bc45295a6a72e108a318d53db53f37a584 --- /dev/null +++ b/results/all-MiniLM-L6-v2-self-optimum-q8/TwitterURLCorpus.json @@ -0,0 +1,49 @@ +{ + "dataset_revision": "8b6510b0b1fa4e4c4f879467980e9be563ec1cdf", + "mteb_dataset_name": "TwitterURLCorpus", + "mteb_version": "1.1.0", + "test": { + "cos_sim": { + "accuracy": 0.8826793961268289, + "accuracy_threshold": 0.674436092376709, + "ap": 0.8468494238463535, + "f1": 0.7654432523997741, + "f1_threshold": 0.6305196285247803, + "precision": 0.7488399499152979, + "recall": 0.78279950723745 + }, + "dot": { + "accuracy": 0.8508363410563899, + "accuracy_threshold": 13.469935417175293, + "ap": 0.7659922234196515, + "f1": 0.7114903218982477, + "f1_threshold": 11.886985778808594, + "precision": 0.656260162601626, + "recall": 0.7768709578072066 + }, + "euclidean": { + "accuracy": 0.8572398804672643, + "accuracy_threshold": 3.583315849304199, + "ap": 0.7846270644934776, + "f1": 0.7020438498699368, + "f1_threshold": 3.890807628631592, + "precision": 0.6784944691854619, + "recall": 0.7272867262088081 + }, + "evaluation_time": 69.57, + "manhattan": { + "accuracy": 0.8571234524779757, + "accuracy_threshold": 55.01682662963867, + "ap": 0.7838805708871157, + "f1": 0.7009824375071123, + "f1_threshold": 59.76050567626953, + "precision": 0.6908411214953271, + "recall": 0.7114259316291962 + }, + "max": { + "accuracy": 0.8826793961268289, + "ap": 0.8468494238463535, + "f1": 0.7654432523997741 + } + } +} \ No newline at end of file diff --git a/results/all-MiniLM-L6-v2/ArxivClusteringP2P.json b/results/all-MiniLM-L6-v2/ArxivClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..72f5db8ffd0cab30026fceafbdd56326de9e8769 --- /dev/null +++ b/results/all-MiniLM-L6-v2/ArxivClusteringP2P.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "a122ad7f3f0291bf49cc6f4d32aa80929df69d5d", + "mteb_dataset_name": "ArxivClusteringP2P", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 3020.73, + "v_measure": 0.46475205553589694, + "v_measure_std": 0.14127490459016118 + } +} \ No newline at end of file diff --git a/results/all-MiniLM-L6-v2/ArxivClusteringS2S.json b/results/all-MiniLM-L6-v2/ArxivClusteringS2S.json new file mode 100644 index 0000000000000000000000000000000000000000..4458396b4f844e860c4e6a7800d13d4b6a81260f --- /dev/null +++ b/results/all-MiniLM-L6-v2/ArxivClusteringS2S.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "f910caf1a6075f7329cdf8c1a6135696f37dbd53", + "mteb_dataset_name": "ArxivClusteringS2S", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 555.92, + "v_measure": 0.37668696140320146, + "v_measure_std": 0.1442262573062312 + } +} \ No newline at end of file diff --git a/results/all-MiniLM-L6-v2/BiorxivClusteringP2P.json b/results/all-MiniLM-L6-v2/BiorxivClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..b71b20f3667f7fedca3261c2b7d26619d1e7c1ae --- /dev/null +++ b/results/all-MiniLM-L6-v2/BiorxivClusteringP2P.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "65b79d1d13f80053f67aca9498d9402c2d9f1f40", + "mteb_dataset_name": "BiorxivClusteringP2P", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 338.42, + "v_measure": 0.38367725707685313, + "v_measure_std": 0.007571234916434439 + } +} \ No newline at end of file diff --git a/results/all-MiniLM-L6-v2/BiorxivClusteringS2S.json b/results/all-MiniLM-L6-v2/BiorxivClusteringS2S.json new file mode 100644 index 0000000000000000000000000000000000000000..17b8cb7b0095e4c8e2fa15b176bfd7940d66c417 --- /dev/null +++ b/results/all-MiniLM-L6-v2/BiorxivClusteringS2S.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "258694dd0231531bc1fd9de6ceb52a0853c6d908", + "mteb_dataset_name": "BiorxivClusteringS2S", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 63.55, + "v_measure": 0.32877100863482717, + "v_measure_std": 0.007542883732950632 + } +} \ No newline at end of file diff --git a/results/all-MiniLM-L6-v2/MedrxivClusteringP2P.json b/results/all-MiniLM-L6-v2/MedrxivClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..6f3975c0bd0f098b4eaf6bbe02340c1cf69d120b --- /dev/null +++ b/results/all-MiniLM-L6-v2/MedrxivClusteringP2P.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "e7a26af6f3ae46b30dde8737f02c07b1505bcc73", + "mteb_dataset_name": "MedrxivClusteringP2P", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 173.82, + "v_measure": 0.34389331859263667, + "v_measure_std": 0.015084000520185471 + } +} \ No newline at end of file diff --git a/results/all-MiniLM-L6-v2/MedrxivClusteringS2S.json b/results/all-MiniLM-L6-v2/MedrxivClusteringS2S.json new file mode 100644 index 0000000000000000000000000000000000000000..58ddbf4ec9d12e1efca9178ab22f6a08115b0495 --- /dev/null +++ b/results/all-MiniLM-L6-v2/MedrxivClusteringS2S.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "35191c8c0dca72d8ff3efcd72aa802307d469663", + "mteb_dataset_name": "MedrxivClusteringS2S", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 36.96, + "v_measure": 0.3186351621903201, + "v_measure_std": 0.017336971951213677 + } +} \ No newline at end of file diff --git a/results/all-MiniLM-L6-v2/RedditClustering.json b/results/all-MiniLM-L6-v2/RedditClustering.json new file mode 100644 index 0000000000000000000000000000000000000000..113fe853cfb5350f2469f1b6bd5626f5245f77d9 --- /dev/null +++ b/results/all-MiniLM-L6-v2/RedditClustering.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "24640382cdbf8abc73003fb0fa6d111a705499eb", + "mteb_dataset_name": "RedditClustering", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 379.65, + "v_measure": 0.5068847759621761, + "v_measure_std": 0.045990849171726154 + } +} \ No newline at end of file diff --git a/results/all-MiniLM-L6-v2/RedditClusteringP2P.json b/results/all-MiniLM-L6-v2/RedditClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..93138db69777a7bc9fa9a1d7800ebefb184b50ee --- /dev/null +++ b/results/all-MiniLM-L6-v2/RedditClusteringP2P.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "282350215ef01743dc01b456c7f5241fa8937f16", + "mteb_dataset_name": "RedditClusteringP2P", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 1400.48, + "v_measure": 0.5479845419577372, + "v_measure_std": 0.11285422691786078 + } +} \ No newline at end of file diff --git a/results/all-MiniLM-L6-v2/SprintDuplicateQuestions.json b/results/all-MiniLM-L6-v2/SprintDuplicateQuestions.json new file mode 100644 index 0000000000000000000000000000000000000000..ce78586d5c64f7f860522ebc2f0cc0c25e1242d1 --- /dev/null +++ b/results/all-MiniLM-L6-v2/SprintDuplicateQuestions.json @@ -0,0 +1,49 @@ +{ + "dataset_revision": "d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46", + "mteb_dataset_name": "SprintDuplicateQuestions", + "mteb_version": "1.1.0", + "test": { + "cos_sim": { + "accuracy": 0.9978514851485148, + "accuracy_threshold": 0.71305251121521, + "ap": 0.9455063045792447, + "f1": 0.890126582278481, + "f1_threshold": 0.71305251121521, + "precision": 0.9015384615384615, + "recall": 0.879 + }, + "dot": { + "accuracy": 0.9978514851485148, + "accuracy_threshold": 0.7130525708198547, + "ap": 0.9455063045792447, + "f1": 0.890126582278481, + "f1_threshold": 0.7130525708198547, + "precision": 0.9015384615384615, + "recall": 0.879 + }, + "euclidean": { + "accuracy": 0.9978514851485148, + "accuracy_threshold": 0.7575584650039673, + "ap": 0.9455063045792447, + "f1": 0.890126582278481, + "f1_threshold": 0.7575584650039673, + "precision": 0.9015384615384615, + "recall": 0.879 + }, + "evaluation_time": 13.21, + "manhattan": { + "accuracy": 0.9978415841584158, + "accuracy_threshold": 11.837488174438477, + "ap": 0.9454002074215007, + "f1": 0.8898989898989899, + "f1_threshold": 11.837488174438477, + "precision": 0.8989795918367347, + "recall": 0.881 + }, + "max": { + "accuracy": 0.9978514851485148, + "ap": 0.9455063045792447, + "f1": 0.890126582278481 + } + } +} \ No newline at end of file diff --git a/results/all-MiniLM-L6-v2/StackExchangeClustering.json b/results/all-MiniLM-L6-v2/StackExchangeClustering.json new file mode 100644 index 0000000000000000000000000000000000000000..060fd1b8fa5b0ba6d8db2314d0344a799f4da8f2 --- /dev/null +++ b/results/all-MiniLM-L6-v2/StackExchangeClustering.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "6cbc1f7b2bc0622f2e39d2c77fa502909748c259", + "mteb_dataset_name": "StackExchangeClustering", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 317.85, + "v_measure": 0.5313685840246208, + "v_measure_std": 0.05607312786136468 + } +} \ No newline at end of file diff --git a/results/all-MiniLM-L6-v2/StackExchangeClusteringP2P.json b/results/all-MiniLM-L6-v2/StackExchangeClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..91d5328cb7f979cc5caf232ab328d693890fc3d4 --- /dev/null +++ b/results/all-MiniLM-L6-v2/StackExchangeClusteringP2P.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "815ca46b2622cec33ccafc3735d572c266efdb44", + "mteb_dataset_name": "StackExchangeClusteringP2P", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 299.37, + "v_measure": 0.3423986915835284, + "v_measure_std": 0.015639293197691355 + } +} \ No newline at end of file diff --git a/results/all-MiniLM-L6-v2/TwentyNewsgroupsClustering.json b/results/all-MiniLM-L6-v2/TwentyNewsgroupsClustering.json new file mode 100644 index 0000000000000000000000000000000000000000..f38dca7ef7c3082c084eb627d7b4d9d55547b588 --- /dev/null +++ b/results/all-MiniLM-L6-v2/TwentyNewsgroupsClustering.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "6125ec4e24fa026cec8a478383ee943acfbd5449", + "mteb_dataset_name": "TwentyNewsgroupsClustering", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 38.48, + "v_measure": 0.46490211003880566, + "v_measure_std": 0.00983303557519297 + } +} \ No newline at end of file diff --git a/results/all-MiniLM-L6-v2/TwitterSemEval2015.json b/results/all-MiniLM-L6-v2/TwitterSemEval2015.json new file mode 100644 index 0000000000000000000000000000000000000000..a1ef4a438e32b6dbc45e6d1a2d3144f94a29cd1b --- /dev/null +++ b/results/all-MiniLM-L6-v2/TwitterSemEval2015.json @@ -0,0 +1,49 @@ +{ + "dataset_revision": "70970daeab8776df92f5ea462b6173c0b46fd2d1", + "mteb_dataset_name": "TwitterSemEval2015", + "mteb_version": "1.1.0", + "test": { + "cos_sim": { + "accuracy": 0.841151576563152, + "accuracy_threshold": 0.760004997253418, + "ap": 0.6785804277968714, + "f1": 0.6408006919560113, + "f1_threshold": 0.7039602994918823, + "precision": 0.602602835231234, + "recall": 0.6841688654353562 + }, + "dot": { + "accuracy": 0.841151576563152, + "accuracy_threshold": 0.7600049376487732, + "ap": 0.6785805581908406, + "f1": 0.6408006919560113, + "f1_threshold": 0.7039604187011719, + "precision": 0.602602835231234, + "recall": 0.6841688654353562 + }, + "euclidean": { + "accuracy": 0.841151576563152, + "accuracy_threshold": 0.6928131580352783, + "ap": 0.678580330833871, + "f1": 0.6408006919560113, + "f1_threshold": 0.7694668769836426, + "precision": 0.602602835231234, + "recall": 0.6841688654353562 + }, + "evaluation_time": 8.91, + "manhattan": { + "accuracy": 0.8396614412588663, + "accuracy_threshold": 10.996787071228027, + "ap": 0.6766935756595975, + "f1": 0.6382363570654138, + "f1_threshold": 12.064617156982422, + "precision": 0.5872312125914432, + "recall": 0.6989445910290237 + }, + "max": { + "accuracy": 0.841151576563152, + "ap": 0.6785805581908406, + "f1": 0.6408006919560113 + } + } +} \ No newline at end of file diff --git a/results/all-MiniLM-L6-v2/TwitterURLCorpus.json b/results/all-MiniLM-L6-v2/TwitterURLCorpus.json new file mode 100644 index 0000000000000000000000000000000000000000..27a51bbeefe888d4e5321698f78bf9701cf54f57 --- /dev/null +++ b/results/all-MiniLM-L6-v2/TwitterURLCorpus.json @@ -0,0 +1,49 @@ +{ + "dataset_revision": "8b6510b0b1fa4e4c4f879467980e9be563ec1cdf", + "mteb_dataset_name": "TwitterURLCorpus", + "mteb_version": "1.1.0", + "test": { + "cos_sim": { + "accuracy": 0.882504754142896, + "accuracy_threshold": 0.6595447659492493, + "ap": 0.8470165834383896, + "f1": 0.7657057281916886, + "f1_threshold": 0.6311962008476257, + "precision": 0.745226643346451, + "recall": 0.7873421619956883 + }, + "dot": { + "accuracy": 0.882504754142896, + "accuracy_threshold": 0.6595448851585388, + "ap": 0.8470166526011678, + "f1": 0.7657057281916886, + "f1_threshold": 0.6311962604522705, + "precision": 0.745226643346451, + "recall": 0.7873421619956883 + }, + "euclidean": { + "accuracy": 0.882504754142896, + "accuracy_threshold": 0.8251729607582092, + "ap": 0.847016568342695, + "f1": 0.7657057281916886, + "f1_threshold": 0.858840823173523, + "precision": 0.745226643346451, + "recall": 0.7873421619956883 + }, + "evaluation_time": 35.6, + "manhattan": { + "accuracy": 0.8827376101214732, + "accuracy_threshold": 12.611679077148438, + "ap": 0.846351848497952, + "f1": 0.7655138674594514, + "f1_threshold": 13.285348892211914, + "precision": 0.7486934118513066, + "recall": 0.7831074838312289 + }, + "max": { + "accuracy": 0.8827376101214732, + "ap": 0.8470166526011678, + "f1": 0.7657057281916886 + } + } +} \ No newline at end of file diff --git a/results/gte-base/ArxivClusteringP2P.json b/results/gte-base/ArxivClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..2c435fd22b5fc03f9e29795ed6226b9ecdec7481 --- /dev/null +++ b/results/gte-base/ArxivClusteringP2P.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "a122ad7f3f0291bf49cc6f4d32aa80929df69d5d", + "mteb_dataset_name": "ArxivClusteringP2P", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 17420.32, + "v_measure": 0.48382102161114754, + "v_measure_std": 0.14018823371221706 + } +} \ No newline at end of file diff --git a/results/gte-base/ArxivClusteringS2S.json b/results/gte-base/ArxivClusteringS2S.json new file mode 100644 index 0000000000000000000000000000000000000000..49c13fa57b37887a168cdeb2821fdac9e5a1a866 --- /dev/null +++ b/results/gte-base/ArxivClusteringS2S.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "f910caf1a6075f7329cdf8c1a6135696f37dbd53", + "mteb_dataset_name": "ArxivClusteringS2S", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 1944.96, + "v_measure": 0.4301558768913956, + "v_measure_std": 0.14253189552031462 + } +} \ No newline at end of file diff --git a/results/gte-base/BiorxivClusteringP2P.json b/results/gte-base/BiorxivClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..c5880b44c3a21298d222735ecb11626baf2a9a37 --- /dev/null +++ b/results/gte-base/BiorxivClusteringP2P.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "65b79d1d13f80053f67aca9498d9402c2d9f1f40", + "mteb_dataset_name": "BiorxivClusteringP2P", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 2292.44, + "v_measure": 0.3881840458148674, + "v_measure_std": 0.010968838534302094 + } +} \ No newline at end of file diff --git a/results/gte-base/BiorxivClusteringS2S.json b/results/gte-base/BiorxivClusteringS2S.json new file mode 100644 index 0000000000000000000000000000000000000000..f7b0497419763d0fa5fe5c81c01261dbbc2ba97d --- /dev/null +++ b/results/gte-base/BiorxivClusteringS2S.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "258694dd0231531bc1fd9de6ceb52a0853c6d908", + "mteb_dataset_name": "BiorxivClusteringS2S", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 240.23, + "v_measure": 0.36831102966298046, + "v_measure_std": 0.010162662522450641 + } +} \ No newline at end of file diff --git a/results/gte-base/MedrxivClusteringP2P.json b/results/gte-base/MedrxivClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..5cc644319c9e8251f9bc5dbda737237bdd862170 --- /dev/null +++ b/results/gte-base/MedrxivClusteringP2P.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "e7a26af6f3ae46b30dde8737f02c07b1505bcc73", + "mteb_dataset_name": "MedrxivClusteringP2P", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 1257.32, + "v_measure": 0.3336670595936883, + "v_measure_std": 0.016041747790479916 + } +} \ No newline at end of file diff --git a/results/gte-base/MedrxivClusteringS2S.json b/results/gte-base/MedrxivClusteringS2S.json new file mode 100644 index 0000000000000000000000000000000000000000..151be65231b8710f0dd012bb5e03d6ad46550f79 --- /dev/null +++ b/results/gte-base/MedrxivClusteringS2S.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "35191c8c0dca72d8ff3efcd72aa802307d469663", + "mteb_dataset_name": "MedrxivClusteringS2S", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 135.05, + "v_measure": 0.3202151715088234, + "v_measure_std": 0.016400664656544005 + } +} \ No newline at end of file diff --git a/results/gte-base/RedditClustering.json b/results/gte-base/RedditClustering.json new file mode 100644 index 0000000000000000000000000000000000000000..0fc9fa985a3d561cf03d4a50a8e4c37dcd2b2ca1 --- /dev/null +++ b/results/gte-base/RedditClustering.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "24640382cdbf8abc73003fb0fa6d111a705499eb", + "mteb_dataset_name": "RedditClustering", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 1456.97, + "v_measure": 0.5894249153617389, + "v_measure_std": 0.04013423216574163 + } +} \ No newline at end of file diff --git a/results/gte-base/RedditClusteringP2P.json b/results/gte-base/RedditClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..d411ae35878676af36d3df5de52eb126418cab25 --- /dev/null +++ b/results/gte-base/RedditClusteringP2P.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "282350215ef01743dc01b456c7f5241fa8937f16", + "mteb_dataset_name": "RedditClusteringP2P", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 7406.94, + "v_measure": 0.6271786948932438, + "v_measure_std": 0.124884922843343 + } +} \ No newline at end of file diff --git a/results/gte-base/SprintDuplicateQuestions.json b/results/gte-base/SprintDuplicateQuestions.json new file mode 100644 index 0000000000000000000000000000000000000000..747499acfe5d871ec68b06d27e4fd93dba00fa85 --- /dev/null +++ b/results/gte-base/SprintDuplicateQuestions.json @@ -0,0 +1,49 @@ +{ + "dataset_revision": "d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46", + "mteb_dataset_name": "SprintDuplicateQuestions", + "mteb_version": "1.1.0", + "test": { + "cos_sim": { + "accuracy": 0.9982178217821782, + "accuracy_threshold": 0.9155436158180237, + "ap": 0.9571220118360753, + "f1": 0.9073120494335737, + "f1_threshold": 0.9155436158180237, + "precision": 0.935244161358811, + "recall": 0.881 + }, + "dot": { + "accuracy": 0.9982178217821782, + "accuracy_threshold": 0.9155436754226685, + "ap": 0.9571219977317563, + "f1": 0.9073120494335737, + "f1_threshold": 0.9155436754226685, + "precision": 0.935244161358811, + "recall": 0.881 + }, + "euclidean": { + "accuracy": 0.9982178217821782, + "accuracy_threshold": 0.4109899401664734, + "ap": 0.9571219793104613, + "f1": 0.9073120494335737, + "f1_threshold": 0.4109899401664734, + "precision": 0.935244161358811, + "recall": 0.881 + }, + "evaluation_time": 30.77, + "manhattan": { + "accuracy": 0.9982178217821782, + "accuracy_threshold": 8.906990051269531, + "ap": 0.956972725802545, + "f1": 0.9077001529831717, + "f1_threshold": 9.168195724487305, + "precision": 0.9261186264308012, + "recall": 0.89 + }, + "max": { + "accuracy": 0.9982178217821782, + "ap": 0.9571220118360753, + "f1": 0.9077001529831717 + } + } +} \ No newline at end of file diff --git a/results/gte-base/StackExchangeClustering.json b/results/gte-base/StackExchangeClustering.json new file mode 100644 index 0000000000000000000000000000000000000000..98de3f10dbe2d287e3f536c09937b9122c83bc79 --- /dev/null +++ b/results/gte-base/StackExchangeClustering.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "6cbc1f7b2bc0622f2e39d2c77fa502909748c259", + "mteb_dataset_name": "StackExchangeClustering", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 1131.23, + "v_measure": 0.6686972648631798, + "v_measure_std": 0.043035802483956075 + } +} \ No newline at end of file diff --git a/results/gte-base/StackExchangeClusteringP2P.json b/results/gte-base/StackExchangeClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..3552f4dc268ce2ab9635cd8fb470a2a2020e98b4 --- /dev/null +++ b/results/gte-base/StackExchangeClusteringP2P.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "815ca46b2622cec33ccafc3735d572c266efdb44", + "mteb_dataset_name": "StackExchangeClusteringP2P", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 1727.31, + "v_measure": 0.3605541867052922, + "v_measure_std": 0.015301455168688237 + } +} \ No newline at end of file diff --git a/results/gte-base/TwentyNewsgroupsClustering.json b/results/gte-base/TwentyNewsgroupsClustering.json new file mode 100644 index 0000000000000000000000000000000000000000..743ffca556fbc7a4672895e6755aed27f05f29cc --- /dev/null +++ b/results/gte-base/TwentyNewsgroupsClustering.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "6125ec4e24fa026cec8a478383ee943acfbd5449", + "mteb_dataset_name": "TwentyNewsgroupsClustering", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 113.29, + "v_measure": 0.5154031629975145, + "v_measure_std": 0.013251582639784319 + } +} \ No newline at end of file diff --git a/results/gte-base/TwitterSemEval2015.json b/results/gte-base/TwitterSemEval2015.json new file mode 100644 index 0000000000000000000000000000000000000000..7226b7fdc77dcdba56d2af61f100f2bbd8a0a198 --- /dev/null +++ b/results/gte-base/TwitterSemEval2015.json @@ -0,0 +1,49 @@ +{ + "dataset_revision": "70970daeab8776df92f5ea462b6173c0b46fd2d1", + "mteb_dataset_name": "TwitterSemEval2015", + "mteb_version": "1.1.0", + "test": { + "cos_sim": { + "accuracy": 0.8564105620790368, + "accuracy_threshold": 0.9224622845649719, + "ap": 0.7246544507254236, + "f1": 0.6796572367648784, + "f1_threshold": 0.9091762900352478, + "precision": 0.6121801649397336, + "recall": 0.7638522427440633 + }, + "dot": { + "accuracy": 0.8564105620790368, + "accuracy_threshold": 0.9224622249603271, + "ap": 0.7246545978977025, + "f1": 0.6796572367648784, + "f1_threshold": 0.9091762900352478, + "precision": 0.6121801649397336, + "recall": 0.7638522427440633 + }, + "euclidean": { + "accuracy": 0.8564105620790368, + "accuracy_threshold": 0.3937961757183075, + "ap": 0.7246547648505838, + "f1": 0.6796572367648784, + "f1_threshold": 0.42620110511779785, + "precision": 0.6121801649397336, + "recall": 0.7638522427440633 + }, + "evaluation_time": 28.44, + "manhattan": { + "accuracy": 0.8567085891398939, + "accuracy_threshold": 8.628108024597168, + "ap": 0.7236803599101469, + "f1": 0.6801418439716312, + "f1_threshold": 9.374834060668945, + "precision": 0.6160599571734475, + "recall": 0.7591029023746702 + }, + "max": { + "accuracy": 0.8567085891398939, + "ap": 0.7246547648505838, + "f1": 0.6801418439716312 + } + } +} \ No newline at end of file diff --git a/results/gte-base/TwitterURLCorpus.json b/results/gte-base/TwitterURLCorpus.json new file mode 100644 index 0000000000000000000000000000000000000000..48e01a4b0e17133a51f65cdafdbca1ed1d20203a --- /dev/null +++ b/results/gte-base/TwitterURLCorpus.json @@ -0,0 +1,49 @@ +{ + "dataset_revision": "8b6510b0b1fa4e4c4f879467980e9be563ec1cdf", + "mteb_dataset_name": "TwitterURLCorpus", + "mteb_version": "1.1.0", + "test": { + "cos_sim": { + "accuracy": 0.887297706368611, + "accuracy_threshold": 0.8953593969345093, + "ap": 0.8551813671848673, + "f1": 0.7795627934272301, + "f1_threshold": 0.8873952627182007, + "precision": 0.744396189408798, + "recall": 0.8182168155220203 + }, + "dot": { + "accuracy": 0.887297706368611, + "accuracy_threshold": 0.8953593969345093, + "ap": 0.8551813435958813, + "f1": 0.7795627934272301, + "f1_threshold": 0.8873952627182007, + "precision": 0.744396189408798, + "recall": 0.8182168155220203 + }, + "euclidean": { + "accuracy": 0.887297706368611, + "accuracy_threshold": 0.457472562789917, + "ap": 0.8551813732980336, + "f1": 0.7795627934272301, + "f1_threshold": 0.47456246614456177, + "precision": 0.744396189408798, + "recall": 0.8182168155220203 + }, + "evaluation_time": 131.42, + "manhattan": { + "accuracy": 0.8869290177358637, + "accuracy_threshold": 10.028573989868164, + "ap": 0.8549390135903003, + "f1": 0.7796471983397568, + "f1_threshold": 10.410341262817383, + "precision": 0.751571877679337, + "recall": 0.8099014474899907 + }, + "max": { + "accuracy": 0.887297706368611, + "ap": 0.8551813732980336, + "f1": 0.7796471983397568 + } + } +} \ No newline at end of file diff --git a/results/optimum/all-MiniLM-L6-v2-q8/ArxivClusteringP2P.json b/results/optimum/all-MiniLM-L6-v2-q8/ArxivClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..4900a4a921293a9000b4622fbf976501695e010e --- /dev/null +++ b/results/optimum/all-MiniLM-L6-v2-q8/ArxivClusteringP2P.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "a122ad7f3f0291bf49cc6f4d32aa80929df69d5d", + "mteb_dataset_name": "ArxivClusteringP2P", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 40102.45, + "v_measure": 0.4618034932273851, + "v_measure_std": 0.14005599148945322 + } +} \ No newline at end of file diff --git a/results/optimum/all-MiniLM-L6-v2-q8/ArxivClusteringS2S.json b/results/optimum/all-MiniLM-L6-v2-q8/ArxivClusteringS2S.json new file mode 100644 index 0000000000000000000000000000000000000000..4da9a839b15a4be2809246501d781bf922055a78 --- /dev/null +++ b/results/optimum/all-MiniLM-L6-v2-q8/ArxivClusteringS2S.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "f910caf1a6075f7329cdf8c1a6135696f37dbd53", + "mteb_dataset_name": "ArxivClusteringS2S", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 1056.15, + "v_measure": 0.3661110691976883, + "v_measure_std": 0.1428983905125625 + } +} \ No newline at end of file diff --git a/results/optimum/all-MiniLM-L6-v2-q8/BiorxivClusteringP2P.json b/results/optimum/all-MiniLM-L6-v2-q8/BiorxivClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..8f9f83d7cde197d369432f73c10e253f16bee01d --- /dev/null +++ b/results/optimum/all-MiniLM-L6-v2-q8/BiorxivClusteringP2P.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "65b79d1d13f80053f67aca9498d9402c2d9f1f40", + "mteb_dataset_name": "BiorxivClusteringP2P", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 2012.27, + "v_measure": 0.3826666857854354, + "v_measure_std": 0.009760375786673237 + } +} \ No newline at end of file diff --git a/results/optimum/all-MiniLM-L6-v2-q8/BiorxivClusteringS2S.json b/results/optimum/all-MiniLM-L6-v2-q8/BiorxivClusteringS2S.json new file mode 100644 index 0000000000000000000000000000000000000000..9f87edd474935608ea74d3c3284f48492d78287f --- /dev/null +++ b/results/optimum/all-MiniLM-L6-v2-q8/BiorxivClusteringS2S.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "258694dd0231531bc1fd9de6ceb52a0853c6d908", + "mteb_dataset_name": "BiorxivClusteringS2S", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 129.47, + "v_measure": 0.32225968859400944, + "v_measure_std": 0.010394710148885584 + } +} \ No newline at end of file diff --git a/results/optimum/all-MiniLM-L6-v2-q8/MedrxivClusteringP2P.json b/results/optimum/all-MiniLM-L6-v2-q8/MedrxivClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..56b5c815c8039abc677842c647217a9f8c308f4e --- /dev/null +++ b/results/optimum/all-MiniLM-L6-v2-q8/MedrxivClusteringP2P.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "e7a26af6f3ae46b30dde8737f02c07b1505bcc73", + "mteb_dataset_name": "MedrxivClusteringP2P", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 1138.47, + "v_measure": 0.3214793378259306, + "v_measure_std": 0.010193796252019919 + } +} \ No newline at end of file diff --git a/results/optimum/all-MiniLM-L6-v2-q8/MedrxivClusteringS2S.json b/results/optimum/all-MiniLM-L6-v2-q8/MedrxivClusteringS2S.json new file mode 100644 index 0000000000000000000000000000000000000000..61fd6c330e6eb2a1ae2aeeac04ad463af2ede1f1 --- /dev/null +++ b/results/optimum/all-MiniLM-L6-v2-q8/MedrxivClusteringS2S.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "35191c8c0dca72d8ff3efcd72aa802307d469663", + "mteb_dataset_name": "MedrxivClusteringS2S", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 72.3, + "v_measure": 0.29467392686663063, + "v_measure_std": 0.009493099815860577 + } +} \ No newline at end of file diff --git a/results/optimum/all-MiniLM-L6-v2-q8/RedditClustering.json b/results/optimum/all-MiniLM-L6-v2-q8/RedditClustering.json new file mode 100644 index 0000000000000000000000000000000000000000..7eaaf612aeec1bc02fe106f7e541e49cfae3b722 --- /dev/null +++ b/results/optimum/all-MiniLM-L6-v2-q8/RedditClustering.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "24640382cdbf8abc73003fb0fa6d111a705499eb", + "mteb_dataset_name": "RedditClustering", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 806.16, + "v_measure": 0.45130152555271275, + "v_measure_std": 0.050225232137712884 + } +} \ No newline at end of file diff --git a/results/optimum/all-MiniLM-L6-v2-q8/RedditClusteringP2P.json b/results/optimum/all-MiniLM-L6-v2-q8/RedditClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..565f6a4765cb8f7cd3002bdbd5f51c370e34a3ac --- /dev/null +++ b/results/optimum/all-MiniLM-L6-v2-q8/RedditClusteringP2P.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "282350215ef01743dc01b456c7f5241fa8937f16", + "mteb_dataset_name": "RedditClusteringP2P", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 5989.02, + "v_measure": 0.5225768488129836, + "v_measure_std": 0.11421440945199224 + } +} \ No newline at end of file diff --git a/results/optimum/all-MiniLM-L6-v2-q8/SprintDuplicateQuestions.json b/results/optimum/all-MiniLM-L6-v2-q8/SprintDuplicateQuestions.json new file mode 100644 index 0000000000000000000000000000000000000000..9b990cc773b5f256a95e0fe3a459b8ffa3525d7c --- /dev/null +++ b/results/optimum/all-MiniLM-L6-v2-q8/SprintDuplicateQuestions.json @@ -0,0 +1,49 @@ +{ + "dataset_revision": "d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46", + "mteb_dataset_name": "SprintDuplicateQuestions", + "mteb_version": "1.1.0", + "test": { + "cos_sim": { + "accuracy": 0.9978613861386139, + "accuracy_threshold": 0.7077131271362305, + "ap": 0.9448484786327728, + "f1": 0.8905775075987842, + "f1_threshold": 0.7067674398422241, + "precision": 0.9024640657084189, + "recall": 0.879 + }, + "dot": { + "accuracy": 0.994950495049505, + "accuracy_threshold": 22.681137084960938, + "ap": 0.8020156400279196, + "f1": 0.7354577642893273, + "f1_threshold": 21.621265411376953, + "precision": 0.744114636642784, + "recall": 0.727 + }, + "euclidean": { + "accuracy": 0.9974455445544554, + "accuracy_threshold": 4.0493879318237305, + "ap": 0.9186645435197903, + "f1": 0.8695214105793451, + "f1_threshold": 4.1963300704956055, + "precision": 0.8761421319796955, + "recall": 0.863 + }, + "evaluation_time": 11.27, + "manhattan": { + "accuracy": 0.9974851485148515, + "accuracy_threshold": 62.31474304199219, + "ap": 0.9193729034580711, + "f1": 0.8695214105793451, + "f1_threshold": 65.15161895751953, + "precision": 0.8761421319796955, + "recall": 0.863 + }, + "max": { + "accuracy": 0.9978613861386139, + "ap": 0.9448484786327728, + "f1": 0.8905775075987842 + } + } +} \ No newline at end of file diff --git a/results/optimum/all-MiniLM-L6-v2-q8/StackExchangeClustering.json b/results/optimum/all-MiniLM-L6-v2-q8/StackExchangeClustering.json new file mode 100644 index 0000000000000000000000000000000000000000..a7cc080020c11843ecf076817bc4599f52f8e660 --- /dev/null +++ b/results/optimum/all-MiniLM-L6-v2-q8/StackExchangeClustering.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "6cbc1f7b2bc0622f2e39d2c77fa502909748c259", + "mteb_dataset_name": "StackExchangeClustering", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 615.37, + "v_measure": 0.4670523695116733, + "v_measure_std": 0.06592905155287662 + } +} \ No newline at end of file diff --git a/results/optimum/all-MiniLM-L6-v2-q8/StackExchangeClusteringP2P.json b/results/optimum/all-MiniLM-L6-v2-q8/StackExchangeClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..e1322cce9771f48ec91681e53546325b37f216bf --- /dev/null +++ b/results/optimum/all-MiniLM-L6-v2-q8/StackExchangeClusteringP2P.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "815ca46b2622cec33ccafc3735d572c266efdb44", + "mteb_dataset_name": "StackExchangeClusteringP2P", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 1483.01, + "v_measure": 0.3444038063243484, + "v_measure_std": 0.013884513241399048 + } +} \ No newline at end of file diff --git a/results/optimum/all-MiniLM-L6-v2-q8/TwentyNewsgroupsClustering.json b/results/optimum/all-MiniLM-L6-v2-q8/TwentyNewsgroupsClustering.json new file mode 100644 index 0000000000000000000000000000000000000000..5ce4a7e29a0f0b16dc119e0f3e7847d0c812dd74 --- /dev/null +++ b/results/optimum/all-MiniLM-L6-v2-q8/TwentyNewsgroupsClustering.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "6125ec4e24fa026cec8a478383ee943acfbd5449", + "mteb_dataset_name": "TwentyNewsgroupsClustering", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 54.34, + "v_measure": 0.4193178041990425, + "v_measure_std": 0.011626728303010321 + } +} \ No newline at end of file diff --git a/results/optimum/all-MiniLM-L6-v2-q8/TwitterSemEval2015.json b/results/optimum/all-MiniLM-L6-v2-q8/TwitterSemEval2015.json new file mode 100644 index 0000000000000000000000000000000000000000..14ece3b5a7a2f0a2cd36a46b06dd8315e118ffeb --- /dev/null +++ b/results/optimum/all-MiniLM-L6-v2-q8/TwitterSemEval2015.json @@ -0,0 +1,49 @@ +{ + "dataset_revision": "70970daeab8776df92f5ea462b6173c0b46fd2d1", + "mteb_dataset_name": "TwitterSemEval2015", + "mteb_version": "1.1.0", + "test": { + "cos_sim": { + "accuracy": 0.8393038087858378, + "accuracy_threshold": 0.7425773739814758, + "ap": 0.6783801443900392, + "f1": 0.638964096527369, + "f1_threshold": 0.6870745420455933, + "precision": 0.5768331562167907, + "recall": 0.7160949868073878 + }, + "dot": { + "accuracy": 0.7894140787983549, + "accuracy_threshold": 29.843990325927734, + "ap": 0.48171195666313793, + "f1": 0.5091934809862098, + "f1_threshold": 21.281982421875, + "precision": 0.4214804565894154, + "recall": 0.6430079155672823 + }, + "euclidean": { + "accuracy": 0.8322703701496096, + "accuracy_threshold": 3.9689555168151855, + "ap": 0.6530047778986019, + "f1": 0.6224239450441609, + "f1_threshold": 4.39756965637207, + "precision": 0.5816139385602934, + "recall": 0.6693931398416887 + }, + "evaluation_time": 15.41, + "manhattan": { + "accuracy": 0.8324491863861239, + "accuracy_threshold": 61.23781204223633, + "ap": 0.6517381509783486, + "f1": 0.622907699665232, + "f1_threshold": 69.10938262939453, + "precision": 0.5695233930913861, + "recall": 0.6873350923482849 + }, + "max": { + "accuracy": 0.8393038087858378, + "ap": 0.6783801443900392, + "f1": 0.638964096527369 + } + } +} \ No newline at end of file diff --git a/results/optimum/all-MiniLM-L6-v2-q8/TwitterURLCorpus.json b/results/optimum/all-MiniLM-L6-v2-q8/TwitterURLCorpus.json new file mode 100644 index 0000000000000000000000000000000000000000..9ee4a10687465f173a9d74546bf5080e47858349 --- /dev/null +++ b/results/optimum/all-MiniLM-L6-v2-q8/TwitterURLCorpus.json @@ -0,0 +1,49 @@ +{ + "dataset_revision": "8b6510b0b1fa4e4c4f879467980e9be563ec1cdf", + "mteb_dataset_name": "TwitterURLCorpus", + "mteb_version": "1.1.0", + "test": { + "cos_sim": { + "accuracy": 0.8822913028292001, + "accuracy_threshold": 0.6609122157096863, + "ap": 0.846976901109128, + "f1": 0.7661019479065602, + "f1_threshold": 0.6262260675430298, + "precision": 0.7433019551049964, + "recall": 0.7903449337850323 + }, + "dot": { + "accuracy": 0.8514960996623588, + "accuracy_threshold": 13.692197799682617, + "ap": 0.7666287617149927, + "f1": 0.7092627336529774, + "f1_threshold": 11.69295597076416, + "precision": 0.64625807268583, + "recall": 0.7858792731752386 + }, + "euclidean": { + "accuracy": 0.8575697597702487, + "accuracy_threshold": 3.5195302963256836, + "ap": 0.7850038350910886, + "f1": 0.7016092645510604, + "f1_threshold": 3.8645129203796387, + "precision": 0.685626102292769, + "recall": 0.7183554049892208 + }, + "evaluation_time": 70.05, + "manhattan": { + "accuracy": 0.8574533317809602, + "accuracy_threshold": 55.26979064941406, + "ap": 0.7843165101218974, + "f1": 0.7006873879258816, + "f1_threshold": 60.24797058105469, + "precision": 0.6805515239477503, + "recall": 0.7220511241145673 + }, + "max": { + "accuracy": 0.8822913028292001, + "ap": 0.846976901109128, + "f1": 0.7661019479065602 + } + } +} \ No newline at end of file diff --git a/results/optimum/all-MiniLM-L6-v2-unquantized/ArxivClusteringP2P.json b/results/optimum/all-MiniLM-L6-v2-unquantized/ArxivClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..d7d37bb641415e015c57da6e574d7ed9548a1c38 --- /dev/null +++ b/results/optimum/all-MiniLM-L6-v2-unquantized/ArxivClusteringP2P.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "a122ad7f3f0291bf49cc6f4d32aa80929df69d5d", + "mteb_dataset_name": "ArxivClusteringP2P", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 19548.39, + "v_measure": 0.4619195099830416, + "v_measure_std": 0.1393985612371939 + } +} \ No newline at end of file diff --git a/results/optimum/all-MiniLM-L6-v2-unquantized/ArxivClusteringS2S.json b/results/optimum/all-MiniLM-L6-v2-unquantized/ArxivClusteringS2S.json new file mode 100644 index 0000000000000000000000000000000000000000..473c173b175e34192799a9b8591bb15049ff59ab --- /dev/null +++ b/results/optimum/all-MiniLM-L6-v2-unquantized/ArxivClusteringS2S.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "f910caf1a6075f7329cdf8c1a6135696f37dbd53", + "mteb_dataset_name": "ArxivClusteringS2S", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 1261.03, + "v_measure": 0.36916118241265555, + "v_measure_std": 0.14259291289231926 + } +} \ No newline at end of file diff --git a/results/optimum/all-MiniLM-L6-v2-unquantized/BiorxivClusteringP2P.json b/results/optimum/all-MiniLM-L6-v2-unquantized/BiorxivClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..8fff05feb4ece3cf97527bf9fd2c7119a6c6121f --- /dev/null +++ b/results/optimum/all-MiniLM-L6-v2-unquantized/BiorxivClusteringP2P.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "65b79d1d13f80053f67aca9498d9402c2d9f1f40", + "mteb_dataset_name": "BiorxivClusteringP2P", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 2432.35, + "v_measure": 0.3859023213662521, + "v_measure_std": 0.01406600428209064 + } +} \ No newline at end of file diff --git a/results/optimum/all-MiniLM-L6-v2-unquantized/BiorxivClusteringS2S.json b/results/optimum/all-MiniLM-L6-v2-unquantized/BiorxivClusteringS2S.json new file mode 100644 index 0000000000000000000000000000000000000000..00ff0de5821188068d6afcb14159041743d1eeb6 --- /dev/null +++ b/results/optimum/all-MiniLM-L6-v2-unquantized/BiorxivClusteringS2S.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "258694dd0231531bc1fd9de6ceb52a0853c6d908", + "mteb_dataset_name": "BiorxivClusteringS2S", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 154.56, + "v_measure": 0.32322412431760067, + "v_measure_std": 0.008083549649938205 + } +} \ No newline at end of file diff --git a/results/optimum/all-MiniLM-L6-v2-unquantized/MedrxivClusteringP2P.json b/results/optimum/all-MiniLM-L6-v2-unquantized/MedrxivClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..abe1892284ae182b5105a483368573aad57b1de0 --- /dev/null +++ b/results/optimum/all-MiniLM-L6-v2-unquantized/MedrxivClusteringP2P.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "e7a26af6f3ae46b30dde8737f02c07b1505bcc73", + "mteb_dataset_name": "MedrxivClusteringP2P", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 1374.84, + "v_measure": 0.3254905871377117, + "v_measure_std": 0.010313809384246527 + } +} \ No newline at end of file diff --git a/results/optimum/all-MiniLM-L6-v2-unquantized/MedrxivClusteringS2S.json b/results/optimum/all-MiniLM-L6-v2-unquantized/MedrxivClusteringS2S.json new file mode 100644 index 0000000000000000000000000000000000000000..be23f0a67552c68842edd4cf29ea0308fcd41f1b --- /dev/null +++ b/results/optimum/all-MiniLM-L6-v2-unquantized/MedrxivClusteringS2S.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "35191c8c0dca72d8ff3efcd72aa802307d469663", + "mteb_dataset_name": "MedrxivClusteringS2S", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 85.98, + "v_measure": 0.29915756420522766, + "v_measure_std": 0.011216563018471815 + } +} \ No newline at end of file diff --git a/results/optimum/all-MiniLM-L6-v2-unquantized/RedditClustering.json b/results/optimum/all-MiniLM-L6-v2-unquantized/RedditClustering.json new file mode 100644 index 0000000000000000000000000000000000000000..74d5924ff684186596d484ac445d0d0d061d6425 --- /dev/null +++ b/results/optimum/all-MiniLM-L6-v2-unquantized/RedditClustering.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "24640382cdbf8abc73003fb0fa6d111a705499eb", + "mteb_dataset_name": "RedditClustering", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 960.49, + "v_measure": 0.4562601827744961, + "v_measure_std": 0.037967391321723803 + } +} \ No newline at end of file diff --git a/results/optimum/all-MiniLM-L6-v2-unquantized/RedditClusteringP2P.json b/results/optimum/all-MiniLM-L6-v2-unquantized/RedditClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..363f79e0726a72317d1bddeba7c33d45c29b596f --- /dev/null +++ b/results/optimum/all-MiniLM-L6-v2-unquantized/RedditClusteringP2P.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "282350215ef01743dc01b456c7f5241fa8937f16", + "mteb_dataset_name": "RedditClusteringP2P", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 7201.67, + "v_measure": 0.5336681782941832, + "v_measure_std": 0.11295961638956403 + } +} \ No newline at end of file diff --git a/results/optimum/all-MiniLM-L6-v2-unquantized/SprintDuplicateQuestions.json b/results/optimum/all-MiniLM-L6-v2-unquantized/SprintDuplicateQuestions.json new file mode 100644 index 0000000000000000000000000000000000000000..eb4638139b399e92e6ce458ca39ea6b3910155c5 --- /dev/null +++ b/results/optimum/all-MiniLM-L6-v2-unquantized/SprintDuplicateQuestions.json @@ -0,0 +1,49 @@ +{ + "dataset_revision": "d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46", + "mteb_dataset_name": "SprintDuplicateQuestions", + "mteb_version": "1.1.0", + "test": { + "cos_sim": { + "accuracy": 0.9978514851485148, + "accuracy_threshold": 0.7130525708198547, + "ap": 0.9455063045792448, + "f1": 0.890126582278481, + "f1_threshold": 0.7130525708198547, + "precision": 0.9015384615384615, + "recall": 0.879 + }, + "dot": { + "accuracy": 0.9949405940594059, + "accuracy_threshold": 22.973613739013672, + "ap": 0.807591492776382, + "f1": 0.7377605428986913, + "f1_threshold": 21.528966903686523, + "precision": 0.715898400752587, + "recall": 0.761 + }, + "euclidean": { + "accuracy": 0.9975247524752475, + "accuracy_threshold": 4.005368232727051, + "ap": 0.922948863946992, + "f1": 0.8700155359917141, + "f1_threshold": 4.0734686851501465, + "precision": 0.9022556390977443, + "recall": 0.84 + }, + "evaluation_time": 13.06, + "manhattan": { + "accuracy": 0.9975247524752475, + "accuracy_threshold": 62.37427520751953, + "ap": 0.9235450475118805, + "f1": 0.8698347107438016, + "f1_threshold": 63.49694061279297, + "precision": 0.8995726495726496, + "recall": 0.842 + }, + "max": { + "accuracy": 0.9978514851485148, + "ap": 0.9455063045792448, + "f1": 0.890126582278481 + } + } +} \ No newline at end of file diff --git a/results/optimum/all-MiniLM-L6-v2-unquantized/StackExchangeClustering.json b/results/optimum/all-MiniLM-L6-v2-unquantized/StackExchangeClustering.json new file mode 100644 index 0000000000000000000000000000000000000000..d54852115a4d1990f7eeb4792df0efb9eb5ecf8f --- /dev/null +++ b/results/optimum/all-MiniLM-L6-v2-unquantized/StackExchangeClustering.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "6cbc1f7b2bc0622f2e39d2c77fa502909748c259", + "mteb_dataset_name": "StackExchangeClustering", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 737.05, + "v_measure": 0.4727910276403759, + "v_measure_std": 0.05819086658404892 + } +} \ No newline at end of file diff --git a/results/optimum/all-MiniLM-L6-v2-unquantized/StackExchangeClusteringP2P.json b/results/optimum/all-MiniLM-L6-v2-unquantized/StackExchangeClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..3cd6423745e8c062eddc7842b2a5bd82baf88fc9 --- /dev/null +++ b/results/optimum/all-MiniLM-L6-v2-unquantized/StackExchangeClusteringP2P.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "815ca46b2622cec33ccafc3735d572c266efdb44", + "mteb_dataset_name": "StackExchangeClusteringP2P", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 1780.19, + "v_measure": 0.3425332156645654, + "v_measure_std": 0.013564685787118119 + } +} \ No newline at end of file diff --git a/results/optimum/all-MiniLM-L6-v2-unquantized/TwentyNewsgroupsClustering.json b/results/optimum/all-MiniLM-L6-v2-unquantized/TwentyNewsgroupsClustering.json new file mode 100644 index 0000000000000000000000000000000000000000..0868f6f0c2809df25b1932a2c56e13ff342b629f --- /dev/null +++ b/results/optimum/all-MiniLM-L6-v2-unquantized/TwentyNewsgroupsClustering.json @@ -0,0 +1,10 @@ +{ + "dataset_revision": "6125ec4e24fa026cec8a478383ee943acfbd5449", + "mteb_dataset_name": "TwentyNewsgroupsClustering", + "mteb_version": "1.1.0", + "test": { + "evaluation_time": 65.36, + "v_measure": 0.42377081093709323, + "v_measure_std": 0.020233734133197876 + } +} \ No newline at end of file diff --git a/results/optimum/all-MiniLM-L6-v2-unquantized/TwitterSemEval2015.json b/results/optimum/all-MiniLM-L6-v2-unquantized/TwitterSemEval2015.json new file mode 100644 index 0000000000000000000000000000000000000000..6a9d55c563fad2fa40af9ec611542dc75a0ffc0d --- /dev/null +++ b/results/optimum/all-MiniLM-L6-v2-unquantized/TwitterSemEval2015.json @@ -0,0 +1,49 @@ +{ + "dataset_revision": "70970daeab8776df92f5ea462b6173c0b46fd2d1", + "mteb_dataset_name": "TwitterSemEval2015", + "mteb_version": "1.1.0", + "test": { + "cos_sim": { + "accuracy": 0.841151576563152, + "accuracy_threshold": 0.7600052356719971, + "ap": 0.6785801239192376, + "f1": 0.6408006919560113, + "f1_threshold": 0.7039604187011719, + "precision": 0.602602835231234, + "recall": 0.6841688654353562 + }, + "dot": { + "accuracy": 0.7892352625618406, + "accuracy_threshold": 28.787059783935547, + "ap": 0.4828865284643087, + "f1": 0.5088373919874313, + "f1_threshold": 20.959426879882812, + "precision": 0.40522364716922116, + "recall": 0.683641160949868 + }, + "euclidean": { + "accuracy": 0.833581689217381, + "accuracy_threshold": 4.012348651885986, + "ap": 0.656114001142199, + "f1": 0.6250154340041981, + "f1_threshold": 4.361933708190918, + "precision": 0.5873752610814574, + "recall": 0.6678100263852242 + }, + "evaluation_time": 18.32, + "manhattan": { + "accuracy": 0.8338797162782381, + "accuracy_threshold": 59.877410888671875, + "ap": 0.6546088131355778, + "f1": 0.6246687545169839, + "f1_threshold": 68.5105972290039, + "precision": 0.5746897163120568, + "recall": 0.6841688654353562 + }, + "max": { + "accuracy": 0.841151576563152, + "ap": 0.6785801239192376, + "f1": 0.6408006919560113 + } + } +} \ No newline at end of file diff --git a/results/optimum/all-MiniLM-L6-v2-unquantized/TwitterURLCorpus.json b/results/optimum/all-MiniLM-L6-v2-unquantized/TwitterURLCorpus.json new file mode 100644 index 0000000000000000000000000000000000000000..9e283d9e20402918bfd56ec2078c6233aa32fcc9 --- /dev/null +++ b/results/optimum/all-MiniLM-L6-v2-unquantized/TwitterURLCorpus.json @@ -0,0 +1,49 @@ +{ + "dataset_revision": "8b6510b0b1fa4e4c4f879467980e9be563ec1cdf", + "mteb_dataset_name": "TwitterURLCorpus", + "mteb_version": "1.1.0", + "test": { + "cos_sim": { + "accuracy": 0.882504754142896, + "accuracy_threshold": 0.6595448851585388, + "ap": 0.8470165722410063, + "f1": 0.7657057281916886, + "f1_threshold": 0.6311963796615601, + "precision": 0.745226643346451, + "recall": 0.7873421619956883 + }, + "dot": { + "accuracy": 0.8516125276516474, + "accuracy_threshold": 14.463157653808594, + "ap": 0.7668983860779237, + "f1": 0.7101665954720207, + "f1_threshold": 12.37724494934082, + "precision": 0.6604210805084746, + "recall": 0.7680166307360641 + }, + "euclidean": { + "accuracy": 0.8597236775720883, + "accuracy_threshold": 3.5921688079833984, + "ap": 0.7900028133299262, + "f1": 0.7074245510090724, + "f1_threshold": 3.922621488571167, + "precision": 0.6814582292930014, + "recall": 0.7354481059439483 + }, + "evaluation_time": 84.53, + "manhattan": { + "accuracy": 0.8594132029339854, + "accuracy_threshold": 55.53254699707031, + "ap": 0.7894101286308219, + "f1": 0.7058084540348802, + "f1_threshold": 60.99834442138672, + "precision": 0.6785308326229043, + "recall": 0.7353711117955035 + }, + "max": { + "accuracy": 0.882504754142896, + "ap": 0.8470165722410063, + "f1": 0.7657057281916886 + } + } +} \ No newline at end of file diff --git a/run_mteb.py b/run_mteb.py index cc0fe9544d74912aed100102fe7c2283c9efabdb..0115f3f5d4d8b403bce478f7a0e2d7fd3254f685 100644 --- a/run_mteb.py +++ b/run_mteb.py @@ -53,14 +53,16 @@ def parse_args(): parser.add_argument("--lang", type=str, default="en") parser.add_argument("--taskname", type=str, default=None) parser.add_argument("--batchsize", type=int, default=128) - parser.add_argument("--device", type=str, default="mps") # sorry :> + parser.add_argument("--device", type=str, default="mps") args = parser.parse_args() return args def main(args): """ - ex: python run_array.py --modelpath ./models/all-MiniLM-L6-v2 + ex: python run_mteb.py --modelpath ./models/all-MiniLM-L6-v2-unquantized + + Optimum/onnx models need to contain string "opt" somewhere in model path or name. """ model = SentenceTransformer(args.modelpath, device=args.device) model_name = args.modelpath.split("/")[-1].split("_")[-1]