lets save progress

models/all-MiniLM-L6-v2-q8/sentence_bert_config.json

@@ -1,4 +0,0 @@
-{
-  "max_seq_length": 256,
-  "do_lower_case": false
-}

ort_sentence_transformers.py

@@ -0,0 +1,44 @@
+from sentence_transformers import SentenceTransformer
+from scipy.spatial import distance
+
+
+device = "mps"
+# Load the models
+# so close, and yet, .! so far ~!~ ~
+providers = [
+    ('CoreMLExecutionProvider', {
+        'device_id': 0,
+    }),
+    'CPUExecutionProvider',
+]
+
+model1 = SentenceTransformer('./models/optimum/all-MiniLM-L6-v2', device=device,
+                             model_args={
+                                    "providers": providers
+                                })
+print("\033[91m", model1.modules(), "\033[0m")
+# model2 = SentenceTransformer('./models/all-MiniLM-L6-v2', device=device)
+
+sentences = [
+    'This framework generates embeddings for each input sentence',
+    'Sentences are passed as a list of string.',
+    'The quick brown fox jumps over the lazy dog.'
+]
+
+
+
+# Get embeddings for each sentence from both models
+embeddings1 = model1.encode(sentences)
+# embeddings2 = model2.encode(sentences)
+
+
+# Compute and print the cosine similarity for each sentence's embeddings from the two models
+for sentence, emb1, emb2 in zip(sentences, embeddings1, range(3)):
+    sim = 1 - distance.cosine(emb1, emb2)  # Cosine similarity is the complement of cosine distance
+    print(f"Sentence: {sentence}")
+    print(f"Cosine Similarity: {sim:.4f}")
+    print("")
+
+# print(model2.device)
+
+# should be working perfectly :))

run_mteb.py

@@ -0,0 +1,92 @@
+import argparse
+import logging
+import os
+import time
+
+from mteb import MTEB
+from sentence_transformers import SentenceTransformer
+
+logging.basicConfig(level=logging.INFO)
+
+logger = logging.getLogger("main")
+
+os.environ["HF_DATASETS_OFFLINE"] = "1"  # 1 for offline
+os.environ["TRANSFORMERS_OFFLINE"] = "1"  # 1 for offline
+os.environ["TRANSFORMERS_CACHE"] = "./transformers_cache/"
+os.environ["HF_DATASETS_CACHE"] = "./hf_datasets_cache/"
+os.environ["HF_MODULES_CACHE"] = "./hf_modules_cache/"
+os.environ["HF_METRICS_CACHE"] = "./hf_metrics_cache/"
+# os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+
+
+TASK_LIST_CLUSTERING = [
+    "ArxivClusteringP2P",
+    "ArxivClusteringS2S",
+    "BiorxivClusteringP2P",
+    "BiorxivClusteringS2S",
+    "MedrxivClusteringP2P",
+    "MedrxivClusteringS2S",
+    "RedditClustering",
+    "RedditClusteringP2P",
+    "StackExchangeClustering",
+    "StackExchangeClusteringP2P",
+    "TwentyNewsgroupsClustering",
+]
+
+TASK_LIST_PAIR_CLASSIFICATION = [
+    "SprintDuplicateQuestions",
+    "TwitterSemEval2015",
+    "TwitterURLCorpus",
+]
+
+TASK_LIST = TASK_LIST_CLUSTERING + TASK_LIST_PAIR_CLASSIFICATION
+
+
+def parse_args():
+    # Parse command line arguments
+    parser = argparse.ArgumentParser()
+    # parser.add_argument("--startid", type=int)
+    # parser.add_argument("--endid", type=int)
+
+    parser.add_argument("--modelpath", type=str, default="./models/")
+    parser.add_argument("--lang", type=str, default="en")
+    parser.add_argument("--taskname", type=str, default=None)
+    parser.add_argument("--batchsize", type=int, default=128)
+    parser.add_argument("--device", type=str, default="mps")  # sorry :>
+    args = parser.parse_args()
+    return args
+
+
+def main(args):
+    """
+    ex: python run_array.py --modelpath ./models/all-MiniLM-L6-v2
+    """
+    model = SentenceTransformer(args.modelpath, device=args.device)
+    model_name = args.modelpath.split("/")[-1].split("_")[-1]
+    if not model_name:
+        print(f"Model name is empty. Make sure not to end modelpath with a /")
+        return
+
+    print(f"Running on {model._target_device} with model {model_name}.")
+
+    for task in TASK_LIST:
+        print("Running task: ", task)
+        # this args. notation seems anti-pythonic
+        evaluation = MTEB(tasks=[task], task_langs=[args.lang])
+        retries = 5
+        for attempt in range(retries):
+            try:
+                evaluation.run(model, output_folder=f"results/{model_name}", batch_size=args.batchsize, eval_splits=["test"])
+                break
+            except ConnectionError:
+                if attempt < retries - 1:
+                    print(f"Connection error occurred during task {task}. Waiting for 1 minute before retrying...")
+                    time.sleep(60)
+                else:
+                    print(f"Failed to execute task {task} after {retries} attempts due to connection errors.")
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    main(args)