upload

CERerankingEvaluator_results.csv

@@ -0,0 +1,127 @@
+epoch,steps,MRR@10
+0,5000,0.5220761904761905
+0,10000,0.5474539682539683
+0,15000,0.5190095238095239
+0,20000,0.5671206349206349
+0,25000,0.5630952380952381
+0,30000,0.565215873015873
+0,35000,0.5821079365079366
+0,40000,0.5496603174603175
+0,45000,0.5716761904761904
+0,50000,0.5707079365079364
+0,55000,0.5709047619047619
+0,60000,0.5643650793650793
+0,65000,0.5586507936507936
+0,70000,0.5792857142857142
+0,75000,0.5964857142857143
+0,80000,0.5759936507936507
+0,85000,0.5640507936507936
+0,90000,0.6016063492063491
+0,95000,0.594984126984127
+0,100000,0.5770507936507936
+0,105000,0.605984126984127
+0,110000,0.6106380952380953
+0,115000,0.5763650793650794
+0,120000,0.5977269841269841
+0,125000,0.5764190476190475
+0,130000,0.5846825396825397
+0,135000,0.5810380952380951
+0,140000,0.5902317460317461
+0,145000,0.6034063492063492
+0,150000,0.5953714285714286
+0,155000,0.5992349206349207
+0,160000,0.6026666666666668
+0,165000,0.6046603174603176
+0,170000,0.5939269841269842
+0,175000,0.6007714285714286
+0,180000,0.574752380952381
+0,185000,0.5923619047619049
+0,190000,0.600431746031746
+0,195000,0.6104984126984127
+0,200000,0.6154095238095239
+0,205000,0.5908285714285714
+0,210000,0.590936507936508
+0,215000,0.6043174603174603
+0,220000,0.6032825396825396
+0,225000,0.6210666666666667
+0,230000,0.6113396825396825
+0,235000,0.6135873015873016
+0,240000,0.6162285714285715
+0,245000,0.6064317460317461
+0,250000,0.6072285714285715
+0,255000,0.6073746031746032
+0,260000,0.6112857142857142
+0,265000,0.6156412698412698
+0,270000,0.6350095238095238
+0,275000,0.6074158730158731
+0,280000,0.6154761904761905
+0,285000,0.6236507936507937
+0,290000,0.6162412698412697
+0,295000,0.616184126984127
+0,300000,0.5997523809523809
+0,305000,0.5937492063492064
+0,310000,0.6227968253968252
+0,315000,0.6274952380952382
+0,320000,0.6269523809523809
+0,325000,0.6306698412698413
+0,330000,0.6235079365079363
+0,335000,0.6206190476190477
+0,340000,0.6209936507936508
+0,345000,0.613095238095238
+0,350000,0.6196952380952381
+0,355000,0.6197301587301588
+0,360000,0.6274634920634922
+0,365000,0.6152730158730159
+0,370000,0.6053968253968254
+0,375000,0.615352380952381
+0,380000,0.6110285714285715
+0,385000,0.621184126984127
+0,390000,0.6025619047619047
+0,395000,0.6122507936507936
+0,400000,0.6189079365079365
+0,405000,0.6252285714285714
+0,410000,0.6022634920634921
+0,415000,0.6053492063492063
+0,420000,0.6239619047619047
+0,425000,0.6127523809523809
+0,430000,0.6231873015873016
+0,435000,0.6233968253968254
+0,440000,0.6186825396825396
+0,445000,0.6279079365079365
+0,450000,0.6075079365079366
+0,455000,0.603352380952381
+0,460000,0.5917142857142857
+0,465000,0.5998285714285714
+0,470000,0.5949492063492064
+0,475000,0.6139714285714285
+0,480000,0.6100507936507936
+0,485000,0.6057619047619048
+0,490000,0.6255714285714286
+0,495000,0.6058158730158729
+0,500000,0.63004126984127
+0,505000,0.6207269841269841
+0,510000,0.6126857142857143
+0,515000,0.6224825396825397
+0,520000,0.6282730158730159
+0,525000,0.6256634920634919
+0,530000,0.6199079365079365
+0,535000,0.6065555555555556
+0,540000,0.6166158730158731
+0,545000,0.6133936507936507
+0,550000,0.6265428571428572
+0,555000,0.6077619047619048
+0,560000,0.6010984126984126
+0,565000,0.6134158730158731
+0,570000,0.6211714285714286
+0,575000,0.6167301587301588
+0,580000,0.6193968253968253
+0,585000,0.605352380952381
+0,590000,0.6013523809523811
+0,595000,0.6070285714285715
+0,600000,0.6075492063492064
+0,605000,0.6051396825396825
+0,610000,0.609984126984127
+0,615000,0.6076412698412699
+0,620000,0.604384126984127
+0,625000,0.6051396825396825
+0,-1,0.6051396825396825

README.md

@@ -0,0 +1,35 @@
+# Cross-Encoder for MS Marco
+
+This model uses [TinyBERT](https://github.com/huawei-noah/Pretrained-Language-Model/tree/master/TinyBERT), a tiny BERT model with only 4 layers. The base model is: General_TinyBERT_v2(4layer-312dim)
+
+It was trained on [MS Marco Passage Ranking](https://github.com/microsoft/MSMARCO-Passage-Ranking) task.
+
+The model can be used for Information Retrieval: Given a query, encode the query will all possible passages (e.g. retrieved with ElasticSearch). Then sort the passages in a decreasing order. See [SBERT.net Information Retrieval](https://github.com/UKPLab/sentence-transformers/tree/master/examples/applications/information-retrieval) for more details. The training code is available here: [SBERT.net Training MS Marco](https://github.com/UKPLab/sentence-transformers/tree/master/examples/training/ms_marco)
+
+## Usage and Performance
+
+Pre-trained models can be used like this:
+```
+from sentence_transformers import CrossEncoder
+model = CrossEncoder('model_name', max_length=512)
+scores = model.predict([('Query', 'Paragraph1'), ('Query', 'Paragraph2') , ('Query', 'Paragraph3')])
+```
+
+In the following table, we provide various pre-trained Cross-Encoders together with their performance on the [TREC Deep Learning 2019](https://microsoft.github.io/TREC-2019-Deep-Learning/) and the [MS Marco Passage Reranking](https://github.com/microsoft/MSMARCO-Passage-Ranking/) dataset. 
+
+
+| Model-Name        | NDCG@10 (TREC DL 19) | MRR@10 (MS Marco Dev)  | Docs / Sec (BertTokenizerFast) | Docs / Sec |
+| ------------- |:-------------| -----| --- | --- |
+| cross-encoder/ms-marco-TinyBERT-L-2  | 67.43 | 30.15  | 9000 | 780
+| cross-encoder/ms-marco-TinyBERT-L-4  | 68.09 | 34.50  | 2900 | 760
+| cross-encoder/ms-marco-TinyBERT-L-6 |  69.57 | 36.13  | 680 | 660
+| cross-encoder/ms-marco-electra-base | 71.99 | 36.41 | 340 | 340
+| *Other models* | | | |
+| nboost/pt-tinybert-msmarco | 63.63 | 28.80 | 2900 | 760
+| nboost/pt-bert-base-uncased-msmarco | 70.94 | 34.75 | 340 | 340|
+| nboost/pt-bert-large-msmarco | 73.36 | 36.48 | 100 | 100 |
+| Capreolus/electra-base-msmarco | 71.23 | | 340 | 340 |
+| amberoad/bert-multilingual-passage-reranking-msmarco | 68.40 | | 330 | 330 
+ 
+ Note: Runtime was computed on a V100 GPU. A bottleneck for smaller models is the standard Python tokenizer from Huggingface in version 3. Replacing it with the fast tokenizer based on Rust, the throughput is significantly improved:
+ 

config.json

@@ -0,0 +1,27 @@
+{
+  "_name_or_path": "nreimers/TinyBERT_L-4_H-312_v2",
+  "architectures": [
+    "BertForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 312,
+  "id2label": {
+    "0": "LABEL_0"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 1200,
+  "label2id": {
+    "LABEL_0": 0
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 4,
+  "pad_token_id": 0,
+  "type_vocab_size": 2,
+  "vocab_size": 30522
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b06785cbb737ac18adf49a4ac3ce4d724479afc9c133d5e83172b12db996dc91
+size 57436041

special_tokens_map.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}

tokenizer_config.json

@@ -0,0 +1 @@
+{"do_lower_case": true, "do_basic_tokenize": true, "never_split": null, "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "tokenize_chinese_chars": true, "strip_accents": null, "model_max_length": 512, "special_tokens_map_file": "/home/ukp-reimers/.cache/torch/transformers/f96b11e14fec8f4be06121e7f6bbe07f82216bf7d75ad76fe3a81251e8895d69.dd8bd9bfd3664b530ea4e645105f557769387b3da9f79bdb55ed556bdd80611d", "tokenizer_file": null, "name_or_path": "nreimers/TinyBERT_L-4_H-312_v2"}

train_script.py

@@ -0,0 +1,192 @@
+from torch.utils.data import DataLoader
+from sentence_transformers import LoggingHandler
+from sentence_transformers.cross_encoder import CrossEncoder
+from sentence_transformers.cross_encoder.evaluation import CEBinaryClassificationEvaluator
+from sentence_transformers import InputExample
+import logging
+from datetime import datetime
+import gzip
+import sys
+import numpy as np
+import os
+from shutil import copyfile
+import csv
+import tqdm
+
+#### Just some code to print debug information to stdout
+logging.basicConfig(format='%(asctime)s - %(message)s',
+                    datefmt='%Y-%m-%d %H:%M:%S',
+                    level=logging.INFO,
+                    handlers=[LoggingHandler()])
+#### /print debug information to stdout
+
+
+#Define our Cross-Encoder
+model_name = sys.argv[1] #'google/electra-small-discriminator'
+train_batch_size = 32
+num_epochs = 1
+model_save_path = 'output/training_ms-marco_cross-encoder-'+model_name.replace("/", "-")+'-'+datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+
+#We set num_labels=1, which predicts a continous score between 0 and 1
+model = CrossEncoder(model_name, num_labels=1, max_length=512)
+
+
+# Write self to path
+os.makedirs(model_save_path, exist_ok=True)
+
+train_script_path = os.path.join(model_save_path, 'train_script.py')
+copyfile(__file__, train_script_path)
+with open(train_script_path, 'a') as fOut:
+    fOut.write("\n\n# Script was called via:\n#python " + " ".join(sys.argv))
+
+
+corpus = {}
+queries = {}
+
+#### Read train file
+with gzip.open('../data/collection.tsv.gz', 'rt') as fIn:
+    for line in fIn:
+        pid, passage = line.strip().split("\t")
+        corpus[pid] = passage
+
+with open('../data/queries.train.tsv', 'r') as fIn:
+    for line in fIn:
+        qid, query = line.strip().split("\t")
+        queries[qid] = query
+
+
+
+pos_neg_ration = (4+1)
+cnt = 0
+train_samples = []
+dev_samples = {}
+
+num_dev_queries = 125
+num_max_dev_negatives = 200
+
+with gzip.open('../data/qidpidtriples.rnd-shuf.train-eval.tsv.gz', 'rt') as fIn:
+    for line in fIn:
+        qid, pos_id, neg_id = line.strip().split()
+
+        if qid not in dev_samples and len(dev_samples) < num_dev_queries:
+            dev_samples[qid] = {'query': queries[qid], 'positive': set(), 'negative': set()}
+
+        if qid in dev_samples:
+            dev_samples[qid]['positive'].add(corpus[pos_id])
+
+            if len(dev_samples[qid]['negative']) < num_max_dev_negatives:
+                dev_samples[qid]['negative'].add(corpus[neg_id])
+
+with gzip.open('../data/qidpidtriples.rnd-shuf.train.tsv.gz', 'rt') as fIn:
+    for line in tqdm.tqdm(fIn, unit_scale=True):
+        cnt += 1
+        qid, pos_id, neg_id = line.strip().split()
+        query = queries[qid]
+        if (cnt % pos_neg_ration) == 0:
+            passage = corpus[pos_id]
+            label = 1
+        else:
+            passage = corpus[neg_id]
+            label = 0
+
+        train_samples.append(InputExample(texts=[query, passage], label=label))
+
+        if len(train_samples) >= 2e7:
+            break
+
+
+
+train_dataloader = DataLoader(train_samples, shuffle=True, batch_size=train_batch_size)
+
+# We add an evaluator, which evaluates the performance during training
+
+class CERerankingEvaluator:
+    def __init__(self, samples, mrr_at_k: int = 10, name: str = ''):
+        self.samples = samples
+        self.name = name
+        self.mrr_at_k = mrr_at_k
+
+        if isinstance(self.samples, dict):
+            self.samples = list(self.samples.values())
+
+        self.csv_file = "CERerankingEvaluator" + ("_" + name if name else '') + "_results.csv"
+        self.csv_headers = ["epoch", "steps", "MRR@{}".format(mrr_at_k)]
+
+    def __call__(self, model, output_path: str = None, epoch: int = -1, steps: int = -1) -> float:
+        if epoch != -1:
+            if steps == -1:
+                out_txt = " after epoch {}:".format(epoch)
+            else:
+                out_txt = " in epoch {} after {} steps:".format(epoch, steps)
+        else:
+            out_txt = ":"
+
+        logging.info("CERerankingEvaluator: Evaluating the model on " + self.name + " dataset" + out_txt)
+
+        all_mrr_scores = []
+        num_queries = 0
+        num_positives = []
+        num_negatives = []
+        for instance in self.samples:
+            query = instance['query']
+            positive = list(instance['positive'])
+            negative = list(instance['negative'])
+            docs = positive + negative
+            is_relevant = [True]*len(positive) + [False]*len(negative)
+
+            if len(positive) == 0 or len(negative) == 0:
+                continue
+
+            num_queries += 1
+            num_positives.append(len(positive))
+            num_negatives.append(len(negative))
+
+            model_input = [[query, doc] for doc in docs]
+            pred_scores = model.predict(model_input, convert_to_numpy=True, show_progress_bar=False)
+            pred_scores_argsort = np.argsort(-pred_scores)  #Sort in decreasing order
+
+            mrr_score = 0
+            for rank, index in enumerate(pred_scores_argsort[0:self.mrr_at_k]):
+                if is_relevant[index]:
+                    mrr_score = 1 / (rank+1)
+
+            all_mrr_scores.append(mrr_score)
+
+        mean_mrr = np.mean(all_mrr_scores)
+        logging.info("Queries: {} \t Positives: Min {:.1f}, Mean {:.1f}, Max {:.1f} \t Negatives: Min {:.1f}, Mean {:.1f}, Max {:.1f}".format(num_queries, np.min(num_positives), np.mean(num_positives), np.max(num_positives), np.min(num_negatives), np.mean(num_negatives), np.max(num_negatives)))
+        logging.info("MRR@{}: {:.2f}".format(self.mrr_at_k, mean_mrr*100))
+
+        if output_path is not None:
+            csv_path = os.path.join(output_path, self.csv_file)
+            output_file_exists = os.path.isfile(csv_path)
+            with open(csv_path, mode="a" if output_file_exists else 'w', encoding="utf-8") as f:
+                writer = csv.writer(f)
+                if not output_file_exists:
+                    writer.writerow(self.csv_headers)
+
+                writer.writerow([epoch, steps, mean_mrr])
+
+        return mean_mrr
+
+
+evaluator = CERerankingEvaluator(dev_samples)
+
+# Configure the training
+warmup_steps = 5000
+logging.info("Warmup-steps: {}".format(warmup_steps))
+
+
+# Train the model
+model.fit(train_dataloader=train_dataloader,
+          evaluator=evaluator,
+          epochs=num_epochs,
+          evaluation_steps=5000,
+          warmup_steps=warmup_steps,
+          output_path=model_save_path,
+          use_amp=True)
+
+#Save latest model
+model.save(model_save_path+'-latest')
+
+# Script was called via:
+#python train_cross-encoder.py nreimers/TinyBERT_L-4_H-312_v2