Upload 136 files

.gitattributes

@@ -32,3 +32,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+qa_output/nq-test/results.jsonl filter=lfs diff=lfs merge=lfs -text
+qa_output/trivia-test/results.jsonl filter=lfs diff=lfs merge=lfs -text
+qa_output/webq-test/results.jsonl filter=lfs diff=lfs merge=lfs -text
+results.jsonl filter=lfs diff=lfs merge=lfs -text

beir_output/arguana.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.17852,0.28888,0.33868,0.39579,0.4317,0.45102,0.45728,0.4609,0.46408
+map,0.17852,0.26079,0.28842,0.31217,0.3223,0.32551,0.32607,0.32626,0.32634
+precision,0.17852,0.12352,0.09829,0.06671,0.04036,0.01807,0.00942,0.00484,0.00099
+recall,0.17852,0.37055,0.49147,0.66714,0.80725,0.90327,0.94168,0.96728,0.99431
+mrr,0.1835,0.26245,0.29061,0.3144,0.32459,0.32779,0.32836,0.32856,0.32864
+recall_cap,0.1835,0.36771,0.49075,0.66643,0.80725,0.90256,0.94168,0.96728,0.99431
+hole,0.66714,0.67354,0.66927,0.67176,0.71486,0.7962,0.83578,0.85851,0.87223

beir_output/arguana.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "arguana",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.17852,
+            "NDCG@3": 0.28888,
+            "NDCG@5": 0.33868,
+            "NDCG@10": 0.39579,
+            "NDCG@20": 0.4317,
+            "NDCG@50": 0.45102,
+            "NDCG@100": 0.45728,
+            "NDCG@200": 0.4609,
+            "NDCG@1000": 0.46408
+        },
+        "map": {
+            "MAP@1": 0.17852,
+            "MAP@3": 0.26079,
+            "MAP@5": 0.28842,
+            "MAP@10": 0.31217,
+            "MAP@20": 0.3223,
+            "MAP@50": 0.32551,
+            "MAP@100": 0.32607,
+            "MAP@200": 0.32626,
+            "MAP@1000": 0.32634
+        },
+        "precision": {
+            "P@1": 0.17852,
+            "P@3": 0.12352,
+            "P@5": 0.09829,
+            "P@10": 0.06671,
+            "P@20": 0.04036,
+            "P@50": 0.01807,
+            "P@100": 0.00942,
+            "P@200": 0.00484,
+            "P@1000": 0.00099
+        },
+        "recall": {
+            "Recall@1": 0.17852,
+            "Recall@3": 0.37055,
+            "Recall@5": 0.49147,
+            "Recall@10": 0.66714,
+            "Recall@20": 0.80725,
+            "Recall@50": 0.90327,
+            "Recall@100": 0.94168,
+            "Recall@200": 0.96728,
+            "Recall@1000": 0.99431
+        },
+        "mrr": {
+            "MRR@1": 0.1835,
+            "MRR@3": 0.26245,
+            "MRR@5": 0.29061,
+            "MRR@10": 0.3144,
+            "MRR@20": 0.32459,
+            "MRR@50": 0.32779,
+            "MRR@100": 0.32836,
+            "MRR@200": 0.32856,
+            "MRR@1000": 0.32864
+        },
+        "recall_cap": {
+            "R_cap@1": 0.1835,
+            "R_cap@3": 0.36771,
+            "R_cap@5": 0.49075,
+            "R_cap@10": 0.66643,
+            "R_cap@20": 0.80725,
+            "R_cap@50": 0.90256,
+            "R_cap@100": 0.94168,
+            "R_cap@200": 0.96728,
+            "R_cap@1000": 0.99431
+        },
+        "hole": {
+            "Hole@1": 0.66714,
+            "Hole@3": 0.67354,
+            "Hole@5": 0.66927,
+            "Hole@10": 0.67176,
+            "Hole@20": 0.71486,
+            "Hole@50": 0.7962,
+            "Hole@100": 0.83578,
+            "Hole@200": 0.85851,
+            "Hole@1000": 0.87223
+        }
+    }
+}

beir_output/beir.json

@@ -0,0 +1,26 @@
+{
+    "eval_beir-msmarco_ndcg@10": 0.21547,
+    "eval_beir-msmarco_recall@10": 0.36814,
+    "eval_beir-msmarco_recall@20": 0.47541,
+    "eval_beir-msmarco_recall@100": 0.68529,
+    "eval_beir-dbpedia-entity_ndcg@10": 0.29269,
+    "eval_beir-dbpedia-entity_recall@10": 0.19543,
+    "eval_beir-dbpedia-entity_recall@20": 0.26224,
+    "eval_beir-dbpedia-entity_recall@100": 0.43193,
+    "eval_beir-fever_ndcg@10": 0.55206,
+    "eval_beir-fever_recall@10": 0.73955,
+    "eval_beir-fever_recall@20": 0.79964,
+    "eval_beir-fever_recall@100": 0.89197,
+    "eval_beir-climate-fever_ndcg@10": 0.1562,
+    "eval_beir-climate-fever_recall@10": 0.19275,
+    "eval_beir-climate-fever_recall@20": 0.25077,
+    "eval_beir-climate-fever_recall@100": 0.42836,
+    "eval_beir-hotpotqa_ndcg@10": 0.49581,
+    "eval_beir-hotpotqa_recall@10": 0.52519,
+    "eval_beir-hotpotqa_recall@20": 0.57731,
+    "eval_beir-hotpotqa_recall@100": 0.68339,
+    "eval_beir-avg_ndcg@10": 0.342446,
+    "eval_beir-avg_recall@10": 0.40421200000000007,
+    "eval_beir-avg_recall@20": 0.473074,
+    "eval_beir-avg_recall@100": 0.624188
+}

beir_output/climate-fever.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.14267,0.12477,0.1338,0.1562,0.17584,0.20196,0.2204,0.23772,0.26753
+map,0.0686,0.09132,0.09911,0.108,0.11332,0.11804,0.12016,0.12139,0.12242
+precision,0.14267,0.0886,0.06788,0.04834,0.03212,0.01846,0.01167,0.00717,0.00202
+recall,0.0686,0.11491,0.14159,0.19275,0.25077,0.34648,0.42836,0.51727,0.6977
+mrr,0.14267,0.18784,0.19924,0.21243,0.2196,0.22405,0.22523,0.22579,0.22612
+recall_cap,0.14267,0.12258,0.14159,0.19275,0.25077,0.34648,0.42836,0.51727,0.6977
+hole,0.63909,0.70684,0.73889,0.7686,0.80199,0.83547,0.85737,0.87996,0.92767

beir_output/climate-fever.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "climate-fever",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.14267,
+            "NDCG@3": 0.12477,
+            "NDCG@5": 0.1338,
+            "NDCG@10": 0.1562,
+            "NDCG@20": 0.17584,
+            "NDCG@50": 0.20196,
+            "NDCG@100": 0.2204,
+            "NDCG@200": 0.23772,
+            "NDCG@1000": 0.26753
+        },
+        "map": {
+            "MAP@1": 0.0686,
+            "MAP@3": 0.09132,
+            "MAP@5": 0.09911,
+            "MAP@10": 0.108,
+            "MAP@20": 0.11332,
+            "MAP@50": 0.11804,
+            "MAP@100": 0.12016,
+            "MAP@200": 0.12139,
+            "MAP@1000": 0.12242
+        },
+        "precision": {
+            "P@1": 0.14267,
+            "P@3": 0.0886,
+            "P@5": 0.06788,
+            "P@10": 0.04834,
+            "P@20": 0.03212,
+            "P@50": 0.01846,
+            "P@100": 0.01167,
+            "P@200": 0.00717,
+            "P@1000": 0.00202
+        },
+        "recall": {
+            "Recall@1": 0.0686,
+            "Recall@3": 0.11491,
+            "Recall@5": 0.14159,
+            "Recall@10": 0.19275,
+            "Recall@20": 0.25077,
+            "Recall@50": 0.34648,
+            "Recall@100": 0.42836,
+            "Recall@200": 0.51727,
+            "Recall@1000": 0.6977
+        },
+        "mrr": {
+            "MRR@1": 0.14267,
+            "MRR@3": 0.18784,
+            "MRR@5": 0.19924,
+            "MRR@10": 0.21243,
+            "MRR@20": 0.2196,
+            "MRR@50": 0.22405,
+            "MRR@100": 0.22523,
+            "MRR@200": 0.22579,
+            "MRR@1000": 0.22612
+        },
+        "recall_cap": {
+            "R_cap@1": 0.14267,
+            "R_cap@3": 0.12258,
+            "R_cap@5": 0.14159,
+            "R_cap@10": 0.19275,
+            "R_cap@20": 0.25077,
+            "R_cap@50": 0.34648,
+            "R_cap@100": 0.42836,
+            "R_cap@200": 0.51727,
+            "R_cap@1000": 0.6977
+        },
+        "hole": {
+            "Hole@1": 0.63909,
+            "Hole@3": 0.70684,
+            "Hole@5": 0.73889,
+            "Hole@10": 0.7686,
+            "Hole@20": 0.80199,
+            "Hole@50": 0.83547,
+            "Hole@100": 0.85737,
+            "Hole@200": 0.87996,
+            "Hole@1000": 0.92767
+        }
+    }
+}

beir_output/cqadupstack.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.191785,0.22964416666666665,0.24790416666666668,0.2689125,0.28783416666666667,0.30862,0.3217433333333333,0.33254666666666666,0.35154250000000004
+map,0.16045,0.20496,0.2168866666666666,0.22648166666666666,0.23229249999999999,0.23614249999999995,0.23753583333333328,0.23825333333333332,0.2388283333333333
+precision,0.191785,0.10717333333333334,0.07819333333333332,0.04877416666666665,0.030159166666666667,0.015413333333333334,0.009021666666666667,0.005186666666666667,0.001340833333333333
+recall,0.16045,0.2544575,0.3006583333333333,0.3638466666666666,0.4333991666666667,0.5287158333333333,0.6019383333333334,0.6707433333333332,0.8166041666666667
+mrr,0.191785,0.23855000000000004,0.2503225,0.25958749999999997,0.2646675,0.2677983333333333,0.26882500000000004,0.26928499999999994,0.2696158333333333
+recall_cap,0.191785,0.2602333333333333,0.30328916666666667,0.36496833333333334,0.43395083333333334,0.52893,0.6020241666666667,0.6707508333333333,0.8166041666666667
+hole,0.7298749999999999,0.8100466666666665,0.8372991666666666,0.8669091666666667,0.8862683333333333,0.9035375,0.9118466666666666,0.9179083333333332,0.9267599999999999

beir_output/cqadupstack.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "cqadupstack",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.191785,
+            "NDCG@3": 0.22964416666666665,
+            "NDCG@5": 0.24790416666666668,
+            "NDCG@10": 0.2689125,
+            "NDCG@20": 0.28783416666666667,
+            "NDCG@50": 0.30862,
+            "NDCG@100": 0.3217433333333333,
+            "NDCG@200": 0.33254666666666666,
+            "NDCG@1000": 0.35154250000000004
+        },
+        "map": {
+            "MAP@1": 0.16045,
+            "MAP@3": 0.20496,
+            "MAP@5": 0.2168866666666666,
+            "MAP@10": 0.22648166666666666,
+            "MAP@20": 0.23229249999999999,
+            "MAP@50": 0.23614249999999995,
+            "MAP@100": 0.23753583333333328,
+            "MAP@200": 0.23825333333333332,
+            "MAP@1000": 0.2388283333333333
+        },
+        "precision": {
+            "P@1": 0.191785,
+            "P@3": 0.10717333333333334,
+            "P@5": 0.07819333333333332,
+            "P@10": 0.04877416666666665,
+            "P@20": 0.030159166666666667,
+            "P@50": 0.015413333333333334,
+            "P@100": 0.009021666666666667,
+            "P@200": 0.005186666666666667,
+            "P@1000": 0.001340833333333333
+        },
+        "recall": {
+            "Recall@1": 0.16045,
+            "Recall@3": 0.2544575,
+            "Recall@5": 0.3006583333333333,
+            "Recall@10": 0.3638466666666666,
+            "Recall@20": 0.4333991666666667,
+            "Recall@50": 0.5287158333333333,
+            "Recall@100": 0.6019383333333334,
+            "Recall@200": 0.6707433333333332,
+            "Recall@1000": 0.8166041666666667
+        },
+        "mrr": {
+            "MRR@1": 0.191785,
+            "MRR@3": 0.23855000000000004,
+            "MRR@5": 0.2503225,
+            "MRR@10": 0.25958749999999997,
+            "MRR@20": 0.2646675,
+            "MRR@50": 0.2677983333333333,
+            "MRR@100": 0.26882500000000004,
+            "MRR@200": 0.26928499999999994,
+            "MRR@1000": 0.2696158333333333
+        },
+        "recall_cap": {
+            "R_cap@1": 0.191785,
+            "R_cap@3": 0.2602333333333333,
+            "R_cap@5": 0.30328916666666667,
+            "R_cap@10": 0.36496833333333334,
+            "R_cap@20": 0.43395083333333334,
+            "R_cap@50": 0.52893,
+            "R_cap@100": 0.6020241666666667,
+            "R_cap@200": 0.6707508333333333,
+            "R_cap@1000": 0.8166041666666667
+        },
+        "hole": {
+            "Hole@1": 0.7298749999999999,
+            "Hole@3": 0.8100466666666665,
+            "Hole@5": 0.8372991666666666,
+            "Hole@10": 0.8669091666666667,
+            "Hole@20": 0.8862683333333333,
+            "Hole@50": 0.9035375,
+            "Hole@100": 0.9118466666666666,
+            "Hole@200": 0.9179083333333332,
+            "Hole@1000": 0.9267599999999999
+        }
+    }
+}

beir_output/dbpedia-entity.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.3575,0.31246,0.30201,0.29269,0.29738,0.31934,0.34713,0.37475,0.42549
+map,0.06375,0.09495,0.11141,0.13299,0.15573,0.1808,0.19181,0.19862,0.20625
+precision,0.4525,0.34833,0.3035,0.23625,0.18662,0.1218,0.08122,0.05351,0.01888
+recall,0.06375,0.11059,0.14503,0.19543,0.26224,0.35463,0.43193,0.51676,0.69154
+mrr,0.4525,0.54667,0.56229,0.57022,0.5761,0.57753,0.57799,0.57814,0.57821
+recall_cap,0.4525,0.36667,0.34317,0.31558,0.31951,0.36545,0.43678,0.52032,0.69215
+hole,0.345,0.4275,0.48,0.55575,0.62588,0.73435,0.80477,0.85945,0.93595

beir_output/dbpedia-entity.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "dbpedia-entity",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.3575,
+            "NDCG@3": 0.31246,
+            "NDCG@5": 0.30201,
+            "NDCG@10": 0.29269,
+            "NDCG@20": 0.29738,
+            "NDCG@50": 0.31934,
+            "NDCG@100": 0.34713,
+            "NDCG@200": 0.37475,
+            "NDCG@1000": 0.42549
+        },
+        "map": {
+            "MAP@1": 0.06375,
+            "MAP@3": 0.09495,
+            "MAP@5": 0.11141,
+            "MAP@10": 0.13299,
+            "MAP@20": 0.15573,
+            "MAP@50": 0.1808,
+            "MAP@100": 0.19181,
+            "MAP@200": 0.19862,
+            "MAP@1000": 0.20625
+        },
+        "precision": {
+            "P@1": 0.4525,
+            "P@3": 0.34833,
+            "P@5": 0.3035,
+            "P@10": 0.23625,
+            "P@20": 0.18662,
+            "P@50": 0.1218,
+            "P@100": 0.08122,
+            "P@200": 0.05351,
+            "P@1000": 0.01888
+        },
+        "recall": {
+            "Recall@1": 0.06375,
+            "Recall@3": 0.11059,
+            "Recall@5": 0.14503,
+            "Recall@10": 0.19543,
+            "Recall@20": 0.26224,
+            "Recall@50": 0.35463,
+            "Recall@100": 0.43193,
+            "Recall@200": 0.51676,
+            "Recall@1000": 0.69154
+        },
+        "mrr": {
+            "MRR@1": 0.4525,
+            "MRR@3": 0.54667,
+            "MRR@5": 0.56229,
+            "MRR@10": 0.57022,
+            "MRR@20": 0.5761,
+            "MRR@50": 0.57753,
+            "MRR@100": 0.57799,
+            "MRR@200": 0.57814,
+            "MRR@1000": 0.57821
+        },
+        "recall_cap": {
+            "R_cap@1": 0.4525,
+            "R_cap@3": 0.36667,
+            "R_cap@5": 0.34317,
+            "R_cap@10": 0.31558,
+            "R_cap@20": 0.31951,
+            "R_cap@50": 0.36545,
+            "R_cap@100": 0.43678,
+            "R_cap@200": 0.52032,
+            "R_cap@1000": 0.69215
+        },
+        "hole": {
+            "Hole@1": 0.345,
+            "Hole@3": 0.4275,
+            "Hole@5": 0.48,
+            "Hole@10": 0.55575,
+            "Hole@20": 0.62588,
+            "Hole@50": 0.73435,
+            "Hole@100": 0.80477,
+            "Hole@200": 0.85945,
+            "Hole@1000": 0.93595
+        }
+    }
+}

beir_output/fever.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.37939,0.48935,0.52415,0.55206,0.56787,0.58048,0.5858,0.58909,0.5935
+map,0.35336,0.45006,0.46989,0.48195,0.48651,0.48864,0.48915,0.48934,0.48946
+precision,0.37939,0.20622,0.14218,0.08059,0.04376,0.01892,0.00985,0.00507,0.00107
+recall,0.35336,0.57215,0.65643,0.73955,0.79964,0.86092,0.89197,0.91386,0.94855
+mrr,0.37939,0.48027,0.50056,0.51251,0.51676,0.51873,0.51918,0.51932,0.5194
+recall_cap,0.37939,0.57333,0.65668,0.73956,0.79964,0.86092,0.89197,0.91386,0.94855
+hole,0.59301,0.76663,0.83507,0.90167,0.94344,0.97351,0.98514,0.99145,0.99724

beir_output/fever.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "fever",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.37939,
+            "NDCG@3": 0.48935,
+            "NDCG@5": 0.52415,
+            "NDCG@10": 0.55206,
+            "NDCG@20": 0.56787,
+            "NDCG@50": 0.58048,
+            "NDCG@100": 0.5858,
+            "NDCG@200": 0.58909,
+            "NDCG@1000": 0.5935
+        },
+        "map": {
+            "MAP@1": 0.35336,
+            "MAP@3": 0.45006,
+            "MAP@5": 0.46989,
+            "MAP@10": 0.48195,
+            "MAP@20": 0.48651,
+            "MAP@50": 0.48864,
+            "MAP@100": 0.48915,
+            "MAP@200": 0.48934,
+            "MAP@1000": 0.48946
+        },
+        "precision": {
+            "P@1": 0.37939,
+            "P@3": 0.20622,
+            "P@5": 0.14218,
+            "P@10": 0.08059,
+            "P@20": 0.04376,
+            "P@50": 0.01892,
+            "P@100": 0.00985,
+            "P@200": 0.00507,
+            "P@1000": 0.00107
+        },
+        "recall": {
+            "Recall@1": 0.35336,
+            "Recall@3": 0.57215,
+            "Recall@5": 0.65643,
+            "Recall@10": 0.73955,
+            "Recall@20": 0.79964,
+            "Recall@50": 0.86092,
+            "Recall@100": 0.89197,
+            "Recall@200": 0.91386,
+            "Recall@1000": 0.94855
+        },
+        "mrr": {
+            "MRR@1": 0.37939,
+            "MRR@3": 0.48027,
+            "MRR@5": 0.50056,
+            "MRR@10": 0.51251,
+            "MRR@20": 0.51676,
+            "MRR@50": 0.51873,
+            "MRR@100": 0.51918,
+            "MRR@200": 0.51932,
+            "MRR@1000": 0.5194
+        },
+        "recall_cap": {
+            "R_cap@1": 0.37939,
+            "R_cap@3": 0.57333,
+            "R_cap@5": 0.65668,
+            "R_cap@10": 0.73956,
+            "R_cap@20": 0.79964,
+            "R_cap@50": 0.86092,
+            "R_cap@100": 0.89197,
+            "R_cap@200": 0.91386,
+            "R_cap@1000": 0.94855
+        },
+        "hole": {
+            "Hole@1": 0.59301,
+            "Hole@3": 0.76663,
+            "Hole@5": 0.83507,
+            "Hole@10": 0.90167,
+            "Hole@20": 0.94344,
+            "Hole@50": 0.97351,
+            "Hole@100": 0.98514,
+            "Hole@200": 0.99145,
+            "Hole@1000": 0.99724
+        }
+    }
+}

beir_output/fiqa.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.22994,0.22025,0.22971,0.25367,0.27826,0.30668,0.32515,0.34001,0.36491
+map,0.11452,0.16638,0.17887,0.19239,0.20087,0.20659,0.20874,0.2099,0.2108
+precision,0.22994,0.14558,0.10833,0.07269,0.04653,0.02429,0.01441,0.00833,0.00215
+recall,0.11452,0.20686,0.24866,0.31546,0.39133,0.50494,0.59399,0.6741,0.83452
+mrr,0.22994,0.28344,0.29879,0.31011,0.31587,0.31959,0.32077,0.32113,0.32139
+recall_cap,0.22994,0.22917,0.25476,0.31595,0.39133,0.50494,0.59399,0.6741,0.83452
+hole,0.71451,0.78961,0.82932,0.86481,0.89252,0.91407,0.92321,0.9284,0.93762

beir_output/fiqa.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "fiqa",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.22994,
+            "NDCG@3": 0.22025,
+            "NDCG@5": 0.22971,
+            "NDCG@10": 0.25367,
+            "NDCG@20": 0.27826,
+            "NDCG@50": 0.30668,
+            "NDCG@100": 0.32515,
+            "NDCG@200": 0.34001,
+            "NDCG@1000": 0.36491
+        },
+        "map": {
+            "MAP@1": 0.11452,
+            "MAP@3": 0.16638,
+            "MAP@5": 0.17887,
+            "MAP@10": 0.19239,
+            "MAP@20": 0.20087,
+            "MAP@50": 0.20659,
+            "MAP@100": 0.20874,
+            "MAP@200": 0.2099,
+            "MAP@1000": 0.2108
+        },
+        "precision": {
+            "P@1": 0.22994,
+            "P@3": 0.14558,
+            "P@5": 0.10833,
+            "P@10": 0.07269,
+            "P@20": 0.04653,
+            "P@50": 0.02429,
+            "P@100": 0.01441,
+            "P@200": 0.00833,
+            "P@1000": 0.00215
+        },
+        "recall": {
+            "Recall@1": 0.11452,
+            "Recall@3": 0.20686,
+            "Recall@5": 0.24866,
+            "Recall@10": 0.31546,
+            "Recall@20": 0.39133,
+            "Recall@50": 0.50494,
+            "Recall@100": 0.59399,
+            "Recall@200": 0.6741,
+            "Recall@1000": 0.83452
+        },
+        "mrr": {
+            "MRR@1": 0.22994,
+            "MRR@3": 0.28344,
+            "MRR@5": 0.29879,
+            "MRR@10": 0.31011,
+            "MRR@20": 0.31587,
+            "MRR@50": 0.31959,
+            "MRR@100": 0.32077,
+            "MRR@200": 0.32113,
+            "MRR@1000": 0.32139
+        },
+        "recall_cap": {
+            "R_cap@1": 0.22994,
+            "R_cap@3": 0.22917,
+            "R_cap@5": 0.25476,
+            "R_cap@10": 0.31595,
+            "R_cap@20": 0.39133,
+            "R_cap@50": 0.50494,
+            "R_cap@100": 0.59399,
+            "R_cap@200": 0.6741,
+            "R_cap@1000": 0.83452
+        },
+        "hole": {
+            "Hole@1": 0.71451,
+            "Hole@3": 0.78961,
+            "Hole@5": 0.82932,
+            "Hole@10": 0.86481,
+            "Hole@20": 0.89252,
+            "Hole@50": 0.91407,
+            "Hole@100": 0.92321,
+            "Hole@200": 0.9284,
+            "Hole@1000": 0.93762
+        }
+    }
+}

beir_output/hotpotqa.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.57691,0.45012,0.47263,0.49581,0.51201,0.52716,0.53597,0.5432,0.55571
+map,0.28845,0.3745,0.38971,0.40177,0.4076,0.41096,0.41204,0.41257,0.41298
+precision,0.57691,0.28157,0.18677,0.10504,0.05773,0.02556,0.01367,0.00726,0.00163
+recall,0.28845,0.42235,0.46691,0.52519,0.57731,0.6391,0.68339,0.72552,0.81472
+mrr,0.57691,0.64267,0.65301,0.66024,0.66314,0.66453,0.66489,0.66503,0.66512
+recall_cap,0.57691,0.42235,0.46691,0.52519,0.57731,0.6391,0.68339,0.72552,0.81472
+hole,0.41702,0.70813,0.80278,0.8845,0.93217,0.96495,0.97742,0.98442,0.99139

beir_output/hotpotqa.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "hotpotqa",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.57691,
+            "NDCG@3": 0.45012,
+            "NDCG@5": 0.47263,
+            "NDCG@10": 0.49581,
+            "NDCG@20": 0.51201,
+            "NDCG@50": 0.52716,
+            "NDCG@100": 0.53597,
+            "NDCG@200": 0.5432,
+            "NDCG@1000": 0.55571
+        },
+        "map": {
+            "MAP@1": 0.28845,
+            "MAP@3": 0.3745,
+            "MAP@5": 0.38971,
+            "MAP@10": 0.40177,
+            "MAP@20": 0.4076,
+            "MAP@50": 0.41096,
+            "MAP@100": 0.41204,
+            "MAP@200": 0.41257,
+            "MAP@1000": 0.41298
+        },
+        "precision": {
+            "P@1": 0.57691,
+            "P@3": 0.28157,
+            "P@5": 0.18677,
+            "P@10": 0.10504,
+            "P@20": 0.05773,
+            "P@50": 0.02556,
+            "P@100": 0.01367,
+            "P@200": 0.00726,
+            "P@1000": 0.00163
+        },
+        "recall": {
+            "Recall@1": 0.28845,
+            "Recall@3": 0.42235,
+            "Recall@5": 0.46691,
+            "Recall@10": 0.52519,
+            "Recall@20": 0.57731,
+            "Recall@50": 0.6391,
+            "Recall@100": 0.68339,
+            "Recall@200": 0.72552,
+            "Recall@1000": 0.81472
+        },
+        "mrr": {
+            "MRR@1": 0.57691,
+            "MRR@3": 0.64267,
+            "MRR@5": 0.65301,
+            "MRR@10": 0.66024,
+            "MRR@20": 0.66314,
+            "MRR@50": 0.66453,
+            "MRR@100": 0.66489,
+            "MRR@200": 0.66503,
+            "MRR@1000": 0.66512
+        },
+        "recall_cap": {
+            "R_cap@1": 0.57691,
+            "R_cap@3": 0.42235,
+            "R_cap@5": 0.46691,
+            "R_cap@10": 0.52519,
+            "R_cap@20": 0.57731,
+            "R_cap@50": 0.6391,
+            "R_cap@100": 0.68339,
+            "R_cap@200": 0.72552,
+            "R_cap@1000": 0.81472
+        },
+        "hole": {
+            "Hole@1": 0.41702,
+            "Hole@3": 0.70813,
+            "Hole@5": 0.80278,
+            "Hole@10": 0.8845,
+            "Hole@20": 0.93217,
+            "Hole@50": 0.96495,
+            "Hole@100": 0.97742,
+            "Hole@200": 0.98442,
+            "Hole@1000": 0.99139
+        }
+    }
+}

beir_output/msmarco.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.09284,0.15537,0.18313,0.21547,0.24287,0.26812,0.28219,0.29379,0.30955
+map,0.09031,0.1381,0.15355,0.16699,0.17454,0.17873,0.17999,0.1806,0.18097
+precision,0.09284,0.06953,0.05564,0.03819,0.02474,0.01256,0.00719,0.00403,0.00095
+recall,0.09031,0.20192,0.26897,0.36814,0.47541,0.5998,0.68529,0.76656,0.90024
+mrr,0.09284,0.14174,0.15716,0.17037,0.1779,0.18201,0.18323,0.18382,0.18417
+recall_cap,0.09284,0.20193,0.26897,0.36814,0.47541,0.5998,0.68529,0.76656,0.90024
+hole,0.90602,0.92927,0.94335,0.9608,0.97418,0.98645,0.99179,0.99493,0.99804

beir_output/msmarco.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "msmarco",
+    "split": "dev",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.09284,
+            "NDCG@3": 0.15537,
+            "NDCG@5": 0.18313,
+            "NDCG@10": 0.21547,
+            "NDCG@20": 0.24287,
+            "NDCG@50": 0.26812,
+            "NDCG@100": 0.28219,
+            "NDCG@200": 0.29379,
+            "NDCG@1000": 0.30955
+        },
+        "map": {
+            "MAP@1": 0.09031,
+            "MAP@3": 0.1381,
+            "MAP@5": 0.15355,
+            "MAP@10": 0.16699,
+            "MAP@20": 0.17454,
+            "MAP@50": 0.17873,
+            "MAP@100": 0.17999,
+            "MAP@200": 0.1806,
+            "MAP@1000": 0.18097
+        },
+        "precision": {
+            "P@1": 0.09284,
+            "P@3": 0.06953,
+            "P@5": 0.05564,
+            "P@10": 0.03819,
+            "P@20": 0.02474,
+            "P@50": 0.01256,
+            "P@100": 0.00719,
+            "P@200": 0.00403,
+            "P@1000": 0.00095
+        },
+        "recall": {
+            "Recall@1": 0.09031,
+            "Recall@3": 0.20192,
+            "Recall@5": 0.26897,
+            "Recall@10": 0.36814,
+            "Recall@20": 0.47541,
+            "Recall@50": 0.5998,
+            "Recall@100": 0.68529,
+            "Recall@200": 0.76656,
+            "Recall@1000": 0.90024
+        },
+        "mrr": {
+            "MRR@1": 0.09284,
+            "MRR@3": 0.14174,
+            "MRR@5": 0.15716,
+            "MRR@10": 0.17037,
+            "MRR@20": 0.1779,
+            "MRR@50": 0.18201,
+            "MRR@100": 0.18323,
+            "MRR@200": 0.18382,
+            "MRR@1000": 0.18417
+        },
+        "recall_cap": {
+            "R_cap@1": 0.09284,
+            "R_cap@3": 0.20193,
+            "R_cap@5": 0.26897,
+            "R_cap@10": 0.36814,
+            "R_cap@20": 0.47541,
+            "R_cap@50": 0.5998,
+            "R_cap@100": 0.68529,
+            "R_cap@200": 0.76656,
+            "R_cap@1000": 0.90024
+        },
+        "hole": {
+            "Hole@1": 0.90602,
+            "Hole@3": 0.92927,
+            "Hole@5": 0.94335,
+            "Hole@10": 0.9608,
+            "Hole@20": 0.97418,
+            "Hole@50": 0.98645,
+            "Hole@100": 0.99179,
+            "Hole@200": 0.99493,
+            "Hole@1000": 0.99804
+        }
+    }
+}

beir_output/nfcorpus.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.36687,0.34358,0.32376,0.29655,0.27597,0.26373,0.27268,0.29088,0.36292
+map,0.04295,0.07304,0.08708,0.10203,0.11246,0.12234,0.12809,0.13292,0.14161
+precision,0.39009,0.33333,0.28669,0.22291,0.16502,0.10415,0.07127,0.04896,0.02023
+recall,0.04295,0.08692,0.11257,0.14464,0.17974,0.22974,0.28415,0.35644,0.60391
+mrr,0.39628,0.46956,0.48256,0.49141,0.49529,0.4971,0.49763,0.49793,0.49811
+recall_cap,0.39628,0.35139,0.32023,0.27996,0.25795,0.25981,0.29779,0.36142,0.60391
+hole,0.06811,0.08669,0.10402,0.11176,0.11316,0.12297,0.12851,0.12817,0.13272

beir_output/nfcorpus.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "nfcorpus",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.36687,
+            "NDCG@3": 0.34358,
+            "NDCG@5": 0.32376,
+            "NDCG@10": 0.29655,
+            "NDCG@20": 0.27597,
+            "NDCG@50": 0.26373,
+            "NDCG@100": 0.27268,
+            "NDCG@200": 0.29088,
+            "NDCG@1000": 0.36292
+        },
+        "map": {
+            "MAP@1": 0.04295,
+            "MAP@3": 0.07304,
+            "MAP@5": 0.08708,
+            "MAP@10": 0.10203,
+            "MAP@20": 0.11246,
+            "MAP@50": 0.12234,
+            "MAP@100": 0.12809,
+            "MAP@200": 0.13292,
+            "MAP@1000": 0.14161
+        },
+        "precision": {
+            "P@1": 0.39009,
+            "P@3": 0.33333,
+            "P@5": 0.28669,
+            "P@10": 0.22291,
+            "P@20": 0.16502,
+            "P@50": 0.10415,
+            "P@100": 0.07127,
+            "P@200": 0.04896,
+            "P@1000": 0.02023
+        },
+        "recall": {
+            "Recall@1": 0.04295,
+            "Recall@3": 0.08692,
+            "Recall@5": 0.11257,
+            "Recall@10": 0.14464,
+            "Recall@20": 0.17974,
+            "Recall@50": 0.22974,
+            "Recall@100": 0.28415,
+            "Recall@200": 0.35644,
+            "Recall@1000": 0.60391
+        },
+        "mrr": {
+            "MRR@1": 0.39628,
+            "MRR@3": 0.46956,
+            "MRR@5": 0.48256,
+            "MRR@10": 0.49141,
+            "MRR@20": 0.49529,
+            "MRR@50": 0.4971,
+            "MRR@100": 0.49763,
+            "MRR@200": 0.49793,
+            "MRR@1000": 0.49811
+        },
+        "recall_cap": {
+            "R_cap@1": 0.39628,
+            "R_cap@3": 0.35139,
+            "R_cap@5": 0.32023,
+            "R_cap@10": 0.27996,
+            "R_cap@20": 0.25795,
+            "R_cap@50": 0.25981,
+            "R_cap@100": 0.29779,
+            "R_cap@200": 0.36142,
+            "R_cap@1000": 0.60391
+        },
+        "hole": {
+            "Hole@1": 0.06811,
+            "Hole@3": 0.08669,
+            "Hole@5": 0.10402,
+            "Hole@10": 0.11176,
+            "Hole@20": 0.11316,
+            "Hole@50": 0.12297,
+            "Hole@100": 0.12851,
+            "Hole@200": 0.12817,
+            "Hole@1000": 0.13272
+        }
+    }
+}

beir_output/nq.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.13268,0.20055,0.23512,0.2742,0.30826,0.3351,0.35119,0.36108,0.37105
+map,0.11602,0.17686,0.19651,0.21343,0.22334,0.22808,0.22966,0.23023,0.23051
+precision,0.13268,0.09695,0.07758,0.05272,0.0342,0.01682,0.00957,0.0052,0.00114
+recall,0.11602,0.25043,0.33099,0.44602,0.57443,0.70237,0.79582,0.8624,0.94284
+mrr,0.13268,0.19559,0.21521,0.231,0.23989,0.24387,0.24519,0.24563,0.24583
+recall_cap,0.13268,0.25053,0.33099,0.44602,0.57443,0.70237,0.79582,0.8624,0.94284
+hole,0.86356,0.90025,0.91952,0.9445,0.96288,0.98023,0.98768,0.99208,0.99636

beir_output/nq.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "nq",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.13268,
+            "NDCG@3": 0.20055,
+            "NDCG@5": 0.23512,
+            "NDCG@10": 0.2742,
+            "NDCG@20": 0.30826,
+            "NDCG@50": 0.3351,
+            "NDCG@100": 0.35119,
+            "NDCG@200": 0.36108,
+            "NDCG@1000": 0.37105
+        },
+        "map": {
+            "MAP@1": 0.11602,
+            "MAP@3": 0.17686,
+            "MAP@5": 0.19651,
+            "MAP@10": 0.21343,
+            "MAP@20": 0.22334,
+            "MAP@50": 0.22808,
+            "MAP@100": 0.22966,
+            "MAP@200": 0.23023,
+            "MAP@1000": 0.23051
+        },
+        "precision": {
+            "P@1": 0.13268,
+            "P@3": 0.09695,
+            "P@5": 0.07758,
+            "P@10": 0.05272,
+            "P@20": 0.0342,
+            "P@50": 0.01682,
+            "P@100": 0.00957,
+            "P@200": 0.0052,
+            "P@1000": 0.00114
+        },
+        "recall": {
+            "Recall@1": 0.11602,
+            "Recall@3": 0.25043,
+            "Recall@5": 0.33099,
+            "Recall@10": 0.44602,
+            "Recall@20": 0.57443,
+            "Recall@50": 0.70237,
+            "Recall@100": 0.79582,
+            "Recall@200": 0.8624,
+            "Recall@1000": 0.94284
+        },
+        "mrr": {
+            "MRR@1": 0.13268,
+            "MRR@3": 0.19559,
+            "MRR@5": 0.21521,
+            "MRR@10": 0.231,
+            "MRR@20": 0.23989,
+            "MRR@50": 0.24387,
+            "MRR@100": 0.24519,
+            "MRR@200": 0.24563,
+            "MRR@1000": 0.24583
+        },
+        "recall_cap": {
+            "R_cap@1": 0.13268,
+            "R_cap@3": 0.25053,
+            "R_cap@5": 0.33099,
+            "R_cap@10": 0.44602,
+            "R_cap@20": 0.57443,
+            "R_cap@50": 0.70237,
+            "R_cap@100": 0.79582,
+            "R_cap@200": 0.8624,
+            "R_cap@1000": 0.94284
+        },
+        "hole": {
+            "Hole@1": 0.86356,
+            "Hole@3": 0.90025,
+            "Hole@5": 0.91952,
+            "Hole@10": 0.9445,
+            "Hole@20": 0.96288,
+            "Hole@50": 0.98023,
+            "Hole@100": 0.98768,
+            "Hole@200": 0.99208,
+            "Hole@1000": 0.99636
+        }
+    }
+}

beir_output/quora.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.6754,0.73528,0.75736,0.77792,0.79031,0.79909,0.80225,0.8041,0.80573
+map,0.5901,0.69448,0.71415,0.72735,0.73302,0.73562,0.73619,0.7364,0.7365
+precision,0.6754,0.32203,0.21556,0.12091,0.06573,0.02839,0.01471,0.00754,0.00155
+recall,0.5901,0.76907,0.82662,0.88573,0.9275,0.96223,0.97682,0.9868,0.99658
+mrr,0.6751,0.74512,0.75582,0.76212,0.76427,0.76504,0.76516,0.76521,0.76523
+recall_cap,0.6751,0.78148,0.83155,0.88718,0.92779,0.96226,0.97682,0.9868,0.99658
+hole,0.305,0.64493,0.74606,0.83775,0.89183,0.92915,0.94396,0.95282,0.96182

beir_output/quora.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "quora",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.6754,
+            "NDCG@3": 0.73528,
+            "NDCG@5": 0.75736,
+            "NDCG@10": 0.77792,
+            "NDCG@20": 0.79031,
+            "NDCG@50": 0.79909,
+            "NDCG@100": 0.80225,
+            "NDCG@200": 0.8041,
+            "NDCG@1000": 0.80573
+        },
+        "map": {
+            "MAP@1": 0.5901,
+            "MAP@3": 0.69448,
+            "MAP@5": 0.71415,
+            "MAP@10": 0.72735,
+            "MAP@20": 0.73302,
+            "MAP@50": 0.73562,
+            "MAP@100": 0.73619,
+            "MAP@200": 0.7364,
+            "MAP@1000": 0.7365
+        },
+        "precision": {
+            "P@1": 0.6754,
+            "P@3": 0.32203,
+            "P@5": 0.21556,
+            "P@10": 0.12091,
+            "P@20": 0.06573,
+            "P@50": 0.02839,
+            "P@100": 0.01471,
+            "P@200": 0.00754,
+            "P@1000": 0.00155
+        },
+        "recall": {
+            "Recall@1": 0.5901,
+            "Recall@3": 0.76907,
+            "Recall@5": 0.82662,
+            "Recall@10": 0.88573,
+            "Recall@20": 0.9275,
+            "Recall@50": 0.96223,
+            "Recall@100": 0.97682,
+            "Recall@200": 0.9868,
+            "Recall@1000": 0.99658
+        },
+        "mrr": {
+            "MRR@1": 0.6751,
+            "MRR@3": 0.74512,
+            "MRR@5": 0.75582,
+            "MRR@10": 0.76212,
+            "MRR@20": 0.76427,
+            "MRR@50": 0.76504,
+            "MRR@100": 0.76516,
+            "MRR@200": 0.76521,
+            "MRR@1000": 0.76523
+        },
+        "recall_cap": {
+            "R_cap@1": 0.6751,
+            "R_cap@3": 0.78148,
+            "R_cap@5": 0.83155,
+            "R_cap@10": 0.88718,
+            "R_cap@20": 0.92779,
+            "R_cap@50": 0.96226,
+            "R_cap@100": 0.97682,
+            "R_cap@200": 0.9868,
+            "R_cap@1000": 0.99658
+        },
+        "hole": {
+            "Hole@1": 0.305,
+            "Hole@3": 0.64493,
+            "Hole@5": 0.74606,
+            "Hole@10": 0.83775,
+            "Hole@20": 0.89183,
+            "Hole@50": 0.92915,
+            "Hole@100": 0.94396,
+            "Hole@200": 0.95282,
+            "Hole@1000": 0.96182
+        }
+    }
+}

beir_output/scidocs.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.181,0.14792,0.12849,0.15413,0.17987,0.2085,0.22665,0.24445,0.28012
+map,0.03693,0.06617,0.07757,0.08875,0.09734,0.10354,0.10595,0.10741,0.10878
+precision,0.181,0.13767,0.1116,0.0793,0.0545,0.03026,0.0184,0.01106,0.00312
+recall,0.03693,0.08423,0.11338,0.16083,0.22063,0.3065,0.37255,0.44818,0.63403
+mrr,0.181,0.24383,0.26043,0.27504,0.2824,0.28623,0.28731,0.28775,0.28804
+recall_cap,0.181,0.13767,0.11338,0.16083,0.22063,0.3065,0.37255,0.44818,0.63403
+hole,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0

beir_output/scidocs.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "scidocs",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.181,
+            "NDCG@3": 0.14792,
+            "NDCG@5": 0.12849,
+            "NDCG@10": 0.15413,
+            "NDCG@20": 0.17987,
+            "NDCG@50": 0.2085,
+            "NDCG@100": 0.22665,
+            "NDCG@200": 0.24445,
+            "NDCG@1000": 0.28012
+        },
+        "map": {
+            "MAP@1": 0.03693,
+            "MAP@3": 0.06617,
+            "MAP@5": 0.07757,
+            "MAP@10": 0.08875,
+            "MAP@20": 0.09734,
+            "MAP@50": 0.10354,
+            "MAP@100": 0.10595,
+            "MAP@200": 0.10741,
+            "MAP@1000": 0.10878
+        },
+        "precision": {
+            "P@1": 0.181,
+            "P@3": 0.13767,
+            "P@5": 0.1116,
+            "P@10": 0.0793,
+            "P@20": 0.0545,
+            "P@50": 0.03026,
+            "P@100": 0.0184,
+            "P@200": 0.01106,
+            "P@1000": 0.00312
+        },
+        "recall": {
+            "Recall@1": 0.03693,
+            "Recall@3": 0.08423,
+            "Recall@5": 0.11338,
+            "Recall@10": 0.16083,
+            "Recall@20": 0.22063,
+            "Recall@50": 0.3065,
+            "Recall@100": 0.37255,
+            "Recall@200": 0.44818,
+            "Recall@1000": 0.63403
+        },
+        "mrr": {
+            "MRR@1": 0.181,
+            "MRR@3": 0.24383,
+            "MRR@5": 0.26043,
+            "MRR@10": 0.27504,
+            "MRR@20": 0.2824,
+            "MRR@50": 0.28623,
+            "MRR@100": 0.28731,
+            "MRR@200": 0.28775,
+            "MRR@1000": 0.28804
+        },
+        "recall_cap": {
+            "R_cap@1": 0.181,
+            "R_cap@3": 0.13767,
+            "R_cap@5": 0.11338,
+            "R_cap@10": 0.16083,
+            "R_cap@20": 0.22063,
+            "R_cap@50": 0.3065,
+            "R_cap@100": 0.37255,
+            "R_cap@200": 0.44818,
+            "R_cap@1000": 0.63403
+        },
+        "hole": {
+            "Hole@1": 0.0,
+            "Hole@3": 0.0,
+            "Hole@5": 0.0,
+            "Hole@10": 0.0,
+            "Hole@20": 0.0,
+            "Hole@50": 0.0,
+            "Hole@100": 0.0,
+            "Hole@200": 0.0,
+            "Hole@1000": 0.0
+        }
+    }
+}

beir_output/scifact.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.50667,0.60342,0.6286,0.65609,0.66614,0.67427,0.67852,0.68338,0.68859
+map,0.47983,0.57024,0.58777,0.59991,0.60339,0.60496,0.60537,0.60564,0.60576
+precision,0.50667,0.24111,0.16133,0.09033,0.04783,0.02,0.0103,0.00535,0.00112
+recall,0.47983,0.66756,0.73033,0.81167,0.84756,0.88589,0.91156,0.94489,0.99
+mrr,0.50667,0.59333,0.6065,0.61726,0.61933,0.62053,0.62085,0.62107,0.62117
+recall_cap,0.50667,0.67111,0.73033,0.81167,0.84756,0.88589,0.91156,0.94489,0.99
+hole,0.47667,0.72333,0.79267,0.85467,0.8995,0.92353,0.93103,0.93655,0.94158

beir_output/scifact.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "scifact",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.50667,
+            "NDCG@3": 0.60342,
+            "NDCG@5": 0.6286,
+            "NDCG@10": 0.65609,
+            "NDCG@20": 0.66614,
+            "NDCG@50": 0.67427,
+            "NDCG@100": 0.67852,
+            "NDCG@200": 0.68338,
+            "NDCG@1000": 0.68859
+        },
+        "map": {
+            "MAP@1": 0.47983,
+            "MAP@3": 0.57024,
+            "MAP@5": 0.58777,
+            "MAP@10": 0.59991,
+            "MAP@20": 0.60339,
+            "MAP@50": 0.60496,
+            "MAP@100": 0.60537,
+            "MAP@200": 0.60564,
+            "MAP@1000": 0.60576
+        },
+        "precision": {
+            "P@1": 0.50667,
+            "P@3": 0.24111,
+            "P@5": 0.16133,
+            "P@10": 0.09033,
+            "P@20": 0.04783,
+            "P@50": 0.02,
+            "P@100": 0.0103,
+            "P@200": 0.00535,
+            "P@1000": 0.00112
+        },
+        "recall": {
+            "Recall@1": 0.47983,
+            "Recall@3": 0.66756,
+            "Recall@5": 0.73033,
+            "Recall@10": 0.81167,
+            "Recall@20": 0.84756,
+            "Recall@50": 0.88589,
+            "Recall@100": 0.91156,
+            "Recall@200": 0.94489,
+            "Recall@1000": 0.99
+        },
+        "mrr": {
+            "MRR@1": 0.50667,
+            "MRR@3": 0.59333,
+            "MRR@5": 0.6065,
+            "MRR@10": 0.61726,
+            "MRR@20": 0.61933,
+            "MRR@50": 0.62053,
+            "MRR@100": 0.62085,
+            "MRR@200": 0.62107,
+            "MRR@1000": 0.62117
+        },
+        "recall_cap": {
+            "R_cap@1": 0.50667,
+            "R_cap@3": 0.67111,
+            "R_cap@5": 0.73033,
+            "R_cap@10": 0.81167,
+            "R_cap@20": 0.84756,
+            "R_cap@50": 0.88589,
+            "R_cap@100": 0.91156,
+            "R_cap@200": 0.94489,
+            "R_cap@1000": 0.99
+        },
+        "hole": {
+            "Hole@1": 0.47667,
+            "Hole@3": 0.72333,
+            "Hole@5": 0.79267,
+            "Hole@10": 0.85467,
+            "Hole@20": 0.8995,
+            "Hole@50": 0.92353,
+            "Hole@100": 0.93103,
+            "Hole@200": 0.93655,
+            "Hole@1000": 0.94158
+        }
+    }
+}

beir_output/senteval-core.json

@@ -0,0 +1,5 @@
+{
+    "eval_senteval-stsb_spearman": 0.7888860081302335,
+    "eval_senteval-sickr_spearman": 0.7277741174377886,
+    "eval_senteval-avg_sts": 0.758330062784011
+}

beir_output/senteval.json

@@ -0,0 +1,99 @@
+{
+    "STSBenchmark": {
+        "train": {
+            "pearson": [
+                0.7671625496286432,
+                0.0
+            ],
+            "spearman": [
+                0.7407051823403011,
+                0.0
+            ],
+            "nsamples": 5749
+        },
+        "dev": {
+            "pearson": [
+                0.7813083270941137,
+                5.741719519093736e-309
+            ],
+            "spearman": [
+                0.7888860081302335,
+                4.6194e-319
+            ],
+            "nsamples": 1500
+        },
+        "test": {
+            "pearson": [
+                0.7624448096598531,
+                1.3012886054724166e-262
+            ],
+            "spearman": [
+                0.7602771570534436,
+                2.908594263406545e-260
+            ],
+            "nsamples": 1379
+        },
+        "all": {
+            "pearson": {
+                "all": 0.7695053681260078,
+                "mean": 0.7703052287942033,
+                "wmean": 0.7688678002986993
+            },
+            "spearman": {
+                "all": 0.7564014032837215,
+                "mean": 0.7632894491746595,
+                "wmean": 0.7522097015584654
+            }
+        }
+    },
+    "SICKRelatedness": {
+        "train": {
+            "pearson": [
+                0.7772208691267224,
+                0.0
+            ],
+            "spearman": [
+                0.6978066630711928,
+                0.0
+            ],
+            "nsamples": 4500
+        },
+        "dev": {
+            "pearson": [
+                0.7909750341980787,
+                2.5296588439552555e-108
+            ],
+            "spearman": [
+                0.7277741174377886,
+                1.3244056458147093e-83
+            ],
+            "nsamples": 500
+        },
+        "test": {
+            "pearson": [
+                0.771314072761317,
+                0.0
+            ],
+            "spearman": [
+                0.6895861583188468,
+                0.0
+            ],
+            "nsamples": 4927
+        },
+        "all": {
+            "pearson": {
+                "all": 0.7749070438706636,
+                "mean": 0.7798366586953728,
+                "wmean": 0.7749819547360027
+            },
+            "spearman": {
+                "all": 0.6952695805138401,
+                "mean": 0.7050556462759427,
+                "wmean": 0.6952360274580659
+            }
+        }
+    },
+    "eval_senteval-stsb_spearman": 0.7888860081302335,
+    "eval_senteval-sickr_spearman": 0.7277741174377886,
+    "eval_senteval-avg_sts": 0.758330062784011
+}

beir_output/trec-covid.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.61,0.58112,0.5741,0.56947,0.53901,0.50299,0.44789,0.38906,0.4155
+map,0.00183,0.00488,0.00756,0.0133,0.02337,0.04848,0.07739,0.11343,0.19245
+precision,0.66,0.64667,0.64,0.622,0.587,0.5424,0.469,0.3914,0.19078
+recall,0.00183,0.00516,0.00821,0.01571,0.02978,0.06582,0.11015,0.17562,0.39938
+mrr,0.66,0.73333,0.73833,0.75694,0.75694,0.7574,0.7574,0.7574,0.7574
+recall_cap,0.66,0.64667,0.636,0.622,0.587,0.5424,0.469,0.39494,0.401
+hole,0.14,0.12,0.14,0.138,0.174,0.2092,0.2448,0.288,0.4139

beir_output/trec-covid.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "trec-covid",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.61,
+            "NDCG@3": 0.58112,
+            "NDCG@5": 0.5741,
+            "NDCG@10": 0.56947,
+            "NDCG@20": 0.53901,
+            "NDCG@50": 0.50299,
+            "NDCG@100": 0.44789,
+            "NDCG@200": 0.38906,
+            "NDCG@1000": 0.4155
+        },
+        "map": {
+            "MAP@1": 0.00183,
+            "MAP@3": 0.00488,
+            "MAP@5": 0.00756,
+            "MAP@10": 0.0133,
+            "MAP@20": 0.02337,
+            "MAP@50": 0.04848,
+            "MAP@100": 0.07739,
+            "MAP@200": 0.11343,
+            "MAP@1000": 0.19245
+        },
+        "precision": {
+            "P@1": 0.66,
+            "P@3": 0.64667,
+            "P@5": 0.64,
+            "P@10": 0.622,
+            "P@20": 0.587,
+            "P@50": 0.5424,
+            "P@100": 0.469,
+            "P@200": 0.3914,
+            "P@1000": 0.19078
+        },
+        "recall": {
+            "Recall@1": 0.00183,
+            "Recall@3": 0.00516,
+            "Recall@5": 0.00821,
+            "Recall@10": 0.01571,
+            "Recall@20": 0.02978,
+            "Recall@50": 0.06582,
+            "Recall@100": 0.11015,
+            "Recall@200": 0.17562,
+            "Recall@1000": 0.39938
+        },
+        "mrr": {
+            "MRR@1": 0.66,
+            "MRR@3": 0.73333,
+            "MRR@5": 0.73833,
+            "MRR@10": 0.75694,
+            "MRR@20": 0.75694,
+            "MRR@50": 0.7574,
+            "MRR@100": 0.7574,
+            "MRR@200": 0.7574,
+            "MRR@1000": 0.7574
+        },
+        "recall_cap": {
+            "R_cap@1": 0.66,
+            "R_cap@3": 0.64667,
+            "R_cap@5": 0.636,
+            "R_cap@10": 0.622,
+            "R_cap@20": 0.587,
+            "R_cap@50": 0.5424,
+            "R_cap@100": 0.469,
+            "R_cap@200": 0.39494,
+            "R_cap@1000": 0.401
+        },
+        "hole": {
+            "Hole@1": 0.14,
+            "Hole@3": 0.12,
+            "Hole@5": 0.14,
+            "Hole@10": 0.138,
+            "Hole@20": 0.174,
+            "Hole@50": 0.2092,
+            "Hole@100": 0.2448,
+            "Hole@200": 0.288,
+            "Hole@1000": 0.4139
+        }
+    }
+}

beir_output/webis-touche2020.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.21429,0.18461,0.18374,0.18628,0.20488,0.25506,0.2983,0.34648,0.4193
+map,0.01776,0.03354,0.04519,0.06783,0.08833,0.10596,0.11535,0.12418,0.13174
+precision,0.22449,0.18367,0.17959,0.16531,0.13673,0.0902,0.06306,0.04439,0.01416
+recall,0.01776,0.04443,0.07258,0.12937,0.20379,0.31054,0.4094,0.54264,0.77146
+mrr,0.22449,0.33673,0.37041,0.39022,0.39343,0.39735,0.39759,0.39796,0.39805
+recall_cap,0.22449,0.18367,0.17959,0.18662,0.20962,0.31054,0.4094,0.54264,0.77146
+hole,0.69388,0.7551,0.76327,0.79184,0.82347,0.87551,0.90592,0.9298,0.96933

beir_output/webis-touche2020.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "webis-touche2020",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.21429,
+            "NDCG@3": 0.18461,
+            "NDCG@5": 0.18374,
+            "NDCG@10": 0.18628,
+            "NDCG@20": 0.20488,
+            "NDCG@50": 0.25506,
+            "NDCG@100": 0.2983,
+            "NDCG@200": 0.34648,
+            "NDCG@1000": 0.4193
+        },
+        "map": {
+            "MAP@1": 0.01776,
+            "MAP@3": 0.03354,
+            "MAP@5": 0.04519,
+            "MAP@10": 0.06783,
+            "MAP@20": 0.08833,
+            "MAP@50": 0.10596,
+            "MAP@100": 0.11535,
+            "MAP@200": 0.12418,
+            "MAP@1000": 0.13174
+        },
+        "precision": {
+            "P@1": 0.22449,
+            "P@3": 0.18367,
+            "P@5": 0.17959,
+            "P@10": 0.16531,
+            "P@20": 0.13673,
+            "P@50": 0.0902,
+            "P@100": 0.06306,
+            "P@200": 0.04439,
+            "P@1000": 0.01416
+        },
+        "recall": {
+            "Recall@1": 0.01776,
+            "Recall@3": 0.04443,
+            "Recall@5": 0.07258,
+            "Recall@10": 0.12937,
+            "Recall@20": 0.20379,
+            "Recall@50": 0.31054,
+            "Recall@100": 0.4094,
+            "Recall@200": 0.54264,
+            "Recall@1000": 0.77146
+        },
+        "mrr": {
+            "MRR@1": 0.22449,
+            "MRR@3": 0.33673,
+            "MRR@5": 0.37041,
+            "MRR@10": 0.39022,
+            "MRR@20": 0.39343,
+            "MRR@50": 0.39735,
+            "MRR@100": 0.39759,
+            "MRR@200": 0.39796,
+            "MRR@1000": 0.39805
+        },
+        "recall_cap": {
+            "R_cap@1": 0.22449,
+            "R_cap@3": 0.18367,
+            "R_cap@5": 0.17959,
+            "R_cap@10": 0.18662,
+            "R_cap@20": 0.20962,
+            "R_cap@50": 0.31054,
+            "R_cap@100": 0.4094,
+            "R_cap@200": 0.54264,
+            "R_cap@1000": 0.77146
+        },
+        "hole": {
+            "Hole@1": 0.69388,
+            "Hole@3": 0.7551,
+            "Hole@5": 0.76327,
+            "Hole@10": 0.79184,
+            "Hole@20": 0.82347,
+            "Hole@50": 0.87551,
+            "Hole@100": 0.90592,
+            "Hole@200": 0.9298,
+            "Hole@1000": 0.96933
+        }
+    }
+}

cc.moco.T0title50.bs1024.gpu8.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+export LOCAL_RANK=0
+export WORLD_SIZE=8
+
+export TOKENIZERS_PARALLELISM=true
+export NUM_WORKER=4
+export MAX_STEPS=100000
+
+export CUDA_LAUNCH_BLOCKING=1
+export NCCL_DEBUG=INFO
+
+export EXP_NAME=cc.T03b_title50.moco-2e14.contriever256-special50.bert-base-uncased.avg.dot.q128d256.step100k.bs1024.lr5e5
+export PROJECT_DIR=output_dir/$EXP_NAME
+mkdir -p $PROJECT_DIR
+cp "$0" $PROJECT_DIR  # copy bash to project dir
+echo $PROJECT_DIR
+
+export WANDB_NAME=$EXP_NAME
+export WANDB_PROJECT=unsup_retrieval_cc
+export WANDB_DIR=$PROJECT_DIR
+mkdir -p $WANDB_DIR/wandb
+
+nohup python -m torch.distributed.launch --nproc_per_node=8 --master_port=31133 --max_restarts=0 train.py --model_name_or_path bert-base-uncased --arch_type moco --train_file /export/home/data/search/upr/cc/T03B_PileCC_title.json --dev_file /export/home/data/pretrain/wiki2021_structure/wiki_psgs_w100.dev.tail2e13.tsv --data_type hf --data_pipeline_name contriever256-special50% --remove_unused_columns False --sim_type dot --queue_size 16384 --momentum 0.9995 --output_dir $PROJECT_DIR --cache_dir /export/home/data/pretrain/.cache --max_steps $MAX_STEPS --warmup_steps 10000 --logging_steps 100 --eval_steps 10000 --save_steps 1000000 --per_device_train_batch_size 128 --per_device_eval_batch_size 128 --dataloader_num_workers $NUM_WORKER --learning_rate 5e-5 --max_q_tokens 128 --max_d_tokens 256 --evaluation_strategy steps --load_best_model_at_end --overwrite_output_dir --do_train --do_eval --run_name $EXP_NAME --fp16 --seed 42 --report_to wandb --wiki_passage_path /export/home/data/search/nq/psgs_w100.tsv --qa_datasets_path /export/home/data/search/nq/qas/*-test.csv,/export/home/data/search/nq/qas/entityqs/test/P*.test.json > $PROJECT_DIR/nohup.log 2>&1 & echo $! > run.pid
+
+#export WANDB_RUN_ID=13h4vrrq
+#nohup python -m torch.distributed.launch --nproc_per_node=8 --master_port=3112 --max_restarts=0 train.py --model_name_or_path bert-base-uncased --arch_type inbatch --reload_model_from $PROJECT_DIR --remove_unused_columns False --sim_type dot --output_dir $PROJECT_DIR --cache_dir /export/home/data/pretrain/.cache --max_steps $MAX_STEPS --do_eval --run_name $EXP_NAME --fp16 --seed 477 --report_to wandb --wiki_passage_path /export/home/data/search/nq/psgs_w100.tsv --qa_datasets_path /export/home/data/search/nq/qas/*-test.csv,/export/home/data/search/nq/qas/entityqs/test/P*.test.json > $PROJECT_DIR/nohup-eval.log 2>&1 & echo $! > run.pid
+

eval_output/checkpoint-100000/arguana.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.17852,0.28888,0.33868,0.39579,0.4317,0.45102,0.45728,0.4609,0.46408
+map,0.17852,0.26079,0.28842,0.31217,0.3223,0.32551,0.32607,0.32626,0.32634
+precision,0.17852,0.12352,0.09829,0.06671,0.04036,0.01807,0.00942,0.00484,0.00099
+recall,0.17852,0.37055,0.49147,0.66714,0.80725,0.90327,0.94168,0.96728,0.99431
+mrr,0.1835,0.26245,0.29061,0.3144,0.32459,0.32779,0.32836,0.32856,0.32864
+recall_cap,0.1835,0.36771,0.49075,0.66643,0.80725,0.90256,0.94168,0.96728,0.99431
+hole,0.66714,0.67354,0.66927,0.67176,0.71486,0.7962,0.83578,0.85851,0.87223

eval_output/checkpoint-100000/arguana.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "arguana",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.17852,
+            "NDCG@3": 0.28888,
+            "NDCG@5": 0.33868,
+            "NDCG@10": 0.39579,
+            "NDCG@20": 0.4317,
+            "NDCG@50": 0.45102,
+            "NDCG@100": 0.45728,
+            "NDCG@200": 0.4609,
+            "NDCG@1000": 0.46408
+        },
+        "map": {
+            "MAP@1": 0.17852,
+            "MAP@3": 0.26079,
+            "MAP@5": 0.28842,
+            "MAP@10": 0.31217,
+            "MAP@20": 0.3223,
+            "MAP@50": 0.32551,
+            "MAP@100": 0.32607,
+            "MAP@200": 0.32626,
+            "MAP@1000": 0.32634
+        },
+        "precision": {
+            "P@1": 0.17852,
+            "P@3": 0.12352,
+            "P@5": 0.09829,
+            "P@10": 0.06671,
+            "P@20": 0.04036,
+            "P@50": 0.01807,
+            "P@100": 0.00942,
+            "P@200": 0.00484,
+            "P@1000": 0.00099
+        },
+        "recall": {
+            "Recall@1": 0.17852,
+            "Recall@3": 0.37055,
+            "Recall@5": 0.49147,
+            "Recall@10": 0.66714,
+            "Recall@20": 0.80725,
+            "Recall@50": 0.90327,
+            "Recall@100": 0.94168,
+            "Recall@200": 0.96728,
+            "Recall@1000": 0.99431
+        },
+        "mrr": {
+            "MRR@1": 0.1835,
+            "MRR@3": 0.26245,
+            "MRR@5": 0.29061,
+            "MRR@10": 0.3144,
+            "MRR@20": 0.32459,
+            "MRR@50": 0.32779,
+            "MRR@100": 0.32836,
+            "MRR@200": 0.32856,
+            "MRR@1000": 0.32864
+        },
+        "recall_cap": {
+            "R_cap@1": 0.1835,
+            "R_cap@3": 0.36771,
+            "R_cap@5": 0.49075,
+            "R_cap@10": 0.66643,
+            "R_cap@20": 0.80725,
+            "R_cap@50": 0.90256,
+            "R_cap@100": 0.94168,
+            "R_cap@200": 0.96728,
+            "R_cap@1000": 0.99431
+        },
+        "hole": {
+            "Hole@1": 0.66714,
+            "Hole@3": 0.67354,
+            "Hole@5": 0.66927,
+            "Hole@10": 0.67176,
+            "Hole@20": 0.71486,
+            "Hole@50": 0.7962,
+            "Hole@100": 0.83578,
+            "Hole@200": 0.85851,
+            "Hole@1000": 0.87223
+        }
+    }
+}

eval_output/checkpoint-100000/beir.json

@@ -0,0 +1,46 @@
+{
+    "eval_beir-nfcorpus_ndcg@10": 0.29655,
+    "eval_beir-nfcorpus_recall@10": 0.14464,
+    "eval_beir-nfcorpus_recall@20": 0.17974,
+    "eval_beir-nfcorpus_recall@100": 0.28415,
+    "eval_beir-fiqa_ndcg@10": 0.25367,
+    "eval_beir-fiqa_recall@10": 0.31546,
+    "eval_beir-fiqa_recall@20": 0.39133,
+    "eval_beir-fiqa_recall@100": 0.59399,
+    "eval_beir-arguana_ndcg@10": 0.39579,
+    "eval_beir-arguana_recall@10": 0.66714,
+    "eval_beir-arguana_recall@20": 0.80725,
+    "eval_beir-arguana_recall@100": 0.94168,
+    "eval_beir-scidocs_ndcg@10": 0.15413,
+    "eval_beir-scidocs_recall@10": 0.16083,
+    "eval_beir-scidocs_recall@20": 0.22063,
+    "eval_beir-scidocs_recall@100": 0.37255,
+    "eval_beir-scifact_ndcg@10": 0.65609,
+    "eval_beir-scifact_recall@10": 0.81167,
+    "eval_beir-scifact_recall@20": 0.84756,
+    "eval_beir-scifact_recall@100": 0.91156,
+    "eval_beir-webis-touche2020_ndcg@10": 0.18628,
+    "eval_beir-webis-touche2020_recall@10": 0.12937,
+    "eval_beir-webis-touche2020_recall@20": 0.20379,
+    "eval_beir-webis-touche2020_recall@100": 0.4094,
+    "eval_beir-cqadupstack_ndcg@10": 0.2689125,
+    "eval_beir-cqadupstack_recall@10": 0.3638466666666666,
+    "eval_beir-cqadupstack_recall@20": 0.4333991666666667,
+    "eval_beir-cqadupstack_recall@100": 0.6019383333333334,
+    "eval_beir-trec-covid_ndcg@10": 0.56947,
+    "eval_beir-trec-covid_recall@10": 0.622,
+    "eval_beir-trec-covid_recall@20": 0.587,
+    "eval_beir-trec-covid_recall@100": 0.469,
+    "eval_beir-quora_ndcg@10": 0.77792,
+    "eval_beir-quora_recall@10": 0.88573,
+    "eval_beir-quora_recall@20": 0.9275,
+    "eval_beir-quora_recall@100": 0.97682,
+    "eval_beir-nq_ndcg@10": 0.2742,
+    "eval_beir-nq_recall@10": 0.44602,
+    "eval_beir-nq_recall@20": 0.57443,
+    "eval_beir-nq_recall@100": 0.79582,
+    "eval_beir-avg_ndcg@10": 0.38330125000000004,
+    "eval_beir-avg_recall@10": 0.4546706666666666,
+    "eval_beir-avg_recall@20": 0.5172629166666667,
+    "eval_beir-avg_recall@100": 0.6356908333333333
+}

eval_output/checkpoint-100000/cqadupstack.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.191785,0.22964416666666665,0.24790416666666668,0.2689125,0.28783416666666667,0.30862,0.3217433333333333,0.33254666666666666,0.35154250000000004
+map,0.16045,0.20496,0.2168866666666666,0.22648166666666666,0.23229249999999999,0.23614249999999995,0.23753583333333328,0.23825333333333332,0.2388283333333333
+precision,0.191785,0.10717333333333334,0.07819333333333332,0.04877416666666665,0.030159166666666667,0.015413333333333334,0.009021666666666667,0.005186666666666667,0.001340833333333333
+recall,0.16045,0.2544575,0.3006583333333333,0.3638466666666666,0.4333991666666667,0.5287158333333333,0.6019383333333334,0.6707433333333332,0.8166041666666667
+mrr,0.191785,0.23855000000000004,0.2503225,0.25958749999999997,0.2646675,0.2677983333333333,0.26882500000000004,0.26928499999999994,0.2696158333333333
+recall_cap,0.191785,0.2602333333333333,0.30328916666666667,0.36496833333333334,0.43395083333333334,0.52893,0.6020241666666667,0.6707508333333333,0.8166041666666667
+hole,0.7298749999999999,0.8100466666666665,0.8372991666666666,0.8669091666666667,0.8862683333333333,0.9035375,0.9118466666666666,0.9179083333333332,0.9267599999999999

eval_output/checkpoint-100000/cqadupstack.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "cqadupstack",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.191785,
+            "NDCG@3": 0.22964416666666665,
+            "NDCG@5": 0.24790416666666668,
+            "NDCG@10": 0.2689125,
+            "NDCG@20": 0.28783416666666667,
+            "NDCG@50": 0.30862,
+            "NDCG@100": 0.3217433333333333,
+            "NDCG@200": 0.33254666666666666,
+            "NDCG@1000": 0.35154250000000004
+        },
+        "map": {
+            "MAP@1": 0.16045,
+            "MAP@3": 0.20496,
+            "MAP@5": 0.2168866666666666,
+            "MAP@10": 0.22648166666666666,
+            "MAP@20": 0.23229249999999999,
+            "MAP@50": 0.23614249999999995,
+            "MAP@100": 0.23753583333333328,
+            "MAP@200": 0.23825333333333332,
+            "MAP@1000": 0.2388283333333333
+        },
+        "precision": {
+            "P@1": 0.191785,
+            "P@3": 0.10717333333333334,
+            "P@5": 0.07819333333333332,
+            "P@10": 0.04877416666666665,
+            "P@20": 0.030159166666666667,
+            "P@50": 0.015413333333333334,
+            "P@100": 0.009021666666666667,
+            "P@200": 0.005186666666666667,
+            "P@1000": 0.001340833333333333
+        },
+        "recall": {
+            "Recall@1": 0.16045,
+            "Recall@3": 0.2544575,
+            "Recall@5": 0.3006583333333333,
+            "Recall@10": 0.3638466666666666,
+            "Recall@20": 0.4333991666666667,
+            "Recall@50": 0.5287158333333333,
+            "Recall@100": 0.6019383333333334,
+            "Recall@200": 0.6707433333333332,
+            "Recall@1000": 0.8166041666666667
+        },
+        "mrr": {
+            "MRR@1": 0.191785,
+            "MRR@3": 0.23855000000000004,
+            "MRR@5": 0.2503225,
+            "MRR@10": 0.25958749999999997,
+            "MRR@20": 0.2646675,
+            "MRR@50": 0.2677983333333333,
+            "MRR@100": 0.26882500000000004,
+            "MRR@200": 0.26928499999999994,
+            "MRR@1000": 0.2696158333333333
+        },
+        "recall_cap": {
+            "R_cap@1": 0.191785,
+            "R_cap@3": 0.2602333333333333,
+            "R_cap@5": 0.30328916666666667,
+            "R_cap@10": 0.36496833333333334,
+            "R_cap@20": 0.43395083333333334,
+            "R_cap@50": 0.52893,
+            "R_cap@100": 0.6020241666666667,
+            "R_cap@200": 0.6707508333333333,
+            "R_cap@1000": 0.8166041666666667
+        },
+        "hole": {
+            "Hole@1": 0.7298749999999999,
+            "Hole@3": 0.8100466666666665,
+            "Hole@5": 0.8372991666666666,
+            "Hole@10": 0.8669091666666667,
+            "Hole@20": 0.8862683333333333,
+            "Hole@50": 0.9035375,
+            "Hole@100": 0.9118466666666666,
+            "Hole@200": 0.9179083333333332,
+            "Hole@1000": 0.9267599999999999
+        }
+    }
+}

eval_output/checkpoint-100000/fiqa.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.22994,0.22025,0.22971,0.25367,0.27826,0.30668,0.32515,0.34001,0.36491
+map,0.11452,0.16638,0.17887,0.19239,0.20087,0.20659,0.20874,0.2099,0.2108
+precision,0.22994,0.14558,0.10833,0.07269,0.04653,0.02429,0.01441,0.00833,0.00215
+recall,0.11452,0.20686,0.24866,0.31546,0.39133,0.50494,0.59399,0.6741,0.83452
+mrr,0.22994,0.28344,0.29879,0.31011,0.31587,0.31959,0.32077,0.32113,0.32139
+recall_cap,0.22994,0.22917,0.25476,0.31595,0.39133,0.50494,0.59399,0.6741,0.83452
+hole,0.71451,0.78961,0.82932,0.86481,0.89252,0.91407,0.92321,0.9284,0.93762

eval_output/checkpoint-100000/fiqa.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "fiqa",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.22994,
+            "NDCG@3": 0.22025,
+            "NDCG@5": 0.22971,
+            "NDCG@10": 0.25367,
+            "NDCG@20": 0.27826,
+            "NDCG@50": 0.30668,
+            "NDCG@100": 0.32515,
+            "NDCG@200": 0.34001,
+            "NDCG@1000": 0.36491
+        },
+        "map": {
+            "MAP@1": 0.11452,
+            "MAP@3": 0.16638,
+            "MAP@5": 0.17887,
+            "MAP@10": 0.19239,
+            "MAP@20": 0.20087,
+            "MAP@50": 0.20659,
+            "MAP@100": 0.20874,
+            "MAP@200": 0.2099,
+            "MAP@1000": 0.2108
+        },
+        "precision": {
+            "P@1": 0.22994,
+            "P@3": 0.14558,
+            "P@5": 0.10833,
+            "P@10": 0.07269,
+            "P@20": 0.04653,
+            "P@50": 0.02429,
+            "P@100": 0.01441,
+            "P@200": 0.00833,
+            "P@1000": 0.00215
+        },
+        "recall": {
+            "Recall@1": 0.11452,
+            "Recall@3": 0.20686,
+            "Recall@5": 0.24866,
+            "Recall@10": 0.31546,
+            "Recall@20": 0.39133,
+            "Recall@50": 0.50494,
+            "Recall@100": 0.59399,
+            "Recall@200": 0.6741,
+            "Recall@1000": 0.83452
+        },
+        "mrr": {
+            "MRR@1": 0.22994,
+            "MRR@3": 0.28344,
+            "MRR@5": 0.29879,
+            "MRR@10": 0.31011,
+            "MRR@20": 0.31587,
+            "MRR@50": 0.31959,
+            "MRR@100": 0.32077,
+            "MRR@200": 0.32113,
+            "MRR@1000": 0.32139
+        },
+        "recall_cap": {
+            "R_cap@1": 0.22994,
+            "R_cap@3": 0.22917,
+            "R_cap@5": 0.25476,
+            "R_cap@10": 0.31595,
+            "R_cap@20": 0.39133,
+            "R_cap@50": 0.50494,
+            "R_cap@100": 0.59399,
+            "R_cap@200": 0.6741,
+            "R_cap@1000": 0.83452
+        },
+        "hole": {
+            "Hole@1": 0.71451,
+            "Hole@3": 0.78961,
+            "Hole@5": 0.82932,
+            "Hole@10": 0.86481,
+            "Hole@20": 0.89252,
+            "Hole@50": 0.91407,
+            "Hole@100": 0.92321,
+            "Hole@200": 0.9284,
+            "Hole@1000": 0.93762
+        }
+    }
+}

eval_output/checkpoint-100000/nfcorpus.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.36687,0.34358,0.32376,0.29655,0.27597,0.26373,0.27268,0.29088,0.36292
+map,0.04295,0.07304,0.08708,0.10203,0.11246,0.12234,0.12809,0.13292,0.14161
+precision,0.39009,0.33333,0.28669,0.22291,0.16502,0.10415,0.07127,0.04896,0.02023
+recall,0.04295,0.08692,0.11257,0.14464,0.17974,0.22974,0.28415,0.35644,0.60391
+mrr,0.39628,0.46956,0.48256,0.49141,0.49529,0.4971,0.49763,0.49793,0.49811
+recall_cap,0.39628,0.35139,0.32023,0.27996,0.25795,0.25981,0.29779,0.36142,0.60391
+hole,0.06811,0.08669,0.10402,0.11176,0.11316,0.12297,0.12851,0.12817,0.13272

eval_output/checkpoint-100000/nfcorpus.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "nfcorpus",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.36687,
+            "NDCG@3": 0.34358,
+            "NDCG@5": 0.32376,
+            "NDCG@10": 0.29655,
+            "NDCG@20": 0.27597,
+            "NDCG@50": 0.26373,
+            "NDCG@100": 0.27268,
+            "NDCG@200": 0.29088,
+            "NDCG@1000": 0.36292
+        },
+        "map": {
+            "MAP@1": 0.04295,
+            "MAP@3": 0.07304,
+            "MAP@5": 0.08708,
+            "MAP@10": 0.10203,
+            "MAP@20": 0.11246,
+            "MAP@50": 0.12234,
+            "MAP@100": 0.12809,
+            "MAP@200": 0.13292,
+            "MAP@1000": 0.14161
+        },
+        "precision": {
+            "P@1": 0.39009,
+            "P@3": 0.33333,
+            "P@5": 0.28669,
+            "P@10": 0.22291,
+            "P@20": 0.16502,
+            "P@50": 0.10415,
+            "P@100": 0.07127,
+            "P@200": 0.04896,
+            "P@1000": 0.02023
+        },
+        "recall": {
+            "Recall@1": 0.04295,
+            "Recall@3": 0.08692,
+            "Recall@5": 0.11257,
+            "Recall@10": 0.14464,
+            "Recall@20": 0.17974,
+            "Recall@50": 0.22974,
+            "Recall@100": 0.28415,
+            "Recall@200": 0.35644,
+            "Recall@1000": 0.60391
+        },
+        "mrr": {
+            "MRR@1": 0.39628,
+            "MRR@3": 0.46956,
+            "MRR@5": 0.48256,
+            "MRR@10": 0.49141,
+            "MRR@20": 0.49529,
+            "MRR@50": 0.4971,
+            "MRR@100": 0.49763,
+            "MRR@200": 0.49793,
+            "MRR@1000": 0.49811
+        },
+        "recall_cap": {
+            "R_cap@1": 0.39628,
+            "R_cap@3": 0.35139,
+            "R_cap@5": 0.32023,
+            "R_cap@10": 0.27996,
+            "R_cap@20": 0.25795,
+            "R_cap@50": 0.25981,
+            "R_cap@100": 0.29779,
+            "R_cap@200": 0.36142,
+            "R_cap@1000": 0.60391
+        },
+        "hole": {
+            "Hole@1": 0.06811,
+            "Hole@3": 0.08669,
+            "Hole@5": 0.10402,
+            "Hole@10": 0.11176,
+            "Hole@20": 0.11316,
+            "Hole@50": 0.12297,
+            "Hole@100": 0.12851,
+            "Hole@200": 0.12817,
+            "Hole@1000": 0.13272
+        }
+    }
+}

eval_output/checkpoint-100000/nq.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.13268,0.20055,0.23512,0.2742,0.30826,0.3351,0.35119,0.36108,0.37105
+map,0.11602,0.17686,0.19651,0.21343,0.22334,0.22808,0.22966,0.23023,0.23051
+precision,0.13268,0.09695,0.07758,0.05272,0.0342,0.01682,0.00957,0.0052,0.00114
+recall,0.11602,0.25043,0.33099,0.44602,0.57443,0.70237,0.79582,0.8624,0.94284
+mrr,0.13268,0.19559,0.21521,0.231,0.23989,0.24387,0.24519,0.24563,0.24583
+recall_cap,0.13268,0.25053,0.33099,0.44602,0.57443,0.70237,0.79582,0.8624,0.94284
+hole,0.86356,0.90025,0.91952,0.9445,0.96288,0.98023,0.98768,0.99208,0.99636

eval_output/checkpoint-100000/nq.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "nq",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.13268,
+            "NDCG@3": 0.20055,
+            "NDCG@5": 0.23512,
+            "NDCG@10": 0.2742,
+            "NDCG@20": 0.30826,
+            "NDCG@50": 0.3351,
+            "NDCG@100": 0.35119,
+            "NDCG@200": 0.36108,
+            "NDCG@1000": 0.37105
+        },
+        "map": {
+            "MAP@1": 0.11602,
+            "MAP@3": 0.17686,
+            "MAP@5": 0.19651,
+            "MAP@10": 0.21343,
+            "MAP@20": 0.22334,
+            "MAP@50": 0.22808,
+            "MAP@100": 0.22966,
+            "MAP@200": 0.23023,
+            "MAP@1000": 0.23051
+        },
+        "precision": {
+            "P@1": 0.13268,
+            "P@3": 0.09695,
+            "P@5": 0.07758,
+            "P@10": 0.05272,
+            "P@20": 0.0342,
+            "P@50": 0.01682,
+            "P@100": 0.00957,
+            "P@200": 0.0052,
+            "P@1000": 0.00114
+        },
+        "recall": {
+            "Recall@1": 0.11602,
+            "Recall@3": 0.25043,
+            "Recall@5": 0.33099,
+            "Recall@10": 0.44602,
+            "Recall@20": 0.57443,
+            "Recall@50": 0.70237,
+            "Recall@100": 0.79582,
+            "Recall@200": 0.8624,
+            "Recall@1000": 0.94284
+        },
+        "mrr": {
+            "MRR@1": 0.13268,
+            "MRR@3": 0.19559,
+            "MRR@5": 0.21521,
+            "MRR@10": 0.231,
+            "MRR@20": 0.23989,
+            "MRR@50": 0.24387,
+            "MRR@100": 0.24519,
+            "MRR@200": 0.24563,
+            "MRR@1000": 0.24583
+        },
+        "recall_cap": {
+            "R_cap@1": 0.13268,
+            "R_cap@3": 0.25053,
+            "R_cap@5": 0.33099,
+            "R_cap@10": 0.44602,
+            "R_cap@20": 0.57443,
+            "R_cap@50": 0.70237,
+            "R_cap@100": 0.79582,
+            "R_cap@200": 0.8624,
+            "R_cap@1000": 0.94284
+        },
+        "hole": {
+            "Hole@1": 0.86356,
+            "Hole@3": 0.90025,
+            "Hole@5": 0.91952,
+            "Hole@10": 0.9445,
+            "Hole@20": 0.96288,
+            "Hole@50": 0.98023,
+            "Hole@100": 0.98768,
+            "Hole@200": 0.99208,
+            "Hole@1000": 0.99636
+        }
+    }
+}

eval_output/checkpoint-100000/quora.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.6754,0.73528,0.75736,0.77792,0.79031,0.79909,0.80225,0.8041,0.80573
+map,0.5901,0.69448,0.71415,0.72735,0.73302,0.73562,0.73619,0.7364,0.7365
+precision,0.6754,0.32203,0.21556,0.12091,0.06573,0.02839,0.01471,0.00754,0.00155
+recall,0.5901,0.76907,0.82662,0.88573,0.9275,0.96223,0.97682,0.9868,0.99658
+mrr,0.6751,0.74512,0.75582,0.76212,0.76427,0.76504,0.76516,0.76521,0.76523
+recall_cap,0.6751,0.78148,0.83155,0.88718,0.92779,0.96226,0.97682,0.9868,0.99658
+hole,0.305,0.64493,0.74606,0.83775,0.89183,0.92915,0.94396,0.95282,0.96182

eval_output/checkpoint-100000/quora.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "quora",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.6754,
+            "NDCG@3": 0.73528,
+            "NDCG@5": 0.75736,
+            "NDCG@10": 0.77792,
+            "NDCG@20": 0.79031,
+            "NDCG@50": 0.79909,
+            "NDCG@100": 0.80225,
+            "NDCG@200": 0.8041,
+            "NDCG@1000": 0.80573
+        },
+        "map": {
+            "MAP@1": 0.5901,
+            "MAP@3": 0.69448,
+            "MAP@5": 0.71415,
+            "MAP@10": 0.72735,
+            "MAP@20": 0.73302,
+            "MAP@50": 0.73562,
+            "MAP@100": 0.73619,
+            "MAP@200": 0.7364,
+            "MAP@1000": 0.7365
+        },
+        "precision": {
+            "P@1": 0.6754,
+            "P@3": 0.32203,
+            "P@5": 0.21556,
+            "P@10": 0.12091,
+            "P@20": 0.06573,
+            "P@50": 0.02839,
+            "P@100": 0.01471,
+            "P@200": 0.00754,
+            "P@1000": 0.00155
+        },
+        "recall": {
+            "Recall@1": 0.5901,
+            "Recall@3": 0.76907,
+            "Recall@5": 0.82662,
+            "Recall@10": 0.88573,
+            "Recall@20": 0.9275,
+            "Recall@50": 0.96223,
+            "Recall@100": 0.97682,
+            "Recall@200": 0.9868,
+            "Recall@1000": 0.99658
+        },
+        "mrr": {
+            "MRR@1": 0.6751,
+            "MRR@3": 0.74512,
+            "MRR@5": 0.75582,
+            "MRR@10": 0.76212,
+            "MRR@20": 0.76427,
+            "MRR@50": 0.76504,
+            "MRR@100": 0.76516,
+            "MRR@200": 0.76521,
+            "MRR@1000": 0.76523
+        },
+        "recall_cap": {
+            "R_cap@1": 0.6751,
+            "R_cap@3": 0.78148,
+            "R_cap@5": 0.83155,
+            "R_cap@10": 0.88718,
+            "R_cap@20": 0.92779,
+            "R_cap@50": 0.96226,
+            "R_cap@100": 0.97682,
+            "R_cap@200": 0.9868,
+            "R_cap@1000": 0.99658
+        },
+        "hole": {
+            "Hole@1": 0.305,
+            "Hole@3": 0.64493,
+            "Hole@5": 0.74606,
+            "Hole@10": 0.83775,
+            "Hole@20": 0.89183,
+            "Hole@50": 0.92915,
+            "Hole@100": 0.94396,
+            "Hole@200": 0.95282,
+            "Hole@1000": 0.96182
+        }
+    }
+}

eval_output/checkpoint-100000/scidocs.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.181,0.14792,0.12849,0.15413,0.17987,0.2085,0.22665,0.24445,0.28012
+map,0.03693,0.06617,0.07757,0.08875,0.09734,0.10354,0.10595,0.10741,0.10878
+precision,0.181,0.13767,0.1116,0.0793,0.0545,0.03026,0.0184,0.01106,0.00312
+recall,0.03693,0.08423,0.11338,0.16083,0.22063,0.3065,0.37255,0.44818,0.63403
+mrr,0.181,0.24383,0.26043,0.27504,0.2824,0.28623,0.28731,0.28775,0.28804
+recall_cap,0.181,0.13767,0.11338,0.16083,0.22063,0.3065,0.37255,0.44818,0.63403
+hole,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0