conditions: - name: bm25-doc-default display: BM25 doc (k1=0.9, b=0.4) display-html: BM25 doc (k1=0.9, b=0.4) display-row: (1a) command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc --topics $topics --output $output --bm25 topics: - topic_key: msmarco-v2-doc-dev eval_key: msmarco-v2-doc-dev scores: - MRR@100: 0.1572 R@1K: 0.8054 - topic_key: msmarco-v2-doc-dev2 eval_key: msmarco-v2-doc-dev2 scores: - MRR@100: 0.1659 R@1K: 0.8029 - topic_key: dl21 eval_key: dl21-doc scores: - MAP@100: 0.2126 nDCG@10: 0.5116 MRR@100: 0.8367 R@100: 0.3195 R@1K: 0.6739 - name: bm25-doc-segmented-default display: BM25 doc segmented (k1=0.9, b=0.4) display-html: BM25 doc segmented (k1=0.9, b=0.4) display-row: (1b) command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented --topics $topics --output $output --bm25 --hits 10000 --max-passage-hits 1000 --max-passage topics: - topic_key: msmarco-v2-doc-dev eval_key: msmarco-v2-doc-dev scores: - MRR@100: 0.1896 R@1K: 0.8542 - topic_key: msmarco-v2-doc-dev2 eval_key: msmarco-v2-doc-dev2 scores: - MRR@100: 0.1930 R@1K: 0.8549 - topic_key: dl21 eval_key: dl21-doc scores: - MAP@100: 0.2436 nDCG@10: 0.5776 MRR@100: 0.8937 R@100: 0.3478 R@1K: 0.6930 - name: bm25-rm3-doc-default display: BM25+RM3 doc (k1=0.9, b=0.4) display-html: BM25+RM3 doc (k1=0.9, b=0.4) display-row: (1c) command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc --topics $topics --output $output --bm25 --rm3 topics: - topic_key: msmarco-v2-doc-dev eval_key: msmarco-v2-doc-dev scores: - MRR@100: 0.0974 R@1K: 0.7699 - topic_key: msmarco-v2-doc-dev2 eval_key: msmarco-v2-doc-dev2 scores: - MRR@100: 0.1033 R@1K: 0.7736 - topic_key: dl21 eval_key: dl21-doc scores: - MAP@100: 0.2452 nDCG@10: 0.5304 MRR@100: 0.7914 R@100: 0.3376 R@1K: 0.7341 - name: bm25-rm3-doc-segmented-default display: BM25+RM3 doc segmented (k1=0.9, b=0.4) display-html: BM25+RM3 doc segmented (k1=0.9, b=0.4) display-row: (1d) command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented --topics $topics --output $output --bm25 --rm3 --hits 10000 --max-passage-hits 1000 --max-passage topics: - topic_key: msmarco-v2-doc-dev eval_key: msmarco-v2-doc-dev scores: - MRR@100: 0.1660 R@1K: 0.8608 - topic_key: msmarco-v2-doc-dev2 eval_key: msmarco-v2-doc-dev2 scores: - MRR@100: 0.1702 R@1K: 0.8639 - topic_key: dl21 eval_key: dl21-doc scores: - MAP@100: 0.2936 nDCG@10: 0.6189 MRR@100: 0.9076 R@100: 0.3890 R@1K: 0.7678 - name: bm25-d2q-t5-doc-default display: BM25 w/ doc2query-T5 doc (k1=0.9, b=0.4) display-html: BM25 w/ doc2query-T5 doc (k1=0.9, b=0.4) display-row: (2a) command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-d2q-t5 --topics $topics --output $output --bm25 topics: - topic_key: msmarco-v2-doc-dev eval_key: msmarco-v2-doc-dev scores: - MRR@100: 0.2011 R@1K: 0.8614 - topic_key: msmarco-v2-doc-dev2 eval_key: msmarco-v2-doc-dev2 scores: - MRR@100: 0.2012 R@1K: 0.8568 - topic_key: dl21 eval_key: dl21-doc scores: - MAP@100: 0.2387 nDCG@10: 0.5792 MRR@100: 0.8866 R@100: 0.3443 R@1K: 0.7066 - name: bm25-d2q-t5-doc-segmented-default display: BM25 w/ doc2query-T5 doc segmented (k1=0.9, b=0.4) display-html: BM25 w/ doc2query-T5 doc segmented (k1=0.9, b=0.4) display-row: (2b) command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-d2q-t5 --topics $topics --output $output --bm25 --hits 10000 --max-passage-hits 1000 --max-passage topics: - topic_key: msmarco-v2-doc-dev eval_key: msmarco-v2-doc-dev scores: - MRR@100: 0.2226 R@1K: 0.8982 - topic_key: msmarco-v2-doc-dev2 eval_key: msmarco-v2-doc-dev2 scores: - MRR@100: 0.2234 R@1K: 0.8952 - topic_key: dl21 eval_key: dl21-doc scores: - MAP@100: 0.2683 nDCG@10: 0.6289 MRR@100: 0.9454 R@100: 0.3656 R@1K: 0.7202 - name: bm25-rm3-d2q-t5-doc-default display: BM25+RM3 w/ doc2query-T5 doc (k1=0.9, b=0.4) display-html: BM25+RM3 w/ doc2query-T5 doc (k1=0.9, b=0.4) display-row: (2c) command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-d2q-t5-docvectors --topics $topics --output $output --bm25 --rm3 topics: - topic_key: msmarco-v2-doc-dev eval_key: msmarco-v2-doc-dev scores: - MRR@100: 0.1141 R@1K: 0.8191 - topic_key: msmarco-v2-doc-dev2 eval_key: msmarco-v2-doc-dev2 scores: - MRR@100: 0.1170 R@1K: 0.8247 - topic_key: dl21 eval_key: dl21-doc scores: - MAP@100: 0.2611 nDCG@10: 0.5375 MRR@100: 0.8255 R@100: 0.3580 R@1K: 0.7574 - name: bm25-rm3-d2q-t5-doc-segmented-default display: BM25+RM3 w/ doc2query-T5 doc segmented (k1=0.9, b=0.4) display-html: BM25+RM3 w/ doc2query-T5 doc segmented (k1=0.9, b=0.4) display-row: (2d) command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-d2q-t5-docvectors --topics $topics --output $output --bm25 --rm3 --hits 10000 --max-passage-hits 1000 --max-passage topics: - topic_key: msmarco-v2-doc-dev eval_key: msmarco-v2-doc-dev scores: - MRR@100: 0.1975 R@1K: 0.9002 - topic_key: msmarco-v2-doc-dev2 eval_key: msmarco-v2-doc-dev2 scores: - MRR@100: 0.1978 R@1K: 0.8972 - topic_key: dl21 eval_key: dl21-doc scores: - MAP@100: 0.3191 nDCG@10: 0.6559 MRR@100: 0.8989 R@100: 0.4131 R@1K: 0.7948 - name: unicoil-noexp display: "uniCOIL (noexp): pre-encoded" display-html: "uniCOIL (noexp): pre-encoded queries" display-row: (3a) command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-unicoil-noexp-0shot --topics $topics --output $output --impact --hits 10000 --max-passage-hits 1000 --max-passage topics: - topic_key: msmarco-v2-doc-dev-unicoil-noexp eval_key: msmarco-v2-doc-dev scores: - MRR@100: 0.2231 R@1K: 0.8987 - topic_key: msmarco-v2-doc-dev2-unicoil-noexp eval_key: msmarco-v2-doc-dev2 scores: - MRR@100: 0.2314 R@1K: 0.8995 - topic_key: dl21-unicoil-noexp eval_key: dl21-doc scores: - MAP@100: 0.2587 nDCG@10: 0.6495 MRR@100: 0.9282 R@100: 0.3563 R@1K: 0.6787 - name: unicoil-noexp-otf display: "uniCOIL (noexp): query inference with PyTorch" display-html: "uniCOIL (noexp): query inference with PyTorch" command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-unicoil-noexp-0shot --topics $topics --encoder castorini/unicoil-noexp-msmarco-passage --output $output --impact --hits 10000 --max-passage-hits 1000 --max-passage topics: - topic_key: msmarco-v2-doc-dev eval_key: msmarco-v2-doc-dev scores: - MRR@100: 0.2232 R@1K: 0.8987 - topic_key: msmarco-v2-doc-dev2 eval_key: msmarco-v2-doc-dev2 scores: - MRR@100: 0.2314 R@1K: 0.8993 - topic_key: dl21 eval_key: dl21-doc scores: - MAP@100: 0.2589 nDCG@10: 0.6501 MRR@100: 0.9282 R@100: 0.3574 R@1K: 0.6782 - name: unicoil display: "uniCOIL (w/ doc2query-T5): pre-encoded" display-html: "uniCOIL (w/ doc2query-T5): pre-encoded queries" display-row: (3b) command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-unicoil-0shot --topics $topics --output $output --impact --hits 10000 --max-passage-hits 1000 --max-passage topics: - topic_key: msmarco-v2-doc-dev-unicoil eval_key: msmarco-v2-doc-dev scores: - MRR@100: 0.2419 R@1K: 0.9122 - topic_key: msmarco-v2-doc-dev2-unicoil eval_key: msmarco-v2-doc-dev2 scores: - MRR@100: 0.2445 R@1K: 0.9172 - topic_key: dl21-unicoil eval_key: dl21-doc scores: - MAP@100: 0.2718 nDCG@10: 0.6783 MRR@100: 0.9684 R@100: 0.3700 R@1K: 0.7069 - name: unicoil-otf display: "uniCOIL (w/ doc2query-T5): query inference with PyTorch" display-html: "uniCOIL (w/ doc2query-T5): query inference with PyTorch" command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-unicoil-0shot --topics $topics --encoder castorini/unicoil-msmarco-passage --output $output --impact --hits 10000 --max-passage-hits 1000 --max-passage topics: - topic_key: msmarco-v2-doc-dev eval_key: msmarco-v2-doc-dev scores: - MRR@100: 0.2419 R@1K: 0.9120 - topic_key: msmarco-v2-doc-dev2 eval_key: msmarco-v2-doc-dev2 scores: - MRR@100: 0.2447 R@1K: 0.9174 - topic_key: dl21 eval_key: dl21-doc scores: - MAP@100: 0.2720 nDCG@10: 0.6782 MRR@100: 0.9684 R@100: 0.3702 R@1K: 0.7071