conditions: - name: bm25-default display: BM25 original passage (k1=0.9, b=0.4) display-html: BM25 original passage (k1=0.9, b=0.4) display-row: (1a) command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage --topics $topics --output $output --bm25 topics: - topic_key: msmarco-v2-passage-dev eval_key: msmarco-v2-passage-dev scores: - MRR@100: 0.0719 R@1K: 0.5733 - topic_key: msmarco-v2-passage-dev2 eval_key: msmarco-v2-passage-dev2 scores: - MRR@100: 0.0802 R@1K: 0.5839 - topic_key: dl21 eval_key: dl21-passage scores: - MAP@100: 0.1357 nDCG@10: 0.4458 MRR@100: 0.5060 R@100: 0.3261 R@1K: 0.6149 - name: bm25-augmented-default display: BM25 augmented passage (k1=0.9, b=0.4) display-html: BM25 augmented passage (k1=0.9, b=0.4) display-row: (1b) command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage-augmented --topics $topics --output $output --bm25 topics: - topic_key: msmarco-v2-passage-dev eval_key: msmarco-v2-passage-dev scores: - MRR@100: 0.0872 R@1K: 0.6925 - topic_key: msmarco-v2-passage-dev2 eval_key: msmarco-v2-passage-dev2 scores: - MRR@100: 0.0917 R@1K: 0.6933 - topic_key: dl21 eval_key: dl21-passage scores: - MAP@100: 0.0977 nDCG@10: 0.3977 MRR@100: 0.5303 R@100: 0.2709 R@1K: 0.5835 - name: bm25-rm3-default display: BM25+RM3 original passage (k1=0.9, b=0.4) display-html: BM25+RM3 original passage (k1=0.9, b=0.4) display-row: (1c) command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage --topics $topics --output $output --bm25 --rm3 topics: - topic_key: msmarco-v2-passage-dev eval_key: msmarco-v2-passage-dev scores: - MRR@100: 0.0630 R@1K: 0.5947 - topic_key: msmarco-v2-passage-dev2 eval_key: msmarco-v2-passage-dev2 scores: - MRR@100: 0.0659 R@1K: 0.6062 - topic_key: dl21 eval_key: dl21-passage scores: - MAP@100: 0.1666 nDCG@10: 0.4455 MRR@100: 0.5202 R@100: 0.3499 R@1K: 0.6616 - name: bm25-rm3-augmented-default display: BM25+RM3 augmented passage (k1=0.9, b=0.4) display-html: BM25+RM3 augmented passage (k1=0.9, b=0.4) display-row: (1d) command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage-augmented --topics $topics --output $output --bm25 --rm3 topics: - topic_key: msmarco-v2-passage-dev eval_key: msmarco-v2-passage-dev scores: - MRR@100: 0.0667 R@1K: 0.6857 - topic_key: msmarco-v2-passage-dev2 eval_key: msmarco-v2-passage-dev2 scores: - MRR@100: 0.0700 R@1K: 0.6826 - topic_key: dl21 eval_key: dl21-passage scores: - MAP@100: 0.1050 nDCG@10: 0.3869 MRR@100: 0.4915 R@100: 0.2807 R@1K: 0.6298 - name: bm25-d2q-t5-default display: BM25 w/ doc2query-T5 original passage (k1=0.9, b=0.4) display-html: BM25 w/ doc2query-T5 original passage (k1=0.9, b=0.4) display-row: (2a) command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage-d2q-t5 --topics $topics --output $output --bm25 topics: - topic_key: msmarco-v2-passage-dev eval_key: msmarco-v2-passage-dev scores: - MRR@100: 0.1072 R@1K: 0.7083 - topic_key: msmarco-v2-passage-dev2 eval_key: msmarco-v2-passage-dev2 scores: - MRR@100: 0.1123 R@1K: 0.7151 - topic_key: dl21 eval_key: dl21-passage scores: - MAP@100: 0.1874 nDCG@10: 0.4816 MRR@100: 0.6848 R@100: 0.4076 R@1K: 0.7078 - name: bm25-d2q-t5-augmented-default display: BM25 w/ doc2query-T5 augmented passage (k1=0.9, b=0.4) display-html: BM25 w/ doc2query-T5 augmented passage (k1=0.9, b=0.4) display-row: (2b) command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage-augmented-d2q-t5 --topics $topics --output $output --bm25 topics: - topic_key: msmarco-v2-passage-dev eval_key: msmarco-v2-passage-dev scores: - MRR@100: 0.1172 R@1K: 0.7647 - topic_key: msmarco-v2-passage-dev2 eval_key: msmarco-v2-passage-dev2 scores: - MRR@100: 0.1170 R@1K: 0.7659 - topic_key: dl21 eval_key: dl21-passage scores: - MAP@100: 0.1649 nDCG@10: 0.4702 MRR@100: 0.6391 R@100: 0.3883 R@1K: 0.6962 - name: bm25-rm3-d2q-t5-default display: BM25+RM3 w/ doc2query-T5 original passage (k1=0.9, b=0.4) display-html: BM25+RM3 w/ doc2query-T5 original passage (k1=0.9, b=0.4) display-row: (2c) command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage-d2q-t5-docvectors --topics $topics --output $output --bm25 --rm3 topics: - topic_key: msmarco-v2-passage-dev eval_key: msmarco-v2-passage-dev scores: - MRR@100: 0.0947 R@1K: 0.7181 - topic_key: msmarco-v2-passage-dev2 eval_key: msmarco-v2-passage-dev2 scores: - MRR@100: 0.0984 R@1K: 0.7222 - topic_key: dl21 eval_key: dl21-passage scores: - MAP@100: 0.2285 nDCG@10: 0.5098 MRR@100: 0.6548 R@100: 0.4499 R@1K: 0.7537 - name: bm25-rm3-d2q-t5-augmented-default display: BM25+RM3 w/ doc2query-T5 augmented passage (k1=0.9, b=0.4) display-html: BM25+RM3 w/ doc2query-T5 augmented passage (k1=0.9, b=0.4) display-row: (2d) command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage-augmented-d2q-t5-docvectors --topics $topics --output $output --bm25 --rm3 topics: - topic_key: msmarco-v2-passage-dev eval_key: msmarco-v2-passage-dev scores: - MRR@100: 0.0883 R@1K: 0.7607 - topic_key: msmarco-v2-passage-dev2 eval_key: msmarco-v2-passage-dev2 scores: - MRR@100: 0.0904 R@1K: 0.7649 - topic_key: dl21 eval_key: dl21-passage scores: - MAP@100: 0.1930 nDCG@10: 0.4812 MRR@100: 0.5958 R@100: 0.4321 R@1K: 0.7672 - name: unicoil display: "uniCOIL (w/ doc2query-T5): pre-encoded" display-html: "uniCOIL (w/ doc2query-T5): pre-encoded queries" display-row: (3b) command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage-unicoil-0shot --topics $topics --output $output --hits 1000 --impact topics: - topic_key: msmarco-v2-passage-dev-unicoil eval_key: msmarco-v2-passage-dev scores: - MRR@100: 0.1499 R@1K: 0.7616 - topic_key: msmarco-v2-passage-dev2-unicoil eval_key: msmarco-v2-passage-dev2 scores: - MRR@100: 0.1577 R@1K: 0.7671 - topic_key: dl21-unicoil eval_key: dl21-passage scores: - MAP@100: 0.2538 nDCG@10: 0.6159 MRR@100: 0.7311 R@100: 0.4731 R@1K: 0.7551 - name: unicoil-otf display: "uniCOIL (w/ doc2query-T5): query inference with PyTorch" display-html: "uniCOIL (w/ doc2query-T5): query inference with PyTorch" command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage-unicoil-0shot --topics $topics --encoder castorini/unicoil-msmarco-passage --output $output --hits 1000 --impact topics: - topic_key: msmarco-v2-passage-dev eval_key: msmarco-v2-passage-dev scores: - MRR@100: 0.1501 R@1K: 0.7613 - topic_key: msmarco-v2-passage-dev2 eval_key: msmarco-v2-passage-dev2 scores: - MRR@100: 0.1576 R@1K: 0.7676 - topic_key: dl21 eval_key: dl21-passage scores: - MAP@100: 0.2539 nDCG@10: 0.6160 MRR@100: 0.7311 R@100: 0.4723 R@1K: 0.7560 - name: unicoil-noexp display: "uniCOIL (noexp): pre-encoded" display-html: "uniCOIL (noexp): pre-encoded queries" display-row: (3a) command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage-unicoil-noexp-0shot --topics $topics --output $output --hits 1000 --impact topics: - topic_key: msmarco-v2-passage-dev-unicoil-noexp eval_key: msmarco-v2-passage-dev scores: - MRR@100: 0.1342 R@1K: 0.7010 - topic_key: msmarco-v2-passage-dev2-unicoil-noexp eval_key: msmarco-v2-passage-dev2 scores: - MRR@100: 0.1385 R@1K: 0.7114 - topic_key: dl21-unicoil-noexp eval_key: dl21-passage scores: - MAP@100: 0.2193 nDCG@10: 0.5756 MRR@100: 0.6991 R@100: 0.4246 R@1K: 0.6897 - name: unicoil-noexp-otf display: "uniCOIL (noexp): query inference with PyTorch" display-html: "uniCOIL (noexp): query inference with PyTorch" command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage-unicoil-noexp-0shot --topics $topics --encoder castorini/unicoil-noexp-msmarco-passage --output $output --hits 1000 --impact topics: - topic_key: msmarco-v2-passage-dev eval_key: msmarco-v2-passage-dev scores: - MRR@100: 0.1343 R@1K: 0.7010 - topic_key: msmarco-v2-passage-dev2 eval_key: msmarco-v2-passage-dev2 scores: - MRR@100: 0.1385 R@1K: 0.7114 - topic_key: dl21 eval_key: dl21-passage scores: - MAP@100: 0.2194 nDCG@10: 0.5759 MRR@100: 0.6991 R@100: 0.4247 R@1K: 0.6893