|
conditions: |
|
- name: bm25-doc-default |
|
display: BM25 doc (k1=0.9, b=0.4) |
|
display-html: BM25 doc (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4) |
|
display-row: (1a) |
|
command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc --topics $topics --output $output --bm25 |
|
topics: |
|
- topic_key: msmarco-v2-doc-dev |
|
eval_key: msmarco-v2-doc-dev |
|
scores: |
|
- MRR@100: 0.1572 |
|
R@1K: 0.8054 |
|
- topic_key: msmarco-v2-doc-dev2 |
|
eval_key: msmarco-v2-doc-dev2 |
|
scores: |
|
- MRR@100: 0.1659 |
|
R@1K: 0.8029 |
|
- topic_key: dl21 |
|
eval_key: dl21-doc |
|
scores: |
|
- MAP@100: 0.2126 |
|
nDCG@10: 0.5116 |
|
MRR@100: 0.8367 |
|
R@100: 0.3195 |
|
R@1K: 0.6739 |
|
- name: bm25-doc-segmented-default |
|
display: BM25 doc segmented (k1=0.9, b=0.4) |
|
display-html: BM25 doc segmented (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4) |
|
display-row: (1b) |
|
command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented --topics $topics --output $output --bm25 --hits 10000 --max-passage-hits 1000 --max-passage |
|
topics: |
|
- topic_key: msmarco-v2-doc-dev |
|
eval_key: msmarco-v2-doc-dev |
|
scores: |
|
- MRR@100: 0.1896 |
|
R@1K: 0.8542 |
|
- topic_key: msmarco-v2-doc-dev2 |
|
eval_key: msmarco-v2-doc-dev2 |
|
scores: |
|
- MRR@100: 0.1930 |
|
R@1K: 0.8549 |
|
- topic_key: dl21 |
|
eval_key: dl21-doc |
|
scores: |
|
- MAP@100: 0.2436 |
|
nDCG@10: 0.5776 |
|
MRR@100: 0.8937 |
|
R@100: 0.3478 |
|
R@1K: 0.6930 |
|
- name: bm25-rm3-doc-default |
|
display: BM25+RM3 doc (k1=0.9, b=0.4) |
|
display-html: BM25+RM3 doc (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4) |
|
display-row: (1c) |
|
command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc --topics $topics --output $output --bm25 --rm3 |
|
topics: |
|
- topic_key: msmarco-v2-doc-dev |
|
eval_key: msmarco-v2-doc-dev |
|
scores: |
|
- MRR@100: 0.0974 |
|
R@1K: 0.7699 |
|
- topic_key: msmarco-v2-doc-dev2 |
|
eval_key: msmarco-v2-doc-dev2 |
|
scores: |
|
- MRR@100: 0.1033 |
|
R@1K: 0.7736 |
|
- topic_key: dl21 |
|
eval_key: dl21-doc |
|
scores: |
|
- MAP@100: 0.2452 |
|
nDCG@10: 0.5304 |
|
MRR@100: 0.7914 |
|
R@100: 0.3376 |
|
R@1K: 0.7341 |
|
- name: bm25-rm3-doc-segmented-default |
|
display: BM25+RM3 doc segmented (k1=0.9, b=0.4) |
|
display-html: BM25+RM3 doc segmented (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4) |
|
display-row: (1d) |
|
command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented --topics $topics --output $output --bm25 --rm3 --hits 10000 --max-passage-hits 1000 --max-passage |
|
topics: |
|
- topic_key: msmarco-v2-doc-dev |
|
eval_key: msmarco-v2-doc-dev |
|
scores: |
|
- MRR@100: 0.1660 |
|
R@1K: 0.8608 |
|
- topic_key: msmarco-v2-doc-dev2 |
|
eval_key: msmarco-v2-doc-dev2 |
|
scores: |
|
- MRR@100: 0.1702 |
|
R@1K: 0.8639 |
|
- topic_key: dl21 |
|
eval_key: dl21-doc |
|
scores: |
|
- MAP@100: 0.2936 |
|
nDCG@10: 0.6189 |
|
MRR@100: 0.9076 |
|
R@100: 0.3890 |
|
R@1K: 0.7678 |
|
- name: bm25-d2q-t5-doc-default |
|
display: BM25 w/ doc2query-T5 doc (k1=0.9, b=0.4) |
|
display-html: BM25 w/ doc2query-T5 doc (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4) |
|
display-row: (2a) |
|
command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-d2q-t5 --topics $topics --output $output --bm25 |
|
topics: |
|
- topic_key: msmarco-v2-doc-dev |
|
eval_key: msmarco-v2-doc-dev |
|
scores: |
|
- MRR@100: 0.2011 |
|
R@1K: 0.8614 |
|
- topic_key: msmarco-v2-doc-dev2 |
|
eval_key: msmarco-v2-doc-dev2 |
|
scores: |
|
- MRR@100: 0.2012 |
|
R@1K: 0.8568 |
|
- topic_key: dl21 |
|
eval_key: dl21-doc |
|
scores: |
|
- MAP@100: 0.2387 |
|
nDCG@10: 0.5792 |
|
MRR@100: 0.8866 |
|
R@100: 0.3443 |
|
R@1K: 0.7066 |
|
- name: bm25-d2q-t5-doc-segmented-default |
|
display: BM25 w/ doc2query-T5 doc segmented (k1=0.9, b=0.4) |
|
display-html: BM25 w/ doc2query-T5 doc segmented (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4) |
|
display-row: (2b) |
|
command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-d2q-t5 --topics $topics --output $output --bm25 --hits 10000 --max-passage-hits 1000 --max-passage |
|
topics: |
|
- topic_key: msmarco-v2-doc-dev |
|
eval_key: msmarco-v2-doc-dev |
|
scores: |
|
- MRR@100: 0.2226 |
|
R@1K: 0.8982 |
|
- topic_key: msmarco-v2-doc-dev2 |
|
eval_key: msmarco-v2-doc-dev2 |
|
scores: |
|
- MRR@100: 0.2234 |
|
R@1K: 0.8952 |
|
- topic_key: dl21 |
|
eval_key: dl21-doc |
|
scores: |
|
- MAP@100: 0.2683 |
|
nDCG@10: 0.6289 |
|
MRR@100: 0.9454 |
|
R@100: 0.3656 |
|
R@1K: 0.7202 |
|
- name: bm25-rm3-d2q-t5-doc-default |
|
display: BM25+RM3 w/ doc2query-T5 doc (k1=0.9, b=0.4) |
|
display-html: BM25+RM3 w/ doc2query-T5 doc (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4) |
|
display-row: (2c) |
|
command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-d2q-t5-docvectors --topics $topics --output $output --bm25 --rm3 |
|
topics: |
|
- topic_key: msmarco-v2-doc-dev |
|
eval_key: msmarco-v2-doc-dev |
|
scores: |
|
- MRR@100: 0.1141 |
|
R@1K: 0.8191 |
|
- topic_key: msmarco-v2-doc-dev2 |
|
eval_key: msmarco-v2-doc-dev2 |
|
scores: |
|
- MRR@100: 0.1170 |
|
R@1K: 0.8247 |
|
- topic_key: dl21 |
|
eval_key: dl21-doc |
|
scores: |
|
- MAP@100: 0.2611 |
|
nDCG@10: 0.5375 |
|
MRR@100: 0.8255 |
|
R@100: 0.3580 |
|
R@1K: 0.7574 |
|
- name: bm25-rm3-d2q-t5-doc-segmented-default |
|
display: BM25+RM3 w/ doc2query-T5 doc segmented (k1=0.9, b=0.4) |
|
display-html: BM25+RM3 w/ doc2query-T5 doc segmented (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4) |
|
display-row: (2d) |
|
command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-d2q-t5-docvectors --topics $topics --output $output --bm25 --rm3 --hits 10000 --max-passage-hits 1000 --max-passage |
|
topics: |
|
- topic_key: msmarco-v2-doc-dev |
|
eval_key: msmarco-v2-doc-dev |
|
scores: |
|
- MRR@100: 0.1975 |
|
R@1K: 0.9002 |
|
- topic_key: msmarco-v2-doc-dev2 |
|
eval_key: msmarco-v2-doc-dev2 |
|
scores: |
|
- MRR@100: 0.1978 |
|
R@1K: 0.8972 |
|
- topic_key: dl21 |
|
eval_key: dl21-doc |
|
scores: |
|
- MAP@100: 0.3191 |
|
nDCG@10: 0.6559 |
|
MRR@100: 0.8989 |
|
R@100: 0.4131 |
|
R@1K: 0.7948 |
|
- name: unicoil-noexp |
|
display: "uniCOIL (noexp): pre-encoded" |
|
display-html: "uniCOIL (noexp): pre-encoded queries" |
|
display-row: (3a) |
|
command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-unicoil-noexp-0shot --topics $topics --output $output --impact --hits 10000 --max-passage-hits 1000 --max-passage |
|
topics: |
|
- topic_key: msmarco-v2-doc-dev-unicoil-noexp |
|
eval_key: msmarco-v2-doc-dev |
|
scores: |
|
- MRR@100: 0.2231 |
|
R@1K: 0.8987 |
|
- topic_key: msmarco-v2-doc-dev2-unicoil-noexp |
|
eval_key: msmarco-v2-doc-dev2 |
|
scores: |
|
- MRR@100: 0.2314 |
|
R@1K: 0.8995 |
|
- topic_key: dl21-unicoil-noexp |
|
eval_key: dl21-doc |
|
scores: |
|
- MAP@100: 0.2587 |
|
nDCG@10: 0.6495 |
|
MRR@100: 0.9282 |
|
R@100: 0.3563 |
|
R@1K: 0.6787 |
|
- name: unicoil-noexp-otf |
|
display: "uniCOIL (noexp): query inference with PyTorch" |
|
display-html: "uniCOIL (noexp): query inference with PyTorch" |
|
command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-unicoil-noexp-0shot --topics $topics --encoder castorini/unicoil-noexp-msmarco-passage --output $output --impact --hits 10000 --max-passage-hits 1000 --max-passage |
|
topics: |
|
- topic_key: msmarco-v2-doc-dev |
|
eval_key: msmarco-v2-doc-dev |
|
scores: |
|
- MRR@100: 0.2232 |
|
R@1K: 0.8987 |
|
- topic_key: msmarco-v2-doc-dev2 |
|
eval_key: msmarco-v2-doc-dev2 |
|
scores: |
|
- MRR@100: 0.2314 |
|
R@1K: 0.8993 |
|
- topic_key: dl21 |
|
eval_key: dl21-doc |
|
scores: |
|
- MAP@100: 0.2589 |
|
nDCG@10: 0.6501 |
|
MRR@100: 0.9282 |
|
R@100: 0.3574 |
|
R@1K: 0.6782 |
|
- name: unicoil |
|
display: "uniCOIL (w/ doc2query-T5): pre-encoded" |
|
display-html: "uniCOIL (w/ doc2query-T5): pre-encoded queries" |
|
display-row: (3b) |
|
command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-unicoil-0shot --topics $topics --output $output --impact --hits 10000 --max-passage-hits 1000 --max-passage |
|
topics: |
|
- topic_key: msmarco-v2-doc-dev-unicoil |
|
eval_key: msmarco-v2-doc-dev |
|
scores: |
|
- MRR@100: 0.2419 |
|
R@1K: 0.9122 |
|
- topic_key: msmarco-v2-doc-dev2-unicoil |
|
eval_key: msmarco-v2-doc-dev2 |
|
scores: |
|
- MRR@100: 0.2445 |
|
R@1K: 0.9172 |
|
- topic_key: dl21-unicoil |
|
eval_key: dl21-doc |
|
scores: |
|
- MAP@100: 0.2718 |
|
nDCG@10: 0.6783 |
|
MRR@100: 0.9684 |
|
R@100: 0.3700 |
|
R@1K: 0.7069 |
|
- name: unicoil-otf |
|
display: "uniCOIL (w/ doc2query-T5): query inference with PyTorch" |
|
display-html: "uniCOIL (w/ doc2query-T5): query inference with PyTorch" |
|
command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-unicoil-0shot --topics $topics --encoder castorini/unicoil-msmarco-passage --output $output --impact --hits 10000 --max-passage-hits 1000 --max-passage |
|
topics: |
|
- topic_key: msmarco-v2-doc-dev |
|
eval_key: msmarco-v2-doc-dev |
|
scores: |
|
- MRR@100: 0.2419 |
|
R@1K: 0.9120 |
|
- topic_key: msmarco-v2-doc-dev2 |
|
eval_key: msmarco-v2-doc-dev2 |
|
scores: |
|
- MRR@100: 0.2447 |
|
R@1K: 0.9174 |
|
- topic_key: dl21 |
|
eval_key: dl21-doc |
|
scores: |
|
- MAP@100: 0.2720 |
|
nDCG@10: 0.6782 |
|
MRR@100: 0.9684 |
|
R@100: 0.3702 |
|
R@1K: 0.7071 |
|
|