upload pyserini

pyserini/2cr/_base.py

@@ -0,0 +1,95 @@
+#
+# Pyserini: Reproducible IR research with sparse and dense representations
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import subprocess
+
+fail_str = '\033[91m[FAIL]\033[0m'
+ok_str = '[OK]'
+okish_str = '\033[94m[OKish]\033[0m'
+
+
+def run_command(cmd):
+ process = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ stdout, stderr = process.communicate()
+ stdout = stdout.decode('utf-8')
+ stderr = stderr.decode('utf-8')
+
+ return stdout, stderr
+
+
+def run_eval_and_return_metric(metric, eval_key, defs, runfile):
+ eval_cmd = f'python -m pyserini.eval.trec_eval {defs} {eval_key} {runfile}'
+ eval_stdout, eval_stderr = run_command(eval_cmd)
+
+ for line in eval_stdout.split('\n'):
+ parts = line.split('\t')
+ if len(parts) == 3 and parts[1] == 'all':
+ return round(float(parts[2]), 4)
+
+ return 0.0
+
+
+def run_dpr_retrieval_eval_and_return_metric(defs, json_file):
+ """Generate dpr retrieval evaluation scores
+
+ Args:
+ defs: topk definitions (e.g., '--topk 5 20')
+ json_file: dpr retrieval json file
+
+ Returns:
+ topk: a dictionary of topk scores (e.g., {"Top5": <score>})
+ """
+ eval_cmd = f'python -m pyserini.eval.evaluate_dpr_retrieval --retrieval {json_file} {defs} '
+ eval_stdout, eval_stderr = run_command(eval_cmd)
+ topk = {}
+ for line in eval_stdout.split('\n'):
+ parts = line.split('\t')
+ if len(parts) == 2 and 'accuracy' in parts[1]:
+ topk.update({parts[0]:round(float(parts[1][10:])*100, 4)})
+ return topk
+
+
+def convert_trec_run_to_dpr_retrieval_json(topics,index,runfile,output):
+ """Convert trec runfile to dpr retrieval json file
+
+ Args:
+ topics: topics field
+ index: index field
+ runfile: input runfile
+ output: output jsonfile
+
+ Returns:
+ exit status: exit status
+ """
+ cmd = f'python -m pyserini.eval.convert_trec_run_to_dpr_retrieval_run --topics {topics} --index {index} --input {runfile} --output {output}'
+ return os.system(cmd)
+
+
+def run_fusion(run_ls, output, k):
+ """run fusion command and return status code
+
+ Args:
+ run_ls: a list of runfile paths
+ output: output path
+ k: topk value
+
+ Returns:
+ status code: status code
+ """
+ run_files = ' '.join(run_ls)
+ cmd = f'python -m pyserini.fusion --runs {run_files} --output {output} --k {k}'
+ return os.system(cmd)

pyserini/2cr/miracl.py

@@ -0,0 +1,447 @@
+#
+# Pyserini: Reproducible IR research with sparse and dense representations
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import argparse
+import math
+import os
+import sys
+import time
+import subprocess
+import pkg_resources
+from collections import defaultdict, OrderedDict
+from string import Template
+
+import yaml
+
+from ._base import run_eval_and_return_metric, ok_str, okish_str, fail_str
+
+languages = [
+ ['ar', 'arabic'],
+ ['bn', 'bengali'],
+ ['en', 'english'],
+ ['es', 'spanish'],
+ ['fa', 'persian'],
+ ['fi', 'finnish'],
+ ['fr', 'french'],
+ ['hi', 'hindi'],
+ ['id', 'indonesian'],
+ ['ja', 'japanese'],
+ ['ko', 'korean'],
+ ['ru', 'russian'],
+ ['sw', 'swahili'],
+ ['te', 'telugu'],
+ ['th', 'thai'],
+ ['zh', 'chinese'],
+ ['de', 'german'],
+ ['yo', 'yoruba']
+]
+
+html_display = OrderedDict()
+html_display['bm25'] = 'BM25'
+html_display['mdpr-tied-pft-msmarco'] = 'mDPR (tied encoders), pre-FT w/ MS MARCO'
+html_display['mdpr-tied-pft-msmarco-ft-all'] = 'mDPR (tied encoders), pre-FT w/ MS MARCO then FT w/ all Mr. TyDi'
+html_display['bm25-mdpr-tied-pft-msmarco-hybrid'] = 'Hybrid of `bm25` and `mdpr-tied-pft-msmarco`'
+html_display['mdpr-tied-pft-msmarco-ft-miracl'] = 'mDPR (tied encoders), pre-FT w/ MS MARCO then in-lang FT w/ MIRACL'
+html_display['mcontriever-tied-pft-msmarco'] = 'mContriever (tied encoders), pre-FT w/ MS MARCO'
+
+models = list(html_display)
+
+trec_eval_metric_definitions = {
+ 'nDCG@10': '-c -M 100 -m ndcg_cut.10',
+ 'R@100': '-c -m recall.100',
+}
+
+
+def format_run_command(raw):
+ return raw.replace('--lang', '\\\n --lang') \
+ .replace('--encoder', '\\\n --encoder') \
+ .replace('--topics', '\\\n --topics') \
+ .replace('--index', '\\\n --index') \
+ .replace('--output ', '\\\n --output ') \
+ .replace('--runs', '\\\n --runs ') \
+ .replace('--batch ', '\\\n --batch ') \
+ .replace('--threads 12', '--threads 12 \\\n ')
+
+
+def format_eval_command(raw):
+ return raw.replace('-c ', '\\\n -c ') \
+ .replace(raw.split()[-1], f'\\\n {raw.split()[-1]}')
+
+
+def read_file(f):
+ fin = open(f, 'r')
+ text = fin.read()
+ fin.close()
+
+ return text
+
+
+def list_conditions():
+ print('Conditions:\n-----------')
+ for condition, _ in html_display.items():
+ print(condition)
+ print('\nLanguages\n---------')
+ for language in languages:
+ print(language[0])
+
+
+def generate_table_rows(table, row_template, commands, eval_commands, table_id, split, metric):
+ row_cnt = 1
+ html_rows = []
+
+ for model in models:
+ s = Template(row_template)
+
+ keys = {}
+ used_langs = 0
+ for lang in languages:
+ keys[lang[0]] = f'{model}.{lang[0]}'
+ used_langs += 1 if table[keys[lang[0]]][split][metric] != 0 else 0
+
+ sum = table[keys["ar"]][split][metric] + \
+ table[keys["bn"]][split][metric] + \
+ table[keys["en"]][split][metric] + \
+ table[keys["es"]][split][metric] + \
+ table[keys["fa"]][split][metric] + \
+ table[keys["fi"]][split][metric] + \
+ table[keys["fr"]][split][metric] + \
+ table[keys["hi"]][split][metric] + \
+ table[keys["id"]][split][metric] + \
+ table[keys["ja"]][split][metric] + \
+ table[keys["ko"]][split][metric] + \
+ table[keys["ru"]][split][metric] + \
+ table[keys["sw"]][split][metric] + \
+ table[keys["te"]][split][metric] + \
+ table[keys["th"]][split][metric] + \
+ table[keys["zh"]][split][metric] + \
+ table[keys["de"]][split][metric] + \
+ table[keys["yo"]][split][metric]
+ avg = sum / used_langs
+
+ s = s.substitute(table_cnt=table_id,
+ row_cnt=row_cnt,
+ model=html_display[model],
+ ar=f'{table[keys["ar"]][split][metric]:.3f}',
+ bn=f'{table[keys["bn"]][split][metric]:.3f}',
+ en=f'{table[keys["en"]][split][metric]:.3f}',
+ es=f'{table[keys["es"]][split][metric]:.3f}',
+ fa=f'{table[keys["fa"]][split][metric]:.3f}',
+ fi=f'{table[keys["fi"]][split][metric]:.3f}',
+ fr=f'{table[keys["fr"]][split][metric]:.3f}',
+ hi=f'{table[keys["hi"]][split][metric]:.3f}',
+ id=f'{table[keys["id"]][split][metric]:.3f}',
+ ja=f'{table[keys["ja"]][split][metric]:.3f}',
+ ko=f'{table[keys["ko"]][split][metric]:.3f}',
+ ru=f'{table[keys["ru"]][split][metric]:.3f}',
+ sw=f'{table[keys["sw"]][split][metric]:.3f}',
+ te=f'{table[keys["te"]][split][metric]:.3f}',
+ th=f'{table[keys["th"]][split][metric]:.3f}',
+ zh=f'{table[keys["zh"]][split][metric]:.3f}',
+ de=f'{table[keys["de"]][split][metric]:.3f}',
+ yo=f'{table[keys["yo"]][split][metric]:.3f}',
+ avg=f'{avg:.3f}',
+ cmd1=f'{commands[keys["ar"]]}',
+ cmd2=f'{commands[keys["bn"]]}',
+ cmd3=f'{commands[keys["en"]]}',
+ cmd4=f'{commands[keys["es"]]}',
+ cmd5=f'{commands[keys["fa"]]}',
+ cmd6=f'{commands[keys["fi"]]}',
+ cmd7=f'{commands[keys["fr"]]}',
+ cmd8=f'{commands[keys["hi"]]}',
+ cmd9=f'{commands[keys["id"]]}',
+ cmd10=f'{commands[keys["ja"]]}',
+ cmd11=f'{commands[keys["ko"]]}',
+ cmd12=f'{commands[keys["ru"]]}',
+ cmd13=f'{commands[keys["sw"]]}',
+ cmd14=f'{commands[keys["te"]]}',
+ cmd15=f'{commands[keys["th"]]}',
+ cmd16=f'{commands[keys["zh"]]}',
+ cmd17=f'{commands[keys["de"]]}',
+ cmd18=f'{commands[keys["yo"]]}',
+ eval_cmd1=f'{eval_commands[keys["ar"]][metric]}',
+ eval_cmd2=f'{eval_commands[keys["bn"]][metric]}',
+ eval_cmd3=f'{eval_commands[keys["en"]][metric]}',
+ eval_cmd4=f'{eval_commands[keys["es"]][metric]}',
+ eval_cmd5=f'{eval_commands[keys["fa"]][metric]}',
+ eval_cmd6=f'{eval_commands[keys["fi"]][metric]}',
+ eval_cmd7=f'{eval_commands[keys["fr"]][metric]}',
+ eval_cmd8=f'{eval_commands[keys["hi"]][metric]}',
+ eval_cmd9=f'{eval_commands[keys["id"]][metric]}',
+ eval_cmd10=f'{eval_commands[keys["ja"]][metric]}',
+ eval_cmd11=f'{eval_commands[keys["ko"]][metric]}',
+ eval_cmd12=f'{eval_commands[keys["ru"]][metric]}',
+ eval_cmd13=f'{eval_commands[keys["sw"]][metric]}',
+ eval_cmd14=f'{eval_commands[keys["te"]][metric]}',
+ eval_cmd15=f'{eval_commands[keys["th"]][metric]}',
+ eval_cmd16=f'{eval_commands[keys["zh"]][metric]}',
+ eval_cmd17=f'{eval_commands[keys["de"]][metric]}',
+ eval_cmd18=f'{eval_commands[keys["yo"]][metric]}'
+ )
+
+ s = s.replace("0.000", "--")
+ html_rows.append(s)
+ row_cnt += 1
+
+ return html_rows
+
+
+def print_results(table, metric, split):
+ print(f'Metric = {metric}, Split = {split}')
+ print(' ' * 35, end='')
+ for lang in languages:
+ print(f'{lang[0]:3} ', end='')
+ print('')
+ for model in models:
+ print(f'{model:33}', end='')
+ for lang in languages:
+ key = f'{model}.{lang[0]}'
+ print(f'{table[key][split][metric]:7.3f}', end='')
+ print('')
+ print('')
+
+
+def extract_topic_fn_from_cmd(cmd):
+ cmd = cmd.split()
+ topic_idx = cmd.index('--topics')
+ return cmd[topic_idx + 1]
+
+
+def generate_report(args):
+ table = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: 0.0)))
+ commands = defaultdict(lambda: '')
+ eval_commands = defaultdict(lambda: defaultdict(lambda: ''))
+
+ html_template = read_file(pkg_resources.resource_filename(__name__, 'miracl_html.template'))
+ table_template = read_file(pkg_resources.resource_filename(__name__, 'miracl_html_table.template'))
+ row_template = read_file(pkg_resources.resource_filename(__name__, 'miracl_html_table_row.template'))
+
+ with open(pkg_resources.resource_filename(__name__, 'miracl.yaml')) as f:
+ yaml_data = yaml.safe_load(f)
+ for condition in yaml_data['conditions']:
+ name = condition['name']
+ eval_key = condition['eval_key']
+ cmd_template = condition['command']
+ cmd_lst = cmd_template.split()
+ lang = name.split('.')[-1]
+ is_hybrid_run = 'hybrid' in name
+
+ for splits in condition['splits']:
+ split = splits['split']
+ if is_hybrid_run:
+ hits = int(cmd_lst[cmd_lst.index('--k') + 1])
+ else:
+ hits = int(cmd_lst[cmd_lst.index('--hits') + 1])
+
+ runfile = os.path.join(args.directory, f'run.miracl.{name}.{split}.txt')
+ if is_hybrid_run:
+ bm25_output = os.path.join(args.directory,
+ f'run.miracl.bm25.{lang}.{split}.top{hits}.txt')
+ mdpr_output = os.path.join(args.directory,
+ f'run.miracl.mdpr-tied-pft-msmarco.{lang}.{split}.top{hits}.txt')
+ expected_args = dict(output=runfile, bm25_output=bm25_output, mdpr_output=mdpr_output)
+ else:
+ expected_args = dict(split=split, output=runfile)
+
+ if not all([f"${k}" in cmd_template or f"${{{k}}}" in cmd_template for k in expected_args]):
+ raise ValueError(f"Not all arguements {list(expected_args)} detected from inputs: {cmd_template}.")
+ cmd = Template(cmd_template).substitute(**expected_args)
+ commands[name] = format_run_command(cmd)
+
+ for expected in splits['scores']:
+ for metric in expected:
+ if str(expected[metric])[-1] == "5":
+ # without adding espilon, there is a chance that f-string would round 0.5 to 0 rather than 1
+ # e.g., 0.8885 -> 0.888 rather than 0.889
+ # add a espilon to the expected score to avoid rounding error
+ expected[metric] += 1e-5
+ table[name][split][metric] = expected[metric]
+
+ eval_cmd = f'python -m pyserini.eval.trec_eval ' + \
+ f'{trec_eval_metric_definitions[metric]} {eval_key}-{split} {runfile}'
+ eval_commands[name][metric] = format_eval_command(eval_cmd)
+
+ tables_html = []
+
+ split = 'dev'
+
+ # Build the table for MRR@100, test queries
+ html_rows = generate_table_rows(table, row_template, commands, eval_commands, 1, split, 'nDCG@10')
+ all_rows = '\n'.join(html_rows)
+ tables_html.append(Template(table_template).substitute(desc=f'nDCG@10, {split} queries', rows=all_rows))
+
+ # Build the table for R@100, test queries
+ html_rows = generate_table_rows(table, row_template, commands, eval_commands, 2, split, 'R@100')
+ all_rows = '\n'.join(html_rows)
+ tables_html.append(Template(table_template).substitute(desc=f'Recall@100, {split} queries', rows=all_rows))
+
+ with open(args.output, 'w') as out:
+ out.write(Template(html_template).substitute(title='MIRACL', tables=' '.join(tables_html)))
+
+
+def run_conditions(args):
+ if args.condition == 'mdpr-tied-pft-msmarco-ft-miracl' and args.language in ['de', 'yo']:
+ print('MIRACL de and yo datasets do not have train splits to finetune with')
+ return
+
+ start = time.time()
+
+ table = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: 0.0)))
+
+ with open(pkg_resources.resource_filename(__name__, 'miracl.yaml')) as f:
+ yaml_data = yaml.safe_load(f)
+ for condition in yaml_data['conditions']:
+ name = condition['name']
+ encoder = name.split('.')[0]
+ lang = name.split('.')[-1]
+ if args.all:
+ pass
+ elif args.condition != encoder:
+ continue
+ elif args.language and args.language != lang:
+ continue
+ eval_key = condition['eval_key']
+ cmd_template = condition['command']
+ cmd_lst = cmd_template.split()
+
+ print(f'condition {name}:')
+ is_hybrid_run = 'hybrid' in name
+
+ for splits in condition['splits']:
+ split = splits['split']
+ if is_hybrid_run:
+ hits = int(cmd_lst[cmd_lst.index('--k') + 1])
+ else:
+ hits = int(cmd_lst[cmd_lst.index('--hits') + 1])
+
+ print(f' - split: {split}')
+
+ runfile = os.path.join(args.directory, f'run.miracl.{name}.{split}.top{hits}.txt')
+ if is_hybrid_run:
+ bm25_output = os.path.join(args.directory,
+ f'run.miracl.bm25.{lang}.{split}.top{hits}.txt')
+ mdpr_output = os.path.join(args.directory,
+ f'run.miracl.mdpr-tied-pft-msmarco.{lang}.{split}.top{hits}.txt')
+ if not os.path.exists(bm25_output):
+ print(f'Missing BM25 file: {bm25_output}')
+ continue
+ if not os.path.exists(mdpr_output):
+ print(f'Missing mDPR file: {mdpr_output}')
+ continue
+ cmd = Template(cmd_template).substitute(split=split, output=runfile, bm25_output=bm25_output,
+ mdpr_output=mdpr_output)
+ else:
+ cmd = Template(cmd_template).substitute(split=split, output=runfile)
+
+ # In the yaml file, the topics are written as something like '--topics miracl-v1.0-ar-${split}'
+ # This works for the dev split because the topics are directly included in Anserini/Pyserini.
+ # For this training split, we have to map the symbol into a file in tools/topics-and-qrels/
+ # Here, we assume that the developer has cloned the miracl repo and placed the topics there.
+ if split == 'train':
+ cmd = cmd.replace(f'--topics miracl-v1.0-{lang}-{split}',
+ f'--topics tools/topics-and-qrels/topics.miracl-v1.0-{lang}-{split}.tsv')
+
+ if args.display_commands:
+ print(f'\n```bash\n{format_run_command(cmd)}\n```\n')
+
+ if not os.path.exists(runfile):
+ if not args.dry_run:
+ rtn = subprocess.run(cmd.split(), capture_output=True)
+ stderr = rtn.stderr.decode()
+ if '--topics' in cmd:
+ topic_fn = extract_topic_fn_from_cmd(cmd)
+ if f'ValueError: Topic {topic_fn} Not Found' in stderr:
+ print(f'Skipping {topic_fn}: file not found.')
+ continue
+
+ for expected in splits['scores']:
+ for metric in expected:
+ if not args.skip_eval:
+ # We have the translate the training qrels into a file located in tools/topics-and-qrels/
+ # because they are not included with Anserini/Pyserini by default.
+ # Here, we assume that the developer has cloned the miracl repo and placed the qrels there.
+ if split == 'train':
+ qrels = f'tools/topics-and-qrels/qrels.{eval_key}-train.tsv'
+ else:
+ qrels = f'{eval_key}-{split}'
+ score = float(run_eval_and_return_metric(metric, qrels,
+ trec_eval_metric_definitions[metric], runfile))
+ if math.isclose(score, float(expected[metric])):
+ result_str = ok_str
+ # Flaky tests
+ elif (name == 'mdpr-tied-pft-msmarco.hi' and split == 'train'
+ and math.isclose(score, float(expected[metric]), abs_tol=2e-4)) or \
+ (name == 'mdpr-tied-pft-msmarco-ft-all.ru'
+ and split == 'dev' and metric == 'nDCG@10'
+ and math.isclose(score, float(expected[metric]), abs_tol=2e-4)) or \
+ (name == 'bm25-mdpr-tied-pft-msmarco-hybrid.te'
+ and split == 'train' and metric == 'nDCG@10'
+ and math.isclose(score, float(expected[metric]), abs_tol=2e-4)) or \
+ (name == 'bm25-mdpr-tied-pft-msmarco-hybrid.zh'
+ and split == 'dev' and metric == 'nDCG@10'
+ and math.isclose(score, float(expected[metric]), abs_tol=2e-4)):
+ result_str = okish_str
+ else:
+ result_str = fail_str + f' expected {expected[metric]:.4f}'
+ print(f' {metric:7}: {score:.4f} {result_str}')
+ table[name][split][metric] = score
+ else:
+ table[name][split][metric] = expected[metric]
+
+ print('')
+
+ for metric in ['nDCG@10', 'R@100']:
+ for split in ['dev', 'train']:
+ print_results(table, metric, split)
+
+ end = time.time()
+ print(f'Total elapsed time: {end - start:.0f}s')
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser(description='Generate regression matrix for MIRACL.')
+ parser.add_argument('--condition', type=str,
+ help='Condition to run', required=False)
+ # To list all conditions
+ parser.add_argument('--list-conditions', action='store_true', default=False, help='List available conditions.')
+ # For generating reports
+ parser.add_argument('--generate-report', action='store_true', default=False, help='Generate report.')
+ parser.add_argument('--output', type=str, help='File to store report.', required=False)
+ # For actually running the experimental conditions
+ parser.add_argument('--all', action='store_true', default=False, help='Run using all languages.')
+ parser.add_argument('--language', type=str, help='Language to run.', required=False)
+ parser.add_argument('--directory', type=str, help='Base directory.', default='', required=False)
+ parser.add_argument('--dry-run', action='store_true', default=False, help='Print out commands but do not execute.')
+ parser.add_argument('--skip-eval', action='store_true', default=False, help='Skip running trec_eval.')
+ parser.add_argument('--display-commands', action='store_true', default=False, help='Display command.')
+ args = parser.parse_args()
+
+ if args.list_conditions:
+ list_conditions()
+ sys.exit()
+
+ if args.generate_report:
+ if not args.output:
+ print(f'Must specify report filename with --output.')
+ sys.exit()
+
+ generate_report(args)
+ sys.exit()
+
+ if args.all and (args.condition or args.language):
+ print('Specifying --all will run all conditions and languages')
+ sys.exit()
+
+ run_conditions(args)

pyserini/2cr/miracl.yaml

@@ -0,0 +1,1180 @@
+conditions:
+ # BM25
+ - name: bm25.ar
+ eval_key: miracl-v1.0-ar
+ command: python -m pyserini.search.lucene --language ar --topics miracl-v1.0-ar-${split} --index miracl-v1.0-ar --output $output --batch 128 --threads 16 --bm25 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.4434
+ R@100: 0.8562
+ - split: dev
+ scores:
+ - nDCG@10: 0.4809
+ R@100: 0.8885
+ - name: bm25.bn
+ eval_key: miracl-v1.0-bn
+ command: python -m pyserini.search.lucene --language bn --topics miracl-v1.0-bn-${split} --index miracl-v1.0-bn --output $output --batch 128 --threads 16 --bm25 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.5122
+ R@100: 0.8934
+ - split: dev
+ scores:
+ - nDCG@10: 0.5079
+ R@100: 0.9088
+ - name: bm25.en
+ eval_key: miracl-v1.0-en
+ command: python -m pyserini.search.lucene --language en --topics miracl-v1.0-en-${split} --index miracl-v1.0-en --output $output --batch 128 --threads 16 --bm25 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.3415
+ R@100: 0.7928
+ - split: dev
+ scores:
+ - nDCG@10: 0.3506
+ R@100: 0.8190
+ - name: bm25.es
+ eval_key: miracl-v1.0-es
+ command: python -m pyserini.search.lucene --language es --topics miracl-v1.0-es-${split} --index miracl-v1.0-es --output $output --batch 128 --threads 16 --bm25 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.3030
+ R@100: 0.7020
+ - split: dev
+ scores:
+ - nDCG@10: 0.3193
+ R@100: 0.7018
+ - name: bm25.fa
+ eval_key: miracl-v1.0-fa
+ command: python -m pyserini.search.lucene --language fa --topics miracl-v1.0-fa-${split} --index miracl-v1.0-fa --output $output --batch 128 --threads 16 --bm25 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.3270
+ R@100: 0.7139
+ - split: dev
+ scores:
+ - nDCG@10: 0.3334
+ R@100: 0.7306
+ - name: bm25.fi
+ eval_key: miracl-v1.0-fi
+ command: python -m pyserini.search.lucene --language fi --topics miracl-v1.0-fi-${split} --index miracl-v1.0-fi --output $output --batch 128 --threads 16 --bm25 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.5106
+ R@100: 0.8471
+ - split: dev
+ scores:
+ - nDCG@10: 0.5513
+ R@100: 0.8910
+ - name: bm25.fr
+ eval_key: miracl-v1.0-fr
+ command: python -m pyserini.search.lucene --language fr --topics miracl-v1.0-fr-${split} --index miracl-v1.0-fr --output $output --batch 128 --threads 16 --bm25 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.2152
+ R@100: 0.6601
+ - split: dev
+ scores:
+ - nDCG@10: 0.1832
+ R@100: 0.6528
+ - name: bm25.hi
+ eval_key: miracl-v1.0-hi
+ command: python -m pyserini.search.lucene --language hi --topics miracl-v1.0-hi-${split} --index miracl-v1.0-hi --output $output --batch 128 --threads 16 --bm25 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.4745
+ R@100: 0.9016
+ - split: dev
+ scores:
+ - nDCG@10: 0.4578
+ R@100: 0.8679
+ - name: bm25.id
+ eval_key: miracl-v1.0-id
+ command: python -m pyserini.search.lucene --language id --topics miracl-v1.0-id-${split} --index miracl-v1.0-id --output $output --batch 128 --threads 16 --bm25 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.4844
+ R@100: 0.9234
+ - split: dev
+ scores:
+ - nDCG@10: 0.4486
+ R@100: 0.9041
+ - name: bm25.ja
+ eval_key: miracl-v1.0-ja
+ command: python -m pyserini.search.lucene --language ja --topics miracl-v1.0-ja-${split} --index miracl-v1.0-ja --output $output --batch 128 --threads 16 --bm25 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.3796
+ R@100: 0.8225
+ - split: dev
+ scores:
+ - nDCG@10: 0.3689
+ R@100: 0.8048
+ - name: bm25.ko
+ eval_key: miracl-v1.0-ko
+ command: python -m pyserini.search.lucene --language ko --topics miracl-v1.0-ko-${split} --index miracl-v1.0-ko --output $output --batch 128 --threads 16 --bm25 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.4279
+ R@100: 0.7572
+ - split: dev
+ scores:
+ - nDCG@10: 0.4190
+ R@100: 0.7831
+ - name: bm25.ru
+ eval_key: miracl-v1.0-ru
+ command: python -m pyserini.search.lucene --language ru --topics miracl-v1.0-ru-${split} --index miracl-v1.0-ru --output $output --batch 128 --threads 16 --bm25 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.3153
+ R@100: 0.6464
+ - split: dev
+ scores:
+ - nDCG@10: 0.3342
+ R@100: 0.6614
+ - name: bm25.sw
+ eval_key: miracl-v1.0-sw
+ command: python -m pyserini.search.lucene --language sw --topics miracl-v1.0-sw-${split} --index miracl-v1.0-sw --output $output --batch 128 --threads 16 --bm25 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.3356
+ R@100: 0.6499
+ - split: dev
+ scores:
+ - nDCG@10: 0.3826
+ R@100: 0.7008
+ - name: bm25.te
+ eval_key: miracl-v1.0-te
+ command: python -m pyserini.search.lucene --language te --topics miracl-v1.0-te-${split} --index miracl-v1.0-te --output $output --batch 128 --threads 16 --bm25 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.4814
+ R@100: 0.8077
+ - split: dev
+ scores:
+ - nDCG@10: 0.4942
+ R@100: 0.8307
+ - name: bm25.th
+ eval_key: miracl-v1.0-th
+ command: python -m pyserini.search.lucene --language th --topics miracl-v1.0-th-${split} --index miracl-v1.0-th --output $output --batch 128 --threads 16 --bm25 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.4629
+ R@100: 0.8768
+ - split: dev
+ scores:
+ - nDCG@10: 0.4838
+ R@100: 0.8874
+ - name: bm25.zh
+ eval_key: miracl-v1.0-zh
+ command: python -m pyserini.search.lucene --language zh --topics miracl-v1.0-zh-${split} --index miracl-v1.0-zh --output $output --batch 128 --threads 16 --bm25 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.2018
+ R@100: 0.5541
+ - split: dev
+ scores:
+ - nDCG@10: 0.1801
+ R@100: 0.5599
+ - name: bm25.de
+ eval_key: miracl-v1.0-de
+ command: python -m pyserini.search.lucene --language de --topics miracl-v1.0-de-${split} --index miracl-v1.0-de --output $output --batch 128 --threads 16 --bm25 --hits 1000
+ splits:
+ - split: dev
+ scores:
+ - nDCG@10: 0.2262
+ R@100: 0.5724
+ - name: bm25.yo
+ eval_key: miracl-v1.0-yo
+ command: python -m pyserini.search.lucene --pretokenized --topics miracl-v1.0-yo-${split} --index miracl-v1.0-yo --output $output --batch 128 --threads 16 --bm25 --hits 1000
+ splits:
+ - split: dev
+ scores:
+ - nDCG@10: 0.4059
+ R@100: 0.7325
+
+ # mdpr-tied-pft-msmarco
+ - name: mdpr-tied-pft-msmarco.ar
+ eval_key: miracl-v1.0-ar
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics miracl-v1.0-ar-${split} --index miracl-v1.0-ar-mdpr-tied-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores: 
+ - nDCG@10: 0.4653
+ R@100: 0.8293 
+ - split: dev 
+ scores: 
+ - nDCG@10: 0.4993
+ R@100: 0.8407
+ - name: mdpr-tied-pft-msmarco.bn
+ eval_key: miracl-v1.0-bn
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics miracl-v1.0-bn-${split} --index miracl-v1.0-bn-mdpr-tied-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores: 
+ - nDCG@10: 0.4362
+ R@100: 0.8045 
+ - split: dev 
+ scores: 
+ - nDCG@10: 0.4427
+ R@100: 0.8193
+ - name: mdpr-tied-pft-msmarco.en
+ eval_key: miracl-v1.0-en
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics miracl-v1.0-en-${split} --index miracl-v1.0-en-mdpr-tied-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores: 
+ - nDCG@10: 0.3986
+ R@100: 0.7779 
+ - split: dev 
+ scores: 
+ - nDCG@10: 0.3938
+ R@100: 0.7675
+ - name: mdpr-tied-pft-msmarco.es
+ eval_key: miracl-v1.0-es
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics miracl-v1.0-es-${split} --index miracl-v1.0-es-mdpr-tied-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.4637
+ R@100: 0.8654
+ - split: dev
+ scores:
+ - nDCG@10: 0.4777
+ R@100: 0.8643
+ - name: mdpr-tied-pft-msmarco.fa
+ eval_key: miracl-v1.0-fa
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics miracl-v1.0-fa-${split} --index miracl-v1.0-fa-mdpr-tied-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.4882
+ R@100: 0.9092
+ - split: dev
+ scores:
+ - nDCG@10: 0.4800
+ R@100: 0.8980
+ - name: mdpr-tied-pft-msmarco.fi
+ eval_key: miracl-v1.0-fi
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics miracl-v1.0-fi-${split} --index miracl-v1.0-fi-mdpr-tied-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores: 
+ - nDCG@10: 0.4426
+ R@100: 0.7611 
+ - split: dev 
+ scores: 
+ - nDCG@10: 0.4721
+ R@100: 0.7877
+ - name: mdpr-tied-pft-msmarco.fr
+ eval_key: miracl-v1.0-fr
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics miracl-v1.0-fr-${split} --index miracl-v1.0-fr-mdpr-tied-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.4372
+ R@100: 0.9268
+ - split: dev
+ scores:
+ - nDCG@10: 0.4352
+ R@100: 0.9154
+ - name: mdpr-tied-pft-msmarco.hi
+ eval_key: miracl-v1.0-hi
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics miracl-v1.0-hi-${split} --index miracl-v1.0-hi-mdpr-tied-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.3685
+ R@100: 0.7780
+ - split: dev
+ scores:
+ - nDCG@10: 0.3830
+ R@100: 0.7755
+ - name: mdpr-tied-pft-msmarco.id
+ eval_key: miracl-v1.0-id
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics miracl-v1.0-id-${split} --index miracl-v1.0-id-mdpr-tied-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores: 
+ - nDCG@10: 0.2549
+ R@100: 0.5610 
+ - split: dev 
+ scores: 
+ - nDCG@10: 0.2719
+ R@100: 0.5734
+ - name: mdpr-tied-pft-msmarco.ja
+ eval_key: miracl-v1.0-ja
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics miracl-v1.0-ja-${split} --index miracl-v1.0-ja-mdpr-tied-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores: 
+ - nDCG@10: 0.4342
+ R@100: 0.8211 
+ - split: dev 
+ scores: 
+ - nDCG@10: 0.4390
+ R@100: 0.8254
+ - name: mdpr-tied-pft-msmarco.ko
+ eval_key: miracl-v1.0-ko
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics miracl-v1.0-ko-${split} --index miracl-v1.0-ko-mdpr-tied-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores: 
+ - nDCG@10: 0.4147
+ R@100: 0.7699 
+ - split: dev 
+ scores: 
+ - nDCG@10: 0.4189
+ R@100: 0.7369
+ - name: mdpr-tied-pft-msmarco.ru
+ eval_key: miracl-v1.0-ru
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics miracl-v1.0-ru-${split} --index miracl-v1.0-ru-mdpr-tied-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores: 
+ - nDCG@10: 0.3812
+ R@100: 0.7854 
+ - split: dev 
+ scores: 
+ - nDCG@10: 0.4073
+ R@100: 0.7972
+ - name: mdpr-tied-pft-msmarco.sw
+ eval_key: miracl-v1.0-sw
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics miracl-v1.0-sw-${split} --index miracl-v1.0-sw-mdpr-tied-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores: 
+ - nDCG@10: 0.2973
+ R@100: 0.5761 
+ - split: dev 
+ scores: 
+ - nDCG@10: 0.2990
+ R@100: 0.6158
+ - name: mdpr-tied-pft-msmarco.te
+ eval_key: miracl-v1.0-te
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics miracl-v1.0-te-${split} --index miracl-v1.0-te-mdpr-tied-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.3723
+ R@100: 0.7698
+ - split: dev
+ scores:
+ - nDCG@10: 0.3557
+ R@100: 0.7619
+ - name: mdpr-tied-pft-msmarco.th
+ eval_key: miracl-v1.0-th
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics miracl-v1.0-th-${split} --index miracl-v1.0-th-mdpr-tied-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.3451
+ R@100: 0.6728
+ - split: dev
+ scores:
+ - nDCG@10: 0.3578
+ R@100: 0.6783
+ - name: mdpr-tied-pft-msmarco.zh
+ eval_key: miracl-v1.0-zh
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics miracl-v1.0-zh-${split} --index miracl-v1.0-zh-mdpr-tied-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.5040
+ R@100: 0.9355
+ - split: dev
+ scores:
+ - nDCG@10: 0.5116
+ R@100: 0.9436
+ - name: mdpr-tied-pft-msmarco.de
+ eval_key: miracl-v1.0-de
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics miracl-v1.0-de-${split} --index miracl-v1.0-de-mdpr-tied-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: dev
+ scores:
+ - nDCG@10: 0.4895
+ R@100: 0.8983
+ - name: mdpr-tied-pft-msmarco.yo
+ eval_key: miracl-v1.0-yo
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics miracl-v1.0-yo-${split} --index miracl-v1.0-yo-mdpr-tied-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: dev
+ scores:
+ - nDCG@10: 0.4439
+ R@100: 0.8403
+
+ # mdpr-tied-pft-msmarco-ft-all
+ - name: mdpr-tied-pft-msmarco-ft-all.ar
+ eval_key: miracl-v1.0-ar
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics miracl-v1.0-ar-${split} --index miracl-v1.0-ar-mdpr-tied-pft-msmarco-ft-all --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.6954
+ R@100: 0.8542
+ - split: dev
+ scores:
+ - nDCG@10: 0.5782
+ R@100: 0.7953
+ - name: mdpr-tied-pft-msmarco-ft-all.bn
+ eval_key: miracl-v1.0-bn
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics miracl-v1.0-bn-${split} --index miracl-v1.0-bn-mdpr-tied-pft-msmarco-ft-all --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.6823
+ R@100: 0.8646
+ - split: dev
+ scores:
+ - nDCG@10: 0.5804
+ R@100: 0.8480
+ - name: mdpr-tied-pft-msmarco-ft-all.en
+ eval_key: miracl-v1.0-en
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics miracl-v1.0-en-${split} --index miracl-v1.0-en-mdpr-tied-pft-msmarco-ft-all --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.3491
+ R@100: 0.5678
+ - split: dev
+ scores:
+ - nDCG@10: 0.2813
+ R@100: 0.5083
+ - name: mdpr-tied-pft-msmarco-ft-all.es
+ eval_key: miracl-v1.0-es
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics miracl-v1.0-es-${split} --index miracl-v1.0-es-mdpr-tied-pft-msmarco-ft-all --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.2488
+ R@100: 0.4799
+ - split: dev
+ scores:
+ - nDCG@10: 0.2509
+ R@100: 0.4706
+ - name: mdpr-tied-pft-msmarco-ft-all.fa
+ eval_key: miracl-v1.0-fa
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics miracl-v1.0-fa-${split} --index miracl-v1.0-fa-mdpr-tied-pft-msmarco-ft-all --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.3809
+ R@100: 0.6899
+ - split: dev
+ scores:
+ - nDCG@10: 0.3836
+ R@100: 0.6863
+ - name: mdpr-tied-pft-msmarco-ft-all.fi
+ eval_key: miracl-v1.0-fi
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics miracl-v1.0-fi-${split} --index miracl-v1.0-fi-mdpr-tied-pft-msmarco-ft-all --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.7738
+ R@100: 0.9081
+ - split: dev
+ scores:
+ - nDCG@10: 0.5694
+ R@100: 0.7984
+ - name: mdpr-tied-pft-msmarco-ft-all.fr
+ eval_key: miracl-v1.0-fr
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics miracl-v1.0-fr-${split} --index miracl-v1.0-fr-mdpr-tied-pft-msmarco-ft-all --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.2989
+ R@100: 0.6197
+ - split: dev
+ scores:
+ - nDCG@10: 0.3010
+ R@100: 0.6005
+ - name: mdpr-tied-pft-msmarco-ft-all.hi
+ eval_key: miracl-v1.0-hi
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics miracl-v1.0-hi-${split} --index miracl-v1.0-hi-mdpr-tied-pft-msmarco-ft-all --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.3336
+ R@100: 0.6388
+ - split: dev
+ scores:
+ - nDCG@10: 0.3286
+ R@100: 0.6371
+ - name: mdpr-tied-pft-msmarco-ft-all.id
+ eval_key: miracl-v1.0-id
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics miracl-v1.0-id-${split} --index miracl-v1.0-id-mdpr-tied-pft-msmarco-ft-all --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.3321
+ R@100: 0.5492
+ - split: dev
+ scores:
+ - nDCG@10: 0.3462
+ R@100: 0.5841
+ - name: mdpr-tied-pft-msmarco-ft-all.ja
+ eval_key: miracl-v1.0-ja
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics miracl-v1.0-ja-${split} --index miracl-v1.0-ja-mdpr-tied-pft-msmarco-ft-all --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.6378
+ R@100: 0.7950
+ - split: dev
+ scores:
+ - nDCG@10: 0.4999
+ R@100: 0.7451
+ - name: mdpr-tied-pft-msmarco-ft-all.ko
+ eval_key: miracl-v1.0-ko
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics miracl-v1.0-ko-${split} --index miracl-v1.0-ko-mdpr-tied-pft-msmarco-ft-all --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.5795
+ R@100: 0.7850
+ - split: dev
+ scores:
+ - nDCG@10: 0.4864
+ R@100: 0.7183
+ - name: mdpr-tied-pft-msmarco-ft-all.ru
+ eval_key: miracl-v1.0-ru
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics miracl-v1.0-ru-${split} --index miracl-v1.0-ru-mdpr-tied-pft-msmarco-ft-all --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.6011
+ R@100: 0.8188
+ - split: dev
+ scores:
+ - nDCG@10: 0.3933
+ R@100: 0.6707
+ - name: mdpr-tied-pft-msmarco-ft-all.sw
+ eval_key: miracl-v1.0-sw
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics miracl-v1.0-sw-${split} --index miracl-v1.0-sw-mdpr-tied-pft-msmarco-ft-all --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.8882
+ R@100: 0.9710
+ - split: dev
+ scores:
+ - nDCG@10: 0.6575
+ R@100: 0.8883
+ - name: mdpr-tied-pft-msmarco-ft-all.te
+ eval_key: miracl-v1.0-te
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics miracl-v1.0-te-${split} --index miracl-v1.0-te-mdpr-tied-pft-msmarco-ft-all --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.8757
+ R@100: 0.9725
+ - split: dev
+ scores:
+ - nDCG@10: 0.7783
+ R@100: 0.9513
+ - name: mdpr-tied-pft-msmarco-ft-all.th
+ eval_key: miracl-v1.0-th
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics miracl-v1.0-th-${split} --index miracl-v1.0-th-mdpr-tied-pft-msmarco-ft-all --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.7761
+ R@100: 0.9241
+ - split: dev
+ scores:
+ - nDCG@10: 0.5975
+ R@100: 0.8360
+ - name: mdpr-tied-pft-msmarco-ft-all.zh
+ eval_key: miracl-v1.0-zh
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics miracl-v1.0-zh-${split} --index miracl-v1.0-zh-mdpr-tied-pft-msmarco-ft-all --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.3446
+ R@100: 0.6608
+ - split: dev
+ scores:
+ - nDCG@10: 0.3575
+ R@100: 0.6725
+ - name: mdpr-tied-pft-msmarco-ft-all.de
+ eval_key: miracl-v1.0-de
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics miracl-v1.0-de-${split} --index miracl-v1.0-de-mdpr-tied-pft-msmarco-ft-all --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: dev
+ scores:
+ - nDCG@10: 0.3219
+ R@100: 0.5990
+ - name: mdpr-tied-pft-msmarco-ft-all.yo
+ eval_key: miracl-v1.0-yo
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics miracl-v1.0-yo-${split} --index miracl-v1.0-yo-mdpr-tied-pft-msmarco-ft-all --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: dev
+ scores:
+ - nDCG@10: 0.5983
+ R@100: 0.8908
+
+ - name: bm25-mdpr-tied-pft-msmarco-hybrid.ar
+ eval_key: miracl-v1.0-ar
+ command: python -m pyserini.fusion --runs ${bm25_output} ${mdpr_output} --output ${output} --method interpolation --alpha 0.5 --depth 1000 --k 1000
+ splits: 
+ - split: train
+ scores:
+ - nDCG@10: 0.6259
+ R@100: 0.9173
+ - split: dev
+ scores:
+ - nDCG@10: 0.6729
+ R@100: 0.9405
+ - name: bm25-mdpr-tied-pft-msmarco-hybrid.bn
+ eval_key: miracl-v1.0-bn
+ command: python -m pyserini.fusion --runs ${bm25_output} ${mdpr_output} --output ${output} --method interpolation --alpha 0.5 --depth 1000 --k 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.6587
+ R@100: 0.9297
+ - split: dev
+ scores:
+ - nDCG@10: 0.6540
+ R@100: 0.9321
+ - name: bm25-mdpr-tied-pft-msmarco-hybrid.en
+ eval_key: miracl-v1.0-en
+ command: python -m pyserini.fusion --runs ${bm25_output} ${mdpr_output} --output ${output} --method interpolation --alpha 0.5 --depth 1000 --k 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.5347
+ R@100: 0.8772
+ - split: dev
+ scores:
+ - nDCG@10: 0.5488
+ R@100: 0.8815
+ - name: bm25-mdpr-tied-pft-msmarco-hybrid.es
+ eval_key: miracl-v1.0-es
+ command: python -m pyserini.fusion --runs ${bm25_output} ${mdpr_output} --output ${output} --method interpolation --alpha 0.5 --depth 1000 --k 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.6234
+ R@100: 0.9425
+ - split: dev
+ scores:
+ - nDCG@10: 0.6413
+ R@100: 0.9479
+ - name: bm25-mdpr-tied-pft-msmarco-hybrid.fa
+ eval_key: miracl-v1.0-fa
+ command: python -m pyserini.fusion --runs ${bm25_output} ${mdpr_output} --output ${output} --method interpolation --alpha 0.5 --depth 1000 --k 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.5890
+ R@100: 0.9433
+ - split: dev
+ scores:
+ - nDCG@10: 0.5935
+ R@100: 0.9374
+ - name: bm25-mdpr-tied-pft-msmarco-hybrid.fi
+ eval_key: miracl-v1.0-fi
+ command: python -m pyserini.fusion --runs ${bm25_output} ${mdpr_output} --output ${output} --method interpolation --alpha 0.5 --depth 1000 --k 1000
+ splits:
+ - split: train
+ scores:
+ scores:
+ - nDCG@10: 0.6164
+ R@100: 0.8506
+ - split: dev
+ scores:
+ - nDCG@10: 0.6716
+ R@100: 0.8949
+ - name: bm25-mdpr-tied-pft-msmarco-hybrid.fr
+ eval_key: miracl-v1.0-fr
+ command: python -m pyserini.fusion --runs ${bm25_output} ${mdpr_output} --output ${output} --method interpolation --alpha 0.5 --depth 1000 --k 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.5299
+ R@100: 0.9709
+ - split: dev
+ scores:
+ - nDCG@10: 0.5233
+ R@100: 0.9647
+ - name: bm25-mdpr-tied-pft-msmarco-hybrid.hi
+ eval_key: miracl-v1.0-hi
+ command: python -m pyserini.fusion --runs ${bm25_output} ${mdpr_output} --output ${output} --method interpolation --alpha 0.5 --depth 1000 --k 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.6217
+ R@100: 0.9059
+ - split: dev
+ scores:
+ - nDCG@10: 0.6157
+ R@100: 0.9115
+ - name: bm25-mdpr-tied-pft-msmarco-hybrid.id
+ eval_key: miracl-v1.0-id
+ command: python -m pyserini.fusion --runs ${bm25_output} ${mdpr_output} --output ${output} --method interpolation --alpha 0.5 --depth 1000 --k 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.4442
+ R@100: 0.7595
+ - split: dev
+ scores:
+ - nDCG@10: 0.4433
+ R@100: 0.7683
+ - name: bm25-mdpr-tied-pft-msmarco-hybrid.ja
+ eval_key: miracl-v1.0-ja
+ command: python -m pyserini.fusion --runs ${bm25_output} ${mdpr_output} --output ${output} --method interpolation --alpha 0.5 --depth 1000 --k 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.5795
+ R@100: 0.9082
+ - split: dev
+ scores:
+ - nDCG@10: 0.5757
+ R@100: 0.9036
+ - name: bm25-mdpr-tied-pft-msmarco-hybrid.ko
+ eval_key: miracl-v1.0-ko
+ command: python -m pyserini.fusion --runs ${bm25_output} ${mdpr_output} --output ${output} --method interpolation --alpha 0.5 --depth 1000 --k 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.5758
+ R@100: 0.8744
+ - split: dev
+ scores:
+ - nDCG@10: 0.6086
+ R@100: 0.8997
+ - name: bm25-mdpr-tied-pft-msmarco-hybrid.ru
+ eval_key: miracl-v1.0-ru
+ command: python -m pyserini.fusion --runs ${bm25_output} ${mdpr_output} --output ${output} --method interpolation --alpha 0.5 --depth 1000 --k 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.4921
+ R@100: 0.8494
+ - split: dev
+ scores:
+ - nDCG@10: 0.5323
+ R@100: 0.8738
+ - name: bm25-mdpr-tied-pft-msmarco-hybrid.sw
+ eval_key: miracl-v1.0-sw
+ command: python -m pyserini.fusion --runs ${bm25_output} ${mdpr_output} --output ${output} --method interpolation --alpha 0.5 --depth 1000 --k 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.4100
+ R@100: 0.6987
+ - split: dev
+ scores:
+ - nDCG@10: 0.4457
+ R@100: 0.7254
+ - name: bm25-mdpr-tied-pft-msmarco-hybrid.te
+ eval_key: miracl-v1.0-te
+ command: python -m pyserini.fusion --runs ${bm25_output} ${mdpr_output} --output ${output} --method interpolation --alpha 0.5 --depth 1000 --k 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.6000
+ R@100: 0.8717
+ - split: dev
+ scores:
+ - nDCG@10: 0.6021
+ R@100: 0.8569
+ - name: bm25-mdpr-tied-pft-msmarco-hybrid.th
+ eval_key: miracl-v1.0-th
+ command: python -m pyserini.fusion --runs ${bm25_output} ${mdpr_output} --output ${output} --method interpolation --alpha 0.5 --depth 1000 --k 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.5669
+ R@100: 0.8195
+ - split: dev
+ scores:
+ - nDCG@10: 0.5990
+ R@100: 0.8228
+ - name: bm25-mdpr-tied-pft-msmarco-hybrid.zh
+ eval_key: miracl-v1.0-zh
+ command: python -m pyserini.fusion --runs ${bm25_output} ${mdpr_output} --output ${output} --method interpolation --alpha 0.5 --depth 1000 --k 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.5209
+ R@100: 0.9576
+ - split: dev
+ scores:
+ - nDCG@10: 0.5254
+ R@100: 0.9587
+ - name: bm25-mdpr-tied-pft-msmarco-hybrid.de
+ eval_key: miracl-v1.0-de
+ command: python -m pyserini.fusion --runs ${bm25_output} ${mdpr_output} --output ${output} --method interpolation --alpha 0.5 --depth 1000 --k 1000
+ splits:
+ - split: dev
+ scores:
+ - nDCG@10: 0.5643
+ R@100: 0.9482
+ - name: bm25-mdpr-tied-pft-msmarco-hybrid.yo
+ eval_key: miracl-v1.0-yo
+ command: python -m pyserini.fusion --runs ${bm25_output} ${mdpr_output} --output ${output} --method interpolation --alpha 0.5 --depth 1000 --k 1000
+ splits:
+ - split: dev
+ scores:
+ - nDCG@10: 0.6114
+ R@100: 0.9496
+
+ # mdpr-tied-pft-msmarco-ft-miracl-ft-miracl 
+ - name: mdpr-tied-pft-msmarco-ft-miracl.ar
+ eval_key: miracl-v1.0-ar
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-miracl-ar --topics miracl-v1.0-ar-${split} --index miracl-v1.0-ar-mdpr-tied-pft-msmarco-ft-miracl-ar --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: dev 
+ scores: 
+ - nDCG@10: 0.7252
+ R@100: 0.9489
+ - name: mdpr-tied-pft-msmarco-ft-miracl.bn
+ eval_key: miracl-v1.0-bn
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-miracl-bn --topics miracl-v1.0-bn-${split} --index miracl-v1.0-bn-mdpr-tied-pft-msmarco-ft-miracl-bn --output $output --batch 128 --threads 16 --hits 1000
+ splits: 
+ - split: dev 
+ scores: 
+ - nDCG@10: 0.6842
+ R@100: 0.9547
+ - name: mdpr-tied-pft-msmarco-ft-miracl.en
+ eval_key: miracl-v1.0-en
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-miracl-en --topics miracl-v1.0-en-${split} --index miracl-v1.0-en-mdpr-tied-pft-msmarco-ft-miracl-en --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: dev 
+ scores: 
+ - nDCG@10: 0.4878
+ R@100: 0.8341
+ - name: mdpr-tied-pft-msmarco-ft-miracl.es
+ eval_key: miracl-v1.0-es
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-miracl-es --topics miracl-v1.0-es-${split} --index miracl-v1.0-es-mdpr-tied-pft-msmarco-ft-miracl-es --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: dev
+ scores:
+ - nDCG@10: 0.5648
+ R@100: 0.9109
+ - name: mdpr-tied-pft-msmarco-ft-miracl.fa
+ eval_key: miracl-v1.0-fa
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-miracl-fa --topics miracl-v1.0-fa-${split} --index miracl-v1.0-fa-mdpr-tied-pft-msmarco-ft-miracl-fa --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: dev
+ scores:
+ - nDCG@10: 0.5934
+ R@100: 0.9133
+ - name: mdpr-tied-pft-msmarco-ft-miracl.fi
+ eval_key: miracl-v1.0-fi
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-miracl-fi --topics miracl-v1.0-fi-${split} --index miracl-v1.0-fi-mdpr-tied-pft-msmarco-ft-miracl-fi --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: dev 
+ scores: 
+ - nDCG@10: 0.7139
+ R@100: 0.9479
+ - name: mdpr-tied-pft-msmarco-ft-miracl.fr
+ eval_key: miracl-v1.0-fr
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-miracl-fr --topics miracl-v1.0-fr-${split} --index miracl-v1.0-fr-mdpr-tied-pft-msmarco-ft-miracl-fr --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: dev
+ scores:
+ - nDCG@10: 0.5893
+ R@100: 0.9537
+ - name: mdpr-tied-pft-msmarco-ft-miracl.hi
+ eval_key: miracl-v1.0-hi
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-miracl-hi --topics miracl-v1.0-hi-${split} --index miracl-v1.0-hi-mdpr-tied-pft-msmarco-ft-miracl-hi --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: dev
+ scores:
+ - nDCG@10: 0.5164
+ R@100: 0.8862
+ - name: mdpr-tied-pft-msmarco-ft-miracl.id
+ eval_key: miracl-v1.0-id
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-miracl-id --topics miracl-v1.0-id-${split} --index miracl-v1.0-id-mdpr-tied-pft-msmarco-ft-miracl-id --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: dev 
+ scores: 
+ - nDCG@10: 0.4959
+ R@100: 0.8642
+ - name: mdpr-tied-pft-msmarco-ft-miracl.ja
+ eval_key: miracl-v1.0-ja
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-miracl-ja --topics miracl-v1.0-ja-${split} --index miracl-v1.0-ja-mdpr-tied-pft-msmarco-ft-miracl-ja --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: dev 
+ scores: 
+ - nDCG@10: 0.6416
+ R@100: 0.9225
+ - name: mdpr-tied-pft-msmarco-ft-miracl.ko
+ eval_key: miracl-v1.0-ko
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-miracl-ko --topics miracl-v1.0-ko-${split} --index miracl-v1.0-ko-mdpr-tied-pft-msmarco-ft-miracl-ko --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: dev 
+ scores: 
+ - nDCG@10: 0.5901
+ R@100: 0.8857
+ - name: mdpr-tied-pft-msmarco-ft-miracl.ru
+ eval_key: miracl-v1.0-ru
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-miracl-ru --topics miracl-v1.0-ru-${split} --index miracl-v1.0-ru-mdpr-tied-pft-msmarco-ft-miracl-ru --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: dev 
+ scores: 
+ - nDCG@10: 0.5974
+ R@100: 0.9099
+ - name: mdpr-tied-pft-msmarco-ft-miracl.sw
+ eval_key: miracl-v1.0-sw
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-miracl-sw --topics miracl-v1.0-sw-${split} --index miracl-v1.0-sw-mdpr-tied-pft-msmarco-ft-miracl-sw --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: dev 
+ scores: 
+ - nDCG@10: 0.6853
+ R@100: 0.9367
+ - name: mdpr-tied-pft-msmarco-ft-miracl.te
+ eval_key: miracl-v1.0-te
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-miracl-te --topics miracl-v1.0-te-${split} --index miracl-v1.0-te-mdpr-tied-pft-msmarco-ft-miracl-te --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: dev
+ scores:
+ - nDCG@10: 0.8037
+ R@100: 0.9616
+ - name: mdpr-tied-pft-msmarco-ft-miracl.th
+ eval_key: miracl-v1.0-th
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-miracl-th --topics miracl-v1.0-th-${split} --index miracl-v1.0-th-mdpr-tied-pft-msmarco-ft-miracl-th --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: dev
+ scores:
+ - nDCG@10: 0.6951
+ R@100: 0.9311
+ - name: mdpr-tied-pft-msmarco-ft-miracl.zh
+ eval_key: miracl-v1.0-zh
+ command: python -m pyserini.search.faiss --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-miracl-zh --topics miracl-v1.0-zh-${split} --index miracl-v1.0-zh-mdpr-tied-pft-msmarco-ft-miracl-zh --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: dev
+ scores:
+ - nDCG@10: 0.6500
+ R@100: 0.9631
+
+ # mcontriever
+ - name: mcontriever-tied-pft-msmarco.ar
+ eval_key: miracl-v1.0-ar
+ command: python -m pyserini.search.faiss --encoder-class contriever --encoder facebook/mcontriever-msmarco --topics miracl-v1.0-ar-${split} --index miracl-v1.0-ar-mcontriever-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.5027
+ R@100: 0.9166
+ - split: dev
+ scores:
+ - nDCG@10: 0.5248
+ R@100: 0.9253
+ - name: mcontriever-tied-pft-msmarco.bn
+ eval_key: miracl-v1.0-bn
+ command: python -m pyserini.search.faiss --encoder-class contriever --encoder facebook/mcontriever-msmarco --topics miracl-v1.0-bn-${split} --index miracl-v1.0-bn-mcontriever-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.5138
+ R@100: 0.9313
+ - split: dev
+ scores:
+ - nDCG@10: 0.5011
+ R@100: 0.9205
+ - name: mcontriever-tied-pft-msmarco.en
+ eval_key: miracl-v1.0-en
+ command: python -m pyserini.search.faiss --encoder-class contriever --encoder facebook/mcontriever-msmarco --topics miracl-v1.0-en-${split} --index miracl-v1.0-en-mcontriever-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.3579
+ R@100: 0.7990
+ - split: dev
+ scores:
+ - nDCG@10: 0.3637
+ R@100: 0.7967
+ - name: mcontriever-tied-pft-msmarco.es
+ eval_key: miracl-v1.0-es
+ command: python -m pyserini.search.faiss --encoder-class contriever --encoder facebook/mcontriever-msmarco --topics miracl-v1.0-es-${split} --index miracl-v1.0-es-mcontriever-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.4081
+ R@100: 0.8339
+ - split: dev
+ scores:
+ - nDCG@10: 0.4184
+ R@100: 0.8411
+ - name: mcontriever-tied-pft-msmarco.fa
+ eval_key: miracl-v1.0-fa
+ command: python -m pyserini.search.faiss --encoder-class contriever --encoder facebook/mcontriever-msmarco --topics miracl-v1.0-fa-${split} --index miracl-v1.0-fa-mcontriever-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.2263
+ R@100: 0.6374
+ - split: dev
+ scores:
+ - nDCG@10: 0.2152
+ R@100: 0.6540
+ - name: mcontriever-tied-pft-msmarco.fi
+ eval_key: miracl-v1.0-fi
+ command: python -m pyserini.search.faiss --encoder-class contriever --encoder facebook/mcontriever-msmarco --topics miracl-v1.0-fi-${split} --index miracl-v1.0-fi-mcontriever-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.5680
+ R@100: 0.9369
+ - split: dev
+ scores:
+ - nDCG@10: 0.6019
+ R@100: 0.9527
+ - name: mcontriever-tied-pft-msmarco.fr
+ eval_key: miracl-v1.0-fr
+ command: python -m pyserini.search.faiss --encoder-class contriever --encoder facebook/mcontriever-msmarco --topics miracl-v1.0-fr-${split} --index miracl-v1.0-fr-mcontriever-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.3332
+ R@100: 0.8341
+ - split: dev
+ scores:
+ - nDCG@10: 0.3140
+ R@100: 0.8243
+ - name: mcontriever-tied-pft-msmarco.hi
+ eval_key: miracl-v1.0-hi
+ command: python -m pyserini.search.faiss --encoder-class contriever --encoder facebook/mcontriever-msmarco --topics miracl-v1.0-hi-${split} --index miracl-v1.0-hi-mcontriever-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.2886
+ R@100: 0.6664
+ - split: dev
+ scores:
+ - nDCG@10: 0.2864
+ R@100: 0.6461
+ - name: mcontriever-tied-pft-msmarco.id
+ eval_key: miracl-v1.0-id
+ command: python -m pyserini.search.faiss --encoder-class contriever --encoder facebook/mcontriever-msmarco --topics miracl-v1.0-id-${split} --index miracl-v1.0-id-mcontriever-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.3748
+ R@100: 0.7955
+ - split: dev
+ scores:
+ - nDCG@10: 0.3915
+ R@100: 0.8015
+ - name: mcontriever-tied-pft-msmarco.ja
+ eval_key: miracl-v1.0-ja
+ command: python -m pyserini.search.faiss --encoder-class contriever --encoder facebook/mcontriever-msmarco --topics miracl-v1.0-ja-${split} --index miracl-v1.0-ja-mcontriever-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.4402
+ R@100: 0.8813
+ - split: dev
+ scores:
+ - nDCG@10: 0.4240
+ R@100: 0.8783
+ - name: mcontriever-tied-pft-msmarco.ko
+ eval_key: miracl-v1.0-ko
+ command: python -m pyserini.search.faiss --encoder-class contriever --encoder facebook/mcontriever-msmarco --topics miracl-v1.0-ko-${split} --index miracl-v1.0-ko-mcontriever-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.4799
+ R@100: 0.8672
+ - split: dev
+ scores:
+ - nDCG@10: 0.4829
+ R@100: 0.8753
+ - name: mcontriever-tied-pft-msmarco.ru
+ eval_key: miracl-v1.0-ru
+ command: python -m pyserini.search.faiss --encoder-class contriever --encoder facebook/mcontriever-msmarco --topics miracl-v1.0-ru-${split} --index miracl-v1.0-ru-mcontriever-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.3811
+ R@100: 0.8369
+ - split: dev
+ scores:
+ - nDCG@10: 0.3913
+ R@100: 0.8500
+ - name: mcontriever-tied-pft-msmarco.sw
+ eval_key: miracl-v1.0-sw
+ command: python -m pyserini.search.faiss --encoder-class contriever --encoder facebook/mcontriever-msmarco --topics miracl-v1.0-sw-${split} --index miracl-v1.0-sw-mcontriever-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.5568
+ R@100: 0.9130
+ - split: dev
+ scores:
+ - nDCG@10: 0.5600
+ R@100: 0.9108
+ - name: mcontriever-tied-pft-msmarco.te
+ eval_key: miracl-v1.0-te
+ command: python -m pyserini.search.faiss --encoder-class contriever --encoder facebook/mcontriever-msmarco --topics miracl-v1.0-te-${split} --index miracl-v1.0-te-mcontriever-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.5260
+ R@100: 0.9457
+ - split: dev
+ scores:
+ - nDCG@10: 0.5283
+ R@100: 0.9612
+ - name: mcontriever-tied-pft-msmarco.th
+ eval_key: miracl-v1.0-th
+ command: python -m pyserini.search.faiss --encoder-class contriever --encoder facebook/mcontriever-msmarco --topics miracl-v1.0-th-${split} --index miracl-v1.0-th-mcontriever-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.5299
+ R@100: 0.9361
+ - split: dev
+ scores:
+ - nDCG@10: 0.5173
+ R@100: 0.9361
+ - name: mcontriever-tied-pft-msmarco.zh
+ eval_key: miracl-v1.0-zh
+ command: python -m pyserini.search.faiss --encoder-class contriever --encoder facebook/mcontriever-msmarco --topics miracl-v1.0-zh-${split} --index miracl-v1.0-zh-mcontriever-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: train
+ scores:
+ - nDCG@10: 0.4283
+ R@100: 0.8745
+ - split: dev
+ scores:
+ - nDCG@10: 0.4097
+ R@100: 0.9026
+ - name: mcontriever-tied-pft-msmarco.de
+ eval_key: miracl-v1.0-de
+ command: python -m pyserini.search.faiss --encoder-class contriever --encoder facebook/mcontriever-msmarco --topics miracl-v1.0-de-${split} --index miracl-v1.0-de-mcontriever-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: dev
+ scores:
+ - nDCG@10: 0.4079
+ R@100: 0.8407
+ - name: mcontriever-tied-pft-msmarco.yo
+ eval_key: miracl-v1.0-yo
+ command: python -m pyserini.search.faiss --encoder-class contriever --encoder facebook/mcontriever-msmarco --topics miracl-v1.0-yo-${split} --index miracl-v1.0-yo-mcontriever-pft-msmarco --output $output --batch 128 --threads 16 --hits 1000
+ splits:
+ - split: dev
+ scores:
+ - nDCG@10: 0.4150
+ R@100: 0.7703

pyserini/2cr/miracl_html.template

@@ -0,0 +1,256 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no" />
+ <meta http-equiv="x-ua-compatible" content="ie=edge" />
+ <title>Pyserini Reproductions</title>
+ <!-- Font Awesome -->
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.11.2/css/all.css" />
+ <!-- Google Fonts Roboto -->
+ <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Roboto:wght@300;400;500;700&display=swap" />
+ <!-- MDB -->
+ <link href="https://cdnjs.cloudflare.com/ajax/libs/mdb-ui-kit/4.0.0/mdb.min.css" rel="stylesheet" />
+
+ <style>
+tr.hide-table-padding td {
+ padding: 0;
+}
+
+.expand-button {
+ position: relative;
+}
+
+.accordion-toggle .expand-button:after {
+ position: absolute;
+ left:.75rem;
+ top: 50%;
+ transform: translate(0, -50%);
+ content: '-';
+}
+
+.accordion-toggle.collapsed .expand-button:after {
+ content: '+';
+}
+
+blockquote.mycode {
+ border-left: 3px solid #ccc;
+ margin-left: 25px;
+ margin-top: 15px;
+ padding-left: 15px;
+}
+
+blockquote.mycode2 {
+ border-left: 3px solid #ccc;
+ margin-left: 25px;
+ padding-top: 10px;
+ padding-bottom: 10px;
+ padding-left: 15px;
+}
+
+tr th.headertop {
+ border-bottom: none;
+ padding-bottom: 0rem
+}
+
+tr th.headerbottom {
+ padding-top: 0rem
+}
+
+.table>:not(caption)>*>*{padding:0.75rem 0.75rem}
+
+.copy-code-button {
+ border-radius: 0;
+ min-width: 55px;
+ background: none repeat scroll 0 0 transparent;
+ background-color: grey;
+ color: #F1F2F3 !important;
+ cursor: pointer;
+ border-style: none;
+ font-family: 'HELVETICA',sans-serif;
+ font-size: 0.8em;
+ font-weight: normal;
+ text-align: center;
+ text-decoration: none;
+ text-indent: 0;
+ text-transform: uppercase;
+ font-weight: 500;
+ line-height: 1.42rem;
+ margin: 0;
+ padding: 3px 8px;
+ position: absolute !important;
+ top: 0 !important;
+ right: 0 !important;
+}
+
+.copy-code-button > span {
+ color: #F1F2F3 !important;
+}
+
+.copy-code-button, ::before, ::after {
+ box-sizing: inherit;
+}
+
+.copy-code-button::before {
+ content: '';
+ display: inline-block;
+ width: 16px;
+ height: 16px;
+ margin-right: 3px;
+ background-size: contain;
+ background-image: url("data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiPz4KPHN2ZyB3aWR0aD0iMTVweCIgaGVpZ2h0PSIxNXB4IiB2aWV3Qm94PSIwIDAgMTUgMTUiIHZlcnNpb249IjEuMSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB4bWxuczp4bGluaz0iaHR0cDovL3d3dy53My5vcmcvMTk5OS94bGluayI+CiAgICA8IS0tIEdlbmVyYXRvcjogU2tldGNoIDUwLjIgKDU1MDQ3KSAtIGh0dHA6Ly93d3cuYm9oZW1pYW5jb2RpbmcuY29tL3NrZXRjaCAtLT4KICAgIDx0aXRsZT5QYWdlIDE8L3RpdGxlPgogICAgPGRlc2M+Q3JlYXRlZCB3aXRoIFNrZXRjaC48L2Rlc2M+CiAgICA8ZGVmcz48L2RlZnM+CiAgICA8ZyBpZD0iRmxvdyIgc3Ryb2tlPSJub25lIiBzdHJva2Utd2lkdGg9IjEiIGZpbGw9Im5vbmUiIGZpbGwtcnVsZT0iZXZlbm9kZCI+CiAgICAgICAgPGcgaWQ9IkJ0dG5faHRtbCIgdHJhbnNmb3JtPSJ0cmFuc2xhdGUoLTgxOS4wMDAwMDAsIC03NTMuMDAwMDAwKSIgZmlsbD0iI0ZGRkZGRiI+CiAgICAgICAgICAgIDxnIGlkPSJHcm91cC0xIiB0cmFuc2Zvcm09InRyYW5zbGF0ZSgzMTEuMDAwMDAwLCA0MDUuMDAwMDAwKSI+CiAgICAgICAgICAgICAgICA8ZyBpZD0iR3JvdXAtMiIgdHJhbnNmb3JtPSJ0cmFuc2xhdGUoNTA4LjAwMDAwMCwgMzQyLjAwMDAwMCkiPgogICAgICAgICAgICAgICAgICAgIDxwYXRoIGQ9Ik0xMy45NzcyNzI3LDYgTDMuNDA5MDkwOTEsNiBDMi44NDQ1NDU0NSw2IDIuMzg2MzYzNjQsNi40NTgxODE4MiAyLjM4NjM2MzY0LDcuMDIyNzI3MjcgTDIuMzg2MzYzNjQsMTcuNTkwOTA5MSBDMi4zODYzNjM2NCwxOC4xNTU0NTQ1IDIuODQ0NTQ1NDUsMTguNjEzNjM2NCAzLjQwOTA5MDkxLDE4LjYxMzYzNjQgTDEzLjk3NzI3MjcsMTguNjEzNjM2NCBDMTQuNTQxODE4MiwxOC42MTM2MzY0IDE1LDE4LjE1NTQ1NDUgMTUsMTcuNTkwOTA5MSBMMTUsNy4wMjI3MjcyNyBDMTUsNi40NTgxODE4MiAxNC41NDE4MTgyLDYgMTMuOTc3MjcyNyw2IFogTTE0LjMxODE4MTgsMTcuNTkwOTA5MSBDMTQuMzE4MTgxOCwxNy43NzkwOTA5IDE0LjE2NTQ1NDUsMTcuOTMxODE4MiAxMy45NzcyNzI3LDE3LjkzMTgxODIgTDMuNDA5MDkwOTEsMTcuOTMxODE4MiBDMy4yMjA5MDkwOSwxNy45MzE4MTgyIDMuMDY4MTgxODIsMTcuNzc5MDkwOSAzLjA2ODE4MTgyLDE3LjU5MDkwOTEgTDMuMDY4MTgxODIsNy4wMjI3MjcyNyBDMy4wNjgxODE4Miw2LjgzNDU0NTQ1IDMuMjIwOTA5MDksNi42ODE4MTgxOCAzLjQwOTA5MDkxLDYuNjgxODE4MTggTDEzLjk3NzI3MjcsNi42ODE4MTgxOCBDMTQuMTY1NDU0NSw2LjY4MTgxODE4IDE0LjMxODE4MTgsNi44MzQ1NDU0NSAxNC4zMTgxODE4LDcuMDIyNzI3MjcgTDE0LjMxODE4MTgsMTcuNTkwOTA5MSBaIE0xMS45MzE4MTgyLDE5Ljk3NzI3MjcgQzExLjkzMTgxODIsMjAuMTY1NDU0NSAxMS43NzkwOTA5LDIwLjMxODE4MTggMTEuNTkwOTA5MSwyMC4zMTgxODE4IEwxLjAyMjcyNzI3LDIwLjMxODE4MTggQzAuODM0NTQ1NDU1LDIwLjMxODE4MTggMC42ODE4MTgxODIsMjAuMTY1NDU0NSAwLjY4MTgxODE4MiwxOS45NzcyNzI3IEwwLjY4MTgxODE4Miw5LjQwOTA5MDkxIEMwLjY4MTgxODE4Miw5LjIyMDkwOTA5IDAuODM0NTQ1NDU1LDkuMDY4MTgxODIgMS4wMjI3MjcyNyw5LjA2ODE4MTgyIEwxLjM2MzYzNjM2LDkuMDY4MTgxODIgTDEuMzYzNjM2MzYsOC4zODYzNjM2NCBMMS4wMjI3MjcyNyw4LjM4NjM2MzY0IEMwLjQ1ODE4MTgxOCw4LjM4NjM2MzY0IDAsOC44NDQ1NDU0NSAwLDkuNDA5MDkwOTEgTDAsMTkuOTc3MjcyNyBDMCwyMC41NDE4MTgyIDAuNDU4MTgxODE4LDIxIDEuMDIyNzI3MjcsMjEgTDExLjU5MDkwOTEsMjEgQzEyLjE1NTQ1NDUsMjEgMTIuNjEzNjM2NCwyMC41NDE4MTgyIDEyLjYxMzYzNjQsMTkuOTc3MjcyNyBMMTIuNjEzNjM2NCwxOS42MzYzNjM2IEwxMS45MzE4MTgyLDE5LjYzNjM2MzYgTDExLjkzMTgxODIsMTkuOTc3MjcyNyBaIiBpZD0iUGFnZS0xIj48L3BhdGg+CiAgICAgICAgICAgICAgICA8L2c+CiAgICAgICAgICAgIDwvZz4KICAgICAgICA8L2c+CiAgICA8L2c+Cjwvc3ZnPg==");
+ background-repeat: no-repeat;
+ position: relative;
+ top: 3px;
+}
+
+.copy-code-button:focus {
+ /* Avoid an ugly focus outline on click in Chrome,
+ but darken the button for accessibility.
+ See https://stackoverflow.com/a/25298082/1481479 */
+ /* background-color: #E6E6E6; */
+ outline: 0;
+}
+
+pre[class*="prettyprint"] {
+ position: relative;
+ overflow: hidden;
+}
+ </style>
+</head>
+<body>
+
+ <!-- Background image -->
+ <div id="intro" class="bg-image vh-100 shadow-1-strong" style="max-height: 150px">
+ <div class="mask" style="
+ background: linear-gradient(
+ 45deg,
+ rgba(29, 236, 197, 0.7),
+ rgba(91, 14, 214, 0.7) 100%
+ );
+ ">
+ <div class="container d-flex align-items-center justify-content-center text-center h-100" style="max-height: 150px">
+ <div class="text-white">
+ <h1 class="mb-3">$title</h1>
+ </div>
+ </div>
+ </div>
+ </div>
+ <!-- Background image -->
+
+ <div class="container my-4">
+
+ $tables
+
+ </ul>
+
+<div style="padding-top: 20px"/>
+
+<h4>Programmatic Execution</h4>
+
+<p>All experimental runs shown in the above table can be programmatically executed based on the instructions below.
+To list all the experimental conditions:</p>
+
+<blockquote class="mycode2"><tt>
+python -m pyserini.2cr.miracl --list-conditions
+</tt></blockquote>
+
+<p>Run all languages for a specific condition and show commands:</p>
+
+<blockquote class="mycode2"><tt>
+python -m pyserini.2cr.miracl --condition bm25 --display-commands
+</tt></blockquote>
+
+<p>Run a particular language for a specific condition and show commands:</p>
+
+<blockquote class="mycode2"><tt>
+python -m pyserini.2cr.miracl --condition bm25 --language ko --display-commands
+</tt></blockquote>
+
+<p>Run all languages for all conditions and show commands:</p>
+
+<blockquote class="mycode2"><tt>
+python -m pyserini.2cr.miracl --all --display-commands
+</tt></blockquote>
+
+<p>With the above commands, run files will be placed in the current directory. Use the option <tt>--directory runs</tt> to place the runs in a sub-directory.</p>
+
+<p>For a specific condition, just show the commands and do not run:</p>
+
+<blockquote class="mycode2"><tt>
+python -m pyserini.2cr.miracl --condition bm25 --display-commands --dry-run
+</tt></blockquote>
+
+<p>This will generate exactly the commands for a specific condition above (corresponding to a row in the table).</p>
+
+<p>For a specific condition and language, just show the commands and do not run:</p>
+
+<blockquote class="mycode2"><tt>
+python -m pyserini.2cr.miracl --condition bm25 --language ko --display-commands --dry-run
+</tt></blockquote>
+
+<p>For all conditions, just show the commands and do not run and skip evaluation:</p>
+
+<blockquote class="mycode2"><tt>
+python -m pyserini.2cr.miracl --all --display-commands --dry-run --skip-eval
+</tt></blockquote>
+
+<p>Finally, to generate this page:</p>
+
+<blockquote class="mycode2"><tt>
+python -m pyserini.2cr.miracl --generate-report --output docs/2cr/miracl.html
+</tt></blockquote>
+
+<p>The output file <tt>miracl.html</tt> should be identical to this page.</p>
+
+<div style="padding-top: 50px"/>
+
+ </div>
+
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.0/jquery.min.js"></script>
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/4.3.1/js/bootstrap.min.js"></script>
+ <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mdb-ui-kit/4.0.0/mdb.min.js"></script>
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.10/clipboard.min.js"></script>
+
+<script>
+document.querySelectorAll('pre').forEach(function (codeBlock) {
+ var button = document.createElement('button');
+ button.className = 'copy-code-button';
+ button.type = 'button';
+ var s = codeBlock.innerText;
+ button.setAttribute('data-clipboard-text',s);
+ button.innerText = 'Copy';
+
+ // var pre = codeBlock.parentNode;
+ codeBlock.classList.add('prettyprint');
+ // pre.parentNode.insertBefore(button, pre);
+ codeBlock.appendChild(button);
+});
+
+var clipboard = new ClipboardJS('.copy-code-button');
+
+clipboard.on('success', function(e) {
+ console.info('Action:', e.action);
+ console.info('Text:', e.text);
+ console.info('Trigger:', e.trigger);
+ e.trigger.textContent = 'Copied';
+ window.setTimeout(function() {
+ e.trigger.textContent = 'Copy';
+ }, 2000);
+ e.clearSelection();
+});
+
+clipboard.on('error', function(e) {
+ console.error('Action:', e.action);
+ console.error('Trigger:', e.trigger);
+ e.trigger.textContent = 'Error Copying';
+ window.setTimeout(function() {
+ e.trigger.textContent = 'Copy';
+ }, 2000);
+ e.clearSelection();
+});
+
+</script>
+
+</body>
+</html>

pyserini/2cr/miracl_html_table.template

@@ -0,0 +1,35 @@
+<div class="table-responsive">
+ <table class="table">
+ <thead>
+ <tr>
+ <th scope="col"></th>
+ <th scope="col">$desc</th>
+ <th scope="col">ar</th>
+ <th scope="col">bn</th>
+ <th scope="col">en</th>
+ <th scope="col">es</th>
+ <th scope="col">fa</th>
+ <th scope="col">fi</th>
+ <th scope="col">fr</th>
+ <th scope="col">hi</th>
+ <th scope="col">id</th>
+ <th scope="col">ja</th>
+ <th scope="col">ko</th>
+ <th scope="col">ru</th>
+ <th scope="col">sw</th>
+ <th scope="col">te</th>
+ <th scope="col">th</th>
+ <th scope="col">zh</th>
+ <th scope="col">de</th>
+ <th scope="col">yo</th>
+ <th scope="col"></th>
+ <th scope="col">avg</th>
+ </tr>
+ </thead>
+ <tbody>
+
+$rows
+
+ </tbody>
+ </table>
+</div>

pyserini/2cr/miracl_html_table_row.template

@@ -0,0 +1,336 @@
+<!-- Condition: $model -->
+<tr class="accordion-toggle collapsed" id="table${table_cnt}-row${row_cnt}" data-toggle="collapse" data-parent="#table${table_cnt}-row${row_cnt}" href="#table${table_cnt}-collapse${row_cnt}">
+<td class="expand-button"></td>
+<td>$model</td>
+<td>$ar</td>
+<td>$bn</td>
+<td>$en</td>
+<td>$es</td>
+<td>$fa</td>
+<td>$fi</td>
+<td>$fr</td>
+<td>$hi</td>
+<td>$id</td>
+<td>$ja</td>
+<td>$ko</td>
+<td>$ru</td>
+<td>$sw</td>
+<td>$te</td>
+<td>$th</td>
+<td>$zh</td>
+<td>$de</td>
+<td>$yo</td>
+<td></td>
+<td>$avg</td>
+</tr>
+<tr class="hide-table-padding">
+<td colspan="22">
+<div id="table${table_cnt}-collapse${row_cnt}" class="collapse in p-3">
+
+<!-- Tabs navs -->
+<ul class="nav nav-tabs mb-3" id="table${table_cnt}-row${row_cnt}-tabs" role="tablist">
+ <li class="nav-item" role="presentation">
+ <a class="nav-link active" id="table${table_cnt}-row${row_cnt}-tab1-header" data-mdb-toggle="tab" href="#table${table_cnt}-row${row_cnt}-tab1" role="tab" aria-controls="table${table_cnt}-row${row_cnt}-tab1" aria-selected="true" style="text-transform:none">ar</a>
+ </li>
+ <li class="nav-item" role="presentation">
+ <a class="nav-link" id="table${table_cnt}-row${row_cnt}-tab2-header" data-mdb-toggle="tab" href="#table${table_cnt}-row${row_cnt}-tab2" role="tab" aria-controls="table${table_cnt}-row${row_cnt}-tab2" aria-selected="false" style="text-transform:none">bn</a>
+ </li>
+ <li class="nav-item" role="presentation">
+ <a class="nav-link" id="table${table_cnt}-row${row_cnt}-tab3-header" data-mdb-toggle="tab" href="#table${table_cnt}-row${row_cnt}-tab3" role="tab" aria-controls="table${table_cnt}-row${row_cnt}-tab3" aria-selected="false" style="text-transform:none">en</a>
+ </li>
+ <li class="nav-item" role="presentation">
+ <a class="nav-link" id="table${table_cnt}-row${row_cnt}-tab4-header" data-mdb-toggle="tab" href="#table${table_cnt}-row${row_cnt}-tab4" role="tab" aria-controls="table${table_cnt}-row${row_cnt}-tab4" aria-selected="false" style="text-transform:none">es</a>
+ </li> 
+ <li class="nav-item" role="presentation">
+ <a class="nav-link" id="table${table_cnt}-row${row_cnt}-tab5-header" data-mdb-toggle="tab" href="#table${table_cnt}-row${row_cnt}-tab5" role="tab" aria-controls="table${table_cnt}-row${row_cnt}-tab5" aria-selected="false" style="text-transform:none">fa</a>
+ </li>
+ <li class="nav-item" role="presentation">
+ <a class="nav-link" id="table${table_cnt}-row${row_cnt}-tab6-header" data-mdb-toggle="tab" href="#table${table_cnt}-row${row_cnt}-tab6" role="tab" aria-controls="table${table_cnt}-row${row_cnt}-tab6" aria-selected="false" style="text-transform:none">fi</a>
+ </li> 
+ <li class="nav-item" role="presentation">
+ <a class="nav-link" id="table${table_cnt}-row${row_cnt}-tab7-header" data-mdb-toggle="tab" href="#table${table_cnt}-row${row_cnt}-tab7" role="tab" aria-controls="table${table_cnt}-row${row_cnt}-tab7" aria-selected="false" style="text-transform:none">fr</a>
+ </li>
+ <li class="nav-item" role="presentation">
+ <a class="nav-link" id="table${table_cnt}-row${row_cnt}-tab8-header" data-mdb-toggle="tab" href="#table${table_cnt}-row${row_cnt}-tab8" role="tab" aria-controls="table${table_cnt}-row${row_cnt}-tab8" aria-selected="false" style="text-transform:none">hi</a>
+ </li> 
+ <li class="nav-item" role="presentation">
+ <a class="nav-link" id="table${table_cnt}-row${row_cnt}-tab9-header" data-mdb-toggle="tab" href="#table${table_cnt}-row${row_cnt}-tab9" role="tab" aria-controls="table${table_cnt}-row${row_cnt}-tab9" aria-selected="false" style="text-transform:none">id</a>
+ </li>
+ <li class="nav-item" role="presentation">
+ <a class="nav-link" id="table${table_cnt}-row${row_cnt}-tab10-header" data-mdb-toggle="tab" href="#table${table_cnt}-row${row_cnt}-tab10" role="tab" aria-controls="table${table_cnt}-row${row_cnt}-tab10" aria-selected="false" style="text-transform:none">ja</a>
+ </li>
+ <li class="nav-item" role="presentation">
+ <a class="nav-link" id="table${table_cnt}-row${row_cnt}-tab11-header" data-mdb-toggle="tab" href="#table${table_cnt}-row${row_cnt}-tab11" role="tab" aria-controls="table${table_cnt}-row${row_cnt}-tab11" aria-selected="false" style="text-transform:none">ko</a>
+ </li>
+ <li class="nav-item" role="presentation">
+ <a class="nav-link" id="table${table_cnt}-row${row_cnt}-tab12-header" data-mdb-toggle="tab" href="#table${table_cnt}-row${row_cnt}-tab12" role="tab" aria-controls="table${table_cnt}-row${row_cnt}-tab12" aria-selected="false" style="text-transform:none">ru</a>
+ </li>
+ <li class="nav-item" role="presentation">
+ <a class="nav-link" id="table${table_cnt}-row${row_cnt}-tab13-header" data-mdb-toggle="tab" href="#table${table_cnt}-row${row_cnt}-tab13" role="tab" aria-controls="table${table_cnt}-row${row_cnt}-tab13" aria-selected="false" style="text-transform:none">sw</a>
+ </li>
+ <li class="nav-item" role="presentation">
+ <a class="nav-link" id="table${table_cnt}-row${row_cnt}-tab14-header" data-mdb-toggle="tab" href="#table${table_cnt}-row${row_cnt}-tab14" role="tab" aria-controls="table${table_cnt}-row${row_cnt}-tab14" aria-selected="false" style="text-transform:none">te</a>
+ </li>
+ <li class="nav-item" role="presentation">
+ <a class="nav-link" id="table${table_cnt}-row${row_cnt}-tab15-header" data-mdb-toggle="tab" href="#table${table_cnt}-row${row_cnt}-tab15" role="tab" aria-controls="table${table_cnt}-row${row_cnt}-tab15" aria-selected="false" style="text-transform:none">th</a>
+ </li>
+ <li class="nav-item" role="presentation">
+ <a class="nav-link" id="table${table_cnt}-row${row_cnt}-tab16-header" data-mdb-toggle="tab" href="#table${table_cnt}-row${row_cnt}-tab16" role="tab" aria-controls="table${table_cnt}-row${row_cnt}-tab16" aria-selected="false" style="text-transform:none">zh</a>
+ </li> 
+ <li class="nav-item" role="presentation">
+ <a class="nav-link" id="table${table_cnt}-row${row_cnt}-tab17-header" data-mdb-toggle="tab" href="#table${table_cnt}-row${row_cnt}-tab17" role="tab" aria-controls="table${table_cnt}-row${row_cnt}-tab17" aria-selected="false" style="text-transform:none">de</a>
+ </li> 
+ <li class="nav-item" role="presentation">
+ <a class="nav-link" id="table${table_cnt}-row${row_cnt}-tab18-header" data-mdb-toggle="tab" href="#table${table_cnt}-row${row_cnt}-tab18" role="tab" aria-controls="table${table_cnt}-row${row_cnt}-tab18" aria-selected="false" style="text-transform:none">yo</a>
+ </li> 
+ 
+</ul>
+<!-- Tabs navs -->
+
+<!-- Tabs content -->
+<div class="tab-content" id="table${table_cnt}-row${row_cnt}-content">
+ <div class="tab-pane fade show active" id="table${table_cnt}-row${row_cnt}-tab1" role="tabpanel" aria-labelledby="table${table_cnt}-row${row_cnt}-tab1">
+Command to generate run:
+
+ <blockquote class="mycode">
+<pre><code>$cmd1
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd1}</code></pre>
+ </blockquote>
+
+ </div>
+ <div class="tab-pane fade" id="table${table_cnt}-row${row_cnt}-tab2" role="tabpanel" aria-labelledby="table${table_cnt}-row${row_cnt}-tab2">
+Command to generate run:
+
+ <blockquote class="mycode">
+<pre><code>$cmd2
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd2}</code></pre>
+ </blockquote>
+
+ </div>
+ <div class="tab-pane fade" id="table${table_cnt}-row${row_cnt}-tab3" role="tabpanel" aria-labelledby="table${table_cnt}-row${row_cnt}-tab3">
+Command to generate run:
+
+ <blockquote class="mycode">
+<pre><code>$cmd3
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd3}</code></pre>
+ </blockquote>
+
+ </div>
+ <div class="tab-pane fade" id="table${table_cnt}-row${row_cnt}-tab4" role="tabpanel" aria-labelledby="table${table_cnt}-row${row_cnt}-tab4">
+Command to generate run:
+
+ <blockquote class="mycode">
+<pre><code>$cmd4
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd4}</code></pre>
+ </blockquote>
+
+ </div>
+ <div class="tab-pane fade" id="table${table_cnt}-row${row_cnt}-tab5" role="tabpanel" aria-labelledby="table${table_cnt}-row${row_cnt}-tab5">
+Command to generate run:
+
+ <blockquote class="mycode">
+<pre><code>$cmd5
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd5}</code></pre>
+ </blockquote>
+
+ </div>
+ <div class="tab-pane fade" id="table${table_cnt}-row${row_cnt}-tab6" role="tabpanel" aria-labelledby="table${table_cnt}-row${row_cnt}-tab6">
+Command to generate run:
+
+ <blockquote class="mycode">
+<pre><code>$cmd6
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd6}</code></pre>
+ </blockquote>
+
+ </div>
+ <div class="tab-pane fade" id="table${table_cnt}-row${row_cnt}-tab7" role="tabpanel" aria-labelledby="table${table_cnt}-row${row_cnt}-tab7">
+Command to generate run:
+
+ <blockquote class="mycode">
+<pre><code>$cmd7
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd7}</code></pre>
+ </blockquote>
+
+ </div>
+ <div class="tab-pane fade" id="table${table_cnt}-row${row_cnt}-tab8" role="tabpanel" aria-labelledby="table${table_cnt}-row${row_cnt}-tab8">
+Command to generate run:
+
+ <blockquote class="mycode">
+<pre><code>$cmd8
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd8}</code></pre>
+ </blockquote>
+
+ </div>
+ <div class="tab-pane fade" id="table${table_cnt}-row${row_cnt}-tab9" role="tabpanel" aria-labelledby="table${table_cnt}-row${row_cnt}-tab9">
+Command to generate run:
+
+ <blockquote class="mycode">
+<pre><code>$cmd9
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd9}</code></pre>
+ </blockquote>
+
+ </div>
+ <div class="tab-pane fade" id="table${table_cnt}-row${row_cnt}-tab10" role="tabpanel" aria-labelledby="table${table_cnt}-row${row_cnt}-tab10">
+Command to generate run:
+
+ <blockquote class="mycode">
+<pre><code>$cmd10
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd10}</code></pre>
+ </blockquote>
+
+ </div>
+ <div class="tab-pane fade" id="table${table_cnt}-row${row_cnt}-tab11" role="tabpanel" aria-labelledby="table${table_cnt}-row${row_cnt}-tab11">
+Command to generate run:
+
+ <blockquote class="mycode">
+<pre><code>$cmd11
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd11}</code></pre>
+ </blockquote>
+
+ </div>
+ <div class="tab-pane fade" id="table${table_cnt}-row${row_cnt}-tab12" role="tabpanel" aria-labelledby="table${table_cnt}-row${row_cnt}-tab12">
+Command to generate run:
+
+ <blockquote class="mycode">
+<pre><code>$cmd12
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd12}</code></pre>
+ </blockquote>
+
+ </div>
+ <div class="tab-pane fade" id="table${table_cnt}-row${row_cnt}-tab13" role="tabpanel" aria-labelledby="table${table_cnt}-row${row_cnt}-tab13">
+Command to generate run:
+
+ <blockquote class="mycode">
+<pre><code>$cmd13
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd13}</code></pre>
+ </blockquote>
+
+ </div>
+ <div class="tab-pane fade" id="table${table_cnt}-row${row_cnt}-tab14" role="tabpanel" aria-labelledby="table${table_cnt}-row${row_cnt}-tab14">
+
+Command to generate run:
+
+ <blockquote class="mycode">
+<pre><code>$cmd14
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd14}</code></pre>
+ </blockquote>
+
+ </div>
+ <div class="tab-pane fade" id="table${table_cnt}-row${row_cnt}-tab15" role="tabpanel" aria-labelledby="table${table_cnt}-row${row_cnt}-tab15">
+
+Command to generate run:
+
+ <blockquote class="mycode">
+<pre><code>$cmd15
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd15}</code></pre>
+ </blockquote>
+
+ </div>
+
+ <div class="tab-pane fade" id="table${table_cnt}-row${row_cnt}-tab16" role="tabpanel" aria-labelledby="table${table_cnt}-row${row_cnt}-tab16">
+Command to generate run:
+
+ <blockquote class="mycode">
+<pre><code>$cmd16
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd16}</code></pre>
+ </blockquote>
+
+ </div>
+
+ <div class="tab-pane fade" id="table${table_cnt}-row${row_cnt}-tab17" role="tabpanel" aria-labelledby="table${table_cnt}-row${row_cnt}-tab17">
+Command to generate run:
+
+ <blockquote class="mycode">
+<pre><code>$cmd17
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd17}</code></pre>
+ </blockquote>
+
+ </div>
+
+ <div class="tab-pane fade" id="table${table_cnt}-row${row_cnt}-tab18" role="tabpanel" aria-labelledby="table${table_cnt}-row${row_cnt}-tab18">
+Command to generate run:
+
+ <blockquote class="mycode">
+<pre><code>$cmd18
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd18}</code></pre>
+ </blockquote>
+
+ </div>
+ 
+</div>
+<!-- Tabs content -->
+
+</div></td>
+</tr>

pyserini/2cr/mrtydi.py

@@ -0,0 +1,330 @@
+#
+# Pyserini: Reproducible IR research with sparse and dense representations
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from collections import defaultdict
+from string import Template
+
+import argparse
+import math
+import os
+import pkg_resources
+import sys
+import time
+import yaml
+
+from ._base import run_eval_and_return_metric, ok_str, okish_str, fail_str
+
+languages = [
+ ['ar', 'arabic'],
+ ['bn', 'bengali'],
+ ['en', 'english'],
+ ['fi', 'finnish'],
+ ['id', 'indonesian'],
+ ['ja', 'japanese'],
+ ['ko', 'korean'],
+ ['ru', 'russian'],
+ ['sw', 'swahili'],
+ ['te', 'telugu'],
+ ['th', 'thai']
+]
+
+models = ['bm25', 'mdpr-split-pft-nq', 'mdpr-tied-pft-nq', 'mdpr-tied-pft-msmarco', 'mdpr-tied-pft-msmarco-ft-all']
+
+html_display = {
+ 'bm25': 'BM25',
+ 'mdpr-split-pft-nq': 'mDPR (split encoders), pre-FT w/ NQ',
+ 'mdpr-tied-pft-nq': 'mDPR (tied encoders), pre-FT w/ NQ',
+ 'mdpr-tied-pft-msmarco': 'mDPR (tied encoders), pre-FT w/ MS MARCO',
+ 'mdpr-tied-pft-msmarco-ft-all': 'mDPR (tied encoders), pre-FT w/ MS MARCO, FT w/ all'
+}
+
+trec_eval_metric_definitions = {
+ 'MRR@100': '-c -M 100 -m recip_rank',
+ 'R@100': '-c -m recall.100',
+}
+
+
+def format_run_command(raw):
+ return raw.replace('--lang', '\\\n --lang')\
+ .replace('--encoder', '\\\n --encoder')\
+ .replace('--topics', '\\\n --topics')\
+ .replace('--index', '\\\n --index')\
+ .replace('--output ', '\\\n --output ')\
+ .replace('--batch ', '\\\n --batch ') \
+ .replace('--threads 12', '--threads 12 \\\n ')
+
+
+def format_eval_command(raw):
+ return raw.replace('-c ', '\\\n -c ')\
+ .replace(raw.split()[-1], f'\\\n {raw.split()[-1]}')
+
+
+def read_file(f):
+ fin = open(f, 'r')
+ text = fin.read()
+ fin.close()
+
+ return text
+
+
+def list_conditions():
+ print('Conditions:\n-----------')
+ for condition in models:
+ print(condition)
+ print('\nLanguages\n---------')
+ for language in languages:
+ print(language[0])
+
+
+def print_results(table, metric, split):
+ print(f'Metric = {metric}, Split = {split}')
+ print(' ' * 32, end='')
+ for lang in languages:
+ print(f'{lang[0]:3} ', end='')
+ print('')
+ for model in models:
+ print(f'{model:30}', end='')
+ for lang in languages:
+ key = f'{model}.{lang[0]}'
+ print(f'{table[key][split][metric]:7.3f}', end='')
+ print('')
+ print('')
+
+
+def generate_table_rows(table, row_template, commands, eval_commands, table_id, split, metric):
+ row_cnt = 1
+ html_rows = []
+
+ for model in models:
+ s = Template(row_template)
+
+ keys = {}
+ for lang in languages:
+ keys[lang[0]] = f'{model}.{lang[0]}'
+
+ sum = table[keys["ar"]][split][metric] + \
+ table[keys["bn"]][split][metric] + \
+ table[keys["en"]][split][metric] + \
+ table[keys["fi"]][split][metric] + \
+ table[keys["id"]][split][metric] + \
+ table[keys["ja"]][split][metric] + \
+ table[keys["ko"]][split][metric] + \
+ table[keys["ru"]][split][metric] + \
+ table[keys["sw"]][split][metric] + \
+ table[keys["te"]][split][metric] + \
+ table[keys["th"]][split][metric]
+ avg = sum / 11
+
+ s = s.substitute(table_cnt=table_id,
+ row_cnt=row_cnt,
+ model=html_display[model],
+ ar=f'{table[keys["ar"]][split][metric]:.3f}',
+ bn=f'{table[keys["bn"]][split][metric]:.3f}',
+ en=f'{table[keys["en"]][split][metric]:.3f}',
+ fi=f'{table[keys["fi"]][split][metric]:.3f}',
+ id=f'{table[keys["id"]][split][metric]:.3f}',
+ ja=f'{table[keys["ja"]][split][metric]:.3f}',
+ ko=f'{table[keys["ko"]][split][metric]:.3f}',
+ ru=f'{table[keys["ru"]][split][metric]:.3f}',
+ sw=f'{table[keys["sw"]][split][metric]:.3f}',
+ te=f'{table[keys["te"]][split][metric]:.3f}',
+ th=f'{table[keys["th"]][split][metric]:.3f}',
+ avg=f'{avg:.3f}',
+ cmd1=f'{commands[keys["ar"]]}',
+ cmd2=f'{commands[keys["bn"]]}',
+ cmd3=f'{commands[keys["en"]]}',
+ cmd4=f'{commands[keys["fi"]]}',
+ cmd5=f'{commands[keys["id"]]}',
+ cmd6=f'{commands[keys["ja"]]}',
+ cmd7=f'{commands[keys["ko"]]}',
+ cmd8=f'{commands[keys["ru"]]}',
+ cmd9=f'{commands[keys["sw"]]}',
+ cmd10=f'{commands[keys["te"]]}',
+ cmd11=f'{commands[keys["th"]]}',
+ eval_cmd1=f'{eval_commands[keys["ar"]][metric]}',
+ eval_cmd2=f'{eval_commands[keys["bn"]][metric]}',
+ eval_cmd3=f'{eval_commands[keys["en"]][metric]}',
+ eval_cmd4=f'{eval_commands[keys["fi"]][metric]}',
+ eval_cmd5=f'{eval_commands[keys["id"]][metric]}',
+ eval_cmd6=f'{eval_commands[keys["ja"]][metric]}',
+ eval_cmd7=f'{eval_commands[keys["ko"]][metric]}',
+ eval_cmd8=f'{eval_commands[keys["ru"]][metric]}',
+ eval_cmd9=f'{eval_commands[keys["sw"]][metric]}',
+ eval_cmd10=f'{eval_commands[keys["te"]][metric]}',
+ eval_cmd11=f'{eval_commands[keys["th"]][metric]}'
+ )
+
+ html_rows.append(s)
+ row_cnt += 1
+
+ return html_rows
+
+
+def generate_report(args):
+ table = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: 0.0)))
+ commands = defaultdict(lambda: '')
+ eval_commands = defaultdict(lambda: defaultdict(lambda: ''))
+
+ html_template = read_file(pkg_resources.resource_filename(__name__, 'mrtydi_html.template'))
+ table_template = read_file(pkg_resources.resource_filename(__name__, 'mrtydi_html_table.template'))
+ row_template = read_file(pkg_resources.resource_filename(__name__, 'mrtydi_html_table_row.template'))
+
+ with open(pkg_resources.resource_filename(__name__, 'mrtydi.yaml')) as f:
+ yaml_data = yaml.safe_load(f)
+ for condition in yaml_data['conditions']:
+ name = condition['name']
+ eval_key = condition['eval_key']
+ cmd_template = condition['command']
+
+ for splits in condition['splits']:
+ split = splits['split']
+
+ runfile = os.path.join(args.directory, f'run.mrtydi.{name}.{split}.txt')
+ cmd = Template(cmd_template).substitute(split=split, output=runfile)
+ commands[name] = format_run_command(cmd)
+
+ for expected in splits['scores']:
+ for metric in expected:
+ table[name][split][metric] = expected[metric]
+
+ eval_cmd = f'python -m pyserini.eval.trec_eval ' + \
+ f'{trec_eval_metric_definitions[metric]} {eval_key}-{split} {runfile}'
+ eval_commands[name][metric] = format_eval_command(eval_cmd)
+
+ tables_html = []
+
+ # Build the table for MRR@100, test queries
+ html_rows = generate_table_rows(table, row_template, commands, eval_commands, 1, 'test', 'MRR@100')
+ all_rows = '\n'.join(html_rows)
+ tables_html.append(Template(table_template).substitute(desc='MRR@100, test queries', rows=all_rows))
+
+ # Build the table for R@100, test queries
+ html_rows = generate_table_rows(table, row_template, commands, eval_commands, 2, 'test', 'R@100')
+ all_rows = '\n'.join(html_rows)
+ tables_html.append(Template(table_template).substitute(desc='Recall@100, test queries', rows=all_rows))
+
+ with open(args.output, 'w') as out:
+ out.write(Template(html_template).substitute(title='Mr.TyDi', tables=' '.join(tables_html)))
+
+
+def run_conditions(args):
+ start = time.time()
+
+ table = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: 0.0)))
+
+ with open(pkg_resources.resource_filename(__name__, 'mrtydi.yaml')) as f:
+ yaml_data = yaml.safe_load(f)
+ for condition in yaml_data['conditions']:
+ name = condition['name']
+ encoder = name.split('.')[0]
+ lang = name.split('.')[-1]
+ if args.all:
+ pass
+ elif args.condition != encoder:
+ continue
+ elif args.language and args.language != lang:
+ continue
+ eval_key = condition['eval_key']
+ cmd_template = condition['command']
+
+ print(f'condition {name}:')
+
+ for splits in condition['splits']:
+ split = splits['split']
+
+ print(f' - split: {split}')
+
+ runfile = os.path.join(args.directory, f'run.mrtydi.{name}.{split}.txt')
+ cmd = Template(cmd_template).substitute(split=split, output=runfile)
+
+ if args.display_commands:
+ print(f'\n```bash\n{format_run_command(cmd)}\n```\n')
+
+ if not os.path.exists(runfile):
+ if not args.dry_run:
+ os.system(cmd)
+
+ for expected in splits['scores']:
+ for metric in expected:
+ if not args.skip_eval:
+ score = float(run_eval_and_return_metric(metric, f'{eval_key}-{split}',
+ trec_eval_metric_definitions[metric], runfile))
+ if math.isclose(score, float(expected[metric])):
+ result_str = ok_str
+ # Flaky test: small difference on orca
+ elif name == 'mdpr-tied-pft-nq.te' and split == 'dev' \
+ and math.isclose(score, float(expected[metric]), abs_tol=2e-4):
+ result_str = okish_str
+ # Flaky test: small difference on orca
+ elif name == 'mdpr-tied-pft-msmarco-ft-all.ko' and split == 'train' \
+ and math.isclose(score, float(expected[metric]), abs_tol=4e-4):
+ result_str = okish_str
+ # Flaky test: small difference on Mac Studio (M1)
+ elif name == 'mdpr-tied-pft-msmarco.th' and split == 'train' \
+ and math.isclose(score, float(expected[metric]), abs_tol=3e-4):
+ result_str = okish_str
+ else:
+ result_str = fail_str + f' expected {expected[metric]:.4f}'
+ print(f' {metric:7}: {score:.4f} {result_str}')
+ table[name][split][metric] = score
+ else:
+ table[name][split][metric] = expected[metric]
+
+ print('')
+
+ for metric in ['MRR@100', 'R@100']:
+ for split in ['test', 'dev', 'train']:
+ print_results(table, metric, split)
+
+ end = time.time()
+ print(f'Total elapsed time: {end - start:.0f}s')
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser(description='Generate regression matrix for MIRACL.')
+ parser.add_argument('--condition', type=str,
+ help='Condition to run', required=False)
+ # To list all conditions
+ parser.add_argument('--list-conditions', action='store_true', default=False, help='List available conditions.')
+ # For generating reports
+ parser.add_argument('--generate-report', action='store_true', default=False, help='Generate report.')
+ parser.add_argument('--output', type=str, help='File to store report.', required=False)
+ # For actually running the experimental conditions
+ parser.add_argument('--all', action='store_true', default=False, help='Run using all languages.')
+ parser.add_argument('--language', type=str, help='Language to run.', required=False)
+ parser.add_argument('--directory', type=str, help='Base directory.', default='', required=False)
+ parser.add_argument('--dry-run', action='store_true', default=False, help='Print out commands but do not execute.')
+ parser.add_argument('--skip-eval', action='store_true', default=False, help='Skip running trec_eval.')
+ parser.add_argument('--display-commands', action='store_true', default=False, help='Display command.')
+ args = parser.parse_args()
+
+ if args.list_conditions:
+ list_conditions()
+ sys.exit()
+
+ if args.generate_report:
+ if not args.output:
+ print(f'Must specify report filename with --output.')
+ sys.exit()
+
+ generate_report(args)
+ sys.exit()
+
+ if args.all and (args.condition or args.language):
+ print('Specifying --all will run all conditions and languages')
+ sys.exit()
+
+ run_conditions(args)

pyserini/2cr/mrtydi.yaml

@@ -0,0 +1,890 @@
+conditions:
+ # mDPR, tied encoders, pFT w/ MS MARCO, FT all
+ - name: mdpr-tied-pft-msmarco-ft-all.ar
+ eval_key: mrtydi-v1.1-arabic
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics mrtydi-v1.1-arabic-${split} --index mrtydi-v1.1-arabic-mdpr-tied-pft-msmarco-ft-all --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.9505
+ R@100: 1.0000
+ - split: dev
+ scores:
+ - MRR@100: 0.6913
+ R@100: 0.9165
+ - split: test
+ scores:
+ - MRR@100: 0.6949
+ R@100: 0.9004
+ - name: mdpr-tied-pft-msmarco-ft-all.bn
+ eval_key: mrtydi-v1.1-bengali
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics mrtydi-v1.1-bengali-${split} --index mrtydi-v1.1-bengali-mdpr-tied-pft-msmarco-ft-all --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.9620
+ R@100: 1.0000
+ - split: dev
+ scores:
+ - MRR@100: 0.5897
+ R@100: 0.8977
+ - split: test
+ scores:
+ - MRR@100: 0.6228
+ R@100: 0.9550
+ - name: mdpr-tied-pft-msmarco-ft-all.en
+ eval_key: mrtydi-v1.1-english
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics mrtydi-v1.1-english-${split} --index mrtydi-v1.1-english-mdpr-tied-pft-msmarco-ft-all --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.8278
+ R@100: 1.0000
+ - split: dev
+ scores:
+ - MRR@100: 0.5357
+ R@100: 0.8884
+ - split: test
+ scores:
+ - MRR@100: 0.4916
+ R@100: 0.8414
+ - name: mdpr-tied-pft-msmarco-ft-all.fi
+ eval_key: mrtydi-v1.1-finnish
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics mrtydi-v1.1-finnish-${split} --index mrtydi-v1.1-finnish-mdpr-tied-pft-msmarco-ft-all --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.9577
+ R@100: 0.9997
+ - split: dev
+ scores:
+ - MRR@100: 0.6626
+ R@100: 0.9171
+ - split: test
+ scores:
+ - MRR@100: 0.5595
+ R@100: 0.8563
+ - name: mdpr-tied-pft-msmarco-ft-all.id
+ eval_key: mrtydi-v1.1-indonesian
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics mrtydi-v1.1-indonesian-${split} --index mrtydi-v1.1-indonesian-mdpr-tied-pft-msmarco-ft-all --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.9469
+ R@100: 1.0000
+ - split: dev
+ scores:
+ - MRR@100: 0.6294
+ R@100: 0.9150
+ - split: test
+ scores:
+ - MRR@100: 0.5783
+ R@100: 0.8609
+ - name: mdpr-tied-pft-msmarco-ft-all.ja
+ eval_key: mrtydi-v1.1-japanese
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics mrtydi-v1.1-japanese-${split} --index mrtydi-v1.1-japanese-mdpr-tied-pft-msmarco-ft-all --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.8802
+ R@100: 0.9997
+ - split: dev
+ scores:
+ - MRR@100: 0.5505
+ R@100: 0.8696
+ - split: test
+ scores:
+ - MRR@100: 0.5007
+ R@100: 0.8130
+ - name: mdpr-tied-pft-msmarco-ft-all.ko
+ eval_key: mrtydi-v1.1-korean
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics mrtydi-v1.1-korean-${split} --index mrtydi-v1.1-korean-mdpr-tied-pft-msmarco-ft-all --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.9195
+ R@100: 1.0000
+ - split: dev
+ scores:
+ - MRR@100: 0.5645
+ R@100: 0.8663
+ - split: test
+ scores:
+ - MRR@100: 0.4861
+ R@100: 0.7854
+ - name: mdpr-tied-pft-msmarco-ft-all.ru
+ eval_key: mrtydi-v1.1-russian
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics mrtydi-v1.1-russian-${split} --index mrtydi-v1.1-russian-mdpr-tied-pft-msmarco-ft-all --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.8473
+ R@100: 0.9994
+ - split: dev
+ scores:
+ - MRR@100: 0.5104
+ R@100: 0.8720
+ - split: test
+ scores:
+ - MRR@100: 0.5161
+ R@100: 0.8432
+ - name: mdpr-tied-pft-msmarco-ft-all.sw
+ eval_key: mrtydi-v1.1-swahili
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics mrtydi-v1.1-swahili-${split} --index mrtydi-v1.1-swahili-mdpr-tied-pft-msmarco-ft-all --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.9515
+ R@100: 1.0000
+ - split: dev
+ scores:
+ - MRR@100: 0.6404
+ R@100: 0.9018
+ - split: test
+ scores:
+ - MRR@100: 0.6438
+ R@100: 0.8756
+ - name: mdpr-tied-pft-msmarco-ft-all.te
+ eval_key: mrtydi-v1.1-telugu
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics mrtydi-v1.1-telugu-${split} --index mrtydi-v1.1-telugu-mdpr-tied-pft-msmarco-ft-all --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.9679
+ R@100: 1.0000
+ - split: dev
+ scores:
+ - MRR@100: 0.7962
+ R@100: 0.9593
+ - split: test
+ scores:
+ - MRR@100: 0.8908
+ R@100: 0.9659
+ - name: mdpr-tied-pft-msmarco-ft-all.th
+ eval_key: mrtydi-v1.1-thai
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics mrtydi-v1.1-thai-${split} --index mrtydi-v1.1-thai-mdpr-tied-pft-msmarco-ft-all --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.9504
+ R@100: 1.0000
+ - split: dev
+ scores:
+ - MRR@100: 0.6670
+ R@100: 0.9114
+ - split: test
+ scores:
+ - MRR@100: 0.6175
+ R@100: 0.8826
+
+ # mDPR, tied encoders, pFT w/ MS MARCO
+ - name: mdpr-tied-pft-msmarco.ar
+ eval_key: mrtydi-v1.1-arabic
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics mrtydi-v1.1-arabic-${split} --index mrtydi-v1.1-arabic-mdpr-tied-pft-msmarco --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.3957
+ R@100: 0.7818
+ - split: dev
+ scores:
+ - MRR@100: 0.3978
+ R@100: 0.7778
+ - split: test
+ scores:
+ - MRR@100: 0.4414
+ R@100: 0.7971
+ - name: mdpr-tied-pft-msmarco.bn
+ eval_key: mrtydi-v1.1-bengali
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics mrtydi-v1.1-bengali-${split} --index mrtydi-v1.1-bengali-mdpr-tied-pft-msmarco --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.2920
+ R@100: 0.7323
+ - split: dev
+ scores:
+ - MRR@100: 0.2993
+ R@100: 0.7318
+ - split: test
+ scores:
+ - MRR@100: 0.3969
+ R@100: 0.7838
+ - name: mdpr-tied-pft-msmarco.en
+ eval_key: mrtydi-v1.1-english
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics mrtydi-v1.1-english-${split} --index mrtydi-v1.1-english-mdpr-tied-pft-msmarco --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.3374
+ R@100: 0.8111
+ - split: dev
+ scores:
+ - MRR@100: 0.3451
+ R@100: 0.7995
+ - split: test
+ scores:
+ - MRR@100: 0.3270
+ R@100: 0.7536
+ - name: mdpr-tied-pft-msmarco.fi
+ eval_key: mrtydi-v1.1-finnish
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics mrtydi-v1.1-finnish-${split} --index mrtydi-v1.1-finnish-mdpr-tied-pft-msmarco --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.3668
+ R@100: 0.7337
+ - split: dev
+ scores:
+ - MRR@100: 0.3636
+ R@100: 0.7371
+ - split: test
+ scores:
+ - MRR@100: 0.2750
+ R@100: 0.6471
+ - name: mdpr-tied-pft-msmarco.id
+ eval_key: mrtydi-v1.1-indonesian
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics mrtydi-v1.1-indonesian-${split} --index mrtydi-v1.1-indonesian-mdpr-tied-pft-msmarco --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.2794
+ R@100: 0.7044
+ - split: dev
+ scores:
+ - MRR@100: 0.2853
+ R@100: 0.7198
+ - split: test
+ scores:
+ - MRR@100: 0.3520
+ R@100: 0.7356
+ - name: mdpr-tied-pft-msmarco.ja
+ eval_key: mrtydi-v1.1-japanese
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics mrtydi-v1.1-japanese-${split} --index mrtydi-v1.1-japanese-mdpr-tied-pft-msmarco --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.3089
+ R@100: 0.7603
+ - split: dev
+ scores:
+ - MRR@100: 0.3108
+ R@100: 0.7597
+ - split: test
+ scores:
+ - MRR@100: 0.3107
+ R@100: 0.7317
+ - name: mdpr-tied-pft-msmarco.ko
+ eval_key: mrtydi-v1.1-korean
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics mrtydi-v1.1-korean-${split} --index mrtydi-v1.1-korean-mdpr-tied-pft-msmarco --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.3003
+ R@100: 0.6907
+ - split: dev
+ scores:
+ - MRR@100: 0.3017
+ R@100: 0.7046
+ - split: test
+ scores:
+ - MRR@100: 0.2820
+ R@100: 0.6172
+ - name: mdpr-tied-pft-msmarco.ru
+ eval_key: mrtydi-v1.1-russian
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics mrtydi-v1.1-russian-${split} --index mrtydi-v1.1-russian-mdpr-tied-pft-msmarco --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.2856
+ R@100: 0.7305
+ - split: dev
+ scores:
+ - MRR@100: 0.2943
+ R@100: 0.7404
+ - split: test
+ scores:
+ - MRR@100: 0.3561
+ R@100: 0.7432
+ - name: mdpr-tied-pft-msmarco.sw
+ eval_key: mrtydi-v1.1-swahili
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics mrtydi-v1.1-swahili-${split} --index mrtydi-v1.1-swahili-mdpr-tied-pft-msmarco --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.2491
+ R@100: 0.5195
+ - split: dev
+ scores:
+ - MRR@100: 0.2447
+ R@100: 0.5266
+ - split: test
+ scores:
+ - MRR@100: 0.3418
+ R@100: 0.6343
+ - name: mdpr-tied-pft-msmarco.te
+ eval_key: mrtydi-v1.1-telugu
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics mrtydi-v1.1-telugu-${split} --index mrtydi-v1.1-telugu-mdpr-tied-pft-msmarco --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.3059
+ R@100: 0.7510
+ - split: dev
+ scores:
+ - MRR@100: 0.2995
+ R@100: 0.7355
+ - split: test
+ scores:
+ - MRR@100: 0.3102
+ R@100: 0.7817
+ - name: mdpr-tied-pft-msmarco.th
+ eval_key: mrtydi-v1.1-thai
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics mrtydi-v1.1-thai-${split} --index mrtydi-v1.1-thai-mdpr-tied-pft-msmarco --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.2334
+ R@100: 0.5851
+ - split: dev
+ scores:
+ - MRR@100: 0.2407
+ R@100: 0.5795
+ - split: test
+ scores:
+ - MRR@100: 0.2693
+ R@100: 0.5945
+
+ # mDPR, tied encoders, pFT w/ NQ
+ - name: mdpr-tied-pft-nq.ar
+ eval_key: mrtydi-v1.1-arabic
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-nq --topics mrtydi-v1.1-arabic-${split} --index mrtydi-v1.1-arabic-mdpr-tied-pft-nq --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.2087
+ R@100: 0.5854
+ - split: dev
+ scores:
+ - MRR@100: 0.2132
+ R@100: 0.5868
+ - split: test
+ scores:
+ - MRR@100: 0.2214
+ R@100: 0.6001
+ - name: mdpr-tied-pft-nq.bn
+ eval_key: mrtydi-v1.1-bengali
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-nq --topics mrtydi-v1.1-bengali-${split} --index mrtydi-v1.1-bengali-mdpr-tied-pft-nq --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.2371
+ R@100: 0.6281
+ - split: dev
+ scores:
+ - MRR@100: 0.2414
+ R@100: 0.6409
+ - split: test
+ scores:
+ - MRR@100: 0.2535
+ R@100: 0.7072
+ - name: mdpr-tied-pft-nq.en
+ eval_key: mrtydi-v1.1-english
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-nq --topics mrtydi-v1.1-english-${split} --index mrtydi-v1.1-english-mdpr-tied-pft-nq --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.2441
+ R@100: 0.7217
+ - split: dev
+ scores:
+ - MRR@100: 0.2359
+ R@100: 0.7187
+ - split: test
+ scores:
+ - MRR@100: 0.2433
+ R@100: 0.6893
+ - name: mdpr-tied-pft-nq.fi
+ eval_key: mrtydi-v1.1-finnish
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-nq --topics mrtydi-v1.1-finnish-${split} --index mrtydi-v1.1-finnish-mdpr-tied-pft-nq --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.2996
+ R@100: 0.6787
+ - split: dev
+ scores:
+ - MRR@100: 0.3252
+ R@100: 0.7037
+ - split: test
+ scores:
+ - MRR@100: 0.2444
+ R@100: 0.6401
+ - name: mdpr-tied-pft-nq.id
+ eval_key: mrtydi-v1.1-indonesian
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-nq --topics mrtydi-v1.1-indonesian-${split} --index mrtydi-v1.1-indonesian-mdpr-tied-pft-nq --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.2706
+ R@100: 0.7322
+ - split: dev
+ scores:
+ - MRR@100: 0.2719
+ R@100: 0.7394
+ - split: test
+ scores:
+ - MRR@100: 0.2815
+ R@100: 0.6914
+ - name: mdpr-tied-pft-nq.ja
+ eval_key: mrtydi-v1.1-japanese
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-nq --topics mrtydi-v1.1-japanese-${split} --index mrtydi-v1.1-japanese-mdpr-tied-pft-nq --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.2165
+ R@100: 0.6043
+ - split: dev
+ scores:
+ - MRR@100: 0.2299
+ R@100: 0.6239
+ - split: test
+ scores:
+ - MRR@100: 0.2058
+ R@100: 0.5734
+ - name: mdpr-tied-pft-nq.ko
+ eval_key: mrtydi-v1.1-korean
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-nq --topics mrtydi-v1.1-korean-${split} --index mrtydi-v1.1-korean-mdpr-tied-pft-nq --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.2527
+ R@100: 0.6556
+ - split: dev
+ scores:
+ - MRR@100: 0.2680
+ R@100: 0.6271
+ - split: test
+ scores:
+ - MRR@100: 0.2234
+ R@100: 0.5499
+ - name: mdpr-tied-pft-nq.ru
+ eval_key: mrtydi-v1.1-russian
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-nq --topics mrtydi-v1.1-russian-${split} --index mrtydi-v1.1-russian-mdpr-tied-pft-nq --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.2160
+ R@100: 0.6262
+ - split: dev
+ scores:
+ - MRR@100: 0.2263
+ R@100: 0.6444
+ - split: test
+ scores:
+ - MRR@100: 0.2501
+ R@100: 0.6181
+ - name: mdpr-tied-pft-nq.sw
+ eval_key: mrtydi-v1.1-swahili
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-nq --topics mrtydi-v1.1-swahili-${split} --index mrtydi-v1.1-swahili-mdpr-tied-pft-nq --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.2383
+ R@100: 0.5707
+ - split: dev
+ scores:
+ - MRR@100: 0.2543
+ R@100: 0.6138
+ - split: test
+ scores:
+ - MRR@100: 0.2621
+ R@100: 0.5965
+ - name: mdpr-tied-pft-nq.te
+ eval_key: mrtydi-v1.1-telugu
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-nq --topics mrtydi-v1.1-telugu-${split} --index mrtydi-v1.1-telugu-mdpr-tied-pft-nq --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.1483
+ R@100: 0.4162
+ - split: dev
+ scores:
+ - MRR@100: 0.1494
+ R@100: 0.3967
+ - split: test
+ scores:
+ - MRR@100: 0.0970
+ R@100: 0.2454
+ - name: mdpr-tied-pft-nq.th
+ eval_key: mrtydi-v1.1-thai
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-nq --topics mrtydi-v1.1-thai-${split} --index mrtydi-v1.1-thai-mdpr-tied-pft-nq --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.1426
+ R@100: 0.4717
+ - split: dev
+ scores:
+ - MRR@100: 0.1618
+ R@100: 0.4637
+ - split: test
+ scores:
+ - MRR@100: 0.1575
+ R@100: 0.4550
+
+ # mDPR, split encoders, pFT w/ NQ
+ - name: mdpr-split-pft-nq.ar
+ eval_key: mrtydi-v1.1-arabic
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder castorini/mdpr-question-nq --topics mrtydi-v1.1-arabic-${split} --index mrtydi-v1.1-arabic-mdpr-nq --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.2510
+ R@100: 0.6384
+ - split: dev
+ scores:
+ - MRR@100: 0.2449
+ R@100: 0.6334
+ - split: test
+ scores:
+ - MRR@100: 0.2907
+ R@100: 0.6502
+ - name: mdpr-split-pft-nq.bn
+ eval_key: mrtydi-v1.1-bengali
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder castorini/mdpr-question-nq --topics mrtydi-v1.1-bengali-${split} --index mrtydi-v1.1-bengali-mdpr-nq --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.2293
+ R@100: 0.6454
+ - split: dev
+ scores:
+ - MRR@100: 0.2367
+ R@100: 0.6511
+ - split: test
+ scores:
+ - MRR@100: 0.2911
+ R@100: 0.7793
+ - name: mdpr-split-pft-nq.en
+ eval_key: mrtydi-v1.1-english
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder castorini/mdpr-question-nq --topics mrtydi-v1.1-english-${split} --index mrtydi-v1.1-english-mdpr-nq --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.2862
+ R@100: 0.7372
+ - split: dev
+ scores:
+ - MRR@100: 0.2821
+ R@100: 0.7437
+ - split: test
+ scores:
+ - MRR@100: 0.2907
+ R@100: 0.6779
+ - name: mdpr-split-pft-nq.fi
+ eval_key: mrtydi-v1.1-finnish
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder castorini/mdpr-question-nq --topics mrtydi-v1.1-finnish-${split} --index mrtydi-v1.1-finnish-mdpr-nq --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.2473
+ R@100: 0.6289
+ - split: dev
+ scores:
+ - MRR@100: 0.2466
+ R@100: 0.6283
+ - split: test
+ scores:
+ - MRR@100: 0.2050
+ R@100: 0.5680
+ - name: mdpr-split-pft-nq.id
+ eval_key: mrtydi-v1.1-indonesian
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder castorini/mdpr-question-nq --topics mrtydi-v1.1-indonesian-${split} --index mrtydi-v1.1-indonesian-mdpr-nq --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.2351
+ R@100: 0.6952
+ - split: dev
+ scores:
+ - MRR@100: 0.2475
+ R@100: 0.7181
+ - split: test
+ scores:
+ - MRR@100: 0.2705
+ R@100: 0.6848
+ - name: mdpr-split-pft-nq.ja
+ eval_key: mrtydi-v1.1-japanese
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder castorini/mdpr-question-nq --topics mrtydi-v1.1-japanese-${split} --index mrtydi-v1.1-japanese-mdpr-nq --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.1967
+ R@100: 0.5983
+ - split: dev
+ scores:
+ - MRR@100: 0.2055
+ R@100: 0.6142
+ - split: test
+ scores:
+ - MRR@100: 0.2119
+ R@100: 0.5840
+ - name: mdpr-split-pft-nq.ko
+ eval_key: mrtydi-v1.1-korean
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder castorini/mdpr-question-nq --topics mrtydi-v1.1-korean-${split} --index mrtydi-v1.1-korean-mdpr-nq --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.2383
+ R@100: 0.6180
+ - split: dev
+ scores:
+ - MRR@100: 0.2343
+ R@100: 0.6238
+ - split: test
+ scores:
+ - MRR@100: 0.2345
+ R@100: 0.5325
+ - name: mdpr-split-pft-nq.ru
+ eval_key: mrtydi-v1.1-russian
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder castorini/mdpr-question-nq --topics mrtydi-v1.1-russian-${split} --index mrtydi-v1.1-russian-mdpr-nq --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.2317
+ R@100: 0.6534
+ - split: dev
+ scores:
+ - MRR@100: 0.2490
+ R@100: 0.6553
+ - split: test
+ scores:
+ - MRR@100: 0.2820
+ R@100: 0.6474
+ - name: mdpr-split-pft-nq.sw
+ eval_key: mrtydi-v1.1-swahili
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder castorini/mdpr-question-nq --topics mrtydi-v1.1-swahili-${split} --index mrtydi-v1.1-swahili-mdpr-nq --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.1457
+ R@100: 0.4481
+ - split: dev
+ scores:
+ - MRR@100: 0.1547
+ R@100: 0.4724
+ - split: test
+ scores:
+ - MRR@100: 0.1883
+ R@100: 0.5281
+ - name: mdpr-split-pft-nq.te
+ eval_key: mrtydi-v1.1-telugu
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder castorini/mdpr-question-nq --topics mrtydi-v1.1-telugu-${split} --index mrtydi-v1.1-telugu-mdpr-nq --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.1489
+ R@100: 0.4905
+ - split: dev
+ scores:
+ - MRR@100: 0.1503
+ R@100: 0.4934
+ - split: test
+ scores:
+ - MRR@100: 0.1099
+ R@100: 0.3661
+ - name: mdpr-split-pft-nq.th
+ eval_key: mrtydi-v1.1-thai
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder castorini/mdpr-question-nq --topics mrtydi-v1.1-thai-${split} --index mrtydi-v1.1-thai-mdpr-nq --output $output --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.1603
+ R@100: 0.4983
+ - split: dev
+ scores:
+ - MRR@100: 0.1584
+ R@100: 0.5083
+ - split: test
+ scores:
+ - MRR@100: 0.1709
+ R@100: 0.5146
+
+ # BM25
+ - name: bm25.ar
+ eval_key: mrtydi-v1.1-arabic
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --language ar --topics mrtydi-v1.1-arabic-${split} --index mrtydi-v1.1-arabic --output $output --bm25 --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.3356
+ R@100: 0.7944
+ - split: dev
+ scores:
+ - MRR@100: 0.3462
+ R@100: 0.7872
+ - split: test
+ scores:
+ - MRR@100: 0.3682
+ R@100: 0.7928
+ - name: bm25.bn
+ eval_key: mrtydi-v1.1-bengali
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --language bn --topics mrtydi-v1.1-bengali-${split} --index mrtydi-v1.1-bengali --output $output --bm25 --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.3566
+ - R@100: 0.8336
+ - split: dev
+ scores:
+ - MRR@100: 0.3385
+ - R@100: 0.8432
+ - split: test
+ scores:
+ - MRR@100: 0.4182
+ - R@100: 0.8694
+ - name: bm25.en
+ eval_key: mrtydi-v1.1-english
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --language en --topics mrtydi-v1.1-english-${split} --index mrtydi-v1.1-english --output $output --bm25 --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.1592
+ - R@100: 0.5785
+ - split: dev
+ scores:
+ - MRR@100: 0.1685
+ - R@100: 0.6196
+ - split: test
+ scores:
+ - MRR@100: 0.1404
+ - R@100: 0.5365
+ - name: bm25.fi
+ eval_key: mrtydi-v1.1-finnish
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --language fi --topics mrtydi-v1.1-finnish-${split} --index mrtydi-v1.1-finnish --output $output --bm25 --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.4101
+ - R@100: 0.8198
+ - split: dev
+ scores:
+ - MRR@100: 0.4136
+ - R@100: 0.8285
+ - split: test
+ scores:
+ - MRR@100: 0.2836
+ - R@100: 0.7196
+ - name: bm25.id
+ eval_key: mrtydi-v1.1-indonesian
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --language id --topics mrtydi-v1.1-indonesian-${split} --index mrtydi-v1.1-indonesian --output $output --bm25 --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.2972
+ - R@100: 0.7948
+ - split: dev
+ scores:
+ - MRR@100: 0.2937
+ - R@100: 0.7827
+ - split: test
+ scores:
+ - MRR@100: 0.3762
+ - R@100: 0.8426
+ - name: bm25.ja
+ eval_key: mrtydi-v1.1-japanese
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --language ja --topics mrtydi-v1.1-japanese-${split} --index mrtydi-v1.1-japanese --output $output --bm25 --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.2262
+ - R@100: 0.7290
+ - split: dev
+ scores:
+ - MRR@100: 0.2250
+ - R@100: 0.7252
+ - split: test
+ scores:
+ - MRR@100: 0.2125
+ - R@100: 0.6431
+ - name: bm25.ko
+ eval_key: mrtydi-v1.1-korean
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --language ko --topics mrtydi-v1.1-korean-${split} --index mrtydi-v1.1-korean --output $output --bm25 --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.2596
+ - R@100: 0.6178
+ - split: dev
+ scores:
+ - MRR@100: 0.2888
+ - R@100: 0.6733
+ - split: test
+ scores:
+ - MRR@100: 0.2848
+ - R@100: 0.6188
+ - name: bm25.ru
+ eval_key: mrtydi-v1.1-russian
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --language ru --topics mrtydi-v1.1-russian-${split} --index mrtydi-v1.1-russian --output $output --bm25 --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.2229
+ - R@100: 0.5779
+ - split: dev
+ scores:
+ - MRR@100: 0.2202
+ - R@100: 0.5760
+ - split: test
+ scores:
+ - MRR@100: 0.3163
+ - R@100: 0.6541
+ - name: bm25.sw
+ eval_key: mrtydi-v1.1-swahili
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --language sw --topics mrtydi-v1.1-swahili-${split} --index mrtydi-v1.1-swahili --output $output --bm25 --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.2610
+ - R@100: 0.5903
+ - split: dev
+ scores:
+ - MRR@100: 0.2693
+ - R@100: 0.5789
+ - split: test
+ scores:
+ - MRR@100: 0.3893
+ - R@100: 0.7642
+ - name: bm25.te
+ eval_key: mrtydi-v1.1-telugu
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --language te --topics mrtydi-v1.1-telugu-${split} --index mrtydi-v1.1-telugu --output $output --bm25 --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.4204
+ - R@100: 0.8229
+ - split: dev
+ scores:
+ - MRR@100: 0.4269
+ - R@100: 0.8362
+ - split: test
+ scores:
+ - MRR@100: 0.5283
+ - R@100: 0.8971
+ - name: bm25.th
+ eval_key: mrtydi-v1.1-thai
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --language th --topics mrtydi-v1.1-thai-${split} --index mrtydi-v1.1-thai --output $output --bm25 --hits 100
+ splits:
+ - split: train
+ scores:
+ - MRR@100: 0.3543
+ - R@100: 0.8349
+ - split: dev
+ scores:
+ - MRR@100: 0.3586
+ - R@100: 0.8536
+ - split: test
+ scores:
+ - MRR@100: 0.4012
+ - R@100: 0.8529

pyserini/2cr/mrtydi_html.template

@@ -0,0 +1,256 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no" />
+ <meta http-equiv="x-ua-compatible" content="ie=edge" />
+ <title>Pyserini Reproductions</title>
+ <!-- Font Awesome -->
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.11.2/css/all.css" />
+ <!-- Google Fonts Roboto -->
+ <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Roboto:wght@300;400;500;700&display=swap" />
+ <!-- MDB -->
+ <link href="https://cdnjs.cloudflare.com/ajax/libs/mdb-ui-kit/4.0.0/mdb.min.css" rel="stylesheet" />
+
+ <style>
+tr.hide-table-padding td {
+ padding: 0;
+}
+
+.expand-button {
+ position: relative;
+}
+
+.accordion-toggle .expand-button:after {
+ position: absolute;
+ left:.75rem;
+ top: 50%;
+ transform: translate(0, -50%);
+ content: '-';
+}
+
+.accordion-toggle.collapsed .expand-button:after {
+ content: '+';
+}
+
+blockquote.mycode {
+ border-left: 3px solid #ccc;
+ margin-left: 25px;
+ margin-top: 15px;
+ padding-left: 15px;
+}
+
+blockquote.mycode2 {
+ border-left: 3px solid #ccc;
+ margin-left: 25px;
+ padding-top: 10px;
+ padding-bottom: 10px;
+ padding-left: 15px;
+}
+
+tr th.headertop {
+ border-bottom: none;
+ padding-bottom: 0rem
+}
+
+tr th.headerbottom {
+ padding-top: 0rem
+}
+
+.table>:not(caption)>*>*{padding:0.75rem 0.75rem}
+
+.copy-code-button {
+ border-radius: 0;
+ min-width: 55px;
+ background: none repeat scroll 0 0 transparent;
+ background-color: grey;
+ color: #F1F2F3 !important;
+ cursor: pointer;
+ border-style: none;
+ font-family: 'HELVETICA',sans-serif;
+ font-size: 0.8em;
+ font-weight: normal;
+ text-align: center;
+ text-decoration: none;
+ text-indent: 0;
+ text-transform: uppercase;
+ font-weight: 500;
+ line-height: 1.42rem;
+ margin: 0;
+ padding: 3px 8px;
+ position: absolute !important;
+ top: 0 !important;
+ right: 0 !important;
+}
+
+.copy-code-button > span {
+ color: #F1F2F3 !important;
+}
+
+.copy-code-button, ::before, ::after {
+ box-sizing: inherit;
+}
+
+.copy-code-button::before {
+ content: '';
+ display: inline-block;
+ width: 16px;
+ height: 16px;
+ margin-right: 3px;
+ background-size: contain;
+ background-image: url("data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiPz4KPHN2ZyB3aWR0aD0iMTVweCIgaGVpZ2h0PSIxNXB4IiB2aWV3Qm94PSIwIDAgMTUgMTUiIHZlcnNpb249IjEuMSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB4bWxuczp4bGluaz0iaHR0cDovL3d3dy53My5vcmcvMTk5OS94bGluayI+CiAgICA8IS0tIEdlbmVyYXRvcjogU2tldGNoIDUwLjIgKDU1MDQ3KSAtIGh0dHA6Ly93d3cuYm9oZW1pYW5jb2RpbmcuY29tL3NrZXRjaCAtLT4KICAgIDx0aXRsZT5QYWdlIDE8L3RpdGxlPgogICAgPGRlc2M+Q3JlYXRlZCB3aXRoIFNrZXRjaC48L2Rlc2M+CiAgICA8ZGVmcz48L2RlZnM+CiAgICA8ZyBpZD0iRmxvdyIgc3Ryb2tlPSJub25lIiBzdHJva2Utd2lkdGg9IjEiIGZpbGw9Im5vbmUiIGZpbGwtcnVsZT0iZXZlbm9kZCI+CiAgICAgICAgPGcgaWQ9IkJ0dG5faHRtbCIgdHJhbnNmb3JtPSJ0cmFuc2xhdGUoLTgxOS4wMDAwMDAsIC03NTMuMDAwMDAwKSIgZmlsbD0iI0ZGRkZGRiI+CiAgICAgICAgICAgIDxnIGlkPSJHcm91cC0xIiB0cmFuc2Zvcm09InRyYW5zbGF0ZSgzMTEuMDAwMDAwLCA0MDUuMDAwMDAwKSI+CiAgICAgICAgICAgICAgICA8ZyBpZD0iR3JvdXAtMiIgdHJhbnNmb3JtPSJ0cmFuc2xhdGUoNTA4LjAwMDAwMCwgMzQyLjAwMDAwMCkiPgogICAgICAgICAgICAgICAgICAgIDxwYXRoIGQ9Ik0xMy45NzcyNzI3LDYgTDMuNDA5MDkwOTEsNiBDMi44NDQ1NDU0NSw2IDIuMzg2MzYzNjQsNi40NTgxODE4MiAyLjM4NjM2MzY0LDcuMDIyNzI3MjcgTDIuMzg2MzYzNjQsMTcuNTkwOTA5MSBDMi4zODYzNjM2NCwxOC4xNTU0NTQ1IDIuODQ0NTQ1NDUsMTguNjEzNjM2NCAzLjQwOTA5MDkxLDE4LjYxMzYzNjQgTDEzLjk3NzI3MjcsMTguNjEzNjM2NCBDMTQuNTQxODE4MiwxOC42MTM2MzY0IDE1LDE4LjE1NTQ1NDUgMTUsMTcuNTkwOTA5MSBMMTUsNy4wMjI3MjcyNyBDMTUsNi40NTgxODE4MiAxNC41NDE4MTgyLDYgMTMuOTc3MjcyNyw2IFogTTE0LjMxODE4MTgsMTcuNTkwOTA5MSBDMTQuMzE4MTgxOCwxNy43NzkwOTA5IDE0LjE2NTQ1NDUsMTcuOTMxODE4MiAxMy45NzcyNzI3LDE3LjkzMTgxODIgTDMuNDA5MDkwOTEsMTcuOTMxODE4MiBDMy4yMjA5MDkwOSwxNy45MzE4MTgyIDMuMDY4MTgxODIsMTcuNzc5MDkwOSAzLjA2ODE4MTgyLDE3LjU5MDkwOTEgTDMuMDY4MTgxODIsNy4wMjI3MjcyNyBDMy4wNjgxODE4Miw2LjgzNDU0NTQ1IDMuMjIwOTA5MDksNi42ODE4MTgxOCAzLjQwOTA5MDkxLDYuNjgxODE4MTggTDEzLjk3NzI3MjcsNi42ODE4MTgxOCBDMTQuMTY1NDU0NSw2LjY4MTgxODE4IDE0LjMxODE4MTgsNi44MzQ1NDU0NSAxNC4zMTgxODE4LDcuMDIyNzI3MjcgTDE0LjMxODE4MTgsMTcuNTkwOTA5MSBaIE0xMS45MzE4MTgyLDE5Ljk3NzI3MjcgQzExLjkzMTgxODIsMjAuMTY1NDU0NSAxMS43NzkwOTA5LDIwLjMxODE4MTggMTEuNTkwOTA5MSwyMC4zMTgxODE4IEwxLjAyMjcyNzI3LDIwLjMxODE4MTggQzAuODM0NTQ1NDU1LDIwLjMxODE4MTggMC42ODE4MTgxODIsMjAuMTY1NDU0NSAwLjY4MTgxODE4MiwxOS45NzcyNzI3IEwwLjY4MTgxODE4Miw5LjQwOTA5MDkxIEMwLjY4MTgxODE4Miw5LjIyMDkwOTA5IDAuODM0NTQ1NDU1LDkuMDY4MTgxODIgMS4wMjI3MjcyNyw5LjA2ODE4MTgyIEwxLjM2MzYzNjM2LDkuMDY4MTgxODIgTDEuMzYzNjM2MzYsOC4zODYzNjM2NCBMMS4wMjI3MjcyNyw4LjM4NjM2MzY0IEMwLjQ1ODE4MTgxOCw4LjM4NjM2MzY0IDAsOC44NDQ1NDU0NSAwLDkuNDA5MDkwOTEgTDAsMTkuOTc3MjcyNyBDMCwyMC41NDE4MTgyIDAuNDU4MTgxODE4LDIxIDEuMDIyNzI3MjcsMjEgTDExLjU5MDkwOTEsMjEgQzEyLjE1NTQ1NDUsMjEgMTIuNjEzNjM2NCwyMC41NDE4MTgyIDEyLjYxMzYzNjQsMTkuOTc3MjcyNyBMMTIuNjEzNjM2NCwxOS42MzYzNjM2IEwxMS45MzE4MTgyLDE5LjYzNjM2MzYgTDExLjkzMTgxODIsMTkuOTc3MjcyNyBaIiBpZD0iUGFnZS0xIj48L3BhdGg+CiAgICAgICAgICAgICAgICA8L2c+CiAgICAgICAgICAgIDwvZz4KICAgICAgICA8L2c+CiAgICA8L2c+Cjwvc3ZnPg==");
+ background-repeat: no-repeat;
+ position: relative;
+ top: 3px;
+}
+
+.copy-code-button:focus {
+ /* Avoid an ugly focus outline on click in Chrome,
+ but darken the button for accessibility.
+ See https://stackoverflow.com/a/25298082/1481479 */
+ /* background-color: #E6E6E6; */
+ outline: 0;
+}
+
+pre[class*="prettyprint"] {
+ position: relative;
+ overflow: hidden;
+}
+ </style>
+</head>
+<body>
+
+ <!-- Background image -->
+ <div id="intro" class="bg-image vh-100 shadow-1-strong" style="max-height: 150px">
+ <div class="mask" style="
+ background: linear-gradient(
+ 45deg,
+ rgba(29, 236, 197, 0.7),
+ rgba(91, 14, 214, 0.7) 100%
+ );
+ ">
+ <div class="container d-flex align-items-center justify-content-center text-center h-100" style="max-height: 150px">
+ <div class="text-white">
+ <h1 class="mb-3">$title</h1>
+ </div>
+ </div>
+ </div>
+ </div>
+ <!-- Background image -->
+
+ <div class="container my-4">
+
+ $tables
+
+ </ul>
+
+ <div style="padding-top: 20px"/>
+
+ <h4>Programmatic Execution</h4>
+
+ <p>All experimental runs shown in the above table can be programmatically executed based on the instructions below.
+ To list all the experimental conditions:</p>
+
+ <blockquote class="mycode2"><tt>
+ python -m pyserini.2cr.mrtydi --list-conditions
+ </tt></blockquote>
+
+ <p>Run all languages for a specific condition and show commands:</p>
+
+ <blockquote class="mycode2"><tt>
+ python -m pyserini.2cr.mrtydi --condition bm25 --display-commands
+ </tt></blockquote>
+
+ <p>Run a particular language for a specific condition and show commands:</p>
+
+ <blockquote class="mycode2"><tt>
+ python -m pyserini.2cr.mrtydi --condition bm25 --language ko --display-commands
+ </tt></blockquote>
+
+ <p>Run all languages for all conditions and show commands:</p>
+
+ <blockquote class="mycode2"><tt>
+ python -m pyserini.2cr.mrtydi --all --display-commands
+ </tt></blockquote>
+
+ <p>With the above commands, run files will be placed in the current directory. Use the option <tt>--directory runs</tt> to place the runs in a sub-directory.</p>
+
+ <p>For a specific condition, just show the commands and do not run:</p>
+
+ <blockquote class="mycode2"><tt>
+ python -m pyserini.2cr.mrtydi --condition bm25 --display-commands --dry-run
+ </tt></blockquote>
+
+ <p>This will generate exactly the commands for a specific condition above (corresponding to a row in the table).</p>
+
+ <p>For a specific condition and language, just show the commands and do not run:</p>
+
+ <blockquote class="mycode2"><tt>
+ python -m pyserini.2cr.mrtydi --condition bm25 --language ko --display-commands --dry-run
+ </tt></blockquote>
+
+ <p>For all conditions, just show the commands and do not run and skip evaluation:</p>
+
+ <blockquote class="mycode2"><tt>
+ python -m pyserini.2cr.mrtydi --all --display-commands --dry-run --skip-eval
+ </tt></blockquote>
+
+ <p>Finally, to generate this page:</p>
+
+ <blockquote class="mycode2"><tt>
+ python -m pyserini.2cr.mrtydi --generate-report --output docs/2cr/mrtydi.html
+ </tt></blockquote>
+
+ <p>The output file <tt>mrtydi.html</tt> should be identical to this page.</p>
+
+ <div style="padding-top: 50px"/>
+
+ </div>
+
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.0/jquery.min.js"></script>
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/4.3.1/js/bootstrap.min.js"></script>
+ <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mdb-ui-kit/4.0.0/mdb.min.js"></script>
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.10/clipboard.min.js"></script>
+
+<script>
+document.querySelectorAll('pre').forEach(function (codeBlock) {
+ var button = document.createElement('button');
+ button.className = 'copy-code-button';
+ button.type = 'button';
+ var s = codeBlock.innerText;
+ button.setAttribute('data-clipboard-text',s);
+ button.innerText = 'Copy';
+
+ // var pre = codeBlock.parentNode;
+ codeBlock.classList.add('prettyprint');
+ // pre.parentNode.insertBefore(button, pre);
+ codeBlock.appendChild(button);
+});
+
+var clipboard = new ClipboardJS('.copy-code-button');
+
+clipboard.on('success', function(e) {
+ console.info('Action:', e.action);
+ console.info('Text:', e.text);
+ console.info('Trigger:', e.trigger);
+ e.trigger.textContent = 'Copied';
+ window.setTimeout(function() {
+ e.trigger.textContent = 'Copy';
+ }, 2000);
+ e.clearSelection();
+});
+
+clipboard.on('error', function(e) {
+ console.error('Action:', e.action);
+ console.error('Trigger:', e.trigger);
+ e.trigger.textContent = 'Error Copying';
+ window.setTimeout(function() {
+ e.trigger.textContent = 'Copy';
+ }, 2000);
+ e.clearSelection();
+});
+
+</script>
+
+</body>
+</html>

pyserini/2cr/mrtydi_html_table.template

@@ -0,0 +1,28 @@
+<div class="table-responsive">
+ <table class="table">
+ <thead>
+ <tr>
+ <th scope="col"></th>
+ <th scope="col">$desc</th>
+ <th scope="col">ar</th>
+ <th scope="col">bn</th>
+ <th scope="col">en</th>
+ <th scope="col">fi</th>
+ <th scope="col">id</th>
+ <th scope="col">ja</th>
+ <th scope="col">ko</th>
+ <th scope="col">ru</th>
+ <th scope="col">sw</th>
+ <th scope="col">te</th>
+ <th scope="col">th</th>
+ <th scope="col"></th>
+ <th scope="col">avg</th>
+ </tr>
+ </thead>
+ <tbody>
+
+$rows
+
+ </tbody>
+ </table>
+</div>

pyserini/2cr/mrtydi_html_table_row.template

@@ -0,0 +1,212 @@
+<!-- Condition: $model -->
+<tr class="accordion-toggle collapsed" id="table${table_cnt}-row${row_cnt}" data-toggle="collapse" data-parent="#table${table_cnt}-row${row_cnt}" href="#table${table_cnt}-collapse${row_cnt}">
+<td class="expand-button"></td>
+<td>$model</td>
+<td>$ar</td>
+<td>$bn</td>
+<td>$en</td>
+<td>$fi</td>
+<td>$id</td>
+<td>$ja</td>
+<td>$ko</td>
+<td>$ru</td>
+<td>$sw</td>
+<td>$te</td>
+<td>$th</td>
+<td></td>
+<td>$avg</td>
+</tr>
+<tr class="hide-table-padding">
+<td></td>
+<td></td>
+<td colspan="13" style="max-width: 600px">
+<div id="table${table_cnt}-collapse${row_cnt}" class="collapse in p-3">
+
+<!-- Tabs navs -->
+<ul class="nav nav-tabs mb-3" id="table${table_cnt}-row${row_cnt}-tabs" role="tablist">
+ <li class="nav-item" role="presentation">
+ <a class="nav-link active" id="table${table_cnt}-row${row_cnt}-tab1-header" data-mdb-toggle="tab" href="#table${table_cnt}-row${row_cnt}-tab1" role="tab" aria-controls="table${table_cnt}-row${row_cnt}-tab1" aria-selected="true" style="text-transform:none">ar</a>
+ </li>
+ <li class="nav-item" role="presentation">
+ <a class="nav-link" id="table${table_cnt}-row${row_cnt}-tab2-header" data-mdb-toggle="tab" href="#table${table_cnt}-row${row_cnt}-tab2" role="tab" aria-controls="table${table_cnt}-row${row_cnt}-tab2" aria-selected="false" style="text-transform:none">bn</a>
+ </li>
+ <li class="nav-item" role="presentation">
+ <a class="nav-link" id="table${table_cnt}-row${row_cnt}-tab3-header" data-mdb-toggle="tab" href="#table${table_cnt}-row${row_cnt}-tab3" role="tab" aria-controls="table${table_cnt}-row${row_cnt}-tab3" aria-selected="false" style="text-transform:none">en</a>
+ </li>
+ <li class="nav-item" role="presentation">
+ <a class="nav-link" id="table${table_cnt}-row${row_cnt}-tab4-header" data-mdb-toggle="tab" href="#table${table_cnt}-row${row_cnt}-tab4" role="tab" aria-controls="table${table_cnt}-row${row_cnt}-tab3" aria-selected="false" style="text-transform:none">fi</a>
+ </li>
+ <li class="nav-item" role="presentation">
+ <a class="nav-link" id="table${table_cnt}-row${row_cnt}-tab5-header" data-mdb-toggle="tab" href="#table${table_cnt}-row${row_cnt}-tab5" role="tab" aria-controls="table${table_cnt}-row${row_cnt}-tab3" aria-selected="false" style="text-transform:none">id</a>
+ </li>
+ <li class="nav-item" role="presentation">
+ <a class="nav-link" id="table${table_cnt}-row${row_cnt}-tab6-header" data-mdb-toggle="tab" href="#table${table_cnt}-row${row_cnt}-tab6" role="tab" aria-controls="table${table_cnt}-row${row_cnt}-tab3" aria-selected="false" style="text-transform:none">ja</a>
+ </li>
+ <li class="nav-item" role="presentation">
+ <a class="nav-link" id="table${table_cnt}-row${row_cnt}-tab7-header" data-mdb-toggle="tab" href="#table${table_cnt}-row${row_cnt}-tab7" role="tab" aria-controls="table${table_cnt}-row${row_cnt}-tab3" aria-selected="false" style="text-transform:none">ko</a>
+ </li>
+ <li class="nav-item" role="presentation">
+ <a class="nav-link" id="table${table_cnt}-row${row_cnt}-tab8-header" data-mdb-toggle="tab" href="#table${table_cnt}-row${row_cnt}-tab8" role="tab" aria-controls="table${table_cnt}-row${row_cnt}-tab3" aria-selected="false" style="text-transform:none">ru</a>
+ </li>
+ <li class="nav-item" role="presentation">
+ <a class="nav-link" id="table${table_cnt}-row${row_cnt}-tab9-header" data-mdb-toggle="tab" href="#table${table_cnt}-row${row_cnt}-tab9" role="tab" aria-controls="table${table_cnt}-row${row_cnt}-tab3" aria-selected="false" style="text-transform:none">sw</a>
+ </li>
+ <li class="nav-item" role="presentation">
+ <a class="nav-link" id="table${table_cnt}-row${row_cnt}-tab10-header" data-mdb-toggle="tab" href="#table${table_cnt}-row${row_cnt}-tab10" role="tab" aria-controls="table${table_cnt}-row${row_cnt}-tab3" aria-selected="false" style="text-transform:none">te</a>
+ </li>
+ <li class="nav-item" role="presentation">
+ <a class="nav-link" id="table${table_cnt}-row${row_cnt}-tab11-header" data-mdb-toggle="tab" href="#table${table_cnt}-row${row_cnt}-tab11" role="tab" aria-controls="table${table_cnt}-row${row_cnt}-tab3" aria-selected="false" style="text-transform:none">th</a>
+ </li>
+</ul>
+<!-- Tabs navs -->
+
+<!-- Tabs content -->
+<div class="tab-content" id="table${table_cnt}-row${row_cnt}-content">
+ <div class="tab-pane fade show active" id="table${table_cnt}-row${row_cnt}-tab1" role="tabpanel" aria-labelledby="table${table_cnt}-row${row_cnt}-tab1">
+Command to generate run:
+
+ <blockquote class="mycode">
+<pre><code>$cmd1
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd1}</code></pre>
+ </blockquote>
+
+ </div>
+ <div class="tab-pane fade" id="table${table_cnt}-row${row_cnt}-tab2" role="tabpanel" aria-labelledby="table${table_cnt}-row${row_cnt}-tab2">
+Command to generate run:
+
+ <blockquote class="mycode">
+<pre><code>$cmd2
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd2}</code></pre>
+ </blockquote>
+
+ </div>
+ <div class="tab-pane fade" id="table${table_cnt}-row${row_cnt}-tab3" role="tabpanel" aria-labelledby="table${table_cnt}-row${row_cnt}-tab3">
+Command to generate run:
+
+ <blockquote class="mycode">
+<pre><code>$cmd3
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd3}</code></pre>
+ </blockquote>
+
+ </div>
+ <div class="tab-pane fade" id="table${table_cnt}-row${row_cnt}-tab4" role="tabpanel" aria-labelledby="table${table_cnt}-row${row_cnt}-tab4">
+Command to generate run:
+
+ <blockquote class="mycode">
+<pre><code>$cmd4
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd4}</code></pre>
+ </blockquote>
+
+ </div>
+ <div class="tab-pane fade" id="table${table_cnt}-row${row_cnt}-tab5" role="tabpanel" aria-labelledby="table${table_cnt}-row${row_cnt}-tab5">
+Command to generate run:
+
+ <blockquote class="mycode">
+<pre><code>$cmd5
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd5}</code></pre>
+ </blockquote>
+
+ </div>
+ <div class="tab-pane fade" id="table${table_cnt}-row${row_cnt}-tab6" role="tabpanel" aria-labelledby="table${table_cnt}-row${row_cnt}-tab6">
+Command to generate run:
+
+ <blockquote class="mycode">
+<pre><code>$cmd6
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd6}</code></pre>
+ </blockquote>
+
+ </div>
+ <div class="tab-pane fade" id="table${table_cnt}-row${row_cnt}-tab7" role="tabpanel" aria-labelledby="table${table_cnt}-row${row_cnt}-tab7">
+Command to generate run:
+
+ <blockquote class="mycode">
+<pre><code>$cmd7
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd7}</code></pre>
+ </blockquote>
+
+ </div>
+ <div class="tab-pane fade" id="table${table_cnt}-row${row_cnt}-tab8" role="tabpanel" aria-labelledby="table${table_cnt}-row${row_cnt}-tab8">
+Command to generate run:
+
+ <blockquote class="mycode">
+<pre><code>$cmd8
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd8}</code></pre>
+ </blockquote>
+
+ </div>
+ <div class="tab-pane fade" id="table${table_cnt}-row${row_cnt}-tab9" role="tabpanel" aria-labelledby="table${table_cnt}-row${row_cnt}-tab9">
+Command to generate run:
+
+ <blockquote class="mycode">
+<pre><code>$cmd9
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd9}</code></pre>
+ </blockquote>
+
+ </div>
+ <div class="tab-pane fade" id="table${table_cnt}-row${row_cnt}-tab10" role="tabpanel" aria-labelledby="table${table_cnt}-row${row_cnt}-tab10">
+Command to generate run:
+
+ <blockquote class="mycode">
+<pre><code>$cmd10
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd10}</code></pre>
+ </blockquote>
+
+ </div>
+ <div class="tab-pane fade" id="table${table_cnt}-row${row_cnt}-tab11" role="tabpanel" aria-labelledby="table${table_cnt}-row${row_cnt}-tab11">
+Command to generate run:
+
+ <blockquote class="mycode">
+<pre><code>$cmd11
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd11}</code></pre>
+ </blockquote>
+
+ </div>
+</div>
+<!-- Tabs content -->
+
+</div></td>
+</tr>

pyserini/2cr/msmarco-v1-doc.yaml

@@ -0,0 +1,539 @@
+conditions:
+ - name: bm25-doc-tuned
+ display: BM25 doc (k1=4.46, b=0.82)
+ display-html: BM25 doc (<i>k<sub><small>1</small></sub></i>=4.46, <i>b</i>=0.82)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-doc --topics $topics --output $output --bm25
+ topics:
+ - topic_key: msmarco-doc-dev
+ eval_key: msmarco-doc-dev
+ scores:
+ - MRR@10: 0.2767
+ R@1K: 0.9357
+ - topic_key: dl19-doc
+ eval_key: dl19-doc
+ scores:
+ - MAP: 0.2336
+ nDCG@10: 0.5233
+ R@1K: 0.6757
+ - topic_key: dl20
+ eval_key: dl20-doc
+ scores:
+ - MAP: 0.3581
+ nDCG@10: 0.5061
+ R@1K: 0.7776
+ - name: bm25-doc-default
+ display: BM25 doc (k1=0.9, b=0.4)
+ display-html: BM25 doc (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
+ display-row: "[<a href=\"#\" data-mdb-toggle=\"tooltip\" title=\"Ma et al. (SIGIR 2021) Document Expansions and Learned Sparse Lexical Representations for MS MARCO V1 and V2.\">1</a>] &mdash; (1a)"
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-doc --topics $topics --output $output --bm25 --k1 0.9 --b 0.4
+ topics:
+ - topic_key: msmarco-doc-dev
+ eval_key: msmarco-doc-dev
+ scores:
+ - MRR@10: 0.2299
+ R@1K: 0.8856
+ - topic_key: dl19-doc
+ eval_key: dl19-doc
+ scores:
+ - MAP: 0.2434
+ nDCG@10: 0.5176
+ R@1K: 0.6966
+ - topic_key: dl20
+ eval_key: dl20-doc
+ scores:
+ - MAP: 0.3793
+ nDCG@10: 0.5286
+ R@1K: 0.8085
+ - name: bm25-doc-segmented-tuned
+ display: BM25 doc segmented (k1=2.16, b=0.61)
+ display-html: BM25 doc segmented (<i>k<sub><small>1</small></sub></i>=2.16, <i>b</i>=0.61)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-doc-segmented --topics $topics --output $output --bm25 --hits 10000 --max-passage-hits 1000 --max-passage
+ topics:
+ - topic_key: msmarco-doc-dev
+ eval_key: msmarco-doc-dev
+ scores:
+ - MRR@10: 0.2756
+ R@1K: 0.9311
+ - topic_key: dl19-doc
+ eval_key: dl19-doc
+ scores:
+ - MAP: 0.2398
+ nDCG@10: 0.5389
+ R@1K: 0.6565
+ - topic_key: dl20
+ eval_key: dl20-doc
+ scores:
+ - MAP: 0.3458
+ nDCG@10: 0.5213
+ R@1K: 0.7725
+ - name: bm25-doc-segmented-default
+ display: BM25 doc segmented (k1=0.9, b=0.4)
+ display-html: BM25 doc segmented (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
+ display-row: "[<a href=\"#\" data-mdb-toggle=\"tooltip\" title=\"Ma et al. (SIGIR 2021) Document Expansions and Learned Sparse Lexical Representations for MS MARCO V1 and V2.\">1</a>] &mdash; (1b)"
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-doc-segmented --topics $topics --output $output --bm25 --k1 0.9 --b 0.4 --hits 10000 --max-passage-hits 1000 --max-passage
+ topics:
+ - topic_key: msmarco-doc-dev
+ eval_key: msmarco-doc-dev
+ scores:
+ - MRR@10: 0.2684
+ R@1K: 0.9178
+ - topic_key: dl19-doc
+ eval_key: dl19-doc
+ scores:
+ - MAP: 0.2449
+ nDCG@10: 0.5302
+ R@1K: 0.6871
+ - topic_key: dl20
+ eval_key: dl20-doc
+ scores:
+ - MAP: 0.3586
+ nDCG@10: 0.5281
+ R@1K: 0.7755
+ - name: bm25-rm3-doc-tuned
+ display: BM25+RM3 doc (k1=4.46, b=0.82)
+ display-html: BM25+RM3 doc (<i>k<sub><small>1</small></sub></i>=4.46, <i>b</i>=0.82)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-doc --topics $topics --output $output --bm25 --rm3
+ topics:
+ - topic_key: msmarco-doc-dev
+ eval_key: msmarco-doc-dev
+ scores:
+ - MRR@10: 0.2227
+ R@1K: 0.9303
+ - topic_key: dl19-doc
+ eval_key: dl19-doc
+ scores:
+ - MAP: 0.2638
+ nDCG@10: 0.5526
+ R@1K: 0.7188
+ - topic_key: dl20
+ eval_key: dl20-doc
+ scores:
+ - MAP: 0.3610
+ nDCG@10: 0.5195
+ R@1K: 0.8180
+ - name: bm25-rm3-doc-default
+ display: BM25+RM3 doc (k1=0.9, b=0.4)
+ display-html: BM25+RM3 doc (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
+ display-row: "[<a href=\"#\" data-mdb-toggle=\"tooltip\" title=\"Ma et al. (SIGIR 2021) Document Expansions and Learned Sparse Lexical Representations for MS MARCO V1 and V2.\">1</a>] &mdash; (1c)"
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-doc --topics $topics --output $output --bm25 --rm3 --k1 0.9 --b 0.4
+ topics:
+ - topic_key: msmarco-doc-dev
+ eval_key: msmarco-doc-dev
+ scores:
+ - MRR@10: 0.1618
+ R@1K: 0.8783
+ - topic_key: dl19-doc
+ eval_key: dl19-doc
+ scores:
+ - MAP: 0.2773
+ nDCG@10: 0.5174
+ R@1K: 0.7507
+ - topic_key: dl20
+ eval_key: dl20-doc
+ scores:
+ - MAP: 0.4015
+ nDCG@10: 0.5254
+ R@1K: 0.8259
+ - name: bm25-rm3-doc-segmented-tuned
+ display: BM25+RM3 doc segmented (k1=2.16, b=0.61)
+ display-html: BM25+RM3 doc segmented (<i>k<sub><small>1</small></sub></i>=2.16, <i>b</i>=0.61)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-doc-segmented --topics $topics --output $output --bm25 --rm3 --hits 10000 --max-passage-hits 1000 --max-passage
+ topics:
+ - topic_key: msmarco-doc-dev
+ eval_key: msmarco-doc-dev
+ scores:
+ - MRR@10: 0.2448
+ R@1K: 0.9359
+ - topic_key: dl19-doc
+ eval_key: dl19-doc
+ scores:
+ - MAP: 0.2655
+ nDCG@10: 0.5392
+ R@1K: 0.7037
+ - topic_key: dl20
+ eval_key: dl20-doc
+ scores:
+ - MAP: 0.3471
+ nDCG@10: 0.5030
+ R@1K: 0.8056
+ - name: bm25-rm3-doc-segmented-default
+ display: BM25+RM3 doc segmented (k1=0.9, b=0.4)
+ display-html: BM25+RM3 doc segmented (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
+ display-row: "[<a href=\"#\" data-mdb-toggle=\"tooltip\" title=\"Ma et al. (SIGIR 2021) Document Expansions and Learned Sparse Lexical Representations for MS MARCO V1 and V2.\">1</a>] &mdash; (1d)"
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-doc-segmented --topics $topics --output $output --bm25 --rm3 --k1 0.9 --b 0.4 --hits 10000 --max-passage-hits 1000 --max-passage
+ topics:
+ - topic_key: msmarco-doc-dev
+ eval_key: msmarco-doc-dev
+ scores:
+ - MRR@10: 0.2413
+ R@1K: 0.9351
+ - topic_key: dl19-doc
+ eval_key: dl19-doc
+ scores:
+ - MAP: 0.2892
+ nDCG@10: 0.5684
+ R@1K: 0.7368
+ - topic_key: dl20
+ eval_key: dl20-doc
+ scores:
+ - MAP: 0.3792
+ nDCG@10: 0.5202
+ R@1K: 0.8023
+ - name: bm25-rocchio-doc-tuned
+ display: BM25+Rocchio doc (k1=4.46, b=0.82)
+ display-html: BM25+Rocchio doc (<i>k<sub><small>1</small></sub></i>=4.46, <i>b</i>=0.82)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-doc --topics $topics --output $output --bm25 --rocchio
+ topics:
+ - topic_key: msmarco-doc-dev
+ eval_key: msmarco-doc-dev
+ scores:
+ - MRR@10: 0.2242
+ R@1K: 0.9314
+ - topic_key: dl19-doc
+ eval_key: dl19-doc
+ scores:
+ - MAP: 0.2657
+ nDCG@10: 0.5584
+ R@1K: 0.7299
+ - topic_key: dl20
+ eval_key: dl20-doc
+ scores:
+ - MAP: 0.3628
+ nDCG@10: 0.5199
+ R@1K: 0.8217
+ - name: bm25-rocchio-doc-default
+ display: BM25+Rocchio doc (k1=0.9, b=0.4)
+ display-html: BM25+Rocchio doc (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-doc --topics $topics --output $output --bm25 --rocchio --k1 0.9 --b 0.4
+ topics:
+ - topic_key: msmarco-doc-dev
+ eval_key: msmarco-doc-dev
+ scores:
+ - MRR@10: 0.1624
+ R@1K: 0.8789
+ - topic_key: dl19-doc
+ eval_key: dl19-doc
+ scores:
+ - MAP: 0.2811
+ nDCG@10: 0.5256
+ R@1K: 0.7546
+ - topic_key: dl20
+ eval_key: dl20-doc
+ scores:
+ - MAP: 0.4089
+ nDCG@10: 0.5192
+ R@1K: 0.8273
+ - name: bm25-rocchio-doc-segmented-tuned
+ display: BM25+Rocchio doc segmented (k1=2.16, b=0.61)
+ display-html: BM25+Rocchio doc segmented (<i>k<sub><small>1</small></sub></i>=2.16, <i>b</i>=0.61)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-doc-segmented --topics $topics --output $output --bm25 --rocchio --hits 10000 --max-passage-hits 1000 --max-passage
+ topics:
+ - topic_key: msmarco-doc-dev
+ eval_key: msmarco-doc-dev
+ scores:
+ - MRR@10: 0.2475
+ R@1K: 0.9395
+ - topic_key: dl19-doc
+ eval_key: dl19-doc
+ scores:
+ - MAP: 0.2672
+ nDCG@10: 0.5421
+ R@1K: 0.7115
+ - topic_key: dl20
+ eval_key: dl20-doc
+ scores:
+ - MAP: 0.3521
+ nDCG@10: 0.4997
+ R@1K: 0.8042
+ - name: bm25-rocchio-doc-segmented-default
+ display: BM25+Rocchio doc segmented (k1=0.9, b=0.4)
+ display-html: BM25+Rocchio doc segmented (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-doc-segmented --topics $topics --output $output --bm25 --rocchio --k1 0.9 --b 0.4 --hits 10000 --max-passage-hits 1000 --max-passage
+ topics:
+ - topic_key: msmarco-doc-dev
+ eval_key: msmarco-doc-dev
+ scores:
+ - MRR@10: 0.2447
+ R@1K: 0.9351
+ - topic_key: dl19-doc
+ eval_key: dl19-doc
+ scores:
+ - MAP: 0.2889
+ nDCG@10: 0.5570
+ R@1K: 0.7423
+ - topic_key: dl20
+ eval_key: dl20-doc
+ scores:
+ - MAP: 0.3830
+ nDCG@10: 0.5226
+ R@1K: 0.8102
+ - name: bm25-d2q-t5-doc-tuned
+ display: BM25 w/ doc2query-T5 doc (k1=4.68, b=0.87)
+ display-html: BM25 w/ doc2query-T5 doc (<i>k<sub><small>1</small></sub></i>=4.68, <i>b</i>=0.87)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-doc-d2q-t5 --topics $topics --output $output --bm25
+ topics:
+ - topic_key: msmarco-doc-dev
+ eval_key: msmarco-doc-dev
+ scores:
+ - MRR@10: 0.3269
+ R@1K: 0.9553
+ - topic_key: dl19-doc
+ eval_key: dl19-doc
+ scores:
+ - MAP: 0.2620
+ nDCG@10: 0.5972
+ R@1K: 0.6867
+ - topic_key: dl20
+ eval_key: dl20-doc
+ scores:
+ - MAP: 0.4099
+ nDCG@10: 0.5852
+ R@1K: 0.8105
+ - name: bm25-d2q-t5-doc-default
+ display: BM25 w/ doc2query-T5 doc (k1=0.9, b=0.4)
+ display-html: BM25 w/ doc2query-T5 doc (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
+ display-row: "[<a href=\"#\" data-mdb-toggle=\"tooltip\" title=\"Ma et al. (SIGIR 2021) Document Expansions and Learned Sparse Lexical Representations for MS MARCO V1 and V2.\">1</a>] &mdash; (2a)"
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-doc-d2q-t5 --topics $topics --output $output --bm25 --k1 0.9 --b 0.4
+ topics:
+ - topic_key: msmarco-doc-dev
+ eval_key: msmarco-doc-dev
+ scores:
+ - MRR@10: 0.2880
+ R@1K: 0.9259
+ - topic_key: dl19-doc
+ eval_key: dl19-doc
+ scores:
+ - MAP: 0.2700
+ nDCG@10: 0.5968
+ R@1K: 0.7190
+ - topic_key: dl20
+ eval_key: dl20-doc
+ scores:
+ - MAP: 0.4230
+ nDCG@10: 0.5885
+ R@1K: 0.8403
+ - name: bm25-d2q-t5-doc-segmented-tuned
+ display: BM25 w/ doc2query-T5 doc segmented (k1=2.56, b=0.59)
+ display-html: BM25 w/ doc2query-T5 doc segmented (<i>k<sub><small>1</small></sub></i>=2.56, <i>b</i>=0.59)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-doc-segmented-d2q-t5 --topics $topics --output $output --bm25 --hits 10000 --max-passage-hits 1000 --max-passage
+ topics:
+ - topic_key: msmarco-doc-dev
+ eval_key: msmarco-doc-dev
+ scores:
+ - MRR@10: 0.3209
+ R@1K: 0.9530
+ - topic_key: dl19-doc
+ eval_key: dl19-doc
+ scores:
+ - MAP: 0.2658
+ nDCG@10: 0.6273
+ R@1K: 0.6707
+ - topic_key: dl20
+ eval_key: dl20-doc
+ scores:
+ - MAP: 0.4047
+ nDCG@10: 0.5943
+ R@1K: 0.7968
+ - name: bm25-d2q-t5-doc-segmented-default
+ display: BM25 w/ doc2query-T5 doc segmented (k1=0.9, b=0.4)
+ display-html: BM25 w/ doc2query-T5 doc segmented (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
+ display-row: "[<a href=\"#\" data-mdb-toggle=\"tooltip\" title=\"Ma et al. (SIGIR 2021) Document Expansions and Learned Sparse Lexical Representations for MS MARCO V1 and V2.\">1</a>] &mdash; (2b)"
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-doc-segmented-d2q-t5 --topics $topics --output $output --bm25 --k1 0.9 --b 0.4 --hits 10000 --max-passage-hits 1000 --max-passage
+ topics:
+ - topic_key: msmarco-doc-dev
+ eval_key: msmarco-doc-dev
+ scores:
+ - MRR@10: 0.3179
+ R@1K: 0.9490
+ - topic_key: dl19-doc
+ eval_key: dl19-doc
+ scores:
+ - MAP: 0.2798
+ nDCG@10: 0.6119
+ R@1K: 0.7165
+ - topic_key: dl20
+ eval_key: dl20-doc
+ scores:
+ - MAP: 0.4150
+ nDCG@10: 0.5957
+ R@1K: 0.8046
+ - name: bm25-rm3-d2q-t5-doc-tuned
+ display: BM25+RM3 w/ doc2query-T5 doc (k1=4.68, b=0.87)
+ display-html: BM25+RM3 w/ doc2query-T5 doc (<i>k<sub><small>1</small></sub></i>=4.68, <i>b</i>=0.87)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-doc-d2q-t5-docvectors --topics $topics --output $output --bm25 --rm3
+ topics:
+ - topic_key: msmarco-doc-dev
+ eval_key: msmarco-doc-dev
+ scores:
+ - MRR@10: 0.2623
+ R@1K: 0.9522
+ - topic_key: dl19-doc
+ eval_key: dl19-doc
+ scores:
+ - MAP: 0.2813
+ nDCG@10: 0.6091
+ R@1K: 0.7184
+ - topic_key: dl20
+ eval_key: dl20-doc
+ scores:
+ - MAP: 0.4100
+ nDCG@10: 0.5745
+ R@1K: 0.8238
+ - name: bm25-rm3-d2q-t5-doc-default
+ display: BM25+RM3 w/ doc2query-T5 doc (k1=0.9, b=0.4)
+ display-html: BM25+RM3 w/ doc2query-T5 doc (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
+ display-row: "[<a href=\"#\" data-mdb-toggle=\"tooltip\" title=\"Ma et al. (SIGIR 2021) Document Expansions and Learned Sparse Lexical Representations for MS MARCO V1 and V2.\">1</a>] &mdash; (2c)"
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-doc-d2q-t5-docvectors --topics $topics --output $output --bm25 --rm3 --k1 0.9 --b 0.4
+ topics:
+ - topic_key: msmarco-doc-dev
+ eval_key: msmarco-doc-dev
+ scores:
+ - MRR@10: 0.1834
+ R@1K: 0.9126
+ - topic_key: dl19-doc
+ eval_key: dl19-doc
+ scores:
+ - MAP: 0.3045
+ nDCG@10: 0.5904
+ R@1K: 0.7737
+ - topic_key: dl20
+ eval_key: dl20-doc
+ scores:
+ - MAP: 0.4230
+ nDCG@10: 0.5427
+ R@1K: 0.8631
+ - name: bm25-rm3-d2q-t5-doc-segmented-tuned
+ display: BM25+RM3 w/ doc2query-T5 doc segmented (k1=2.56, b=0.59)
+ display-html: BM25+RM3 w/ doc2query-T5 doc segmented (<i>k<sub><small>1</small></sub></i>=2.56, <i>b</i>=0.59)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-doc-segmented-d2q-t5-docvectors --topics $topics --output $output --bm25 --rm3 --hits 10000 --max-passage-hits 1000 --max-passage
+ topics:
+ - topic_key: msmarco-doc-dev
+ eval_key: msmarco-doc-dev
+ scores:
+ - MRR@10: 0.2973
+ R@1K: 0.9563
+ - topic_key: dl19-doc
+ eval_key: dl19-doc
+ scores:
+ - MAP: 0.2892
+ nDCG@10: 0.6247
+ R@1K: 0.7069
+ - topic_key: dl20
+ eval_key: dl20-doc
+ scores:
+ - MAP: 0.4016
+ nDCG@10: 0.5711
+ R@1K: 0.8156
+ - name: bm25-rm3-d2q-t5-doc-segmented-default
+ display: BM25+RM3 w/ doc2query-T5 doc segmented (k1=0.9, b=0.4)
+ display-html: BM25+RM3 w/ doc2query-T5 doc segmented (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
+ display-row: "[<a href=\"#\" data-mdb-toggle=\"tooltip\" title=\"Ma et al. (SIGIR 2021) Document Expansions and Learned Sparse Lexical Representations for MS MARCO V1 and V2.\">1</a>] &mdash; (2d)"
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-doc-segmented-d2q-t5-docvectors --topics $topics --output $output --bm25 --rm3 --k1 0.9 --b 0.4 --hits 10000 --max-passage-hits 1000 --max-passage
+ topics:
+ - topic_key: msmarco-doc-dev
+ eval_key: msmarco-doc-dev
+ scores:
+ - MRR@10: 0.2803
+ R@1K: 0.9551
+ - topic_key: dl19-doc
+ eval_key: dl19-doc
+ scores:
+ - MAP: 0.3030
+ nDCG@10: 0.6290
+ R@1K: 0.7483
+ - topic_key: dl20
+ eval_key: dl20-doc
+ scores:
+ - MAP: 0.4271
+ nDCG@10: 0.5851
+ R@1K: 0.8266
+ - name: unicoil-noexp-pytorch
+ display: "uniCOIL (noexp): query inference with PyTorch"
+ display-html: "uniCOIL (noexp): query inference with PyTorch"
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-doc-segmented-unicoil-noexp --topics $topics --encoder castorini/unicoil-noexp-msmarco-passage --output $output --impact --hits 10000 --max-passage-hits 1000 --max-passage
+ topics:
+ - topic_key: msmarco-doc-dev
+ eval_key: msmarco-doc-dev
+ scores:
+ - MRR@10: 0.3410
+ R@1K: 0.9420
+ - topic_key: dl19-doc
+ eval_key: dl19-doc
+ scores:
+ - MAP: 0.2661
+ nDCG@10: 0.6347
+ R@1K: 0.6385
+ - topic_key: dl20
+ eval_key: dl20-doc
+ scores:
+ - MAP: 0.3698
+ nDCG@10: 0.5906
+ R@1K: 0.7621
+ - name: unicoil-noexp
+ display: "uniCOIL (noexp): pre-encoded"
+ display-html: "uniCOIL (noexp): pre-encoded queries"
+ display-row: "[<a href=\"#\" data-mdb-toggle=\"tooltip\" title=\"Ma et al. (SIGIR 2021) Document Expansions and Learned Sparse Lexical Representations for MS MARCO V1 and V2.\">1</a>] &mdash; (3a)"
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-doc-segmented-unicoil-noexp --topics $topics --output $output --impact --hits 10000 --max-passage-hits 1000 --max-passage
+ topics:
+ - topic_key: msmarco-doc-dev-unicoil-noexp
+ eval_key: msmarco-doc-dev
+ scores:
+ - MRR@10: 0.3409
+ R@1K: 0.9420
+ - topic_key: dl19-doc-unicoil-noexp
+ eval_key: dl19-doc
+ scores:
+ - MAP: 0.2665
+ nDCG@10: 0.6349
+ R@1K: 0.6391
+ - topic_key: dl20-unicoil-noexp
+ eval_key: dl20-doc
+ scores:
+ - MAP: 0.3698
+ nDCG@10: 0.5893
+ R@1K: 0.7623
+ - name: unicoil-pytorch
+ display: "uniCOIL (w/ doc2query-T5): query inference with PyTorch"
+ display-html: "uniCOIL (w/ doc2query-T5): query inference with PyTorch"
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-doc-segmented-unicoil --topics $topics --encoder castorini/unicoil-msmarco-passage --output $output --impact --hits 10000 --max-passage-hits 1000 --max-passage
+ topics:
+ - topic_key: msmarco-doc-dev
+ eval_key: msmarco-doc-dev
+ scores:
+ - MRR@10: 0.3532
+ R@1K: 0.9546
+ - topic_key: dl19-doc
+ eval_key: dl19-doc
+ scores:
+ - MAP: 0.2789
+ nDCG@10: 0.6396
+ R@1K: 0.6654
+ - topic_key: dl20
+ eval_key: dl20-doc
+ scores:
+ - MAP: 0.3881
+ nDCG@10: 0.6030
+ R@1K: 0.7866
+ - name: unicoil
+ display: "uniCOIL (w/ doc2query-T5): pre-encoded"
+ display-html: "uniCOIL (w/ doc2query-T5): pre-encoded queries"
+ display-row: "[<a href=\"#\" data-mdb-toggle=\"tooltip\" title=\"Ma et al. (SIGIR 2021) Document Expansions and Learned Sparse Lexical Representations for MS MARCO V1 and V2.\">1</a>] &mdash; (3b)"
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-doc-segmented-unicoil --topics $topics --output $output --impact --hits 10000 --max-passage-hits 1000 --max-passage
+ topics:
+ - topic_key: msmarco-doc-dev-unicoil
+ eval_key: msmarco-doc-dev
+ scores:
+ - MRR@10: 0.3531
+ R@1K: 0.9546
+ - topic_key: dl19-doc-unicoil
+ eval_key: dl19-doc
+ scores:
+ - MAP: 0.2789
+ nDCG@10: 0.6396
+ R@1K: 0.6652
+ - topic_key: dl20-unicoil
+ eval_key: dl20-doc
+ scores:
+ - MAP: 0.3882
+ nDCG@10: 0.6033
+ R@1K: 0.7869

pyserini/2cr/msmarco-v1-passage.yaml

@@ -0,0 +1,764 @@
+conditions:
+ - name: bm25-rocchio-d2q-t5-tuned
+ display: BM25+Rocchio w/ doc2query-T5 (k1=2.18, b=0.86)
+ display-html: BM25+Rocchio w/ doc2query-T5 (<i>k<sub><small>1</small></sub></i>=2.18, <i>b</i>=0.86)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-passage-d2q-t5-docvectors --topics $topics --output $output --bm25 --rocchio
+ topics:
+ - topic_key: msmarco-passage-dev-subset
+ eval_key: msmarco-passage-dev-subset
+ scores:
+ - MRR@10: 0.2395
+ R@1K: 0.9535
+ - topic_key: dl19-passage
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.4339
+ nDCG@10: 0.6559
+ R@1K: 0.8465
+ - topic_key: dl20
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.4376
+ nDCG@10: 0.6224
+ R@1K: 0.8641
+ - name: bm25-rocchio-d2q-t5-default
+ display: BM25+Rocchio w/ doc2query-T5 (k1=0.9, b=0.4)
+ display-html: BM25+Rocchio w/ doc2query-T5 (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-passage-d2q-t5-docvectors --topics $topics --output $output --bm25 --rocchio --k1 0.9 --b 0.4
+ topics:
+ - topic_key: msmarco-passage-dev-subset
+ eval_key: msmarco-passage-dev-subset
+ scores:
+ - MRR@10: 0.2158
+ R@1K: 0.9467
+ - topic_key: dl19-passage
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.4469
+ nDCG@10: 0.6538
+ R@1K: 0.8855
+ - topic_key: dl20
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.4246
+ nDCG@10: 0.6102
+ R@1K: 0.8675
+ - name: bm25-rocchio-default
+ display: BM25+Rocchio (k1=0.9, b=0.4)
+ display-html: BM25+Rocchio (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-passage --topics $topics --output $output --bm25 --k1 0.9 --b 0.4 --rocchio
+ topics:
+ - topic_key: msmarco-passage-dev-subset
+ eval_key: msmarco-passage-dev-subset
+ scores:
+ - MRR@10: 0.1595
+ R@1K: 0.8620
+ - topic_key: dl19-passage
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.3474
+ nDCG@10: 0.5275
+ R@1K: 0.8007
+ - topic_key: dl20
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.3115
+ nDCG@10: 0.4910
+ R@1K: 0.8156
+ - name: bm25-rocchio-tuned
+ display: BM25+Rocchio (k1=0.82, b=0.68)
+ display-html: BM25+Rocchio (<i>k<sub><small>1</small></sub></i>=0.82, <i>b</i>=0.68)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-passage --topics $topics --output $output --bm25 --rocchio
+ topics:
+ - topic_key: msmarco-passage-dev-subset
+ eval_key: msmarco-passage-dev-subset
+ scores:
+ - MRR@10: 0.1684
+ R@1K: 0.8726
+ - topic_key: dl19-passage
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.3396
+ nDCG@10: 0.5275
+ R@1K: 0.7948
+ - topic_key: dl20
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.3120
+ nDCG@10: 0.4908
+ R@1K: 0.8327
+ - name: distilbert-kd-tasb-pytorch
+ display: "DistilBERT KD TASB: query inference with PyTorch"
+ display-html: "DistilBERT KD TASB: query inference with PyTorch"
+ display-row: "[5]"
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --index msmarco-v1-passage.distilbert-dot-tas_b-b256 --topics $topics --encoder sebastian-hofstaetter/distilbert-dot-tas_b-b256-msmarco --output $output
+ topics:
+ - topic_key: msmarco-passage-dev-subset
+ eval_key: msmarco-passage-dev-subset
+ scores:
+ - MRR@10: 0.3444
+ R@1K: 0.9771
+ - topic_key: dl19-passage
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.4590
+ nDCG@10: 0.7210
+ R@1K: 0.8406
+ - topic_key: dl20
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.4698
+ nDCG@10: 0.6854
+ R@1K: 0.8727
+ - name: distilbert-kd-tasb
+ display: "DistilBERT KD TASB: pre-encoded"
+ display-html: "DistilBERT KD TASB: pre-encoded queries"
+ display-row: "[5]"
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --index msmarco-v1-passage.distilbert-dot-tas_b-b256 --topics $topics --encoded-queries distilbert_tas_b-$topics --output $output
+ topics:
+ - topic_key: msmarco-passage-dev-subset
+ eval_key: msmarco-passage-dev-subset
+ scores:
+ - MRR@10: 0.3444
+ R@1K: 0.9771
+ - topic_key: dl19-passage
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.4590
+ nDCG@10: 0.7210
+ R@1K: 0.8406
+ - topic_key: dl20
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.4698
+ nDCG@10: 0.6854
+ R@1K: 0.8727
+ - name: distilbert-kd-pytorch
+ display: "DistilBERT KD: query inference with PyTorch"
+ display-html: "DistilBERT KD: query inference with PyTorch"
+ display-row: "[4]"
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --index msmarco-v1-passage.distilbert-dot-margin-mse-t2 --topics $topics --encoder sebastian-hofstaetter/distilbert-dot-margin_mse-T2-msmarco --output $output
+ topics:
+ - topic_key: msmarco-passage-dev-subset
+ eval_key: msmarco-passage-dev-subset
+ scores:
+ - MRR@10: 0.3251
+ R@1K: 0.9553
+ - topic_key: dl19-passage
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.4053
+ nDCG@10: 0.6994
+ R@1K: 0.7653
+ - topic_key: dl20
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.4159
+ nDCG@10: 0.6447
+ R@1K: 0.7953
+ - name: distilbert-kd
+ display: "DistilBERT KD: pre-encoded"
+ display-html: "DistilBERT KD: pre-encoded queries"
+ display-row: "[4]"
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --index msmarco-v1-passage.distilbert-dot-margin-mse-t2 --topics $topics --encoded-queries distilbert_kd-$topics --output $output
+ topics:
+ - topic_key: msmarco-passage-dev-subset
+ eval_key: msmarco-passage-dev-subset
+ scores:
+ - MRR@10: 0.3251
+ R@1K: 0.9553
+ - topic_key: dl19-passage
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.4053
+ nDCG@10: 0.6994
+ R@1K: 0.7653
+ - topic_key: dl20
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.4159
+ nDCG@10: 0.6447
+ R@1K: 0.7953
+ - name: ance-pytorch
+ display: "ANCE: query inference with PyTorch"
+ display-html: "ANCE: query inference with PyTorch"
+ display-row: "[3]"
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --index msmarco-v1-passage.ance --topics $topics --encoder castorini/ance-msmarco-passage --output $output
+ topics:
+ - topic_key: msmarco-passage-dev-subset
+ eval_key: msmarco-passage-dev-subset
+ scores:
+ - MRR@10: 0.3302
+ R@1K: 0.9587
+ - topic_key: dl19-passage
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.3710
+ nDCG@10: 0.6452
+ R@1K: 0.7554
+ - topic_key: dl20
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.4076
+ nDCG@10: 0.6458
+ R@1K: 0.7764
+ - name: ance
+ display: "ANCE: pre-encoded"
+ display-html: "ANCE: pre-encoded queries"
+ display-row: "[3]"
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --index msmarco-v1-passage.ance --topics $topics --encoded-queries ance-$topics --output $output
+ topics:
+ - topic_key: msmarco-passage-dev-subset
+ eval_key: msmarco-passage-dev-subset
+ scores:
+ - MRR@10: 0.3302
+ R@1K: 0.9584
+ - topic_key: dl19-passage
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.3710
+ nDCG@10: 0.6452
+ R@1K: 0.7554
+ - topic_key: dl20
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.4076
+ nDCG@10: 0.6458
+ R@1K: 0.7764
+ - name: bm25-tuned
+ display: BM25 (k1=0.82, b=0.68)
+ display-html: BM25 (<i>k<sub><small>1</small></sub></i>=0.82, <i>b</i>=0.68)
+ command: python -m pyserini.search.lucene --topics $topics --index msmarco-v1-passage --output $output --bm25
+ topics:
+ - topic_key: msmarco-passage-dev-subset
+ eval_key: msmarco-passage-dev-subset
+ scores:
+ - MRR@10: 0.1875
+ R@1K: 0.8573
+ - topic_key: dl19-passage
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.2903
+ nDCG@10: 0.4973
+ R@1K: 0.7450
+ - topic_key: dl20
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.2876
+ nDCG@10: 0.4876
+ R@1K: 0.8031
+ - name: bm25-rm3-tuned
+ display: BM25+RM3 (k1=0.82, b=0.68)
+ display-html: BM25+RM3 (<i>k<sub><small>1</small></sub></i>=0.82, <i>b</i>=0.68)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-passage --topics $topics --output $output --bm25 --rm3
+ topics:
+ - topic_key: msmarco-passage-dev-subset
+ eval_key: msmarco-passage-dev-subset
+ scores:
+ - MRR@10: 0.1646
+ R@1K: 0.8704
+ - topic_key: dl19-passage
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.3339
+ nDCG@10: 0.5147
+ R@1K: 0.7950
+ - topic_key: dl20
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.3017
+ nDCG@10: 0.4924
+ R@1K: 0.8292
+ - name: bm25-default
+ display: BM25 (k1=0.9, b=0.4)
+ display-html: BM25 (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
+ display-row: "[<a href=\"#\" data-mdb-toggle=\"tooltip\" title=\"Ma et al. (SIGIR 2021) Document Expansions and Learned Sparse Lexical Representations for MS MARCO V1 and V2.\">1</a>] &mdash; (1a)"
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-passage --topics $topics --output $output --bm25 --k1 0.9 --b 0.4
+ topics:
+ - topic_key: msmarco-passage-dev-subset
+ eval_key: msmarco-passage-dev-subset
+ scores:
+ - MRR@10: 0.1840
+ R@1K: 0.8526
+ - topic_key: dl19-passage
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.3013
+ nDCG@10: 0.5058
+ R@1K: 0.7501
+ - topic_key: dl20
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.2856
+ nDCG@10: 0.4796
+ R@1K: 0.7863
+ - name: bm25-rm3-default
+ display: BM25+RM3 (k1=0.9, b=0.4)
+ display-html: BM25+RM3 (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
+ display-row: "[<a href=\"#\" data-mdb-toggle=\"tooltip\" title=\"Ma et al. (SIGIR 2021) Document Expansions and Learned Sparse Lexical Representations for MS MARCO V1 and V2.\">1</a>] &mdash; (1b)"
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-passage --topics $topics --output $output --bm25 --k1 0.9 --b 0.4 --rm3
+ topics:
+ - topic_key: msmarco-passage-dev-subset
+ eval_key: msmarco-passage-dev-subset
+ scores:
+ - MRR@10: 0.1566
+ R@1K: 0.8606
+ - topic_key: dl19-passage
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.3416
+ nDCG@10: 0.5216
+ R@1K: 0.8136
+ - topic_key: dl20
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.3006
+ nDCG@10: 0.4896
+ R@1K: 0.8236
+ - name: bm25-d2q-t5-tuned
+ display: BM25 w/ doc2query-T5 (k1=2.18, b=0.86)
+ display-html: BM25 w/ doc2query-T5 (<i>k<sub><small>1</small></sub></i>=2.18, <i>b</i>=0.86)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-passage-d2q-t5 --topics $topics --output $output --bm25
+ topics:
+ - topic_key: msmarco-passage-dev-subset
+ eval_key: msmarco-passage-dev-subset
+ scores:
+ - MRR@10: 0.2816
+ R@1K: 0.9506
+ - topic_key: dl19-passage
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.4046
+ nDCG@10: 0.6336
+ R@1K: 0.8134
+ - topic_key: dl20
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.4171
+ nDCG@10: 0.6265
+ R@1K: 0.8393
+ - name: bm25-d2q-t5-default
+ display: BM25 w/ doc2query-T5 (k1=0.9, b=0.4)
+ display-html: BM25 w/ doc2query-T5 (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
+ display-row: "[<a href=\"#\" data-mdb-toggle=\"tooltip\" title=\"Ma et al. (SIGIR 2021) Document Expansions and Learned Sparse Lexical Representations for MS MARCO V1 and V2.\">1</a>] &mdash; (2a)"
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-passage-d2q-t5 --topics $topics --output $output --bm25 --k1 0.9 --b 0.4
+ topics:
+ - topic_key: msmarco-passage-dev-subset
+ eval_key: msmarco-passage-dev-subset
+ scores:
+ - MRR@10: 0.2723
+ R@1K: 0.9470
+ - topic_key: dl19-passage
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.4034
+ nDCG@10: 0.6417
+ R@1K: 0.8310
+ - topic_key: dl20
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.4074
+ nDCG@10: 0.6187
+ R@1K: 0.8452
+ - name: bm25-rm3-d2q-t5-tuned
+ display: BM25+RM3 w/ doc2query-T5 (k1=2.18, b=0.86)
+ display-html: BM25+RM3 w/ doc2query-T5 (<i>k<sub><small>1</small></sub></i>=2.18, <i>b</i>=0.86)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-passage-d2q-t5-docvectors --topics $topics --output $output --bm25 --rm3
+ topics:
+ - topic_key: msmarco-passage-dev-subset
+ eval_key: msmarco-passage-dev-subset
+ scores:
+ - MRR@10: 0.2382
+ R@1K: 0.9528
+ - topic_key: dl19-passage
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.4377
+ nDCG@10: 0.6537
+ R@1K: 0.8443
+ - topic_key: dl20
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.4348
+ nDCG@10: 0.6235
+ R@1K: 0.8605
+ - name: bm25-rm3-d2q-t5-default
+ display: BM25+RM3 w/ doc2query-T5 (k1=0.9, b=0.4)
+ display-html: BM25+RM3 w/ doc2query-T5 (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
+ display-row: "[<a href=\"#\" data-mdb-toggle=\"tooltip\" title=\"Ma et al. (SIGIR 2021) Document Expansions and Learned Sparse Lexical Representations for MS MARCO V1 and V2.\">1</a>] &mdash; (2b)"
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-passage-d2q-t5-docvectors --topics $topics --output $output --bm25 --rm3 --k1 0.9 --b 0.4
+ topics:
+ - topic_key: msmarco-passage-dev-subset
+ eval_key: msmarco-passage-dev-subset
+ scores:
+ - MRR@10: 0.2139
+ R@1K: 0.9460
+ - topic_key: dl19-passage
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.4483
+ nDCG@10: 0.6586
+ R@1K: 0.8863
+ - topic_key: dl20
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.4286
+ nDCG@10: 0.6131
+ R@1K: 0.8700
+ - name: unicoil-pytorch
+ display: "uniCOIL (w/ doc2query-T5): query inference with PyTorch"
+ display-html: "uniCOIL (w/ doc2query-T5): query inference with PyTorch"
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-passage-unicoil --topics $topics --encoder castorini/unicoil-msmarco-passage --output $output --hits 1000 --impact
+ topics:
+ - topic_key: msmarco-passage-dev-subset
+ eval_key: msmarco-passage-dev-subset
+ scores:
+ - MRR@10: 0.3509
+ R@1K: 0.9581
+ - topic_key: dl19-passage
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.4617
+ nDCG@10: 0.7027
+ R@1K: 0.8291
+ - topic_key: dl20
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.4429
+ nDCG@10: 0.6745
+ R@1K: 0.8433
+ - name: unicoil-onnx
+ display: "uniCOIL (w/ doc2query-T5): query inference with ONNX"
+ display-html: "uniCOIL (w/ doc2query-T5): query inference with ONNX"
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-passage-unicoil --topics $topics --onnx-encoder UniCoil --output $output --hits 1000 --impact
+ topics:
+ - topic_key: msmarco-passage-dev-subset
+ eval_key: msmarco-passage-dev-subset
+ scores:
+ - MRR@10: 0.3509
+ R@1K: 0.9581
+ - topic_key: dl19-passage
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.4617
+ nDCG@10: 0.7027
+ R@1K: 0.8291
+ - topic_key: dl20
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.4429
+ nDCG@10: 0.6745
+ R@1K: 0.8433
+ - name: unicoil
+ display: "uniCOIL (w/ doc2query-T5): pre-encoded"
+ display-html: "uniCOIL (w/ doc2query-T5): pre-encoded queries"
+ display-row: "[<a href=\"#\" data-mdb-toggle=\"tooltip\" title=\"Ma et al. (SIGIR 2021) Document Expansions and Learned Sparse Lexical Representations for MS MARCO V1 and V2.\">1</a>] &mdash; (3b)"
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-passage-unicoil --topics $topics --output $output --hits 1000 --impact
+ topics:
+ - topic_key: msmarco-passage-dev-subset-unicoil
+ eval_key: msmarco-passage-dev-subset
+ scores:
+ - MRR@10: 0.3516
+ R@1K: 0.9582
+ - topic_key: dl19-passage-unicoil
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.4612
+ nDCG@10: 0.7024
+ R@1K: 0.8292
+ - topic_key: dl20-unicoil
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.4430
+ nDCG@10: 0.6745
+ R@1K: 0.8430
+ - name: unicoil-noexp-pytorch
+ display: "uniCOIL (noexp): query inference with PyTorch"
+ display-html: "uniCOIL (noexp): query inference with PyTorch"
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-passage-unicoil-noexp --topics $topics --encoder castorini/unicoil-noexp-msmarco-passage --output $output --hits 1000 --impact
+ topics:
+ - topic_key: msmarco-passage-dev-subset
+ eval_key: msmarco-passage-dev-subset
+ scores:
+ - MRR@10: 0.3153
+ R@1K: 0.9239
+ - topic_key: dl19-passage
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.4033
+ nDCG@10: 0.6434
+ R@1K: 0.7752
+ - topic_key: dl20
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.4022
+ nDCG@10: 0.6524
+ R@1K: 0.7861
+ - name: unicoil-noexp-onnx
+ display: "uniCOIL (noexp): query inference with ONNX"
+ display-html: "uniCOIL (noexp): query inference with ONNX"
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-passage-unicoil-noexp --topics $topics --onnx-encoder UniCoil --output $output --hits 1000 --impact
+ topics:
+ - topic_key: msmarco-passage-dev-subset
+ eval_key: msmarco-passage-dev-subset
+ scores:
+ - MRR@10: 0.3119
+ R@1K: 0.9239
+ - topic_key: dl19-passage
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.4061
+ nDCG@10: 0.6531
+ R@1K: 0.7809
+ - topic_key: dl20
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.3909
+ nDCG@10: 0.6388
+ R@1K: 0.7915
+ - name: unicoil-noexp
+ display: "uniCOIL (noexp): pre-encoded"
+ display-html: "uniCOIL (noexp): pre-encoded queries"
+ display-row: "[<a href=\"#\" data-mdb-toggle=\"tooltip\" title=\"Ma et al. (SIGIR 2021) Document Expansions and Learned Sparse Lexical Representations for MS MARCO V1 and V2.\">1</a>] &mdash; (3a)"
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-passage-unicoil-noexp --topics $topics --output $output --hits 1000 --impact
+ topics:
+ - topic_key: msmarco-passage-dev-subset-unicoil-noexp
+ eval_key: msmarco-passage-dev-subset
+ scores:
+ - MRR@10: 0.3153
+ R@1K: 0.9239
+ - topic_key: dl19-passage-unicoil-noexp
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.4033
+ nDCG@10: 0.6433
+ R@1K: 0.7752
+ - topic_key: dl20-unicoil-noexp
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.4021
+ nDCG@10: 0.6523
+ R@1K: 0.7861
+ - name: splade-pp-ed-onnx
+ display: "SPLADE++ EnsembleDistil: query inference with ONNX"
+ display-html: "SPLADE++ EnsembleDistil: query inference with ONNX"
+ display-row: "[2]"
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-passage-splade-pp-ed --topics $topics --onnx-encoder SpladePlusPlusEnsembleDistil --output $output --hits 1000 --impact
+ topics:
+ - topic_key: msmarco-passage-dev-subset
+ eval_key: msmarco-passage-dev-subset
+ scores:
+ - MRR@10: 0.3830
+ R@1K: 0.9831
+ - topic_key: dl19-passage
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.5054
+ nDCG@10: 0.7320
+ R@1K: 0.8724
+ - topic_key: dl20
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.5002
+ nDCG@10: 0.7198
+ R@1K: 0.8995
+ - name: splade-pp-sd-onnx
+ display: "SPLADE++ SelfDistil: query inference with ONNX"
+ display-html: "SPLADE++ SelfDistil: query inference with ONNX"
+ display-row: "[2]"
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v1-passage-splade-pp-sd --topics $topics --onnx-encoder SpladePlusPlusSelfDistil --output $output --hits 1000 --impact
+ topics:
+ - topic_key: msmarco-passage-dev-subset
+ eval_key: msmarco-passage-dev-subset
+ scores:
+ - MRR@10: 0.3778
+ R@1K: 0.9846
+ - topic_key: dl19-passage
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.4997
+ nDCG@10: 0.7356
+ R@1K: 0.8758
+ - topic_key: dl20
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.5140
+ nDCG@10: 0.7285
+ R@1K: 0.9023 
+ - name: tct_colbert-v2-hnp-pytorch
+ display: "TCT_ColBERT-V2-HN+: query inference with PyTorch"
+ display-html: "TCT_ColBERT-V2-HN+: query inference with PyTorch"
+ display-row: "[6]"
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --index msmarco-v1-passage.tct_colbert-v2-hnp --topics $topics --encoder castorini/tct_colbert-v2-hnp-msmarco --output $output
+ topics:
+ - topic_key: msmarco-passage-dev-subset
+ eval_key: msmarco-passage-dev-subset
+ scores:
+ - MRR@10: 0.3584
+ R@1K: 0.9695
+ - topic_key: dl19-passage
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.4469
+ nDCG@10: 0.7204
+ R@1K: 0.8261
+ - topic_key: dl20
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.4754
+ nDCG@10: 0.6882
+ R@1K: 0.8429
+ - name: tct_colbert-v2-hnp
+ display: "TCT_ColBERT-V2-HN+: pre-encoded"
+ display-html: "TCT_ColBERT-V2-HN+: pre-encoded queries"
+ display-row: "[6]"
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --index msmarco-v1-passage.tct_colbert-v2-hnp --topics $topics --encoded-queries tct_colbert-v2-hnp-$topics --output $output
+ topics:
+ - topic_key: msmarco-passage-dev-subset
+ eval_key: msmarco-passage-dev-subset
+ scores:
+ - MRR@10: 0.3584
+ R@1K: 0.9695
+ - topic_key: dl19-passage
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.4469
+ nDCG@10: 0.7204
+ R@1K: 0.8261
+ - topic_key: dl20
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.4754
+ nDCG@10: 0.6882
+ R@1K: 0.8429
+ - name: slimr
+ display: "SLIM: query inference with PyTorch"
+ display-html: "SLIM: query inference with PyTorch"
+ display-row: "[7]"
+ command: python -m pyserini.search.lucene --threads 16 --batch 128 --index msmarco-v1-passage-slimr --topics $topics --encoder castorini/slimr-msmarco-passage --encoded-corpus scipy-sparse-vectors.msmarco-v1-passage-slimr --output $output --output-format msmarco --hits 1000 --impact --min-idf 3
+ topics:
+ - topic_key: msmarco-passage-dev-subset
+ eval_key: msmarco-passage-dev-subset
+ scores:
+ - MRR@10: 0.3581
+ R@1K: 0.9620
+ - topic_key: dl19-passage
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.4509
+ nDCG@10: 0.7010
+ R@1K: 0.8241
+ - topic_key: dl20
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.4419
+ nDCG@10: 0.6403
+ R@1K: 0.8543
+ - name: slimr-pp
+ display: "SLIM++: query inference with PyTorch"
+ display-html: "SLIM++: query inference with PyTorch"
+ display-row: "[7]"
+ command: python -m pyserini.search.lucene --threads 16 --batch 128 --index msmarco-v1-passage-slimr-pp --topics $topics --encoder castorini/slimr-pp-msmarco-passage --encoded-corpus scipy-sparse-vectors.msmarco-v1-passage-slimr-pp --output $output --output-format msmarco --hits 1000 --impact --min-idf 3
+ topics:
+ - topic_key: msmarco-passage-dev-subset
+ eval_key: msmarco-passage-dev-subset
+ scores:
+ - MRR@10: 0.4032
+ R@1K: 0.9680
+ - topic_key: dl19-passage
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.4687
+ nDCG@10: 0.7140
+ R@1K: 0.8415
+ - topic_key: dl20
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.4906
+ nDCG@10: 0.7021
+ R@1K: 0.8551
+ - name: aggretriever-distilbert-pytorch
+ display: "Aggretriever-DistilBERT: query inference with PyTorch"
+ display-html: "Aggretriever-DistilBERT: query inference with PyTorch"
+ display-row: "[8]"
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --index msmarco-v1-passage.aggretriever-distilbert --topics $topics --encoder castorini/aggretriever-distilbert --output $output
+ topics:
+ - topic_key: msmarco-passage-dev-subset
+ eval_key: msmarco-passage-dev-subset
+ scores:
+ - MRR@10: 0.3412
+ R@1K: 0.9604
+ - topic_key: dl19-passage
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.4301
+ nDCG@10: 0.6816
+ R@1K: 0.8023
+ - topic_key: dl20
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.4329
+ nDCG@10: 0.6726
+ R@1K: 0.8351
+ - name: aggretriever-cocondenser-pytorch
+ display: "Aggretriever-coCondenser: query inference with PyTorch"
+ display-html: "Aggretriever-coCondenser: query inference with PyTorch"
+ display-row: "[8]"
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --index msmarco-v1-passage.aggretriever-cocondenser --topics $topics --encoder castorini/aggretriever-cocondenser --output $output
+ topics:
+ - topic_key: msmarco-passage-dev-subset
+ eval_key: msmarco-passage-dev-subset
+ scores:
+ - MRR@10: 0.3619
+ R@1K: 0.9735
+ - topic_key: dl19-passage
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.4350
+ nDCG@10: 0.6837
+ R@1K: 0.8078
+ - topic_key: dl20
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.4710
+ nDCG@10: 0.6972
+ R@1K: 0.8555
+ - name: openai-ada2
+ display: "OpenAI ada2: pre-encoded queries"
+ display-html: "OpenAI ada2: pre-encoded queries"
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 128 --index msmarco-v1-passage.openai-ada2 --topics $topics --encoded-queries openai-ada2-$topics --output $output
+ topics:
+ - topic_key: msmarco-passage-dev-subset
+ eval_key: msmarco-passage-dev-subset
+ scores:
+ - MRR@10: 0.3435
+ R@1K: 0.9858
+ - topic_key: dl19-passage
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.4788
+ nDCG@10: 0.7035
+ R@1K: 0.8629
+ - topic_key: dl20
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.4771
+ nDCG@10: 0.6759
+ R@1K: 0.8705
+ - name: openai-ada2-hyde
+ display: "HyDE-OpenAI ada2: pre-encoded queries"
+ display-html: "HyDE-OpenAI ada2: pre-encoded queries"
+ command: python -m pyserini.search.faiss --threads 16 --batch-size 128 --index msmarco-v1-passage.openai-ada2 --topics $topics --encoded-queries openai-ada2-$topics-hyde --output $output
+ topics:
+ - topic_key: dl19-passage
+ eval_key: dl19-passage
+ scores:
+ - MAP: 0.5125
+ nDCG@10: 0.7163
+ R@1K: 0.9002
+ - topic_key: dl20
+ eval_key: dl20-passage
+ scores:
+ - MAP: 0.4938
+ nDCG@10: 0.6666
+ R@1K: 0.8919

pyserini/2cr/msmarco-v2-doc.yaml

@@ -0,0 +1,287 @@
+conditions:
+ - name: bm25-doc-default
+ display: BM25 doc (k1=0.9, b=0.4)
+ display-html: BM25 doc (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
+ display-row: (1a)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc --topics $topics --output $output --bm25
+ topics:
+ - topic_key: msmarco-v2-doc-dev
+ eval_key: msmarco-v2-doc-dev
+ scores:
+ - MRR@100: 0.1572
+ R@1K: 0.8054
+ - topic_key: msmarco-v2-doc-dev2
+ eval_key: msmarco-v2-doc-dev2
+ scores:
+ - MRR@100: 0.1659
+ R@1K: 0.8029
+ - topic_key: dl21
+ eval_key: dl21-doc
+ scores:
+ - MAP@100: 0.2126
+ nDCG@10: 0.5116
+ MRR@100: 0.8367
+ R@100: 0.3195
+ R@1K: 0.6739
+ - name: bm25-doc-segmented-default
+ display: BM25 doc segmented (k1=0.9, b=0.4)
+ display-html: BM25 doc segmented (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
+ display-row: (1b)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented --topics $topics --output $output --bm25 --hits 10000 --max-passage-hits 1000 --max-passage
+ topics:
+ - topic_key: msmarco-v2-doc-dev
+ eval_key: msmarco-v2-doc-dev
+ scores:
+ - MRR@100: 0.1896
+ R@1K: 0.8542
+ - topic_key: msmarco-v2-doc-dev2
+ eval_key: msmarco-v2-doc-dev2
+ scores:
+ - MRR@100: 0.1930
+ R@1K: 0.8549
+ - topic_key: dl21
+ eval_key: dl21-doc
+ scores:
+ - MAP@100: 0.2436
+ nDCG@10: 0.5776
+ MRR@100: 0.8937
+ R@100: 0.3478
+ R@1K: 0.6930
+ - name: bm25-rm3-doc-default
+ display: BM25+RM3 doc (k1=0.9, b=0.4)
+ display-html: BM25+RM3 doc (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
+ display-row: (1c)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc --topics $topics --output $output --bm25 --rm3
+ topics:
+ - topic_key: msmarco-v2-doc-dev
+ eval_key: msmarco-v2-doc-dev
+ scores:
+ - MRR@100: 0.0974
+ R@1K: 0.7699
+ - topic_key: msmarco-v2-doc-dev2
+ eval_key: msmarco-v2-doc-dev2
+ scores:
+ - MRR@100: 0.1033
+ R@1K: 0.7736
+ - topic_key: dl21
+ eval_key: dl21-doc
+ scores:
+ - MAP@100: 0.2452
+ nDCG@10: 0.5304
+ MRR@100: 0.7914
+ R@100: 0.3376
+ R@1K: 0.7341
+ - name: bm25-rm3-doc-segmented-default
+ display: BM25+RM3 doc segmented (k1=0.9, b=0.4)
+ display-html: BM25+RM3 doc segmented (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
+ display-row: (1d)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented --topics $topics --output $output --bm25 --rm3 --hits 10000 --max-passage-hits 1000 --max-passage
+ topics:
+ - topic_key: msmarco-v2-doc-dev
+ eval_key: msmarco-v2-doc-dev
+ scores:
+ - MRR@100: 0.1660
+ R@1K: 0.8608
+ - topic_key: msmarco-v2-doc-dev2
+ eval_key: msmarco-v2-doc-dev2
+ scores:
+ - MRR@100: 0.1702
+ R@1K: 0.8639
+ - topic_key: dl21
+ eval_key: dl21-doc
+ scores:
+ - MAP@100: 0.2936
+ nDCG@10: 0.6189
+ MRR@100: 0.9076
+ R@100: 0.3890
+ R@1K: 0.7678
+ - name: bm25-d2q-t5-doc-default
+ display: BM25 w/ doc2query-T5 doc (k1=0.9, b=0.4)
+ display-html: BM25 w/ doc2query-T5 doc (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
+ display-row: (2a)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-d2q-t5 --topics $topics --output $output --bm25
+ topics:
+ - topic_key: msmarco-v2-doc-dev
+ eval_key: msmarco-v2-doc-dev
+ scores:
+ - MRR@100: 0.2011
+ R@1K: 0.8614
+ - topic_key: msmarco-v2-doc-dev2
+ eval_key: msmarco-v2-doc-dev2
+ scores:
+ - MRR@100: 0.2012
+ R@1K: 0.8568
+ - topic_key: dl21
+ eval_key: dl21-doc
+ scores:
+ - MAP@100: 0.2387
+ nDCG@10: 0.5792
+ MRR@100: 0.8866
+ R@100: 0.3443
+ R@1K: 0.7066
+ - name: bm25-d2q-t5-doc-segmented-default
+ display: BM25 w/ doc2query-T5 doc segmented (k1=0.9, b=0.4)
+ display-html: BM25 w/ doc2query-T5 doc segmented (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
+ display-row: (2b)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-d2q-t5 --topics $topics --output $output --bm25 --hits 10000 --max-passage-hits 1000 --max-passage
+ topics:
+ - topic_key: msmarco-v2-doc-dev
+ eval_key: msmarco-v2-doc-dev
+ scores:
+ - MRR@100: 0.2226
+ R@1K: 0.8982
+ - topic_key: msmarco-v2-doc-dev2
+ eval_key: msmarco-v2-doc-dev2
+ scores:
+ - MRR@100: 0.2234
+ R@1K: 0.8952
+ - topic_key: dl21
+ eval_key: dl21-doc
+ scores:
+ - MAP@100: 0.2683
+ nDCG@10: 0.6289
+ MRR@100: 0.9454
+ R@100: 0.3656
+ R@1K: 0.7202
+ - name: bm25-rm3-d2q-t5-doc-default
+ display: BM25+RM3 w/ doc2query-T5 doc (k1=0.9, b=0.4)
+ display-html: BM25+RM3 w/ doc2query-T5 doc (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
+ display-row: (2c)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-d2q-t5-docvectors --topics $topics --output $output --bm25 --rm3
+ topics:
+ - topic_key: msmarco-v2-doc-dev
+ eval_key: msmarco-v2-doc-dev
+ scores:
+ - MRR@100: 0.1141
+ R@1K: 0.8191
+ - topic_key: msmarco-v2-doc-dev2
+ eval_key: msmarco-v2-doc-dev2
+ scores:
+ - MRR@100: 0.1170
+ R@1K: 0.8247
+ - topic_key: dl21
+ eval_key: dl21-doc
+ scores:
+ - MAP@100: 0.2611
+ nDCG@10: 0.5375
+ MRR@100: 0.8255
+ R@100: 0.3580
+ R@1K: 0.7574
+ - name: bm25-rm3-d2q-t5-doc-segmented-default
+ display: BM25+RM3 w/ doc2query-T5 doc segmented (k1=0.9, b=0.4)
+ display-html: BM25+RM3 w/ doc2query-T5 doc segmented (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
+ display-row: (2d)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-d2q-t5-docvectors --topics $topics --output $output --bm25 --rm3 --hits 10000 --max-passage-hits 1000 --max-passage
+ topics:
+ - topic_key: msmarco-v2-doc-dev
+ eval_key: msmarco-v2-doc-dev
+ scores:
+ - MRR@100: 0.1975
+ R@1K: 0.9002
+ - topic_key: msmarco-v2-doc-dev2
+ eval_key: msmarco-v2-doc-dev2
+ scores:
+ - MRR@100: 0.1978
+ R@1K: 0.8972
+ - topic_key: dl21
+ eval_key: dl21-doc
+ scores:
+ - MAP@100: 0.3191
+ nDCG@10: 0.6559
+ MRR@100: 0.8989
+ R@100: 0.4131
+ R@1K: 0.7948
+ - name: unicoil-noexp
+ display: "uniCOIL (noexp): pre-encoded"
+ display-html: "uniCOIL (noexp): pre-encoded queries"
+ display-row: (3a)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-unicoil-noexp-0shot --topics $topics --output $output --impact --hits 10000 --max-passage-hits 1000 --max-passage
+ topics:
+ - topic_key: msmarco-v2-doc-dev-unicoil-noexp
+ eval_key: msmarco-v2-doc-dev
+ scores:
+ - MRR@100: 0.2231
+ R@1K: 0.8987
+ - topic_key: msmarco-v2-doc-dev2-unicoil-noexp
+ eval_key: msmarco-v2-doc-dev2
+ scores:
+ - MRR@100: 0.2314
+ R@1K: 0.8995
+ - topic_key: dl21-unicoil-noexp
+ eval_key: dl21-doc
+ scores:
+ - MAP@100: 0.2587
+ nDCG@10: 0.6495
+ MRR@100: 0.9282
+ R@100: 0.3563
+ R@1K: 0.6787
+ - name: unicoil-noexp-otf
+ display: "uniCOIL (noexp): query inference with PyTorch"
+ display-html: "uniCOIL (noexp): query inference with PyTorch"
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-unicoil-noexp-0shot --topics $topics --encoder castorini/unicoil-noexp-msmarco-passage --output $output --impact --hits 10000 --max-passage-hits 1000 --max-passage
+ topics:
+ - topic_key: msmarco-v2-doc-dev
+ eval_key: msmarco-v2-doc-dev
+ scores:
+ - MRR@100: 0.2232
+ R@1K: 0.8987
+ - topic_key: msmarco-v2-doc-dev2
+ eval_key: msmarco-v2-doc-dev2
+ scores:
+ - MRR@100: 0.2314
+ R@1K: 0.8993
+ - topic_key: dl21
+ eval_key: dl21-doc
+ scores:
+ - MAP@100: 0.2589
+ nDCG@10: 0.6501
+ MRR@100: 0.9282
+ R@100: 0.3574
+ R@1K: 0.6782
+ - name: unicoil
+ display: "uniCOIL (w/ doc2query-T5): pre-encoded"
+ display-html: "uniCOIL (w/ doc2query-T5): pre-encoded queries"
+ display-row: (3b)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-unicoil-0shot --topics $topics --output $output --impact --hits 10000 --max-passage-hits 1000 --max-passage
+ topics:
+ - topic_key: msmarco-v2-doc-dev-unicoil
+ eval_key: msmarco-v2-doc-dev
+ scores:
+ - MRR@100: 0.2419
+ R@1K: 0.9122
+ - topic_key: msmarco-v2-doc-dev2-unicoil
+ eval_key: msmarco-v2-doc-dev2
+ scores:
+ - MRR@100: 0.2445
+ R@1K: 0.9172
+ - topic_key: dl21-unicoil
+ eval_key: dl21-doc
+ scores:
+ - MAP@100: 0.2718
+ nDCG@10: 0.6783
+ MRR@100: 0.9684
+ R@100: 0.3700
+ R@1K: 0.7069
+ - name: unicoil-otf
+ display: "uniCOIL (w/ doc2query-T5): query inference with PyTorch"
+ display-html: "uniCOIL (w/ doc2query-T5): query inference with PyTorch"
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-unicoil-0shot --topics $topics --encoder castorini/unicoil-msmarco-passage --output $output --impact --hits 10000 --max-passage-hits 1000 --max-passage
+ topics:
+ - topic_key: msmarco-v2-doc-dev
+ eval_key: msmarco-v2-doc-dev
+ scores:
+ - MRR@100: 0.2419
+ R@1K: 0.9120
+ - topic_key: msmarco-v2-doc-dev2
+ eval_key: msmarco-v2-doc-dev2
+ scores:
+ - MRR@100: 0.2447
+ R@1K: 0.9174
+ - topic_key: dl21
+ eval_key: dl21-doc
+ scores:
+ - MAP@100: 0.2720
+ nDCG@10: 0.6782
+ MRR@100: 0.9684
+ R@100: 0.3702
+ R@1K: 0.7071

pyserini/2cr/msmarco-v2-passage.yaml

@@ -0,0 +1,287 @@
+conditions:
+ - name: bm25-default
+ display: BM25 original passage (k1=0.9, b=0.4)
+ display-html: BM25 original passage (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
+ display-row: (1a)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage --topics $topics --output $output --bm25
+ topics:
+ - topic_key: msmarco-v2-passage-dev
+ eval_key: msmarco-v2-passage-dev
+ scores:
+ - MRR@100: 0.0719
+ R@1K: 0.5733
+ - topic_key: msmarco-v2-passage-dev2
+ eval_key: msmarco-v2-passage-dev2
+ scores:
+ - MRR@100: 0.0802
+ R@1K: 0.5839
+ - topic_key: dl21
+ eval_key: dl21-passage
+ scores:
+ - MAP@100: 0.1357
+ nDCG@10: 0.4458
+ MRR@100: 0.5060
+ R@100: 0.3261
+ R@1K: 0.6149
+ - name: bm25-augmented-default
+ display: BM25 augmented passage (k1=0.9, b=0.4)
+ display-html: BM25 augmented passage (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
+ display-row: (1b)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage-augmented --topics $topics --output $output --bm25
+ topics:
+ - topic_key: msmarco-v2-passage-dev
+ eval_key: msmarco-v2-passage-dev
+ scores:
+ - MRR@100: 0.0872
+ R@1K: 0.6925
+ - topic_key: msmarco-v2-passage-dev2
+ eval_key: msmarco-v2-passage-dev2
+ scores:
+ - MRR@100: 0.0917
+ R@1K: 0.6933
+ - topic_key: dl21
+ eval_key: dl21-passage
+ scores:
+ - MAP@100: 0.0977
+ nDCG@10: 0.3977
+ MRR@100: 0.5303
+ R@100: 0.2709
+ R@1K: 0.5835
+ - name: bm25-rm3-default
+ display: BM25+RM3 original passage (k1=0.9, b=0.4)
+ display-html: BM25+RM3 original passage (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
+ display-row: (1c)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage --topics $topics --output $output --bm25 --rm3
+ topics:
+ - topic_key: msmarco-v2-passage-dev
+ eval_key: msmarco-v2-passage-dev
+ scores:
+ - MRR@100: 0.0630
+ R@1K: 0.5947
+ - topic_key: msmarco-v2-passage-dev2
+ eval_key: msmarco-v2-passage-dev2
+ scores:
+ - MRR@100: 0.0659
+ R@1K: 0.6062
+ - topic_key: dl21
+ eval_key: dl21-passage
+ scores:
+ - MAP@100: 0.1666
+ nDCG@10: 0.4455
+ MRR@100: 0.5202
+ R@100: 0.3499
+ R@1K: 0.6616
+ - name: bm25-rm3-augmented-default
+ display: BM25+RM3 augmented passage (k1=0.9, b=0.4)
+ display-html: BM25+RM3 augmented passage (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
+ display-row: (1d)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage-augmented --topics $topics --output $output --bm25 --rm3
+ topics:
+ - topic_key: msmarco-v2-passage-dev
+ eval_key: msmarco-v2-passage-dev
+ scores:
+ - MRR@100: 0.0667
+ R@1K: 0.6857
+ - topic_key: msmarco-v2-passage-dev2
+ eval_key: msmarco-v2-passage-dev2
+ scores:
+ - MRR@100: 0.0700
+ R@1K: 0.6826
+ - topic_key: dl21
+ eval_key: dl21-passage
+ scores:
+ - MAP@100: 0.1050
+ nDCG@10: 0.3869
+ MRR@100: 0.4915
+ R@100: 0.2807
+ R@1K: 0.6298
+ - name: bm25-d2q-t5-default
+ display: BM25 w/ doc2query-T5 original passage (k1=0.9, b=0.4)
+ display-html: BM25 w/ doc2query-T5 original passage (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
+ display-row: (2a)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage-d2q-t5 --topics $topics --output $output --bm25
+ topics:
+ - topic_key: msmarco-v2-passage-dev
+ eval_key: msmarco-v2-passage-dev
+ scores:
+ - MRR@100: 0.1072
+ R@1K: 0.7083
+ - topic_key: msmarco-v2-passage-dev2
+ eval_key: msmarco-v2-passage-dev2
+ scores:
+ - MRR@100: 0.1123
+ R@1K: 0.7151
+ - topic_key: dl21
+ eval_key: dl21-passage
+ scores:
+ - MAP@100: 0.1874
+ nDCG@10: 0.4816
+ MRR@100: 0.6848
+ R@100: 0.4076
+ R@1K: 0.7078
+ - name: bm25-d2q-t5-augmented-default
+ display: BM25 w/ doc2query-T5 augmented passage (k1=0.9, b=0.4)
+ display-html: BM25 w/ doc2query-T5 augmented passage (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
+ display-row: (2b)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage-augmented-d2q-t5 --topics $topics --output $output --bm25
+ topics:
+ - topic_key: msmarco-v2-passage-dev
+ eval_key: msmarco-v2-passage-dev
+ scores:
+ - MRR@100: 0.1172
+ R@1K: 0.7647
+ - topic_key: msmarco-v2-passage-dev2
+ eval_key: msmarco-v2-passage-dev2
+ scores:
+ - MRR@100: 0.1170
+ R@1K: 0.7659
+ - topic_key: dl21
+ eval_key: dl21-passage
+ scores:
+ - MAP@100: 0.1649
+ nDCG@10: 0.4702
+ MRR@100: 0.6391
+ R@100: 0.3883
+ R@1K: 0.6962
+ - name: bm25-rm3-d2q-t5-default
+ display: BM25+RM3 w/ doc2query-T5 original passage (k1=0.9, b=0.4)
+ display-html: BM25+RM3 w/ doc2query-T5 original passage (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
+ display-row: (2c)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage-d2q-t5-docvectors --topics $topics --output $output --bm25 --rm3
+ topics:
+ - topic_key: msmarco-v2-passage-dev
+ eval_key: msmarco-v2-passage-dev
+ scores:
+ - MRR@100: 0.0947
+ R@1K: 0.7181
+ - topic_key: msmarco-v2-passage-dev2
+ eval_key: msmarco-v2-passage-dev2
+ scores:
+ - MRR@100: 0.0984
+ R@1K: 0.7222
+ - topic_key: dl21
+ eval_key: dl21-passage
+ scores:
+ - MAP@100: 0.2285
+ nDCG@10: 0.5098
+ MRR@100: 0.6548
+ R@100: 0.4499
+ R@1K: 0.7537
+ - name: bm25-rm3-d2q-t5-augmented-default
+ display: BM25+RM3 w/ doc2query-T5 augmented passage (k1=0.9, b=0.4)
+ display-html: BM25+RM3 w/ doc2query-T5 augmented passage (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)
+ display-row: (2d)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage-augmented-d2q-t5-docvectors --topics $topics --output $output --bm25 --rm3
+ topics:
+ - topic_key: msmarco-v2-passage-dev
+ eval_key: msmarco-v2-passage-dev
+ scores:
+ - MRR@100: 0.0883
+ R@1K: 0.7607
+ - topic_key: msmarco-v2-passage-dev2
+ eval_key: msmarco-v2-passage-dev2
+ scores:
+ - MRR@100: 0.0904
+ R@1K: 0.7649
+ - topic_key: dl21
+ eval_key: dl21-passage
+ scores:
+ - MAP@100: 0.1930
+ nDCG@10: 0.4812
+ MRR@100: 0.5958
+ R@100: 0.4321
+ R@1K: 0.7672
+ - name: unicoil
+ display: "uniCOIL (w/ doc2query-T5): pre-encoded"
+ display-html: "uniCOIL (w/ doc2query-T5): pre-encoded queries"
+ display-row: (3b)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage-unicoil-0shot --topics $topics --output $output --hits 1000 --impact
+ topics:
+ - topic_key: msmarco-v2-passage-dev-unicoil
+ eval_key: msmarco-v2-passage-dev
+ scores:
+ - MRR@100: 0.1499
+ R@1K: 0.7616
+ - topic_key: msmarco-v2-passage-dev2-unicoil
+ eval_key: msmarco-v2-passage-dev2
+ scores:
+ - MRR@100: 0.1577
+ R@1K: 0.7671
+ - topic_key: dl21-unicoil
+ eval_key: dl21-passage
+ scores:
+ - MAP@100: 0.2538
+ nDCG@10: 0.6159
+ MRR@100: 0.7311
+ R@100: 0.4731
+ R@1K: 0.7551
+ - name: unicoil-otf
+ display: "uniCOIL (w/ doc2query-T5): query inference with PyTorch"
+ display-html: "uniCOIL (w/ doc2query-T5): query inference with PyTorch"
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage-unicoil-0shot --topics $topics --encoder castorini/unicoil-msmarco-passage --output $output --hits 1000 --impact
+ topics:
+ - topic_key: msmarco-v2-passage-dev
+ eval_key: msmarco-v2-passage-dev
+ scores:
+ - MRR@100: 0.1501
+ R@1K: 0.7613
+ - topic_key: msmarco-v2-passage-dev2
+ eval_key: msmarco-v2-passage-dev2
+ scores:
+ - MRR@100: 0.1576
+ R@1K: 0.7676
+ - topic_key: dl21
+ eval_key: dl21-passage
+ scores:
+ - MAP@100: 0.2539
+ nDCG@10: 0.6160
+ MRR@100: 0.7311
+ R@100: 0.4723
+ R@1K: 0.7560
+ - name: unicoil-noexp
+ display: "uniCOIL (noexp): pre-encoded"
+ display-html: "uniCOIL (noexp): pre-encoded queries"
+ display-row: (3a)
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage-unicoil-noexp-0shot --topics $topics --output $output --hits 1000 --impact
+ topics:
+ - topic_key: msmarco-v2-passage-dev-unicoil-noexp
+ eval_key: msmarco-v2-passage-dev
+ scores:
+ - MRR@100: 0.1342
+ R@1K: 0.7010
+ - topic_key: msmarco-v2-passage-dev2-unicoil-noexp
+ eval_key: msmarco-v2-passage-dev2
+ scores:
+ - MRR@100: 0.1385
+ R@1K: 0.7114
+ - topic_key: dl21-unicoil-noexp
+ eval_key: dl21-passage
+ scores:
+ - MAP@100: 0.2193
+ nDCG@10: 0.5756
+ MRR@100: 0.6991
+ R@100: 0.4246
+ R@1K: 0.6897
+ - name: unicoil-noexp-otf
+ display: "uniCOIL (noexp): query inference with PyTorch"
+ display-html: "uniCOIL (noexp): query inference with PyTorch"
+ command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-passage-unicoil-noexp-0shot --topics $topics --encoder castorini/unicoil-noexp-msmarco-passage --output $output --hits 1000 --impact
+ topics:
+ - topic_key: msmarco-v2-passage-dev
+ eval_key: msmarco-v2-passage-dev
+ scores:
+ - MRR@100: 0.1343
+ R@1K: 0.7010
+ - topic_key: msmarco-v2-passage-dev2
+ eval_key: msmarco-v2-passage-dev2
+ scores:
+ - MRR@100: 0.1385
+ R@1K: 0.7114
+ - topic_key: dl21
+ eval_key: dl21-passage
+ scores:
+ - MAP@100: 0.2194
+ nDCG@10: 0.5759
+ MRR@100: 0.6991
+ R@100: 0.4247
+ R@1K: 0.6893

pyserini/2cr/msmarco.py

@@ -0,0 +1,600 @@
+#
+# Pyserini: Reproducible IR research with sparse and dense representations
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import argparse
+import math
+import os
+import re
+import sys
+import time
+from collections import defaultdict
+from string import Template
+
+import pkg_resources
+import yaml
+
+from ._base import run_eval_and_return_metric, ok_str, okish_str, fail_str
+
+# The models: the rows of the results table will be ordered this way.
+models = {
+ # MS MARCO v1 passage
+ 'msmarco-v1-passage':
+ ['bm25-default',
+ 'bm25-rm3-default',
+ 'bm25-rocchio-default',
+ '',
+ 'bm25-tuned',
+ 'bm25-rm3-tuned',
+ 'bm25-rocchio-tuned',
+ '',
+ 'bm25-d2q-t5-default',
+ 'bm25-rm3-d2q-t5-default',
+ 'bm25-rocchio-d2q-t5-default',
+ '',
+ 'bm25-d2q-t5-tuned',
+ 'bm25-rm3-d2q-t5-tuned',
+ 'bm25-rocchio-d2q-t5-tuned',
+ '',
+ 'unicoil',
+ 'unicoil-pytorch',
+ 'unicoil-onnx',
+ 'unicoil-noexp',
+ 'unicoil-noexp-pytorch',
+ 'unicoil-noexp-onnx',
+ '',
+ 'splade-pp-ed-onnx',
+ 'splade-pp-sd-onnx',
+ '',
+ 'ance',
+ 'ance-pytorch',
+ '',
+ 'distilbert-kd',
+ 'distilbert-kd-pytorch',
+ 'distilbert-kd-tasb',
+ 'distilbert-kd-tasb-pytorch',
+ '',
+ 'tct_colbert-v2-hnp',
+ 'tct_colbert-v2-hnp-pytorch',
+ '',
+ 'slimr',
+ 'slimr-pp',
+ '',
+ 'aggretriever-distilbert-pytorch',
+ 'aggretriever-cocondenser-pytorch',
+ '',
+ 'openai-ada2',
+ 'openai-ada2-hyde'],
+
+ # MS MARCO v1 doc
+ 'msmarco-v1-doc':
+ ['bm25-doc-default',
+ 'bm25-doc-segmented-default',
+ 'bm25-rm3-doc-default',
+ 'bm25-rm3-doc-segmented-default',
+ 'bm25-rocchio-doc-default',
+ 'bm25-rocchio-doc-segmented-default',
+ '',
+ 'bm25-doc-tuned',
+ 'bm25-doc-segmented-tuned',
+ 'bm25-rm3-doc-tuned',
+ 'bm25-rm3-doc-segmented-tuned',
+ 'bm25-rocchio-doc-tuned',
+ 'bm25-rocchio-doc-segmented-tuned',
+ '',
+ 'bm25-d2q-t5-doc-default',
+ 'bm25-d2q-t5-doc-segmented-default',
+ 'bm25-rm3-d2q-t5-doc-default',
+ 'bm25-rm3-d2q-t5-doc-segmented-default',
+ '',
+ 'bm25-d2q-t5-doc-tuned',
+ 'bm25-d2q-t5-doc-segmented-tuned',
+ 'bm25-rm3-d2q-t5-doc-tuned',
+ 'bm25-rm3-d2q-t5-doc-segmented-tuned',
+ '',
+ 'unicoil-noexp',
+ 'unicoil-noexp-pytorch',
+ '',
+ 'unicoil',
+ 'unicoil-pytorch'],
+
+ # MS MARCO v2 passage
+ 'msmarco-v2-passage':
+ ['bm25-default',
+ 'bm25-augmented-default',
+ 'bm25-rm3-default',
+ 'bm25-rm3-augmented-default',
+ '',
+ 'bm25-d2q-t5-default',
+ 'bm25-d2q-t5-augmented-default',
+ 'bm25-rm3-d2q-t5-default',
+ 'bm25-rm3-d2q-t5-augmented-default',
+ '',
+ 'unicoil-noexp',
+ 'unicoil',
+ '',
+ 'unicoil-noexp-otf',
+ 'unicoil-otf'],
+
+ # MS MARCO v2 doc
+ 'msmarco-v2-doc':
+ ['bm25-doc-default',
+ 'bm25-doc-segmented-default',
+ 'bm25-rm3-doc-default',
+ 'bm25-rm3-doc-segmented-default',
+ '',
+ 'bm25-d2q-t5-doc-default',
+ 'bm25-d2q-t5-doc-segmented-default',
+ 'bm25-rm3-d2q-t5-doc-default',
+ 'bm25-rm3-d2q-t5-doc-segmented-default',
+ '',
+ 'unicoil-noexp',
+ 'unicoil',
+ '',
+ 'unicoil-noexp-otf',
+ 'unicoil-otf'
+ ]
+}
+
+trec_eval_metric_definitions = {
+ 'msmarco-v1-passage': {
+ 'msmarco-passage-dev-subset': {
+ 'MRR@10': '-c -M 10 -m recip_rank',
+ 'R@1K': '-c -m recall.1000'
+ },
+ 'dl19-passage': {
+ 'MAP': '-c -l 2 -m map',
+ 'nDCG@10': '-c -m ndcg_cut.10',
+ 'R@1K': '-c -l 2 -m recall.1000'
+ },
+ 'dl20-passage': {
+ 'MAP': '-c -l 2 -m map',
+ 'nDCG@10': '-c -m ndcg_cut.10',
+ 'R@1K': '-c -l 2 -m recall.1000'
+ }
+ },
+ 'msmarco-v1-doc': {
+ 'msmarco-doc-dev': {
+ 'MRR@10': '-c -M 100 -m recip_rank',
+ 'R@1K': '-c -m recall.1000'
+ },
+ 'dl19-doc': {
+ 'MAP': '-c -M 100 -m map',
+ 'nDCG@10': '-c -m ndcg_cut.10',
+ 'R@1K': '-c -m recall.1000'
+ },
+ 'dl20-doc': {
+ 'MAP': '-c -M 100 -m map',
+ 'nDCG@10': '-c -m ndcg_cut.10',
+ 'R@1K': '-c -m recall.1000'
+ }
+ },
+ 'msmarco-v2-passage': {
+ 'msmarco-v2-passage-dev': {
+ 'MRR@100': '-c -M 100 -m recip_rank',
+ 'R@1K': '-c -m recall.1000'
+ },
+ 'msmarco-v2-passage-dev2': {
+ 'MRR@100': '-c -M 100 -m recip_rank',
+ 'R@1K': '-c -m recall.1000'
+ },
+ 'dl21-passage': {
+ 'MAP@100': '-c -l 2 -M 100 -m map',
+ 'nDCG@10': '-c -m ndcg_cut.10',
+ 'MRR@100': '-c -l 2 -M 100 -m recip_rank',
+ 'R@100': '-c -l 2 -m recall.100',
+ 'R@1K': '-c -l 2 -m recall.1000'
+ }
+ },
+ 'msmarco-v2-doc': {
+ 'msmarco-v2-doc-dev': {
+ 'MRR@100': '-c -M 100 -m recip_rank',
+ 'R@1K': '-c -m recall.1000'
+ },
+ 'msmarco-v2-doc-dev2': {
+ 'MRR@100': '-c -M 100 -m recip_rank',
+ 'R@1K': '-c -m recall.1000'
+ },
+ 'dl21-doc': {
+ 'MAP@100': '-c -M 100 -m map',
+ 'nDCG@10': '-c -m ndcg_cut.10',
+ 'MRR@100': '-c -M 100 -m recip_rank',
+ 'R@100': '-c -m recall.100',
+ 'R@1K': '-c -m recall.1000'
+ }
+ }
+}
+
+
+def find_msmarco_table_topic_set_key_v1(topic_key):
+ # E.g., we want to map variants like 'dl19-passage-unicoil' and 'dl19-passage' both into 'dl19'
+ key = ''
+ if topic_key.startswith('dl19'):
+ key = 'dl19'
+ elif topic_key.startswith('dl20'):
+ key = 'dl20'
+ elif topic_key.startswith('msmarco'):
+ key = 'dev'
+
+ return key
+
+
+def find_msmarco_table_topic_set_key_v2(topic_key):
+ key = ''
+ if topic_key.endswith('dev') or topic_key.endswith('dev-unicoil') or topic_key.endswith('dev-unicoil-noexp'):
+ key = 'dev'
+ elif topic_key.endswith('dev2') or topic_key.endswith('dev2-unicoil') or topic_key.endswith('dev2-unicoil-noexp'):
+ key = 'dev2'
+ elif topic_key.startswith('dl21'):
+ key = 'dl21'
+
+ return key
+
+
+def format_command(raw):
+ # After "--output foo.txt" are additional options like "--hits 1000 --impact".
+ # We want these on a separate line for better readability, but note that sometimes that might
+ # be the end of the command, in which case we don't want to add an extra line break.
+ return raw.replace('--topics', '\\\n --topics') \
+ .replace('--threads', '\\\n --threads')\
+ .replace('--index', '\\\n --index')\
+ .replace('--output ', '\\\n --output ')\
+ .replace('--encoder', '\\\n --encoder')\
+ .replace('--onnx-encoder', '\\\n --onnx-encoder')\
+ .replace('--encoded-corpus', '\\\n --encoded-corpus')\
+ .replace('.txt ', '.txt \\\n ')
+
+
+def read_file(f):
+ fin = open(f, 'r')
+ text = fin.read()
+ fin.close()
+
+ return text
+
+
+def list_conditions(args):
+ for condition in models[args.collection]:
+ if condition == '':
+ continue
+ print(condition)
+
+
+def generate_report(args):
+ yaml_file = pkg_resources.resource_filename(__name__, f'{args.collection}.yaml')
+
+ if args.collection == 'msmarco-v1-passage':
+ html_template = read_file(pkg_resources.resource_filename(__name__, 'msmarco_html_v1_passage.template'))
+ row_template = read_file(pkg_resources.resource_filename(__name__, 'msmarco_html_row_v1.template'))
+ elif args.collection == 'msmarco-v1-doc':
+ html_template = read_file(pkg_resources.resource_filename(__name__, 'msmarco_html_v1_doc.template'))
+ row_template = read_file(pkg_resources.resource_filename(__name__, 'msmarco_html_row_v1.template'))
+ elif args.collection == 'msmarco-v2-passage':
+ html_template = read_file(pkg_resources.resource_filename(__name__, 'msmarco_html_v2_passage.template'))
+ row_template = read_file(pkg_resources.resource_filename(__name__, 'msmarco_html_row_v2.template'))
+ elif args.collection == 'msmarco-v2-doc':
+ html_template = read_file(pkg_resources.resource_filename(__name__, 'msmarco_html_v2_doc.template'))
+ row_template = read_file(pkg_resources.resource_filename(__name__, 'msmarco_html_row_v2.template'))
+ else:
+ raise ValueError(f'Unknown corpus: {args.collection}')
+
+ table = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: 0.0)))
+ commands = defaultdict(lambda: defaultdict(lambda: ''))
+ eval_commands = defaultdict(lambda: defaultdict(lambda: ''))
+
+ table_keys = {}
+ row_ids = {}
+
+ with open(yaml_file) as f:
+ yaml_data = yaml.safe_load(f)
+ for condition in yaml_data['conditions']:
+ name = condition['name']
+ display = condition['display-html']
+ row_id = condition['display-row'] if 'display-row' in condition else ''
+ cmd_template = condition['command']
+
+ row_ids[name] =row_id
+ table_keys[name] = display
+
+ for topic_set in condition['topics']:
+ topic_key = topic_set['topic_key']
+ eval_key = topic_set['eval_key']
+
+ if args.collection == 'msmarco-v1-passage' or args.collection == 'msmarco-v1-doc':
+ short_topic_key = find_msmarco_table_topic_set_key_v1(topic_key)
+ else:
+ short_topic_key = find_msmarco_table_topic_set_key_v2(topic_key)
+
+ runfile = f'run.{args.collection}.{name}.{short_topic_key}.txt'
+ cmd = Template(cmd_template).substitute(topics=topic_key, output=runfile)
+ commands[name][short_topic_key] = cmd
+
+ for expected in topic_set['scores']:
+ for metric in expected:
+ eval_cmd = f'python -m pyserini.eval.trec_eval ' + \
+ f'{trec_eval_metric_definitions[args.collection][eval_key][metric]} {eval_key} {runfile}'
+ eval_commands[name][short_topic_key] += eval_cmd + '\n'
+ table[name][short_topic_key][metric] = expected[metric]
+
+ if args.collection == 'msmarco-v1-passage' or args.collection == 'msmarco-v1-doc':
+ row_cnt = 1
+
+ html_rows = []
+ for name in models[args.collection]:
+ if not name:
+ # Add blank row for spacing
+ html_rows.append('<tr><td style="border-bottom: 0"></td></tr>')
+ continue
+ s = Template(row_template)
+ s = s.substitute(row_cnt=row_cnt,
+ condition_name=table_keys[name],
+ row=row_ids[name],
+ s1=f'{table[name]["dl19"]["MAP"]:.4f}' if table[name]['dl19']['MAP'] != 0 else '-',
+ s2=f'{table[name]["dl19"]["nDCG@10"]:.4f}' if table[name]['dl19']['nDCG@10'] != 0 else '-',
+ s3=f'{table[name]["dl19"]["R@1K"]:.4f}' if table[name]['dl19']['R@1K'] != 0 else '-',
+ s4=f'{table[name]["dl20"]["MAP"]:.4f}' if table[name]['dl20']['MAP'] != 0 else '-',
+ s5=f'{table[name]["dl20"]["nDCG@10"]:.4f}' if table[name]['dl20']['nDCG@10'] != 0 else '-',
+ s6=f'{table[name]["dl20"]["R@1K"]:.4f}' if table[name]['dl20']['R@1K'] != 0 else '-',
+ s7=f'{table[name]["dev"]["MRR@10"]:.4f}' if table[name]['dev']['MRR@10'] != 0 else '-',
+ s8=f'{table[name]["dev"]["R@1K"]:.4f}' if table[name]['dev']['R@1K'] != 0 else '-',
+ cmd1=format_command(commands[name]['dl19']),
+ cmd2=format_command(commands[name]['dl20']),
+ cmd3=format_command(commands[name]['dev']),
+ eval_cmd1=eval_commands[name]['dl19'],
+ eval_cmd2=eval_commands[name]['dl20'],
+ eval_cmd3=eval_commands[name]['dev']
+ )
+
+ # If we don't have scores, we want to remove the commands also. Use simple regexp substitution.
+ if table[name]['dl19']['MAP'] == 0:
+ s = re.sub(re.compile('Command to generate run on TREC 2019 queries:.*?</div>',
+ re.MULTILINE | re.DOTALL),
+ 'Not available.</div>', s)
+ if table[name]['dl20']['MAP'] == 0:
+ s = re.sub(re.compile('Command to generate run on TREC 2020 queries:.*?</div>',
+ re.MULTILINE | re.DOTALL),
+ 'Not available.</div>', s)
+ if table[name]['dev']['MRR@10'] == 0:
+ s = re.sub(re.compile('Command to generate run on dev queries:.*?</div>',
+ re.MULTILINE | re.DOTALL),
+ 'Not available.</div>', s)
+
+ html_rows.append(s)
+ row_cnt += 1
+
+ all_rows = '\n'.join(html_rows)
+ if args.collection == 'msmarco-v1-passage':
+ full_name = 'MS MARCO V1 Passage'
+ else:
+ full_name = 'MS MARCO V1 Document'
+
+ with open(args.output, 'w') as out:
+ out.write(Template(html_template).substitute(title=full_name, rows=all_rows))
+ else:
+ row_cnt = 1
+
+ html_rows = []
+ for name in models[args.collection]:
+ if not name:
+ # Add blank row for spacing
+ html_rows.append('<tr><td style="border-bottom: 0"></td></tr>')
+ continue
+ s = Template(row_template)
+ s = s.substitute(row_cnt=row_cnt,
+ condition_name=table_keys[name],
+ row=row_ids[name],
+ s1=f'{table[name]["dl21"]["MAP@100"]:.4f}',
+ s2=f'{table[name]["dl21"]["nDCG@10"]:.4f}',
+ s3=f'{table[name]["dl21"]["MRR@100"]:.4f}',
+ s4=f'{table[name]["dl21"]["R@100"]:.4f}',
+ s5=f'{table[name]["dl21"]["R@1K"]:.4f}',
+ s6=f'{table[name]["dev"]["MRR@100"]:.4f}',
+ s7=f'{table[name]["dev"]["R@1K"]:.4f}',
+ s8=f'{table[name]["dev2"]["MRR@100"]:.4f}',
+ s9=f'{table[name]["dev2"]["R@1K"]:.4f}',
+ cmd1=format_command(commands[name]['dl21']),
+ cmd2=format_command(commands[name]['dev']),
+ cmd3=format_command(commands[name]['dev2']),
+ eval_cmd1=eval_commands[name]['dl21'],
+ eval_cmd2=eval_commands[name]['dev'],
+ eval_cmd3=eval_commands[name]['dev2']
+ )
+ html_rows.append(s)
+ row_cnt += 1
+
+ all_rows = '\n'.join(html_rows)
+ if args.collection == 'msmarco-v2-passage':
+ full_name = 'MS MARCO V2 Passage'
+ else:
+ full_name = 'MS MARCO V2 Document'
+
+ with open(args.output, 'w') as out:
+ out.write(Template(html_template).substitute(title=full_name, rows=all_rows))
+
+
+def run_conditions(args):
+ start = time.time()
+
+ table = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: 0.0)))
+ table_keys = {}
+
+ yaml_file = pkg_resources.resource_filename(__name__, f'{args.collection}.yaml')
+
+ with open(yaml_file) as f:
+ yaml_data = yaml.safe_load(f)
+ for condition in yaml_data['conditions']:
+ # Either we're running all conditions, or running only the condition specified in --condition
+ if not args.all:
+ if not condition['name'] == args.condition:
+ continue
+
+ name = condition['name']
+ display = condition['display']
+ cmd_template = condition['command']
+
+ print(f'# Running condition "{name}": {display}\n')
+ for topic_set in condition['topics']:
+ topic_key = topic_set['topic_key']
+ eval_key = topic_set['eval_key']
+
+ short_topic_key = ''
+ if args.collection == 'msmarco-v1-passage' or args.collection == 'msmarco-v1-doc':
+ short_topic_key = find_msmarco_table_topic_set_key_v1(topic_key)
+ else:
+ short_topic_key = find_msmarco_table_topic_set_key_v2(topic_key)
+
+ print(f' - topic_key: {topic_key}')
+
+ runfile = os.path.join(args.directory, f'run.{args.collection}.{name}.{short_topic_key}.txt')
+ cmd = Template(cmd_template).substitute(topics=topic_key, output=runfile)
+
+ if args.display_commands:
+ print(f'\n```bash\n{format_command(cmd)}\n```\n')
+
+ if not os.path.exists(runfile):
+ if not args.dry_run:
+ os.system(cmd)
+
+ for expected in topic_set['scores']:
+ for metric in expected:
+ table_keys[name] = display
+ if not args.skip_eval:
+ # If the runfile doesn't exist, we can't evaluate.
+ # This would be the case if --dry-run were set.
+ if not os.path.exists(runfile):
+ continue
+
+ score = float(
+ run_eval_and_return_metric(
+ metric,
+ eval_key,
+ trec_eval_metric_definitions[args.collection][eval_key][metric],
+ runfile))
+ if math.isclose(score, float(expected[metric])):
+ result_str = ok_str
+ # Flaky tests
+ elif args.collection == 'msmarco-v1-passage' \
+ and topic_key == 'msmarco-passage-dev-subset' and name == 'ance-pytorch' \
+ and metric == 'MRR@10' and abs(score-float(expected[metric])) <= 0.0001:
+ result_str = okish_str
+ else:
+ result_str = fail_str + f' expected {expected[metric]:.4f}'
+ print(f' {metric:7}: {score:.4f} {result_str}')
+ table[name][short_topic_key][metric] = score
+ else:
+ table[name][short_topic_key][metric] = expected[metric]
+
+ if not args.skip_eval:
+ print('')
+
+ if args.collection == 'msmarco-v1-passage' or args.collection == 'msmarco-v1-doc':
+ print(' ' * 69 + 'TREC 2019' + ' ' * 16 + 'TREC 2020' + ' ' * 12 + 'MS MARCO dev')
+ print(' ' * 62 + 'MAP nDCG@10 R@1K MAP nDCG@10 R@1K MRR@10 R@1K')
+ print(' ' * 62 + '-' * 22 + ' ' + '-' * 22 + ' ' + '-' * 14)
+
+ if args.condition:
+ # If we've used --condition to specify a specific condition, print out only that row.
+ name = args.condition
+ print(f'{table_keys[name]:60}' +
+ f'{table[name]["dl19"]["MAP"]:8.4f}{table[name]["dl19"]["nDCG@10"]:8.4f}{table[name]["dl19"]["R@1K"]:8.4f} ' +
+ f'{table[name]["dl20"]["MAP"]:8.4f}{table[name]["dl20"]["nDCG@10"]:8.4f}{table[name]["dl20"]["R@1K"]:8.4f} ' +
+ f'{table[name]["dev"]["MRR@10"]:8.4f}{table[name]["dev"]["R@1K"]:8.4f}')
+ else:
+ # Otherwise, print out all rows
+ for name in models[args.collection]:
+ if not name:
+ print('')
+ continue
+ print(f'{table_keys[name]:60}' +
+ f'{table[name]["dl19"]["MAP"]:8.4f}{table[name]["dl19"]["nDCG@10"]:8.4f}{table[name]["dl19"]["R@1K"]:8.4f} ' +
+ f'{table[name]["dl20"]["MAP"]:8.4f}{table[name]["dl20"]["nDCG@10"]:8.4f}{table[name]["dl20"]["R@1K"]:8.4f} ' +
+ f'{table[name]["dev"]["MRR@10"]:8.4f}{table[name]["dev"]["R@1K"]:8.4f}')
+ else:
+ print(' ' * 77 + 'TREC 2021' + ' ' * 18 + 'MS MARCO dev' + ' ' * 6 + 'MS MARCO dev2')
+ print(' ' * 62 + 'MAP@100 nDCG@10 MRR@100 R@100 R@1K MRR@100 R@1K MRR@100 R@1K')
+ print(' ' * 62 + '-' * 38 + ' ' + '-' * 14 + ' ' + '-' * 14)
+
+ if args.condition:
+ # If we've used --condition to specify a specific condition, print out only that row.
+ name = args.condition
+ print(f'{table_keys[name]:60}' +
+ f'{table[name]["dl21"]["MAP@100"]:8.4f}{table[name]["dl21"]["nDCG@10"]:8.4f}' +
+ f'{table[name]["dl21"]["MRR@100"]:8.4f}{table[name]["dl21"]["R@100"]:8.4f}{table[name]["dl21"]["R@1K"]:8.4f} ' +
+ f'{table[name]["dev"]["MRR@100"]:8.4f}{table[name]["dev"]["R@1K"]:8.4f} ' +
+ f'{table[name]["dev2"]["MRR@100"]:8.4f}{table[name]["dev2"]["R@1K"]:8.4f}')
+ else:
+ # Otherwise, print out all rows
+ for name in models[args.collection]:
+ if not name:
+ print('')
+ continue
+ print(f'{table_keys[name]:60}' +
+ f'{table[name]["dl21"]["MAP@100"]:8.4f}{table[name]["dl21"]["nDCG@10"]:8.4f}' +
+ f'{table[name]["dl21"]["MRR@100"]:8.4f}{table[name]["dl21"]["R@100"]:8.4f}{table[name]["dl21"]["R@1K"]:8.4f} ' +
+ f'{table[name]["dev"]["MRR@100"]:8.4f}{table[name]["dev"]["R@1K"]:8.4f} ' +
+ f'{table[name]["dev2"]["MRR@100"]:8.4f}{table[name]["dev2"]["R@1K"]:8.4f}')
+
+ end = time.time()
+
+ print('\n')
+ print(f'Total elapsed time: {end - start:.0f}s')
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser(description='Generate regression matrix for MS MARCO corpora.')
+ parser.add_argument('--collection', type=str,
+ help='Collection = {v1-passage, v1-doc, v2-passage, v2-doc}.', required=True)
+ # To list all conditions
+ parser.add_argument('--list-conditions', action='store_true', default=False, help='List available conditions.')
+ # For generating reports
+ parser.add_argument('--generate-report', action='store_true', default=False, help='Generate report.')
+ parser.add_argument('--output', type=str, help='File to store report.', required=False)
+ # For actually running the experimental conditions
+ parser.add_argument('--all', action='store_true', default=False, help='Run all conditions.')
+ parser.add_argument('--condition', type=str, help='Condition to run.', required=False)
+ parser.add_argument('--directory', type=str, help='Base directory.', default='', required=False)
+ parser.add_argument('--dry-run', action='store_true', default=False, help='Print out commands but do not execute.')
+ parser.add_argument('--skip-eval', action='store_true', default=False, help='Skip running trec_eval.')
+ parser.add_argument('--display-commands', action='store_true', default=False, help='Display command.')
+ args = parser.parse_args()
+
+ if args.collection == 'v1-passage':
+ args.collection = 'msmarco-v1-passage'
+ elif args.collection == 'v1-doc':
+ args.collection = 'msmarco-v1-doc'
+ elif args.collection == 'v2-passage':
+ args.collection = 'msmarco-v2-passage'
+ elif args.collection == 'v2-doc':
+ args.collection = 'msmarco-v2-doc'
+ else:
+ raise ValueError(f'Unknown corpus: {args.collection}')
+
+ if args.list_conditions:
+ list_conditions(args)
+ sys.exit()
+
+ if args.generate_report:
+ if not args.output:
+ print(f'Must specify report filename with --output.')
+ sys.exit()
+
+ generate_report(args)
+ sys.exit()
+
+ if not args.all and not args.condition:
+ print(f'Must specify a specific condition using --condition or use --all to run all conditions.')
+ sys.exit()
+
+ run_conditions(args)

pyserini/2cr/msmarco_html_row_v1.template

@@ -0,0 +1,81 @@
+<!-- Condition: ${condition_name} -->
+<tr class="accordion-toggle collapsed" id="row${row_cnt}" data-toggle="collapse" data-parent="#row${row_cnt}" href="#collapse${row_cnt}">
+<td class="expand-button"></td>
+<td style="min-width: 85px">$row</td>
+<td style="min-width: 400px">${condition_name}</td>
+<td>$s1</td>
+<td>$s2</td>
+<td>$s3</td>
+<td></td>
+<td>$s4</td>
+<td>$s5</td>
+<td>$s6</td>
+<td></td>
+<td>$s7</td>
+<td>$s8</td>
+</tr>
+<tr class="hide-table-padding">
+<td></td>
+<td colspan="11">
+<div id="collapse${row_cnt}" class="collapse in p-3">
+
+<!-- Tabs navs -->
+<ul class="nav nav-tabs mb-3" id="row${row_cnt}-tabs" role="tablist">
+ <li class="nav-item" role="presentation">
+ <a class="nav-link active" id="row${row_cnt}-tab1-header" data-mdb-toggle="tab" href="#row${row_cnt}-tab1" role="tab" aria-controls="row${row_cnt}-tab1" aria-selected="true" style="text-transform:none">TREC 2019</a>
+ </li>
+ <li class="nav-item" role="presentation">
+ <a class="nav-link" id="row${row_cnt}-tab2-header" data-mdb-toggle="tab" href="#row${row_cnt}-tab2" role="tab" aria-controls="row${row_cnt}-tab2" aria-selected="false" style="text-transform:none">TREC 2020</a>
+ </li>
+ <li class="nav-item" role="presentation">
+ <a class="nav-link" id="row${row_cnt}-tab3-header" data-mdb-toggle="tab" href="#row${row_cnt}-tab3" role="tab" aria-controls="row${row_cnt}-tab3" aria-selected="false" style="text-transform:none">dev</a>
+ </li>
+</ul>
+<!-- Tabs navs -->
+
+<!-- Tabs content -->
+<div class="tab-content" id="row${row_cnt}-content">
+ <div class="tab-pane fade show active" id="row${row_cnt}-tab1" role="tabpanel" aria-labelledby="row${row_cnt}-tab1">
+Command to generate run on TREC 2019 queries:
+
+ <blockquote class="mycode">
+<pre><code>$cmd1
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd1}</code></pre>
+ </blockquote>
+
+ </div>
+ <div class="tab-pane fade" id="row${row_cnt}-tab2" role="tabpanel" aria-labelledby="row${row_cnt}-tab2">
+ Command to generate run on TREC 2020 queries:
+
+ <blockquote class="mycode">
+<pre><code>$cmd2
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd2}</code></pre>
+ </blockquote>
+
+ </div>
+ <div class="tab-pane fade" id="row${row_cnt}-tab3" role="tabpanel" aria-labelledby="row${row_cnt}-tab3">
+ Command to generate run on dev queries:
+
+ <blockquote class="mycode">
+<pre><code>$cmd3
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd3}</code></pre>
+ </blockquote>
+
+ </div>
+</div>
+<!-- Tabs content -->
+
+</div></td>
+</tr>

pyserini/2cr/msmarco_html_row_v2.template

@@ -0,0 +1,82 @@
+<!-- Condition: ${condition_name} -->
+<tr class="accordion-toggle collapsed" id="row${row_cnt}" data-toggle="collapse" data-parent="#row${row_cnt}" href="#collapse${row_cnt}">
+<td class="expand-button"></td>
+<td>$row</td>
+<td style="min-width: 400px">${condition_name}</td>
+<td>$s1</td>
+<td>$s2</td>
+<td>$s3</td>
+<td>$s4</td>
+<td>$s5</td>
+<td></td>
+<td>$s6</td>
+<td>$s7</td>
+<td></td>
+<td>$s8</td>
+<td>$s9</td>
+</tr>
+<tr class="hide-table-padding">
+<td></td>
+<td colspan="12">
+<div id="collapse${row_cnt}" class="collapse in p-3">
+
+<!-- Tabs navs -->
+<ul class="nav nav-tabs mb-3" id="row${row_cnt}-tabs" role="tablist">
+ <li class="nav-item" role="presentation">
+ <a class="nav-link active" id="row${row_cnt}-tab1-header" data-mdb-toggle="tab" href="#row${row_cnt}-tab1" role="tab" aria-controls="row${row_cnt}-tab1" aria-selected="true" style="text-transform:none">TREC 2021</a>
+ </li>
+ <li class="nav-item" role="presentation">
+ <a class="nav-link" id="row${row_cnt}-tab2-header" data-mdb-toggle="tab" href="#row${row_cnt}-tab2" role="tab" aria-controls="row${row_cnt}-tab2" aria-selected="false" style="text-transform:none">dev</a>
+ </li>
+ <li class="nav-item" role="presentation">
+ <a class="nav-link" id="row${row_cnt}-tab3-header" data-mdb-toggle="tab" href="#row${row_cnt}-tab3" role="tab" aria-controls="row${row_cnt}-tab3" aria-selected="false" style="text-transform:none">dev2</a>
+ </li>
+</ul>
+<!-- Tabs navs -->
+
+<!-- Tabs content -->
+<div class="tab-content" id="row${row_cnt}-content">
+ <div class="tab-pane fade show active" id="row${row_cnt}-tab1" role="tabpanel" aria-labelledby="row${row_cnt}-tab1">
+Command to generate run on TREC 2021 queries:
+
+ <blockquote class="mycode">
+<pre><code>$cmd1
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd1}</code></pre>
+ </blockquote>
+
+ </div>
+ <div class="tab-pane fade" id="row${row_cnt}-tab2" role="tabpanel" aria-labelledby="row${row_cnt}-tab2">
+ Command to generate run on dev queries:
+
+ <blockquote class="mycode">
+<pre><code>$cmd2
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd2}</code></pre>
+ </blockquote>
+
+ </div>
+ <div class="tab-pane fade" id="row${row_cnt}-tab3" role="tabpanel" aria-labelledby="row${row_cnt}-tab3">
+ Command to generate run on dev2 queries:
+
+ <blockquote class="mycode">
+<pre><code>$cmd3
+</code></pre></blockquote>
+Evaluation commands:
+
+ <blockquote class="mycode">
+<pre><code>${eval_cmd3}</code></pre>
+ </blockquote>
+
+ </div>
+</div>
+<!-- Tabs content -->
+
+</div></td>
+</tr>

pyserini/2cr/msmarco_html_v1_doc.template

@@ -0,0 +1,296 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no" />
+ <meta http-equiv="x-ua-compatible" content="ie=edge" />
+ <title>Pyserini Reproductions: MS MARCO V1 Document</title>
+ <!-- Font Awesome -->
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.11.2/css/all.css" />
+ <!-- Google Fonts Roboto -->
+ <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Roboto:wght@300;400;500;700&display=swap" />
+ <!-- MDB -->
+ <link href="https://cdnjs.cloudflare.com/ajax/libs/mdb-ui-kit/4.0.0/mdb.min.css" rel="stylesheet" />
+
+ <style>
+tr.hide-table-padding td {
+ padding: 0;
+}
+
+.expand-button {
+ position: relative;
+}
+
+.accordion-toggle .expand-button:after {
+ position: absolute;
+ left:.75rem;
+ top: 50%;
+ transform: translate(0, -50%);
+ content: '-';
+}
+
+.accordion-toggle.collapsed .expand-button:after {
+ content: '+';
+}
+
+blockquote.mycode {
+ border-left: 3px solid #ccc;
+ margin-left: 25px;
+ margin-top: 15px;
+ padding-left: 15px;
+}
+
+blockquote.mycode2 {
+ border-left: 3px solid #ccc;
+ margin-left: 25px;
+ padding-top: 10px;
+ padding-bottom: 10px;
+ padding-left: 15px;
+}
+
+tr th.headertop {
+ border-bottom: none;
+ padding-bottom: 0rem
+}
+
+tr th.headerbottom {
+ padding-top: 0rem
+}
+
+.table>:not(caption)>*>*{padding:0.75rem 0.75rem}
+
+.copy-code-button {
+ border-radius: 0;
+ min-width: 55px;
+ background: none repeat scroll 0 0 transparent;
+ background-color: grey;
+ color: #F1F2F3 !important;
+ cursor: pointer;
+ border-style: none;
+ font-family: 'HELVETICA',sans-serif;
+ font-size: 0.8em;
+ font-weight: normal;
+ text-align: center;
+ text-decoration: none;
+ text-indent: 0;
+ text-transform: uppercase;
+ font-weight: 500;
+ line-height: 1.42rem;
+ margin: 0;
+ padding: 3px 8px;
+ position: absolute !important;
+ top: 0 !important;
+ right: 0 !important;
+}
+
+.copy-code-button > span {
+ color: #F1F2F3 !important;
+}
+
+.copy-code-button, ::before, ::after {
+ box-sizing: inherit;
+}
+
+.copy-code-button::before {
+ content: '';
+ display: inline-block;
+ width: 16px;
+ height: 16px;
+ margin-right: 3px;
+ background-size: contain;
+ background-image: url("data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiPz4KPHN2ZyB3aWR0aD0iMTVweCIgaGVpZ2h0PSIxNXB4IiB2aWV3Qm94PSIwIDAgMTUgMTUiIHZlcnNpb249IjEuMSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB4bWxuczp4bGluaz0iaHR0cDovL3d3dy53My5vcmcvMTk5OS94bGluayI+CiAgICA8IS0tIEdlbmVyYXRvcjogU2tldGNoIDUwLjIgKDU1MDQ3KSAtIGh0dHA6Ly93d3cuYm9oZW1pYW5jb2RpbmcuY29tL3NrZXRjaCAtLT4KICAgIDx0aXRsZT5QYWdlIDE8L3RpdGxlPgogICAgPGRlc2M+Q3JlYXRlZCB3aXRoIFNrZXRjaC48L2Rlc2M+CiAgICA8ZGVmcz48L2RlZnM+CiAgICA8ZyBpZD0iRmxvdyIgc3Ryb2tlPSJub25lIiBzdHJva2Utd2lkdGg9IjEiIGZpbGw9Im5vbmUiIGZpbGwtcnVsZT0iZXZlbm9kZCI+CiAgICAgICAgPGcgaWQ9IkJ0dG5faHRtbCIgdHJhbnNmb3JtPSJ0cmFuc2xhdGUoLTgxOS4wMDAwMDAsIC03NTMuMDAwMDAwKSIgZmlsbD0iI0ZGRkZGRiI+CiAgICAgICAgICAgIDxnIGlkPSJHcm91cC0xIiB0cmFuc2Zvcm09InRyYW5zbGF0ZSgzMTEuMDAwMDAwLCA0MDUuMDAwMDAwKSI+CiAgICAgICAgICAgICAgICA8ZyBpZD0iR3JvdXAtMiIgdHJhbnNmb3JtPSJ0cmFuc2xhdGUoNTA4LjAwMDAwMCwgMzQyLjAwMDAwMCkiPgogICAgICAgICAgICAgICAgICAgIDxwYXRoIGQ9Ik0xMy45NzcyNzI3LDYgTDMuNDA5MDkwOTEsNiBDMi44NDQ1NDU0NSw2IDIuMzg2MzYzNjQsNi40NTgxODE4MiAyLjM4NjM2MzY0LDcuMDIyNzI3MjcgTDIuMzg2MzYzNjQsMTcuNTkwOTA5MSBDMi4zODYzNjM2NCwxOC4xNTU0NTQ1IDIuODQ0NTQ1NDUsMTguNjEzNjM2NCAzLjQwOTA5MDkxLDE4LjYxMzYzNjQgTDEzLjk3NzI3MjcsMTguNjEzNjM2NCBDMTQuNTQxODE4MiwxOC42MTM2MzY0IDE1LDE4LjE1NTQ1NDUgMTUsMTcuNTkwOTA5MSBMMTUsNy4wMjI3MjcyNyBDMTUsNi40NTgxODE4MiAxNC41NDE4MTgyLDYgMTMuOTc3MjcyNyw2IFogTTE0LjMxODE4MTgsMTcuNTkwOTA5MSBDMTQuMzE4MTgxOCwxNy43NzkwOTA5IDE0LjE2NTQ1NDUsMTcuOTMxODE4MiAxMy45NzcyNzI3LDE3LjkzMTgxODIgTDMuNDA5MDkwOTEsMTcuOTMxODE4MiBDMy4yMjA5MDkwOSwxNy45MzE4MTgyIDMuMDY4MTgxODIsMTcuNzc5MDkwOSAzLjA2ODE4MTgyLDE3LjU5MDkwOTEgTDMuMDY4MTgxODIsNy4wMjI3MjcyNyBDMy4wNjgxODE4Miw2LjgzNDU0NTQ1IDMuMjIwOTA5MDksNi42ODE4MTgxOCAzLjQwOTA5MDkxLDYuNjgxODE4MTggTDEzLjk3NzI3MjcsNi42ODE4MTgxOCBDMTQuMTY1NDU0NSw2LjY4MTgxODE4IDE0LjMxODE4MTgsNi44MzQ1NDU0NSAxNC4zMTgxODE4LDcuMDIyNzI3MjcgTDE0LjMxODE4MTgsMTcuNTkwOTA5MSBaIE0xMS45MzE4MTgyLDE5Ljk3NzI3MjcgQzExLjkzMTgxODIsMjAuMTY1NDU0NSAxMS43NzkwOTA5LDIwLjMxODE4MTggMTEuNTkwOTA5MSwyMC4zMTgxODE4IEwxLjAyMjcyNzI3LDIwLjMxODE4MTggQzAuODM0NTQ1NDU1LDIwLjMxODE4MTggMC42ODE4MTgxODIsMjAuMTY1NDU0NSAwLjY4MTgxODE4MiwxOS45NzcyNzI3IEwwLjY4MTgxODE4Miw5LjQwOTA5MDkxIEMwLjY4MTgxODE4Miw5LjIyMDkwOTA5IDAuODM0NTQ1NDU1LDkuMDY4MTgxODIgMS4wMjI3MjcyNyw5LjA2ODE4MTgyIEwxLjM2MzYzNjM2LDkuMDY4MTgxODIgTDEuMzYzNjM2MzYsOC4zODYzNjM2NCBMMS4wMjI3MjcyNyw4LjM4NjM2MzY0IEMwLjQ1ODE4MTgxOCw4LjM4NjM2MzY0IDAsOC44NDQ1NDU0NSAwLDkuNDA5MDkwOTEgTDAsMTkuOTc3MjcyNyBDMCwyMC41NDE4MTgyIDAuNDU4MTgxODE4LDIxIDEuMDIyNzI3MjcsMjEgTDExLjU5MDkwOTEsMjEgQzEyLjE1NTQ1NDUsMjEgMTIuNjEzNjM2NCwyMC41NDE4MTgyIDEyLjYxMzYzNjQsMTkuOTc3MjcyNyBMMTIuNjEzNjM2NCwxOS42MzYzNjM2IEwxMS45MzE4MTgyLDE5LjYzNjM2MzYgTDExLjkzMTgxODIsMTkuOTc3MjcyNyBaIiBpZD0iUGFnZS0xIj48L3BhdGg+CiAgICAgICAgICAgICAgICA8L2c+CiAgICAgICAgICAgIDwvZz4KICAgICAgICA8L2c+CiAgICA8L2c+Cjwvc3ZnPg==");
+ background-repeat: no-repeat;
+ position: relative;
+ top: 3px;
+}
+
+.copy-code-button:focus {
+ /* Avoid an ugly focus outline on click in Chrome,
+ but darken the button for accessibility.
+ See https://stackoverflow.com/a/25298082/1481479 */
+ /* background-color: #E6E6E6; */
+ outline: 0;
+}
+
+pre[class*="prettyprint"] {
+ position: relative;
+ overflow: hidden;
+}
+ </style>
+</head>
+<body>
+
+ <!-- Background image -->
+ <div id="intro" class="bg-image vh-100 shadow-1-strong" style="max-height: 150px">
+ <div class="mask" style="
+ background: linear-gradient(
+ 45deg,
+ rgba(29, 236, 197, 0.7),
+ rgba(91, 14, 214, 0.7) 100%
+ );
+ ">
+ <div class="container d-flex align-items-center justify-content-center text-center h-100" style="max-height: 150px">
+ <div class="text-white">
+ <h1 class="mb-3">$title</h1>
+ </div>
+ </div>
+ </div>
+ </div>
+ <!-- Background image -->
+
+<div class="container my-4">
+
+<p>The two-click<a href="#" data-mdb-toggle="tooltip" title="What are the two clicks, you ask? Copy and paste!"><sup>*</sup></a> reproduction matrix below provides commands for reproducing experimental results reported in a number of papers, denoted by the references in square brackets.
+Instructions for programmatic execution are shown at the bottom of this page (scroll down).</p>
+
+<div class="table-responsive">
+ <table class="table">
+ <thead>
+ <tr>
+ <th class="headertop"></th>
+ <th class="headertop"></th>
+ <th class="headertop"></th>
+ <th class="headertop" colspan="4"><b>TREC 2019</b></th>
+ <th class="headertop" colspan="4"><b>TREC 2020</b></th>
+ <th class="headertop" colspan="3"><b>dev</b></th>
+ </tr>
+ <tr>
+ <th class="headerbottom" scope="col"></th>
+ <th class="headerbottom" scope="col"></th>
+ <th class="headerbottom" scope="col"></th>
+ <th class="headerbottom" scope="col"><br/>AP@100</th>
+ <th class="headerbottom" scope="col">nDCG@10</th>
+ <th class="headerbottom" scope="col">R@1K</th>
+ <th class="headerbottom" scope="col"></th>
+ <th class="headerbottom" scope="col"><br/>AP@100</th>
+ <th class="headerbottom" scope="col">nDCG@10</th>
+ <th class="headerbottom" scope="col">R@1K</th>
+ <th class="headerbottom" scope="col"></th>
+ <th class="headerbottom" scope="col">RR@100</th>
+ <th class="headerbottom" scope="col">R@1K</th>
+ </tr>
+ </thead>
+ <tbody>
+
+$rows
+
+ </tbody>
+ </table>
+</div>
+
+<ul style="list-style-type:none; padding-top: 25px">
+
+<li><p>[1] Xueguang Ma, Ronak Pradeep, Rodrigo Nogueira, and Jimmy Lin.
+<a href="https://cs.uwaterloo.ca/~jimmylin/publications/Ma_etal_SIGIR2022.pdf">Document Expansions and Learned Sparse Lexical Representations for MS MARCO V1 and V2.</a>
+<i>Proceedings of the 45th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR 2022)</i>, July 2022.</p>
+
+<p>&nbsp; &nbsp; &nbsp; &nbsp;Numbers in parentheses correspond to rows in Table 2 of the paper.</p></li>
+
+</ul>
+
+<div style="padding-top: 20px"/>
+
+<h4>Programmatic Execution</h4>
+
+<p>All experimental runs shown in the above table can be programmatically executed based on the instructions below.
+To list all the experimental conditions:</p>
+
+<blockquote class="mycode2"><tt>
+python -m pyserini.2cr.msmarco --collection v1-doc --list-conditions
+</tt></blockquote>
+
+<p>These conditions correspond to the table rows above.</p>
+
+<p>For all conditions, just show the commands in a "dry run":</p>
+
+<blockquote class="mycode2"><tt>
+python -m pyserini.2cr.msmarco --collection v1-doc --all --display-commands --dry-run
+</tt></blockquote>
+
+<p>To actually run all the experimental conditions:</p>
+
+<blockquote class="mycode2"><tt>
+python -m pyserini.2cr.msmarco --collection v1-doc --all --display-commands
+</tt></blockquote>
+
+<p>With the above command, run files will be placed in the current directory.
+Use the option <tt>--directory runs/</tt> to place the runs in a sub-directory.</p>
+
+<p>To show the commands for a specific condition:</p>
+
+<blockquote class="mycode2"><tt>
+python -m pyserini.2cr.msmarco --collection v1-doc --condition bm25-doc-default --display-commands --dry-run
+</tt></blockquote>
+
+<p>This will generate exactly the commands for a specific condition above (corresponding to a row in the table).</p>
+
+<p>To actually run a specific condition:</p>
+
+<blockquote class="mycode2"><tt>
+python -m pyserini.2cr.msmarco --collection v1-doc --condition bm25-doc-default --display-commands
+</tt></blockquote>
+
+<p>Again, with the above command, run files will be placed in the current directory.
+Use the option <tt>--directory runs/</tt> to place the runs in a sub-directory.</p>
+
+<p>Finally, to generate this page:</p>
+
+<blockquote class="mycode2"><tt>
+python -m pyserini.2cr.msmarco --collection v1-doc --generate-report --output msmarco-v1-doc.html
+</tt></blockquote>
+
+<p>The output file <tt>msmarco-v1-doc.html</tt> should be identical to this page.</p>
+
+<div style="padding-top: 50px"/>
+
+ </div>
+
+
+
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.0/jquery.min.js"></script>
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/4.3.1/js/bootstrap.min.js"></script>
+ <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mdb-ui-kit/4.0.0/mdb.min.js"></script>
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.10/clipboard.min.js"></script>
+
+<script>
+document.querySelectorAll('pre').forEach(function (codeBlock) {
+ var button = document.createElement('button');
+ button.className = 'copy-code-button';
+ button.type = 'button';
+ var s = codeBlock.innerText;
+ button.setAttribute('data-clipboard-text',s);
+ button.innerText = 'Copy';
+
+ // var pre = codeBlock.parentNode;
+ codeBlock.classList.add('prettyprint');
+ // pre.parentNode.insertBefore(button, pre);
+ codeBlock.appendChild(button);
+});
+
+var clipboard = new ClipboardJS('.copy-code-button');
+
+clipboard.on('success', function(e) {
+ console.info('Action:', e.action);
+ console.info('Text:', e.text);
+ console.info('Trigger:', e.trigger);
+ e.trigger.textContent = 'Copied';
+ window.setTimeout(function() {
+ e.trigger.textContent = 'Copy';
+ }, 2000);
+ e.clearSelection();
+});
+
+clipboard.on('error', function(e) {
+ console.error('Action:', e.action);
+ console.error('Trigger:', e.trigger);
+ e.trigger.textContent = 'Error Copying';
+ window.setTimeout(function() {
+ e.trigger.textContent = 'Copy';
+ }, 2000);
+ e.clearSelection();
+});
+
+</script>
+
+</body>
+</html>

pyserini/2cr/msmarco_html_v1_passage.template

@@ -0,0 +1,325 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no" />
+ <meta http-equiv="x-ua-compatible" content="ie=edge" />
+ <title>Pyserini Reproductions: MS MARCO V1 Passage</title>
+ <!-- Font Awesome -->
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.11.2/css/all.css" />
+ <!-- Google Fonts Roboto -->
+ <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Roboto:wght@300;400;500;700&display=swap" />
+ <!-- MDB -->
+ <link href="https://cdnjs.cloudflare.com/ajax/libs/mdb-ui-kit/4.0.0/mdb.min.css" rel="stylesheet" />
+
+ <style>
+tr.hide-table-padding td {
+ padding: 0;
+}
+
+.expand-button {
+ position: relative;
+}
+
+.accordion-toggle .expand-button:after {
+ position: absolute;
+ left:.75rem;
+ top: 50%;
+ transform: translate(0, -50%);
+ content: '-';
+}
+
+.accordion-toggle.collapsed .expand-button:after {
+ content: '+';
+}
+
+blockquote.mycode {
+ border-left: 3px solid #ccc;
+ margin-left: 25px;
+ margin-top: 15px;
+ padding-left: 15px;
+}
+
+blockquote.mycode2 {
+ border-left: 3px solid #ccc;
+ margin-left: 25px;
+ padding-top: 10px;
+ padding-bottom: 10px;
+ padding-left: 15px;
+}
+
+tr th.headertop {
+ border-bottom: none;
+ padding-bottom: 0rem
+}
+
+tr th.headerbottom {
+ padding-top: 0rem
+}
+
+.table>:not(caption)>*>*{padding:0.75rem 0.75rem}
+
+.copy-code-button {
+ border-radius: 0;
+ min-width: 55px;
+ background: none repeat scroll 0 0 transparent;
+ background-color: grey;
+ color: #F1F2F3 !important;
+ cursor: pointer;
+ border-style: none;
+ font-family: 'HELVETICA',sans-serif;
+ font-size: 0.8em;
+ font-weight: normal;
+ text-align: center;
+ text-decoration: none;
+ text-indent: 0;
+ text-transform: uppercase;
+ font-weight: 500;
+ line-height: 1.42rem;
+ margin: 0;
+ padding: 3px 8px;
+ position: absolute !important;
+ top: 0 !important;
+ right: 0 !important;
+}
+
+.copy-code-button > span {
+ color: #F1F2F3 !important;
+}
+
+.copy-code-button, ::before, ::after {
+ box-sizing: inherit;
+}
+
+.copy-code-button::before {
+ content: '';
+ display: inline-block;
+ width: 16px;
+ height: 16px;
+ margin-right: 3px;
+ background-size: contain;
+ background-image: url("data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiPz4KPHN2ZyB3aWR0aD0iMTVweCIgaGVpZ2h0PSIxNXB4IiB2aWV3Qm94PSIwIDAgMTUgMTUiIHZlcnNpb249IjEuMSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB4bWxuczp4bGluaz0iaHR0cDovL3d3dy53My5vcmcvMTk5OS94bGluayI+CiAgICA8IS0tIEdlbmVyYXRvcjogU2tldGNoIDUwLjIgKDU1MDQ3KSAtIGh0dHA6Ly93d3cuYm9oZW1pYW5jb2RpbmcuY29tL3NrZXRjaCAtLT4KICAgIDx0aXRsZT5QYWdlIDE8L3RpdGxlPgogICAgPGRlc2M+Q3JlYXRlZCB3aXRoIFNrZXRjaC48L2Rlc2M+CiAgICA8ZGVmcz48L2RlZnM+CiAgICA8ZyBpZD0iRmxvdyIgc3Ryb2tlPSJub25lIiBzdHJva2Utd2lkdGg9IjEiIGZpbGw9Im5vbmUiIGZpbGwtcnVsZT0iZXZlbm9kZCI+CiAgICAgICAgPGcgaWQ9IkJ0dG5faHRtbCIgdHJhbnNmb3JtPSJ0cmFuc2xhdGUoLTgxOS4wMDAwMDAsIC03NTMuMDAwMDAwKSIgZmlsbD0iI0ZGRkZGRiI+CiAgICAgICAgICAgIDxnIGlkPSJHcm91cC0xIiB0cmFuc2Zvcm09InRyYW5zbGF0ZSgzMTEuMDAwMDAwLCA0MDUuMDAwMDAwKSI+CiAgICAgICAgICAgICAgICA8ZyBpZD0iR3JvdXAtMiIgdHJhbnNmb3JtPSJ0cmFuc2xhdGUoNTA4LjAwMDAwMCwgMzQyLjAwMDAwMCkiPgogICAgICAgICAgICAgICAgICAgIDxwYXRoIGQ9Ik0xMy45NzcyNzI3LDYgTDMuNDA5MDkwOTEsNiBDMi44NDQ1NDU0NSw2IDIuMzg2MzYzNjQsNi40NTgxODE4MiAyLjM4NjM2MzY0LDcuMDIyNzI3MjcgTDIuMzg2MzYzNjQsMTcuNTkwOTA5MSBDMi4zODYzNjM2NCwxOC4xNTU0NTQ1IDIuODQ0NTQ1NDUsMTguNjEzNjM2NCAzLjQwOTA5MDkxLDE4LjYxMzYzNjQgTDEzLjk3NzI3MjcsMTguNjEzNjM2NCBDMTQuNTQxODE4MiwxOC42MTM2MzY0IDE1LDE4LjE1NTQ1NDUgMTUsMTcuNTkwOTA5MSBMMTUsNy4wMjI3MjcyNyBDMTUsNi40NTgxODE4MiAxNC41NDE4MTgyLDYgMTMuOTc3MjcyNyw2IFogTTE0LjMxODE4MTgsMTcuNTkwOTA5MSBDMTQuMzE4MTgxOCwxNy43NzkwOTA5IDE0LjE2NTQ1NDUsMTcuOTMxODE4MiAxMy45NzcyNzI3LDE3LjkzMTgxODIgTDMuNDA5MDkwOTEsMTcuOTMxODE4MiBDMy4yMjA5MDkwOSwxNy45MzE4MTgyIDMuMDY4MTgxODIsMTcuNzc5MDkwOSAzLjA2ODE4MTgyLDE3LjU5MDkwOTEgTDMuMDY4MTgxODIsNy4wMjI3MjcyNyBDMy4wNjgxODE4Miw2LjgzNDU0NTQ1IDMuMjIwOTA5MDksNi42ODE4MTgxOCAzLjQwOTA5MDkxLDYuNjgxODE4MTggTDEzLjk3NzI3MjcsNi42ODE4MTgxOCBDMTQuMTY1NDU0NSw2LjY4MTgxODE4IDE0LjMxODE4MTgsNi44MzQ1NDU0NSAxNC4zMTgxODE4LDcuMDIyNzI3MjcgTDE0LjMxODE4MTgsMTcuNTkwOTA5MSBaIE0xMS45MzE4MTgyLDE5Ljk3NzI3MjcgQzExLjkzMTgxODIsMjAuMTY1NDU0NSAxMS43NzkwOTA5LDIwLjMxODE4MTggMTEuNTkwOTA5MSwyMC4zMTgxODE4IEwxLjAyMjcyNzI3LDIwLjMxODE4MTggQzAuODM0NTQ1NDU1LDIwLjMxODE4MTggMC42ODE4MTgxODIsMjAuMTY1NDU0NSAwLjY4MTgxODE4MiwxOS45NzcyNzI3IEwwLjY4MTgxODE4Miw5LjQwOTA5MDkxIEMwLjY4MTgxODE4Miw5LjIyMDkwOTA5IDAuODM0NTQ1NDU1LDkuMDY4MTgxODIgMS4wMjI3MjcyNyw5LjA2ODE4MTgyIEwxLjM2MzYzNjM2LDkuMDY4MTgxODIgTDEuMzYzNjM2MzYsOC4zODYzNjM2NCBMMS4wMjI3MjcyNyw4LjM4NjM2MzY0IEMwLjQ1ODE4MTgxOCw4LjM4NjM2MzY0IDAsOC44NDQ1NDU0NSAwLDkuNDA5MDkwOTEgTDAsMTkuOTc3MjcyNyBDMCwyMC41NDE4MTgyIDAuNDU4MTgxODE4LDIxIDEuMDIyNzI3MjcsMjEgTDExLjU5MDkwOTEsMjEgQzEyLjE1NTQ1NDUsMjEgMTIuNjEzNjM2NCwyMC41NDE4MTgyIDEyLjYxMzYzNjQsMTkuOTc3MjcyNyBMMTIuNjEzNjM2NCwxOS42MzYzNjM2IEwxMS45MzE4MTgyLDE5LjYzNjM2MzYgTDExLjkzMTgxODIsMTkuOTc3MjcyNyBaIiBpZD0iUGFnZS0xIj48L3BhdGg+CiAgICAgICAgICAgICAgICA8L2c+CiAgICAgICAgICAgIDwvZz4KICAgICAgICA8L2c+CiAgICA8L2c+Cjwvc3ZnPg==");
+ background-repeat: no-repeat;
+ position: relative;
+ top: 3px;
+}
+
+.copy-code-button:focus {
+ /* Avoid an ugly focus outline on click in Chrome,
+ but darken the button for accessibility.
+ See https://stackoverflow.com/a/25298082/1481479 */
+ /* background-color: #E6E6E6; */
+ outline: 0;
+}
+
+pre[class*="prettyprint"] {
+ position: relative;
+ overflow: hidden;
+}
+ </style>
+</head>
+<body>
+
+ <!-- Background image -->
+ <div id="intro" class="bg-image vh-100 shadow-1-strong" style="max-height: 150px">
+ <div class="mask" style="
+ background: linear-gradient(
+ 45deg,
+ rgba(29, 236, 197, 0.7),
+ rgba(91, 14, 214, 0.7) 100%
+ );
+ ">
+ <div class="container d-flex align-items-center justify-content-center text-center h-100" style="max-height: 150px">
+ <div class="text-white">
+ <h1 class="mb-3">$title</h1>
+ </div>
+ </div>
+ </div>
+ </div>
+ <!-- Background image -->
+
+<div class="container my-4">
+
+<p>The two-click<a href="#" data-mdb-toggle="tooltip" title="What are the two clicks, you ask? Copy and paste!"><sup>*</sup></a> reproduction matrix below provides commands for reproducing experimental results reported in a number of papers, denoted by the references in square brackets.
+Instructions for programmatic execution are shown at the bottom of this page (scroll down).</p>
+
+<div class="table-responsive">
+ <table class="table">
+ <thead>
+ <tr>
+ <th class="headertop"></th>
+ <th class="headertop"></th>
+ <th class="headertop"></th>
+ <th class="headertop" colspan="4"><b>TREC 2019</b></th>
+ <th class="headertop" colspan="4"><b>TREC 2020</b></th>
+ <th class="headertop" colspan="3"><b>dev</b></th>
+ </tr>
+ <tr>
+ <th class="headerbottom" scope="col"></th>
+ <th class="headerbottom" scope="col"></th>
+ <th class="headerbottom" scope="col"></th>
+ <th class="headerbottom" scope="col"><br/>AP</th>
+ <th class="headerbottom" scope="col">nDCG@10</th>
+ <th class="headerbottom" scope="col">R@1K</th>
+ <th class="headerbottom" scope="col"></th>
+ <th class="headerbottom" scope="col"><br/>AP</th>
+ <th class="headerbottom" scope="col">nDCG@10</th>
+ <th class="headerbottom" scope="col">R@1K</th>
+ <th class="headerbottom" scope="col"></th>
+ <th class="headerbottom" scope="col">RR@10</th>
+ <th class="headerbottom" scope="col">R@1K</th>
+
+ </tr>
+ </thead>
+ <tbody>
+
+$rows
+
+ </tbody>
+ </table>
+</div>
+
+<ul style="list-style-type:none; padding-top: 25px">
+
+<li><p>[1] Xueguang Ma, Ronak Pradeep, Rodrigo Nogueira, and Jimmy Lin.
+<a href="https://cs.uwaterloo.ca/~jimmylin/publications/Ma_etal_SIGIR2022.pdf">Document Expansions and Learned Sparse Lexical Representations for MS MARCO V1 and V2.</a>
+<i>Proceedings of the 45th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR 2022)</i>, July 2022.</p>
+
+<p>&nbsp; &nbsp; &nbsp; &nbsp;Numbers in parentheses correspond to rows in Table 1 of the paper.</p></li>
+
+<li><p>[2] Thibault Formal, Carlos Lassance, Benjamin Piwowarski, and Stéphane Clinchant
+<a href="https://arxiv.org/abs/2205.04733">From Distillation to Hard Negative Sampling: Making Sparse Neural IR Models More Effective.</a>
+<i>Proceedings of the 45th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR 2022), May 2022.</i></p></li>
+
+<li><p>[3] Lee Xiong, Chenyan Xiong, Ye Li, Kwok-Fung Tang, Jialin Liu, Paul N. Bennett, Junaid Ahmed, and Arnold Overwijk.
+<a href="https://openreview.net/forum?id=zeFrfgyZln">Approximate Nearest Neighbor Negative Contrastive Learning for Dense Text Retrieval.</a>
+<i>Proceedings of the 9th International Conference on Learning Representations (ICLR 2021), May 2021.</i></p></li>
+
+<li><p>[4] Sebastian Hofstätter, Sophia Althammer, Michael Schröder, Mete Sertkan, and Allan Hanbury.
+<a href="https://arxiv.org/abs/2010.02666">Improving Efficient Neural Ranking Models with Cross-Architecture Knowledge Distillation.</a>
+<i>arXiv:2010.02666</i>, October 2020.</p></li>
+
+<li><p>[5] Sebastian Hofstätter, Sheng-Chieh Lin, Jheng-Hong Yang, Jimmy Lin, and Allan Hanbury.
+<a href="https://dl.acm.org/doi/10.1145/3404835.3462891">Efficiently Teaching an Effective Dense Retriever with Balanced Topic Aware Sampling.</a>
+<i>Proceedings of the 44th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR 2021)</i>, pages 113-122, July 2021.</p></li>
+
+<li><p>[6] Sheng-Chieh Lin, Jheng-Hong Yang, and Jimmy Lin.
+<a href="https://aclanthology.org/2021.repl4nlp-1.17/">In-Batch Negatives for Knowledge Distillation with Tightly-Coupled Teachers for Dense Retrieval.</a>
+<i>Proceedings of the 6th Workshop on Representation Learning for NLP (RepL4NLP-2021)</i>, pages 163-173, August 2021.</p></li>
+
+<li><p>[7] Minghan Li, Sheng-Chieh Lin, Xueguang Ma, Jimmy Lin.
+<a href="https://arxiv.org/abs/2302.06587">SLIM: Sparsified Late Interaction for Multi-Vector Retrieval with Inverted Indexes.</a>
+<i>arXiv:2302.06587</i>, Feburary 2023.</p></li>
+
+<li><p>[8] Sheng-Chieh Lin, Minghan Li and Jimmy Lin.
+<a href="https://arxiv.org/abs/2208.00511">Aggretriever: A Simple Approach to Aggregate Textual Representation for Robust Dense Passage Retrieval.</a>
+<i>arXiv:2208.00511</i>, July 2022.</p></li>
+
+</ul>
+
+<div style="padding-top: 20px"/>
+
+<h4>Programmatic Execution</h4>
+
+<p>All experimental runs shown in the above table can be programmatically executed based on the instructions below.
+To list all the experimental conditions:</p>
+
+<blockquote class="mycode2"><tt>
+python -m pyserini.2cr.msmarco --collection v1-passage --list-conditions
+</tt></blockquote>
+
+<p>These conditions correspond to the table rows above.</p>
+
+<p>For all conditions, just show the commands in a "dry run":</p>
+
+<blockquote class="mycode2"><tt>
+python -m pyserini.2cr.msmarco --collection v1-passage --all --display-commands --dry-run
+</tt></blockquote>
+
+<p>To actually run all the experimental conditions:</p>
+
+<blockquote class="mycode2"><tt>
+python -m pyserini.2cr.msmarco --collection v1-passage --all --display-commands
+</tt></blockquote>
+
+<p>With the above command, run files will be placed in the current directory.
+Use the option <tt>--directory runs/</tt> to place the runs in a sub-directory.</p>
+
+<p>To show the commands for a specific condition:</p>
+
+<blockquote class="mycode2"><tt>
+python -m pyserini.2cr.msmarco --collection v1-passage --condition bm25-default --display-commands --dry-run
+</tt></blockquote>
+
+<p>This will generate exactly the commands for a specific condition above (corresponding to a row in the table).</p>
+
+<p>To actually run a specific condition:</p>
+
+<blockquote class="mycode2"><tt>
+python -m pyserini.2cr.msmarco --collection v1-passage --condition bm25-default --display-commands
+</tt></blockquote>
+
+<p>Again, with the above command, run files will be placed in the current directory.
+Use the option <tt>--directory runs/</tt> to place the runs in a sub-directory.</p>
+
+<p>Finally, to generate this page:</p>
+
+<blockquote class="mycode2"><tt>
+python -m pyserini.2cr.msmarco --collection v1-passage --generate-report --output msmarco-v1-passage.html
+</tt></blockquote>
+
+<p>The output file <tt>msmarco-v1-passage.html</tt> should be identical to this page.</p>
+
+<div style="padding-top: 50px"/>
+
+ </div>
+
+
+
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.0/jquery.min.js"></script>
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/4.3.1/js/bootstrap.min.js"></script>
+ <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mdb-ui-kit/4.0.0/mdb.min.js"></script>
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.10/clipboard.min.js"></script>
+
+<script>
+document.querySelectorAll('pre').forEach(function (codeBlock) {
+ var button = document.createElement('button');
+ button.className = 'copy-code-button';
+ button.type = 'button';
+ var s = codeBlock.innerText;
+ button.setAttribute('data-clipboard-text',s);
+ button.innerText = 'Copy';
+
+ // var pre = codeBlock.parentNode;
+ codeBlock.classList.add('prettyprint');
+ // pre.parentNode.insertBefore(button, pre);
+ codeBlock.appendChild(button);
+});
+
+var clipboard = new ClipboardJS('.copy-code-button');
+
+clipboard.on('success', function(e) {
+ console.info('Action:', e.action);
+ console.info('Text:', e.text);
+ console.info('Trigger:', e.trigger);
+ e.trigger.textContent = 'Copied';
+ window.setTimeout(function() {
+ e.trigger.textContent = 'Copy';
+ }, 2000);
+ e.clearSelection();
+});
+
+clipboard.on('error', function(e) {
+ console.error('Action:', e.action);
+ console.error('Trigger:', e.trigger);
+ e.trigger.textContent = 'Error Copying';
+ window.setTimeout(function() {
+ e.trigger.textContent = 'Copy';
+ }, 2000);
+ e.clearSelection();
+});
+
+</script>
+
+</body>
+</html>

pyserini/2cr/msmarco_html_v2_doc.template

@@ -0,0 +1,292 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no" />
+ <meta http-equiv="x-ua-compatible" content="ie=edge" />
+ <title>Pyserini Reproductions: MS MARCO V2 Document</title>
+ <!-- Font Awesome -->
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.11.2/css/all.css" />
+ <!-- Google Fonts Roboto -->
+ <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Roboto:wght@300;400;500;700&display=swap" />
+ <!-- MDB -->
+ <link href="https://cdnjs.cloudflare.com/ajax/libs/mdb-ui-kit/4.0.0/mdb.min.css" rel="stylesheet" />
+
+ <style>
+tr.hide-table-padding td {
+ padding: 0;
+}
+
+.expand-button {
+ position: relative;
+}
+
+.accordion-toggle .expand-button:after {
+ position: absolute;
+ left:.75rem;
+ top: 50%;
+ transform: translate(0, -50%);
+ content: '-';
+}
+
+.accordion-toggle.collapsed .expand-button:after {
+ content: '+';
+}
+
+blockquote.mycode {
+ border-left: 3px solid #ccc;
+ margin-left: 25px;
+ margin-top: 15px;
+ padding-left: 15px;
+}
+
+blockquote.mycode2 {
+ border-left: 3px solid #ccc;
+ margin-left: 25px;
+ padding-top: 10px;
+ padding-bottom: 10px;
+ padding-left: 15px;
+}
+
+tr th.headertop {
+ border-bottom: none;
+ padding-bottom: 0rem
+}
+
+tr th.headerbottom {
+ padding-top: 0rem
+}
+
+.table>:not(caption)>*>*{padding:0.75rem 0.75rem}
+
+.copy-code-button {
+ border-radius: 0;
+ min-width: 55px;
+ background: none repeat scroll 0 0 transparent;
+ background-color: grey;
+ color: #F1F2F3 !important;
+ cursor: pointer;
+ border-style: none;
+ font-family: 'HELVETICA',sans-serif;
+ font-size: 0.8em;
+ font-weight: normal;
+ text-align: center;
+ text-decoration: none;
+ text-indent: 0;
+ text-transform: uppercase;
+ font-weight: 500;
+ line-height: 1.42rem;
+ margin: 0;
+ padding: 3px 8px;
+ position: absolute !important;
+ top: 0 !important;
+ right: 0 !important;
+}
+
+.copy-code-button > span {
+ color: #F1F2F3 !important;
+}
+
+.copy-code-button, ::before, ::after {
+ box-sizing: inherit;
+}
+
+.copy-code-button::before {
+ content: '';
+ display: inline-block;
+ width: 16px;
+ height: 16px;
+ margin-right: 3px;
+ background-size: contain;
+ background-image: url("data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiPz4KPHN2ZyB3aWR0aD0iMTVweCIgaGVpZ2h0PSIxNXB4IiB2aWV3Qm94PSIwIDAgMTUgMTUiIHZlcnNpb249IjEuMSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB4bWxuczp4bGluaz0iaHR0cDovL3d3dy53My5vcmcvMTk5OS94bGluayI+CiAgICA8IS0tIEdlbmVyYXRvcjogU2tldGNoIDUwLjIgKDU1MDQ3KSAtIGh0dHA6Ly93d3cuYm9oZW1pYW5jb2RpbmcuY29tL3NrZXRjaCAtLT4KICAgIDx0aXRsZT5QYWdlIDE8L3RpdGxlPgogICAgPGRlc2M+Q3JlYXRlZCB3aXRoIFNrZXRjaC48L2Rlc2M+CiAgICA8ZGVmcz48L2RlZnM+CiAgICA8ZyBpZD0iRmxvdyIgc3Ryb2tlPSJub25lIiBzdHJva2Utd2lkdGg9IjEiIGZpbGw9Im5vbmUiIGZpbGwtcnVsZT0iZXZlbm9kZCI+CiAgICAgICAgPGcgaWQ9IkJ0dG5faHRtbCIgdHJhbnNmb3JtPSJ0cmFuc2xhdGUoLTgxOS4wMDAwMDAsIC03NTMuMDAwMDAwKSIgZmlsbD0iI0ZGRkZGRiI+CiAgICAgICAgICAgIDxnIGlkPSJHcm91cC0xIiB0cmFuc2Zvcm09InRyYW5zbGF0ZSgzMTEuMDAwMDAwLCA0MDUuMDAwMDAwKSI+CiAgICAgICAgICAgICAgICA8ZyBpZD0iR3JvdXAtMiIgdHJhbnNmb3JtPSJ0cmFuc2xhdGUoNTA4LjAwMDAwMCwgMzQyLjAwMDAwMCkiPgogICAgICAgICAgICAgICAgICAgIDxwYXRoIGQ9Ik0xMy45NzcyNzI3LDYgTDMuNDA5MDkwOTEsNiBDMi44NDQ1NDU0NSw2IDIuMzg2MzYzNjQsNi40NTgxODE4MiAyLjM4NjM2MzY0LDcuMDIyNzI3MjcgTDIuMzg2MzYzNjQsMTcuNTkwOTA5MSBDMi4zODYzNjM2NCwxOC4xNTU0NTQ1IDIuODQ0NTQ1NDUsMTguNjEzNjM2NCAzLjQwOTA5MDkxLDE4LjYxMzYzNjQgTDEzLjk3NzI3MjcsMTguNjEzNjM2NCBDMTQuNTQxODE4MiwxOC42MTM2MzY0IDE1LDE4LjE1NTQ1NDUgMTUsMTcuNTkwOTA5MSBMMTUsNy4wMjI3MjcyNyBDMTUsNi40NTgxODE4MiAxNC41NDE4MTgyLDYgMTMuOTc3MjcyNyw2IFogTTE0LjMxODE4MTgsMTcuNTkwOTA5MSBDMTQuMzE4MTgxOCwxNy43NzkwOTA5IDE0LjE2NTQ1NDUsMTcuOTMxODE4MiAxMy45NzcyNzI3LDE3LjkzMTgxODIgTDMuNDA5MDkwOTEsMTcuOTMxODE4MiBDMy4yMjA5MDkwOSwxNy45MzE4MTgyIDMuMDY4MTgxODIsMTcuNzc5MDkwOSAzLjA2ODE4MTgyLDE3LjU5MDkwOTEgTDMuMDY4MTgxODIsNy4wMjI3MjcyNyBDMy4wNjgxODE4Miw2LjgzNDU0NTQ1IDMuMjIwOTA5MDksNi42ODE4MTgxOCAzLjQwOTA5MDkxLDYuNjgxODE4MTggTDEzLjk3NzI3MjcsNi42ODE4MTgxOCBDMTQuMTY1NDU0NSw2LjY4MTgxODE4IDE0LjMxODE4MTgsNi44MzQ1NDU0NSAxNC4zMTgxODE4LDcuMDIyNzI3MjcgTDE0LjMxODE4MTgsMTcuNTkwOTA5MSBaIE0xMS45MzE4MTgyLDE5Ljk3NzI3MjcgQzExLjkzMTgxODIsMjAuMTY1NDU0NSAxMS43NzkwOTA5LDIwLjMxODE4MTggMTEuNTkwOTA5MSwyMC4zMTgxODE4IEwxLjAyMjcyNzI3LDIwLjMxODE4MTggQzAuODM0NTQ1NDU1LDIwLjMxODE4MTggMC42ODE4MTgxODIsMjAuMTY1NDU0NSAwLjY4MTgxODE4MiwxOS45NzcyNzI3IEwwLjY4MTgxODE4Miw5LjQwOTA5MDkxIEMwLjY4MTgxODE4Miw5LjIyMDkwOTA5IDAuODM0NTQ1NDU1LDkuMDY4MTgxODIgMS4wMjI3MjcyNyw5LjA2ODE4MTgyIEwxLjM2MzYzNjM2LDkuMDY4MTgxODIgTDEuMzYzNjM2MzYsOC4zODYzNjM2NCBMMS4wMjI3MjcyNyw4LjM4NjM2MzY0IEMwLjQ1ODE4MTgxOCw4LjM4NjM2MzY0IDAsOC44NDQ1NDU0NSAwLDkuNDA5MDkwOTEgTDAsMTkuOTc3MjcyNyBDMCwyMC41NDE4MTgyIDAuNDU4MTgxODE4LDIxIDEuMDIyNzI3MjcsMjEgTDExLjU5MDkwOTEsMjEgQzEyLjE1NTQ1NDUsMjEgMTIuNjEzNjM2NCwyMC41NDE4MTgyIDEyLjYxMzYzNjQsMTkuOTc3MjcyNyBMMTIuNjEzNjM2NCwxOS42MzYzNjM2IEwxMS45MzE4MTgyLDE5LjYzNjM2MzYgTDExLjkzMTgxODIsMTkuOTc3MjcyNyBaIiBpZD0iUGFnZS0xIj48L3BhdGg+CiAgICAgICAgICAgICAgICA8L2c+CiAgICAgICAgICAgIDwvZz4KICAgICAgICA8L2c+CiAgICA8L2c+Cjwvc3ZnPg==");
+ background-repeat: no-repeat;
+ position: relative;
+ top: 3px;
+}
+
+.copy-code-button:focus {
+ /* Avoid an ugly focus outline on click in Chrome,
+ but darken the button for accessibility.
+ See https://stackoverflow.com/a/25298082/1481479 */
+ /* background-color: #E6E6E6; */
+ outline: 0;
+}
+
+pre[class*="prettyprint"] {
+ position: relative;
+ overflow: hidden;
+}
+ </style>
+</head>
+<body>
+
+ <!-- Background image -->
+ <div id="intro" class="bg-image vh-100 shadow-1-strong" style="max-height: 150px">
+ <div class="mask" style="
+ background: linear-gradient(
+ 45deg,
+ rgba(29, 236, 197, 0.7),
+ rgba(91, 14, 214, 0.7) 100%
+ );
+ ">
+ <div class="container d-flex align-items-center justify-content-center text-center h-100" style="max-height: 150px">
+ <div class="text-white">
+ <h1 class="mb-3">$title</h1>
+ </div>
+ </div>
+ </div>
+ </div>
+ <!-- Background image -->
+
+<div class="container my-4">
+
+<p>The two-click<a href="#" data-mdb-toggle="tooltip" title="What are the two clicks, you ask? Copy and paste!"><sup>*</sup></a> reproduction matrix below provides commands for reproducing experimental results reported in the following paper.
+Numbered rows correspond to tables in the paper; additional conditions are provided for comparison purposes.</p>
+
+<p class="note note-light">Xueguang Ma, Ronak Pradeep, Rodrigo Nogueira, and Jimmy Lin. <a href="https://cs.uwaterloo.ca/~jimmylin/publications/Ma_etal_SIGIR2022.pdf">Document Expansions and Learned Sparse Lexical Representations for MS MARCO V1 and V2.</a>
+<i>Proceedings of the 45th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR 2022)</i>, July 2022.</p>
+
+<p>Instructions for programmatic execution are shown at the bottom of this page (scroll down).</p>
+
+<div class="table-responsive">
+ <table class="table">
+ <thead>
+ <tr>
+ <th class="headertop"></th>
+ <th class="headertop"></th>
+ <th class="headertop"></th>
+ <th class="headertop" colspan="6"><b>TREC 2021</b></th>
+ <th class="headertop" colspan="3"><b>dev</b></th>
+ <th class="headertop" colspan="3"><b>dev2</b></th>
+ </tr>
+ <tr>
+ <th class="headerbottom" scope="col"></th>
+ <th class="headerbottom" scope="col"></th>
+ <th class="headerbottom" scope="col"></th>
+ <th class="headerbottom" scope="col"><br/>AP</th>
+ <th class="headerbottom" scope="col">nDCG@10</th>
+ <th class="headerbottom" scope="col">RR@100</th>
+ <th class="headerbottom" scope="col">R@100</th>
+ <th class="headerbottom" scope="col">R@1K</th>
+ <th class="headerbottom" scope="col"></th>
+ <th class="headerbottom" scope="col">RR@100</th>
+ <th class="headerbottom" scope="col">R@1K</th>
+ <th class="headerbottom" scope="col"></th>
+ <th class="headerbottom" scope="col">RR@100</th>
+ <th class="headerbottom" scope="col">R@1K</th>
+ </tr>
+ </thead>
+ <tbody>
+
+$rows
+
+ </tbody>
+ </table>
+</div>
+
+<div style="padding-top: 20px"/>
+
+<h4>Programmatic Execution</h4>
+
+<p>All experimental runs shown in the above table can be programmatically executed based on the instructions below.
+To list all the experimental conditions:</p>
+
+<blockquote class="mycode2"><tt>
+python -m pyserini.2cr.msmarco --collection v2-doc --list-conditions
+</tt></blockquote>
+
+<p>These conditions correspond to the table rows above.</p>
+
+<p>For all conditions, just show the commands in a "dry run":</p>
+
+<blockquote class="mycode2"><tt>
+python -m pyserini.2cr.msmarco --collection v2-doc --all --display-commands --dry-run
+</tt></blockquote>
+
+<p>To actually run all the experimental conditions:</p>
+
+<blockquote class="mycode2"><tt>
+python -m pyserini.2cr.msmarco --collection v2-doc --all --display-commands
+</tt></blockquote>
+
+<p>With the above command, run files will be placed in the current directory.
+Use the option <tt>--directory runs/</tt> to place the runs in a sub-directory.</p>
+
+<p>To show the commands for a specific condition:</p>
+
+<blockquote class="mycode2"><tt>
+python -m pyserini.2cr.msmarco --collection v2-doc --condition bm25-doc-default --display-commands --dry-run
+</tt></blockquote>
+
+<p>This will generate exactly the commands for a specific condition above (corresponding to a row in the table).</p>
+
+<p>To actually run a specific condition:</p>
+
+<blockquote class="mycode2"><tt>
+python -m pyserini.2cr.msmarco --collection v2-doc --condition bm25-doc-default --display-commands
+</tt></blockquote>
+
+<p>Again, with the above command, run files will be placed in the current directory.
+Use the option <tt>--directory runs/</tt> to place the runs in a sub-directory.</p>
+
+<p>Finally, to generate this page:</p>
+
+<blockquote class="mycode2"><tt>
+python -m pyserini.2cr.msmarco --collection v2-doc --generate-report --output msmarco-v2-doc.html
+</tt></blockquote>
+
+<p>The output file <tt>msmarco-v2-doc.html</tt> should be identical to this page.</p>
+
+<div style="padding-top: 50px"/>
+
+ </div>
+
+
+
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.0/jquery.min.js"></script>
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/4.3.1/js/bootstrap.min.js"></script>
+ <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mdb-ui-kit/4.0.0/mdb.min.js"></script>
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.10/clipboard.min.js"></script>
+
+<script>
+document.querySelectorAll('pre').forEach(function (codeBlock) {
+ var button = document.createElement('button');
+ button.className = 'copy-code-button';
+ button.type = 'button';
+ var s = codeBlock.innerText;
+ button.setAttribute('data-clipboard-text',s);
+ button.innerText = 'Copy';
+
+ // var pre = codeBlock.parentNode;
+ codeBlock.classList.add('prettyprint');
+ // pre.parentNode.insertBefore(button, pre);
+ codeBlock.appendChild(button);
+});
+
+var clipboard = new ClipboardJS('.copy-code-button');
+
+clipboard.on('success', function(e) {
+ console.info('Action:', e.action);
+ console.info('Text:', e.text);
+ console.info('Trigger:', e.trigger);
+ e.trigger.textContent = 'Copied';
+ window.setTimeout(function() {
+ e.trigger.textContent = 'Copy';
+ }, 2000);
+ e.clearSelection();
+});
+
+clipboard.on('error', function(e) {
+ console.error('Action:', e.action);
+ console.error('Trigger:', e.trigger);
+ e.trigger.textContent = 'Error Copying';
+ window.setTimeout(function() {
+ e.trigger.textContent = 'Copy';
+ }, 2000);
+ e.clearSelection();
+});
+
+</script>
+
+</body>
+</html>

pyserini/2cr/msmarco_html_v2_passage.template

@@ -0,0 +1,292 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no" />
+ <meta http-equiv="x-ua-compatible" content="ie=edge" />
+ <title>Pyserini Reproductions: MS MARCO V2 Passage</title>
+ <!-- Font Awesome -->
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.11.2/css/all.css" />
+ <!-- Google Fonts Roboto -->
+ <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Roboto:wght@300;400;500;700&display=swap" />
+ <!-- MDB -->
+ <link href="https://cdnjs.cloudflare.com/ajax/libs/mdb-ui-kit/4.0.0/mdb.min.css" rel="stylesheet" />
+
+ <style>
+tr.hide-table-padding td {
+ padding: 0;
+}
+
+.expand-button {
+ position: relative;
+}
+
+.accordion-toggle .expand-button:after {
+ position: absolute;
+ left:.75rem;
+ top: 50%;
+ transform: translate(0, -50%);
+ content: '-';
+}
+
+.accordion-toggle.collapsed .expand-button:after {
+ content: '+';
+}
+
+blockquote.mycode {
+ border-left: 3px solid #ccc;
+ margin-left: 25px;
+ margin-top: 15px;
+ padding-left: 15px;
+}
+
+blockquote.mycode2 {
+ border-left: 3px solid #ccc;
+ margin-left: 25px;
+ padding-top: 10px;
+ padding-bottom: 10px;
+ padding-left: 15px;
+}
+
+tr th.headertop {
+ border-bottom: none;
+ padding-bottom: 0rem
+}
+
+tr th.headerbottom {
+ padding-top: 0rem
+}
+
+.table>:not(caption)>*>*{padding:0.75rem 0.75rem}
+
+.copy-code-button {
+ border-radius: 0;
+ min-width: 55px;
+ background: none repeat scroll 0 0 transparent;
+ background-color: grey;
+ color: #F1F2F3 !important;
+ cursor: pointer;
+ border-style: none;
+ font-family: 'HELVETICA',sans-serif;
+ font-size: 0.8em;
+ font-weight: normal;
+ text-align: center;
+ text-decoration: none;
+ text-indent: 0;
+ text-transform: uppercase;
+ font-weight: 500;
+ line-height: 1.42rem;
+ margin: 0;
+ padding: 3px 8px;
+ position: absolute !important;
+ top: 0 !important;
+ right: 0 !important;
+}
+
+.copy-code-button > span {
+ color: #F1F2F3 !important;
+}
+
+.copy-code-button, ::before, ::after {
+ box-sizing: inherit;
+}
+
+.copy-code-button::before {
+ content: '';
+ display: inline-block;
+ width: 16px;
+ height: 16px;
+ margin-right: 3px;
+ background-size: contain;
+ background-image: url("data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiPz4KPHN2ZyB3aWR0aD0iMTVweCIgaGVpZ2h0PSIxNXB4IiB2aWV3Qm94PSIwIDAgMTUgMTUiIHZlcnNpb249IjEuMSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB4bWxuczp4bGluaz0iaHR0cDovL3d3dy53My5vcmcvMTk5OS94bGluayI+CiAgICA8IS0tIEdlbmVyYXRvcjogU2tldGNoIDUwLjIgKDU1MDQ3KSAtIGh0dHA6Ly93d3cuYm9oZW1pYW5jb2RpbmcuY29tL3NrZXRjaCAtLT4KICAgIDx0aXRsZT5QYWdlIDE8L3RpdGxlPgogICAgPGRlc2M+Q3JlYXRlZCB3aXRoIFNrZXRjaC48L2Rlc2M+CiAgICA8ZGVmcz48L2RlZnM+CiAgICA8ZyBpZD0iRmxvdyIgc3Ryb2tlPSJub25lIiBzdHJva2Utd2lkdGg9IjEiIGZpbGw9Im5vbmUiIGZpbGwtcnVsZT0iZXZlbm9kZCI+CiAgICAgICAgPGcgaWQ9IkJ0dG5faHRtbCIgdHJhbnNmb3JtPSJ0cmFuc2xhdGUoLTgxOS4wMDAwMDAsIC03NTMuMDAwMDAwKSIgZmlsbD0iI0ZGRkZGRiI+CiAgICAgICAgICAgIDxnIGlkPSJHcm91cC0xIiB0cmFuc2Zvcm09InRyYW5zbGF0ZSgzMTEuMDAwMDAwLCA0MDUuMDAwMDAwKSI+CiAgICAgICAgICAgICAgICA8ZyBpZD0iR3JvdXAtMiIgdHJhbnNmb3JtPSJ0cmFuc2xhdGUoNTA4LjAwMDAwMCwgMzQyLjAwMDAwMCkiPgogICAgICAgICAgICAgICAgICAgIDxwYXRoIGQ9Ik0xMy45NzcyNzI3LDYgTDMuNDA5MDkwOTEsNiBDMi44NDQ1NDU0NSw2IDIuMzg2MzYzNjQsNi40NTgxODE4MiAyLjM4NjM2MzY0LDcuMDIyNzI3MjcgTDIuMzg2MzYzNjQsMTcuNTkwOTA5MSBDMi4zODYzNjM2NCwxOC4xNTU0NTQ1IDIuODQ0NTQ1NDUsMTguNjEzNjM2NCAzLjQwOTA5MDkxLDE4LjYxMzYzNjQgTDEzLjk3NzI3MjcsMTguNjEzNjM2NCBDMTQuNTQxODE4MiwxOC42MTM2MzY0IDE1LDE4LjE1NTQ1NDUgMTUsMTcuNTkwOTA5MSBMMTUsNy4wMjI3MjcyNyBDMTUsNi40NTgxODE4MiAxNC41NDE4MTgyLDYgMTMuOTc3MjcyNyw2IFogTTE0LjMxODE4MTgsMTcuNTkwOTA5MSBDMTQuMzE4MTgxOCwxNy43NzkwOTA5IDE0LjE2NTQ1NDUsMTcuOTMxODE4MiAxMy45NzcyNzI3LDE3LjkzMTgxODIgTDMuNDA5MDkwOTEsMTcuOTMxODE4MiBDMy4yMjA5MDkwOSwxNy45MzE4MTgyIDMuMDY4MTgxODIsMTcuNzc5MDkwOSAzLjA2ODE4MTgyLDE3LjU5MDkwOTEgTDMuMDY4MTgxODIsNy4wMjI3MjcyNyBDMy4wNjgxODE4Miw2LjgzNDU0NTQ1IDMuMjIwOTA5MDksNi42ODE4MTgxOCAzLjQwOTA5MDkxLDYuNjgxODE4MTggTDEzLjk3NzI3MjcsNi42ODE4MTgxOCBDMTQuMTY1NDU0NSw2LjY4MTgxODE4IDE0LjMxODE4MTgsNi44MzQ1NDU0NSAxNC4zMTgxODE4LDcuMDIyNzI3MjcgTDE0LjMxODE4MTgsMTcuNTkwOTA5MSBaIE0xMS45MzE4MTgyLDE5Ljk3NzI3MjcgQzExLjkzMTgxODIsMjAuMTY1NDU0NSAxMS43NzkwOTA5LDIwLjMxODE4MTggMTEuNTkwOTA5MSwyMC4zMTgxODE4IEwxLjAyMjcyNzI3LDIwLjMxODE4MTggQzAuODM0NTQ1NDU1LDIwLjMxODE4MTggMC42ODE4MTgxODIsMjAuMTY1NDU0NSAwLjY4MTgxODE4MiwxOS45NzcyNzI3IEwwLjY4MTgxODE4Miw5LjQwOTA5MDkxIEMwLjY4MTgxODE4Miw5LjIyMDkwOTA5IDAuODM0NTQ1NDU1LDkuMDY4MTgxODIgMS4wMjI3MjcyNyw5LjA2ODE4MTgyIEwxLjM2MzYzNjM2LDkuMDY4MTgxODIgTDEuMzYzNjM2MzYsOC4zODYzNjM2NCBMMS4wMjI3MjcyNyw4LjM4NjM2MzY0IEMwLjQ1ODE4MTgxOCw4LjM4NjM2MzY0IDAsOC44NDQ1NDU0NSAwLDkuNDA5MDkwOTEgTDAsMTkuOTc3MjcyNyBDMCwyMC41NDE4MTgyIDAuNDU4MTgxODE4LDIxIDEuMDIyNzI3MjcsMjEgTDExLjU5MDkwOTEsMjEgQzEyLjE1NTQ1NDUsMjEgMTIuNjEzNjM2NCwyMC41NDE4MTgyIDEyLjYxMzYzNjQsMTkuOTc3MjcyNyBMMTIuNjEzNjM2NCwxOS42MzYzNjM2IEwxMS45MzE4MTgyLDE5LjYzNjM2MzYgTDExLjkzMTgxODIsMTkuOTc3MjcyNyBaIiBpZD0iUGFnZS0xIj48L3BhdGg+CiAgICAgICAgICAgICAgICA8L2c+CiAgICAgICAgICAgIDwvZz4KICAgICAgICA8L2c+CiAgICA8L2c+Cjwvc3ZnPg==");
+ background-repeat: no-repeat;
+ position: relative;
+ top: 3px;
+}
+
+.copy-code-button:focus {
+ /* Avoid an ugly focus outline on click in Chrome,
+ but darken the button for accessibility.
+ See https://stackoverflow.com/a/25298082/1481479 */
+ /* background-color: #E6E6E6; */
+ outline: 0;
+}
+
+pre[class*="prettyprint"] {
+ position: relative;
+ overflow: hidden;
+}
+ </style>
+</head>
+<body>
+
+ <!-- Background image -->
+ <div id="intro" class="bg-image vh-100 shadow-1-strong" style="max-height: 150px">
+ <div class="mask" style="
+ background: linear-gradient(
+ 45deg,
+ rgba(29, 236, 197, 0.7),
+ rgba(91, 14, 214, 0.7) 100%
+ );
+ ">
+ <div class="container d-flex align-items-center justify-content-center text-center h-100" style="max-height: 150px">
+ <div class="text-white">
+ <h1 class="mb-3">$title</h1>
+ </div>
+ </div>
+ </div>
+ </div>
+ <!-- Background image -->
+
+<div class="container my-4">
+
+<p>The two-click<a href="#" data-mdb-toggle="tooltip" title="What are the two clicks, you ask? Copy and paste!"><sup>*</sup></a> reproduction matrix below provides commands for reproducing experimental results reported in the following paper.
+Numbered rows correspond to tables in the paper; additional conditions are provided for comparison purposes.</p>
+
+<p class="note note-light">Xueguang Ma, Ronak Pradeep, Rodrigo Nogueira, and Jimmy Lin. <a href="https://cs.uwaterloo.ca/~jimmylin/publications/Ma_etal_SIGIR2022.pdf">Document Expansions and Learned Sparse Lexical Representations for MS MARCO V1 and V2.</a>
+<i>Proceedings of the 45th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR 2022)</i>, July 2022.</p>
+
+<p>Instructions for programmatic execution are shown at the bottom of this page (scroll down).</p>
+
+<div class="table-responsive">
+ <table class="table">
+ <thead>
+ <tr>
+ <th class="headertop"></th>
+ <th class="headertop"></th>
+ <th class="headertop"></th>
+ <th class="headertop" colspan="6"><b>TREC 2021</b></th>
+ <th class="headertop" colspan="3"><b>dev</b></th>
+ <th class="headertop" colspan="3"><b>dev2</b></th>
+ </tr>
+ <tr>
+ <th class="headerbottom" scope="col"></th>
+ <th class="headerbottom" scope="col"></th>
+ <th class="headerbottom" scope="col"></th>
+ <th class="headerbottom" scope="col"><br/>AP</th>
+ <th class="headerbottom" scope="col">nDCG@10</th>
+ <th class="headerbottom" scope="col">RR@100</th>
+ <th class="headerbottom" scope="col">R@100</th>
+ <th class="headerbottom" scope="col">R@1K</th>
+ <th class="headerbottom" scope="col"></th>
+ <th class="headerbottom" scope="col">RR@100</th>
+ <th class="headerbottom" scope="col">R@1K</th>
+ <th class="headerbottom" scope="col"></th>
+ <th class="headerbottom" scope="col">RR@100</th>
+ <th class="headerbottom" scope="col">R@1K</th>
+ </tr>
+ </thead>
+ <tbody>
+
+$rows
+
+ </tbody>
+ </table>
+</div>
+
+<div style="padding-top: 20px"/>
+
+<h4>Programmatic Execution</h4>
+
+<p>All experimental runs shown in the above table can be programmatically executed based on the instructions below.
+To list all the experimental conditions:</p>
+
+<blockquote class="mycode2"><tt>
+python -m pyserini.2cr.msmarco --collection v2-passage --list-conditions
+</tt></blockquote>
+
+<p>These conditions correspond to the table rows above.</p>
+
+<p>For all conditions, just show the commands in a "dry run":</p>
+
+<blockquote class="mycode2"><tt>
+python -m pyserini.2cr.msmarco --collection v2-passage --all --display-commands --dry-run
+</tt></blockquote>
+
+<p>To actually run all the experimental conditions:</p>
+
+<blockquote class="mycode2"><tt>
+python -m pyserini.2cr.msmarco --collection v2-passage --all --display-commands
+</tt></blockquote>
+
+<p>With the above command, run files will be placed in the current directory.
+Use the option <tt>--directory runs/</tt> to place the runs in a sub-directory.</p>
+
+<p>To show the commands for a specific condition:</p>
+
+<blockquote class="mycode2"><tt>
+python -m pyserini.2cr.msmarco --collection v2-passage --condition bm25-default --display-commands --dry-run
+</tt></blockquote>
+
+<p>This will generate exactly the commands for a specific condition above (corresponding to a row in the table).</p>
+
+<p>To actually run a specific condition:</p>
+
+<blockquote class="mycode2"><tt>
+python -m pyserini.2cr.msmarco --collection v2-passage --condition bm25-default --display-commands
+</tt></blockquote>
+
+<p>Again, with the above command, run files will be placed in the current directory.
+Use the option <tt>--directory runs/</tt> to place the runs in a sub-directory.</p>
+
+<p>Finally, to generate this page:</p>
+
+<blockquote class="mycode2"><tt>
+python -m pyserini.2cr.msmarco --collection v2-passage --generate-report --output msmarco-v2-passage.html
+</tt></blockquote>
+
+<p>The output file <tt>msmarco-v2-passage.html</tt> should be identical to this page.</p>
+
+<div style="padding-top: 50px"/>
+
+ </div>
+
+
+
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.0/jquery.min.js"></script>
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/4.3.1/js/bootstrap.min.js"></script>
+ <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mdb-ui-kit/4.0.0/mdb.min.js"></script>
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.10/clipboard.min.js"></script>
+
+<script>
+document.querySelectorAll('pre').forEach(function (codeBlock) {
+ var button = document.createElement('button');
+ button.className = 'copy-code-button';
+ button.type = 'button';
+ var s = codeBlock.innerText;
+ button.setAttribute('data-clipboard-text',s);
+ button.innerText = 'Copy';
+
+ // var pre = codeBlock.parentNode;
+ codeBlock.classList.add('prettyprint');
+ // pre.parentNode.insertBefore(button, pre);
+ codeBlock.appendChild(button);
+});
+
+var clipboard = new ClipboardJS('.copy-code-button');
+
+clipboard.on('success', function(e) {
+ console.info('Action:', e.action);
+ console.info('Text:', e.text);
+ console.info('Trigger:', e.trigger);
+ e.trigger.textContent = 'Copied';
+ window.setTimeout(function() {
+ e.trigger.textContent = 'Copy';
+ }, 2000);
+ e.clearSelection();
+});
+
+clipboard.on('error', function(e) {
+ console.error('Action:', e.action);
+ console.error('Trigger:', e.trigger);
+ e.trigger.textContent = 'Error Copying';
+ window.setTimeout(function() {
+ e.trigger.textContent = 'Copy';
+ }, 2000);
+ e.clearSelection();
+});
+
+</script>
+
+</body>
+</html>

pyserini/__init__.py

@@ -0,0 +1 @@
+

pyserini/analysis/__init__.py

@@ -0,0 +1,19 @@
+#
+# Pyserini: Reproducible IR research with sparse and dense representations
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from ._base import get_lucene_analyzer, Analyzer, JAnalyzer, JAnalyzerUtils, JDefaultEnglishAnalyzer, JWhiteSpaceAnalyzer
+
+__all__ = ['get_lucene_analyzer', 'Analyzer', 'JAnalyzer', 'JAnalyzerUtils', 'JDefaultEnglishAnalyzer', 'JWhiteSpaceAnalyzer']

pyserini/analysis/_base.py

@@ -0,0 +1,166 @@
+#
+# Pyserini: Reproducible IR research with sparse and dense representations
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from typing import List
+
+from ..pyclass import autoclass
+
+# Wrappers around Lucene classes
+JAnalyzer = autoclass('org.apache.lucene.analysis.Analyzer')
+JArabicAnalyzer = autoclass('org.apache.lucene.analysis.ar.ArabicAnalyzer')
+JBengaliAnalyzer = autoclass('org.apache.lucene.analysis.bn.BengaliAnalyzer')
+JCJKAnalyzer = autoclass('org.apache.lucene.analysis.cjk.CJKAnalyzer')
+JDanishAnalyzer = autoclass('org.apache.lucene.analysis.da.DanishAnalyzer')
+JDefaultEnglishAnalyzer = autoclass('io.anserini.analysis.DefaultEnglishAnalyzer')
+JDutchAnalyzer = autoclass('org.apache.lucene.analysis.nl.DutchAnalyzer')
+JFinnishAnalyzer = autoclass('org.apache.lucene.analysis.fi.FinnishAnalyzer')
+JFrenchAnalyzer = autoclass('org.apache.lucene.analysis.fr.FrenchAnalyzer')
+JGermanAnalyzer = autoclass('org.apache.lucene.analysis.de.GermanAnalyzer')
+JHindiAnalyzer = autoclass('org.apache.lucene.analysis.hi.HindiAnalyzer')
+JHungarianAnalyzer = autoclass('org.apache.lucene.analysis.hu.HungarianAnalyzer')
+JIndonesianAnalyzer = autoclass('org.apache.lucene.analysis.id.IndonesianAnalyzer')
+JItalianAnalyzer = autoclass('org.apache.lucene.analysis.it.ItalianAnalyzer')
+JJapaneseAnalyzer = autoclass('org.apache.lucene.analysis.ja.JapaneseAnalyzer')
+JNorwegianAnalyzer = autoclass('org.apache.lucene.analysis.no.NorwegianAnalyzer')
+JPortugueseAnalyzer = autoclass('org.apache.lucene.analysis.pt.PortugueseAnalyzer')
+JRussianAnalyzer = autoclass('org.apache.lucene.analysis.ru.RussianAnalyzer')
+JSpanishAnalyzer = autoclass('org.apache.lucene.analysis.es.SpanishAnalyzer')
+JSwedishAnalyzer = autoclass('org.apache.lucene.analysis.sv.SwedishAnalyzer')
+JTeluguAnalyzer = autoclass('org.apache.lucene.analysis.te.TeluguAnalyzer')
+JThaiAnalyzer = autoclass('org.apache.lucene.analysis.th.ThaiAnalyzer')
+JTurkishAnalyzer = autoclass('org.apache.lucene.analysis.tr.TurkishAnalyzer')
+JWhiteSpaceAnalyzer = autoclass('org.apache.lucene.analysis.core.WhitespaceAnalyzer')
+JCharArraySet = autoclass('org.apache.lucene.analysis.CharArraySet')
+
+# Wrappers around Anserini classes
+JAnalyzerUtils = autoclass('io.anserini.analysis.AnalyzerUtils')
+JDefaultEnglishAnalyzer = autoclass('io.anserini.analysis.DefaultEnglishAnalyzer')
+JTweetAnalyzer = autoclass('io.anserini.analysis.TweetAnalyzer')
+JHuggingFaceTokenizerAnalyzer = autoclass('io.anserini.analysis.HuggingFaceTokenizerAnalyzer')
+
+
+def get_lucene_analyzer(language: str='en', stemming: bool=True, stemmer: str='porter', stopwords: bool=True, huggingFaceTokenizer: str=None) -> JAnalyzer:
+ """Create a Lucene ``Analyzer`` with specific settings.
+
+ Parameters
+ ----------
+ language : str
+ Name of analyzer.
+ stemming : bool
+ Set to stem.
+ stemmer : str
+ Stemmer to use.
+ stopwords : bool
+ Set to filter stopwords.
+ huggingFaceTokenizer: str
+ a huggingface model id or path to a tokenizer.json file
+
+ Returns
+ -------
+ JAnalyzer
+ Java ``Analyzer`` with specified settings.
+ """
+ if language.lower() == 'ar':
+ return JArabicAnalyzer()
+ elif language.lower() == 'bn':
+ return JBengaliAnalyzer()
+ elif language.lower() in ['zh', 'ko']:
+ return JCJKAnalyzer()
+ elif language.lower() == 'da':
+ return JDanishAnalyzer()
+ elif language.lower() == 'nl':
+ return JDutchAnalyzer()
+ elif language.lower() == 'fi':
+ return JFinnishAnalyzer()
+ elif language.lower() == 'fr':
+ return JFrenchAnalyzer()
+ elif language.lower() == 'de':
+ return JGermanAnalyzer()
+ elif language.lower() == 'hi':
+ return JHindiAnalyzer()
+ elif language.lower() == 'hu':
+ return JHungarianAnalyzer()
+ elif language.lower() == 'id':
+ return JIndonesianAnalyzer()
+ elif language.lower() == 'it':
+ return JItalianAnalyzer()
+ elif language.lower() == 'ja':
+ return JJapaneseAnalyzer()
+ elif language.lower() == 'no':
+ return JNorwegianAnalyzer()
+ elif language.lower() == 'pt':
+ return JPortugueseAnalyzer()
+ elif language.lower() == 'ru':
+ return JRussianAnalyzer()
+ elif language.lower() == 'es':
+ return JSpanishAnalyzer()
+ elif language.lower() == 'te':
+ return JTeluguAnalyzer()
+ elif language.lower() == 'th':
+ return JThaiAnalyzer()
+ elif language.lower() == 'tr':
+ return JTurkishAnalyzer()
+ elif language.lower() == 'tweet':
+ return JTweetAnalyzer()
+ elif language.lower() == 'hgf_tokenizer':
+ return JHuggingFaceTokenizerAnalyzer(huggingFaceTokenizer)
+ elif language.lower() == 'en':
+ if stemming:
+ if stopwords:
+ return JDefaultEnglishAnalyzer.newStemmingInstance(stemmer)
+ else:
+ return JDefaultEnglishAnalyzer.newStemmingInstance(stemmer, JCharArraySet.EMPTY_SET)
+ else:
+ if stopwords:
+ return JDefaultEnglishAnalyzer.newNonStemmingInstance()
+ else:
+ return JDefaultEnglishAnalyzer.newNonStemmingInstance(JCharArraySet.EMPTY_SET)
+ else:
+ raise ValueError('Invalid configuration.')
+
+
+class Analyzer:
+ """Python wrapper around a Lucene ``Analyzer`` to simplify analysis.
+
+ Parameters
+ ----------
+ analyzer : JAnalyzer
+ Lucene ``Analyzer``.
+ """
+
+ def __init__(self, analyzer):
+ if not isinstance(analyzer, JAnalyzer):
+ raise TypeError('Invalid JAnalyzer!')
+ self.analyzer = analyzer
+
+ def analyze(self, text: str) -> List[str]:
+ """Analyze a piece of text.
+
+ Parameters
+ ----------
+ text : str
+ Text to analyze.
+
+ Returns
+ -------
+ List[str]
+ List of tokens corresponding to the output of the analyzer.
+ """
+ results = JAnalyzerUtils.analyze(self.analyzer, text)
+ tokens = []
+ for token in results.toArray():
+ tokens.append(token)
+ return tokens

pyserini/collection/__init__.py

@@ -0,0 +1,20 @@
+#
+# Pyserini: Reproducible IR research with sparse and dense representations
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from ._base import Collection, FileSegment, SourceDocument
+from ._collection_support import Cord19Article
+
+__all__ = ['Collection', 'FileSegment', 'SourceDocument', 'Cord19Article']

pyserini/collection/_base.py

@@ -0,0 +1,153 @@
+#
+# Pyserini: Reproducible IR research with sparse and dense representations
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import logging
+import re
+from enum import Enum
+
+from ..multithreading import Counters
+from ..pyclass import autoclass, cast, JPaths
+
+logger = logging.getLogger(__name__)
+
+
+JFileSegment = autoclass('io.anserini.collection.FileSegment')
+JSourceDocument = autoclass('io.anserini.collection.SourceDocument')
+
+
+class JCollections(Enum):
+ AclAnthology = autoclass('io.anserini.collection.AclAnthology')
+ CarCollection = autoclass('io.anserini.collection.CarCollection')
+ Cord19AbstractCollection = autoclass('io.anserini.collection.Cord19AbstractCollection')
+ ClueWeb09Collection = autoclass('io.anserini.collection.ClueWeb09Collection')
+ ClueWeb12Collection = autoclass('io.anserini.collection.ClueWeb12Collection')
+ HtmlCollection = autoclass('io.anserini.collection.HtmlCollection')
+ JsonCollection = autoclass('io.anserini.collection.JsonCollection')
+ NewYorkTimesCollection = autoclass('io.anserini.collection.NewYorkTimesCollection')
+ TrecCollection = autoclass('io.anserini.collection.TrecCollection')
+ TrecwebCollection = autoclass('io.anserini.collection.TrecwebCollection')
+ TweetCollection = autoclass('io.anserini.collection.TweetCollection')
+ WashingtonPostCollection = autoclass('io.anserini.collection.WashingtonPostCollection')
+ WikipediaCollection = autoclass('io.anserini.collection.WikipediaCollection')
+
+
+class Collection:
+ """
+ Iterable wrapper class for Anserini's DocumentCollection.
+
+ Parameters
+ ----------
+ collection_class : str
+ Name of collection class to instantiate
+ collection_path : str
+ Path to directory containing collection
+ """
+
+ def __init__(self, collection_class, collection_path):
+ self.counters = Counters()
+ self.collection_class = collection_class
+ self.collection_path = JPaths.get(collection_path)
+ self.object = self._get_collection()
+ self.collection_iterator = self.object.iterator()
+
+ def _get_collection(self):
+ try:
+ return JCollections[self.collection_class].value(self.collection_path)
+ except:
+ raise ValueError(self.collection_class)
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ if self.collection_iterator.hasNext():
+ fs = self.collection_iterator.next()
+ return FileSegment(self, fs, fs.getSegmentPath())
+ else:
+ raise StopIteration
+
+
+class FileSegment:
+ """
+ Iterable wrapper class for Anserini's FileSegment.
+
+ Parameters
+ ----------
+ collection : Collection
+ Parent collection of the file segment
+ segment : JFileSegment
+ FileSegment object to create wrapper from
+ segment_path : str
+ Path to file backing the file segment
+ """
+
+ def __init__(self, collection, segment, segment_path):
+ self.collection = collection
+ try:
+ self.object = cast(collection.object.getClass().getName() +
+ '$Segment', segment)
+ except:
+ logger.exception('Exception from casting FileSegment type...')
+ self.object = cast('io.anserini.collection.FileSegment', segment)
+
+ self.segment_iterator = self.object.iterator()
+ self.segment_path = segment_path
+ self.segment_name = re.sub(r'\\|\/', '-', collection.collection_path.relativize(segment_path).toString())
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ if self.object.iterator().hasNext():
+ d = self.object.iterator().next()
+ return SourceDocument(self, d)
+ else:
+ # log if iteration stopped by error
+ if self.object.getErrorStatus():
+ logger.error(self.segment_name + ': Error from segment iteration, stopping...')
+ self.collection.counters.errors.increment()
+
+ # stop iteration and log skipped documents
+ skipped = self.object.getSkippedCount()
+ if skipped > 0:
+ self.collection.counters.skips.increment(skipped)
+ logger.warning(self.segment_name + ': ' + str(skipped) + ' documents skipped')
+ self.object.close()
+ raise StopIteration
+
+
+class SourceDocument:
+ """
+ Wrapper class for Anserini's SourceDocument.
+
+ Parameters
+ ----------
+
+ segment : FileSegment
+ Parent segment of the source document
+ document : io.anserini.collection.SourceDocument
+ SourceDocument object to create wrapper from
+ """
+
+ def __init__(self, segment, document):
+ if not isinstance(document, JSourceDocument):
+ raise TypeError('Invalid JSourceDocument!')
+ self.segment = segment
+ self.object = document
+ self.id = self.object.id()
+ self.indexable = self.object.indexable()
+ self.contents = self.object.contents()
+ self.raw = self.object.raw()

pyserini/collection/_collection_support.py

@@ -0,0 +1,78 @@
+#
+# Pyserini: Reproducible IR research with sparse and dense representations
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Implementations of support for specific collections.
+
+import json
+
+
+class Cord19Article:
+ """Wrapper class for a raw JSON article from AI2's COVID-19 Open Research Dataset (CORD-19).
+
+ Parameters
+ ----------
+ doc : str
+ A JSON string of a CORD-19 article.
+ """
+
+ def __init__(self, doc):
+ self.json = json.loads(doc)
+ # Performs some basic error checking, throws an exception if user tries to instantiate with something
+ # that isn't from CORD-19.
+ if 'cord_uid' in self.json:
+ self.full_text = False
+ elif 'paper_id' in self.json:
+ self.full_text = True
+ else:
+ raise TypeError
+
+ def is_full_text(self):
+ return self.json['has_full_text']
+
+ def cord_uid(self):
+ return self.json['cord_uid']
+
+ def bib_entries(self):
+ return self.json['bib_entries']
+
+ def title(self):
+ try:
+ if self.is_full_text():
+ return self.json['metadata']['title']
+ else:
+ return self.json['csv_metadata']['title']
+ except KeyError:
+ return ''
+
+ def abstract(self):
+ try:
+ # For a full-text article, we can grab the abstract from two independent sources, the metadata or the
+ # actual full text. Here, we make the decision to use the metadata, even for full text.
+ return self.json['csv_metadata']['abstract']
+ except KeyError:
+ return ''
+
+ def metadata(self):
+ return self.json['csv_metadata']
+
+ def body(self):
+ try:
+ if self.is_full_text():
+ return [entry['text'] for entry in self.json['body_text']]
+ else:
+ return []
+ except KeyError:
+ return ''

pyserini/demo/acl.py

@@ -0,0 +1,124 @@
+#
+# Pyserini: Reproducible IR research with sparse and dense representations
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+This script provides an interactive web interface demo for retrieval on the ACL dataset.
+It requires `flask` (`pip install flask~=2.2.0`).
+An example command looks like `python -m pyserini.demo.acl` that starts up a server on port 8080.
+The demo can be accessed via "http://localhost:8080" in a web browser.
+Additional arguments include:
+ --port [PORT] --hits [Number of hits]
+ --k1 [BM25 k1] --b [BM25 b] --device [cpu, cuda]
+"""
+import json
+import logging
+from argparse import ArgumentParser
+from functools import partial
+from typing import Callable, Optional, Tuple, Union
+
+from flask import Flask, render_template, request, flash, jsonify
+from pyserini.search import LuceneSearcher, FaissSearcher, AutoQueryEncoder
+
+logging.basicConfig(
+ format='%(asctime)s | %(levelname)s | %(name)s | %(message)s',
+ datefmt='%Y-%m-%d %H:%M:%S',
+ level=logging.INFO,
+)
+logger = logging.getLogger('acl-demo')
+
+VERSION = '1.0'
+Searcher = Union[FaissSearcher, LuceneSearcher]
+
+
+def create_app(k: int, load_searcher_fn: Callable[[str], Tuple[Searcher, str]]):
+ app = Flask(__name__)
+
+ lang = 'en'
+ searcher, retriever = load_searcher_fn(lang)
+
+ @app.route('/')
+ def index():
+ nonlocal lang, searcher, retriever
+ return render_template('acl.html', lang=lang, retriever=retriever)
+
+ @app.route('/search', methods=['GET', 'POST'])
+ def search():
+ nonlocal lang, searcher, retriever
+ query = request.form['q']
+ if not query:
+ search_results = []
+ flash('Question is required')
+ else:
+ hits = searcher.search(query, k=k)
+ docs = [searcher.doc(hit.docid) for hit in hits]
+ search_results = [
+ {
+ 'rank': r + 1,
+ 'docid': hit.docid,
+ 'doc': docs[r].contents(),
+ 'score': hit.score,
+ }
+ for r, hit in enumerate(hits)
+ ]
+ return render_template(
+ 'acl.html', search_results=search_results, query=query, lang=lang, retriever=retriever
+ )
+
+
+ return app
+
+
+def _load_sparse_searcher(language: str, k1: Optional[float]=None, b: Optional[float]=None) -> (Searcher, str):
+ searcher = LuceneSearcher('indexes/lucene-index-acl-paragraph')
+ searcher.set_language(language)
+ if k1 is not None and b is not None:
+ searcher.set_bm25(k1, b)
+ retriever_name = f'BM25 (k1={k1}, b={b})'
+ else:
+ retriever_name = 'BM25'
+
+ return searcher, retriever_name
+
+
+def main():
+ parser = ArgumentParser()
+
+ parser.add_argument('--k1', type=float, help='BM25 k1 parameter.')
+ parser.add_argument('--b', type=float, help='BM25 b parameter.')
+ parser.add_argument('--hits', type=int, default=10, help='Number of hits returned by the retriever')
+ parser.add_argument(
+ '--device',
+ type=str,
+ default='cpu',
+ help='Device to run query encoder, cpu or [cuda:0, cuda:1, ...] (used only when index is based on FAISS)',
+ )
+ parser.add_argument(
+ '--port',
+ default=8080,
+ type=int,
+ help='Web server port',
+ )
+
+ args = parser.parse_args()
+
+ load_fn = partial(_load_sparse_searcher, k1=args.k1, b=args.b)
+
+ app = create_app(args.hits, load_fn)
+ app.run(host='0.0.0.0', port=args.port)
+
+
+if __name__ == '__main__':
+ main()

pyserini/demo/dpr.py

@@ -0,0 +1,105 @@
+#
+# Pyserini: Reproducible IR research with sparse and dense representations
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import cmd
+import json
+import random
+
+from pyserini.search.lucene import LuceneSearcher
+from pyserini.search.faiss import FaissSearcher, DprQueryEncoder
+from pyserini.search.hybrid import HybridSearcher
+from pyserini import search
+
+
+class DPRDemo(cmd.Cmd):
+ nq_dev_topics = list(search.get_topics('dpr-nq-dev').values())
+ trivia_dev_topics = list(search.get_topics('dpr-trivia-dev').values())
+
+ ssearcher = LuceneSearcher.from_prebuilt_index('wikipedia-dpr')
+ searcher = ssearcher
+
+ encoder = DprQueryEncoder("facebook/dpr-question_encoder-multiset-base")
+ index = 'wikipedia-dpr-multi-bf'
+ dsearcher = FaissSearcher.from_prebuilt_index(
+ index,
+ encoder
+ )
+ hsearcher = HybridSearcher(dsearcher, ssearcher)
+
+ k = 10
+ prompt = '>>> '
+
+ def precmd(self, line):
+ if line[0] == '/':
+ line = line[1:]
+ return line
+
+ def do_help(self, arg):
+ print(f'/help : returns this message')
+ print(f'/k [NUM] : sets k (number of hits to return) to [NUM]')
+ print(f'/mode [MODE] : sets retriever type to [MODE] (one of sparse, dense, hybrid)')
+ print(f'/random [COLLECTION]: returns results for a random question from the dev subset [COLLECTION] (one of nq, trivia).')
+
+ def do_k(self, arg):
+ print(f'setting k = {int(arg)}')
+ self.k = int(arg)
+
+ def do_mode(self, arg):
+ if arg == "sparse":
+ self.searcher = self.ssearcher
+ elif arg == "dense":
+ self.searcher = self.dsearcher
+ elif arg == "hybrid":
+ self.searcher = self.hsearcher
+ else:
+ print(
+ f'Mode "{arg}" is invalid. Mode should be one of [sparse, dense, hybrid].')
+ return
+ print(f'setting retriver = {arg}')
+
+ def do_random(self, arg):
+ if arg == "nq":
+ topics = self.nq_dev_topics
+ elif arg == "trivia":
+ topics = self.trivia_dev_topics
+ else:
+ print(
+ f'Collection "{arg}" is invalid. Collection should be one of [nq, trivia].')
+ return
+ q = random.choice(topics)['title']
+ print(f'question: {q}')
+ self.default(q)
+
+ def do_EOF(self, line):
+ return True
+
+ def default(self, q):
+ hits = self.searcher.search(q, self.k)
+
+ for i in range(0, len(hits)):
+ raw_doc = None
+ if isinstance(self.searcher, LuceneSearcher):
+ raw_doc = hits[i].raw
+ else:
+ doc = self.searcher.doc(hits[i].docid)
+ if doc:
+ raw_doc = doc.raw()
+ jsondoc = json.loads(raw_doc)
+ print(f'{i + 1:2} {hits[i].score:.5f} {jsondoc["contents"]}')
+
+
+if __name__ == '__main__':
+ DPRDemo().cmdloop()

pyserini/demo/miracl.py

@@ -0,0 +1,149 @@
+#
+# Pyserini: Reproducible IR research with sparse and dense representations
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+This script provides an interactive web interface demo for retrieval on the MIRACL dataset.
+It requires `flask` (`pip install flask~=2.2.0`).
+An example command looks like `python -m pyserini.demo.miracl` that starts up a server on port 8080.
+The demo can be accessed via "http://localhost:8080" in a web browser.
+Additional arguments include:
+ --port [PORT] --hits [Number of hits] --index [BM25 or mdpr-tied-pft-msmarco]
+ --k1 [BM25 k1] --b [BM25 b] --device [cpu, cuda]
+"""
+import json
+import logging
+from argparse import ArgumentParser
+from functools import partial
+from typing import Callable, Optional, Tuple, Union
+
+from flask import Flask, render_template, request, flash, jsonify
+from pyserini.search import LuceneSearcher, FaissSearcher, AutoQueryEncoder
+
+logging.basicConfig(
+ format='%(asctime)s | %(levelname)s | %(name)s | %(message)s',
+ datefmt='%Y-%m-%d %H:%M:%S',
+ level=logging.INFO,
+)
+logger = logging.getLogger('miracl-demo')
+
+VERSION = '1.0'
+LANGUAGES = ('ar', 'bn', 'en', 'es', 'fa', 'fi', 'fr', 'hi', 'id', 'ja', 'ko', 'ru', 'sw', 'te', 'th', 'zh')
+Searcher = Union[FaissSearcher, LuceneSearcher]
+
+
+def create_app(k: int, load_searcher_fn: Callable[[str], Tuple[Searcher, str]]):
+ app = Flask(__name__)
+
+ lang = LANGUAGES[0]
+ searcher, retriever = load_searcher_fn(lang)
+
+ @app.route('/')
+ def index():
+ nonlocal lang, searcher, retriever
+ return render_template('miracl.html', lang=lang, retriever=retriever)
+
+ @app.route('/search', methods=['GET', 'POST'])
+ def search():
+ nonlocal lang, searcher, retriever
+ query = request.form['q']
+ if not query:
+ search_results = []
+ flash('Question is required')
+ else:
+ hits = searcher.search(query, k=k)
+ docs = [json.loads(searcher.doc(hit.docid).raw()) for hit in hits]
+ search_results = [
+ {
+ 'rank': r + 1,
+ 'docid': hit.docid,
+ 'doc': docs[r]['text'],
+ 'title': docs[r]['title'],
+ 'score': hit.score,
+ }
+ for r, hit in enumerate(hits)
+ ]
+ return render_template(
+ 'miracl.html', search_results=search_results, query=query, lang=lang, retriever=retriever
+ )
+
+ @app.route('/lang', methods=['GET'])
+ def change_language():
+ nonlocal lang, searcher, retriever
+ new_lang = request.args.get('new_lang', '', type=str)
+ if not new_lang or new_lang not in LANGUAGES:
+ return
+
+ lang = new_lang
+ searcher, retriever = load_searcher_fn(lang)
+ return jsonify(lang=lang)
+
+ return app
+
+
+def _load_sparse_searcher(language: str, k1: Optional[float]=None, b: Optional[float]=None) -> (Searcher, str):
+ searcher = LuceneSearcher.from_prebuilt_index(f'miracl-v{VERSION}-{language}')
+ searcher.set_language(language)
+ if k1 is not None and b is not None:
+ searcher.set_bm25(k1, b)
+ retriever_name = f'BM25 (k1={k1}, b={b})'
+ else:
+ retriever_name = 'BM25'
+
+ return searcher, retriever_name
+
+
+def _load_faiss_searcher(language: str, device: str) -> (Searcher, str):
+ query_encoder = AutoQueryEncoder(encoder_dir='castorini/mdpr-tied-pft-msmarco', device=device)
+ searcher = FaissSearcher.from_prebuilt_index(
+ f'miracl-v{VERSION}-{language}-mdpr-tied-pft-msmarco', query_encoder
+ )
+ retriever_name = 'mDPR-pFT-MSMARCO'
+ return searcher, retriever_name
+
+
+def main():
+ parser = ArgumentParser()
+
+ parser.add_argument('--index', default='BM25', choices=('BM25', 'mdpr-tied-pft-msmarco'), help='Index type.')
+ parser.add_argument('--k1', type=float, help='BM25 k1 parameter.')
+ parser.add_argument('--b', type=float, help='BM25 b parameter.')
+ parser.add_argument('--hits', type=int, default=10, help='Number of hits returned by the retriever')
+ parser.add_argument(
+ '--device',
+ type=str,
+ default='cpu',
+ help='Device to run query encoder, cpu or [cuda:0, cuda:1, ...] (used only when index is based on FAISS)',
+ )
+ parser.add_argument(
+ '--port',
+ default=8080,
+ type=int,
+ help='Web server port',
+ )
+
+ args = parser.parse_args()
+
+ if args.index == 'mdpr-tied-pft-msmarco':
+ load_fn = partial(_load_faiss_searcher, device=args.device)
+ else:
+ load_fn = partial(_load_sparse_searcher, k1=args.k1, b=args.b)
+
+ app = create_app(args.hits, load_fn)
+ app.run(host='0.0.0.0', port=args.port)
+
+
+if __name__ == '__main__':
+ main()

pyserini/demo/msmarco.py

@@ -0,0 +1,118 @@
+#
+# Pyserini: Reproducible IR research with sparse and dense representations
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import cmd
+import json
+import os
+import random
+
+from pyserini.search.lucene import LuceneSearcher
+from pyserini.search.faiss import FaissSearcher, TctColBertQueryEncoder, AnceQueryEncoder
+from pyserini.search.hybrid import HybridSearcher
+from pyserini import search
+
+
+class MsMarcoDemo(cmd.Cmd):
+ dev_topics = list(search.get_topics('msmarco-passage-dev-subset').values())
+
+ ssearcher = LuceneSearcher.from_prebuilt_index('msmarco-passage')
+ dsearcher = None
+ hsearcher = None
+ searcher = ssearcher
+
+ k = 10
+ prompt = '>>> '
+
+ # https://stackoverflow.com/questions/35213134/command-prefixes-in-python-cli-using-cmd-in-pythons-standard-library
+ def precmd(self, line):
+ if line[0] == '/':
+ line = line[1:]
+ return line
+
+ def do_help(self, arg):
+ print(f'/help : returns this message')
+ print(f'/k [NUM] : sets k (number of hits to return) to [NUM]')
+ print(f'/model [MODEL] : sets encoder to use the model [MODEL] (one of tct, ance)')
+ print(f'/mode [MODE] : sets retriever type to [MODE] (one of sparse, dense, hybrid)')
+ print(f'/random : returns results for a random question from dev subset')
+
+ def do_k(self, arg):
+ print(f'setting k = {int(arg)}')
+ self.k = int(arg)
+
+ def do_mode(self, arg):
+ if arg == "sparse":
+ self.searcher = self.ssearcher
+ elif arg == "dense":
+ if self.dsearcher is None:
+ print(f'Specify model through /model before using dense retrieval.')
+ return
+ self.searcher = self.dsearcher
+ elif arg == "hybrid":
+ if self.hsearcher is None:
+ print(f'Specify model through /model before using hybrid retrieval.')
+ return
+ self.searcher = self.hsearcher
+ else:
+ print(
+ f'Mode "{arg}" is invalid. Mode should be one of [sparse, dense, hybrid].')
+ return
+ print(f'setting retriver = {arg}')
+
+ def do_model(self, arg):
+ if arg == "tct":
+ encoder = TctColBertQueryEncoder("castorini/tct_colbert-msmarco")
+ index = "msmarco-passage-tct_colbert-hnsw"
+ elif arg == "ance":
+ encoder = AnceQueryEncoder("castorini/ance-msmarco-passage")
+ index = "msmarco-passage-ance-bf"
+ else:
+ print(
+ f'Model "{arg}" is invalid. Model should be one of [tct, ance].')
+ return
+
+ self.dsearcher = FaissSearcher.from_prebuilt_index(
+ index,
+ encoder
+ )
+ self.hsearcher = HybridSearcher(self.dsearcher, self.ssearcher)
+ print(f'setting model = {arg}')
+
+ def do_random(self, arg):
+ q = random.choice(self.dev_topics)['title']
+ print(f'question: {q}')
+ self.default(q)
+
+ def do_EOF(self, line):
+ return True
+
+ def default(self, q):
+ hits = self.searcher.search(q, self.k)
+
+ for i in range(0, len(hits)):
+ raw_doc = None
+ if isinstance(self.searcher, LuceneSearcher):
+ raw_doc = hits[i].raw
+ else:
+ doc = self.searcher.doc(hits[i].docid)
+ if doc:
+ raw_doc = doc.raw()
+ jsondoc = json.loads(raw_doc)
+ print(f'{i + 1:2} {hits[i].score:.5f} {jsondoc["contents"]}')
+
+
+if __name__ == '__main__':
+ MsMarcoDemo().cmdloop()

pyserini/demo/templates/acl.html

@@ -0,0 +1,74 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8">
+ <meta property="og:title" content="ACL 🌍🙌🌏">
+ <link href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" rel="stylesheet"
+ integrity="sha384-Zenh87qX5JnK2Jl0vWa8Ck2rdkQ2Bzep5IDxbcnCeuOxjzrPF/et3URy9Bv1WTRi" crossorigin="anonymous">
+ <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/font/bootstrap-icons.css">
+ <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.bundle.min.js"
+ integrity="sha384-OERcA2EqjJCMA+/3y+gxIOqMEjwtxJY7qPCqsdltbNJuaOe923+mo//f6V8Qbsw3"
+ crossorigin="anonymous"></script>
+ <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/jquery.min.js"></script>
+
+ <script>
+ $SCRIPT_ROOT = {{ request.script_root|tojson }};
+ </script>
+ <title>ACL 🌍🙌🌏 Demo</title>
+</head>
+<body>
+
+<div style="display: flex; align-items: center; gap: 10px;">
+ <h2>ACL</h2>
+ <img src="https://aclanthology.org/images/acl-logo.svg" alt="acl logo" width="50px">
+ <h2>Demo</h2>
+</div>
+<br/>
+
+<div class="container text-center">
+ {% for message in get_flashed_messages() %}
+ <div class="alert">{{ message }}</div>
+ {% endfor %}
+
+ <form action="/search" method="post">
+ <div class="row-cols-3">
+ <div class="input-group mb-3">
+ <input type="text" class="form-control" placeholder="Enter a Question" aria-label="Question" name="q"
+ aria-describedby="button-addon2" value="{{ query if query else '' }}">
+ <button class="btn btn-outline-secondary" type="submit" id="button-addon2"><i class="bi bi-search"></i>
+ </button>
+ </div>
+ </div>
+ </form>
+
+ {% if search_results %}
+ <div class="row">
+ <table class="table">
+ <thead>
+ <tr>
+ <th scope="col">#</th>
+ <th scope="col">Score</th>
+ <th scope="col">Passage ID</th>
+ <th scope="col">Content</th>
+ </tr>
+ </thead>
+ <tbody class="table-group-divider">
+ {% for res in search_results %}
+ <tr class="{{ 'table-secondary' if res['rank'] % 2 else 'table-light' }}">
+ <th scope="row">{{ res["rank"] }}</th>
+ <td>{{ "%.2f"|format(res["score"]) }}</td>
+ <td>{{ res["docid"] }}</td>
+
+ <td style="word-wrap: break-word;min-width: 600px;max-width: 600px;"
+ class="text-{{ 'end' if lang in ('ar', 'fa') else 'start' }}">
+ <small>{{ res["doc"] }}</small>
+ </td>
+ </tr>
+ {% endfor %}
+ </tbody>
+ </table>
+ </div>
+ {% endif %}
+</div>
+</body>
+</html>

pyserini/demo/templates/miracl.html

@@ -0,0 +1,127 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8">
+ <meta property="og:title" content="MIRACL 🌍🙌🌏">
+ <link href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-Zenh87qX5JnK2Jl0vWa8Ck2rdkQ2Bzep5IDxbcnCeuOxjzrPF/et3URy9Bv1WTRi" crossorigin="anonymous">
+ <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/font/bootstrap-icons.css">
+ <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.bundle.min.js" integrity="sha384-OERcA2EqjJCMA+/3y+gxIOqMEjwtxJY7qPCqsdltbNJuaOe923+mo//f6V8Qbsw3" crossorigin="anonymous"></script>
+ <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/jquery.min.js"></script>
+
+ <script>
+ $SCRIPT_ROOT = {{ request.script_root|tojson }};
+
+ $( document ).ready(function() {
+ $("#loading").hide();
+ $('#language').val("{{lang}}");
+ });
+
+ $(function() {
+ $('#language').on('change', function() {
+ $.getJSON($SCRIPT_ROOT + '/lang', {
+ new_lang: this.value,
+ }, function(data) {
+ $("#language").removeAttr('disabled');
+ $("#loading").hide();
+ });
+
+ $(this).attr('disabled','disabled');
+ $("#loading").show();
+
+ return false;
+ });
+ });
+ </script>
+ <title>MIRACL 🌍🙌🌏 Demo</title>
+</head>
+<body>
+ <h2>MIRACL 🌍🙌🌏 Demo</h2>
+ <h4>Multilingual Information Retrieval Across a Continuum of Languages</h4>
+
+ <br/>
+
+ <p class="lead">
+ <a href="http://miracl.ai/">MIRACL</a> is a multilingual dataset for ad hoc retrieval that consists of 18 different languages, collectively encompassing over three billion native speakers around the world.
+ </p>
+
+ <div class="row g-3 align-items-center">
+ <label class="col-auto" for="language">This demo running on the language</label>
+ <div class="col-auto">
+ <select class="form-select form-select-sm" aria-label=".form-select-sm" id="language">
+ <option value="ar">Arabic</option>
+ <option value="bn">Bengali</option>
+ <option value="en">English</option>
+ <option value="es">Spanish</option>
+ <option value="fa">Persian</option>
+ <option value="fi">Finnish</option>
+ <option value="fr">French</option>
+ <option value="hi">Hindi</option>
+ <option value="id">Indonesian</option>
+ <option value="ja">Japanese</option>
+ <option value="ko">Korean</option>
+ <option value="ru">Russian</option>
+ <option value="sw">Swahili</option>
+ <option value="te">Telugu</option>
+ <option value="th">Thai</option>
+ <option value="zh">Chinese</option>
+ </select>
+ </div>
+ <div class="col-auto">
+ <div class="spinner-border text-secondary" role="status" id="loading">
+ <span class="visually-hidden">Loading...</span>
+ </div>
+ </div>
+ <div class="col-auto">
+ <span>
+ retrieves passages using <em>{{retriever}}</em>.
+ </span>
+ </div>
+ </div>
+
+ <br/>
+
+ <div class="container text-center">
+ {% for message in get_flashed_messages() %}
+ <div class="alert">{{ message }}</div>
+ {% endfor %}
+
+ <form action="/search" method="post">
+ <div class="row-cols-3">
+ <div class="input-group mb-3">
+ <input type="text" class="form-control" placeholder="Enter a Question" aria-label="Question" name="q" aria-describedby="button-addon2" value="{{query if query else ''}}">
+ <button class="btn btn-outline-secondary" type="submit" id="button-addon2"><i class="bi bi-search"></i></button>
+ </div>
+ </div>
+ </form>
+
+ {% if search_results %}
+ <div class="row">
+ <table class="table">
+ <thead>
+ <tr>
+ <th scope="col">#</th>
+ <th scope="col">Score</th>
+ <th scope="col">Passage ID</th>
+ <th scope="col">Title</th>
+ <th scope="col">Content</th>
+ </tr>
+ </thead>
+ <tbody class="table-group-divider">
+ {% for res in search_results %}
+ <tr class="{{'table-secondary' if res['rank'] % 2 else 'table-light'}}">
+ <th scope="row">{{res["rank"]}}</th>
+ <td>{{"%.2f"|format(res["score"])}}</td>
+ <td>{{res["docid"]}}</td>
+ <td>{{res["title"]}}</td>
+ <td style="word-wrap: break-word;min-width: 600px;max-width: 600px;" class="text-{{'end' if lang in ('ar', 'fa') else 'start'}}">
+ <small>{{res["doc"]}}</small>
+ </td>
+ </tr>
+ {% endfor %}
+ </tbody>
+ </table>
+ </div>
+ {% endif %}
+ </div>
+</body>
+</html>

pyserini/dsearch.py

@@ -0,0 +1,46 @@
+#
+# Pyserini: Reproducible IR research with sparse and dense representations
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Deprecated. The package ``pyserini.dsearch` has been renamed `pyserini.search.faiss`. Stubs are retained here for
+redirection purpose to ensure that code in existing published papers remain function (with warnings)."""
+
+import os
+import sys
+
+import pyserini.search.faiss
+from pyserini.search.faiss import TctColBertQueryEncoder
+
+__all__ = ['SimpleDenseSearcher', 'BinaryDenseSearcher', 'TctColBertQueryEncoder']
+
+
+class SimpleDenseSearcher(pyserini.search.faiss.FaissSearcher):
+ def __new__(cls, *args, **kwargs):
+ print('pyserini.dsearch.SimpleDenseSearcher class has been deprecated, '
+ 'please use FaissSearcher from pyserini.search.faiss instead')
+ return super().__new__(cls)
+
+
+class BinaryDenseSearcher(pyserini.search.faiss.BinaryDenseSearcher):
+ def __new__(cls, *args, **kwargs):
+ print('pyserini.dsearch.BinaryDenseSearcher class has been deprecated, '
+ 'please use BinaryDenseSearcher from pyserini.search.faiss instead')
+ return super().__new__(cls)
+
+
+if __name__ == "__main__":
+ print('WARNING: pyserini.dsearch is deprecated, please use pyserini.search.faiss instead!')
+ args = " ".join(sys.argv[1:])
+ os.system(f'python -m pyserini.search.faiss {args}')

pyserini/encode/__init__.py

@@ -0,0 +1,28 @@
+#
+# Pyserini: Reproducible IR research with sparse and dense representations
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from ._base import DocumentEncoder, QueryEncoder, JsonlCollectionIterator,\
+ RepresentationWriter, FaissRepresentationWriter, JsonlRepresentationWriter, PcaEncoder
+from ._ance import AnceEncoder, AnceDocumentEncoder, AnceQueryEncoder
+from ._auto import AutoQueryEncoder, AutoDocumentEncoder
+from ._dpr import DprDocumentEncoder, DprQueryEncoder
+from ._tct_colbert import TctColBertDocumentEncoder, TctColBertQueryEncoder
+from ._aggretriever import AggretrieverDocumentEncoder, AggretrieverQueryEncoder
+from ._unicoil import UniCoilEncoder, UniCoilDocumentEncoder, UniCoilQueryEncoder
+from ._cached_data import CachedDataQueryEncoder
+from ._tok_freq import TokFreqQueryEncoder
+from ._splade import SpladeQueryEncoder
+from ._slim import SlimQueryEncoder

pyserini/encode/__main__.py

@@ -0,0 +1,147 @@
+#
+# Pyserini: Reproducible IR research with sparse and dense representations
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import argparse
+import sys
+
+from pyserini.encode import JsonlRepresentationWriter, FaissRepresentationWriter, JsonlCollectionIterator
+from pyserini.encode import DprDocumentEncoder, TctColBertDocumentEncoder, AnceDocumentEncoder, AggretrieverDocumentEncoder, AutoDocumentEncoder
+from pyserini.encode import UniCoilDocumentEncoder
+
+
+encoder_class_map = {
+ "dpr": DprDocumentEncoder,
+ "tct_colbert": TctColBertDocumentEncoder,
+ "aggretriever": AggretrieverDocumentEncoder,
+ "ance": AnceDocumentEncoder,
+ "sentence-transformers": AutoDocumentEncoder,
+ "unicoil": UniCoilDocumentEncoder,
+ "auto": AutoDocumentEncoder,
+}
+ALLOWED_POOLING_OPTS = ["cls","mean"]
+
+def init_encoder(encoder, encoder_class, device):
+ _encoder_class = encoder_class
+
+ # determine encoder_class
+ if encoder_class is not None:
+ encoder_class = encoder_class_map[encoder_class]
+ else:
+ # if any class keyword was matched in the given encoder name,
+ # use that encoder class
+ for class_keyword in encoder_class_map:
+ if class_keyword in encoder.lower():
+ encoder_class = encoder_class_map[class_keyword]
+ break
+
+ # if none of the class keyword was matched,
+ # use the AutoDocumentEncoder
+ if encoder_class is None:
+ encoder_class = AutoDocumentEncoder
+
+ # prepare arguments to encoder class
+ kwargs = dict(model_name=encoder, device=device)
+ if (_encoder_class == "sentence-transformers") or ("sentence-transformers" in encoder):
+ kwargs.update(dict(pooling='mean', l2_norm=True))
+ if (_encoder_class == "contriever") or ("contriever" in encoder):
+ kwargs.update(dict(pooling='mean', l2_norm=False))
+ return encoder_class(**kwargs)
+
+
+def parse_args(parser, commands):
+ # Divide argv by commands
+ split_argv = [[]]
+ for c in sys.argv[1:]:
+ if c in commands.choices:
+ split_argv.append([c])
+ else:
+ split_argv[-1].append(c)
+ # Initialize namespace
+ args = argparse.Namespace()
+ for c in commands.choices:
+ setattr(args, c, None)
+ # Parse each command
+ parser.parse_args(split_argv[0], namespace=args) # Without command
+ for argv in split_argv[1:]: # Commands
+ n = argparse.Namespace()
+ setattr(args, argv[0], n)
+ parser.parse_args(argv, namespace=n)
+ return args
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ commands = parser.add_subparsers(title='sub-commands')
+ input_parser = commands.add_parser('input')
+ input_parser.add_argument('--corpus', type=str,
+ help='directory that contains corpus files to be encoded, in jsonl format.',
+ required=True)
+ input_parser.add_argument('--fields', help='fields that contents in jsonl has (in order)',
+ nargs='+', default=['text'], required=False)
+ input_parser.add_argument('--docid-field',
+ help='name of document id field name. If you have a custom id with a name other than "id", "_id" or "docid", then use this argument',
+ default=None, required=False)
+ input_parser.add_argument('--delimiter', help='delimiter for the fields', default='\n', required=False)
+ input_parser.add_argument('--shard-id', type=int, help='shard-id 0-based', default=0, required=False)
+ input_parser.add_argument('--shard-num', type=int, help='number of shards', default=1, required=False)
+
+ output_parser = commands.add_parser('output')
+ output_parser.add_argument('--embeddings', type=str, help='directory to store encoded corpus', required=True)
+ output_parser.add_argument('--to-faiss', action='store_true', default=False)
+
+ encoder_parser = commands.add_parser('encoder')
+ encoder_parser.add_argument('--encoder', type=str, help='encoder name or path', required=True)
+ encoder_parser.add_argument('--encoder-class', type=str, required=False, default=None,
+ choices=["dpr", "bpr", "tct_colbert", "ance", "sentence-transformers", "auto"],
+ help='which query encoder class to use. `default` would infer from the args.encoder')
+ encoder_parser.add_argument('--fields', help='fields to encode', nargs='+', default=['text'], required=False)
+ encoder_parser.add_argument('--batch-size', type=int, help='batch size', default=64, required=False)
+ encoder_parser.add_argument('--max-length', type=int, help='max length', default=256, required=False)
+ encoder_parser.add_argument('--dimension', type=int, help='dimension', default=768, required=False)
+ encoder_parser.add_argument('--device', type=str, help='device cpu or cuda [cuda:0, cuda:1...]',
+ default='cuda:0', required=False)
+ encoder_parser.add_argument('--fp16', action='store_true', default=False)
+ encoder_parser.add_argument('--add-sep', action='store_true', default=False)
+ encoder_parser.add_argument('--pooling', type=str, default='cls', help='for auto classes, allow the ability to dictate pooling strategy', required=False)
+
+ args = parse_args(parser, commands)
+ delimiter = args.input.delimiter.replace("\\n", "\n") # argparse would add \ prior to the passed '\n\n'
+
+ encoder = init_encoder(args.encoder.encoder, args.encoder.encoder_class, device=args.encoder.device)
+ if type(encoder).__name__ == "AutoDocumentEncoder":
+ if args.encoder.pooling in ALLOWED_POOLING_OPTS:
+ encoder.pooling = args.encoder.pooling
+ else:
+ raise ValueError(f"Only allowed to use pooling types {ALLOWED_POOLING_OPTS}. You entered {args.encoder.pooling}")
+ if args.output.to_faiss:
+ embedding_writer = FaissRepresentationWriter(args.output.embeddings, dimension=args.encoder.dimension)
+ else:
+ embedding_writer = JsonlRepresentationWriter(args.output.embeddings)
+ collection_iterator = JsonlCollectionIterator(args.input.corpus, args.input.fields, args.input.docid_field, delimiter)
+
+ with embedding_writer:
+ for batch_info in collection_iterator(args.encoder.batch_size, args.input.shard_id, args.input.shard_num):
+ kwargs = {
+ 'texts': batch_info['text'],
+ 'titles': batch_info['title'] if 'title' in args.encoder.fields else None,
+ 'expands': batch_info['expand'] if 'expand' in args.encoder.fields else None,
+ 'fp16': args.encoder.fp16,
+ 'max_length': args.encoder.max_length,
+ 'add_sep': args.encoder.add_sep,
+ }
+ embeddings = encoder.encode(**kwargs)
+ batch_info['vector'] = embeddings
+ embedding_writer.write(batch_info, args.input.fields)