!ref suite

tasks.py

@@ -65,6 +65,7 @@ class Task:
  few_shot: int = 0
  few_shot_from: Optional[str] = None
  # results: dict[str, Any] = field(default_factory=dict)
+ outputs: Optional[list] = field(default_factory=list)
 
  def __post_init__(self):
  names = (
@@ -142,31 +143,21 @@ class Task:
  )
  return metric
 
+ @cached_property
+ def result(self) -> dict:
+ assert self.outputs, "Please run the task first."
+ return self.metric._compute(
+ responses=self.outputs, references=self.dataset[self.label_column]
+ )
+
  # @cache
  def run(
  self,
  pipeline,
  ):
- if (outputs := pipeline(self.samples)) is None:
- logging.warning("pipeline returns None")
- return
- self.outputs = outputs
- try:
- try:
- result = self.metric._compute(
- responses=outputs, references=self.dataset[self.label_column]
- )
- except Exception as e:
- result = self.metric.compute(
- responses=outputs, references=self.dataset[self.label_column]
- )
- except Exception as e:
- result = outputs
- # if log:
- # name = name or pipeline.__name__
- # self.results[name] = result
+ self.outputs = self.outputs or pipeline(self.samples)
 
- return result
+ return self.result
 
 
 def multichoice(responses: Any, references: list[str]):

tlem.py

@@ -12,6 +12,7 @@ import datasets
 import pandas as pd
 from .tasks import *
 from .utils import *
+from itertools import chain
 
 
 class ReasoningMetric(evaluate.Metric):
@@ -70,33 +71,35 @@ class ReasoningMetric(evaluate.Metric):
 class Suite(EvaluationSuite):
  task_class = Task
 
+ def __getitem__(self, key) -> Task:
+ match key:
+ case str():
+ return self.suite[key]
+ # case _:
+ # return list(chain(*self.suite.values()))[key]
+
  def run(
  self,
  model_or_pipeline: Any,
+ suite=None,
  ) -> dict[str, float]:
  self.assert_suite_nonempty()
+ if suite is None:
+ suite = self.suite
+
+ self.suite: dict[str, list[Task]]
+ results = defaultdict(dict)
+ for category, tasks in (bar := tqdm(self.suite.items())):
+ bar.desc = f"complete {category}."
+ if isinstance(tasks, dict):
+ results[category] = self.run(model_or_pipeline, tasks)
+ else:
+ for task in tasks:
+ results[category].update(task.run(model_or_pipeline))
+ results[category] = np.mean(list(results[category].values()))
+ return results
 
- def run_tasks(tasks):
- for task in (bar := tqdm(tasks, leave=False)):
- bar.desc = f"complete {task.name}."
- if task.name not in self.cached_result:
- self.cached_result[task.name] = task.run(model_or_pipeline)
- results = [self.cached_result[task.name] for task in tasks]
- return pd.DataFrame(results).mean().to_dict()
-
- if isinstance(self.suite, dict):
- for category, tasks in (bar := tqdm(self.suite.items())):
- bar.desc = f"complete {category}."
- logging.warning(f"Combined results {category}: {run_tasks(tasks)}")
- else:
- logging.warning(f"Combined results: {run_tasks(self.suite)}")
-
- return self.cached_result
-
- def add(self, name):
- self.load(name)
-
- def load(self, name):
+ def get_suite(self, name) -> dict[str, Task]:
  chat = False
  match name:
  case _ if "chat" in name:
@@ -106,6 +109,8 @@ class Suite(EvaluationSuite):
  suite = MMLU.suite(chat=chat)
  case _ if name.startswith("cmmlu"):
  suite = CMMLU.suite(chat=chat)
+ case _ if name.startswith("ceval"):
+ suite = CEVAL.suite(chat=chat)
  case "gsm8k":
  suite = Task(
  dataset_name=("gsm8k", "main"),
@@ -123,8 +128,7 @@ class Suite(EvaluationSuite):
  suite = DROP.suite()
  case "winogrande":
  suite = Winogrande.suite()
- case _ if name.startswith("ceval"):
- suite = CEVAL.suite(chat=chat)
+
  case "mt_bench":
  suite = Task(
  dataset_name="SUSTech/mt_bench_judge",
@@ -135,16 +139,39 @@ class Suite(EvaluationSuite):
  case "MATH" | "competition_math":
  suite = Task(
  dataset_name="hendrycks/competition_math",
- split="test",
- prompt="This is a math problem, please think step by step and slove it: {input_column}, simplify your final answer as much as possible and surround them with $ in TeX form",
+ prompt="This is a math problem, please think step by step and slove it: {input_column}. Simplify your final answer as much as possible and surround them with '$' in TeX form",
  metric_name=("sustech/tlem", "MATH"),
  input_column="problem",
  label_column="solution",
  )
+ if isinstance(suite, Task):
+ suite = [suite]
+ if isinstance(suite, list):
+ suite = {name: suite}
+
+ return suite
 
- self.suite = [suite] if isinstance(suite, Task) else suite
+ def singleton(self, task):
+ try:
+ return self.tasks[self.tasks.index(task)]
+
+ except Exception as e:
+ self.tasks.append(task)
+ return self.tasks[-1]
+
+ def drop_duplicates(self, suite):
+ for category, tasks in suite.items():
+ if isinstance(tasks, dict):
+ suite[category] = self.drop_duplicates(tasks)
+ else:
+ suite[category] = [self.singleton(task) for task in tasks]
+ return suite
+
+ def load(self, name):
+ self.suite.update(self.get_suite(name))
+ self.suite = self.drop_duplicates(self.suite)
 
  def __init__(self, name="tlem"):
  super().__init__(name)
- self.cached_result = {}
- self.suite = []
+ self.tasks = []
+ self.suite = {}

utils.py

@@ -138,13 +138,13 @@ def extract_numeric(string, pattern=NUMERIC_IN_EN) -> str:
 def remove_boxed(s):
  if "\\boxed " in s:
  left = "\\boxed "
- assert s[: len(left)] == left
+ assert s[: len(left)] == left, s
  return s[len(left) :]
 
  left = "\\boxed{"
 
- assert s[: len(left)] == left
- assert s[-1] == "}"
+ assert s[: len(left)] == left, s
+ assert s[-1] == "}", s
 
  return s[len(left) : -1]