update

tasks.py

@@ -59,7 +59,7 @@ class Task:
  dataset_name: str | tuple[str, str] = ("gsm8k", "main")
  split: str = "test"
  # metrics: list[str] = field(default_factory=list)
- metric_name: str | tuple[str, str] = ("sustech/tlem", "gsm8k")
+ metric_name: str | tuple[str, str] = ("sustech/tlem", "mmlu")
  input_column: str = "question"
  label_column: str = ""
  prompt: Optional[Callable | str] = None
@@ -147,12 +147,12 @@ class Task:
  if isinstance(self.metric_name, str)
  else load(*self.metric_name)
  )
- return metric
+ return metric._compute
 
  @cached_property
  def result(self) -> dict:
  assert self.outputs, "Please run the task first."
- results = self.metric._compute(
+ results = self.metric(
  responses=self.outputs, references=self.dataset[self.label_column]
  )
  # logging.info(f"{self.name}:{results}")

tlem.py

@@ -72,6 +72,17 @@ class ReasoningMetric(evaluate.Metric):
 class Suite(EvaluationSuite):
  task_class = Task
  utils = utils
+ supported_datasets = [
+ "arc",
+ "hellaswag",
+ "mmlu-chat",
+ "winogrande",
+ "gsm8k",
+ "cmmlu-chat",
+ "ceval-chat",
+ "bbh",
+ "drop",
+ ]
 
  def __getitem__(self, key) -> Task:
  match key:
@@ -171,19 +182,26 @@ class Suite(EvaluationSuite):
  ]:
  suite[name] = self.get_suite(name)
  case "tlem":
- suite = {}
- for name in [
- "arc",
- "hellaswag",
- "mmlu-chat",
- "winogrande",
- "gsm8k",
- "cmmlu-chat",
- "ceval-chat",
- # "truthful_qa",
- "drop",
- ]:
- suite[name] = self.get_suite(name)
+ suite = {
+ name: self.get_suite(name)
+ for name in [
+ "arc",
+ "hellaswag",
+ "mmlu-chat",
+ "winogrande",
+ "gsm8k",
+ "cmmlu-chat",
+ "ceval-chat",
+ "bbh",
+ ]
+ }
+ case "all":
+ suite = {name: self.get_suite(name) for name in self.supported_datasets}
+ case _:
+ raise NotImplementedError(
+ f"{name} is not supported in {self.supported_datasets}"
+ )
+
  if isinstance(suite, Task):
  suite = [suite]
  if isinstance(suite, list):
@@ -195,6 +213,7 @@ class Suite(EvaluationSuite):
  try:
  return self.tasks[self.tasks.index(task)]
  except ValueError:
+ logging.debug(f"add {task.name} to suite.")
  self.tasks.append(task)
  return self.tasks[-1]
 
@@ -212,6 +231,7 @@ class Suite(EvaluationSuite):
  def load(self, name):
  self.suite.update(self.get_suite(name))
  self.suite = self.drop_duplicates(self.suite)
+ # return self
 
  def __init__(self, name="tlem"):
  super().__init__(name)