hieunguyen1053 commited on
Commit
0cf0987
1 Parent(s): f8b127b

Update src/tasks.py

Browse files
Files changed (1) hide show
  1. src/tasks.py +8 -8
src/tasks.py CHANGED
@@ -12,7 +12,7 @@ class Task:
12
 
13
  class Lambada(Task):
14
  code = "lambada_vi"
15
- name = "LAMBADA"
16
  metric = "ppl"
17
  higher_is_better = False
18
  num_fewshot = 0
@@ -21,7 +21,7 @@ class Lambada(Task):
21
 
22
  class Arc(Task):
23
  code = "arc_vi"
24
- name = "ARC"
25
  metric = "acc_norm"
26
  higher_is_better = True
27
  num_fewshot = 25
@@ -30,7 +30,7 @@ class Arc(Task):
30
 
31
  class HellaSwag(Task):
32
  code = "hellaswag_vi"
33
- name = "HellaSwag"
34
  metric = "acc_norm"
35
  higher_is_better = True
36
  num_fewshot = 10
@@ -39,7 +39,7 @@ class HellaSwag(Task):
39
 
40
  class MMLU(Task):
41
  code = "mmlu_vi"
42
- name = "MMLU"
43
  metric = "acc_norm"
44
  higher_is_better = True
45
  num_fewshot = 5
@@ -48,7 +48,7 @@ class MMLU(Task):
48
 
49
  class TruthfulQA(Task):
50
  code = "truthfulqa_vi"
51
- name = "TruthfulQA"
52
  metric = "mc2"
53
  higher_is_better = True
54
  num_fewshot = 0
@@ -75,7 +75,7 @@ class IWSLT2023_en_vi(Task):
75
 
76
  class WikipediaQA(Task):
77
  code = "wikipediaqa_vi"
78
- name = "Wikipedia QA"
79
  metric = "acc_norm"
80
  higher_is_better = True
81
  num_fewshot = 5
@@ -84,11 +84,11 @@ class WikipediaQA(Task):
84
 
85
  class Comprehension(Task):
86
  code = "comprehension_vi"
87
- name = "Comprehension"
88
  metric = "acc_norm"
89
  higher_is_better = True
90
  num_fewshot = 0
91
  private_test: bool = True
92
 
93
 
94
- TASKS = [Arc, HellaSwag, MMLU, TruthfulQA, Grade12Exams] + [Lambada, WikipediaQA, Comprehension]
 
12
 
13
  class Lambada(Task):
14
  code = "lambada_vi"
15
+ name = "LAMBADA-vi"
16
  metric = "ppl"
17
  higher_is_better = False
18
  num_fewshot = 0
 
21
 
22
  class Arc(Task):
23
  code = "arc_vi"
24
+ name = "ARC-vi"
25
  metric = "acc_norm"
26
  higher_is_better = True
27
  num_fewshot = 25
 
30
 
31
  class HellaSwag(Task):
32
  code = "hellaswag_vi"
33
+ name = "HellaSwag-vi"
34
  metric = "acc_norm"
35
  higher_is_better = True
36
  num_fewshot = 10
 
39
 
40
  class MMLU(Task):
41
  code = "mmlu_vi"
42
+ name = "MMLU-vi"
43
  metric = "acc_norm"
44
  higher_is_better = True
45
  num_fewshot = 5
 
48
 
49
  class TruthfulQA(Task):
50
  code = "truthfulqa_vi"
51
+ name = "TruthfulQA-vi"
52
  metric = "mc2"
53
  higher_is_better = True
54
  num_fewshot = 0
 
75
 
76
  class WikipediaQA(Task):
77
  code = "wikipediaqa_vi"
78
+ name = "GeneralKnowledgeQA-vi"
79
  metric = "acc_norm"
80
  higher_is_better = True
81
  num_fewshot = 5
 
84
 
85
  class Comprehension(Task):
86
  code = "comprehension_vi"
87
+ name = "ComprehensionQA-vi"
88
  metric = "acc_norm"
89
  higher_is_better = True
90
  num_fewshot = 0
91
  private_test: bool = True
92
 
93
 
94
+ TASKS = [Arc, HellaSwag, MMLU, TruthfulQA] + [Lambada, WikipediaQA, Comprehension]