yqsong commited on
Commit
8ff2c2a
1 Parent(s): 196f8b3
Files changed (2) hide show
  1. execution_accuracy.py +5 -5
  2. tests.py +0 -17
execution_accuracy.py CHANGED
@@ -94,13 +94,13 @@ class ExecutionAccuracy(evaluate.Metric):
94
  for prediction, reference, db_url in zip(predictions, references, db_urls):
95
  db = Database(db_url)
96
  try:
97
- pred = db.query(predictions).as_dict()
98
  except Exception as e:
99
- pred = []
100
  try:
101
- ref = db.query(references).as_dict()
102
  except Exception as e:
103
- ref = []
104
  pred = [tuple(x.values()) for x in pred]
105
  ref = [tuple(x.values()) for x in ref]
106
  if len(pred) == len(ref):
@@ -110,5 +110,5 @@ class ExecutionAccuracy(evaluate.Metric):
110
  cnt += 1
111
  accuracy = cnt / len(predictions)
112
  return {
113
- "execution accuracy": accuracy,
114
  }
 
94
  for prediction, reference, db_url in zip(predictions, references, db_urls):
95
  db = Database(db_url)
96
  try:
97
+ pred = db.query(prediction).as_dict()
98
  except Exception as e:
99
+ pred = {}
100
  try:
101
+ ref = db.query(reference).as_dict()
102
  except Exception as e:
103
+ ref = {}
104
  pred = [tuple(x.values()) for x in pred]
105
  ref = [tuple(x.values()) for x in ref]
106
  if len(pred) == len(ref):
 
110
  cnt += 1
111
  accuracy = cnt / len(predictions)
112
  return {
113
+ "execution_accuracy": accuracy,
114
  }
tests.py DELETED
@@ -1,17 +0,0 @@
1
- test_cases = [
2
- {
3
- "predictions": [0, 0],
4
- "references": [1, 1],
5
- "result": {"metric_score": 0}
6
- },
7
- {
8
- "predictions": [1, 1],
9
- "references": [1, 1],
10
- "result": {"metric_score": 1}
11
- },
12
- {
13
- "predictions": [1, 0],
14
- "references": [1, 1],
15
- "result": {"metric_score": 0.5}
16
- }
17
- ]