Spaces:
Running
Running
fix
Browse files- execution_accuracy.py +5 -5
- tests.py +0 -17
execution_accuracy.py
CHANGED
@@ -94,13 +94,13 @@ class ExecutionAccuracy(evaluate.Metric):
|
|
94 |
for prediction, reference, db_url in zip(predictions, references, db_urls):
|
95 |
db = Database(db_url)
|
96 |
try:
|
97 |
-
pred = db.query(
|
98 |
except Exception as e:
|
99 |
-
pred =
|
100 |
try:
|
101 |
-
ref = db.query(
|
102 |
except Exception as e:
|
103 |
-
ref =
|
104 |
pred = [tuple(x.values()) for x in pred]
|
105 |
ref = [tuple(x.values()) for x in ref]
|
106 |
if len(pred) == len(ref):
|
@@ -110,5 +110,5 @@ class ExecutionAccuracy(evaluate.Metric):
|
|
110 |
cnt += 1
|
111 |
accuracy = cnt / len(predictions)
|
112 |
return {
|
113 |
-
"
|
114 |
}
|
|
|
94 |
for prediction, reference, db_url in zip(predictions, references, db_urls):
|
95 |
db = Database(db_url)
|
96 |
try:
|
97 |
+
pred = db.query(prediction).as_dict()
|
98 |
except Exception as e:
|
99 |
+
pred = {}
|
100 |
try:
|
101 |
+
ref = db.query(reference).as_dict()
|
102 |
except Exception as e:
|
103 |
+
ref = {}
|
104 |
pred = [tuple(x.values()) for x in pred]
|
105 |
ref = [tuple(x.values()) for x in ref]
|
106 |
if len(pred) == len(ref):
|
|
|
110 |
cnt += 1
|
111 |
accuracy = cnt / len(predictions)
|
112 |
return {
|
113 |
+
"execution_accuracy": accuracy,
|
114 |
}
|
tests.py
DELETED
@@ -1,17 +0,0 @@
|
|
1 |
-
test_cases = [
|
2 |
-
{
|
3 |
-
"predictions": [0, 0],
|
4 |
-
"references": [1, 1],
|
5 |
-
"result": {"metric_score": 0}
|
6 |
-
},
|
7 |
-
{
|
8 |
-
"predictions": [1, 1],
|
9 |
-
"references": [1, 1],
|
10 |
-
"result": {"metric_score": 1}
|
11 |
-
},
|
12 |
-
{
|
13 |
-
"predictions": [1, 0],
|
14 |
-
"references": [1, 1],
|
15 |
-
"result": {"metric_score": 0.5}
|
16 |
-
}
|
17 |
-
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|