tdoehmen commited on
Commit
6853b2a
1 Parent(s): 5051da6
Files changed (1) hide show
  1. app.py +22 -22
app.py CHANGED
@@ -22,12 +22,12 @@ def run_evaluation(model_name):
22
  try:
23
  # Start manifest server in background with explicit CUDA_VISIBLE_DEVICES
24
  manifest_cmd = f"""
25
- CUDA_VISIBLE_DEVICES=0 HF_TOKEN={os.environ['HF_TOKEN']} cd duckdb-nsql/ &&
26
- python -m manifest.api.app
27
- --model_type huggingface
28
- --model_generation_type text-generation
29
- --model_name_or_path {model_name}
30
- --fp16
31
  --device 0
32
  """
33
  manifest_process = subprocess.Popen(manifest_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -35,16 +35,16 @@ def run_evaluation(model_name):
35
 
36
  # Run inference
37
  inference_cmd = f"""
38
- cd duckdb-nsql/ &&
39
- python eval/predict.py
40
- predict
41
- eval/data/dev.json
42
- eval/data/tables.json
43
- --output-dir output/
44
- --stop-tokens ';'
45
- --overwrite-manifest
46
- --manifest-client huggingface
47
- --manifest-connection http://localhost:5000
48
  --prompt-format duckdbinstgraniteshort
49
  """
50
  inference_result = subprocess.run(inference_cmd, shell=True, check=True, capture_output=True, text=True)
@@ -59,12 +59,12 @@ def run_evaluation(model_name):
59
 
60
  # Run evaluation
61
  eval_cmd = f"""
62
- cd duckdb-nsql/ &&
63
- python eval/evaluate.py evaluate
64
- --gold eval/data/dev.json
65
- --db eval/data/databases/
66
- --tables eval/data/tables.json
67
- --output-dir output/
68
  --pred output/{json_file}
69
  """
70
  eval_result = subprocess.run(eval_cmd, shell=True, check=True, capture_output=True, text=True)
 
22
  try:
23
  # Start manifest server in background with explicit CUDA_VISIBLE_DEVICES
24
  manifest_cmd = f"""
25
+ CUDA_VISIBLE_DEVICES=0 HF_TOKEN={os.environ['HF_TOKEN']} cd duckdb-nsql/ && \
26
+ python -m manifest.api.app \
27
+ --model_type huggingface \
28
+ --model_generation_type text-generation \
29
+ --model_name_or_path {model_name} \
30
+ --fp16 \
31
  --device 0
32
  """
33
  manifest_process = subprocess.Popen(manifest_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
35
 
36
  # Run inference
37
  inference_cmd = f"""
38
+ cd duckdb-nsql/ && \
39
+ python eval/predict.py \
40
+ predict \
41
+ eval/data/dev.json \
42
+ eval/data/tables.json \
43
+ --output-dir output/ \
44
+ --stop-tokens ';' \
45
+ --overwrite-manifest \
46
+ --manifest-client huggingface \
47
+ --manifest-connection http://localhost:5000 \
48
  --prompt-format duckdbinstgraniteshort
49
  """
50
  inference_result = subprocess.run(inference_cmd, shell=True, check=True, capture_output=True, text=True)
 
59
 
60
  # Run evaluation
61
  eval_cmd = f"""
62
+ cd duckdb-nsql/ && \
63
+ python eval/evaluate.py evaluate \
64
+ --gold eval/data/dev.json \
65
+ --db eval/data/databases/ \
66
+ --tables eval/data/tables.json \
67
+ --output-dir output/ \
68
  --pred output/{json_file}
69
  """
70
  eval_result = subprocess.run(eval_cmd, shell=True, check=True, capture_output=True, text=True)