Revert "eval (#6)"

This reverts commit 7f272e4a20ca97427179a37281c7a2a62e203e89.

.gitattributes

@@ -47,7 +47,3 @@ rng_state_2.pth filter=lfs diff=lfs merge=lfs -text
 model-00003-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text
 rng_state_4.pth filter=lfs diff=lfs merge=lfs -text
 model-00002-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text
-*.csv filter=lfs diff=lfs merge=lfs -text
-*.json filter=lfs diff=lfs merge=lfs -text
-*.jsonl filter=lfs diff=lfs merge=lfs -text
-*.sqlite filter=lfs diff=lfs merge=lfs -text

added_tokens.json

@@ -1,3 +1,24 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:58b54bbe36fc752f79a24a271ef66a0a0830054b4dfad94bde757d851968060b
-size 605
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

config.json

@@ -1,3 +1,28 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:82c97ddf1f855ce947c97a62859a064efb2499cb7cbb82aa9c67bdc65b678c17
-size 709
+{
+  "_name_or_path": "Qwen/Qwen2.5-7B-Instruct",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.44.2",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}

evaluation/.gitignore

@@ -1,190 +0,0 @@
-
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-*.sql
-*.sqlite
-*.splite
-*.desc
-*.txt
-*.DS_Store
-.DS_Store
-!eval_retriever/data/*.json
-!eval_retriever/preds/*.json
-!reject_eval/*.json
-!evalset/*/*.json
-!evalset/*.json
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-cover/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-.pybuilder/
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-#   For a library or package, you might want to ignore these files since the code is
-#   intended to run in multiple environments; otherwise, check them in:
-# .python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# poetry
-#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-#poetry.lock
-
-# pdm
-#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-#pdm.lock
-#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
-#   in version control.
-#   https://pdm.fming.dev/#use-with-ide
-.pdm.toml
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/
-
-# pytype static type analyzer
-.pytype/
-
-# Cython debug symbols
-cython_debug/
-
-# PyCharm
-#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
-#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-#  and can be added to the global gitignore or merged into this file.  For a more nuclear
-#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
-
-output/
-images/
-.vscode
-vllm_encoder/
-
-!table_related_benchmarks/evalset/bird_data/*.sql
-!table_related_benchmarks/evalset/spider_data/*.sql
-
-table_related_benchmarks/evalset/spider_data/test_database/*
-table_related_benchmarks/evalset/bird_data/dev_databases/*
-table_related_benchmarks/evalset/spider_data/dev_database/*
-
-
-!table_related_benchmarks/evalset/spider_data/test_database/README.md
-!table_related_benchmarks/evalset/bird_data/dev_databases/README.md
-!table_related_benchmarks/evalset/spider_data/dev_database/README.md

evaluation/README.md

@@ -1,181 +0,0 @@
-# Benchmarks evaluations for tablegpt
-
-<p align="center">
-    <a href="#-About">🔥About</a> •
-    <a href="#-Usage">💻Usage</a> •
-</p>
-
-## About
-
-</div>
-
-This is a repo opened for evaluation on different table-related benchmarks for tablegpt. 
-
-Given the complexity of table QA tasks and the uncertainty of input instructions,  we provide evaluation datasets and scripts for 7 capabilities: 
-
-- ✨Code correction based on tables 
-- ✨Refusal of ambiguous questions
-- ✨Table & field recall in multi-table scenarios
-- ✨Table QA output code executable
-- ✨Table-Bench.
-- ✨Text2Sql.
-- ✨TableInstruct, which includes a series of table-related evaluation benchmarks.
-
-In addition, we have integrated other general abilities benchmarks like HumanEval, MBPP and MMLU/CMMLU.
-We have built an inference method based on the  local model path using vLLM as the backend, and defined a set of example prompts templates for the above benchmarks. 
-
-## Usage
-
-</div>
-</details>
-
-⏬ To use this framework, please first install the repository from GitHub:
-
-```shell
-git clone https://github.com/tablegpt/tablegpt-eval
-cd tablegpt-eval
-pip install -r requirements.txt
-```
-
-</div>
-</details>
-
-[!Tips]
-1. You can run all the benchmarks with the default params by running command `bash run_benchmarks.sh`.
-2. If you want more configuration options for running parameters, refer to the typical Python script.
-3. Download the .db files before running text2sql evaluation scripts. Download urls refer to `/table_related_benchmarks/evalset/bird_data/dev_databases/README.md`(Bird dev) & `table_related_benchmarks/evalset/spider_data/dev_database/README.md` (Spider dev) & `table_related_benchmarks/evalset/spider_data/test_database/README.md` (Spider test).
-
-
-### Code correction eval
-
-We provide a non-executable eval dataset based on the Python language. Eval dataset path:
-
-```python
-table_related_benchmarks/evalset/code_correction_test/correction_set.json
-```
-
-We use the  ***executable_pass_rate*** and ***absolute_match_rate***  of the corrected code in pass-1 to evaluate the model's code correction ability. You can perform code-correction evaluation by running the following Python command:
-
-```bash
-python table_related_benchmarks/run_code_correction_eval.py \
-		--model_path  <EVAL MODEL PATH> \
-		--template  <CHAT_TEMPLATE_NAME, support [llama3, baichuan, chatglm, None], default None> \
-    	--eval_results_save_path <PATH TO SAVE THE EVAL RESULTS> \
-        --gpus_num <NUMBER OF GPU TO RUN INFERENCE> \
-        --temperature <ONE OF THE INFERENCE PARAMETER>
-```
-
-### Ambiguous reject eval
-
-We provide 298 table-based queries, with a ratio of about 1:3 between queries marked as ambiguous (to be rejected) and queries that should be accepted and correctly answered. Dataset path:
-
-```python
-# test queries
-evalset/reject_test/test_query.json
-# queries with ground truth
-evalset/reject_test/ground_truth.json
-```
-
-We use **accuracy**, **recall**, and **F1 score** as metrics to evaluate the LLM's ability in this task. You can perform reject evaluation by  running the following Python command:
-
-```bash
-python table_related_benchmarks/run_reject_eval.py \
-    --model_path <EVAL MODEL PATH>  \
-    --save_path <LLM OUTPUT CONTENT SAVE PATH> \
-    --gpus_num <NUMBER OF GPU TO RUN INFERENCE> \
-    --temperature <ONE OF THE INFERENCE PARAMETER>
-```
-
-### Table&Fields recall eval
-
-The provided eval dataset path:
-
-```python
-table_related_benchmarks/evalset/retrieval_test/recall_set.json
-```
-
-We use a series of evaluation metrics such as **recall**, **precision**, **Jaccard similarity**, and **Hamming loss** to assess the LLM's performance in table and field retrieval tasks.  You can perform recall evaluation by  running the following Python command:
-
-```bash
-python table_related_benchmarks/run_recall_eval.py \
-    --model_path <EVAL MODEL PATH> \
-    --temperature <TEMPERATURE> \
-    --gpus_num <NUMBER OF GPU TO RUN INFERENCE> 
-```
-
-### Table QA executable 
-
-Provide 2178 table based queries,  eval dataset path:
-
-```python
-table_related_benchmarks/evalset/table_qa_execuate_test/tableqa_samples_with_paths.jsonl
-```
-
-We employ ***executable_pass_rate***  of pass-1 to employ the model's tableQA code generation ability. You can perform tableQA evaluation by  running the following Python command:
-
-```bash
-python table_related_benchmarks/run_tableqa_execution_eval.py \
-    --model_path <EVAL MODEL PATH> \
-    --temperature <ONE OF THE INFERENCE PARAMETER> \
-    --gpus_num <NUMBER OF GPU TO RUN INFERENCE> 
-```
-
-### TableBench evaluation
-
-The provided eval dataset path:
-
-```python
-table_related_benchmarks/evalset/TableBench
-```
-
-In the evaluation of TableBench, Rough-L was used to assess general QA questions, while pass@1 was used as the evaluation metric for visualization-type samples.  You can perform TableBench evaluation by the following command:
-
-```bash
-python table_related_benchmarks/run_table_bench_eval.py \
-    --model_path <EVAL MODEL PATH> \
-    --temperature <ONE OF THE INFERENCE PARAMETER> \
-    --gpus_num <NUMBER OF GPU TO RUN INFERENCE> 
-```
-
-### TableInstruct
-
-The provided eval dataset path:
-
-```python
-table_related_benchmarks/evalset/TableInstruct
-```
-
-You can perform TableInstruct evaluation by the following command:
-
-```bash
-python table_related_benchmarks/run_table_instruct_eval.py \
-    --model_path <EVAL MODEL PATH> \
-    --temperature <ONE OF THE INFERENCE PARAMETER> \
-    --gpus_num <NUMBER OF GPU TO RUN INFERENCE> 
-```
-
-### Text2Sql
-```bash
-python table_related_benchmarks/run_text2sql_eval.py --model_path <EVAL MODEL PATH> 
-```
-
-### HumanEval
-Perform HumanEval evaluation by the following command:
-
-```bash
-python general_benchmarks/HumanEval/eval_instruct_vllm.py --model_path <EVAL MODEL PATH> 
-```
-
-### MBPP
-Perform MBPP evaluation by the following command:
-
-```bash
-python general_benchmarks/MBPP/eval_instruct_vllm.py --model_path <EVAL MODEL PATH> 
-```
-
-### MMLU & CMMLU
-
-```bash
-python general_benchmarks/MMLU/evaluator.py --task <mmlu or cmmlu> --lang <en or zh> --model_path <EVAL MODEL PATH> 
-```
-

evaluation/general_benchmarks/HumanEval/README.md

@@ -1,74 +0,0 @@
-## 1. Introduction
-
-We provide a test script to evaluate the performance of the **deepseek-coder** model on code generation benchmarks. We select the widely-used benchmarks: **[HumanEval-Python](https://huggingface.co/datasets/openai_humaneval), [HumanEval-Multilingual](https://huggingface.co/datasets/nuprl/MultiPL-E)**.
-
-
-
-## 2. Setup
-
-```
-pip install accelerate
-pip install attrdict
-pip install transformers
-pip install pytorch
-```
-
-
-## 3. Evaluation
-
-We've created a sample script, **eval.sh**, that demonstrates how to test the **DeepSeek-Coder-1.3b-Base** model on the HumanEval dataset leveraging **8** GPUs. If your use case involves a different model or dataset, simply adjust the script to fit your needs.
-
-Additionally, for various programming languages, the execution path may differ. Please ensure you update the appropriate paths in the **humaneval/execution.py** file accordingly.
-
-```bash
-MODEL_NAME_OR_PATH="deepseek-ai/deepseek-coder-1.3b-base"
-DATASET_ROOT="data/"
-LANGUAGE="python"
-python -m accelerate.commands.launch --config_file test_config.yaml eval_pal.py --logdir ${MODEL_NAME_OR_PATH} --language ${LANGUAGE} --dataroot ${DATASET_ROOT} 
-```
-
-To evaluate the instruction-based model, please follow the script below:
-```bash
-LANG="python"
-OUPUT_DIR="output"
-MODEL="deepseek-coder-33b-instruct"
-
-CUDA_VISIBLE_DEVICES=0,1 python eval_instruct.py \
-    --model "deepseek-ai/$MODEL" \
-    --output_path "$OUPUT_DIR/${LANG}.$MODEL.jsonl" \
-    --language $LANG \
-    --temp_dir $OUPUT_DIR
-```
-
-## 4. Experimental Results
-
-We report experimental results here for 8 main-stream programming languages, **python**, **c++**, **java**, **PHP**, **TypeScript**, **C#**, **Bash**, and **JavaScript**. For all open-source models, we utilize this repository to obtain the performance of the models on the HumanEval dataset. We set the maximum input length to **4096** and the maximum output length to **500**, and employ the **greedy search strategy**.
-
-
-#### (1) Multilingual Base Models
-
-| Model             | Size | Python | C++   | Java | PHP  | TS   | C#   | Bash | JS   | Avg  |
-|-------------------|------|--------|-------|------|------|------|------|------|------|------|
-| code-cushman-001  | 12B  | 33.5%  | 31.9% | 30.6%| 28.9%| 31.3%| 22.1%| 11.7%| -    | -    |
-| CodeShell         | 7B   | 35.4%  | 32.9% | 34.2%| 31.7%| 30.2%| 38.0%| 7.0% | 33.5%| 30.4%|
-| CodeGeeX2         | 6B   | 36.0%  | 29.2% | 25.9%| 23.6%| 20.8%| 29.7%| 6.3% | 24.8%| 24.5%|
-| StarCoderBase     | 16B  | 31.7%  | 31.1% | 28.5%| 25.4%| 34.0%| 34.8%| 8.9% | 29.8%| 28.0%|
-| CodeLLama         | 7B   | 31.7%  | 29.8% | 34.2%| 23.6%| 36.5%| 36.7%| 12.0%| 29.2%| 29.2%|
-| CodeLLama         | 13B  | 36.0%  | 37.9% | 38.0%| 34.2%| 45.2%| 43.0%| 16.5%| 32.3%| 35.4%|
-| CodeLLama         | 34B  | 48.2%  | 44.7% | 44.9%| 41.0%| 42.1%| 48.7%| 15.8%| 42.2%| 41.0%|
-| | | | |  |  |  |  |  |  | |
-| DeepSeek-Coder-Base| 1.3B   | 34.8%  | 31.1% | 32.3%| 24.2%| 28.9%| 36.7%| 10.1%| 28.6%| 28.3%|
-| DeepSeek-Coder-Base| 5.7B   | 48.7%  | 45.3% | 41.1%| 39.7%| 44.7%| 41.1%| 27.8%| 42.2%| 41.3%|
-| DeepSeek-Coder-Base| 6.7B   | 49.4%  | 50.3% | 43.0%| 38.5%| 49.7%| 50.0%| 28.5%| 48.4%| 44.7%|
-| DeepSeek-Coder-Base|33B  | **56.1%**  | **58.4%** | **51.9%**| **44.1%**| **52.8%**| **51.3%**| **32.3%**| **55.3%**| **50.3%**|
-
-#### (2) Instruction-Tuned Models
-| Model               | Size | Python | C++   | Java | PHP  | TS   | C#   | Bash | JS   | Avg  |
-|---------------------|------|--------|-------|------|------|------|------|------|------|------|
-| GPT-3.5-Turbo         | -    | 76.2%  | 63.4% | 69.2%| 60.9%| 69.1%| 70.8%| 42.4%| 67.1%| 64.9%|
-| GPT-4               | -    | **84.1%**  | **76.4%** | **81.6%**| **77.2%**| **77.4%**| **79.1%**| **58.2%**| **78.0%**| **76.5%**|
-| | | | |  |  |  |  |  |  | |
-| DeepSeek-Coder-Instruct | 1.3B  | 65.2%      | 45.3%    | 51.9%    | 45.3%    | 59.7%   |55.1%    | 12.7%    | 52.2%    | 48.4%    |
-| DeepSeek-Coder-Instruct | 6.7B  | 78.9%  | 63.4% | 68.4% | 68.9%| 67.2%| 72.8%| 36.7%| 72.7%| 66.1%|
-| DeepSeek-Coder-Instruct | 33B | **79.3%**  | **68.9%** | **73.4%** | **72.7%**| **67.9%**| **74.1%**| **43.0%**| **73.9%**| **69.2%**|
-

evaluation/general_benchmarks/HumanEval/data/humaneval-cpp.jsonl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8717eabdf202137158c84506144b0fb1e73d5ecccbe5363ec79009ca014df629
-size 388688

evaluation/general_benchmarks/HumanEval/data/humaneval-cs-bu.jsonl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d281b53b24e0f44cb76f1b1a8702b1ca668ff2a29c7621276ee8b658f5c124c6
-size 448701

evaluation/general_benchmarks/HumanEval/data/humaneval-cs.jsonl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1c3fe18be10addc2d0b96311f4db192ae3232e08628d17768d889d6ab87be224
-size 452021

evaluation/general_benchmarks/HumanEval/data/humaneval-d.jsonl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:226938c015f90d713a3e30d8f174f4a6d2c88820cf50512379a16890dda70332
-size 289365

evaluation/general_benchmarks/HumanEval/data/humaneval-go.jsonl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7f5dfe3ff001049b1221e9d21a8119b5cbc38eb87c97fefc5d57fa7adc1df888
-size 432325

evaluation/general_benchmarks/HumanEval/data/humaneval-java.jsonl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b21b015763452ec4f9f4e0e6425148ec331ace4da1232c8b4d441186185f6265
-size 454059

evaluation/general_benchmarks/HumanEval/data/humaneval-jl.jsonl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ef21bd920889c9c1ab0e87a825cc26bd895e7a715f569f8ba7de577f870b6815
-size 268754

evaluation/general_benchmarks/HumanEval/data/humaneval-js.jsonl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:634e2eee8d6e22de07c121b972207683dd96be76256bf44bdfc1a3386b739287
-size 297853

evaluation/general_benchmarks/HumanEval/data/humaneval-lua.jsonl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d2e91f3603f6aee63db2c2d5754d165397603a8c9bd6130842af7988b27a96fc
-size 298314

evaluation/general_benchmarks/HumanEval/data/humaneval-php.jsonl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:bebd86875b1d8e65a8b7e692ed7bdf64612b44aa388825ea4dab40c7047c786b
-size 388096

evaluation/general_benchmarks/HumanEval/data/humaneval-pl.jsonl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:949048bff3eaea7ae47cd5759042c088a7c227d53c74fe95a80728fd5aefbf77
-size 437506

evaluation/general_benchmarks/HumanEval/data/humaneval-python.jsonl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0eae07adadbb00d51962fdb78b9e2a26bfa8ade85dc54eb57cae9bffed2f5c54
-size 342974

evaluation/general_benchmarks/HumanEval/data/humaneval-r.jsonl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c2a3268ea54d2bfb8ce65bb2c808437ac0d9934c6caf99fcf75d6b6a4fb3f911
-size 311904

evaluation/general_benchmarks/HumanEval/data/humaneval-rb.jsonl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:4204395b835019e32d55513325a0fad01c6d382fd1eb97b516f0458d00058302
-size 312806

evaluation/general_benchmarks/HumanEval/data/humaneval-rkt.jsonl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:083325daf55daf431ae6d6d17cf017afb18a6ec790d4c841b7c2b4752c5807ff
-size 312006

evaluation/general_benchmarks/HumanEval/data/humaneval-rs.jsonl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e33f562838e973b7e6b8d76dbc1fb84b076d7426629b4cdc624f12678778d2fa
-size 306470

evaluation/general_benchmarks/HumanEval/data/humaneval-scala.jsonl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:482b72a16613a563755d642e37792da68471399789b069fba5b1249e831445f3
-size 384243

evaluation/general_benchmarks/HumanEval/data/humaneval-sh.jsonl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:2f702106e9ce4aa7385de568d9248d6c57c90382d782b162cb9e072fbd01ccf8
-size 274180

evaluation/general_benchmarks/HumanEval/data/humaneval-swift.jsonl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e4418d6f532e4298969e727690cda80dc88b94e4abf836f8ff79ac18a737eaaa
-size 344436

evaluation/general_benchmarks/HumanEval/data/humaneval-ts.jsonl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:4b1af1050c9f226335d54a8c80553c39454a525cace3e67bdcfcc9092ba02637
-size 304732

evaluation/general_benchmarks/HumanEval/eval.sh

@@ -1,4 +0,0 @@
-MODEL_NAME_OR_PATH="/data3/models/DeepSeek/deepseek-coder-6.7b-base"
-DATASET_ROOT="HumanEval/data"
-LANGUAGE="python"
-CUDA_VISIBLE_DEVICES=5,6,7 python -m accelerate.commands.launch --config_file HumanEval/test_config.yaml HumanEval/eval_pal.py --model_path ${MODEL_NAME_OR_PATH} --language ${LANGUAGE} --dataroot ${DATASET_ROOT}

evaluation/general_benchmarks/HumanEval/eval_base_vllm.py

@@ -1,162 +0,0 @@
-import json
-import os
-import time
-from argparse import ArgumentParser
-# from accelerate import Accelerator
-# from accelerate import DistributedDataParallelKwargs
-from pathlib import Path
-
-import numpy as np
-import torch
-import torch.nn.functional as F
-from human_eval.evaluation import evaluate_functional_correctness
-from tqdm import tqdm
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from utils.dataset import HumanEvalDataset
-from utils.utils import cleanup_code
-from vllm import LLM, SamplingParams
-
-
-class HumanEval:
-    """
-    HumanEval evaluation class.
-    """
-
-    def __init__(
-        self,
-        data_root,
-        language="python",
-        log_dir=None,
-        issft=False,
-        inference_increment=True,
-        n_sample=1,
-        k_sample=1,
-    ):
-        self.data_root = data_root
-        self.k = k_sample
-        self.n_sample = n_sample
-        self.language = language
-        self.log_dir = log_dir
-        self.sft = issft
-        self.inference_increment = inference_increment
-        os.makedirs(self.log_dir, exist_ok=True)
-
-    @torch.no_grad()
-    def eval_model(self, args):
-        """
-        Evaluate the model on HumanEval.
-        """
-        assert (
-            self.log_dir is not None
-        ), "log_dir should not be None when evaluating humaneval"
-        dataset = HumanEvalDataset(
-            self.data_root,
-            sample_num=self.n_sample,
-            language=self.language,
-            issft=self.sft,
-        )
-        model_name_or_path = args.model_path
-        print("model", model_name_or_path)
-        tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
-        print(
-            "load tokenizer {} from {} over.".format(
-                tokenizer.__class__, model_name_or_path
-            )
-        )
-
-        llm = LLM(
-            model=model_name_or_path,
-            tensor_parallel_size=1,
-            max_model_len=4096,
-            trust_remote_code=True,
-            enforce_eager=True,
-        )
-        sampling_params = SamplingParams(
-            temperature=0,
-            max_tokens=1024,
-            top_p=0.95,
-            stop_token_ids=[tokenizer.eos_token_id],
-        )
-        messages_list = []
-        for i in range(len(dataset)):
-            data = dataset[i]
-            prompt = data["prompt"].strip()
-            messages_list.append(prompt)
-        outputs = llm.generate(messages_list, sampling_params=sampling_params)
-        assert len(dataset) == len(outputs), "dataset and outputs different lengths."
-        log_file = os.path.join(self.log_dir, f"{self.language}.json")
-        tmpfile = open(log_file, "w")
-        for i, output in enumerate(tqdm(outputs)):
-            data = dataset[i]
-            output = output.outputs[0].text
-            output = cleanup_code(
-                output,
-                self.language,
-                "humaneval",
-                self.sft,
-                dataset.stopwords,
-            )
-            # sft mode does not need original prompt
-            if not self.sft:
-                suffixprediction = data["original_prompt"] + "\n" + output
-            res = {
-                "task_id": data["task_id"],
-                "generation": suffixprediction,
-                "prompt": data["original_prompt"],
-            }
-            tmpfile.write(json.dumps(res) + "\n")
-
-        tmpfile.close()
-        # calculate the final score of pass@k
-        self._calculate_final_score(log_file)
-        return
-
-    def _calculate_final_score(self, logfilepath):
-        """
-        Calculate the final score.
-        """
-        res = evaluate_functional_correctness(
-            input_file=logfilepath,
-            problem_file=os.path.join(
-                self.data_root, f"humaneval-{self.language}.jsonl"
-            ),
-            tmp_dir=self.log_dir,
-            language=self.language,
-        )
-        print("score is", res["pass@%d" % self.k])
-        os.remove(logfilepath)
-        return
-
-
-if __name__ == "__main__":
-    parser = ArgumentParser()
-    parser.add_argument("--logdir", type=str, default="")
-    parser.add_argument(
-        "--model_path",
-        type=str,
-        help="model name or path",
-        default="/data0/pretrained-models/qwen2-7b",
-    )
-
-    parser.add_argument("--language", type=str, default="python")
-    parser.add_argument(
-        "--dataroot",
-        type=str,
-        default="HumanEval/data",
-    )
-    args = parser.parse_args()
-
-    logdir = args.logdir
-    language = args.language
-
-    if logdir == "":
-        logdir = "output/tmp/"
-
-    evaluator = HumanEval(
-        data_root=args.dataroot,
-        log_dir=logdir,
-        n_sample=1,
-        language=language,
-    )
-    os.environ["TOKENIZERS_PARALLELISM"] = "false"
-    evaluator.eval_model(args)

evaluation/general_benchmarks/HumanEval/eval_instruct.py

@@ -1,168 +0,0 @@
-import argparse
-import json
-import os
-from pathlib import Path
-
-import torch
-from tqdm import tqdm
-
-data_abs_dir = Path(__file__).parent / "data"
-
-from human_eval.evaluation import evaluate_functional_correctness
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from utils.utils import extract_generation_code, languge_settings
-
-
-def build_deepseekcoder_instruction(languge: str, question: str):
-    return """
-Please continue to complete the function. You are not allowed to modify the given code and do the completion only. Please return all completed function in a codeblock. Here is the given code to do completion:
-```{}
-{}
-```
-""".strip().format(
-        languge.lower(), question.strip()
-    )
-
-
-def generate_one(example, lang, tokenizer, model):
-    prompt = build_deepseekcoder_instruction(
-        languge_settings[lang]["full_name"], example["prompt"]
-    )
-    inputs = tokenizer.apply_chat_template(
-        [{"role": "user", "content": prompt}],
-        return_tensors="pt",
-        add_generation_prompt=True,
-    ).to(model.device)
-
-    stop_id = tokenizer.convert_tokens_to_ids("<|EOT|>")
-    assert isinstance(stop_id, int), "Invalid tokenizer, EOT id not found"
-
-    outputs = model.generate(
-        inputs,
-        max_new_tokens=1024,
-        do_sample=False,
-        # top_p=0.95,
-        # temperature=temperature,
-        pad_token_id=stop_id,
-        eos_token_id=stop_id,
-    )
-
-    output = tokenizer.decode(outputs[0][len(inputs[0]) :], skip_special_tokens=True)
-    example["output"] = output
-
-    return extract_generation_code(example, lang_code=lang)
-
-
-def generate_main(args):
-    model_name_or_path = args.model
-    lang = args.language
-    saved_path = args.output_path
-    temp_dir = args.temp_dir
-    os.makedirs(temp_dir, exist_ok=True)
-    problem_file = os.path.join(data_abs_dir, f"humaneval-{lang}.jsonl")
-
-    print("model", model_name_or_path)
-    tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
-    print(
-        "load tokenizer {} from {} over.".format(
-            tokenizer.__class__, model_name_or_path
-        )
-    )
-    model = AutoModelForCausalLM.from_pretrained(
-        model_name_or_path,
-        torch_dtype=torch.bfloat16,
-        device_map="auto",
-        # use_flash_attention_2=True
-    )
-    model.eval()
-    examples = [json.loads(x) for x in open(problem_file) if x.strip()]
-    print("Read {} examples for evaluation over.".format(len(examples)))
-
-    generated_examples = []
-    for ex in tqdm(examples, desc="Generating"):
-        gen_example = generate_one(ex, args.language, tokenizer, model)
-        generated_examples.append(gen_example)
-
-    print("Generate all over!!!")
-    with open(saved_path, "w", encoding="utf-8") as fw:
-        for ex in generated_examples:
-            fw.write(json.dumps(ex) + "\n")
-        print(
-            "Save {} processed examples into {} over!".format(
-                len(generated_examples), saved_path
-            )
-        )
-
-    result = evaluate_functional_correctness(
-        input_file=saved_path,
-        tmp_dir=temp_dir,
-        n_workers=8,
-        timeout=3.0,
-        problem_file=problem_file,
-        language=lang,
-    )
-    print(lang, result, model_name_or_path)
-    pass
-
-
-def evaluation_only(args):
-    lang = args.language
-    temp_dir = args.temp_dir
-    assert os.path.exists(args.output_path), "Not fond output file: {}".format(
-        args.output_path
-    )
-    os.makedirs(temp_dir, exist_ok=True)
-
-    output_name = os.path.basename(args.output_path)
-    output_examples = [json.loads(x) for x in open(args.output_path) if x.strip()]
-
-    processed_examples = [
-        extract_generation_code(ex, lang) for ex in tqdm(output_examples, "Processing")
-    ]
-    processed_path = os.path.join(temp_dir, output_name)
-    with open(processed_path, "w", encoding="utf-8") as fw:
-        for ex in processed_examples:
-            fw.write(json.dumps(ex) + "\n")
-        print(
-            "Save {} processed examples into {} over!".format(
-                len(processed_examples), processed_path
-            )
-        )
-
-    problem_file = os.path.join(data_abs_dir, f"humaneval-{lang}.jsonl")
-    from human_eval.evaluation import evaluate_functional_correctness
-
-    result = evaluate_functional_correctness(
-        input_file=processed_path,
-        tmp_dir=temp_dir,
-        n_workers=8,
-        timeout=3.0,
-        problem_file=problem_file,
-        language=lang,
-    )
-    print(lang, result)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "--model",
-        type=str,
-        help="model name or path",
-        default="/data0/pretrained-models/deepseek-coder-6.7b-instruct",
-    )
-    parser.add_argument(
-        "--output_path",
-        type=str,
-        help="output path of your generation",
-        default="/home/qyhuang/DeepSeek-Coder/outputs/deepseek-chat.json",
-    )
-    parser.add_argument("--language", type=str, help="langauge", default="python")
-    parser.add_argument(
-        "--temp_dir", type=str, help="temp dir for evaluation", default="tmp"
-    )
-    args = parser.parse_args()
-
-    os.environ["TOKENIZERS_PARALLELISM"] = "false"
-    generate_main(args)
-    pass

evaluation/general_benchmarks/HumanEval/eval_instruct_vllm.py

@@ -1,225 +0,0 @@
-import argparse
-import json
-import os
-import shutil
-from pathlib import Path
-
-import torch
-import transformers
-from human_eval.evaluation import evaluate_functional_correctness
-from tqdm import tqdm
-from transformers import AutoTokenizer
-from utils.utils import extract_generation_code, languge_settings
-from vllm import LLM, SamplingParams
-
-data_abs_dir = Path(__file__).parent / "data"
-
-
-def build_deepseekcoder_instruction(languge: str, question: str):
-    return """
-Please continue to complete the function. You are not allowed to modify the given code and do the completion only. Please return all completed function in a codeblock. Here is the given code to do completion:
-```{}
-{}
-```
-""".strip().format(
-        languge.lower(), question.strip()
-    )
-
-
-def create_dir(output_dir):
-    if os.path.exists(output_dir):
-        if not os.access(output_dir, os.W_OK):
-            shutil.rmtree(output_dir)
-            os.makedirs(output_dir)
-            os.chmod(output_dir, 0o777)
-            print("not write permission, makedir:", output_dir)
-        else:
-            print(f"{output_dir} exists!")
-    else:
-        os.makedirs(output_dir)
-        os.chmod(output_dir, 0o777)
-        print("makedir:", output_dir)
-
-
-def get_client_res(messages, example, output_key, open_ai_key=False):
-    try:
-        if open_ai_key:
-            from openai import AzureOpenAI, OpenAI
-            try:
-                api_key = os.environ["OPENAI_API_KEY"]
-            except KeyError:
-                print("环境变量 OPENAI_API_KEY 未设置")
-                api_key = "default_value"
-
-            client = AzureOpenAI(
-                api_key=api_key,
-                api_version="2024-07-01-preview",
-                azure_endpoint="https://zju-tablegpt.openai.azure.com/",
-            )
-            chat_response = client.chat.completions.create(
-                model="gpt-4o",
-                # model="gpt-4o-mini",
-                messages=messages,
-                top_p=0.95,
-                temperature=0,
-                max_tokens=1024,
-                timeout=40,
-            )
-        else:
-            # Set OpenAI's API key and API base to use vLLM's API server.
-            openai_api_key = "EMPTY"
-            openai_api_base = "http://localhost:8080/v1"
-
-            client = OpenAI(
-                api_key=openai_api_key,
-                base_url=openai_api_base,
-            )
-            chat_response = client.chat.completions.create(
-                model="qwen2-7b-sft",
-                messages=messages,
-                top_p=0.3,
-                temperature=0.1,
-                max_tokens=1024,
-            )
-        example[output_key] = chat_response.choices[0].message.content
-    except Exception as e:
-        print(f"An unexpected error occurred: {e}")
-        example[output_key] = None
-    example["input"] = messages
-    return example
-
-
-
-def generate_main(args):
-    model_name_or_path = args.model_path
-    lang = args.language
-    temp_dir = args.temp_dir
-    create_dir(temp_dir)
-    # os.makedirs(temp_dir, exist_ok=True)
-    problem_file = os.path.join(data_abs_dir, f"humaneval-{lang}.jsonl")
-    if not args.api:
-        print("model", model_name_or_path)
-        tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
-        print(
-            "load tokenizer {} from {} over.".format(
-                tokenizer.__class__, model_name_or_path
-            )
-        )
-        llm_args = {
-            "model": model_name_or_path,
-            "gpu_memory_utilization": 0.95,
-            "trust_remote_code": True,
-            "tensor_parallel_size": args.gpus_num,
-            "dtype": "half",
-            "max_model_len": 8192,
-            "enforce_eager": True,
-        }
-
-        llm = LLM(**llm_args)
-        sampling_params = SamplingParams(
-            temperature=0,
-            max_tokens=1024,
-            top_p=0.95,
-            stop_token_ids=[tokenizer.eos_token_id],
-        )
-
-    examples = [json.loads(x) for x in open(problem_file) if x.strip()]
-    print("Read {} examples for evaluation over.".format(len(examples)))
-    messages_list = []
-    for example in tqdm(examples, desc="Generating"):
-        prompt = build_deepseekcoder_instruction(
-            languge_settings[lang]["full_name"], example["prompt"]
-        )
-        message = [{"role": "user", "content": prompt}]
-        if args.api:
-            messages_list.append(message)
-        else:
-            messages_list.append(
-                tokenizer.apply_chat_template(
-                    message, tokenize=False, add_generation_prompt=True
-                )
-            )
-    if args.api:
-        from joblib import Parallel, delayed
-        examples_ = Parallel(n_jobs=24)(
-            delayed(get_client_res)(inp, examples[i], "output",open_ai_key=True)
-            for i, inp in enumerate(tqdm(messages_list))
-        )
-
-        # 请求错误的重新请求
-        examples = []
-        for example in examples_:
-            if example["output"] == None:
-                example = get_client_res(
-                    example["input"], example, "output", open_ai_key=True
-                )
-            del example["input"]
-            examples.append(example)
-
-        generated_examples = []
-        for example in examples:
-            example = extract_generation_code(example, lang_code=lang)
-            generated_examples.append(example)
-    else:
-        outputs = llm.generate(messages_list, sampling_params=sampling_params)
-        generated_examples = []
-        for i, output in enumerate(tqdm(outputs)):
-            output = output.outputs[0].text
-            example = examples[i]
-            example["output"] = output
-            example = extract_generation_code(example, lang_code=lang)
-            generated_examples.append(example)
-
-    print("Generate all over!!!")
-    # os.makedirs(args.save_dir, exist_ok=True)
-    create_dir(args.save_dir)
-    saved_path = os.path.join(args.save_dir, "results_humaneval.json")
-    with open(saved_path, "w", encoding="utf-8") as fw:
-        for ex in generated_examples:
-            fw.write(json.dumps(ex) + "\n")
-        print(
-            "Save {} processed examples into {} over!".format(
-                len(generated_examples), saved_path
-            )
-        )
-
-    result = evaluate_functional_correctness(
-        input_file=saved_path,
-        tmp_dir=temp_dir,
-        n_workers=8,
-        timeout=3.0,
-        problem_file=problem_file,
-        language=lang,
-        out_path=saved_path,
-    )
-    print(lang, result, model_name_or_path)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "--model_path",
-        type=str,
-        help="model name or path",
-        default="/data4/sft_output/qwen2-instruct-0709/checkpoint-1400",
-    )
-    parser.add_argument(
-        "--gpus_num", type=int, default=1, help="the number of GPUs you want to use."
-    )
-    parser.add_argument(
-        "--save_dir",
-        type=str,
-        help="output path of your generation",
-        default="output",
-    )
-    parser.add_argument("--api", action="store_true", help="infer api type")
-    parser.add_argument("--language", type=str, help="langauge", default="python")
-    parser.add_argument(
-        "--temp_dir", type=str, help="temp dir for evaluation", default="output/tmp"
-    )
-    parser.add_argument("--seed", type=int, help="seed", default=42)
-    args = parser.parse_args()
-
-    os.environ["TOKENIZERS_PARALLELISM"] = "false"
-    transformers.set_seed(args.seed)
-    generate_main(args)

evaluation/general_benchmarks/HumanEval/eval_pal.py

@@ -1,62 +0,0 @@
-import json
-import os
-import subprocess
-import sys
-from argparse import ArgumentParser
-from pathlib import Path
-
-import numpy as np
-import pandas as pd
-import torch
-import torch.distributed as dist
-import torch.nn.functional as F
-from accelerate import Accelerator, DistributedDataParallelKwargs
-from humaneval import HumanEval as evaltor
-from transformers import AutoModelForCausalLM, AutoTokenizer
-
-if __name__ == "__main__":
-    kwargs_handlers = [DistributedDataParallelKwargs(find_unused_parameters=True)]
-    accelerator = Accelerator(mixed_precision="bf16", kwargs_handlers=kwargs_handlers)
-
-    parser = ArgumentParser()
-    parser.add_argument("--logdir", type=str, default="./output")
-    parser.add_argument(
-        "--model_path",
-        type=str,
-        default="/data3/models/DeepSeek/deepseek-coder-6.7b-base",
-    )
-    parser.add_argument("--language", type=str, default="python")
-    parser.add_argument("--dataroot", type=str, default="HumanEval/data")
-    args = parser.parse_args()
-
-    logdir = args.logdir
-    language = args.language
-    model_path = args.model_path
-
-    if logdir == "":
-        logdir = "tmp/"
-    tokenizer = dict(
-        cls=AutoTokenizer,
-        model_path=model_path,
-    )
-
-    dataroot = args.dataroot
-
-    evaluator = evaltor(
-        data_root=dataroot,
-        max_seq_len=4096,
-        tokenizer_cfg=tokenizer,
-        log_dir=logdir,
-        n_sample=1,
-        batch_size=1,
-        language=language,
-        max_gen_len=500,
-    )
-    model = AutoModelForCausalLM.from_pretrained(
-        model_path,
-        device_map=accelerator.device,
-        trust_remote_code=True,
-        torch_dtype=torch.bfloat16,
-    )
-    os.environ["TOKENIZERS_PARALLELISM"] = "false"
-    evaluator.eval_model(model, accelerator)

evaluation/general_benchmarks/HumanEval/human_eval/data.py

@@ -1,48 +0,0 @@
-import gzip
-import json
-import os
-from typing import Dict, Iterable
-
-ROOT = os.path.dirname(os.path.abspath(__file__))
-HUMAN_EVAL = os.path.join(ROOT, "..", "data", "HumanEval.jsonl.gz")
-
-
-def read_problems(evalset_file: str = HUMAN_EVAL) -> Dict[str, Dict]:
-    return {task["task_id"]: task for task in stream_jsonl(evalset_file)}
-
-
-def stream_jsonl(filename: str) -> Iterable[Dict]:
-    """
-    Parses each jsonl line and yields it as a dictionary
-    """
-    if filename.endswith(".gz"):
-        with open(filename, "rb") as gzfp:
-            with gzip.open(gzfp, "rt") as fp:
-                for line in fp:
-                    if any(not x.isspace() for x in line):
-                        yield json.loads(line)
-    else:
-        with open(filename, "r", encoding="utf-8") as fp:
-            for line in fp:
-                if any(not x.isspace() for x in line):
-                    yield json.loads(line)
-
-
-def write_jsonl(filename: str, data: Iterable[Dict], append: bool = False):
-    """
-    Writes an iterable of dictionaries to jsonl
-    """
-    if append:
-        mode = "ab"
-    else:
-        mode = "wb"
-    filename = os.path.expanduser(filename)
-    if filename.endswith(".gz"):
-        with open(filename, mode) as fp:
-            with gzip.GzipFile(fileobj=fp, mode="wb") as gzfp:
-                for x in data:
-                    gzfp.write((json.dumps(x) + "\n").encode("utf-8"))
-    else:
-        with open(filename, mode) as fp:
-            for x in data:
-                fp.write((json.dumps(x) + "\n").encode("utf-8"))

evaluation/general_benchmarks/HumanEval/human_eval/evaluate_functional_correctness.py

@@ -1,32 +0,0 @@
-import sys
-
-import fire
-
-from .data import HUMAN_EVAL
-from .evaluation import evaluate_functional_correctness
-
-
-def entry_point(
-    sample_file: str,
-    k: str = "1,10,100",
-    n_workers: int = 4,
-    timeout: float = 3.0,
-    problem_file: str = "",
-    is_mbpp: bool = False,
-):
-    """
-    Evaluates the functional correctness of generated samples, and writes
-    results to f"{sample_file}_results.jsonl.gz"
-    """
-    k = list(map(int, k.split(",")))
-    results = evaluate_functional_correctness(
-        sample_file, k, n_workers, timeout, problem_file, is_mbpp
-    )
-    print(results)
-
-
-def main():
-    fire.Fire(entry_point)
-
-
-sys.exit(main())

evaluation/general_benchmarks/HumanEval/human_eval/evaluation.py

@@ -1,351 +0,0 @@
-import gzip
-import itertools
-import json
-import os
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from typing import *
-
-import numpy as np
-from tqdm.auto import tqdm
-
-from human_eval.data import stream_jsonl
-from human_eval.execution import check_correctness
-
-IMPORT_HELPER = {
-    "python": [
-        "import math",
-        "import re",
-        "import sys",
-        "import copy",
-        "import datetime",
-        "import itertools",
-        "import collections",
-        "import heapq",
-        "import functools",
-        "import hashlib",
-        "import numpy",
-        "import numpy as np",
-        "import string",
-        "from typing import *",
-        "from collections import *",
-    ],
-    "go": [
-        "math",
-        "strings",
-        "fmt",
-        "strconv",
-        "time",
-        "bytes",
-        "regexp",
-        "sort",
-        "math/rand",
-        "crypto/md5",
-    ],
-    "cpp": [
-        "#include<stdlib.h>",
-        "#include<algorithm>",
-        "#include<math.h>",
-        "#include<stdio.h>",
-        "#include<vector>",
-        "#include<string>",
-        "#include<climits>",
-        "#include<cstring>",
-        "#include<iostream>",
-        "#include<cassert>",
-    ],
-    "cs": [
-        "using System.Numerics;",
-        "using System.Diagnostics;",
-        "using System.Collections.Generic;",
-        "using System.Linq;",
-        "using System.Text;",
-        "using System.Security.Cryptography;",
-        "using System.Collections.Generic;",
-    ],
-}
-
-
-LANGUAGE_NAME = {
-    "cpp": "CPP",
-    "go": "Go",
-    "java": "Java",
-    "js": "JavaScript",
-    "python": "Python",
-}
-
-
-def read_dataset(
-    data_file: str = None,
-    dataset_type: str = "humaneval",
-    num_shot=None,
-) -> Dict:
-    """
-    Reads a dataset and returns a dictionary of tasks.
-    """
-    if num_shot is not None:
-        print(f"{num_shot}-shot setting...")
-    if "humaneval" in dataset_type.lower():
-        if data_file is None:
-            current_path = os.path.dirname(os.path.abspath(__file__))
-            data_file = os.path.join(
-                current_path,
-                "..",
-                "humaneval-x",
-                "python",
-                "data",
-                "humaneval_python.jsonl.gz",
-            )
-        dataset = {task["task_id"]: task for task in stream_jsonl(data_file)}
-    else:
-        raise f"Dataset: {dataset_type} not supported."
-
-    return dataset
-
-
-def estimate_pass_at_k(
-    num_samples: Union[int, List[int], np.ndarray],
-    num_correct: Union[List[int], np.ndarray],
-    k: int,
-) -> np.ndarray:
-    """
-    Estimates pass@k of each problem and returns them in an array.
-    """
-
-    def estimator(n: int, c: int, k: int) -> float:
-        """
-        Calculates 1 - comb(n - c, k) / comb(n, k).
-        """
-        if n - c < k:
-            return 1.0
-        return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1))
-
-    if isinstance(num_samples, int):
-        num_samples_it = itertools.repeat(num_samples, len(num_correct))
-    else:
-        assert len(num_samples) == len(num_correct)
-        num_samples_it = iter(num_samples)
-
-    return np.array(
-        [estimator(int(n), int(c), k) for n, c in zip(num_samples_it, num_correct)]
-    )
-
-
-def process_humaneval_test(
-    sample, problems, example_test=False, is_mbpp=False, language="python"
-):
-    """
-    Processes a sample for evaluation.
-    """
-    task_id = sample["task_id"]
-    if is_mbpp:
-        return sample["generation"] + "\n" + "\n".join(problems[task_id]["test"])
-
-    prompt = sample["prompt"]
-    if (
-        example_test
-        and "example_test" in problems[task_id]
-        and problems[task_id]["example_test"] != ""
-    ):
-        test = problems[task_id]["example_test"]
-    else:
-        test = problems[task_id]["test"]
-    code = sample["generation"]
-
-    # Pre-process for different languages
-    if language == "python":
-        test_setup = "\n".join(IMPORT_HELPER["python"]) + "\n"
-        test_string = test_setup + code + "\n" + test + "\n"
-    elif language == "cpp":
-        test_set_up = ""
-        for s in IMPORT_HELPER["cpp"]:
-            if s not in prompt:
-                test_set_up += s + "\n"
-        test_string = test_set_up + "\n" + code + "\n" + test
-    elif language == "java":
-        test_string = code + "\n" + test
-    elif language == "cs":
-        test_set_up = ""
-        for s in IMPORT_HELPER["cs"]:
-            test_set_up += s + "\n"
-        test_string = test_set_up + "\n" + code + "\n" + test
-    elif language in ["js", "javascript", "ts", "sh", "go"]:
-        test_string = code + "\n" + test
-    elif language == "go232":
-        import_string = problems[task_id]["import"]
-        prompt = prompt.replace(import_string, "")
-        if example_test and "example_test" in problems[task_id]:
-            test = problems[task_id]["example_test"]
-        else:
-            test = problems[task_id]["test"]
-        test_setup = problems[task_id]["test_setup"]
-        other_pkgs = []
-        for pkg in IMPORT_HELPER["go"]:
-            if pkg not in test_setup:
-                p = pkg.split("/")[-1]
-                if p + "." in code:
-                    other_pkgs.append(f'"{pkg}"')
-        if other_pkgs:
-            import_other_pkgs = (
-                "import (\n" + "    ".join([p + "\n" for p in other_pkgs]) + ")"
-            )
-            test_string = (
-                test_setup
-                + "\n"
-                + import_other_pkgs
-                + "\n"
-                + prompt
-                + code
-                + "\n"
-                + test
-            )
-        else:
-            test_string = test_setup + "\n" + prompt + code + "\n" + test
-    elif language == "rust":
-        main = "\nfn main(){ \n } \n"
-        declaration = problems[task_id]["declaration"]
-        test_string = main + declaration + prompt + code + test
-    elif language == "php":
-        if code[:5] != "<?php":
-            code = "<?php\n" + code
-        test_string = code + "\n" + test + "?>"
-    return test_string
-
-
-def stream_jsonl_all(filename: str) -> Iterable[Dict]:
-    """
-    Streams a JSONL file.
-    """
-    results = []
-    if filename.endswith(".gz"):
-        fp = gzip.open(open(filename, "rb"), "rt")
-    else:
-        fp = open(filename, "r")
-    for line in fp:
-        if any(not x.isspace() for x in line):
-            results.append(json.loads(line))
-    fp.close()
-
-    return results
-
-
-def evaluate_functional_correctness(
-    input_file: str = None,
-    tmp_dir: str = "./",
-    n_workers: int = 32,
-    timeout: float = 10.0,
-    problem_file: str = "../data/humaneval_python.jsonl.gz",
-    out_path: str = None,
-    k: List[int] = [1, 10, 100],
-    test_groundtruth: bool = False,
-    example_test: bool = False,
-    is_mbpp: bool = False,
-    language: str = "python",
-):
-    """
-    Evaluates the functional correctness of a model.
-    """
-    if example_test:
-        print("Example test...")
-
-    problems = read_dataset(problem_file, dataset_type="humaneval")
-    sample_jsonl = stream_jsonl_all(input_file)
-
-    with ThreadPoolExecutor(max_workers=n_workers) as executor:
-
-        futures = []
-        completion_id = Counter()
-        n_samples = 0
-        # results = defaultdict(list)
-        results = {}
-
-        if test_groundtruth:
-            print("Testing ground truth...")
-            for sample in tqdm(problems.values()):
-                task_id = sample["task_id"]
-                lang = task_id.split("/")[0].lower()
-                if lang == "javascript":
-                    lang = "js"
-                tmp_dir_ = os.path.join(tmp_dir, lang, "evaluation")
-                sample["generation"] = sample["canonical_solution"]
-                sample["test_code"] = process_humaneval_test(
-                    sample, problems, example_test, language
-                )
-                if sample["test_code"] is None:
-                    continue
-                args = (
-                    task_id,
-                    sample,
-                    lang,
-                    timeout,
-                    tmp_dir_,
-                    completion_id[task_id],
-                )
-                future = executor.submit(check_correctness, *args)
-                futures.append(future)
-                completion_id[task_id] += 1
-                n_samples += 1
-        else:
-            print("Reading samples...")
-            for sample in tqdm(sample_jsonl):
-                task_id = sample["task_id"]
-                if not is_mbpp:
-                    lang = language
-                if not is_mbpp and lang == "javascript":
-                    lang = "js"
-                if is_mbpp:
-                    lang = "python"
-                tmp_dir_ = os.path.join(tmp_dir, lang, "evaluation")
-                sample["task_id"] = task_id
-                sample["test_code"] = process_humaneval_test(
-                    sample, problems, example_test, is_mbpp, language
-                )
-                if sample["test_code"] is None:
-                    continue
-                if "completion_id" in sample:
-                    completion_id_ = sample["completion_id"]
-                else:
-                    completion_id_ = completion_id[task_id]
-                args = (task_id, sample, lang, timeout, tmp_dir_, completion_id_)
-                future = executor.submit(check_correctness, *args)
-                futures.append(future)
-                completion_id[task_id] += 1
-                n_samples += 1
-
-        if len(completion_id) == len(problems):
-            evaluate_pass_at_k = True
-        else:
-            evaluate_pass_at_k = False
-
-        print("Running test suites...")
-        for future in tqdm(as_completed(futures), total=len(futures)):
-            result = future.result()
-            # results[result["task_id"]].append((result["completion_id"], result))
-            results[result["task_id"]] = result
-
-    # Calculate pass@k.
-    total, correct = [], []
-    for result in results.values():
-        # passed = [r[1]["passed"] for r in result]
-        passed = [result["passed"]]
-        total.append(len(passed))
-        correct.append(sum(passed))
-    total = np.array(total)
-    correct = np.array(correct)
-
-    if evaluate_pass_at_k:
-        ks = k
-        pass_at_k = {
-            f"pass@{k}": estimate_pass_at_k(total, correct, k).mean()
-            for k in ks
-            if (total >= k).all()
-        }
-        print(pass_at_k)
-    else:
-        print("Total:", np.sum(total))
-        print("Correct:", np.sum(correct))
-
-    if out_path:
-        with open(out_path, "w") as f:
-            json.dump(list(results.values()), f, ensure_ascii=False)
-
-    return pass_at_k

evaluation/general_benchmarks/HumanEval/human_eval/execution.py

@@ -1,817 +0,0 @@
-import contextlib
-import faulthandler
-import gzip
-import io
-import json
-import multiprocessing
-import os
-import platform
-import random
-import signal
-import subprocess
-import tempfile
-import traceback
-from typing import *
-
-java_exec = ""
-node_exec = ""
-tsc_exec = ""
-go_exec = ""
-php_exec = ""
-cs_exec = ""
-
-
-def check_correctness(
-    task_id: str,
-    sample: dict,
-    language_type: str,
-    timeout: float = 3.0,
-    tmp_dir: str = None,
-    completion_id: Optional[int] = None,
-) -> Dict:
-    """
-    Evaluates the functional correctness of a completion by running the test
-    suite provided in the problem.
-    """
-
-    def unsafe_execute(tmp_dir):
-        random_id = random.randint(1, 100000)
-        if "python" in language_type.lower():
-            with create_tempdir():
-
-                # These system calls are needed when cleaning up tempdir.
-                import os
-                import shutil
-
-                rmtree = shutil.rmtree
-                rmdir = os.rmdir
-                chdir = os.chdir
-
-                # Disable functionalities that can make destructive changes to the test.
-                reliability_guard()
-
-                try:
-                    exec_globals = {}
-                    with swallow_io():
-                        with time_limit(timeout):
-                            # WARNING
-                            # This program exists to execute untrusted model-generated code. Although
-                            # it is highly unlikely that model-generated code will do something overtly
-                            # malicious in response to this test suite, model-generated code may act
-                            # destructively due to a lack of model capability or alignment.
-                            # Users are strongly encouraged to sandbox this evaluation suite so that it
-                            # does not perform destructive actions on their host or network.
-                            # Once you have read this disclaimer and taken appropriate precautions,
-                            # uncomment the following line and proceed at your own risk:
-                            exec(sample["test_code"], exec_globals)
-                        result.append("passed")
-                except TimeoutException:
-                    result.append("timed out")
-                except AssertionError as e:
-                    result.append(f"failed: AssertionError")
-                except BaseException as e:
-                    result.append(f"failed: {e}")
-                # print(sample["test_code"])
-                # print(result)
-                # Needed for cleaning up.
-                shutil.rmtree = rmtree
-                os.rmdir = rmdir
-                os.chdir = chdir
-
-        elif "go" in language_type.lower():
-            assert (
-                tmp_dir is not None
-            ), "Go should be evaluated in a dir where necessary module files installed."
-
-            import os
-            import shutil
-
-            if "tmp" not in tmp_dir:
-                tmp_dir = os.path.join(tmp_dir, "tmp")
-            tmp_dir = os.path.join(tmp_dir, f"{task_id.replace('/', '-')}-{random_id}")
-            if not os.path.exists(tmp_dir):
-                os.makedirs(tmp_dir)
-            origin_path = os.getcwd()
-            os.chdir(tmp_dir)
-            open(f"main_test.go", "w").write(sample["test_code"])
-            try:
-                exec_result = None
-                with time_limit(timeout):
-                    # WARNING
-                    # This program exists to execute untrusted model-generated code. Although
-                    # it is highly unlikely that model-generated code will do something overtly
-                    # malicious in response to this test suite, model-generated code may act
-                    # destructively due to a lack of model capability or alignment.
-                    # Users are strongly encouraged to sandbox this evaluation suite so that it
-                    # does not perform destructive actions on their host or network.
-                    # Once you have read this disclaimer and taken appropriate precautions,
-                    # uncomment the following line and proceed at your own risk:
-                    exec_result = subprocess.run(
-                        [
-                            f"{go_exec}go",
-                            "test",
-                            f"-timeout={timeout}s",
-                            "main_test.go",
-                        ],
-                        timeout=timeout,
-                        capture_output=True,
-                    )
-
-                if exec_result.returncode == 0:
-                    result.append("passed")
-                else:
-                    if exec_result.stderr:
-                        try:
-                            err = exec_result.stderr.decode()
-                        except:
-                            err = exec_result.stderr
-                    else:
-                        try:
-                            err = exec_result.stdout.decode()
-                        except:
-                            err = exec_result.stdout
-                    result.append(f"failed: {err}")
-
-            except TimeoutException:
-                result.append("timed out")
-            os.chdir(origin_path)
-            shutil.rmtree(tmp_dir)
-        elif "js" in language_type.lower():
-            import os
-            import shutil
-
-            if "tmp" not in tmp_dir:
-                tmp_dir = os.path.join(tmp_dir, "tmp")
-            tmp_dir = os.path.join(tmp_dir, f"{task_id.replace('/', '-')}-{random_id}")
-            if not os.path.exists(tmp_dir):
-                os.makedirs(tmp_dir)
-            origin_path = os.getcwd()
-            os.chdir(tmp_dir)
-            open(f"test.js", "w").write(sample["test_code"])
-            try:
-                exec_result = None
-                with time_limit(timeout):
-                    # WARNING
-                    # This program exists to execute untrusted model-generated code. Although
-                    # it is highly unlikely that model-generated code will do something overtly
-                    # malicious in response to this test suite, model-generated code may act
-                    # destructively due to a lack of model capability or alignment.
-                    # Users are strongly encouraged to sandbox this evaluation suite so that it
-                    # does not perform destructive actions on their host or network.
-                    # Once you have read this disclaimer and taken appropriate precautions,
-                    # uncomment the following line and proceed at your own risk:
-                    exec_result = subprocess.run(
-                        [f"{node_exec}node", "test.js"],
-                        timeout=timeout,
-                        capture_output=True,
-                    )
-
-                if exec_result.stderr.decode():
-                    err = exec_result.stderr.decode()
-                    result.append(f"failed: {err}")
-                elif exec_result.stdout.decode():
-                    err = exec_result.stdout.decode()
-                    result.append(f"failed: {err}")
-                else:
-                    result.append("passed")
-
-            except TimeoutException:
-                result.append("timed out")
-            os.chdir(origin_path)
-            shutil.rmtree(tmp_dir)
-        elif "cpp" in language_type.lower():
-            import os
-            import shutil
-
-            origin_path = os.getcwd()
-            if "tmp" not in tmp_dir:
-                tmp_dir = os.path.join(tmp_dir, "tmp")
-            tmp_dir = os.path.join(tmp_dir, f"{task_id.replace('/', '-')}-{random_id}")
-            if not os.path.exists(tmp_dir):
-                os.makedirs(tmp_dir)
-
-            os.chdir(tmp_dir)
-            open(f"test.cpp", "w").write(sample["test_code"])
-            if "162" in task_id:
-                compilation_result = subprocess.run(
-                    ["/usr/bin/g++", "-std=c++17", "test.cpp", "-lcrypto", "-lssl"],
-                    timeout=timeout,
-                    capture_output=True,
-                )
-            else:
-                compilation_result = subprocess.run(
-                    ["/usr/bin/g++", "-std=c++17", "test.cpp"],
-                    timeout=timeout,
-                    capture_output=True,
-                )
-            if compilation_result.returncode != 0:
-                if compilation_result.stderr:
-                    err = compilation_result.stderr.decode()
-                else:
-                    err = compilation_result.stdout.decode()
-                result.append(f"failed: compilation error: {err}")
-            else:
-                try:
-                    exec_result = None
-                    with time_limit(timeout):
-                        # WARNING
-                        # This program exists to execute untrusted model-generated code. Although
-                        # it is highly unlikely that model-generated code will do something overtly
-                        # malicious in response to this test suite, model-generated code may act
-                        # destructively due to a lack of model capability or alignment.
-                        # Users are strongly encouraged to sandbox this evaluation suite so that it
-                        # does not perform destructive actions on their host or network.
-                        # Once you have read this disclaimer and taken appropriate precautions,
-                        # uncomment the following line and proceed at your own risk:
-                        exec_result = subprocess.run(
-                            ["./a.out"], timeout=timeout, capture_output=True
-                        )
-
-                    if exec_result.returncode == 0:
-                        result.append("passed")
-                    else:
-                        if exec_result.stderr:
-                            try:
-                                err = exec_result.stderr.decode()
-                            except:
-                                err = exec_result.stderr
-                        else:
-                            try:
-                                err = exec_result.stdout.decode()
-                            except:
-                                err = exec_result.stdout
-                        result.append(f"failed: {err}")
-                except TimeoutException:
-                    result.append("timed out")
-            # print(result[-1])
-            # print(sample["test_code"])
-            os.chdir(origin_path)
-            shutil.rmtree(tmp_dir)
-        elif "php" in language_type.lower():
-            import os
-            import shutil
-
-            origin_path = os.getcwd()
-            if "tmp" not in tmp_dir:
-                tmp_dir = os.path.join(tmp_dir, "tmp")
-            tmp_dir = os.path.join(tmp_dir, f"{task_id.replace('/', '-')}-{random_id}")
-            if not os.path.exists(tmp_dir):
-                os.makedirs(tmp_dir)
-
-            os.chdir(tmp_dir)
-            open(f"test.php", "w").write(sample["test_code"])
-            try:
-                exec_result = None
-                with time_limit(timeout):
-                    cmd = f"{php_exec}php -f test.php"
-                    exec_result = subprocess.run(
-                        cmd, timeout=timeout, capture_output=True, shell=True
-                    )
-
-                if exec_result.returncode == 0:
-                    result.append("passed")
-                else:
-                    if exec_result.stderr:
-                        try:
-                            err = exec_result.stderr.decode()
-                        except:
-                            err = exec_result.stderr
-                    else:
-                        try:
-                            err = exec_result.stdout.decode()
-                        except:
-                            err = exec_result.stdout
-                    result.append(f"failed: {err}")
-            except TimeoutException:
-                result.append("timed out")
-            print(result[-1])
-            print(sample["test_code"])
-            os.chdir(origin_path)
-            shutil.rmtree(tmp_dir)
-        elif "sh" in language_type.lower():
-            import os
-            import shutil
-
-            origin_path = os.getcwd()
-            if "tmp" not in tmp_dir:
-                tmp_dir = os.path.join(tmp_dir, "tmp")
-            tmp_dir = os.path.join(tmp_dir, f"{task_id.replace('/', '-')}-{random_id}")
-            if not os.path.exists(tmp_dir):
-                os.makedirs(tmp_dir)
-
-            os.chdir(tmp_dir)
-            open(f"test.sh", "w").write(sample["test_code"])
-            try:
-                exec_result = None
-                with time_limit(timeout):
-                    cmd = "/bin/bash test.sh"
-                    exec_result = subprocess.run(
-                        cmd, timeout=10, capture_output=True, shell=True
-                    )
-
-                if exec_result.returncode == 0:
-                    result.append("passed")
-                else:
-                    if exec_result.stderr:
-                        try:
-                            err = exec_result.stderr.decode()
-                        except:
-                            err = exec_result.stderr
-                    else:
-                        try:
-                            err = exec_result.stdout.decode()
-                        except:
-                            err = exec_result.stdout
-                    result.append(f"failed: {err}")
-            except TimeoutException:
-                result.append("timed out")
-            # print(result[-1])
-            # print(sample["test_code"])
-            os.chdir(origin_path)
-            shutil.rmtree(tmp_dir)
-        elif "ts" in language_type.lower():
-            import os
-            import shutil
-
-            origin_path = os.getcwd()
-            if "tmp" not in tmp_dir:
-                tmp_dir = os.path.join(tmp_dir, "tmp")
-            tmp_dir = os.path.join(tmp_dir, f"{task_id.replace('/', '-')}-{random_id}")
-            if not os.path.exists(tmp_dir):
-                os.makedirs(tmp_dir)
-
-            os.chdir(tmp_dir)
-            env = {"PATH": f"{node_exec}:" + subprocess.os.environ["PATH"]}
-            open(f"test.ts", "w").write(sample["test_code"])
-            cmd = f"{tsc_exec}tsc test.ts --target ES2015 --lib ES2015,DOM"
-            compilation_result = subprocess.run(
-                cmd, timeout=timeout, capture_output=True, env=env, shell=True
-            )
-            if compilation_result.returncode != 0:
-                if compilation_result.stderr:
-                    err = compilation_result.stderr.decode()
-                else:
-                    err = compilation_result.stdout.decode()
-                result.append(f"failed: compilation error: {err}")
-            else:
-                try:
-                    exec_result = None
-                    with time_limit(timeout):
-                        exec_result = subprocess.run(
-                            [f"{node_exec}node", "test.js"],
-                            timeout=timeout,
-                            capture_output=True,
-                        )
-
-                    if exec_result.returncode == 0:
-                        result.append("passed")
-                    else:
-                        if exec_result.stderr:
-                            try:
-                                err = exec_result.stderr.decode()
-                            except:
-                                err = exec_result.stderr
-                        else:
-                            try:
-                                err = exec_result.stdout.decode()
-                            except:
-                                err = exec_result.stdout
-                        result.append(f"failed: {err}")
-                except TimeoutException:
-                    result.append("timed out")
-            if result[-1] != "passed":
-                env = {"PATH": f"{node_exec}:" + subprocess.os.environ["PATH"]}
-                cmd = f"{tsc_exec}tsc test.ts"
-                compilation_result = subprocess.run(
-                    cmd, timeout=timeout, capture_output=True, env=env, shell=True
-                )
-                if compilation_result.returncode != 0:
-                    if compilation_result.stderr:
-                        err = compilation_result.stderr.decode()
-                    else:
-                        err = compilation_result.stdout.decode()
-                    result[-1] = f"failed: compilation error: {err}"
-                else:
-                    try:
-                        exec_result = None
-                        with time_limit(timeout):
-                            exec_result = subprocess.run(
-                                [f"{node_exec}node", "test.js"],
-                                timeout=timeout,
-                                capture_output=True,
-                            )
-
-                        if exec_result.returncode == 0:
-                            result[-1] = "passed"
-                        else:
-                            if exec_result.stderr:
-                                try:
-                                    err = exec_result.stderr.decode()
-                                except:
-                                    err = exec_result.stderr
-                            else:
-                                try:
-                                    err = exec_result.stdout.decode()
-                                except:
-                                    err = exec_result.stdout
-                            result[-1] = f"failed: {err}"
-                    except TimeoutException:
-                        result[-1] = "timed out"
-
-            os.chdir(origin_path)
-            shutil.rmtree(tmp_dir)
-        elif "cs" in language_type.lower():
-            import os
-            import shutil
-
-            origin_path = os.getcwd()
-            if "tmp" not in tmp_dir:
-                tmp_dir = os.path.join(tmp_dir, "tmp")
-            tmp_dir = os.path.join(tmp_dir, f"{task_id.replace('/', '-')}-{random_id}")
-            if not os.path.exists(tmp_dir):
-                os.makedirs(tmp_dir)
-            os.chdir(tmp_dir)
-            open(f"Program.cs", "w").write(sample["test_code"])
-            cmd = f"{cs_exec}mcs -d:DEBUG Program.cs"
-            compilation_result = subprocess.run(cmd, shell=True, capture_output=True)
-            if compilation_result.returncode != 0:
-                if compilation_result.stderr:
-                    err = compilation_result.stderr.decode()
-                else:
-                    err = compilation_result.stdout.decode()
-                result.append(f"failed: compilation error: {err}")
-            else:
-                try:
-                    exec_result = None
-                    cmd = f"{cs_exec}mono Program.exe"
-                    env = dict(MONO_TRACE_LISTENER="Console.Error")
-                    with time_limit(timeout):
-                        exec_result = subprocess.run(
-                            cmd,
-                            timeout=timeout,
-                            shell=True,
-                            capture_output=True,
-                            env=env,
-                        )
-
-                    if "Fail" not in exec_result.stderr.decode():
-                        result.append("passed")
-                    else:
-                        if exec_result.stderr:
-                            try:
-                                err = exec_result.stderr.decode()
-                            except:
-                                err = exec_result.stderr
-                        else:
-                            try:
-                                err = exec_result.stdout.decode()
-                            except:
-                                err = exec_result.stdout
-                        result.append(f"failed: {err}")
-                except TimeoutException:
-                    result.append("timed out")
-                except Exception as e:
-                    result.append(f"failed: {e}")
-            os.chdir(origin_path)
-            shutil.rmtree(tmp_dir)
-        elif "rust" in language_type.lower():
-            import os
-
-            WD: str = os.path.dirname(os.path.abspath(__file__))
-            RUST_DIR: str = os.path.join(WD, "rust")
-            RUST_SRC: str = os.path.join(RUST_DIR, "src")
-            RUST_BIN: str = os.path.join(RUST_SRC, "bin")
-            RUST_TMP_DIR: str = os.path.join(RUST_DIR, "tmp")
-            RUST_LOGS: str = os.path.join(RUST_TMP_DIR, "logs")
-            RUST_EXT: str = ".rs"
-
-            # Create mandatory tmp directories
-            os.makedirs(RUST_TMP_DIR, exist_ok=True)
-            os.makedirs(RUST_LOGS, exist_ok=True)
-            os.makedirs(RUST_SRC, exist_ok=True)
-            os.makedirs(RUST_BIN, exist_ok=True)
-
-            with tempfile.NamedTemporaryFile(dir=RUST_BIN, delete=False) as f:
-                # temporal file name
-                file_prefix = sample["task_id"].lower().replace("/", "_")
-                file_name: str = file_prefix + RUST_EXT
-
-                os.rename(f.name, os.path.join(RUST_BIN, file_name))
-
-                # Sample to pure Rust function
-                rust_code: str = sample["test_code"]
-
-                # dump the rust source code in the target temporal file
-                f.write(rust_code.encode("utf-8"))
-
-            # Proceed towards Rust binaries compilation. Therefore move to Rust module root dir.
-            os.chdir(RUST_DIR)
-
-            # Two possible outcomes
-            # Pass OR Fail compilation
-            log_filename: str = file_prefix + ".jsonl"
-            log_path: str = os.path.join(RUST_LOGS, log_filename)
-            cargo_check: str = (
-                "cargo check --bin "
-                + file_prefix
-                + " --message-format json >> "
-                + log_path
-            )
-            # Compilation build status
-            returned_val_compilation: int
-
-            # Overwrite file content
-            if os.path.exists(log_path):
-                if (file_size := os.path.getsize(log_path)) >= 0:
-                    os.remove(log_path)
-                    returned_val_compilation = os.system(cargo_check)
-
-            else:
-                returned_val_compilation = os.system(cargo_check)
-
-            # 0 means success
-            if returned_val_compilation == 0:
-
-                # Execution pipeline
-                cargo_test: str = (
-                    "cargo test --bin "
-                    + file_prefix
-                    + " --message-format json >> "
-                    + log_path
-                )
-                returned_val_execution = os.system(cargo_test)
-
-                if returned_val_execution == 0:
-                    result.append("passed")
-                else:
-                    result.append(f"failed: execution error")
-
-            else:
-                result.append(f"failed: compilation error")
-
-        elif "java" in language_type.lower():
-            assert tmp_dir is not None, "Java should be evaluated in a temporary dir."
-
-            import os
-            import shutil
-
-            if "tmp" not in tmp_dir:
-                tmp_dir = os.path.join(tmp_dir, "tmp")
-            tmp_dir = os.path.join(tmp_dir, f"{task_id.replace('/', '-')}-{random_id}")
-            if not os.path.exists(tmp_dir):
-                os.makedirs(tmp_dir)
-            open(os.path.join(tmp_dir, "Problem.java"), "w").write(sample["test_code"])
-            origin_path = os.getcwd()
-            os.system(f"cp ./javatuples-1.2.jar {tmp_dir}/")
-            os.chdir(tmp_dir)
-            res = "failed: unknown error"
-            compile_returncode = -1
-            for _ in range(5):
-                try:
-                    cmd = f"{java_exec}javac -cp javatuples-1.2.jar Problem.java"
-                    compilation_result = subprocess.run(
-                        cmd, timeout=60, capture_output=True, shell=True
-                    )
-                    compile_returncode = compilation_result.returncode
-                    break
-                except subprocess.TimeoutExpired as e:
-                    continue
-            if compile_returncode != 0:
-                res = "failed: compilation error"
-            else:
-                exec_result = None
-                try:
-                    # WARNING
-                    # This program exists to execute untrusted model-generated code. Although
-                    # it is highly unlikely that model-generated code will do something overtly
-                    # malicious in response to this test suite, model-generated code may act
-                    # destructively due to a lack of model capability or alignment.
-                    # Users are strongly encouraged to sandbox this evaluation suite so that it
-                    # does not perform destructive actions on their host or network.
-                    # Once you have read this disclaimer and taken appropriate precautions,
-                    # uncomment the following line and proceed at your own risk:
-                    cmd = f"{java_exec}java -ea -cp .:javatuples-1.2.jar Problem"
-                    exec_result = subprocess.run(
-                        cmd, timeout=timeout, capture_output=True, shell=True
-                    )
-                    if exec_result.returncode == 0:
-                        res = "passed"
-                    elif exec_result.returncode == 1:
-                        if "AssertionError" in exec_result.stderr.decode(
-                            "unicode-escape"
-                        ):
-                            res = "failed: wrong answer"
-                        else:
-                            res = f"failed: {exec_result.stderr.decode()}"
-                except subprocess.TimeoutExpired as e:
-                    res = "time out"
-                except BaseException as e:
-                    res = f"failed: {e}"
-
-            result.append(res)
-            os.chdir(origin_path)
-            shutil.rmtree(tmp_dir)
-
-    manager = multiprocessing.Manager()
-    result = manager.list()
-
-    p = multiprocessing.Process(target=unsafe_execute, args=(tmp_dir,))
-    p.start()
-    p.join(timeout=timeout + 1)
-    if p.is_alive():
-        p.kill()
-
-    if not result:
-        result.append("timed out")
-
-    return {
-        "task_id": task_id,
-        "completion_id": completion_id,
-        "result": result[0],
-        "passed": result[0] == "passed",
-        "finish": -1 if "finish" not in sample else sample["finish"],
-        "test_code": sample["test_code"],
-        "prompt": sample["prompt"],
-        # "canonical_solution"         : sample["canonical_solution"],
-        #     "test"         : sample["test"],
-        #     "text"         : sample["text"],
-        #     "output"         : sample["output"],
-        #     "generation"         : sample["generation"],
-    }
-
-
-# Copyright (c) OpenAI (https://openai.com)
-
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-
-
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-# ============================================================================
-@contextlib.contextmanager
-def time_limit(seconds: float):
-    def signal_handler(signum, frame):
-        raise TimeoutException("Timed out!")
-
-    signal.setitimer(signal.ITIMER_REAL, seconds)
-    signal.signal(signal.SIGALRM, signal_handler)
-    try:
-        yield
-    finally:
-        signal.setitimer(signal.ITIMER_REAL, 0)
-
-
-@contextlib.contextmanager
-def swallow_io():
-    stream = WriteOnlyStringIO()
-    with contextlib.redirect_stdout(stream):
-        with contextlib.redirect_stderr(stream):
-            with redirect_stdin(stream):
-                yield
-
-
-@contextlib.contextmanager
-def create_tempdir():
-    with tempfile.TemporaryDirectory() as dirname:
-        with chdir(dirname):
-            yield dirname
-
-
-class TimeoutException(Exception):
-    pass
-
-
-class WriteOnlyStringIO(io.StringIO):
-    """StringIO that throws an exception when it's read from"""
-
-    def read(self, *args, **kwargs):
-        raise IOError
-
-    def readline(self, *args, **kwargs):
-        raise IOError
-
-    def readlines(self, *args, **kwargs):
-        raise IOError
-
-    def readable(self, *args, **kwargs):
-        """Returns True if the IO object can be read."""
-        return False
-
-
-class redirect_stdin(contextlib._RedirectStream):  # type: ignore
-    _stream = "stdin"
-
-
-@contextlib.contextmanager
-def chdir(root):
-    if root == ".":
-        yield
-        return
-    cwd = os.getcwd()
-    os.chdir(root)
-    try:
-        yield
-    except BaseException as exc:
-        raise exc
-    finally:
-        os.chdir(cwd)
-
-
-def reliability_guard(maximum_memory_bytes: Optional[int] = None):
-    """
-    This disables various destructive functions and prevents the generated code
-    from interfering with the test (e.g. fork bomb, killing other processes,
-    removing filesystem files, etc.)
-
-    WARNING
-    This function is NOT a security sandbox. Untrusted code, including, model-
-    generated code, should not be blindly executed outside of one. See the
-    Codex paper for more information about OpenAI's code sandbox, and proceed
-    with caution.
-    """
-
-    if maximum_memory_bytes is not None:
-        import resource
-
-        resource.setrlimit(
-            resource.RLIMIT_AS, (maximum_memory_bytes, maximum_memory_bytes)
-        )
-        resource.setrlimit(
-            resource.RLIMIT_DATA, (maximum_memory_bytes, maximum_memory_bytes)
-        )
-        if not platform.uname().system == "Darwin":
-            resource.setrlimit(
-                resource.RLIMIT_STACK, (maximum_memory_bytes, maximum_memory_bytes)
-            )
-
-    faulthandler.disable()
-
-    import builtins
-
-    builtins.exit = None
-    builtins.quit = None
-
-    import os
-
-    os.environ["OMP_NUM_THREADS"] = "1"
-
-    os.kill = None
-    os.system = None
-    os.putenv = None
-    os.remove = None
-    os.removedirs = None
-    os.rmdir = None
-    os.fchdir = None
-    os.setuid = None
-    os.fork = None
-    os.forkpty = None
-    os.killpg = None
-    os.rename = None
-    os.renames = None
-    os.truncate = None
-    os.replace = None
-    os.unlink = None
-    os.fchmod = None
-    os.fchown = None
-    os.chmod = None
-    os.chown = None
-    os.chroot = None
-    os.fchdir = None
-    os.lchflags = None
-    os.lchmod = None
-    os.lchown = None
-    os.getcwd = None
-    os.chdir = None
-
-    import shutil
-
-    shutil.rmtree = None
-    shutil.move = None
-    shutil.chown = None
-
-    import subprocess
-
-    subprocess.Popen = None  # type: ignore
-
-    __builtins__["help"] = None
-
-    import sys
-
-    sys.modules["ipdb"] = None
-    sys.modules["joblib"] = None
-    sys.modules["resource"] = None
-    sys.modules["psutil"] = None
-    sys.modules["tkinter"] = None

evaluation/general_benchmarks/HumanEval/humaneval.py

@@ -1,217 +0,0 @@
-import datetime
-import json
-import multiprocessing
-import os
-import re
-import string
-import subprocess
-import time
-
-import numpy as np
-import torch
-import torch.distributed as dist
-# from attrdict import AttrDict
-from human_eval.evaluation import evaluate_functional_correctness
-from transformers import AutoTokenizer
-from utils.dataset import HumanEvalDataset
-from utils.utils import cleanup_code
-
-
-class HumanEval:
-    """
-    HumanEval evaluation class.
-    """
-
-    def __init__(
-        self,
-        data_root,
-        max_seq_len=2048,
-        language="python",
-        max_gen_len=200,
-        batch_size=512,
-        log_dir=None,
-        temperature=0,
-        issft=False,
-        top_p=0.95,
-        model_name="",
-        inference_increment=True,
-        tokenizer_cfg=None,
-        n_sample=40,
-        k_sample=1,
-    ):
-        self.data_root = data_root
-        self.max_seq_len = max_seq_len
-        self.max_gen_len = max_gen_len
-        self.batch_size = batch_size
-        self.k = k_sample
-        self.n_sample = n_sample
-        self.language = language
-        self.log_dir = log_dir
-        self.sft = issft
-        self.temperature = temperature
-        self.top_p = top_p
-        self.model_name = tokenizer_cfg["model_path"].replace("/", "_")
-        self.inference_increment = inference_increment
-        os.makedirs(self.log_dir, exist_ok=True)
-        tokenizer_cls = tokenizer_cfg.pop("cls")
-        try:
-            self.tokenizer = AutoTokenizer.from_pretrained(
-                tokenizer_cfg.pop("model_path"), trust_remote_code=True
-            )
-        except Exception as e:
-            print(e)
-            assert False
-
-    @torch.no_grad()
-    def eval_model(self, gpt, accelerator):
-        """
-        Evaluate the model on HumanEval.
-        """
-        assert (
-            self.log_dir is not None
-        ), "log_dir should not be None when evaluating humaneval"
-        dataset = HumanEvalDataset(
-            self.data_root,
-            sample_num=self.n_sample,
-            language=self.language,
-            issft=self.sft,
-        )
-        nprompt = len(dataset) // self.n_sample
-        dp_rank = accelerator.process_index
-        dp_size = accelerator.num_processes
-        if self.k > 1:
-            assert self.n_sample >= 100, "HumanEval PASS@100 needs n_sample >= 100"
-        gpt.eval()
-        # each process will process a subset of the dataset
-        prompt_indices_split = np.array_split(range(nprompt), dp_size)
-        prompt_indices = prompt_indices_split[dp_rank]
-        indices = [
-            x * self.n_sample + j for x in prompt_indices for j in range(self.n_sample)
-        ]
-        all_num = len(indices)
-        processed_num = 0
-        log_file = os.path.join(
-            self.log_dir,
-            f"{self.model_name}_rank{dp_rank}_bs{self.batch_size}_shot_log_{self.language}.json",
-        )
-        tmpfile = open(log_file, "w")
-        start_time = time.time()
-        # split the dataset into batches and construct a list of inputs
-        for idx in range(0, len(indices), self.batch_size):
-            prompt_list = []
-            prompt_lens = []
-            orriginal_prompt_list = []
-            tokenized_prompt_lens = []
-            taskid = []
-            # get the prompts from the dataset
-            for j in indices[idx : idx + self.batch_size]:
-                data = dataset[j]
-                fprompt = data["prompt"].strip()
-                prompt_list.append(fprompt)
-                tmp = self.tokenizer.encode(fprompt)
-                orriginal_prompt_list.append(data["original_prompt"])
-                prompt_lens.append(len(fprompt))
-                tokenized_prompt_lens.append(tmp)
-                taskid.append(data["task_id"])
-            input_ids = torch.tensor(tokenized_prompt_lens).to(accelerator.device)
-            # generate the code
-            if self.temperature != 0:
-                decoded = gpt.generate(
-                    input_ids=input_ids,
-                    max_new_tokens=self.max_gen_len,
-                    do_sample=True,
-                    eos_token_id=self.tokenizer.eos_token_id,
-                    temperature=self.temperature,
-                    top_p=self.top_p,
-                    pad_token_id=self.tokenizer.eos_token_id,
-                )
-            else:
-                decoded = gpt.generate(
-                    input_ids=input_ids,
-                    max_new_tokens=self.max_gen_len,
-                    do_sample=False,
-                    eos_token_id=self.tokenizer.eos_token_id,
-                    pad_token_id=self.tokenizer.eos_token_id,
-                )
-            # save the results to a file
-            for local_idx, text in enumerate(decoded):
-                prediction = decoded[local_idx]
-                prediction = self.tokenizer.decode(prediction, skip_special_tokens=True)
-                suffixprediction = prediction[prompt_lens[local_idx] :]
-                suffixprediction = cleanup_code(
-                    suffixprediction,
-                    self.language,
-                    "humaneval",
-                    self.sft,
-                    dataset.stopwords,
-                )
-                # sft mode does not need original prompt
-                if not self.sft:
-                    suffixprediction = (
-                        orriginal_prompt_list[local_idx] + "\n" + suffixprediction
-                    )
-                res = {
-                    "task_id": taskid[local_idx],
-                    "generation": suffixprediction,
-                    "prompt": orriginal_prompt_list[local_idx],
-                    "wholecode": prediction,
-                }
-                tmpfile.write(json.dumps(res) + "\n")
-                tmpfile.flush()
-                processed_num += 1
-            self.log_score(dp_rank, processed_num, all_num, start_time, self.batch_size)
-        tmpfile.close()
-        accelerator.wait_for_everyone()
-        # calculate the final score of pass@k
-        self._calculate_final_score(accelerator)
-        accelerator.wait_for_everyone()
-        return
-
-    def log_score(self, dp_rank, processed_num, all_num, start_time, bs):
-        """
-        Log the score.
-        """
-        mem = torch.cuda.max_memory_allocated() / (1 << 30)
-        avg_time = (time.time() - start_time) / processed_num * bs
-        print(
-            f"DP RANK:{dp_rank} process_num/all_num:{int(processed_num)}/{all_num} "
-            f"avg_time_per_batch:{avg_time:.2f} s "
-            f"still_need:{((all_num - processed_num) // bs + 1) * avg_time / 60:.2f} m",
-            f"mem:{mem:.3f} GiB bs:{bs}",
-            flush=True,
-        )
-        if processed_num == all_num:
-            print(
-                f"EVAL DONE! Process time {(time.time() - start_time) / 60:.2f} m",
-                flush=True,
-            )
-
-    def _calculate_final_score(self, accelerator):
-        """
-        Calculate the final score.
-        """
-        if accelerator.is_local_main_process:
-            logfilepath = os.path.join(self.log_dir, f"final_{self.model_name}.jsonl")
-            logfile = open(logfilepath, "w")
-            for i in range(accelerator.num_processes):
-                tmplogfile = os.path.join(
-                    self.log_dir,
-                    f"{self.model_name}_rank{i}_bs{self.batch_size}_shot_log_{self.language}.json",
-                )
-                logfile.write(open(tmplogfile).read().strip() + "\n")
-                os.remove(tmplogfile)
-            logfile.close()
-            timeout = 10
-            runlang = self.language
-            res = evaluate_functional_correctness(
-                input_file=logfilepath,
-                problem_file=os.path.join(
-                    self.data_root, f"humaneval-{self.language}.jsonl"
-                ),
-                tmp_dir=self.log_dir,
-                timeout=timeout,
-                language=runlang,
-            )
-            print("score is", res["pass@%d" % self.k])
-            os.remove(logfilepath)
-        return

evaluation/general_benchmarks/HumanEval/test_config.yaml

@@ -1,15 +0,0 @@
-compute_environment: LOCAL_MACHINE
-distributed_type: MULTI_GPU
-downcast_bf16: 'no'
-gpu_ids: all
-machine_rank: 0
-main_training_function: main
-mixed_precision: 'no'
-num_machines: 1
-num_processes: 3
-rdzv_backend: static
-same_network: true
-tpu_env: []
-tpu_use_cluster: false
-tpu_use_sudo: false
-use_cpu: false

evaluation/general_benchmarks/HumanEval/utils/dataset.py

@@ -1,72 +0,0 @@
-import json
-import os
-
-import numpy as np
-
-
-class HumanEvalDataset:
-
-    def __init__(self, root, sample_num=1, language="python", issft=False):
-        """
-        root: the path to the HumanEval dataset
-        sample_num: the number of samples for each prompt
-        language: the language of the HumanEval dataset
-        issft: whether to use the SFT setting
-        """
-        self.root = root
-        self.data = open(
-            os.path.join(self.root, f"humaneval-{language}.jsonl")
-        ).readlines()
-
-        tmp = self.get_qa_only_data(self.data, issft)
-        self.clean_data = []
-        for i in range(len(tmp)):
-            for j in range(sample_num):
-                self.clean_data.append(tmp[i])
-        self.stopwords = self.clean_data[0]["stopwords"]
-        np.random.seed(1234)
-        print(f"Read HumanEval from {root}, number of samples {len(self.clean_data)}")
-
-    def get_qa_only_data(self, data_json, sft=False):
-        """
-        data_json: the jsonl file of HumanEval
-        sft: whether to use the SFT setting
-        return: a list of dict, each dict contains the prompt, task_id and stopwords
-        """
-        ans = []
-        for line in data_json:
-            line = json.loads(line)
-            prompt = line["prompt"].strip()
-            if "prefix" in line:
-                origin_prompt = line["prefix"]
-            else:
-                origin_prompt = line["prompt"]
-
-            if sft:
-                prompt = f"""Below is an instruction that describes a task, paired with an input that provides further context.\nWrite a response that appropriately completes the request.\n\n### Instruction:\nWrite a program to perform the given task.\n\nInput:\n{prompt}\n\n### Response:\n"""
-            if "stop_tokens" in line:
-                s = line["stop_tokens"]
-            else:
-                s = []
-            ans.append(
-                {
-                    "prompt": prompt,
-                    "task_id": line["task_id"],
-                    "original_prompt": origin_prompt,
-                    "stopwords": s,
-                }
-            )
-        return ans
-
-    def __len__(self):
-        """
-        return the number of samples in the dataset
-        """
-        return len(self.clean_data)
-
-    def __getitem__(self, index):
-        """
-        return the sample at index
-        """
-        sample = self.clean_data[index]
-        return sample

evaluation/general_benchmarks/HumanEval/utils/utils.py

@@ -1,161 +0,0 @@
-import re
-
-languge_settings = {
-    "python": {
-        "full_name": "Python",
-        "indent": 4,
-    },
-    "cpp": {
-        "full_name": "cpp",
-        "indent": 0,
-        "main": "int main()",
-    },
-    "java": {
-        "full_name": "Java",
-        "indent": 4,
-        "main": "public static void main",
-    },
-    "cs": {
-        "full_name": "csharp",
-        "indent": 0,
-        "main": "public static void Main",
-    },
-    "php": {
-        "full_name": "PHP",
-        "indent": 0,
-    },
-    "ts": {
-        "full_name": "TypeScript",
-        "indent": 0,
-    },
-    "js": {"full_name": "JavaScript", "indent": 0},
-    "sh": {"full_name": "Bash", "indent": 0},
-}
-
-
-def get_function_name(question: str, lang: str):
-    func_lines = [x for x in question.strip().split("\n") if x.strip()]
-
-    if lang.lower() == "python":
-        func_idx = [
-            i for i in range(len(func_lines)) if func_lines[i].startswith("def ")
-        ][-1]
-        func_name = func_lines[func_idx].split("(")[0].strip()
-        func_prefix = "\n".join(func_lines[:func_idx])
-        return func_name, func_prefix
-
-    func_name = func_lines[-1].split("{")[0].strip()
-    func_prefix = "\n".join(func_lines[:-1])
-    return func_name, func_prefix
-
-
-def extract_generation_code(example: str, lang_code: str, verbose: bool = False):
-    task_id = example["task_id"]
-    output = example.get("output", example.get("gpt_completion"))
-    question = example["prompt"].strip()
-    setting = languge_settings[lang_code]
-    lang = setting["full_name"]
-    indent = setting["indent"]
-
-    try:
-        code_block: str = re.findall(
-            f"```{lang.lower()}\n(.*?)```", output, re.DOTALL | re.IGNORECASE
-        )[0]
-        if verbose:
-            print(">>> Task: {}\n{}".format(task_id, code_block))
-
-        # Remove main
-        if setting.get("main", None) and setting["main"] in code_block:
-            main_start = code_block.index(setting["main"])
-            code_block = code_block[:main_start]
-
-        func_name, func_prefix = get_function_name(question, lang)
-
-        try:
-            start = code_block.lower().index(func_name.lower())
-            indent = 0
-            while start - indent >= 0 and code_block[start - indent - 1] == " ":
-                indent += 1
-
-            try:
-                end = code_block.rindex("\n" + " " * indent + "}")
-            except:
-                end = len(code_block)
-        except:
-            start = 0
-            try:
-                end = code_block.rindex("\n" + " " * indent + "}")
-            except:
-                end = len(code_block)
-
-        body = code_block[start:end]
-
-        if lang_code.lower() in ["php", "ts", "js"]:
-            body += "\n" + " " * indent + "}"
-
-        generation = func_prefix + "\n" + body + "\n"
-        example["generation"] = generation
-
-    except Exception as ex:
-        print(
-            "Failed to extract code block with error `{}`:\n>>> Task: {}\n>>> Output:\n{}".format(
-                ex, task_id, output
-            )
-        )
-        example["generation"] = example["prompt"] + "\n" + output
-
-    return example
-
-
-def cleanup_code(
-    code: str,
-    language_type: str = None,
-    dataset: str = None,
-    issft: bool = False,
-    stop_words=[],
-):
-    """
-    Cleans up the generated code.
-    """
-
-    if language_type.lower() == "python":
-        if issft:
-            code = _clean_python_code_for_sft(code)
-        stop_words = ["\ndef", "\nclass", "\nif", "\n#", "\nprint"]
-        code = _truncate_code_at_stopwords(code, stop_words)
-    elif language_type.lower() == "ts":
-        code = _truncate_code_at_stopwords(
-            code,
-            stop_words
-            + [
-                "\nexport",
-                "\nimport",
-                "\nexport default",
-                "\nimport default",
-                "\nconsole.log",
-            ],
-        )
-    else:
-        code = _truncate_code_at_stopwords(code, stop_words)
-
-    return code
-
-
-def _clean_python_code_for_sft(code):
-    code = code.replace("\r", "")
-    if "```python" in code:
-        code_start_idx = code.index("```python")
-        code = code[code_start_idx:].replace("```python", "").strip()
-        end_idx = code.find("```") if "```" in code else len(code)
-        code = code[:end_idx].strip()
-
-    return code
-
-
-def _truncate_code_at_stopwords(code, stop_words):
-    min_stop_idx = len(code)
-    for stop_word in stop_words:
-        stop_index = code.find(stop_word)
-        if 0 <= stop_index < min_stop_idx:
-            min_stop_idx = stop_index
-    return code[:min_stop_idx]

evaluation/general_benchmarks/MATH/LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2024 Zhibin Gou
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

evaluation/general_benchmarks/MATH/README.md

@@ -1,52 +0,0 @@
-### Requirements
-You can install the required packages with the following command:
-```bash
-cd latex2sympy
-pip install -e .
-cd ..
-pip install -r requirements.txt 
-pip install vllm==0.5.1 --no-build-isolation
-pip install transformers==4.42.3
-```
-
-### Evaluation
-You can evaluate Qwen2.5/Qwen2-Math-Instruct series model with the following command:
-```bash
-# Qwen2.5-Math-Instruct Series
-PROMPT_TYPE="qwen25-math-cot"
-# Qwen2.5-Math-1.5B-Instruct
-export CUDA_VISIBLE_DEVICES="0"
-MODEL_NAME_OR_PATH="Qwen/Qwen2.5-Math-1.5B-Instruct"
-bash sh/eval.sh $PROMPT_TYPE $MODEL_NAME_OR_PATH
-
-# Qwen2.5-Math-7B-Instruct
-export CUDA_VISIBLE_DEVICES="0"
-MODEL_NAME_OR_PATH="Qwen/Qwen2.5-Math-7B-Instruct"
-bash sh/eval.sh $PROMPT_TYPE $MODEL_NAME_OR_PATH
-
-# Qwen2.5-Math-72B-Instruct
-export CUDA_VISIBLE_DEVICES="0,1,2,3"
-MODEL_NAME_OR_PATH="Qwen/Qwen2.5-Math-72B-Instruct"
-bash sh/eval.sh $PROMPT_TYPE $MODEL_NAME_OR_PATH
-
-
-# Qwen2-Math-Instruct Series
-PROMPT_TYPE="qwen-boxed"
-# Qwen2-Math-1.5B-Instruct
-export CUDA_VISIBLE_DEVICES="0"
-MODEL_NAME_OR_PATH="Qwen/Qwen2-Math-1.5B-Instruct"
-bash sh/eval.sh $PROMPT_TYPE $MODEL_NAME_OR_PATH
-
-# Qwen2-Math-7B-Instruct
-export CUDA_VISIBLE_DEVICES="0"
-MODEL_NAME_OR_PATH="Qwen/Qwen2-Math-7B-Instruct"
-bash sh/eval.sh $PROMPT_TYPE $MODEL_NAME_OR_PATH
-
-# Qwen2-Math-72B-Instruct
-export CUDA_VISIBLE_DEVICES="0,1,2,3"
-MODEL_NAME_OR_PATH="Qwen/Qwen2-Math-72B-Instruct"
-bash sh/eval.sh $PROMPT_TYPE $MODEL_NAME_OR_PATH
-```
-
-## Acknowledgement
-The codebase is adapted from [math-evaluation-harness](https://github.com/ZubinGou/math-evaluation-harness).

evaluation/general_benchmarks/MATH/data/aime24/test.jsonl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:af2b8bd2aa911b6333ad0df32f3ca05c7ae8ed10f1731f4372c8ae26990bf7ac
-size 156944