huihui-ai's picture
Upload 18 files
0f6478f verified
raw
history blame
1.2 kB
# Install required package
pip install antlr4-python3-runtime==4.11 immutabledict langdetect nltk lm_eval
python -c "import nltk; nltk.download('punkt')"
MODEL_PATHS=(
huihui-ai/Qwen2.5-14B-Instruct-abliterated-v2
)
for MODEL_PATH in "${MODEL_PATHS[@]}"; do
MODEL_NAME=$(basename "$MODEL_PATH")
MODEL_DIR="./results/$MODEL_NAME"
mkdir -p "$MODEL_DIR"
MODEL_ARGS="trust_remote_code=True,pretrained=$MODEL_PATH,dtype=bfloat16"
BASE_COMMAND="accelerate launch -m lm_eval --model hf --model_args $MODEL_ARGS --batch_size 4 --fewshot_as_multiturn --apply_chat_template"
# IFEval
$BASE_COMMAND --tasks leaderboard_ifeval --fewshot_as_multiturn --output_path "$MODEL_DIR/ifeval"
# BBH (Big-Bench Hard)
$BASE_COMMAND --tasks leaderboard_bbh --num_fewshot 3 --fewshot_as_multiturn --output_path "$MODEL_DIR/bbh"
# GPQA
$BASE_COMMAND --tasks leaderboard_gpqa --fewshot_as_multiturn --output_path "$MODEL_DIR/gpqa"
# MMLU-Pro
$BASE_COMMAND --tasks leaderboard_mmlu_pro --num_fewshot 5 --fewshot_as_multiturn --output_path "$MODEL_DIR/mmlu_pro"
# TruthfulQA
$BASE_COMMAND --tasks truthfulqa_mc2 --fewshot_as_multiturn --output_path "$MODEL_DIR/truthfulqa"
done