{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "090b49e7-5cbb-4378-bb96-8cd5f5d0a520",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install -r https://huggingface.co/flunardelli/llm-metaeval/raw/main/requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "106470b4-1be2-4790-a5bf-485a1e3bb2d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "import os\n",
    "from huggingface_hub import login, upload_folder\n",
    "# from google.colab import userdata\n",
    "import shutil\n",
    "\n",
    "#HF_TOKEN = userdata.get('HF_TOKEN')\n",
    "HF_TOKEN = \"HUGGINGFACE_WRITE_TOKEN\"\n",
    "login(HF_TOKEN, True)\n",
    "BASE_DATASET='mmlu'\n",
    "REPO_ID='flunardelli/llm-metaeval'\n",
    "BASE_FOLDER=f\"./{BASE_DATASET}/\"#{datetime.now().strftime('%Y-%m-%dT%H-%M-%S')}\n",
    "OUTPUT_FOLDER=os.path.join(BASE_FOLDER,'output')\n",
    "TASK_FOLDER=os.path.join(BASE_FOLDER,'tasks')\n",
    "#shutil.rmtree(BASE_FOLDER)\n",
    "os.makedirs(OUTPUT_FOLDER)\n",
    "os.makedirs(TASK_FOLDER)\n",
    "os.environ['HF_TOKEN'] = HF_TOKEN\n",
    "os.environ['OUTPUT_FOLDER'] = OUTPUT_FOLDER\n",
    "os.environ['TASK_FOLDER'] = TASK_FOLDER\n",
    "\n",
    "def hf_upload_folder(folder_path):\n",
    "  upload_folder(\n",
    "      folder_path=folder_path,\n",
    "      path_in_repo=\"evals/\",\n",
    "      repo_id=REPO_ID,\n",
    "      token=HF_TOKEN,\n",
    "      repo_type=\"dataset\"\n",
    "  )\n",
    "\n",
    "def create_task(content, filename):\n",
    "  filename_path = os.path.join(TASK_FOLDER,filename)\n",
    "  with open(filename_path, \"w\") as f:\n",
    "    f.write(content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4855d529-22d7-4eaf-b6bd-1cc40b0223a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "YAML_mmlu_en_us_string = \"\"\"\n",
    "task: mmlu_all\n",
    "dataset_path: cais/mmlu\n",
    "dataset_name: all\n",
    "description: \"MMLU dataset\"\n",
    "test_split: test\n",
    "fewshot_split: dev\n",
    "fewshot_config:\n",
    "  sampler: first_n\n",
    "num_fewshot: 5\n",
    "output_type: multiple_choice\n",
    "doc_to_text: \"{{question.strip()}}\\nA. {{choices[0]}}\\nB. {{choices[1]}}\\nC. {{choices[2]}}\\nD. {{choices[3]}}\\nAnswer:\"\n",
    "doc_to_choice: [\"A\", \"B\", \"C\", \"D\"]\n",
    "doc_to_target: answer\n",
    "metric_list:\n",
    "  - metric: acc\n",
    "    aggregation: mean\n",
    "    higher_is_better: true\n",
    "\"\"\"\n",
    "create_task(YAML_mmlu_en_us_string, 'mmlu_en_us.yaml')\n",
    "os.environ['TASKS'] = 'mmlu_all'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf3725e2-3a26-49b7-a5ba-71ecb5ba28bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "!accelerate launch -m lm_eval \\\n",
    "--model hf --model_args pretrained=mistralai/Mixtral-8x7B-Instruct-v0.1,revision=41bd4c9e7e4fb318ca40e721131d4933966c2cc1,trust_remote_code=False,dtype=bfloat16,parallelize=True \\\n",
    "--tasks $TASKS \\\n",
    "--include_path $TASK_FOLDER/. --output $OUTPUT_FOLDER  --log_samples \\\n",
    "--batch_size auto \n",
    "#--limit 10 \\"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "367d4e84-ba2a-42fb-a5a9-b303ab63dea5",
   "metadata": {},
   "outputs": [],
   "source": [
    "hf_upload_folder(BASE_FOLDER)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e27edf4-793a-41f9-b664-2592082a3e58",
   "metadata": {},
   "outputs": [],
   "source": [
    "!accelerate launch -m lm_eval  \\\n",
    "--model hf --model_args pretrained=mistralai/Mixtral-8x22B-v0.1,revision=b03e260818710044a2f088d88fab12bb220884fb,trust_remote_code=False,dtype=bfloat16,parallelize=True \\\n",
    "--tasks $TASKS \\\n",
    "--include_path $TASK_FOLDER/. --output $OUTPUT_FOLDER --log_samples \\\n",
    "--batch_size auto\n",
    "#--limit 10 \\"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2bc08732-dea6-43a4-8d31-c407069a6eef",
   "metadata": {},
   "outputs": [],
   "source": [
    "hf_upload_folder(BASE_FOLDER)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}