notebooks/.ipynb_aml_checkpoints/DAEDRA-checkpoint2024-0-29-14-26-30Z.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# DAEDRA: Determining Adverse Event Disposition for Regulatory Affairs\n",
        "\n",
        "DAEDRA is a language model intended to predict the disposition (outcome) of an adverse event based on the text of the event report. Intended to be used to classify reports in passive reporting systems, it is trained on the [VAERS](https://vaers.hhs.gov/) dataset, which contains reports of adverse events following vaccination in the United States."
      ],
      "metadata": {}
    },
    {
      "cell_type": "code",
      "source": [
        "%pip install accelerate -U"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": "Requirement already satisfied: accelerate in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (0.26.1)\nRequirement already satisfied: torch>=1.10.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from accelerate) (1.12.0)\nRequirement already satisfied: packaging>=20.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from accelerate) (23.1)\nRequirement already satisfied: numpy>=1.17 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from accelerate) (1.23.5)\nRequirement already satisfied: pyyaml in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from accelerate) (6.0)\nRequirement already satisfied: psutil in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from accelerate) (5.9.5)\nRequirement already satisfied: safetensors>=0.3.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from accelerate) (0.4.2)\nRequirement already satisfied: huggingface-hub in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from accelerate) (0.20.3)\nRequirement already satisfied: typing_extensions in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from torch>=1.10.0->accelerate) (4.6.3)\nRequirement already satisfied: tqdm>=4.42.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from huggingface-hub->accelerate) (4.65.0)\nRequirement already satisfied: filelock in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from huggingface-hub->accelerate) (3.13.1)\nRequirement already satisfied: requests in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from huggingface-hub->accelerate) (2.31.0)\nRequirement already satisfied: fsspec>=2023.5.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from huggingface-hub->accelerate) (2023.10.0)\nRequirement already satisfied: idna<4,>=2.5 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from requests->huggingface-hub->accelerate) (3.4)\nRequirement already satisfied: charset-normalizer<4,>=2 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from requests->huggingface-hub->accelerate) (3.1.0)\nRequirement already satisfied: certifi>=2017.4.17 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from requests->huggingface-hub->accelerate) (2023.5.7)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from requests->huggingface-hub->accelerate) (1.26.16)\nNote: you may need to restart the kernel to use updated packages.\n"
        }
      ],
      "execution_count": 1,
      "metadata": {
        "gather": {
          "logged": 1706475754655
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "tags": []
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%pip install transformers datasets shap watermark wandb evaluate codecarbon"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": "Requirement already satisfied: transformers in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (4.37.1)\nRequirement already satisfied: datasets in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (2.16.1)\nRequirement already satisfied: shap in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (0.44.1)\nRequirement already satisfied: watermark in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (2.4.3)\nRequirement already satisfied: wandb in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (0.16.2)\nRequirement already satisfied: evaluate in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (0.4.1)\nRequirement already satisfied: codecarbon in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (2.3.3)\nRequirement already satisfied: safetensors>=0.3.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from transformers) (0.4.2)\nRequirement already satisfied: regex!=2019.12.17 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from transformers) (2023.12.25)\nRequirement already satisfied: filelock in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from transformers) (3.13.1)\nRequirement already satisfied: pyyaml>=5.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from transformers) (6.0)\nRequirement already satisfied: numpy>=1.17 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from transformers) (1.23.5)\nRequirement already satisfied: packaging>=20.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from transformers) (23.1)\nRequirement already satisfied: tqdm>=4.27 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from transformers) (4.65.0)\nRequirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from transformers) (0.20.3)\nRequirement already satisfied: tokenizers<0.19,>=0.14 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from transformers) (0.15.1)\nRequirement already satisfied: requests in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from transformers) (2.31.0)\nRequirement already satisfied: pyarrow>=8.0.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from datasets) (9.0.0)\nRequirement already satisfied: aiohttp in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from datasets) (3.9.1)\nRequirement already satisfied: xxhash in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from datasets) (3.4.1)\nRequirement already satisfied: multiprocess in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from datasets) (0.70.15)\nRequirement already satisfied: pyarrow-hotfix in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from datasets) (0.6)\nRequirement already satisfied: pandas in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from datasets) (2.0.2)\nRequirement already satisfied: dill<0.3.8,>=0.3.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from datasets) (0.3.7)\nRequirement already satisfied: fsspec[http]<=2023.10.0,>=2023.1.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from datasets) (2023.10.0)\nRequirement already satisfied: scipy in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from shap) (1.10.1)\nRequirement already satisfied: slicer==0.0.7 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from shap) (0.0.7)\nRequirement already satisfied: cloudpickle in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from shap) (2.2.1)\nRequirement already satisfied: scikit-learn in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from shap) (1.2.2)\nRequirement already satisfied: numba in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from shap) (0.58.1)\nRequirement already satisfied: importlib-metadata>=1.4 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from watermark) (6.7.0)\nRequirement already satisfied: ipython>=6.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from watermark) (8.12.2)\nRequirement already satisfied: setuptools in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from watermark) (65.6.3)\nRequirement already satisfied: protobuf!=4.21.0,<5,>=3.12.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from wandb) (3.19.6)\nRequirement already satisfied: setproctitle in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from wandb) (1.3.3)\nRequirement already satisfied: psutil>=5.0.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from wandb) (5.9.5)\nRequirement already satisfied: sentry-sdk>=1.0.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from wandb) (1.39.2)\nRequirement already satisfied: typing-extensions in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from wandb) (4.6.3)\nRequirement already satisfied: Click!=8.0.0,>=7.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from wandb) (8.1.3)\nRequirement already satisfied: docker-pycreds>=0.4.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from wandb) (0.4.0)\nRequirement already satisfied: appdirs>=1.4.3 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from wandb) (1.4.4)\nRequirement already satisfied: GitPython!=3.1.29,>=1.0.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from wandb) (3.1.31)\nRequirement already satisfied: responses<0.19 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from evaluate) (0.18.0)\nRequirement already satisfied: pynvml in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from codecarbon) (11.5.0)\nRequirement already satisfied: py-cpuinfo in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from codecarbon) (9.0.0)\nRequirement already satisfied: prometheus-client in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from codecarbon) (0.19.0)\nRequirement already satisfied: arrow in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from codecarbon) (1.3.0)\nRequirement already satisfied: rapidfuzz in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from codecarbon) (3.6.1)\nRequirement already satisfied: six>=1.4.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from docker-pycreds>=0.4.0->wandb) (1.16.0)\nRequirement already satisfied: yarl<2.0,>=1.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from aiohttp->datasets) (1.9.4)\nRequirement already satisfied: async-timeout<5.0,>=4.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from aiohttp->datasets) (4.0.3)\nRequirement already satisfied: aiosignal>=1.1.2 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from aiohttp->datasets) (1.3.1)\nRequirement already satisfied: multidict<7.0,>=4.5 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from aiohttp->datasets) (6.0.4)\nRequirement already satisfied: frozenlist>=1.1.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from aiohttp->datasets) (1.4.1)\nRequirement already satisfied: attrs>=17.3.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from aiohttp->datasets) (23.1.0)\nRequirement already satisfied: gitdb<5,>=4.0.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from GitPython!=3.1.29,>=1.0.0->wandb) (4.0.10)\nRequirement already satisfied: zipp>=0.5 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from importlib-metadata>=1.4->watermark) (3.15.0)\nRequirement already satisfied: pygments>=2.4.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from ipython>=6.0->watermark) (2.15.1)\nRequirement already satisfied: decorator in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from ipython>=6.0->watermark) (5.1.1)\nRequirement already satisfied: jedi>=0.16 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from ipython>=6.0->watermark) (0.18.2)\nRequirement already satisfied: pexpect>4.3 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from ipython>=6.0->watermark) (4.8.0)\nRequirement already satisfied: matplotlib-inline in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from ipython>=6.0->watermark) (0.1.6)\nRequirement already satisfied: pickleshare in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from ipython>=6.0->watermark) (0.7.5)\nRequirement already satisfied: stack-data in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from ipython>=6.0->watermark) (0.6.2)\nRequirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from ipython>=6.0->watermark) (3.0.30)\nRequirement already satisfied: backcall in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from ipython>=6.0->watermark) (0.2.0)\nRequirement already satisfied: traitlets>=5 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from ipython>=6.0->watermark) (5.9.0)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from requests->transformers) (1.26.16)\nRequirement already satisfied: idna<4,>=2.5 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from requests->transformers) (3.4)\nRequirement already satisfied: charset-normalizer<4,>=2 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from requests->transformers) (3.1.0)\nRequirement already satisfied: certifi>=2017.4.17 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from requests->transformers) (2023.5.7)\nRequirement already satisfied: python-dateutil>=2.7.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from arrow->codecarbon) (2.8.2)\nRequirement already satisfied: types-python-dateutil>=2.8.10 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from arrow->codecarbon) (2.8.19.20240106)\nRequirement already satisfied: llvmlite<0.42,>=0.41.0dev0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from numba->shap) (0.41.1)\nRequirement already satisfied: pytz>=2020.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from pandas->datasets) (2023.3)\nRequirement already satisfied: tzdata>=2022.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from pandas->datasets) (2023.3)\nRequirement already satisfied: threadpoolctl>=2.0.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from scikit-learn->shap) (3.1.0)\nRequirement already satisfied: joblib>=1.1.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from scikit-learn->shap) (1.2.0)\nRequirement already satisfied: smmap<6,>=3.0.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from gitdb<5,>=4.0.1->GitPython!=3.1.29,>=1.0.0->wandb) (5.0.0)\nRequirement already satisfied: parso<0.9.0,>=0.8.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from jedi>=0.16->ipython>=6.0->watermark) (0.8.3)\nRequirement already satisfied: ptyprocess>=0.5 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from pexpect>4.3->ipython>=6.0->watermark) (0.7.0)\nRequirement already satisfied: wcwidth in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=6.0->watermark) (0.2.6)\nRequirement already satisfied: asttokens>=2.1.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from stack-data->ipython>=6.0->watermark) (2.2.1)\nRequirement already satisfied: pure-eval in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from stack-data->ipython>=6.0->watermark) (0.2.2)\nRequirement already satisfied: executing>=1.2.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from stack-data->ipython>=6.0->watermark) (1.2.0)\nNote: you may need to restart the kernel to use updated packages.\n"
        }
      ],
      "execution_count": 2,
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import torch\n",
        "import os\n",
        "from typing import List, Union\n",
        "from transformers import AutoTokenizer, Trainer, AutoModelForSequenceClassification, TrainingArguments, DataCollatorWithPadding, pipeline\n",
        "from datasets import load_dataset, Dataset, DatasetDict\n",
        "import shap\n",
        "import wandb\n",
        "import evaluate\n",
        "from codecarbon import EmissionsTracker\n",
        "import logging\n",
        "\n",
        "wandb.finish()\n",
        "\n",
        "logging.getLogger('codecarbon').propagate = False\n",
        "\n",
        "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n",
        "tracker = EmissionsTracker()\n",
        "\n",
        "%load_ext watermark"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": "/anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n2024-01-29 04:43:58.191236: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\nTo enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n2024-01-29 04:43:59.182154: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory\n2024-01-29 04:43:59.182291: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory\n2024-01-29 04:43:59.182304: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n[codecarbon INFO @ 04:44:02] [setup] RAM Tracking...\n[codecarbon INFO @ 04:44:02] [setup] GPU Tracking...\n[codecarbon INFO @ 04:44:02] Tracking Nvidia GPU via pynvml\n[codecarbon INFO @ 04:44:02] [setup] CPU Tracking...\n[codecarbon WARNING @ 04:44:02] No CPU tracking mode found. Falling back on CPU constant mode.\n[codecarbon WARNING @ 04:44:03] We saw that you have a Intel(R) Xeon(R) CPU E5-2690 v4 @ 2.60GHz but we don't know it. Please contact us.\n[codecarbon INFO @ 04:44:03] CPU Model on constant consumption mode: Intel(R) Xeon(R) CPU E5-2690 v4 @ 2.60GHz\n[codecarbon INFO @ 04:44:03] >>> Tracker's metadata:\n[codecarbon INFO @ 04:44:03]   Platform system: Linux-5.15.0-1040-azure-x86_64-with-glibc2.10\n[codecarbon INFO @ 04:44:03]   Python version: 3.8.5\n[codecarbon INFO @ 04:44:03]   CodeCarbon version: 2.3.3\n[codecarbon INFO @ 04:44:03]   Available RAM : 440.883 GB\n[codecarbon INFO @ 04:44:03]   CPU count: 24\n[codecarbon INFO @ 04:44:03]   CPU model: Intel(R) Xeon(R) CPU E5-2690 v4 @ 2.60GHz\n[codecarbon INFO @ 04:44:03]   GPU count: 4\n[codecarbon INFO @ 04:44:03]   GPU model: 4 x Tesla V100-PCIE-16GB\n[codecarbon WARNING @ 04:44:03] Cloud provider 'azure' do not publish electricity carbon intensity. Using country value instead.\n"
        }
      ],
      "execution_count": 3,
      "metadata": {
        "datalore": {
          "hide_input_from_viewers": false,
          "hide_output_from_viewers": false,
          "node_id": "caZjjFP0OyQNMVgZDiwswE",
          "report_properties": {
            "rowId": "un8W7ez7ZwoGb5Co6nydEV"
          },
          "type": "CODE"
        },
        "gather": {
          "logged": 1706503443742
        },
        "tags": []
      }
    },
    {
      "cell_type": "code",
      "source": [
        "device: str = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "\n",
        "SEED: int = 42\n",
        "\n",
        "BATCH_SIZE: int = 32\n",
        "EPOCHS: int = 5\n",
        "model_ckpt: str = \"distilbert-base-uncased\"\n",
        "\n",
        "# WandB configuration\n",
        "os.environ[\"WANDB_PROJECT\"] = \"DAEDRA multiclass model training\" \n",
        "os.environ[\"WANDB_LOG_MODEL\"] = \"checkpoint\"  # log all model checkpoints\n",
        "os.environ[\"WANDB_NOTEBOOK_NAME\"] = \"DAEDRA.ipynb\""
      ],
      "outputs": [],
      "execution_count": 4,
      "metadata": {
        "collapsed": false,
        "gather": {
          "logged": 1706503443899
        },
        "jupyter": {
          "outputs_hidden": false
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%watermark --iversion"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": "shap    : 0.44.1\nnumpy   : 1.23.5\npandas  : 2.0.2\nlogging : 0.5.1.2\ntorch   : 1.12.0\nevaluate: 0.4.1\nwandb   : 0.16.2\nre      : 2.2.1\n\n"
        }
      ],
      "execution_count": 5,
      "metadata": {
        "collapsed": false,
        "jupyter": {
          "outputs_hidden": false
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!nvidia-smi"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": "Mon Jan 29 04:44:03 2024       \r\n+---------------------------------------------------------------------------------------+\r\n| NVIDIA-SMI 535.129.03             Driver Version: 535.129.03   CUDA Version: 12.2     |\r\n|-----------------------------------------+----------------------+----------------------+\r\n| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |\r\n| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |\r\n|                                         |                      |               MIG M. |\r\n|=========================================+======================+======================|\r\n|   0  Tesla V100-PCIE-16GB           Off | 00000001:00:00.0 Off |                  Off |\r\n| N/A   26C    P0              25W / 250W |      4MiB / 16384MiB |      0%      Default |\r\n|                                         |                      |                  N/A |\r\n+-----------------------------------------+----------------------+----------------------+\r\n|   1  Tesla V100-PCIE-16GB           Off | 00000002:00:00.0 Off |                  Off |\r\n| N/A   25C    P0              23W / 250W |      4MiB / 16384MiB |      0%      Default |\r\n|                                         |                      |                  N/A |\r\n+-----------------------------------------+----------------------+----------------------+\r\n|   2  Tesla V100-PCIE-16GB           Off | 00000003:00:00.0 Off |                  Off |\r\n| N/A   26C    P0              25W / 250W |      4MiB / 16384MiB |      0%      Default |\r\n|                                         |                      |                  N/A |\r\n+-----------------------------------------+----------------------+----------------------+\r\n|   3  Tesla V100-PCIE-16GB           Off | 00000004:00:00.0 Off |                  Off |\r\n| N/A   27C    P0              25W / 250W |      4MiB / 16384MiB |      0%      Default |\r\n|                                         |                      |                  N/A |\r\n+-----------------------------------------+----------------------+----------------------+\r\n                                                                                         \r\n+---------------------------------------------------------------------------------------+\r\n| Processes:                                                                            |\r\n|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |\r\n|        ID   ID                                                             Usage      |\r\n|=======================================================================================|\r\n|  No running processes found                                                           |\r\n+---------------------------------------------------------------------------------------+\r\n"
        }
      ],
      "execution_count": 6,
      "metadata": {
        "datalore": {
          "hide_input_from_viewers": true,
          "hide_output_from_viewers": true,
          "node_id": "UU2oOJhwbIualogG1YyCMd",
          "type": "CODE"
        }
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Loading the data set"
      ],
      "metadata": {
        "datalore": {
          "hide_input_from_viewers": false,
          "hide_output_from_viewers": false,
          "node_id": "t45KHugmcPVaO0nuk8tGJ9",
          "report_properties": {
            "rowId": "40nN9Hvgi1clHNV5RAemI5"
          },
          "type": "MD"
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "dataset = load_dataset(\"chrisvoncsefalvay/vaers-outcomes\")"
      ],
      "outputs": [],
      "execution_count": 7,
      "metadata": {
        "collapsed": false,
        "gather": {
          "logged": 1706503446033
        },
        "jupyter": {
          "outputs_hidden": false
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "dataset"
      ],
      "outputs": [
        {
          "output_type": "execute_result",
          "execution_count": 8,
          "data": {
            "text/plain": "DatasetDict({\n    train: Dataset({\n        features: ['id', 'text', 'label'],\n        num_rows: 1270444\n    })\n    test: Dataset({\n        features: ['id', 'text', 'label'],\n        num_rows: 272238\n    })\n    val: Dataset({\n        features: ['id', 'text', 'label'],\n        num_rows: 272238\n    })\n})"
          },
          "metadata": {}
        }
      ],
      "execution_count": 8,
      "metadata": {
        "collapsed": false,
        "gather": {
          "logged": 1706503446252
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "SUBSAMPLING = 1.0\n",
        "\n",
        "if SUBSAMPLING < 1:\n",
        "    _ = DatasetDict()\n",
        "    for each in dataset.keys():\n",
        "        _[each] = dataset[each].shuffle(seed=SEED).select(range(int(len(dataset[each]) * SUBSAMPLING)))\n",
        "\n",
        "    dataset = _"
      ],
      "outputs": [],
      "execution_count": 9,
      "metadata": {
        "gather": {
          "logged": 1706503446498
        }
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Tokenisation and encoding"
      ],
      "metadata": {}
    },
    {
      "cell_type": "code",
      "source": [
        "def encode_ds(ds: Union[Dataset, DatasetDict], tokenizer_model: str = model_ckpt) -> Union[Dataset, DatasetDict]:\n",
        "    return ds_enc"
      ],
      "outputs": [],
      "execution_count": 10,
      "metadata": {
        "gather": {
          "logged": 1706503446633
        }
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Evaluation metrics"
      ],
      "metadata": {}
    },
    {
      "cell_type": "code",
      "source": [
        "accuracy = evaluate.load(\"accuracy\")\n",
        "precision, recall = evaluate.load(\"precision\"), evaluate.load(\"recall\")\n",
        "f1 = evaluate.load(\"f1\")"
      ],
      "outputs": [],
      "execution_count": 11,
      "metadata": {
        "gather": {
          "logged": 1706503446863
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def compute_metrics(eval_pred):\n",
        "    predictions, labels = eval_pred\n",
        "    predictions = np.argmax(predictions, axis=1)\n",
        "    return {\n",
        "        'accuracy': accuracy.compute(predictions=predictions, references=labels)[\"accuracy\"],\n",
        "        'precision_macroaverage': precision.compute(predictions=predictions, references=labels, average='macro')[\"precision\"],\n",
        "        'precision_microaverage': precision.compute(predictions=predictions, references=labels, average='micro')[\"precision\"],\n",
        "        'recall_macroaverage': recall.compute(predictions=predictions, references=labels, average='macro')[\"recall\"],\n",
        "        'recall_microaverage': recall.compute(predictions=predictions, references=labels, average='micro')[\"recall\"],\n",
        "        'f1_microaverage': f1.compute(predictions=predictions, references=labels, average='micro')[\"f1\"]\n",
        "    }"
      ],
      "outputs": [],
      "execution_count": 12,
      "metadata": {
        "gather": {
          "logged": 1706503447004
        }
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Training"
      ],
      "metadata": {}
    },
    {
      "cell_type": "markdown",
      "source": [
        "We specify a label map – this has to be done manually, even if `Datasets` has a function for it, as `AutoModelForSequenceClassification` requires an object with a length :("
      ],
      "metadata": {}
    },
    {
      "cell_type": "code",
      "source": [
        "label_map = {i: label for i, label in enumerate(dataset[\"test\"].features[\"label\"].names)}"
      ],
      "outputs": [],
      "execution_count": 13,
      "metadata": {
        "gather": {
          "logged": 1706503447186
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(model_ckpt)\n",
        "\n",
        "cols = dataset[\"train\"].column_names\n",
        "cols.remove(\"label\")\n",
        "ds_enc = dataset.map(lambda x: tokenizer(x[\"text\"], truncation=True), batched=True, remove_columns=cols)\n"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": "Map: 100%|██████████| 272238/272238 [01:45<00:00, 2592.04 examples/s]\n"
        }
      ],
      "execution_count": 14,
      "metadata": {
        "gather": {
          "logged": 1706503552083
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "model = AutoModelForSequenceClassification.from_pretrained(model_ckpt, \n",
        "    num_labels=len(dataset[\"test\"].features[\"label\"].names), \n",
        "    id2label=label_map, \n",
        "    label2id={v:k for k,v in label_map.items()})\n",
        "\n",
        "args = TrainingArguments(\n",
        "    output_dir=\"vaers\",\n",
        "    evaluation_strategy=\"epoch\",\n",
        "    save_strategy=\"epoch\",\n",
        "    learning_rate=2e-5,\n",
        "    per_device_train_batch_size=BATCH_SIZE,\n",
        "    per_device_eval_batch_size=BATCH_SIZE,\n",
        "    num_train_epochs=EPOCHS,\n",
        "    weight_decay=.01,\n",
        "    logging_steps=1,\n",
        "    load_best_model_at_end=True,\n",
        "    run_name=f\"daedra-training\",\n",
        "    report_to=[\"wandb\"])\n",
        "\n",
        "trainer = Trainer(\n",
        "        model=model,\n",
        "        args=args,\n",
        "        train_dataset=ds_enc[\"train\"],\n",
        "        eval_dataset=ds_enc[\"test\"],\n",
        "        tokenizer=tokenizer,\n",
        "        compute_metrics=compute_metrics)"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight', 'pre_classifier.bias', 'pre_classifier.weight']\nYou should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
        }
      ],
      "execution_count": 15,
      "metadata": {
        "gather": {
          "logged": 1706503554669
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "if SUBSAMPLING != 1.0:\n",
        "    wandb_tag: List[str] = [f\"subsample-{SUBSAMPLING}\"]\n",
        "else:\n",
        "    wandb_tag: List[str] = [f\"full_sample\"]\n",
        "\n",
        "wandb_tag.append(f\"batch_size-{BATCH_SIZE}\")\n",
        "wandb_tag.append(f\"base:{model_ckpt}\")\n",
        "    \n",
        "wandb.init(name=\"daedra_training_run\", tags=wandb_tag, magic=True)"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mchrisvoncsefalvay\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m wandb.init() arguments ignored because wandb magic has already been initialized\n"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "Tracking run with wandb version 0.16.2"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "Run data is saved locally in <code>/mnt/batch/tasks/shared/LS_root/mounts/clusters/daedra-hptrain-cvc/code/Users/kristof.csefalvay/daedra/notebooks/wandb/run-20240129_044555-kjhyoltp</code>"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "Syncing run <strong><a href='https://wandb.ai/chrisvoncsefalvay/DAEDRA%20multiclass%20model%20training/runs/kjhyoltp' target=\"_blank\">daedra_training_run</a></strong> to <a href='https://wandb.ai/chrisvoncsefalvay/DAEDRA%20multiclass%20model%20training' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": " View project at <a href='https://wandb.ai/chrisvoncsefalvay/DAEDRA%20multiclass%20model%20training' target=\"_blank\">https://wandb.ai/chrisvoncsefalvay/DAEDRA%20multiclass%20model%20training</a>"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": " View run at <a href='https://wandb.ai/chrisvoncsefalvay/DAEDRA%20multiclass%20model%20training/runs/kjhyoltp' target=\"_blank\">https://wandb.ai/chrisvoncsefalvay/DAEDRA%20multiclass%20model%20training/runs/kjhyoltp</a>"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "Finishing last run (ID:kjhyoltp) before initializing another..."
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "W&B sync reduced upload amount by 26.5%             "
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": " View run <strong style=\"color:#cdcd00\">daedra_training_run</strong> at: <a href='https://wandb.ai/chrisvoncsefalvay/DAEDRA%20multiclass%20model%20training/runs/kjhyoltp' target=\"_blank\">https://wandb.ai/chrisvoncsefalvay/DAEDRA%20multiclass%20model%20training/runs/kjhyoltp</a><br/> View job at <a href='https://wandb.ai/chrisvoncsefalvay/DAEDRA%20multiclass%20model%20training/jobs/QXJ0aWZhY3RDb2xsZWN0aW9uOjEzNDcyMTQwMw==/version_details/v1' target=\"_blank\">https://wandb.ai/chrisvoncsefalvay/DAEDRA%20multiclass%20model%20training/jobs/QXJ0aWZhY3RDb2xsZWN0aW9uOjEzNDcyMTQwMw==/version_details/v1</a><br/>Synced 5 W&B file(s), 0 media file(s), 2 artifact file(s) and 0 other file(s)"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "Find logs at: <code>./wandb/run-20240129_044555-kjhyoltp/logs</code>"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "Successfully finished last run (ID:kjhyoltp). Initializing new run:<br/>"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "Tracking run with wandb version 0.16.2"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "Run data is saved locally in <code>/mnt/batch/tasks/shared/LS_root/mounts/clusters/daedra-hptrain-cvc/code/Users/kristof.csefalvay/daedra/notebooks/wandb/run-20240129_044558-ed51hqn6</code>"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "Syncing run <strong><a href='https://wandb.ai/chrisvoncsefalvay/DAEDRA%20multiclass%20model%20training/runs/ed51hqn6' target=\"_blank\">daedra_training_run</a></strong> to <a href='https://wandb.ai/chrisvoncsefalvay/DAEDRA%20multiclass%20model%20training' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": " View project at <a href='https://wandb.ai/chrisvoncsefalvay/DAEDRA%20multiclass%20model%20training' target=\"_blank\">https://wandb.ai/chrisvoncsefalvay/DAEDRA%20multiclass%20model%20training</a>"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": " View run at <a href='https://wandb.ai/chrisvoncsefalvay/DAEDRA%20multiclass%20model%20training/runs/ed51hqn6' target=\"_blank\">https://wandb.ai/chrisvoncsefalvay/DAEDRA%20multiclass%20model%20training/runs/ed51hqn6</a>"
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "execution_count": 16,
          "data": {
            "text/html": "<button onClick=\"this.nextSibling.style.display='block';this.style.display='none';\">Display W&B run</button><iframe src='https://wandb.ai/chrisvoncsefalvay/DAEDRA%20multiclass%20model%20training/runs/ed51hqn6?jupyter=true' style='border:none;width:100%;height:420px;display:none;'></iframe>",
            "text/plain": "<wandb.sdk.wandb_run.Run at 0x7f19d09fdbe0>"
          },
          "metadata": {}
        }
      ],
      "execution_count": 16,
      "metadata": {
        "gather": {
          "logged": 1706503566090
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "tracker.start()\n",
        "trainer.train()\n",
        "tracker.stop()\n"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": "Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "\n    <div>\n      \n      <progress value='183' max='49630' style='width:300px; height:20px; vertical-align: middle;'></progress>\n      [  183/49630 01:56 < 8:51:06, 1.55 it/s, Epoch 0.02/5]\n    </div>\n    <table border=\"1\" class=\"dataframe\">\n  <thead>\n <tr style=\"text-align: left;\">\n      <th>Epoch</th>\n      <th>Training Loss</th>\n      <th>Validation Loss</th>\n    </tr>\n  </thead>\n  <tbody>\n  </tbody>\n</table><p>"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": "[codecarbon INFO @ 04:46:20] Energy consumed for RAM : 0.000690 kWh. RAM Power : 165.33123922348022 W\n[codecarbon INFO @ 04:46:20] Energy consumed for all GPUs : 0.001499 kWh. Total GPU Power : 359.1829830586385 W\n[codecarbon INFO @ 04:46:20] Energy consumed for all CPUs : 0.000177 kWh. Total CPU Power : 42.5 W\n[codecarbon INFO @ 04:46:20] 0.002366 kWh of electricity used since the beginning.\n[codecarbon INFO @ 04:46:35] Energy consumed for RAM : 0.001378 kWh. RAM Power : 165.33123922348022 W\n[codecarbon INFO @ 04:46:35] Energy consumed for all GPUs : 0.004078 kWh. Total GPU Power : 619.6193403526773 W\n[codecarbon INFO @ 04:46:35] Energy consumed for all CPUs : 0.000355 kWh. Total CPU Power : 42.5 W\n[codecarbon INFO @ 04:46:35] 0.005811 kWh of electricity used since the beginning.\n[codecarbon INFO @ 04:46:50] Energy consumed for RAM : 0.002066 kWh. RAM Power : 165.33123922348022 W\n[codecarbon INFO @ 04:46:50] Energy consumed for all GPUs : 0.006632 kWh. Total GPU Power : 613.6554096062922 W\n[codecarbon INFO @ 04:46:50] Energy consumed for all CPUs : 0.000532 kWh. Total CPU Power : 42.5 W\n[codecarbon INFO @ 04:46:50] 0.009230 kWh of electricity used since the beginning.\n[codecarbon INFO @ 04:47:05] Energy consumed for RAM : 0.002754 kWh. RAM Power : 165.33123922348022 W\n[codecarbon INFO @ 04:47:05] Energy consumed for all GPUs : 0.009249 kWh. Total GPU Power : 628.5574609453653 W\n[codecarbon INFO @ 04:47:05] Energy consumed for all CPUs : 0.000709 kWh. Total CPU Power : 42.5 W\n[codecarbon INFO @ 04:47:05] 0.012712 kWh of electricity used since the beginning.\n[codecarbon INFO @ 04:47:20] Energy consumed for RAM : 0.003442 kWh. RAM Power : 165.33123922348022 W\n[codecarbon INFO @ 04:47:20] Energy consumed for all GPUs : 0.011850 kWh. Total GPU Power : 624.8454173521444 W\n[codecarbon INFO @ 04:47:20] Energy consumed for all CPUs : 0.000886 kWh. Total CPU Power : 42.5 W\n[codecarbon INFO @ 04:47:20] 0.016178 kWh of electricity used since the beginning.\n[codecarbon INFO @ 04:47:35] Energy consumed for RAM : 0.004130 kWh. RAM Power : 165.33123922348022 W\n[codecarbon INFO @ 04:47:35] Energy consumed for all GPUs : 0.014490 kWh. Total GPU Power : 634.7378588005432 W\n[codecarbon INFO @ 04:47:35] Energy consumed for all CPUs : 0.001063 kWh. Total CPU Power : 42.5 W\n[codecarbon INFO @ 04:47:35] 0.019683 kWh of electricity used since the beginning.\n[codecarbon INFO @ 04:47:50] Energy consumed for RAM : 0.004818 kWh. RAM Power : 165.33123922348022 W\n[codecarbon INFO @ 04:47:50] Energy consumed for all GPUs : 0.017140 kWh. Total GPU Power : 636.6500188212152 W\n[codecarbon INFO @ 04:47:50] Energy consumed for all CPUs : 0.001240 kWh. Total CPU Power : 42.5 W\n[codecarbon INFO @ 04:47:50] 0.023197 kWh of electricity used since the beginning.\n[codecarbon INFO @ 04:48:05] Energy consumed for RAM : 0.005506 kWh. RAM Power : 165.33123922348022 W\n[codecarbon INFO @ 04:48:05] Energy consumed for all GPUs : 0.019771 kWh. Total GPU Power : 631.881788173399 W\n[codecarbon INFO @ 04:48:05] Energy consumed for all CPUs : 0.001417 kWh. Total CPU Power : 42.5 W\n[codecarbon INFO @ 04:48:05] 0.026694 kWh of electricity used since the beginning.\n"
        }
      ],
      "execution_count": 17,
      "metadata": {
        "gather": {
          "logged": 1706486541798
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "wandb.finish()"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1706486541918
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "variant = \"full_sample\" if SUBSAMPLING == 1.0 else f\"subsample-{SUBSAMPLING}\"\n",
        "tokenizer._tokenizer.save(\"tokenizer.json\")\n",
        "tokenizer.push_to_hub(\"chrisvoncsefalvay/daedra\")\n",
        "sample = \"full sample\" if SUBSAMPLING == 1.0 else f\"{SUBSAMPLING * 100}% of the full sample\"\n",
        "\n",
        "model.push_to_hub(\"chrisvoncsefalvay/daedra\", \n",
        "                  variant=variant,\n",
        "                  commit_message=f\"DAEDRA model trained on {sample} of the VAERS dataset (training set size: {dataset['train'].num_rows:,})\")"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1706486541928
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "variant = \"full_sample\" if SUBSAMPLING == 1.0 else f\"subsample-{SUBSAMPLING}\"\n",
        "tokenizer._tokenizer.save(\"tokenizer.json\")\n",
        "tokenizer.push_to_hub(\"chrisvoncsefalvay/daedra\")\n",
        "sample = \"full sample\" if SUBSAMPLING == 1.0 else f\"{SUBSAMPLING * 100}% of the full sample\"\n",
        "\n",
        "model.push_to_hub(\"chrisvoncsefalvay/daedra\", \n",
        "                  variant=variant,\n",
        "                  commit_message=f\"DAEDRA model trained on {sample} of the VAERS dataset (training set size: {dataset['train'].num_rows:,})\")"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {}
    }
  ],
  "metadata": {
    "datalore": {
      "base_environment": "default",
      "computation_mode": "JUPYTER",
      "package_manager": "pip",
      "packages": [
        {
          "name": "datasets",
          "source": "PIP",
          "version": "2.16.1"
        },
        {
          "name": "torch",
          "source": "PIP",
          "version": "2.1.2"
        },
        {
          "name": "accelerate",
          "source": "PIP",
          "version": "0.26.1"
        }
      ],
      "report_row_ids": [
        "un8W7ez7ZwoGb5Co6nydEV",
        "40nN9Hvgi1clHNV5RAemI5",
        "TgRD90H5NSPpKS41OeXI1w",
        "ZOm5BfUs3h1EGLaUkBGeEB",
        "kOP0CZWNSk6vqE3wkPp7Vc",
        "W4PWcOu2O2pRaZyoE2W80h",
        "RolbOnQLIftk0vy9mIcz5M",
        "8OPhUgbaNJmOdiq5D3a6vK",
        "5Qrt3jSvSrpK6Ne1hS6shL",
        "hTq7nFUrovN5Ao4u6dIYWZ",
        "I8WNZLpJ1DVP2wiCW7YBIB",
        "SawhU3I9BewSE1XBPstpNJ",
        "80EtLEl2FIE4FqbWnUD3nT"
      ],
      "version": 3
    },
    "kernel_info": {
      "name": "python38-azureml-pt-tf"
    },
    "kernelspec": {
      "display_name": "azureml_py38_PT_TF",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.8.5",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "microsoft": {
      "host": {
        "AzureML": {
          "notebookHasBeenCompleted": true
        }
      },
      "ms_spell_check": {
        "ms_spell_check_language": "en"
      }
    },
    "nteract": {
      "version": "[email protected]"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 4
}