model_usage.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nvidia-smi output:\n",
      "\n",
      "Fri Nov  8 21:11:36 2024       \n",
      "+-----------------------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 560.35.03              Driver Version: 560.35.03      CUDA Version: 12.6     |\n",
      "|-----------------------------------------+------------------------+----------------------+\n",
      "| GPU  Name                 Persistence-M | Bus-Id          Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp   Perf          Pwr:Usage/Cap |           Memory-Usage | GPU-Util  Compute M. |\n",
      "|                                         |                        |               MIG M. |\n",
      "|=========================================+========================+======================|\n",
      "|   0  NVIDIA A40                     On  |   00000000:17:00.0 Off |                    0 |\n",
      "|  0%   53C    P0             85W /  300W |    3717MiB /  46068MiB |      0%      Default |\n",
      "|                                         |                        |                  N/A |\n",
      "+-----------------------------------------+------------------------+----------------------+\n",
      "|   1  NVIDIA A40                     On  |   00000000:65:00.0 Off |                    0 |\n",
      "|  0%   55C    P0             86W /  300W |    3195MiB /  46068MiB |      0%      Default |\n",
      "|                                         |                        |                  N/A |\n",
      "+-----------------------------------------+------------------------+----------------------+\n",
      "|   2  NVIDIA A40                     On  |   00000000:CA:00.0 Off |                    0 |\n",
      "|  0%   45C    P0             65W /  300W |       4MiB /  46068MiB |      0%      Default |\n",
      "|                                         |                        |                  N/A |\n",
      "+-----------------------------------------+------------------------+----------------------+\n",
      "|   3  NVIDIA A40                     On  |   00000000:E3:00.0 Off |                    0 |\n",
      "|  0%   38C    P8             25W /  300W |       4MiB /  46068MiB |      0%      Default |\n",
      "|                                         |                        |                  N/A |\n",
      "+-----------------------------------------+------------------------+----------------------+\n",
      "                                                                                         \n",
      "+-----------------------------------------------------------------------------------------+\n",
      "| Processes:                                                                              |\n",
      "|  GPU   GI   CI        PID   Type   Process name                              GPU Memory |\n",
      "|        ID   ID                                                               Usage      |\n",
      "|=========================================================================================|\n",
      "|    0   N/A  N/A     96617      C   ...10/models_citation/myenv/bin/python       3708MiB |\n",
      "|    1   N/A  N/A   1230721      C   /home/historynlp/venv/bin/python3            1414MiB |\n",
      "|    1   N/A  N/A   2212129      C   /home/historynlp/venv/bin/python3            1766MiB |\n",
      "+-----------------------------------------------------------------------------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import subprocess\n",
    "\n",
    "def run_nvidia_smi():\n",
    "    try:\n",
    "        # Run the nvidia-smi command\n",
    "        result = subprocess.run(['nvidia-smi'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)\n",
    "\n",
    "        # Check if the command was successful\n",
    "        if result.returncode == 0:\n",
    "            print(\"nvidia-smi output:\\n\")\n",
    "            print(result.stdout)\n",
    "        else:\n",
    "            print(\"Error running nvidia-smi:\\n\")\n",
    "            print(result.stderr)\n",
    "\n",
    "    except Exception as e:\n",
    "        print(f\"An error occurred: {e}\")\n",
    "\n",
    "# Call the function to run nvidia-smi\n",
    "run_nvidia_smi()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/smartinez1/.venv/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "/home/smartinez1/.venv/lib/python3.10/site-packages/torchvision/io/image.py:13: UserWarning: Failed to load image Python extension: libtorch_cuda_cu.so: cannot open shared object file: No such file or directory\n",
      "  warn(f\"Failed to load image Python extension: {e}\")\n",
      "2024-11-08 21:11:38.912317: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F AVX512_VNNI FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2024-11-08 21:11:39.068802: I tensorflow/core/util/port.cc:104] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
      "2024-11-08 21:11:39.677787: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory\n",
      "2024-11-08 21:11:39.677842: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory\n",
      "2024-11-08 21:11:39.677848: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "gpu_index = 2\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = f\"{gpu_index}\"\n",
    "from huggingface_hub import login\n",
    "from transformers import BitsAndBytesConfig, AutoModelForCausalLM, AutoTokenizer\n",
    "import torch\n",
    "from peft import PeftModel, PeftConfig\n",
    "from transformers import AutoModelForCausalLM, pipeline\n",
    "import logging\n",
    "# Suppress all warnings\n",
    "logging.getLogger(\"transformers\").setLevel(logging.CRITICAL) #weird warning when using model for inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of available CUDA devices: 1\n",
      "\n",
      "Device 0: NVIDIA A40\n"
     ]
    }
   ],
   "source": [
    "# Check if CUDA is available\n",
    "if torch.cuda.is_available():\n",
    "    num_devices = torch.cuda.device_count()\n",
    "    print(f\"Number of available CUDA devices: {num_devices}\")\n",
    "    \n",
    "    for i in range(num_devices):\n",
    "        device_name = torch.cuda.get_device_name(i)\n",
    "        print(f\"\\nDevice {i}: {device_name}\")\n",
    "else:\n",
    "    print(\"CUDA is not available.\")\n",
    "# Specify the device (0 for GPU or -1 for CPU)\n",
    "device = 0 if torch.cuda.is_available() else -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading checkpoint shards: 100%|██████████████████| 4/4 [00:02<00:00,  1.55it/s]\n"
     ]
    }
   ],
   "source": [
    "config = PeftConfig.from_pretrained(\"smartinez1/Llama-3.1-8B-FINLLM\")\n",
    "base_model = AutoModelForCausalLM.from_pretrained(\"meta-llama/Llama-3.1-8B\")\n",
    "model = PeftModel.from_pretrained(base_model, \"smartinez1/Llama-3.1-8B-FINLLM\")\n",
    "# Load the tokenizer associated with the base model\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"meta-llama/Llama-3.1-8B\")\n",
    "# Define the unique padding token for fine-tuning\n",
    "custom_pad_token = \"<|finetune_right_pad_id|>\"\n",
    "tokenizer.add_special_tokens({'pad_token': custom_pad_token})\n",
    "pad_token_id = tokenizer.pad_token_id"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The input should be given as presented in the Challenge website."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "User: Provide a link for Regulation A (Extensions of Credit by Federal Reserve Banks) law\n",
      "AI: Regulation A (Extensions of Credit by Federal Reserve Banks): https://www.federalreserve.gov/newsevents/pressreleases/files/bcreg20240705c1.pdf\n",
      "--------------------------------------------------\n",
      "User: Define the following term: Insurance Scores.\n",
      "AI: A credit score based on an insurance underwriting model that evaluates the risk of an individual for insurance purposes.\n",
      "--------------------------------------------------\n",
      "User: Expand the following acronym into its full form: ESCB.\n",
      "AI: European System of Central Banks\n",
      "--------------------------------------------------\n",
      "User: Provide a concise answer to the following question: Which countries currently have bilateral FTAs in effect with the U.S.?\n",
      "AI: As of June 2023, the U.S. has bilateral FTAs in effect with 20 countries: Australia, Bahrain, Canada, Chile, Colombia, Costa Rica, Dominican Republic, El Salvador, Guatemala, Honduras, Israel, Jordan, Mexico, Morocco, Nicaragua, Oman, Panama, Peru, Singapore, and South Korea.\n",
      "--------------------------------------------------\n",
      "User: Given the following text, only list the following for each: specific Organizations, Legislations, Dates, Monetary Values, and Statistics When can counterparties start notifying the national competent authorities (NCAs) of their intention to apply the reporting exemption in accordance with Article 9(1) EMIR, as amended by Regulation 2019/834?\n",
      "AI: From 1 January 2021\n",
      "--------------------------------------------------\n",
      "User: Provide a concise answer to the following question: What type of license is the Apache License, Version 2.0?\n",
      "AI: The Apache License, Version 2.0 is a permissive free software license.\n",
      "--------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "# Set up the text generation pipeline with the PEFT model, specifying the device\n",
    "generator = pipeline(\"text-generation\", model=model, tokenizer=tokenizer, device=device)\n",
    "\n",
    "# List of user inputs\n",
    "user_inputs = [\n",
    "    \"Provide a link for Regulation A (Extensions of Credit by Federal Reserve Banks) law\",\n",
    "    \"Define the following term: Insurance Scores.\",\n",
    "    \"Expand the following acronym into its full form: ESCB.\",\n",
    "    \"Provide a concise answer to the following question: Which countries currently have bilateral FTAs in effect with the U.S.?\",\n",
    "    \"\"\"Given the following text, only list the following for each: specific Organizations, Legislations, Dates, Monetary Values, \n",
    "    and Statistics When can counterparties start notifying the national competent authorities (NCAs) of their intention to apply \n",
    "    the reporting exemption in accordance with Article 9(1) EMIR, as amended by Regulation 2019/834?\"\"\",\n",
    "    \"Provide a concise answer to the following question: What type of license is the Apache License, Version 2.0?\"\n",
    "]\n",
    "\n",
    "# Define the prompt template\n",
    "prompt_template = \"\"\"Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
    "\n",
    "### Instruction:\n",
    "{}\n",
    "\n",
    "### Answer:\n",
    "\"\"\"\n",
    "\n",
    "# Loop over each user input and generate a response\n",
    "for user_input in user_inputs:\n",
    "    # Format the user input into the prompt\n",
    "    prompt = prompt_template.format(user_input)\n",
    "\n",
    "    # Generate a response from the model\n",
    "    response = generator(prompt, max_length=200, num_return_sequences=1, do_sample=True)\n",
    "\n",
    "    # Extract and clean up the AI's response\n",
    "    response_str = response[0]['generated_text'].split('### Answer:')[1].strip()\n",
    "    cut_ind = response_str.find(\"#\")  # Remove extra information after the response\n",
    "    response_str = response_str[:cut_ind].strip() if cut_ind != -1 else response_str\n",
    "\n",
    "    # Display the AI's response\n",
    "    print(f\"User: {user_input}\")\n",
    "    print(f\"AI: {response_str}\")\n",
    "    print(\"-\" * 50)  # Separator for clarity\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}