commit cleaned notebook

notebooks/HuggingFace-Inference.ipynb

@@ -1,8 +1,83 @@
 {
  "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "2e6b851c",
+ "metadata": {},
+ "source": [
+ "## Import Packages"
+ ]
+ },
  {
  "cell_type": "code",
- "execution_count": 5,
+ "execution_count": 1,
+ "id": "32d8be79",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "2023-06-09 21:59:52.885485: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F AVX512_VNNI FMA\n",
+ "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
+ "2023-06-09 21:59:53.039141: I tensorflow/core/util/port.cc:104] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
+ "2023-06-09 21:59:53.827918: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/nvidia/lib:/usr/local/nvidia/lib64\n",
+ "2023-06-09 21:59:53.828006: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/nvidia/lib:/usr/local/nvidia/lib64\n",
+ "2023-06-09 21:59:53.828014: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n",
+ "===================================BUG REPORT===================================\n",
+ "Welcome to bitsandbytes. For bug reports, please run\n",
+ "\n",
+ "python -m bitsandbytes\n",
+ "\n",
+ " and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
+ "================================================================================\n",
+ "bin /opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/bitsandbytes/libbitsandbytes_cuda113_nocublaslt.so\n",
+ "CUDA SETUP: CUDA runtime path found: /opt/conda/envs/media-reco-env-3-8/lib/libcudart.so\n",
+ "CUDA SETUP: Highest compute capability among GPUs detected: 7.0\n",
+ "CUDA SETUP: Detected CUDA version 113\n",
+ "CUDA SETUP: Loading binary /opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/bitsandbytes/libbitsandbytes_cuda113_nocublaslt.so...\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: Found duplicate ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] files: {PosixPath('/opt/conda/envs/media-reco-env-3-8/lib/libcudart.so'), PosixPath('/opt/conda/envs/media-reco-env-3-8/lib/libcudart.so.11.0')}.. We'll flip a coin and try one of these, in order to fail forward.\n",
+ "Either way, this might cause trouble in the future:\n",
+ "If you get `CUDA error: invalid device function` errors, the above might be the cause and the solution is to make sure only one ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] in the paths that we search based on your env.\n",
+ " warn(msg)\n",
+ "/opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: Compute capability < 7.5 detected! Only slow 8-bit matmul is supported for your GPU!\n",
+ " warn(msg)\n"
+ ]
+ }
+ ],
+ "source": [
+ "import os\n",
+ "os.chdir(\"..\")\n",
+ "\n",
+ "import torch\n",
+ "from transformers import GenerationConfig, LlamaTokenizer, LlamaForCausalLM\n",
+ "from peft import PeftModel, PeftConfig"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "76b3d3ff",
+ "metadata": {},
+ "source": [
+ "## Utilities"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
  "id": "9837afb7",
  "metadata": {},
  "outputs": [],
@@ -27,6 +102,34 @@
  "### Response:\"\"\""
  ]
  },
+ {
+ "cell_type": "markdown",
+ "id": "0d3b0d75",
+ "metadata": {},
+ "source": [
+ "## Configs"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "id": "fb3cba34",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "MODEL_NAME = \".\"\n",
+ "BASE_MODEL = \"decapoda-research/llama-7b-hf\"\n",
+ "LOAD_FINETUNED = False"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "8dc85b3d",
+ "metadata": {},
+ "source": [
+ "## Load Model & Tokenizer"
+ ]
+ },
  {
  "cell_type": "code",
  "execution_count": 4,
@@ -43,7 +146,7 @@
  {
  "data": {
  "application/vnd.jupyter.widget-view+json": {
- "model_id": "bff8fb0a005e4635a07ecf6fe0fdba88",
+ "model_id": "f11486f431fd48799d91ef69e586e410",
  "version_major": 2,
  "version_minor": 0
  },
@@ -53,223 +156,9 @@
  },
  "metadata": {},
  "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800000; text-decoration-color: #800000\">╭─────────────────────────────── </span><span style=\"color: #800000; text-decoration-color: #800000; font-weight: bold\">Traceback </span><span style=\"color: #bf7f7f; text-decoration-color: #bf7f7f; font-weight: bold\">(most recent call last)</span><span style=\"color: #800000; text-decoration-color: #800000\"> ────────────────────────────────╮</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/transformers/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">modeling_utils.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">45</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #0000ff; text-decoration-color: #0000ff\">9</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">load_state_dict</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 456 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ </span>) <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 457 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">return</span> safe_load_file(checkpoint_file) <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 458 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">try</span>: <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">❱ </span> 459 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">return</span> torch.load(checkpoint_file, map_location=<span style=\"color: #808000; text-decoration-color: #808000\">\"cpu\"</span>) <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 460 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">except</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">Exception</span> <span style=\"color: #0000ff; text-decoration-color: #0000ff\">as</span> e: <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 461 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">try</span>: <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 462 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">with</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">open</span>(checkpoint_file) <span style=\"color: #0000ff; text-decoration-color: #0000ff\">as</span> f: <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/torch/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">serialization.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">791</span> in <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #00ff00; text-decoration-color: #00ff00\">load</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 788 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">if</span> <span style=\"color: #808000; text-decoration-color: #808000\">'encoding'</span> <span style=\"color: #ff00ff; text-decoration-color: #ff00ff\">not</span> <span style=\"color: #ff00ff; text-decoration-color: #ff00ff\">in</span> pickle_load_args.keys(): <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 789 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ </span>pickle_load_args[<span style=\"color: #808000; text-decoration-color: #808000\">'encoding'</span>] = <span style=\"color: #808000; text-decoration-color: #808000\">'utf-8'</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 790 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ </span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">❱ </span> 791 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">with</span> _open_file_like(f, <span style=\"color: #808000; text-decoration-color: #808000\">'rb'</span>) <span style=\"color: #0000ff; text-decoration-color: #0000ff\">as</span> opened_file: <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 792 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">if</span> _is_zipfile(opened_file): <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 793 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"># The zipfile reader is going to advance the current file position.</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 794 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"># If we want to actually tail call to torch.jit.load, we need to</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/torch/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">serialization.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">271</span> in <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #00ff00; text-decoration-color: #00ff00\">_open_file_like</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 268 </span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 269 </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">def</span> <span style=\"color: #00ff00; text-decoration-color: #00ff00\">_open_file_like</span>(name_or_buffer, mode): <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 270 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">if</span> _is_path(name_or_buffer): <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">❱ </span> 271 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">return</span> _open_file(name_or_buffer, mode) <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 272 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">else</span>: <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 273 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">if</span> <span style=\"color: #808000; text-decoration-color: #808000\">'w'</span> <span style=\"color: #ff00ff; text-decoration-color: #ff00ff\">in</span> mode: <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 274 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">return</span> _open_buffer_writer(name_or_buffer) <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/torch/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">serialization.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">252</span> in <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #00ff00; text-decoration-color: #00ff00\">__init__</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 249 </span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 250 </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">class</span> <span style=\"color: #00ff00; text-decoration-color: #00ff00; text-decoration: underline\">_open_file</span>(_opener): <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 251 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">def</span> <span style=\"color: #00ff00; text-decoration-color: #00ff00\">__init__</span>(<span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>, name, mode): <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">❱ </span> 252 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ </span><span style=\"color: #00ffff; text-decoration-color: #00ffff\">super</span>().<span style=\"color: #00ff00; text-decoration-color: #00ff00\">__init__</span>(<span style=\"color: #00ffff; text-decoration-color: #00ffff\">open</span>(name, mode)) <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 253 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ </span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 254 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">def</span> <span style=\"color: #00ff00; text-decoration-color: #00ff00\">__exit__</span>(<span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>, *args): <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 255 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ </span><span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>.file_like.close() <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">╰──────────────────────────────────────────────────────────────────────────────────────────────────╯</span>\n",
- "<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-weight: bold\">FileNotFoundError: </span><span style=\"font-weight: bold\">[</span>Errno <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span><span style=\"font-weight: bold\">]</span> No such file or directory: \n",
- "<span style=\"color: #008000; text-decoration-color: #008000\">'../decapoda-research/llama-7b-hf/pytorch_model-00001-of-00033.bin'</span>\n",
- "\n",
- "<span style=\"font-style: italic\">During handling of the above exception, another exception occurred:</span>\n",
- "\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">╭─────────────────────────────── </span><span style=\"color: #800000; text-decoration-color: #800000; font-weight: bold\">Traceback </span><span style=\"color: #bf7f7f; text-decoration-color: #bf7f7f; font-weight: bold\">(most recent call last)</span><span style=\"color: #800000; text-decoration-color: #800000\"> ────────────────────────────────╮</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">&lt;module&gt;</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">16</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">13 </span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">14 </span>tokenizer = LlamaTokenizer.from_pretrained(MODEL_NAME) <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">15 </span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">❱ </span>16 model = LlamaForCausalLM.from_pretrained( <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">17 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ </span>BASE_MODEL, <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">18 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ </span>load_in_8bit=<span style=\"color: #0000ff; text-decoration-color: #0000ff\">True</span>, <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">19 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ </span>torch_dtype=torch.float16, <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/transformers/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">modeling_utils.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">28</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #0000ff; text-decoration-color: #0000ff\">70</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">from_pretrained</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">2867 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ │ </span>mismatched_keys, <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">2868 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ │ </span>offload_index, <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">2869 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ │ </span>error_msgs, <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">❱ </span>2870 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ </span>) = <span style=\"color: #00ffff; text-decoration-color: #00ffff\">cls</span>._load_pretrained_model( <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">2871 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ │ </span>model, <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">2872 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ │ </span>state_dict, <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">2873 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ │ </span>loaded_state_dict_keys, <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"># XXX: rename?</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/transformers/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">modeling_utils.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">32</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #0000ff; text-decoration-color: #0000ff\">02</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">_load_pretrained_model</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">3199 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ │ </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"># Skip the load for shards that only contain disk-offloaded weights when</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">3200 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ │ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">if</span> shard_file <span style=\"color: #ff00ff; text-decoration-color: #ff00ff\">in</span> disk_only_shard_files: <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">3201 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ │ │ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">continue</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">❱ </span>3202 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ │ </span>state_dict = load_state_dict(shard_file) <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">3203 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ │ </span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">3204 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ │ </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"># Mistmatched keys contains tuples key/shape1/shape2 of weights in the c</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">3205 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ │ </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"># matching the weights in the model.</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/transformers/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">modeling_utils.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">46</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #0000ff; text-decoration-color: #0000ff\">2</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">load_state_dict</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 459 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">return</span> torch.load(checkpoint_file, map_location=<span style=\"color: #808000; text-decoration-color: #808000\">\"cpu\"</span>) <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 460 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">except</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">Exception</span> <span style=\"color: #0000ff; text-decoration-color: #0000ff\">as</span> e: <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 461 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">try</span>: <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">❱ </span> 462 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">with</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">open</span>(checkpoint_file) <span style=\"color: #0000ff; text-decoration-color: #0000ff\">as</span> f: <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 463 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ │ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">if</span> f.read(<span style=\"color: #0000ff; text-decoration-color: #0000ff\">7</span>) == <span style=\"color: #808000; text-decoration-color: #808000\">\"version\"</span>: <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 464 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ │ │ </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">raise</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">OSError</span>( <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 465 </span><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│ │ │ │ │ │ </span><span style=\"color: #808000; text-decoration-color: #808000\">\"You seem to have cloned a repository without having git-lfs ins</span> <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
- "<span style=\"color: #800000; text-decoration-color: #800000\">╰──────────────────────────────────────────────────────────────────────────────────────────────────╯</span>\n",
- "<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-weight: bold\">FileNotFoundError: </span><span style=\"font-weight: bold\">[</span>Errno <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span><span style=\"font-weight: bold\">]</span> No such file or directory: \n",
- "<span style=\"color: #008000; text-decoration-color: #008000\">'../decapoda-research/llama-7b-hf/pytorch_model-00001-of-00033.bin'</span>\n",
- "</pre>\n"
- ],
- "text/plain": [
- "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2;33m/opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/transformers/\u001b[0m\u001b[1;33mmodeling_utils.py\u001b[0m:\u001b[94m45\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[94m9\u001b[0m in \u001b[92mload_state_dict\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 456 \u001b[0m\u001b[2m│ │ │ \u001b[0m) \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 457 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m safe_load_file(checkpoint_file) \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 458 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mtry\u001b[0m: \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 459 \u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m torch.load(checkpoint_file, map_location=\u001b[33m\"\u001b[0m\u001b[33mcpu\u001b[0m\u001b[33m\"\u001b[0m) \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 460 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mexcept\u001b[0m \u001b[96mException\u001b[0m \u001b[94mas\u001b[0m e: \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 461 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mtry\u001b[0m: \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 462 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mwith\u001b[0m \u001b[96mopen\u001b[0m(checkpoint_file) \u001b[94mas\u001b[0m f: \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2;33m/opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/torch/\u001b[0m\u001b[1;33mserialization.py\u001b[0m:\u001b[94m791\u001b[0m in \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[92mload\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 788 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mif\u001b[0m \u001b[33m'\u001b[0m\u001b[33mencoding\u001b[0m\u001b[33m'\u001b[0m \u001b[95mnot\u001b[0m \u001b[95min\u001b[0m pickle_load_args.keys(): \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 789 \u001b[0m\u001b[2m│ │ \u001b[0mpickle_load_args[\u001b[33m'\u001b[0m\u001b[33mencoding\u001b[0m\u001b[33m'\u001b[0m] = \u001b[33m'\u001b[0m\u001b[33mutf-8\u001b[0m\u001b[33m'\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 790 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 791 \u001b[2m│ \u001b[0m\u001b[94mwith\u001b[0m _open_file_like(f, \u001b[33m'\u001b[0m\u001b[33mrb\u001b[0m\u001b[33m'\u001b[0m) \u001b[94mas\u001b[0m opened_file: \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 792 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m _is_zipfile(opened_file): \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 793 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[2m# The zipfile reader is going to advance the current file position.\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 794 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[2m# If we want to actually tail call to torch.jit.load, we need to\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2;33m/opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/torch/\u001b[0m\u001b[1;33mserialization.py\u001b[0m:\u001b[94m271\u001b[0m in \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[92m_open_file_like\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 268 \u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 269 \u001b[0m\u001b[94mdef\u001b[0m \u001b[92m_open_file_like\u001b[0m(name_or_buffer, mode): \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 270 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mif\u001b[0m _is_path(name_or_buffer): \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 271 \u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m _open_file(name_or_buffer, mode) \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 272 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 273 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[33m'\u001b[0m\u001b[33mw\u001b[0m\u001b[33m'\u001b[0m \u001b[95min\u001b[0m mode: \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 274 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mreturn\u001b[0m _open_buffer_writer(name_or_buffer) \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2;33m/opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/torch/\u001b[0m\u001b[1;33mserialization.py\u001b[0m:\u001b[94m252\u001b[0m in \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[92m__init__\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 249 \u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 250 \u001b[0m\u001b[94mclass\u001b[0m \u001b[4;92m_open_file\u001b[0m(_opener): \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 251 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92m__init__\u001b[0m(\u001b[96mself\u001b[0m, name, mode): \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 252 \u001b[2m│ │ \u001b[0m\u001b[96msuper\u001b[0m().\u001b[92m__init__\u001b[0m(\u001b[96mopen\u001b[0m(name, mode)) \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 253 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 254 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92m__exit__\u001b[0m(\u001b[96mself\u001b[0m, *args): \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 255 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m.file_like.close() \u001b[31m│\u001b[0m\n",
- "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
- "\u001b[1;91mFileNotFoundError: \u001b[0m\u001b[1m[\u001b[0mErrno \u001b[1;36m2\u001b[0m\u001b[1m]\u001b[0m No such file or directory: \n",
- "\u001b[32m'../decapoda-research/llama-7b-hf/pytorch_model-00001-of-00033.bin'\u001b[0m\n",
- "\n",
- "\u001b[3mDuring handling of the above exception, another exception occurred:\u001b[0m\n",
- "\n",
- "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n",
- "\u001b[31m│\u001b[0m in \u001b[92m<module>\u001b[0m:\u001b[94m16\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m13 \u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m14 \u001b[0mtokenizer = LlamaTokenizer.from_pretrained(MODEL_NAME) \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m15 \u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m16 model = LlamaForCausalLM.from_pretrained( \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m17 \u001b[0m\u001b[2m│ \u001b[0mBASE_MODEL, \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m18 \u001b[0m\u001b[2m│ \u001b[0mload_in_8bit=\u001b[94mTrue\u001b[0m, \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m19 \u001b[0m\u001b[2m│ \u001b[0mtorch_dtype=torch.float16, \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2;33m/opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/transformers/\u001b[0m\u001b[1;33mmodeling_utils.py\u001b[0m:\u001b[94m28\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[94m70\u001b[0m in \u001b[92mfrom_pretrained\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m2867 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mmismatched_keys, \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m2868 \u001b[0m\u001b[2m│ │ │ │ \u001b[0moffload_index, \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m2869 \u001b[0m\u001b[2m│ │ │ │ \u001b[0merror_msgs, \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m2870 \u001b[2m│ │ │ \u001b[0m) = \u001b[96mcls\u001b[0m._load_pretrained_model( \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m2871 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mmodel, \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m2872 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mstate_dict, \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m2873 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mloaded_state_dict_keys, \u001b[2m# XXX: rename?\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2;33m/opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/transformers/\u001b[0m\u001b[1;33mmodeling_utils.py\u001b[0m:\u001b[94m32\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[94m02\u001b[0m in \u001b[92m_load_pretrained_model\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m3199 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[2m# Skip the load for shards that only contain disk-offloaded weights when\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m3200 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[94mif\u001b[0m shard_file \u001b[95min\u001b[0m disk_only_shard_files: \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m3201 \u001b[0m\u001b[2m│ │ │ │ │ \u001b[0m\u001b[94mcontinue\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m3202 \u001b[2m│ │ │ │ \u001b[0mstate_dict = load_state_dict(shard_file) \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m3203 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m3204 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[2m# Mistmatched keys contains tuples key/shape1/shape2 of weights in the c\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m3205 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[2m# matching the weights in the model.\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2;33m/opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/transformers/\u001b[0m\u001b[1;33mmodeling_utils.py\u001b[0m:\u001b[94m46\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[94m2\u001b[0m in \u001b[92mload_state_dict\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 459 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m torch.load(checkpoint_file, map_location=\u001b[33m\"\u001b[0m\u001b[33mcpu\u001b[0m\u001b[33m\"\u001b[0m) \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 460 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mexcept\u001b[0m \u001b[96mException\u001b[0m \u001b[94mas\u001b[0m e: \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 461 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mtry\u001b[0m: \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 462 \u001b[2m│ │ │ \u001b[0m\u001b[94mwith\u001b[0m \u001b[96mopen\u001b[0m(checkpoint_file) \u001b[94mas\u001b[0m f: \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 463 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[94mif\u001b[0m f.read(\u001b[94m7\u001b[0m) == \u001b[33m\"\u001b[0m\u001b[33mversion\u001b[0m\u001b[33m\"\u001b[0m: \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 464 \u001b[0m\u001b[2m│ │ │ │ │ \u001b[0m\u001b[94mraise\u001b[0m \u001b[96mOSError\u001b[0m( \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 465 \u001b[0m\u001b[2m│ │ │ │ │ │ \u001b[0m\u001b[33m\"\u001b[0m\u001b[33mYou seem to have cloned a repository without having git-lfs ins\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
- "\u001b[1;91mFileNotFoundError: \u001b[0m\u001b[1m[\u001b[0mErrno \u001b[1;36m2\u001b[0m\u001b[1m]\u001b[0m No such file or directory: \n",
- "\u001b[32m'../decapoda-research/llama-7b-hf/pytorch_model-00001-of-00033.bin'\u001b[0m\n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
  }
  ],
  "source": [
- "import torch\n",
- "from transformers import GenerationConfig, LlamaTokenizer, LlamaForCausalLM\n",
- "from peft import PeftModel, PeftConfig\n",
- "\n",
- "\n",
- "MODEL_NAME = \"../\"\n",
- "BASE_MODEL = \"../decapoda-research/llama-7b-hf\"\n",
- "# MODEL_NAME = f\"lora-alpaca/conversations/GPU/{MODEL_NAME}\"\n",
- "# MODEL_NAME = \"chainyo/alpaca-lora-7b\"\n",
- "# MODEL_NAME = \"decapoda-research/llama-7b-hf\"\n",
- "\n",
  "config = PeftConfig.from_pretrained(MODEL_NAME)\n",
  "\n",
  "tokenizer = LlamaTokenizer.from_pretrained(MODEL_NAME)\n",
@@ -280,25 +169,23 @@
  " torch_dtype=torch.float16,\n",
  " device_map=\"auto\",\n",
  ")\n",
- "\n",
- "# model = PeftModel.from_pretrained(model, MODEL_NAME)"
+ " \n",
+ "# model.eval()\n",
+ "# if torch.__version__ >= \"2\":\n",
+ "# model = torch.compile(model)"
  ]
  },
  {
- "cell_type": "code",
- "execution_count": null,
- "id": "71dfae0f",
+ "cell_type": "markdown",
+ "id": "3bb51813",
  "metadata": {},
- "outputs": [],
  "source": [
- "model.eval()\n",
- "if torch.__version__ >= \"2\":\n",
- " model = torch.compile(model)"
+ "## Generation Examples"
  ]
  },
  {
  "cell_type": "code",
- "execution_count": null,
+ "execution_count": 5,
  "id": "10372ae3",
  "metadata": {},
  "outputs": [],
@@ -313,12 +200,35 @@
  ")"
  ]
  },
+ {
+ "cell_type": "markdown",
+ "id": "153062cf",
+ "metadata": {},
+ "source": [
+ "### Example 1"
+ ]
+ },
  {
  "cell_type": "code",
- "execution_count": null,
+ "execution_count": 6,
  "id": "a84a4f9e",
  "metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+ "\n",
+ "### Instruction:\n",
+ "I have two pieces of apples and 3 pieces of oranges. How many pieces of fruits do I have?\n",
+ "\n",
+ "### Response:\n",
+ "I have two pieces of apples and 3 pieces of oranges. How many pieces of fruits do I have?\n",
+ "I have two pieces of\n"
+ ]
+ }
+ ],
  "source": [
  "instruction = \"I have two pieces of apples and 3 pieces of oranges. How many pieces of fruits do I have?\"\n",
  "input_ctxt = None # For some tasks, you can provide an input context to help the model generate a better response.\n",
@@ -340,39 +250,36 @@
  ]
  },
  {
- "cell_type": "code",
- "execution_count": null,
- "id": "65117ac7",
+ "cell_type": "markdown",
+ "id": "a4642cdb",
  "metadata": {},
- "outputs": [],
  "source": [
- "instruction = \"What is the capital city of Greece and with which countries does Greece border?\"\n",
- "input_ctxt = None # For some tasks, you can provide an input context to help the model generate a better response.\n",
- "\n",
- "prompt = generate_prompt(instruction, input_ctxt)\n",
- "input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids\n",
- "input_ids = input_ids.to(model.device)\n",
- "\n",
- "with torch.no_grad():\n",
- " outputs = model.generate(\n",
- " input_ids=input_ids,\n",
- " generation_config=generation_config,\n",
- " return_dict_in_generate=True,\n",
- " output_scores=True,\n",
- " )\n",
- "\n",
- "response = tokenizer.decode(outputs.sequences[0], skip_special_tokens=True)\n",
- "print(response)"
+ "### Example 2"
  ]
  },
  {
  "cell_type": "code",
- "execution_count": null,
- "id": "2ff7a5e5",
+ "execution_count": 7,
+ "id": "65117ac7",
  "metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+ "\n",
+ "### Instruction:\n",
+ "What is the capital city of Greece and with which countries does Greece border?\n",
+ "\n",
+ "### Response:\n",
+ "The capital city of Greece is Athens and it borders Turkey, Bulgaria, Macedonia, Albania, and the Aegean Sea.\n",
+ "\n"
+ ]
+ }
+ ],
  "source": [
- "instruction = \"How can I cook Adobo?\"\n",
+ "instruction = \"What is the capital city of Greece and with which countries does Greece border?\"\n",
  "input_ctxt = None # For some tasks, you can provide an input context to help the model generate a better response.\n",
  "\n",
  "prompt = generate_prompt(instruction, input_ctxt)\n",
@@ -392,39 +299,35 @@
  ]
  },
  {
- "cell_type": "code",
- "execution_count": null,
- "id": "b2b504da",
- "metadata": {},
- "outputs": [],
- "source": []
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "9cba7db1",
- "metadata": {},
- "outputs": [],
- "source": []
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "af3a477a",
+ "cell_type": "markdown",
+ "id": "79fc59d2",
  "metadata": {},
- "outputs": [],
- "source": []
+ "source": [
+ "### Example 3"
+ ]
  },
  {
  "cell_type": "code",
- "execution_count": null,
- "id": "44fdd7ee",
+ "execution_count": 8,
+ "id": "2ff7a5e5",
  "metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+ "\n",
+ "### Instruction:\n",
+ "How can I cook Adobo?\n",
+ "\n",
+ "### Response:\n",
+ "You can cook Adobo by adding garlic, soy sauce, vinegar, bay leaves, peppercorns, and water\n"
+ ]
+ }
+ ],
  "source": [
- "%%time\n",
- "\n",
- "instruction = \"Which are the tags of the following article: 'A year ago, Russia invaded Ukraine in a major escalation of the Russo-Ukrainian War, which had begun in 2014. The invasion has resulted in thousands of deaths, and instigated Europe's largest refugee crisis since World War II.\"\n",
+ "instruction = \"How can I cook Adobo?\"\n",
  "input_ctxt = None # For some tasks, you can provide an input context to help the model generate a better response.\n",
  "\n",
  "prompt = generate_prompt(instruction, input_ctxt)\n",
@@ -444,43 +347,35 @@
  ]
  },
  {
- "cell_type": "code",
- "execution_count": null,
- "id": "1f3a96aa",
+ "cell_type": "markdown",
+ "id": "3080312d",
  "metadata": {},
- "outputs": [],
  "source": [
- "%%time\n",
- "\n",
- "instruction = \"Translate the following text from English to Greek: 'My name is George. I am 22 years old and I live with my parents.'\"\n",
- "input_ctxt = None # For some tasks, you can provide an input context to help the model generate a better response.\n",
- "\n",
- "prompt = generate_prompt(instruction, input_ctxt)\n",
- "input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids\n",
- "input_ids = input_ids.to(model.device)\n",
- "\n",
- "with torch.no_grad():\n",
- " outputs = model.generate(\n",
- " input_ids=input_ids,\n",
- " generation_config=generation_config,\n",
- " return_dict_in_generate=True,\n",
- " output_scores=True,\n",
- " )\n",
- "\n",
- "response = tokenizer.decode(outputs.sequences[0], skip_special_tokens=True)\n",
- "print(response)"
+ "### Example 4"
  ]
  },
  {
  "cell_type": "code",
- "execution_count": null,
- "id": "b87f4120",
+ "execution_count": 9,
+ "id": "b0e4b6f5",
  "metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+ "\n",
+ "### Instruction:\n",
+ "Which are the tags of the following article: 'A year ago, Russia invaded Ukraine in a major escalation of the Russo-Ukrainian War, which had begun in 2014. The invasion has resulted in thousands of deaths, and instigated Europe's largest refugee crisis since World War II.'?\n",
+ "\n",
+ "### Response:\n",
+ "The tags of the following article: 'A year ago, Russia invaded Ukraine in a major escalation of the Russo-Ukrainian\n"
+ ]
+ }
+ ],
  "source": [
- "%%time\n",
- "\n",
- "instruction = \"Ποιά είναι η πρωτεύουσα της Ελλάδας?\"\n",
+ "instruction = \"Which are the tags of the following article: 'A year ago, Russia invaded Ukraine in a major escalation of the Russo-Ukrainian War, which had begun in 2014. The invasion has resulted in thousands of deaths, and instigated Europe's largest refugee crisis since World War II.'?\"\n",
  "input_ctxt = None # For some tasks, you can provide an input context to help the model generate a better response.\n",
  "\n",
  "prompt = generate_prompt(instruction, input_ctxt)\n",
@@ -500,111 +395,55 @@
  ]
  },
  {
- "cell_type": "code",
- "execution_count": null,
- "id": "520edf24",
- "metadata": {},
- "outputs": [],
- "source": []
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "a2fdfc6b",
+ "cell_type": "markdown",
+ "id": "c6c7a56b",
  "metadata": {},
- "outputs": [],
  "source": [
- "%%time\n",
- "\n",
- "instruction = \"Translate the following text from Italian to English: 'Alla vigilia della sfida contro l'Inter, Luciano Spalletti risponde alle recenti parole del presidente De Laurentiis che ha messo in dubbio il suo futuro sulla panchina del Napoli.'\"\n",
- "input_ctxt = None # For some tasks, you can provide an input context to help the model generate a better response.\n",
- "\n",
- "prompt = generate_prompt(instruction, input_ctxt)\n",
- "input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids\n",
- "input_ids = input_ids.to(model.device)\n",
- "\n",
- "with torch.no_grad():\n",
- " outputs = model.generate(\n",
- " input_ids=input_ids,\n",
- " generation_config=generation_config,\n",
- " return_dict_in_generate=True,\n",
- " output_scores=True,\n",
- " )\n",
- "\n",
- "response = tokenizer.decode(outputs.sequences[0], skip_special_tokens=True)\n",
- "print(response)"
+ "## Let's Load the Fine-Tuned version"
  ]
  },
  {
  "cell_type": "code",
- "execution_count": null,
- "id": "6b87cfde",
- "metadata": {
- "scrolled": true
- },
+ "execution_count": 10,
+ "id": "9cba7db1",
+ "metadata": {},
  "outputs": [],
  "source": [
- "%%time\n",
- "\n",
- "instruction = \"What is the capital city of Greece and with which countries does Greece border?\"\n",
- "input_ctxt = None # For some tasks, you can provide an input context to help the model generate a better response.\n",
- "\n",
- "prompt = generate_prompt(instruction, input_ctxt)\n",
- "input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids\n",
- "input_ids = input_ids.to(model.device)\n",
- "\n",
- "with torch.no_grad():\n",
- " outputs = model.generate(\n",
- " input_ids=input_ids,\n",
- " generation_config=generation_config,\n",
- " return_dict_in_generate=True,\n",
- " output_scores=True,\n",
- " )\n",
- "\n",
- "response = tokenizer.decode(outputs.sequences[0], skip_special_tokens=True)\n",
- "print(response)"
+ "model = PeftModel.from_pretrained(model, MODEL_NAME)"
  ]
  },
  {
- "cell_type": "code",
- "execution_count": null,
- "id": "08f25326",
+ "cell_type": "markdown",
+ "id": "5d664394",
  "metadata": {},
- "outputs": [],
  "source": [
- "%%time\n",
- "\n",
- "instruction = \"I have two oranges and 3 apples. How many pieces of fruits I have in total?\"\n",
- "input_ctxt = \"Question Answering\" # For some tasks, you can provide an input context to help the model generate a better response.\n",
- "\n",
- "prompt = generate_prompt(instruction, input_ctxt)\n",
- "input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids\n",
- "input_ids = input_ids.to(model.device)\n",
- "\n",
- "with torch.no_grad():\n",
- " outputs = model.generate(\n",
- " input_ids=input_ids,\n",
- " generation_config=generation_config,\n",
- " return_dict_in_generate=True,\n",
- " output_scores=True,\n",
- " )\n",
- "\n",
- "response = tokenizer.decode(outputs.sequences[0], skip_special_tokens=True)\n",
- "print(response)"
+ "### Example 1"
  ]
  },
  {
  "cell_type": "code",
- "execution_count": null,
- "id": "218815c4",
- "metadata": {
- "scrolled": true
- },
- "outputs": [],
+ "execution_count": 11,
+ "id": "af3a477a",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+ "\n",
+ "### Instruction:\n",
+ "I have two pieces of apples and 3 pieces of oranges. How many pieces of fruits do I have?\n",
+ "\n",
+ "### Response:\n",
+ "If you have 2 pieces of apples and 3 pieces of oranges, then you have 5 pieces of fruits.\n",
+ "\n",
+ "###\n"
+ ]
+ }
+ ],
  "source": [
- "%%time\n",
- "\n",
- "instruction = \"Which are the tags of the following article: 'Prozess in Winterthur: Handwerker (69) wegen uraltem Sex-Heftli vor Gericht. Ein 69-jähriger Handwerker stand in Winterthur vor Gericht, weil bei ihm ein 35 Jahre altes Heftchen mit explizitem Inhalt gefunden wurde. Die Anklage scheiterte. Die Polizei führte bei einem Winterthurer eine Hausdurchsuchung durch, nachdem US-Behörden den Schweizer Behörden einen Hinweis auf ein verbotenes pornografisches Bild gaben. Allerdings fand sich auf den elektronischen Geräten des Mannes nicht der kleinste Hinweis auf weitere Bilder oder Videos im Zusammenhang mit Kinderpornografie, Sex mit Tieren oder mit Gewaltdarstellungen. Das Strafverfahren wurde eingestellt. «Jung und froh mit nacktem Po». Aber: Bei der Hausdurchsuchung stellten die Beamten ein 35 Jahre altes Sexheftli des Orion-Verlags in den Lagerräumen des Handwerkers sicher, wie der «Tages-Anzeiger» berichtet. Das Heftchen «Jung und froh mit nacktem Po» enthielt auf mehr als zehn Seiten ganzseitige Fotos nackter Mädchen und Jungen im Alter von drei bis fünfzehn Jahren.'?\"\n",
+ "instruction = \"I have two pieces of apples and 3 pieces of oranges. How many pieces of fruits do I have?\"\n",
  "input_ctxt = None # For some tasks, you can provide an input context to help the model generate a better response.\n",
  "\n",
  "prompt = generate_prompt(instruction, input_ctxt)\n",
@@ -624,44 +463,36 @@
  ]
  },
  {
- "cell_type": "code",
- "execution_count": null,
- "id": "4fdd7591",
+ "cell_type": "markdown",
+ "id": "a07d76da",
  "metadata": {},
- "outputs": [],
  "source": [
- "%%time\n",
- "\n",
- "instruction = \"Which are the tags of the following article: 'For those now grappling with Alzheimer’s, lecanemab holds out the promise of slowing the disease’s progress. Are the modest benefits worth the risks? (C1)\\nAfter many decades of little or no progress in treating the dementia associated with Alzheimer’s, a new drug now offers hope to patients and caregivers. Lecanemab, announced late last month, was found in clinical trials to slow cognitive decline in early-stage Alzheimer’s patients. “It’s an extremely encouraging result,” says Dr. David Wolk, co-director of the University of Pennsylvania’s Penn Memory Center'?\"\n",
- "input_ctxt = \"Question Answering\" # For some tasks, you can provide an input context to help the model generate a better response.\n",
- "\n",
- "prompt = generate_prompt(instruction, input_ctxt)\n",
- "input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids\n",
- "input_ids = input_ids.to(model.device)\n",
- "\n",
- "with torch.no_grad():\n",
- " outputs = model.generate(\n",
- " input_ids=input_ids,\n",
- " generation_config=generation_config,\n",
- " return_dict_in_generate=True,\n",
- " output_scores=True,\n",
- " )\n",
- "\n",
- "response = tokenizer.decode(outputs.sequences[0], skip_special_tokens=True)\n",
- "print(response)"
+ "### Example 2"
  ]
  },
  {
  "cell_type": "code",
- "execution_count": null,
- "id": "0ff82833",
+ "execution_count": 12,
+ "id": "5f59b3f4",
  "metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+ "\n",
+ "### Instruction:\n",
+ "What is the capital city of Greece and with which countries does Greece border?\n",
+ "\n",
+ "### Response:\n",
+ "Greece borders Albania, Bulgaria, Turkey, Macedonia, and the Aegean Sea. The capital of Greece is Athens.\n"
+ ]
+ }
+ ],
  "source": [
- "%%time\n",
- "\n",
- "instruction = \"Which characteristic is least likely to be affected by an individual's environment? (A) height (B) weight (C) skin color (D) eye color\"\n",
- "input_ctxt = \"Tags\" # For some tasks, you can provide an input context to help the model generate a better response.\n",
+ "instruction = \"What is the capital city of Greece and with which countries does Greece border?\"\n",
+ "input_ctxt = None # For some tasks, you can provide an input context to help the model generate a better response.\n",
  "\n",
  "prompt = generate_prompt(instruction, input_ctxt)\n",
  "input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids\n",
@@ -680,44 +511,42 @@
  ]
  },
  {
- "cell_type": "code",
- "execution_count": null,
- "id": "840e70c5",
+ "cell_type": "markdown",
+ "id": "ecdb6550",
  "metadata": {},
- "outputs": [],
  "source": [
- "%%time\n",
- "\n",
- "instruction = \"A student mixed some yellow sulfur powder with some iron filings. She was able to take the iron out of the sulfur by using a magnet. She then remixed the iron and sulfur in a test tube and heated it. After it cooled, she removed the substance from the test tube but could not separate the iron from the sulfur using the magnet. Which type of change occurred in the material when it was heated? (A) a physical change because the iron reacted with the sulfur (B) a chemical change because a new substance was formed (C) a physical change because a compound was formed (D) a chemical change because a magnet had to be used\"\n",
- "input_ctxt = None # For some tasks, you can provide an input context to help the model generate a better response.\n",
- "\n",
- "prompt = generate_prompt(instruction, input_ctxt)\n",
- "input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids\n",
- "input_ids = input_ids.to(model.device)\n",
- "\n",
- "with torch.no_grad():\n",
- " outputs = model.generate(\n",
- " input_ids=input_ids,\n",
- " generation_config=generation_config,\n",
- " return_dict_in_generate=True,\n",
- " output_scores=True,\n",
- " )\n",
- "\n",
- "response = tokenizer.decode(outputs.sequences[0], skip_special_tokens=True)\n",
- "print(response)"
+ "### Example 3"
  ]
  },
  {
  "cell_type": "code",
- "execution_count": null,
- "id": "54139a84",
+ "execution_count": 13,
+ "id": "5ac57290",
  "metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+ "\n",
+ "### Instruction:\n",
+ "How can I cook Adobo?\n",
+ "\n",
+ "### Response:\n",
+ "Here's a recipe for adobo:\n",
+ "\n",
+ "\\begin{code}\n",
+ "\n",
+ "\\begin{pre}\n",
+ "\n",
+ "\\end{code\n"
+ ]
+ }
+ ],
  "source": [
- "%%time\n",
- "\n",
- "instruction = \"Translate the following from English to Greek: 'My name is George. I am 22 years old and I live with my parents.'\"\n",
- "input_ctxt = \"Question Answering\" # For some tasks, you can provide an input context to help the model generate a better response.\n",
+ "instruction = \"How can I cook Adobo?\"\n",
+ "input_ctxt = None # For some tasks, you can provide an input context to help the model generate a better response.\n",
  "\n",
  "prompt = generate_prompt(instruction, input_ctxt)\n",
  "input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids\n",
@@ -736,43 +565,37 @@
  ]
  },
  {
- "cell_type": "code",
- "execution_count": null,
- "id": "c88de39f",
+ "cell_type": "markdown",
+ "id": "00859e35",
  "metadata": {},
- "outputs": [],
  "source": [
- "%%time\n",
- "\n",
- "instruction = \"Translate the following from English to Spanish: 'My name is George. I am 22 years old and I live with my parents.'\"\n",
- "input_ctxt = None # For some tasks, you can provide an input context to help the model generate a better response.\n",
- "\n",
- "prompt = generate_prompt(instruction, input_ctxt)\n",
- "input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids\n",
- "input_ids = input_ids.to(model.device)\n",
- "\n",
- "with torch.no_grad():\n",
- " outputs = model.generate(\n",
- " input_ids=input_ids,\n",
- " generation_config=generation_config,\n",
- " return_dict_in_generate=True,\n",
- " output_scores=True,\n",
- " )\n",
- "\n",
- "response = tokenizer.decode(outputs.sequences[0], skip_special_tokens=True)\n",
- "print(response)"
+ "### Example 4"
  ]
  },
  {
  "cell_type": "code",
- "execution_count": null,
- "id": "fbccda31",
+ "execution_count": 14,
+ "id": "801165b6",
  "metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
+ "\n",
+ "### Instruction:\n",
+ "Which are the tags of the following article: 'A year ago, Russia invaded Ukraine in a major escalation of the Russo-Ukrainian War, which had begun in 2014. The invasion has resulted in thousands of deaths, and instigated Europe's largest refugee crisis since World War II.'?\n",
+ "\n",
+ "### Response:\n",
+ "\n",
+ "### Instruction:\n",
+ "Which are the tags of the following article: 'A year ago, Russia invaded Ukraine in a major escal\n"
+ ]
+ }
+ ],
  "source": [
- "%%time\n",
- "\n",
- "instruction = \"Translate the following from English to Tagalog: 'I love you. How is your day? Have you eaten?'\"\n",
+ "instruction = \"Which are the tags of the following article: 'A year ago, Russia invaded Ukraine in a major escalation of the Russo-Ukrainian War, which had begun in 2014. The invasion has resulted in thousands of deaths, and instigated Europe's largest refugee crisis since World War II.'?\"\n",
  "input_ctxt = None # For some tasks, you can provide an input context to help the model generate a better response.\n",
  "\n",
  "prompt = generate_prompt(instruction, input_ctxt)\n",
@@ -791,14 +614,6 @@
  "print(response)"
  ]
  },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "aa6e355b",
- "metadata": {},
- "outputs": [],
- "source": []
- },
  {
  "cell_type": "code",
  "execution_count": null,