{ "cells": [ { "cell_type": "markdown", "id": "15908f0e", "metadata": {}, "source": [ "## Import Packages" ] }, { "cell_type": "code", "execution_count": 1, "id": "94f0ccef", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "===================================BUG REPORT===================================\n", "Welcome to bitsandbytes. For bug reports, please run\n", "\n", "python -m bitsandbytes\n", "\n", " and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n", "================================================================================\n", "bin /opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/bitsandbytes/libbitsandbytes_cuda112_nocublaslt.so\n", "CUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching in backup paths...\n", "CUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so\n", "CUDA SETUP: Highest compute capability among GPUs detected: 7.0\n", "CUDA SETUP: Detected CUDA version 112\n", "CUDA SETUP: Loading binary /opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/bitsandbytes/libbitsandbytes_cuda112_nocublaslt.so...\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: /opt/conda/envs/media-reco-env-3-8 did not contain ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] as expected! Searching further paths...\n", " warn(msg)\n", "/opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/usr/local/nvidia/lib'), PosixPath('/usr/local/nvidia/lib64')}\n", " warn(msg)\n", "/opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: /usr/local/nvidia/lib:/usr/local/nvidia/lib64 did not contain ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] as expected! Searching further paths...\n", " warn(msg)\n", "/opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('//162.21.251.11'), PosixPath('http'), PosixPath('8080')}\n", " warn(msg)\n", "/opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('module'), PosixPath('//matplotlib_inline.backend_inline')}\n", " warn(msg)\n", "/opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: Found duplicate ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] files: {PosixPath('/usr/local/cuda/lib64/libcudart.so'), PosixPath('/usr/local/cuda/lib64/libcudart.so.11.0')}.. We'll flip a coin and try one of these, in order to fail forward.\n", "Either way, this might cause trouble in the future:\n", "If you get `CUDA error: invalid device function` errors, the above might be the cause and the solution is to make sure only one ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] in the paths that we search based on your env.\n", " warn(msg)\n", "/opt/conda/envs/media-reco-env-3-8/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: Compute capability < 7.5 detected! Only slow 8-bit matmul is supported for your GPU!\n", " warn(msg)\n" ] } ], "source": [ "import os\n", "os.chdir(\"..\")\n", "\n", "import torch\n", "from transformers import GenerationConfig, LlamaTokenizer, LlamaForCausalLM\n", "from peft import PeftModel, PeftConfig" ] }, { "cell_type": "markdown", "id": "58b927f4", "metadata": {}, "source": [ "## Utilities" ] }, { "cell_type": "code", "execution_count": 2, "id": "9837afb7", "metadata": {}, "outputs": [], "source": [ "def generate_prompt(instruction: str, input_ctxt: str = None) -> str:\n", " if input_ctxt:\n", " return f\"\"\"Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n", "\n", "### Instruction:\n", "{instruction}\n", "\n", "### Input:\n", "{input_ctxt}\n", "\n", "### Response:\"\"\"\n", " else:\n", " return f\"\"\"Below is an instruction that describes a task. Write a response that appropriately completes the request.\n", "\n", "### Instruction:\n", "{instruction}\n", "\n", "### Response:\"\"\"" ] }, { "cell_type": "markdown", "id": "b37f5f57", "metadata": {}, "source": [ "## Configs" ] }, { "cell_type": "code", "execution_count": 3, "id": "b53f6c18", "metadata": {}, "outputs": [], "source": [ "MODEL_NAME = \".\"\n", "BASE_MODEL = \"decapoda-research/llama-7b-hf\"\n", "LOAD_FINETUNED = False" ] }, { "cell_type": "markdown", "id": "ec8111a9", "metadata": {}, "source": [ "## Load Model & Tokenizer" ] }, { "cell_type": "code", "execution_count": 4, "id": "1cb5103c", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "4bef02b4785d497da6871876fe12e757", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Loading checkpoint shards: 0%| | 0/33 [00:00= \"2\":\n", "# model = torch.compile(model)" ] }, { "cell_type": "markdown", "id": "d265647e", "metadata": {}, "source": [ "## Generation Examples" ] }, { "cell_type": "code", "execution_count": 5, "id": "10372ae3", "metadata": {}, "outputs": [], "source": [ "generation_config = GenerationConfig(\n", " temperature=0.2,\n", " top_p=0.95,\n", " top_k=40,\n", " num_beams=4,\n", " max_new_tokens=40,\n", " repetition_penalty=1.7,\n", ")" ] }, { "cell_type": "markdown", "id": "bf54befd", "metadata": {}, "source": [ "## Examples with Base (decapoda-research/llama-7b-hf) model" ] }, { "cell_type": "markdown", "id": "1f6e7df1", "metadata": {}, "source": [ "### Example 1" ] }, { "cell_type": "code", "execution_count": 6, "id": "a84a4f9e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n", "\n", "### Instruction:\n", "I have two pieces of apples and 3 pieces of oranges. How many pieces of fruits do I have?\n", "\n", "### Response:\n", "I have 2 pieces of apples and 3 pieces of oranges.\n", "\n", "### Instruction:\n", "I have 2 pieces of apples and 3 pieces of oranges\n" ] } ], "source": [ "instruction = \"I have two pieces of apples and 3 pieces of oranges. How many pieces of fruits do I have?\"\n", "input_ctxt = None # For some tasks, you can provide an input context to help the model generate a better response.\n", "\n", "prompt = generate_prompt(instruction, input_ctxt)\n", "input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids\n", "input_ids = input_ids.to(model.device)\n", "\n", "with torch.no_grad():\n", " outputs = model.generate(\n", " input_ids=input_ids,\n", " generation_config=generation_config,\n", " return_dict_in_generate=True,\n", " output_scores=True,\n", " )\n", "\n", "response = tokenizer.decode(outputs.sequences[0], skip_special_tokens=True)\n", "print(response)" ] }, { "cell_type": "markdown", "id": "8143ca1f", "metadata": {}, "source": [ "### Example 2" ] }, { "cell_type": "code", "execution_count": 7, "id": "65117ac7", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n", "\n", "### Instruction:\n", "What is the capital city of Greece and with which countries does Greece border?\n", "\n", "### Response:\n", "Athens is the capital city of Greece and it borders Albania, Macedonia, Bulgaria, and Turkey.\n", "\n", "### Instruction:\n", "What is the capital city of\n" ] } ], "source": [ "instruction = \"What is the capital city of Greece and with which countries does Greece border?\"\n", "input_ctxt = None # For some tasks, you can provide an input context to help the model generate a better response.\n", "\n", "prompt = generate_prompt(instruction, input_ctxt)\n", "input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids\n", "input_ids = input_ids.to(model.device)\n", "\n", "with torch.no_grad():\n", " outputs = model.generate(\n", " input_ids=input_ids,\n", " generation_config=generation_config,\n", " return_dict_in_generate=True,\n", " output_scores=True,\n", " )\n", "\n", "response = tokenizer.decode(outputs.sequences[0], skip_special_tokens=True)\n", "print(response)" ] }, { "cell_type": "markdown", "id": "447f75f9", "metadata": {}, "source": [ "### Example 3" ] }, { "cell_type": "code", "execution_count": 8, "id": "2ff7a5e5", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n", "\n", "### Instruction:\n", "Como cocinar supa de pescado?\n", "\n", "### Response:\n", "Cocinar supa de pescado es muy fácil. Primero, tienes que cortar el pescado en trozos pequeños. Luego, ponlo en\n" ] } ], "source": [ "instruction = \"Como cocinar supa de pescado?\"\n", "input_ctxt = None # For some tasks, you can provide an input context to help the model generate a better response.\n", "\n", "prompt = generate_prompt(instruction, input_ctxt)\n", "input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids\n", "input_ids = input_ids.to(model.device)\n", "\n", "with torch.no_grad():\n", " outputs = model.generate(\n", " input_ids=input_ids,\n", " generation_config=generation_config,\n", " return_dict_in_generate=True,\n", " output_scores=True,\n", " )\n", "\n", "response = tokenizer.decode(outputs.sequences[0], skip_special_tokens=True)\n", "print(response)" ] }, { "cell_type": "markdown", "id": "c0f1fc51", "metadata": {}, "source": [ "### Example 4" ] }, { "cell_type": "code", "execution_count": 9, "id": "4073cb6d", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n", "\n", "### Instruction:\n", "Which are the tags of the following article: 'A year ago, Russia invaded Ukraine in a major escalation of the Russo-Ukrainian War, which had begun in 2014. The invasion has resulted in thousands of deaths, and instigated Europe's largest refugee crisis since World War II.'?\n", "\n", "### Response:\n", "The tags of the following article: 'A year ago, Russia invaded Ukraine in a major escalation of the Russo-Ukrainian War, which had begun in 2\n" ] } ], "source": [ "instruction = \"Which are the tags of the following article: 'A year ago, Russia invaded Ukraine in a major escalation of the Russo-Ukrainian War, which had begun in 2014. The invasion has resulted in thousands of deaths, and instigated Europe's largest refugee crisis since World War II.'?\"\n", "input_ctxt = None # For some tasks, you can provide an input context to help the model generate a better response.\n", "\n", "prompt = generate_prompt(instruction, input_ctxt)\n", "input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids\n", "input_ids = input_ids.to(model.device)\n", "\n", "with torch.no_grad():\n", " outputs = model.generate(\n", " input_ids=input_ids,\n", " generation_config=generation_config,\n", " return_dict_in_generate=True,\n", " output_scores=True,\n", " )\n", "\n", "response = tokenizer.decode(outputs.sequences[0], skip_special_tokens=True)\n", "print(response)" ] }, { "cell_type": "markdown", "id": "180873e7", "metadata": {}, "source": [ "### Example 5" ] }, { "cell_type": "code", "execution_count": 10, "id": "10b6036b", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n", "\n", "### Instruction:\n", "Ποιά είναι η μεγαλύτερη πόλη της Ελλάδας?\n", "\n", "### Response:\n", "Η πόλη ππππππππππππππππππππππππππππππππ\n" ] } ], "source": [ "instruction = \"Ποιά είναι η μεγαλύτερη πόλη της Ελλάδας?\"\n", "input_ctxt = None # For some tasks, you can provide an input context to help the model generate a better response.\n", "\n", "prompt = generate_prompt(instruction, input_ctxt)\n", "input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids\n", "input_ids = input_ids.to(model.device)\n", "\n", "with torch.no_grad():\n", " outputs = model.generate(\n", " input_ids=input_ids,\n", " generation_config=generation_config,\n", " return_dict_in_generate=True,\n", " output_scores=True,\n", " )\n", "\n", "response = tokenizer.decode(outputs.sequences[0], skip_special_tokens=True)\n", "print(response)" ] }, { "cell_type": "markdown", "id": "64bc9354", "metadata": {}, "source": [ "## Examples with Fine-Tuned model" ] }, { "cell_type": "markdown", "id": "df08ac5a", "metadata": {}, "source": [ "## Let's Load the Fine-Tuned version" ] }, { "cell_type": "code", "execution_count": 11, "id": "9cba7db1", "metadata": {}, "outputs": [], "source": [ "model = PeftModel.from_pretrained(model, MODEL_NAME)" ] }, { "cell_type": "markdown", "id": "5bc70c31", "metadata": {}, "source": [ "### Example 1" ] }, { "cell_type": "code", "execution_count": 12, "id": "af3a477a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n", "\n", "### Instruction:\n", "I have two pieces of apples and 3 pieces of oranges. How many pieces of fruits do I have?\n", "\n", "### Response:as you have 2 pieces of apples and 3 pieces of oranges, you have a total of 5 pieces of fruits. If you have 2 pieces of apples and \n" ] } ], "source": [ "instruction = \"I have two pieces of apples and 3 pieces of oranges. How many pieces of fruits do I have?\"\n", "input_ctxt = None # For some tasks, you can provide an input context to help the model generate a better response.\n", "\n", "prompt = generate_prompt(instruction, input_ctxt)\n", "input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids\n", "input_ids = input_ids.to(model.device)\n", "\n", "with torch.no_grad():\n", " outputs = model.generate(\n", " input_ids=input_ids,\n", " generation_config=generation_config,\n", " return_dict_in_generate=True,\n", " output_scores=True,\n", " )\n", "\n", "response = tokenizer.decode(outputs.sequences[0], skip_special_tokens=True)\n", "print(response)" ] }, { "cell_type": "markdown", "id": "622b3c0a", "metadata": {}, "source": [ "### Example 2" ] }, { "cell_type": "code", "execution_count": 17, "id": "eab112ae", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n", "\n", "### Instruction:\n", "What is the capital city of Greece and with which countries does Greece border?\n", "\n", "### Response:y\n", "Athens is the capital of Greece and it borders Albania, Bulgaria, Turkey, Macedonia, and the Aegean Sea.\n", "\n", "### Instruction:\n", "\n" ] } ], "source": [ "instruction = \"What is the capital city of Greece and with which countries does Greece border?\"\n", "input_ctxt = None # For some tasks, you can provide an input context to help the model generate a better response.\n", "\n", "prompt = generate_prompt(instruction, input_ctxt)\n", "input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids\n", "input_ids = input_ids.to(model.device)\n", "\n", "with torch.no_grad():\n", " outputs = model.generate(\n", " input_ids=input_ids,\n", " generation_config=generation_config,\n", " return_dict_in_generate=True,\n", " output_scores=True,\n", " )\n", "\n", "response = tokenizer.decode(outputs.sequences[0], skip_special_tokens=True)\n", "print(response)" ] }, { "cell_type": "markdown", "id": "fb0e6d9e", "metadata": {}, "source": [ "### Example 3" ] }, { "cell_type": "code", "execution_count": 14, "id": "df571d56", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n", "\n", "### Instruction:\n", "Como cocinar supa de pescado?\n", "\n", "### Response:así es como cocinar supa de pescado: 1. Caliente el aceite en una sartén. 2. Agrega la cebolla, el\n" ] } ], "source": [ "instruction = \"Como cocinar supa de pescado?\"\n", "input_ctxt = None # For some tasks, you can provide an input context to help the model generate a better response.\n", "\n", "prompt = generate_prompt(instruction, input_ctxt)\n", "input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids\n", "input_ids = input_ids.to(model.device)\n", "\n", "with torch.no_grad():\n", " outputs = model.generate(\n", " input_ids=input_ids,\n", " generation_config=generation_config,\n", " return_dict_in_generate=True,\n", " output_scores=True,\n", " )\n", "\n", "response = tokenizer.decode(outputs.sequences[0], skip_special_tokens=True)\n", "print(response)" ] }, { "cell_type": "markdown", "id": "8d3aa375", "metadata": {}, "source": [ "### Example 4" ] }, { "cell_type": "code", "execution_count": 15, "id": "4975198b", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n", "\n", "### Instruction:\n", "Which are the tags of the following article: 'A year ago, Russia invaded Ukraine in a major escalation of the Russo-Ukrainian War, which had begun in 2014. The invasion has resulted in thousands of deaths, and instigated Europe's largest refugee crisis since World War II.'?\n", "\n", "### Response:english, russia, ukraine, war, europe, refugee\n", "\n", "### Instruction:\n", "Thank you for your help!\n" ] } ], "source": [ "instruction = \"Which are the tags of the following article: 'A year ago, Russia invaded Ukraine in a major escalation of the Russo-Ukrainian War, which had begun in 2014. The invasion has resulted in thousands of deaths, and instigated Europe's largest refugee crisis since World War II.'?\"\n", "input_ctxt = None # For some tasks, you can provide an input context to help the model generate a better response.\n", "\n", "prompt = generate_prompt(instruction, input_ctxt)\n", "input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids\n", "input_ids = input_ids.to(model.device)\n", "\n", "with torch.no_grad():\n", " outputs = model.generate(\n", " input_ids=input_ids,\n", " generation_config=generation_config,\n", " return_dict_in_generate=True,\n", " output_scores=True,\n", " )\n", "\n", "response = tokenizer.decode(outputs.sequences[0], skip_special_tokens=True)\n", "print(response)" ] }, { "cell_type": "markdown", "id": "acd8e768", "metadata": {}, "source": [ "### Example 5" ] }, { "cell_type": "code", "execution_count": 16, "id": "b05c3934", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n", "\n", "### Instruction:\n", "Ποιά είναι η μεγαλύτερη πόλη της Ελλάδας?\n", "\n", "### Response:\n", "Η Αθήνα είναι η μεγαλύτερερερερερερερερ\n" ] } ], "source": [ "instruction = \"Ποιά είναι η μεγαλύτερη πόλη της Ελλάδας?\"\n", "input_ctxt = None # For some tasks, you can provide an input context to help the model generate a better response.\n", "\n", "prompt = generate_prompt(instruction, input_ctxt)\n", "input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids\n", "input_ids = input_ids.to(model.device)\n", "\n", "with torch.no_grad():\n", " outputs = model.generate(\n", " input_ids=input_ids,\n", " generation_config=generation_config,\n", " return_dict_in_generate=True,\n", " output_scores=True,\n", " )\n", "\n", "response = tokenizer.decode(outputs.sequences[0], skip_special_tokens=True)\n", "print(response)" ] }, { "cell_type": "code", "execution_count": null, "id": "569838c8", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python [conda env:media-reco-env-3-8]", "language": "python", "name": "conda-env-media-reco-env-3-8-py" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.0" } }, "nbformat": 4, "nbformat_minor": 5 }