Spaces:
Runtime error
Runtime error
import torch | |
from peft import PeftModel | |
import transformers | |
import gradio as gr | |
assert ( | |
"LlamaTokenizer" in transformers._import_structure["models.llama"] | |
), "LLaMA is now in HuggingFace's main branch.\nPlease reinstall it: pip uninstall transformers && pip install git+https://github.com/huggingface/transformers.git" | |
from transformers import LlamaTokenizer, LlamaForCausalLM, GenerationConfig | |
tokenizer = LlamaTokenizer.from_pretrained("decapoda-research/llama-7b-hf") | |
BASE_MODEL = "TheBloke/vicuna-7B-1.1-HF" | |
LORA_WEIGHTS = "RinInori/vicuna_finetuned_6_sentiments" #Fine-tuned Alpaca model for sentiment analysis | |
if torch.cuda.is_available(): | |
device = "cuda" | |
else: | |
device = "cpu" | |
try: | |
if torch.backends.mps.is_available(): | |
device = "mps" | |
except: | |
pass | |
if device == "cuda": | |
model = LlamaForCausalLM.from_pretrained( | |
BASE_MODEL, | |
load_in_8bit=False, | |
torch_dtype=torch.float16, | |
device_map="auto", | |
) | |
model = PeftModel.from_pretrained( | |
model, LORA_WEIGHTS, torch_dtype=torch.float16, force_download=True | |
) | |
elif device == "mps": | |
model = LlamaForCausalLM.from_pretrained( | |
BASE_MODEL, | |
device_map={"": device}, | |
torch_dtype=torch.float16, | |
) | |
model = PeftModel.from_pretrained( | |
model, | |
LORA_WEIGHTS, | |
device_map={"": device}, | |
torch_dtype=torch.float16, | |
) | |
else: | |
model = LlamaForCausalLM.from_pretrained( | |
BASE_MODEL, device_map={"": device}, low_cpu_mem_usage=True | |
) | |
model = PeftModel.from_pretrained( | |
model, | |
LORA_WEIGHTS, | |
device_map={"": device}, | |
) | |
def generate_prompt(instruction, input=None): | |
if input: | |
return f"""Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request. | |
### Instruction: | |
{instruction} | |
### Input: | |
{input} | |
### Response:""" | |
else: | |
return f"""Below is an instruction that describes a task. Write a response that appropriately completes the request. | |
### Instruction : | |
{instruction} | |
### Response :""" | |
if device != "cpu": | |
model.half() | |
model.eval() | |
if torch.__version__ >= "2": | |
model = torch.compile(model) | |
def evaluate( | |
instruction, | |
input=None, | |
temperature=0.1, | |
top_p=0.75, | |
top_k=40, | |
num_beams=4, | |
max_new_tokens=128, | |
**kwargs, | |
): | |
prompt = generate_prompt(instruction, input) | |
inputs = tokenizer(prompt, return_tensors="pt") | |
input_ids = inputs["input_ids"].to(device) | |
generation_config = GenerationConfig( | |
temperature=temperature, | |
top_p=top_p, | |
top_k=top_k, | |
num_beams=num_beams, | |
**kwargs, | |
) | |
with torch.no_grad(): | |
generation_output = model.generate( | |
input_ids=input_ids, | |
generation_config=generation_config, | |
return_dict_in_generate=True, | |
output_scores=True, | |
max_new_tokens=max_new_tokens, | |
) | |
s = generation_output.sequences[0] | |
output = tokenizer.decode(s) | |
return output.split("### Response:")[1].strip() | |
g = gr.Interface( | |
fn=evaluate, | |
inputs=[ | |
gr.components.Textbox( | |
lines=2, label="Instruction", placeholder="Classify text as one of these 6 emotions: anger,fear,joy,love,sadness,or surprise" | |
), | |
gr.components.Textbox(lines=2, label="Input", placeholder="I am crying"), | |
gr.components.Slider(minimum=0, maximum=1, value=0.1, label="Temperature"), | |
gr.components.Slider(minimum=0, maximum=1, value=0.7, label="Top p"), | |
gr.components.Slider(minimum=0, maximum=100, step=1, value=40, label="Top k"), | |
gr.components.Slider(minimum=1, maximum=4, step=1, value=4, label="Beams"), | |
gr.components.Slider( | |
minimum=1, maximum=256, step=1, value=64, label="Max tokens" | |
), | |
], | |
outputs=[ | |
gr.inputs.Textbox( | |
lines=5, | |
label="Output", | |
) | |
], | |
title="Fine-tuned version of Vicuna Model", | |
description="This model is a fine-tuned version of the Vicuna model for sentiment analysis. https://github.com/hennypurwadi/Vicuna_finetune_sentiment_analysis \ | |
Base model is https://huggingface.co/TheBloke/vicuna-7B-1.1-HF \ | |
It is fine-tuned and trained on a dataset to classify text as one of these six different emotions: anger, fear, joy, love, sadness, or surprise. \ | |
The model was trained and tested on a labeled dataset from Kaggle (https://www.kaggle.com/datasets/praveengovi/emotions-dataset-for-nlp)", | |
) | |
g.queue(concurrency_count=1) | |
g.launch() | |