Spaces:
Runtime error
Runtime error
File size: 2,617 Bytes
b061c53 5323eae b061c53 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 |
"""
Try out gradio.Chatinterface.
colab gradio-chatinterface.
%%writefile reuirements.txt
gradio
transformers
sentencepiece
torch
"""
# pylint: disable=line-too-long, missing-module-docstring, missing-function-docstring
# import torch
from time import time
import gradio as gr
from about_time import about_time
from examples_list import examples_list
from transformers import AutoModel, AutoTokenizer # AutoModelForCausalLM,
# device = "cuda" if torch.cuda.is_available() else "cpu"
# tokenizer = AutoTokenizer.from_pretrained("stabilityai/StableBeluga2", use_fast=False)
# model = AutoModelForCausalLM.from_pretrained("stabilityai/StableBeluga2", torch_dtype=torch.float16, low_cpu_mem_usage=True, device_map="auto")
# system_prompt = "### System:\nYou are Stable Beluga, an AI that follows instructions extremely well. Help as much as you can. Remember, be safe, and don't do anything illegal.\n\n"
# pipeline = pipeline(task="text-generation", model="meta-llama/Llama-2-7b")
tokenizer = AutoTokenizer.from_pretrained(
"THUDM/chatglm2-6b-int4", trust_remote_code=True
)
chat_model = AutoModel.from_pretrained(
"THUDM/chatglm2-6b-int4", trust_remote_code=True # 3.92G
).float()
def chat(message, history):
# prompt = f"{system_prompt}### User: {message}\n\n### Assistant:\n"
# inputs = tokenizer(prompt, return_tensors="pt").to(device=device)
# output = model.generate(**inputs, do_sample=True, top_p=0.95, top_k=0, max_new_tokens=256)
# return tokenizer.decode(output[0], skip_special_tokens=True)
flag = 1
then = time()
prefix = ""
prelude = 0.0
with about_time() as dur:
for response, _ in chat_model.stream_chat(
tokenizer, message, history, max_length=2048, top_p=0.7, temperature=0.95
):
if flag:
flag = 0
prelude = time() - then
prefix = f"{prelude:.2f}s"
yield f"{prefix} {response}"
suffix = f"\n(time elapsed: {dur.duration_human}, {(time() - prelude)/len(response):.2f}s/char)"
yield f"{response}{suffix}"
chatbot = gr.Chatbot([], label="Bot", height=450)
textbox = gr.Textbox('', scale=10, label='', lines=2, placeholder="Ask me anything")
submit_btn = gr.Button(value="▶️ Send", scale=1, min_width=0, variant="primary")
interf = gr.ChatInterface(
chat,
chatbot=chatbot,
textbox=textbox,
submit_btn=submit_btn,
title="Llama-2-70b Locally Hosted",
examples=examples_list,
theme=gr.themes.Glass(text_size="sm", spacing_size="sm"),
).queue(max_size=5)
if __name__ == "__main__":
interf.launch(debug=True)
|