Zenithwang commited on
Commit
b262a14
1 Parent(s): a294443

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +100 -1
app.py CHANGED
@@ -1,3 +1,102 @@
 
1
  import gradio as gr
 
 
 
2
 
3
- gr.load("models/infly/OpenCoder-8B-Instruct").launch()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import spaces
2
  import gradio as gr
3
+ import torch
4
+ from transformers import AutoModelForCausalLM, AutoTokenizer, StoppingCriteria, StoppingCriteriaList, TextIteratorStreamer
5
+ from threading import Thread
6
 
7
+ model_path = 'sail/Sailor-7B-Chat'
8
+
9
+ # Loading the tokenizer and model from Hugging Face's model hub.
10
+ tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
11
+ model = AutoModelForCausalLM.from_pretrained(model_path, trust_remote_code=True, torch_dtype=torch.bfloat16)
12
+
13
+ # using CUDA for an optimal experience
14
+ device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
15
+ model = model.to(device)
16
+
17
+ # Defining a custom stopping criteria class for the model's text generation.
18
+ class StopOnTokens(StoppingCriteria):
19
+ def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
20
+ stop_ids = [151645] # IDs of tokens where the generation should stop.
21
+ for stop_id in stop_ids:
22
+ if input_ids[0][-1] == stop_id: # Checking if the last generated token is a stop token.
23
+ return True
24
+ return False
25
+
26
+
27
+ system_role= 'system'
28
+ user_role = 'question'
29
+ assistant_role = "answer"
30
+
31
+ sft_start_token = "<|im_start|>"
32
+ sft_end_token = "<|im_end|>"
33
+ ct_end_token = "<|endoftext|>"
34
+
35
+ system_prompt= \
36
+ 'You are an AI assistant named Sailor created by Sea AI Lab. \
37
+ Your answer should be friendly, unbiased, faithful, informative and detailed.'
38
+ system_prompt = f"<|im_start|>{system_role}\n{system_prompt}<|im_end|>"
39
+
40
+ # Function to generate model predictions.
41
+
42
+ @spaces.GPU()
43
+ def predict(message, history):
44
+ # history = []
45
+ history_transformer_format = history + [[message, ""]]
46
+ stop = StopOnTokens()
47
+
48
+ # Formatting the input for the model.
49
+ messages = system_prompt + sft_end_token.join([sft_end_token.join([f"\n{sft_start_token}{user_role}\n" + item[0], f"\n{sft_start_token}{assistant_role}\n" + item[1]])
50
+ for item in history_transformer_format])
51
+ model_inputs = tokenizer([messages], return_tensors="pt").to(device)
52
+ streamer = TextIteratorStreamer(tokenizer, timeout=10., skip_prompt=True, skip_special_tokens=True)
53
+ generate_kwargs = dict(
54
+ model_inputs,
55
+ streamer=streamer,
56
+ max_new_tokens=512,
57
+ do_sample=True,
58
+ top_p= 0.75,
59
+ top_k= 60,
60
+ temperature=0.2,
61
+ num_beams=1,
62
+ stopping_criteria=StoppingCriteriaList([stop]),
63
+ repetition_penalty=1.1,
64
+ )
65
+ t = Thread(target=model.generate, kwargs=generate_kwargs)
66
+ t.start() # Starting the generation in a separate thread.
67
+ partial_message = ""
68
+ for new_token in streamer:
69
+ partial_message += new_token
70
+ if sft_end_token in partial_message: # Breaking the loop if the stop token is generated.
71
+ break
72
+ yield partial_message
73
+
74
+
75
+ css = """
76
+ full-height {
77
+ height: 100%;
78
+ }
79
+ """
80
+
81
+ prompt_examples = [
82
+ 'How to cook a fish?',
83
+ 'Cara memanggang ikan',
84
+ 'วิธีย่างปลา',
85
+ 'Cách nướng cá'
86
+ ]
87
+
88
+ placeholder = """
89
+ <div style="opacity: 0.5;">
90
+ <img src="https://raw.githubusercontent.com/sail-sg/sailor-llm/main/misc/banner.jpg" style="width:30%;">
91
+ <br>Sailor models are designed to understand and generate text across diverse linguistic landscapes of these SEA regions:
92
+ <br>🇮🇩Indonesian, 🇹🇭Thai, 🇻🇳Vietnamese, 🇲🇾Malay, and 🇱🇦Lao.
93
+ </div>
94
+ """
95
+
96
+ chatbot = gr.Chatbot(label='Sailor', placeholder=placeholder)
97
+ with gr.Blocks(theme=gr.themes.Soft(), fill_height=True) as demo:
98
+ # gr.Markdown("""<center><font size=8>Sailor-Chat Bot⚓</center>""")
99
+ gr.Markdown("""<p align="center"><img src="https://github.com/sail-sg/sailor-llm/raw/main/misc/wide_sailor_banner.jpg" style="height: 110px"/><p>""")
100
+ gr.ChatInterface(predict, chatbot=chatbot, fill_height=True, examples=prompt_examples, css=css)
101
+
102
+ demo.launch() # Launching the web interface.