Tomoniai commited on
Commit
f070899
·
1 Parent(s): e2f671e

Create app.py

Browse files
Files changed (1) hide show
  1. app.py +51 -0
app.py ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from huggingface_hub import InferenceClient
2
+ import gradio as gr
3
+
4
+ client = InferenceClient("mistralai/Mistral-7B-Instruct-v0.1")
5
+
6
+ def format_prompt(message, history):
7
+ system_prompt = "You are Mistral, a gentle and a useful AI assistant. My input is "
8
+ prompt = "<s>"
9
+ prompt += f"[INST] {system_prompt} [/INST]"
10
+ for user_prompt, bot_response in history:
11
+ prompt += f"[INST] {user_prompt} [/INST]"
12
+ prompt += f" {bot_response}</s> "
13
+ prompt += f"[INST] {message} [/INST]"
14
+ return prompt
15
+
16
+ def generate(
17
+ prompt, history, temperature=0.3, max_new_tokens=256, top_p=0.95, repetition_penalty=1.0,
18
+ ):
19
+ temperature = float(temperature)
20
+ top_p = float(top_p)
21
+
22
+ generate_kwargs = dict(
23
+ temperature=temperature,
24
+ max_new_tokens=max_new_tokens,
25
+ top_p=top_p,
26
+ repetition_penalty=repetition_penalty,
27
+ do_sample=True,
28
+ seed=42,
29
+ )
30
+
31
+ formatted_prompt = format_prompt(prompt, history)
32
+
33
+ stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
34
+ output = ""
35
+
36
+ for response in stream:
37
+ output += response.token.text
38
+ yield output
39
+ return output
40
+
41
+
42
+ mychatbot = gr.Chatbot(show_label=False, show_copy_button=True,)
43
+
44
+ demo = gr.ChatInterface(fn=generate,
45
+ chatbot=mychatbot,
46
+ title="Mistral 7b v0.1 Instruct Chat",
47
+ retry_btn=None,
48
+ undo_btn=None
49
+ )
50
+
51
+ demo.queue().launch()