gorkemgoknar commited on
Commit
d7ce43f
1 Parent(s): 56ad9c6

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +28 -0
app.py CHANGED
@@ -208,6 +208,34 @@ def format_prompt_zephyr(message, history, system_message=system_message):
208
  print(prompt)
209
  return prompt
210
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
211
  def generate_local(
212
  prompt,
213
  history,
 
208
  print(prompt)
209
  return prompt
210
 
211
+ def generate(
212
+ prompt, history, temperature=0.9, max_new_tokens=256, top_p=0.95, repetition_penalty=1.0,
213
+ ):
214
+ temperature = float(temperature)
215
+ if temperature < 1e-2:
216
+ temperature = 1e-2
217
+ top_p = float(top_p)
218
+
219
+ generate_kwargs = dict(
220
+ temperature=temperature,
221
+ max_tokens=max_new_tokens,
222
+ max_new_tokens=max_new_tokens,
223
+ top_p=top_p,
224
+ stop=LLM_STOP_WORDS
225
+ )
226
+ formatted_prompt = format_prompt(prompt, history)
227
+
228
+ del generate_kwargs["max_tokens"]
229
+ del generate_kwargs["stop"]
230
+
231
+ stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
232
+ output = ""
233
+
234
+ for response in stream:
235
+ output += response.token.text
236
+ yield output
237
+ return output
238
+
239
  def generate_local(
240
  prompt,
241
  history,