Update app.py

app.py

@@ -3,12 +3,8 @@ import gradio as gr
 import torch
 import bitsandbytes
 from unsloth import FastLanguageModel
-from transformers import TextStreamer, StoppingCriteriaList, StoppingCriteria
-
-"""
-For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
-"""
-#client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
+from transformers import TextStreamer, StoppingCriteriaList, StoppingCriteria, TextIteratorStreamer
+from threading import Thread
 
 
 model, tokenizer = FastLanguageModel.from_pretrained(
@@ -20,90 +16,6 @@ model, tokenizer = FastLanguageModel.from_pretrained(
 
 FastLanguageModel.for_inference(model)
 
-'''
-alpaca_prompt = """Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.
-
-### Instruction:
-{}
-
-### Input:
-{}
-
-### Response:
-{}"""
-
-def texto_Patol(input):
-  FastLanguageModel.for_inference(model)
-  inputs = tokenizer(
-  [
-      alpaca_prompt.format(
-          input, # instruction
-          "", # input
-          "", # output - leave this blank for generation!
-      )
-  ], return_tensors = "pt")
-#.to("cuda")
-
-  outputs = model.generate(**inputs, max_new_tokens = 2048, use_cache = True)
-  return tokenizer.batch_decode(outputs)
-'''
-
-"""
-def respond(
-    message,
-    history: list[tuple[str, str]],
-    system_message,
-    max_tokens,
-    temperature,
-    top_p,
-):
-    messages = [{"role": "system", "content": system_message}]
-
-    for val in history:
-        if val[0]:
-            messages.append({"role": "user", "content": val[0]})
-        if val[1]:
-            messages.append({"role": "assistant", "content": val[1]})
-
-    messages.append({"role": "user", "content": message})
-
-    response = ""
-
-    for message in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-    ):
-        token = message.choices[0].delta.content
-
-        response += token
-        yield response
-
-"""
-
-"""
-For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
-"""
-
-"""
-demo = gr.ChatInterface(
-    respond,
-    additional_inputs=[
-        gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
-        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
-        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-        gr.Slider(
-            minimum=0.1,
-            maximum=1.0,
-            value=0.95,
-            step=0.05,
-            label="Top-p (nucleus sampling)",
-        ),
-    ],
-)
-"""
 class StopOnTokens(StoppingCriteria):
     def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
         stop_ids = [29, 0]
@@ -116,13 +28,27 @@ def predict(message, history):
     history_transformer_format = history + [[message, ""]]
     stop = StopOnTokens()
 
+    '''
+    inputs = tokenizer(
+    [
+        alpaca_prompt.format(
+            message, # instruction
+            "", # input
+            "", # output - leave this blank for generation!
+        )
+    ], return_tensors = "pt").to("cuda")
+
+    text_streamer = TextStreamer(tokenizer)
+    _ = model.generate(**inputs, streamer = text_streamer, max_new_tokens = 2048)
+    '''
+
     messages = "".join(["".join(["\n<human>:"+item[0], "\n<bot>:"+item[1]])
                 for item in history_transformer_format])
 
     model_inputs = tokenizer([messages], return_tensors="pt").to("cuda")
     streamer = TextIteratorStreamer(tokenizer, timeout=10., skip_prompt=True, skip_special_tokens=True)
     generate_kwargs = dict(
-        model_inputs=inputs,
+        model_inputs,
         streamer=streamer,
         max_new_tokens=2048,
         #do_sample=True,
@@ -143,17 +69,4 @@ def predict(message, history):
 
 gr.ChatInterface(predict).launch(debug=True)
 
-"""
-demo = gr.Interface(fn=texto_Patol,
-                    inputs=[gr.Textbox(label="Ingresa una pregunta acerca de Patología", lines=2)],
-                    outputs=[gr.HighlightedText(label="Respuesta")],
-                    title="Chat de Patología en Español",
-                    description="Utiliza la primera versión del modelo ajustado Patologia_lora_model1 haciendo preguntas de patología",
-                    allow_flagging="never",
-                    #Here we introduce a new tag, examples, easy to use examples for your application
-                    examples=["¿Cuál es el mecanismo de acción del crizotinib?", "Cuál es la mutación accionable más frecuente en cáncer de mama"])
-demo.launch(share=True, debug=True)
-"""
-
-#if __name__ == "__main__":
-#    demo.launch()
+gr.close_all()