Updated Gradio app with new model

app.py

@@ -1,10 +1,71 @@
 
-import gradio as gr
+from peft import PeftModel, PeftConfig
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import torch
+from transformers import AutoTokenizer
+from peft import PeftModel, PeftConfig
+
+config = PeftConfig.from_pretrained("TohidA/LlamaInstructMona")
+model = AutoModelForCausalLM.from_pretrained("mlabonne/llama-2-7b-miniguanaco")
+model = PeftModel.from_pretrained(model, "TohidA/LlamaInstructMona")
+
+if torch.cuda.is_available():
+    model = model.cuda()
+
+tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)
+
+def prompt(instruction, input=''):
+    if input=='':
+        return f"Below is an instruction that describes a task. Write a response that appropriately completes the request. \n\n### Instruction:\n{instruction} \n\n### Response:\n"
+    return f"Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request. \n\n### Instruction:\n{instruction}\n\n### Input:\n{input}\n\n### Response:\n"
 
-def greet(name):
-    return "Hello " + name + "!!"
+tokenizer.pad_token = tokenizer.eos_token
+tokenizer.pad_token_id = tokenizer.eos_token_id
 
-iface = gr.Interface(fn=greet, inputs="text", outputs="text")
+def instruct(instruction, input='', temperature=0.7, top_p=0.95, top_k=4, max_new_tokens=128, do_sample=False, penalty_alpha=0.6, repetition_penalty=1., stop="\n\n"):
+    input_ids = tokenizer(prompt(instruction, input).strip(), return_tensors='pt').input_ids.cuda()
+    with torch.cuda.amp.autocast():
+        outputs = model.generate(
+            input_ids=input_ids,
+            return_dict_in_generate=True,
+            output_scores=True,
+            max_new_tokens=max_new_tokens,
+            temperature=temperature,
+            top_p=top_p,
+            top_k=top_k,
+            do_sample=do_sample,
+            repetition_penalty=repetition_penalty
+        )
+    if stop=="":
+        return tokenizer.decode(outputs.sequences[0], skip_special_tokens=True).split("### Response:")[1].strip(), prompt(instruction, input)
+    return tokenizer.decode(outputs.sequences[0], skip_special_tokens=True).split("### Response:")[1].strip().split(stop)[0].strip(), prompt(instruction, input)
     
-if __name__ == "__main__":
-    iface.launch()
+
+import locale
+locale.getpreferredencoding = lambda: "UTF-8"
+
+import gradio as gr
+
+input_text = gr.Textbox(label="Input")
+instruction_text = gr.Textbox(label="Instruction")
+temperature = gr.Slider(label="Temperature", minimum=0, maximum=1, value=0.7, step=0.05)
+top_p = gr.Slider(label="Top-P", minimum=0, maximum=1, value=0.95, step=0.01)
+top_k = gr.Slider(label="Top-K", minimum=0, maximum=128, value=40, step=1)
+max_new_tokens = gr.Slider(label="Tokens", minimum=1, maximum=256, value=64)
+do_sample = gr.Checkbox(label="Do Sample", value=True)
+penalty_alpha = gr.Slider(minimum=0, maximum=1, value=0.5)
+repetition_penalty = gr.Slider(minimum=1., maximum=2., value=1., step=0.1)
+stop = gr.Textbox(label="Stopping Criteria", value="")
+
+output_prompt = gr.Textbox(label="Prompt")
+output_text = gr.Textbox(label="Output")
+description = """
+The [TohidA/InstructLlamaMONA-withMONAdataset](https://hf.co/TohidA/LlamaInstructMona). A Llama chat 7B model finetuned on an [instruction dataset](https://huggingface.co/mlabonne/llama-2-7b-miniguanaco), then finetuned with the RL/PPO using a [Reward model](https://huggingface.co/TohidA/MONAreward) which is a BERT classifier trained on [Monda dataset](https://huggingface.co/datasets/TohidA/MONA), with [low rank adaptation](https://arxiv.org/abs/2106.09685) for a single epoch.
+"""
+gr.Interface(fn=instruct,
+             inputs=[instruction_text, input_text, temperature, top_p, top_k, max_new_tokens, do_sample, penalty_alpha, repetition_penalty, stop],
+             outputs=[output_text, output_prompt],
+             title="InstructLlamaMONA 7B Gradio Demo", description=description).launch(
+    debug=True,
+    share=True
+)