Updated gradio app

app.py

@@ -1,3 +1,76 @@
 import gradio as gr
 
-gr.load("models/Joaoffg/ELM").launch()
+def evaluate(instruction):
+ # Generate a response:
+ input = None
+ prompt = prompter.generate_prompt(instruction, input)
+ inputs = tokenizer(prompt, return_tensors="pt")
+ #inputs = inputs.to("cuda:0")
+ input_ids = inputs["input_ids"]
+ 
+ #play around with generation strategies for better/diverse sequences. https://huggingface.co/docs/transformers/generation_strategies
+ temperature=0.2
+ top_p=0.95
+ top_k=25
+ num_beams=1
+ # num_beam_groups=num_beams #see: 'Diverse beam search decoding'
+ max_new_tokens=256
+ repetition_penalty = 2.0
+ do_sample = True # allow 'beam sample': do_sample=True, num_beams > 1
+ num_return_sequences = 1 #generate multiple candidates, takes longer..
+ 
+ generation_config = transformers.GenerationConfig(
+ temperature=temperature,
+ top_p=top_p,
+ top_k=top_k,
+ num_beams=num_beams,
+ repetition_penalty=repetition_penalty,
+ do_sample=do_sample,
+ min_new_tokens=32,
+ num_return_sequences=num_return_sequences,
+ pad_token_id = 0
+ # num_beam_groups=num_beam_groups
+ )
+ 
+ generate_params = {
+ "input_ids": input_ids,
+ "generation_config": generation_config,
+ "return_dict_in_generate": True,
+ "output_scores": True,
+ "max_new_tokens": max_new_tokens,
+ }
+ with torch.no_grad():
+ generation_output = model.generate(
+ input_ids=input_ids,
+ generation_config=generation_config,
+ return_dict_in_generate=True,
+ output_scores=True,
+ max_new_tokens=max_new_tokens,
+ )
+ 
+ 
+ print(f'Instruction: {instruction}')
+ 
+ for i,s in enumerate(generation_output.sequences):
+ output = tokenizer.decode(s,skip_special_tokens=True)
+ # print(output)
+ return(f' {prompter.get_response(output)}')
+
+gr.Interface(
+ fn=evaluate,
+ inputs=[
+ gr.components.Textbox(
+ lines=2,
+ label="Instruction",
+ placeholder="Explain economic growth.",
+ ),
+ ],
+ outputs=[
+ gr.components.Textbox(
+ lines=5,
+ label="Output",
+ )
+ ],
+ title="🌲 ELM - Erasmian Language Model",
+ description="ELM is a 900M parameter language model finetuned to follow instruction. It is trained on Erasmus University academic outputs and the [Stanford Alpaca](https://github.com/tatsu-lab/stanford_alpaca) dataset. For more information, please visit [the GitHub repository](https://github.com/Joaoffg/ELM).", # noqa: E501
+ ).queue().gr.load("models/Joaoffg/ELM").launch()