app.py

import random
from transformers import AutoTokenizer, AutoModelForCausalLM
import gradio as gr
from gramformer import Gramformer

tokenizer = AutoTokenizer.from_pretrained("docto/Docto-Bot")
model = AutoModelForCausalLM.from_pretrained("docto/Docto-Bot")
special_token = '<|endoftext|>'

gf = Gramformer(models = 1, use_gpu=False) # 1=corrector, 2=detector

def get_reply(userinput):
        prompt_text = f'Question: {userinput}\nAnswer:'
        encoded_prompt = tokenizer.encode(prompt_text,
                                  add_special_tokens = False,
                                  return_tensors = 'pt')

        output_sequences = model.generate(
            input_ids = encoded_prompt,
            max_length = 500,
            temperature = 0.9,
            top_k = 20,
            top_p = 0.9,
            repetition_penalty = 1,
            do_sample = True,
            num_return_sequences = 4
        )

        # result = tokenizer.decode(random.choice(output_sequences))
        # result = result[result.index("Answer: "):result.index(special_token)]
        try:
            result = tokenizer.decode(random.choice(output_sequences))
            result = result[result.index("Answer: "):result.index(special_token)]
            corrected_sentence = gf.correct(result[8:], max_candidates=1)
            return corrected_sentence 
        except:
            return "Sorry! I don\'t Know"
            
iface = gr.Interface(fn=get_reply, inputs=["text"], outputs=["textbox"]).launch()