Update app.py

app.py

@@ -11,22 +11,36 @@ model = GPT2LMHeadModel.from_pretrained(model_repo_id)
 tokenizer = GPT2Tokenizer.from_pretrained(model_repo_id)
 
 # Define the prediction function
-def predict(text):
-    try:
-        # Encode the input text
-        inputs = tokenizer.encode(text, return_tensors="pt")
-
-        # Generate output using the model
-        with torch.no_grad():
-            outputs = model.generate(inputs, max_length=50, do_sample=True)
-
-        # Decode the generated output
-        prediction = tokenizer.decode(outputs[0], skip_special_tokens=True)
-
-        return prediction.strip()  # Return the clean output
-    except Exception as e:
-        # Handle and print any exceptions for debugging
-        return f"An error occurred: {str(e)}"
+def generate_answer(question):
+    input_ids = tokenizer.encode(question, return_tensors="pt").to("cuda")
+
+    # Create the attention mask and pad token id
+    attention_mask = torch.ones_like(input_ids).to("cuda")
+    pad_token_id = tokenizer.eos_token_id
+
+    #output = model[0].generate(
+    output = model.generate(
+        input_ids,
+        max_new_tokens=100,
+        num_return_sequences=1,
+        attention_mask=attention_mask,
+        pad_token_id=pad_token_id
+    )
+    decoded_output = tokenizer.decode(output[0], skip_special_tokens=True)
+    start_index = decoded_output.find("Answer")
+    end_index = decoded_output.find("<ANSWER_ENDED>")
+
+    if end_index != -1:
+        # Extract the text between "Answer" and "<ANSWER_ENDED>"
+        answer_text = decoded_output[start_index + len("Answer"):end_index].strip()
+        return answer_text
+    else:
+        # If "<ANSWER_ENDED>" is not found, return the text following "Answer"
+        answer_text = decoded_output[start_index + len("Answer"):].strip()
+        return answer_text
+
+    #return tokenizer.decode(output[0], skip_special_tokens=True)
+    #return tokenizer.decode(output, skip_special_tokens=True)
 
 # Gradio interface setup
 interface = gr.Interface(