Update app.py

app.py

@@ -8,8 +8,6 @@ device = torch.device("cpu")  # Ensure it's using CPU only
 # Load model and tokenizer
 model_name = "hosseinhimself/ISANG-v1.0-8B"  # Replace with your model name
 tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False)
-
-# Load the model for inference on CPU
 model = AutoModelForCausalLM.from_pretrained(model_name).to(device)
 
 # Define the Alpaca-style prompt template
@@ -24,23 +22,29 @@ You are ISANG, a multilingual large language model made by ISANG AI. You only re
 ### Response:
 {}"""
 
-EOS_TOKEN = tokenizer.eos_token  # Must add EOS_TOKEN
-
-# Define a function to generate responses
+# Function to generate responses
 def generate_response(input_text, max_tokens=1024, temperature=0.7, history=[]):
-    # Prepare the inputs for the model with the history
-    prompt = f"Chat History: {history[-2:]}\nUser: {input_text}\nAI:"
+    # Retain only the last two exchanges for context
+    if len(history) > 2:
+        history = history[-2:]
+    
+    # Format the prompt
+    prompt = "\n".join(history + [f"User: {input_text}\nAI:"])
     
-    # Tokenize the input and prepare it for inference
-    inputs = tokenizer(prompt, return_tensors="pt").to(device)  # Ensure using CPU
+    # Tokenize the input
+    inputs = tokenizer(prompt, return_tensors="pt").to(device)
     
-    # Set the max new tokens and temperature parameters for model generation
-    output = model.generate(**inputs, max_new_tokens=max_tokens, temperature=temperature, use_cache=True)
+    # Generate model output
+    output = model.generate(
+        inputs.input_ids,
+        max_new_tokens=max_tokens,
+        temperature=temperature
+    )
     
-    # Decode the model output and remove special tokens
-    response = tokenizer.decode(output[0], skip_special_tokens=True)
+    # Decode the model output
+    response = tokenizer.decode(output[0], skip_special_tokens=True).strip()
     
-    # Update history with the new conversation
+    # Update the history
     history.append(f"User: {input_text}")
     history.append(f"AI: {response}")
     
@@ -59,7 +63,7 @@ iface = gr.Interface(
     title="ISANG Chatbot",
     description="A chatbot powered by ISANG-v1.0-8B model. Chat with me!",
     theme="huggingface",  # Purple theme
-    live=True
+    live=False  # Set to False since live updates aren't required
 )
 
 # Launch the interface