Update app.py

app.py

@@ -1,22 +1,19 @@
 from transformers import AutoTokenizer, AutoModelForCausalLM
 import gradio as gr
-from unsloth import FastLanguageModel
+import torch
 
 def load_model(model_name="masonchu/qwen2.5-7b-lora-unsloth_nomerge"):
     # Load tokenizer
     tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
-
-    # Load model with appropriate settings for inference using unsloth
-    model, _ = FastLanguageModel.from_pretrained(
+    
+    # Load model with standard transformers settings
+    model = AutoModelForCausalLM.from_pretrained(
         model_name,
-        max_seq_length=2048,
-        dtype=None,
-        load_in_4bit=True,
+        device_map="auto",
+        torch_dtype=torch.float16,
+        trust_remote_code=True
     )
-
-    # Enable faster inference
-    FastLanguageModel.for_inference(model)
-
+    
     return model, tokenizer
 
 def predict(message, history):
@@ -26,19 +23,22 @@ def predict(message, history):
         human, assistant = msg
         prompt += f"### Instruction:\n{human}\n\n### Input:\n\n### Response:\n{assistant}\n\n"
     prompt += f"### Instruction:\n{message}\n\n### Input:\n\n### Response:\n"
-
+    
     # Generate response
     inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
     outputs = model.generate(
         **inputs,
         max_new_tokens=512,
+        temperature=0,
+        top_p=0.9,
+        repetition_penalty=1.1,
         use_cache=True
     )
-
+    
     # Get response
     full_response = tokenizer.decode(outputs[0], skip_special_tokens=True)
     response = full_response.split("### Response:\n")[-1].strip()
-
+    
     return response
 
 # Load model globally
@@ -51,7 +51,7 @@ demo = gr.ChatInterface(
     predict,
     title="春笋科技 AI 助手",
     description="基于 Qwen2.5 模型训练的企业智能助手",
-    examples=["春笋科技公司的创始人是谁？", "春笋科技员工有多少天带薪年假？", "公司的技术实力如何？"],
+    examples=["你是谁？", "介绍一下公司的福利政策", "公司的技术实力如何？"],
     theme="soft"
 )