Update app.py

app.py

@@ -3,9 +3,17 @@ from transformers import AutoTokenizer
 import re
 from peft import PeftModel, PeftConfig
 from transformers import AutoModelForCausalLM
+from transformers import BitsAndBytesConfig
+
+nf4_config = BitsAndBytesConfig(
+   load_in_4bit=True,
+   bnb_4bit_quant_type="nf4",
+   bnb_4bit_use_double_quant=True,
+   bnb_4bit_compute_dtype=torch.bfloat16
+)
 
 config = PeftConfig.from_pretrained("mohamedemam/Arabic-meeting-summarization")
-model = AutoModelForCausalLM.from_pretrained("bigscience/bloomz-3b")
+model = AutoModelForCausalLM.from_pretrained("bigscience/bloomz-3b",quantization_config=nf4_config)
 model = PeftModel.from_pretrained(model, "mohamedemam/Arabic-meeting-summarization")
 # Load the tokenizer and model
 model_name ="bigscience/bloomz-3b"
@@ -26,7 +34,7 @@ for i in range(len(example_contexts)):
 # Function to generate questions and answers with configurable parameters
 def generate_qa(context, temperature, top_p,num_seq,l_p, num_b):
     input_text = context
-    input_ids = tokenizer(input_text, return_tensors='pt')
+    input_ids = tokenizer(text=input_text, return_tensors='pt')
     
     # Generate with configurable parameters
     output = model.generate(
@@ -35,7 +43,7 @@ def generate_qa(context, temperature, top_p,num_seq,l_p, num_b):
         top_p=top_p,
         num_return_sequences=num_seq,
    
-        max_length=100,
+        max_new_tokens=60,
         num_beams=num_b,
         length_penalty=l_p,    
         do_sample=True,
@@ -49,7 +57,6 @@ def generate_qa(context, temperature, top_p,num_seq,l_p, num_b):
 iface = gr.Interface(
     fn=generate_qa,
     inputs=[
-        gr.inputs.Dropdown(example_contexts, label="Choose an Example"),
         gr.inputs.Slider(minimum=0.0, maximum=5, default=2.1, step=0.01, label="Temperature"),
         gr.inputs.Slider(minimum=0.0, maximum=1, default=0.5, step=0.01, label="Top-p"),     
         gr.inputs.Slider(minimum=1, maximum=20, default=3, step=1, label="num of sequance"),