Update app.py

app.py

@@ -8,8 +8,8 @@ import time
 model_id = "facebook/opt-6.7b"
 assistant_id = "facebook/opt-125m"
 
-model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.float16, device_map="auto")
-assistant_model = AutoModelForCausalLM.from_pretrained(assistant_id).to(device=model.device, dtype=torch.float16)
+model = AutoModelForCausalLM.from_pretrained(model_id, load_in_8bit=True, device_map="auto")
+assistant_model = AutoModelForCausalLM.from_pretrained(assistant_id).to(device=model.device, dtype=torch.bfloat16)
 tokenizer = AutoTokenizer.from_pretrained(model_id)
 
 def run_generation(user_text, use_assistant, temperature, max_new_tokens):