try flash attention

app.py

@@ -36,6 +36,7 @@ model = AutoModelForCausalLM.from_pretrained(
     model_id,
     device_map="auto",
     torch_dtype=torch.bfloat16,
+    attn_implementation="flash_attention_2",
     trust_remote_code=True,
 )
 model.config.sliding_window = 4096