Flash attention test

app.py

@@ -21,8 +21,9 @@ app.add_middleware(
 model_id = "vikhyatk/moondream2"
 revision = "2024-08-26"
 model = AutoModelForCausalLM.from_pretrained(
-    model_id, trust_remote_code=True, revision=revision
-)
+    model_id, trust_remote_code=True, revision=revision,
+    torch_dtype=torch.float16, attn_implementation="flash_attention_2"
+).to("cuda")
 tokenizer = AutoTokenizer.from_pretrained(model_id, revision=revision)
 
 @app.get("/")