chore

Signed-off-by: Suvaditya Mukherjee <[email protected]>

app.py

@@ -194,14 +194,14 @@ if __name__ == "__main__":
         model_name,
         torch_dtype=torch.bfloat16,
         device_map="auto",
-        # quantization_config=QuantoConfig(
-        #     weights="int4",
-        # )
-        quantization_config = BitsAndBytesConfig(
-            load_in_8bit=True,
-            # bnb_4bit_compute_dtype=torch.float16,
-            # bnb_4bit_quant_type="nf4"
+        quantization_config=QuantoConfig(
+            weights="int4",
         )
+        # quantization_config = BitsAndBytesConfig(
+        #     load_in_8bit=True,
+        #     # bnb_4bit_compute_dtype=torch.float16,
+        #     # bnb_4bit_quant_type="nf4"
+        # )
     )
 
     tokenizer = AutoTokenizer.from_pretrained(model_name)