cpu

inference.py

@@ -5,9 +5,9 @@ import torch
 
 def load_model(
     model_name: str,
-    max_seq_length: int,
-    dtype: torch.dtype,
-    load_in_4bit: bool
+    max_seq_length: int = 2048,
+    dtype: torch.dtype = torch.float32,
+    load_in_4bit: bool = False
 ) -> Tuple[AutoModelForCausalLM, any]:
     """
     Load and initialize the language model for inference.