Update main.py

main.py

@@ -21,9 +21,9 @@ embeddingModel = AutoModel.from_pretrained('./multilingual-e5-base')
 
 # chatGpt replacement
 inferenceTokenizer = AutoTokenizer.from_pretrained(
-    "./ct2fast-flan-alpaca-xl")
+    "./ct2fast-flan-alpaca-xxl")
 inferenceTranslator = Translator(
-    "./ct2fast-flan-alpaca-xl", compute_type="int8", device="cpu")
+    "./ct2fast-flan-alpaca-xxl", compute_type="int8", device="cpu")
 
 
 class EmbeddingRequest(BaseModel):
@@ -79,7 +79,7 @@ async def inference(request: InferenceRequest):
         inferenceTokenizer.encode(input_text))
 
     results = inferenceTranslator.translate_batch(
-        [input_tokens], max_input_length=0, max_decoding_length=max_length, num_hypotheses=1, repetition_penalty=1.3, sampling_topk=30, sampling_temperature=1.1, use_vmap=True)
+        [input_tokens], max_input_length=0, max_decoding_length=max_length, num_hypotheses=1, repetition_penalty=1.3, sampling_topk=30, sampling_temperature=1.1, use_vmap=True, disable_unk=True)
 
     output_tokens = results[0].hypotheses[0]
     output_text = inferenceTokenizer.decode(