Update main.py

main.py

@@ -23,7 +23,7 @@ embeddingModel = AutoModel.from_pretrained('./multilingual-e5-base')
 inferenceTokenizer = AutoTokenizer.from_pretrained(
     "./ct2fast-flan-alpaca-xl")
 inferenceTranslator = Translator(
-    "./ct2fast-flan-alpaca-xl", compute_type="int8", device="cpu", inter_threads=1, intra_threads=2)
+    "./ct2fast-flan-alpaca-xl", compute_type="int8", device="cpu")
 
 
 class EmbeddingRequest(BaseModel):
@@ -79,7 +79,7 @@ async def inference(request: InferenceRequest):
         inferenceTokenizer.encode(input_text))
 
     results = inferenceTranslator.translate_batch(
-        [input_tokens], max_input_length=0, max_decoding_length=max_length, num_hypotheses=1, repetition_penalty=1.3, sampling_topk=30, sampling_temperature=1.1, use_vmap=True, disable_unk=True)
+        [input_tokens], max_input_length=0, max_decoding_length=max_length, num_hypotheses=1, repetition_penalty=1.3, sampling_topk=40, sampling_temperature=0.2, use_vmap=False)
 
     output_tokens = results[0].hypotheses[0]
     output_text = inferenceTokenizer.decode(