updated scripts

src/backend/model_operations.py

@@ -299,8 +299,8 @@ class SummaryGenerator:
             payload = {
                 "model": self.model_id,
                 # "max_tokens": 4096,
-                'max_new_tokens': 250,
-                "temperature": 0.0,
+                'max_new_tokens': 50,
+                # "temperature": 0.0,
                 # 'repetition_penalty': 1.1 if 'mixtral' in self.model_id.lower() else 1
             }
             # if 'mixtral' in self.model_id.lower():
@@ -343,8 +343,8 @@ class SummaryGenerator:
                 model=self.model_id.replace('openai/',''),
                 messages=[{"role": "system", "content": system_prompt},
                         {"role": "user", "content": user_prompt}],
-                temperature=0.0,
-                max_tokens=250,
+                # temperature=0.0,
+                max_tokens=50,
             )   
             result = response['choices'][0]['message']['content']
             # print()
@@ -358,7 +358,7 @@ class SummaryGenerator:
                 "temperature": 0,
                 "top_p": 0.95, # cannot change
                 "top_k": 0,
-                "max_output_tokens": 250,
+                "max_output_tokens": 50,
                 # "response_mime_type": "application/json",
             }
             safety_settings = [
@@ -411,7 +411,7 @@ class SummaryGenerator:
                             # mistralai/Mistral-7B-Instruct-v0.1",
                             messages=[{"role": "system", "content": system_prompt},
                                 {"role": "user", "content": user_prompt}],
-                            temperature=0.0,
+                            #temperature=0.0,
                             max_tokens=1024,
                             api_base="https://api-inference.huggingface.co/models/" + self.model_id)
                 print("模型返回结果",response)
@@ -441,9 +441,9 @@ class SummaryGenerator:
                 )
 
                 generation_args = {
-                    "max_new_tokens": 250,
+                    "max_new_tokens": 50,
                     "return_full_text": False,
-                    "temperature": 0.0,
+                    #"temperature": 0.0,
                     "do_sample": False,
                 }
 
@@ -455,7 +455,7 @@ class SummaryGenerator:
                 print(prompt)
                 input_ids = self.tokenizer(prompt, return_tensors="pt").to('cuda')
                 with torch.no_grad():
-                    outputs = self.local_model.generate(**input_ids, max_new_tokens=250, do_sample=True, temperature=0.01, pad_token_id=self.tokenizer.eos_token_id)
+                    outputs = self.local_model.generate(**input_ids, max_new_tokens=50, do_sample=True, pad_token_id=self.tokenizer.eos_token_id)
                 result = self.tokenizer.decode(outputs[0], skip_special_tokens=True)
                 result = result.replace(prompt[0], '')
                 print(result)