Update app.py

app.py

@@ -94,9 +94,21 @@ def evaluate(
         num_beams=num_beams,
         **kwargs,
     )
-    with torch.cuda.amp.autocast():
-        output_tokens = model.generate(**inputs, generation_config=generation_config)
-    output = tokenizer.decode(output_tokens[0], skip_special_tokens=True)
+    # with torch.cuda.amp.autocast():
+    #     output_tokens = model.generate(**inputs, generation_config=generation_config)
+    # output = tokenizer.decode(output_tokens[0], skip_special_tokens=True)
+
+    with torch.no_grad():
+        generation_output = model.generate(
+            input_ids=input_ids,
+            generation_config=generation_config,
+            return_dict_in_generate=True,
+            output_scores=True,
+            max_new_tokens=max_new_tokens,
+        )
+    s = generation_output.sequences[0]
+    output = tokenizer.decode(s, skip_special_tokens=True)
+
     return output[len(input_ids):]