Update app.py

app.py

@@ -28,7 +28,8 @@ if HUGGINGFACE_HUB_TOKEN:
     login(token=HUGGINGFACE_HUB_TOKEN)
 
 os.system("git config --global credential.helper store")
-huggingface_hub.login(token=HUGGINGFACE_HUB_TOKEN, add_to_git_credential=True)
+if HUGGINGFACE_HUB_TOKEN:
+    huggingface_hub.login(token=HUGGINGFACE_HUB_TOKEN, add_to_git_credential=True)
 
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 logger = logging.getLogger(__name__)
@@ -169,19 +170,15 @@ model_loader = GCSModelLoader(bucket)
 
 async def generate_stream(model, tokenizer, input_text, generation_config):
     inputs = tokenizer(input_text, return_tensors="pt").to(model.device)
-
-    async def token_stream():
-        generation_stream = model.generate(
-            **inputs,
-            generation_config=generation_config,
-            stream=True,
-        )
-        async for output in generation_stream:
-            token_id = output[-1]
-            token = tokenizer.decode(token_id, skip_special_tokens=True)
-            yield {"token": token}
-
-    return token_stream()
+    generation_stream = model.generate(
+        **inputs,
+        generation_config=generation_config,
+        stream=True,
+    )
+    async for output in generation_stream:
+        token_id = output[-1]
+        token = tokenizer.decode(token_id, skip_special_tokens=True)
+        yield {"token": token}
 
 def generate_non_stream(model, tokenizer, input_text, generation_config):
     inputs = tokenizer(input_text, return_tensors="pt").to(model.device)
@@ -216,8 +213,8 @@ async def generate(request: GenerateRequest):
         generation_config_kwargs = generation_params.copy()
         generation_config_kwargs['pad_token_id'] = tokenizer.pad_token_id
         generation_config_kwargs['eos_token_id'] = tokenizer.eos_token_id
-        generation_config_kwargs['sep_token_id'] = tokenizer.sep_token_id
-        generation_config_kwargs['unk_token_id'] = tokenizer.unk_token_id
+        generation_config_kwargs['sep_token_id'] = tokenizer.sep_token_id if tokenizer.sep_token_id else tokenizer.eos_token_id
+        generation_config_kwargs['unk_token_id'] = tokenizer.unk_token_id if tokenizer.unk_token_id else tokenizer.eos_token_id
 
         model = model_loader.load_model(model_name, config)
         if not model: