Update app.py

app.py

@@ -30,7 +30,10 @@ def utf8_tokens(tokens):
 def tokenize_text(tokenizer_name_1, tokenizer_name_2, text, hf_token=None):
     def tokenize_with_model(tokenizer_name):
         try:
-            tokenizer = AutoTokenizer.from_pretrained(tokenizer_name, use_auth_token=hf_token)
+            if hf_token:
+                tokenizer = AutoTokenizer.from_pretrained(tokenizer_name, use_auth_token=hf_token)
+            else:
+                tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
             tokens = tokenizer.tokenize(text)
             word_count = len(text.split())
             token_count = len(tokens)