grascii
/

gregg-vision-v0.2.1

vision-encoder-decoder

image-text-to-text

Inference Endpoints

Model card Files Files and versions Community

chanicpanic commited on Dec 13, 2024

Commit

9efb814

·

verified ·

1 Parent(s): 7fa875a

Fix tokenizer special tokens

Files changed (2) hide show

special_tokens_map.json +5 -7
tokenizer_config.json +6 -2

special_tokens_map.json CHANGED Viewed

@@ -1,9 +1,7 @@
 {
-  "pad_token": {
-    "content": "<pad>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  }
 }

 {
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
 }

tokenizer_config.json CHANGED Viewed

@@ -41,12 +41,16 @@
       "special": true
     }
   },
-  "clean_up_tokenization_spaces": true,
   "model_input_names": [
     "labels",
     "pixel_values"
   ],
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": "<pad>",
-  "tokenizer_class": "PreTrainedTokenizerFast"
 }

       "special": true
     }
   },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
   "model_input_names": [
     "labels",
     "pixel_values"
   ],
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": "<pad>",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "<unk>"
 }