answerdotai
/

ModernBERT-base

Model card Files Files and versions Community

Tom Aarsen commited on 10 days ago

Commit

2b01747

•

1 Parent(s): 51c9b61

Also update tokenizer/special_tokens_map

Files changed (2) hide show

special_tokens_map.json +1 -1
tokenizer.json +1 -1

special_tokens_map.json CHANGED Viewed

@@ -8,7 +8,7 @@
   },
   "mask_token": {
     "content": "[MASK]",
-    "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false

   },
   "mask_token": {
     "content": "[MASK]",
+    "lstrip": true,
     "normalized": false,
     "rstrip": false,
     "single_word": false

tokenizer.json CHANGED Viewed

@@ -295,7 +295,7 @@
       "id": 50284,
       "content": "[MASK]",
       "single_word": false,
-      "lstrip": false,
       "rstrip": false,
       "normalized": false,
       "special": true

       "id": 50284,
       "content": "[MASK]",
       "single_word": false,
+      "lstrip": true,
       "rstrip": false,
       "normalized": false,
       "special": true