Whitespace

tokenizer.json

@@ -33,10 +33,17 @@
   ],
   "normalizer": null,
   "pre_tokenizer": {
-    "type": "ByteLevel",
-    "add_prefix_space": false,
-    "trim_offsets": true,
-    "use_regex": true
+    "type": "Sequence",
+    "pretokenizers": [
+    {
+      "type": "Whitespace"
+    },
+    {
+      "type": "ByteLevel",
+      "add_prefix_space": false,
+      "trim_offsets": true,
+      "use_regex": true
+    }]
   },
   "post_processor": {
     "type": "ByteLevel",
@@ -11828,4 +11835,4 @@
       "ìĭ ł"
     ]
   }
-}
+}