v3

README.md

@@ -1,9 +1,9 @@
 ---
+language:
+- en
 license: apache-2.0
 datasets:
 - BEE-spoke-data/bees-internal
-language:
-- en
 ---
 
 # BeeTokenizer

tokenizer_config.json

@@ -64,7 +64,7 @@
  "single_word": false,
  "special": true
  },
- "32100": {
+ "32072": {
  "content": " ",
  "lstrip": false,
  "normalized": true,
@@ -72,7 +72,7 @@
  "single_word": false,
  "special": false
  },
- "32101": {
+ "32073": {
  "content": " ",
  "lstrip": false,
  "normalized": true,
@@ -80,7 +80,7 @@
  "single_word": false,
  "special": false
  },
- "32102": {
+ "32074": {
  "content": " ",
  "lstrip": false,
  "normalized": true,
@@ -88,7 +88,7 @@
  "single_word": false,
  "special": false
  },
- "32103": {
+ "32075": {
  "content": " ",
  "lstrip": false,
  "normalized": true,
@@ -96,7 +96,7 @@
  "single_word": false,
  "special": false
  },
- "32104": {
+ "32076": {
  "content": " ",
  "lstrip": false,
  "normalized": true,
@@ -104,7 +104,7 @@
  "single_word": false,
  "special": false
  },
- "32105": {
+ "32077": {
  "content": " ",
  "lstrip": false,
  "normalized": true,
@@ -112,7 +112,7 @@
  "single_word": false,
  "special": false
  },
- "32106": {
+ "32078": {
  "content": " ",
  "lstrip": false,
  "normalized": true,
@@ -120,7 +120,7 @@
  "single_word": false,
  "special": false
  },
- "32107": {
+ "32079": {
  "content": " ",
  "lstrip": false,
  "normalized": true,
@@ -128,7 +128,7 @@
  "single_word": false,
  "special": false
  },
- "32108": {
+ "32080": {
  "content": " ",
  "lstrip": false,
  "normalized": true,
@@ -136,7 +136,7 @@
  "single_word": false,
  "special": false
  },
- "32109": {
+ "32081": {
  "content": " ",
  "lstrip": false,
  "normalized": true,
@@ -144,7 +144,7 @@
  "single_word": false,
  "special": false
  },
- "32110": {
+ "32082": {
  "content": " ",
  "lstrip": false,
  "normalized": true,
@@ -152,7 +152,7 @@
  "single_word": false,
  "special": false
  },
- "32111": {
+ "32083": {
  "content": " ",
  "lstrip": false,
  "normalized": true,
@@ -160,7 +160,7 @@
  "single_word": false,
  "special": false
  },
- "32112": {
+ "32084": {
  "content": " ",
  "lstrip": false,
  "normalized": true,
@@ -168,7 +168,7 @@
  "single_word": false,
  "special": false
  },
- "32113": {
+ "32085": {
  "content": " ",
  "lstrip": false,
  "normalized": true,
@@ -176,7 +176,7 @@
  "single_word": false,
  "special": false
  },
- "32114": {
+ "32086": {
  "content": " ",
  "lstrip": false,
  "normalized": true,
@@ -184,7 +184,7 @@
  "single_word": false,
  "special": false
  },
- "32115": {
+ "32087": {
  "content": " ",
  "lstrip": false,
  "normalized": true,
@@ -192,7 +192,7 @@
  "single_word": false,
  "special": false
  },
- "32116": {
+ "32088": {
  "content": " ",
  "lstrip": false,
  "normalized": true,
@@ -200,7 +200,7 @@
  "single_word": false,
  "special": false
  },
- "32117": {
+ "32089": {
  "content": " ",
  "lstrip": false,
  "normalized": true,
@@ -208,7 +208,7 @@
  "single_word": false,
  "special": false
  },
- "32118": {
+ "32090": {
  "content": " ",
  "lstrip": false,
  "normalized": true,
@@ -216,7 +216,7 @@
  "single_word": false,
  "special": false
  },
- "32119": {
+ "32091": {
  "content": " ",
  "lstrip": false,
  "normalized": true,
@@ -224,7 +224,7 @@
  "single_word": false,
  "special": false
  },
- "32120": {
+ "32092": {
  "content": " ",
  "lstrip": false,
  "normalized": true,
@@ -232,7 +232,7 @@
  "single_word": false,
  "special": false
  },
- "32121": {
+ "32093": {
  "content": " ",
  "lstrip": false,
  "normalized": true,
@@ -240,7 +240,7 @@
  "single_word": false,
  "special": false
  },
- "32122": {
+ "32094": {
  "content": " ",
  "lstrip": false,
  "normalized": true,
@@ -248,7 +248,7 @@
  "single_word": false,
  "special": false
  },
- "32123": {
+ "32095": {
  "content": " ",
  "lstrip": false,
  "normalized": true,
@@ -256,7 +256,7 @@
  "single_word": false,
  "special": false
  },
- "32124": {
+ "32096": {
  "content": " ",
  "lstrip": false,
  "normalized": true,
@@ -264,7 +264,7 @@
  "single_word": false,
  "special": false
  },
- "32125": {
+ "32097": {
  "content": " ",
  "lstrip": false,
  "normalized": true,
@@ -272,7 +272,7 @@
  "single_word": false,
  "special": false
  },
- "32126": {
+ "32098": {
  "content": " ",
  "lstrip": false,
  "normalized": true,
@@ -280,7 +280,7 @@
  "single_word": false,
  "special": false
  },
- "32127": {
+ "32099": {
  "content": " ",
  "lstrip": false,
  "normalized": true,