Upload 5 files

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "eos_token": "</s>",
+  "mask_token": "…",
+  "unk_token": "<unk>"
+}

token_fraction_describe.txt

@@ -0,0 +1,16 @@
+              token          text    token/text
+count  19980.000000  19980.000000  19980.000000
+mean      87.750000    181.553804      0.494357
+std      237.185668    374.992707      0.100927
+min        7.000000     41.000000      0.134615
+10%       24.000000     46.000000      0.387381
+25%       34.000000     64.000000      0.432000
+33%       42.000000     85.000000      0.449829
+50%       66.000000    139.000000      0.483740
+67%       97.000000    205.000000      0.521950
+75%      117.000000    245.000000      0.545455
+80%      132.000000    277.000000      0.562044
+90%      176.100000    371.000000      0.619048
+95%      221.000000    461.000000      0.674419
+99%      325.210000    666.000000      0.804878
+max    32162.000000  49156.000000      1.024390

tokenizer_config.json

@@ -0,0 +1,42 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "…",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8255": {
+      "content": "<0x{i:02X}>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "</s>",
+  "mask_token": "…",
+  "model_max_length": 1000000000000000019884624838656,
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "<unk>"
+}