Upload tokenizer

added_tokens.json

@@ -0,0 +1,4 @@
+{
+ "<eos>": 50258,
+ "<sos>": 50257
+}

special_tokens_map.json

@@ -0,0 +1,22 @@
+{
+ "additional_special_tokens": [
+ {
+ "content": "<sos>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false
+ },
+ {
+ "content": "<eos>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false
+ }
+ ],
+ "bos_token": "<|endoftext|>",
+ "eos_token": "<|endoftext|>",
+ "pad_token": "<|endoftext|>",
+ "unk_token": "<|endoftext|>"
+}

tokenizer_config.json

@@ -0,0 +1,41 @@
+{
+ "add_eos_token": true,
+ "add_prefix_space": false,
+ "added_tokens_decoder": {
+ "50256": {
+ "content": "<|endoftext|>",
+ "lstrip": false,
+ "normalized": true,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "50257": {
+ "content": "<sos>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "50258": {
+ "content": "<eos>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ }
+ },
+ "additional_special_tokens": [
+ "<sos>",
+ "<eos>"
+ ],
+ "bos_token": "<|endoftext|>",
+ "clean_up_tokenization_spaces": true,
+ "eos_token": "<|endoftext|>",
+ "model_max_length": 1024,
+ "pad_token": "<|endoftext|>",
+ "tokenizer_class": "GPT2Tokenizer",
+ "unk_token": "<|endoftext|>"
+}