Upload tokenizer

added_tokens.json

@@ -537,5 +537,7 @@
  "\\xlongequal": 32634,
  "\\xrightarrow": 32635,
  "\\your": 32636,
- "\\zeta": 32637
+ "\\zeta": 32637,
+ "{": 32639,
+ "}": 32640
 }

tokenizer.json

@@ -5794,6 +5794,24 @@
  "rstrip": false,
  "normalized": true,
  "special": false
+ },
+ {
+ "id": 32639,
+ "content": "{",
+ "single_word": false,
+ "lstrip": false,
+ "rstrip": false,
+ "normalized": true,
+ "special": false
+ },
+ {
+ "id": 32640,
+ "content": "}",
+ "single_word": false,
+ "lstrip": false,
+ "rstrip": false,
+ "normalized": true,
+ "special": false
  }
  ],
  "normalizer": {

tokenizer_config.json

@@ -105,7 +105,7 @@
  "eos_token": "</s>",
  "extra_ids": 100,
  "model_max_length": 512,
- "name_or_path": "./models/definition_naming_model_2024_06_03_2",
+ "name_or_path": "./models/definition_naming_model_2024_06_07",
  "pad_token": "<pad>",
  "special_tokens_map_file": null,
  "tokenizer_class": "T5Tokenizer",