Fill-Mask
Transformers
PyTorch
eurobert
code
custom_code
DuarteMRAlves commited on
Commit
1556890
·
verified ·
1 Parent(s): ce5fe03

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -5
tokenizer_config.json CHANGED
@@ -2055,15 +2055,11 @@
2055
  "eos_token": "<|end_of_text|>",
2056
  "extra_special_tokens": {},
2057
  "mask_token": "<|mask|>",
2058
- "max_length": 512,
2059
  "model_input_names": [
2060
  "input_ids",
2061
  "attention_mask"
2062
  ],
2063
- "model_max_length": 131072,
2064
- "pad_to_multiple_of": null,
2065
  "pad_token": "<|end_of_text|>",
2066
- "pad_token_type_id": 0,
2067
- "padding_side": "right",
2068
  "tokenizer_class": "PreTrainedTokenizer"
2069
  }
 
2055
  "eos_token": "<|end_of_text|>",
2056
  "extra_special_tokens": {},
2057
  "mask_token": "<|mask|>",
 
2058
  "model_input_names": [
2059
  "input_ids",
2060
  "attention_mask"
2061
  ],
2062
+ "model_max_length": 1000000000000000019884624838656,
 
2063
  "pad_token": "<|end_of_text|>",
 
 
2064
  "tokenizer_class": "PreTrainedTokenizer"
2065
  }