tokenizer hf 4.46.3
Browse files- tokenization_chatglm.py +1 -0
tokenization_chatglm.py
CHANGED
@@ -266,6 +266,7 @@ class ChatGLM4Tokenizer(PreTrainedTokenizer):
|
|
266 |
padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
|
267 |
pad_to_multiple_of: Optional[int] = None,
|
268 |
return_attention_mask: Optional[bool] = None,
|
|
|
269 |
) -> dict:
|
270 |
"""
|
271 |
Pad encoded inputs (on left/right and up to predefined length or max length in the batch)
|
|
|
266 |
padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
|
267 |
pad_to_multiple_of: Optional[int] = None,
|
268 |
return_attention_mask: Optional[bool] = None,
|
269 |
+
**kwargs
|
270 |
) -> dict:
|
271 |
"""
|
272 |
Pad encoded inputs (on left/right and up to predefined length or max length in the batch)
|