d7bba61 401f01a d7bba61
1
2
3
4
5
6
7
8
9
10
11
12
from transformers import GPTNeoXTokenizerFast class MambaTokenizerFast(GPTNeoXTokenizerFast): def __init__( self, **kwargs, ): super().__init__(**kwargs) self.pad_token_id = self.eos_token_id
