Pernekhan commited on
Commit
95d9654
1 Parent(s): e4788f3

Set legacy to True after initialization

Browse files

tokenizer.tokenize() method doens't respect legacy if it's not set explicitly after initialization with legacy=False

Files changed (1) hide show
  1. preprocessing/1/model.py +1 -0
preprocessing/1/model.py CHANGED
@@ -73,6 +73,7 @@ class TritonPythonModel:
73
  elif tokenizer_type == 'llama':
74
  self.tokenizer = LlamaTokenizer.from_pretrained(
75
  tokenizer_dir, legacy=False, padding_side='left')
 
76
  else:
77
  raise AttributeError(
78
  f'Unexpected tokenizer type: {tokenizer_type}')
 
73
  elif tokenizer_type == 'llama':
74
  self.tokenizer = LlamaTokenizer.from_pretrained(
75
  tokenizer_dir, legacy=False, padding_side='left')
76
+ self.tokenizer.legacy = True
77
  else:
78
  raise AttributeError(
79
  f'Unexpected tokenizer type: {tokenizer_type}')