Set legacy to True after initialization
Browse filestokenizer.tokenize() method doens't respect legacy if it's not set explicitly after initialization with legacy=False
- preprocessing/1/model.py +1 -0
preprocessing/1/model.py
CHANGED
@@ -73,6 +73,7 @@ class TritonPythonModel:
|
|
73 |
elif tokenizer_type == 'llama':
|
74 |
self.tokenizer = LlamaTokenizer.from_pretrained(
|
75 |
tokenizer_dir, legacy=False, padding_side='left')
|
|
|
76 |
else:
|
77 |
raise AttributeError(
|
78 |
f'Unexpected tokenizer type: {tokenizer_type}')
|
|
|
73 |
elif tokenizer_type == 'llama':
|
74 |
self.tokenizer = LlamaTokenizer.from_pretrained(
|
75 |
tokenizer_dir, legacy=False, padding_side='left')
|
76 |
+
self.tokenizer.legacy = True
|
77 |
else:
|
78 |
raise AttributeError(
|
79 |
f'Unexpected tokenizer type: {tokenizer_type}')
|