echo840 pbaylies commited on
Commit
8a44704
1 Parent(s): 3580119

Update tokenization_qwen.py (#1)

Browse files

- Update tokenization_qwen.py (15b1e871187cf7293e6ea234a1b42ee044e3f117)


Co-authored-by: Peter Baylies <[email protected]>

Files changed (1) hide show
  1. tokenization_qwen.py +1 -1
tokenization_qwen.py CHANGED
@@ -111,7 +111,6 @@ class QWenTokenizer(PreTrainedTokenizer):
111
  quad_end_tag='</quad>',
112
  **kwargs,
113
  ):
114
- super().__init__(**kwargs)
115
  self.image_start_tag = image_start_tag
116
  self.image_end_tag = image_end_tag
117
  self.image_pad_tag = image_pad_tag
@@ -128,6 +127,7 @@ class QWenTokenizer(PreTrainedTokenizer):
128
  image_start_tag, image_end_tag,
129
  image_pad_tag
130
  )
 
131
 
132
  self.errors = errors # how to handle errors in decoding
133
 
 
111
  quad_end_tag='</quad>',
112
  **kwargs,
113
  ):
 
114
  self.image_start_tag = image_start_tag
115
  self.image_end_tag = image_end_tag
116
  self.image_pad_tag = image_pad_tag
 
127
  image_start_tag, image_end_tag,
128
  image_pad_tag
129
  )
130
+ super().__init__(**kwargs)
131
 
132
  self.errors = errors # how to handle errors in decoding
133