echo840 commited on
Commit
bb3eee6
1 Parent(s): e12c976

Update tokenization_qwen.py

Browse files
Files changed (1) hide show
  1. tokenization_qwen.py +2 -1
tokenization_qwen.py CHANGED
@@ -111,7 +111,7 @@ class QWenTokenizer(PreTrainedTokenizer):
111
  quad_end_tag='</quad>',
112
  **kwargs,
113
  ):
114
- super().__init__(**kwargs)
115
  self.image_start_tag = image_start_tag
116
  self.image_end_tag = image_end_tag
117
  self.image_pad_tag = image_pad_tag
@@ -128,6 +128,7 @@ class QWenTokenizer(PreTrainedTokenizer):
128
  image_start_tag, image_end_tag,
129
  image_pad_tag
130
  )
 
131
 
132
  self.errors = errors # how to handle errors in decoding
133
 
 
111
  quad_end_tag='</quad>',
112
  **kwargs,
113
  ):
114
+ # super().__init__(**kwargs)
115
  self.image_start_tag = image_start_tag
116
  self.image_end_tag = image_end_tag
117
  self.image_pad_tag = image_pad_tag
 
128
  image_start_tag, image_end_tag,
129
  image_pad_tag
130
  )
131
+ super().__init__(**kwargs)
132
 
133
  self.errors = errors # how to handle errors in decoding
134