Commit
·
1ff269b
1
Parent(s):
c967539
Upload processor
Browse files- processor.py +5 -5
processor.py
CHANGED
@@ -20,12 +20,12 @@ class GIAProcessor(GitProcessor):
|
|
20 |
}
|
21 |
return result
|
22 |
|
23 |
-
def __call__(self,
|
24 |
-
if text
|
25 |
-
encoded_text = self.tokenizer(text, return_tensors=return_tensors)
|
26 |
encoding = self._group_texts(encoded_text)
|
27 |
-
elif text
|
28 |
-
encoding = super().__call__(text, images, return_tensors, **kwargs)
|
29 |
|
30 |
return encoding
|
31 |
|
|
|
20 |
}
|
21 |
return result
|
22 |
|
23 |
+
def __call__(self, examples, return_tensors=None, **kwargs):
|
24 |
+
if "text" in examples and not "images" in examples:
|
25 |
+
encoded_text = self.tokenizer(examples["text"], return_tensors=return_tensors)
|
26 |
encoding = self._group_texts(encoded_text)
|
27 |
+
elif "text" in examples and "images" in examples:
|
28 |
+
encoding = super().__call__(examples["text"], examples["images"], return_tensors, **kwargs)
|
29 |
|
30 |
return encoding
|
31 |
|