Upload processor

processor.py

@@ -3,27 +3,26 @@ from transformers import GitProcessor
 
 
 class GIAProcessor(GitProcessor):
-    def __init__(self, image_processor, tokenizer, max_input_size):
+    def __init__(self, image_processor, tokenizer):
         super().__init__(image_processor, tokenizer)
-        self.max_input_size = max_input_size
 
-    def _cut_text(self, examples):
+    def _cut_text(self, examples, max_input_size):
         results = {
             "input_ids": [],
             "attention_mask": []
         }
         for i in range(len(examples["input_ids"])):
             _input_size = len(examples["input_ids"][i])
-            for j in range(max(1, _input_size // self.max_input_size)):
-                results["input_ids"].append(examples["input_ids"][i][j*self.max_input_size:(j + 1) * self.max_input_size])
-                results["attention_mask"].append(examples["attention_mask"][i][j * self.max_input_size:(j + 1) * self.max_input_size])
+            for j in range(max(1, _input_size // max_input_size)):
+                results["input_ids"].append(examples["input_ids"][i][j*max_input_size:(j + 1) * max_input_size])
+                results["attention_mask"].append(examples["attention_mask"][i][j * max_input_size:(j + 1) * max_input_size])
 
         return results
 
-    def __call__(self, examples, return_tensors=None, **kwargs):
+    def __call__(self, examples, max_input_size, return_tensors=None, **kwargs):
         if "text" in examples and not "images" in examples:
             encoded_text = self.tokenizer(examples["text"], return_tensors=return_tensors)
-            encoding = self._cut_text(encoded_text)
+            encoding = self._cut_text(encoded_text, max_input_size)
         elif "text" in examples and "images" in examples:
             encoding = super().__call__(examples["text"], examples["images"], return_tensors, **kwargs)