Upload folder using huggingface_hub

modeling_sentence_tokenizer.py

@@ -11,12 +11,14 @@ class SentenceTokenizerConfig(PretrainedConfig):
         min_length=32,
         max_length=64,
         n_overlap=3,
+        roll=False,
         **kwargs
     ):
         super().__init__(**kwargs)
         self.min_length = min_length
         self.max_length = max_length
         self.n_overlap = n_overlap
+        self.roll = roll
 
 class SentenceTokenizer(PreTrainedModel):
     config_class = SentenceTokenizerConfig
@@ -27,6 +29,7 @@ class SentenceTokenizer(PreTrainedModel):
         self.min_length = config.min_length
         self.max_length = config.max_length
         self.n_overlap = config.n_overlap
+        self.roll = config.roll
 
     def split_text_into_sentences(self, text):
         split_text = re.split(r'([^가-힣] )', text)
@@ -117,10 +120,10 @@ class SentenceTokenizer(PreTrainedModel):
         filtered_text = [s + sp for s, sp in zip(split_text[::2], split_text[1::2] + [''])] 
         return filtered_text
     
-    def overlap(self, chunks, roll=False):
+    def overlap(self, chunks):
         if not chunks:
             return []
-        if roll==True:
+        if self.roll:
             chunks = [chunks[-1]] + chunks + [chunks[0]]
         res = []
         total_idx = 0
@@ -139,7 +142,7 @@ class SentenceTokenizer(PreTrainedModel):
 
         return res
 
-    def decode_overlap(self, chunks, roll=False):
+    def decode_overlap(self, chunks):
         if not chunks:
             return ""
         
@@ -162,7 +165,7 @@ class SentenceTokenizer(PreTrainedModel):
             most_common_char, _ = Counter(index_char_map[i]).most_common(1)[0]
             reconstructed_text.append(most_common_char)
         res = "".join(reconstructed_text)
-        if roll==True:
+        if self.roll:
             res = res[len(chunks[0][2]):-len(chunks[-1][2])]
     
         return res

sentence_tokenizer/config.json

@@ -3,13 +3,20 @@
     "SentenceTokenizer"
   ],
   "auto_map": {
-    "AutoConfig": "modeling_sentence_tokenizer.SentenceTokenizerConfig",
-    "AutoModel": "modeling_sentence_tokenizer.SentenceTokenizer"
+    "AutoConfig": [
+      "modeling_sentence_tokenizer.SentenceTokenizerConfig",
+      null
+    ],
+    "AutoModel": [
+      "modeling_sentence_tokenizer.SentenceTokenizer",
+      null
+    ]
   },
   "max_length": 64,
   "min_length": 32,
   "model_type": "sentence_tokenizer",
   "n_overlap": 3,
+  "roll": false,
   "torch_dtype": "float32",
-  "transformers_version": "4.48.0"
+  "transformers_version": "4.50.2"
 }

sentence_tokenizer/modeling_sentence_tokenizer.py

@@ -11,12 +11,14 @@ class SentenceTokenizerConfig(PretrainedConfig):
         min_length=32,
         max_length=64,
         n_overlap=3,
+        roll=False,
         **kwargs
     ):
         super().__init__(**kwargs)
         self.min_length = min_length
         self.max_length = max_length
         self.n_overlap = n_overlap
+        self.roll = roll
 
 class SentenceTokenizer(PreTrainedModel):
     config_class = SentenceTokenizerConfig
@@ -27,6 +29,7 @@ class SentenceTokenizer(PreTrainedModel):
         self.min_length = config.min_length
         self.max_length = config.max_length
         self.n_overlap = config.n_overlap
+        self.roll = config.roll
 
     def split_text_into_sentences(self, text):
         split_text = re.split(r'([^가-힣] )', text)
@@ -117,10 +120,10 @@ class SentenceTokenizer(PreTrainedModel):
         filtered_text = [s + sp for s, sp in zip(split_text[::2], split_text[1::2] + [''])] 
         return filtered_text
     
-    def overlap(self, chunks, roll=False):
+    def overlap(self, chunks):
         if not chunks:
             return []
-        if roll==True:
+        if self.roll:
             chunks = [chunks[-1]] + chunks + [chunks[0]]
         res = []
         total_idx = 0
@@ -139,7 +142,7 @@ class SentenceTokenizer(PreTrainedModel):
 
         return res
 
-    def decode_overlap(self, chunks, roll=False):
+    def decode_overlap(self, chunks):
         if not chunks:
             return ""
         
@@ -162,7 +165,7 @@ class SentenceTokenizer(PreTrainedModel):
             most_common_char, _ = Counter(index_char_map[i]).most_common(1)[0]
             reconstructed_text.append(most_common_char)
         res = "".join(reconstructed_text)
-        if roll==True:
+        if self.roll:
             res = res[len(chunks[0][2]):-len(chunks[-1][2])]
     
-        return res
+        return res