dataset ingles

app.py

@@ -331,16 +331,16 @@ class ModeloDataset:
         new_tokens=[]
         ig_tokens=[] 
         for token in tokens:
-            print('tokensss:',tokens,caracter)
+            print('token_texto:',token,caracter)
             ind=len(new_tokens)
             if i<len(tokens):
-                 if  token.startswith(caracter):
+                 if not token.startswith(caracter):
                     
                     new_tokens.append(token)
                     
                     i=i+1
                  else:
-                    new_tokens[ind-1] = (new_tokens[ind-1] + token)
+                    new_tokens[ind-1] = (new_tokens[ind-1] + token.replace(caracter,''))
                     ig_tokens.append(i)
                     
                     i=i+1
@@ -348,6 +348,32 @@ class ModeloDataset:
                 new_tokens,
                 ig_tokens
                 )  
+
+    def reordenacion_tokens_es(self,tokens,caracter):
+    
+     i=0
+     new_tokens=[]
+     ig_tokens=[] #ignorar estos indices del array de indentificadores
+     for token in tokens:
+           ind=len(new_tokens)
+           if i<len(tokens):
+                if  token.startswith(caracter):
+                    
+                    new_tokens.append(token)
+                    
+                    i=i+1
+                else:
+                #if i==0: new_tokens.append(token)
+                #else:                          
+                    new_tokens[ind-1] = (new_tokens[ind-1] + token.replace(caracter,''))
+                    ig_tokens.append(i)
+                
+                    i=i+1
+     return (
+                new_tokens,
+                ig_tokens
+                )       
+
     def reordenacion_identificadores(self,ig_tokens,predicted_tokens_classes, tamano):
         x=0
         new_identificadores=[]
@@ -449,29 +475,42 @@ class ModeloDataset:
         else:
             
             print('idioma:',idioma)            
-            self.tokenizer = AutoTokenizer.from_pretrained("FacebookAI/xlm-roberta-large-finetuned-conll03-english")
-            tokenized_text=[self.tokenizer.tokenize(sentence[:500]) for sentence in _sentences]
-        
-            ids = [self.tokenizer.convert_tokens_to_ids(x) for x in tokenized_text]
+            self.tokenizer = AutoTokenizer.from_pretrained("dayannex/distilbert-tuned-4labels")
+            self.model = AutoModelForTokenClassification.from_pretrained("dayannex/distilbert-tuned-4labels")     
+
+            inputs=[self.tokenizer(sentence[:500], return_tensors="pt") for sentence in _sentences]
+            print('inputs',inputs)
+            #ids = [self.tokenizer.convert_tokens_to_ids(x) for x in tokenized_text]
             
 
-            MAX_LEN=128
-            ids=pad_sequences(ids,maxlen=MAX_LEN,dtype="long",truncating="post", padding="post")
-            input_ids = torch.tensor(ids) 
+            #MAX_LEN=128
+            #ids=pad_sequences(ids,maxlen=MAX_LEN,dtype="long",truncating="post", padding="post")
+            #input_ids = torch.tensor(ids) 
             
-                    
-            self.model = AutoModelForTokenClassification.from_pretrained("FacebookAI/xlm-roberta-large-finetuned-conll03-english")
             with torch.no_grad():
-             logits = self.model(input_ids).logits
-            predicted_token_class_ids = logits.argmax(-1)
+                outputs = self.model(**inputs)
+            logits = outputs.logits   
+            predicted_token_class_ids = torch.argmax(logits, dim=2)     
+            
+            #predicted_token_class_ids = predicted_token_class_ids[0].tolist()
             i=0
             _predicted_tokens_classes=[]
             for a in predicted_token_class_ids:
                     
-                    _predicted_tokens_classes.append([self.model.config.id2label[t.item()] for t in predicted_token_class_ids[i]])
-                    i=i+1  
-            labels = predicted_token_class_ids
-            loss = self.model(input_ids, labels=labels).loss
+                    _predicted_tokens_classes.append( [self.model.config.id2label[label_id] for label_id in predicted_token_class_ids[i]])
+                    i=i+1 
+            print('_predicted_tokens_classes:',_predicted_tokens_classes[0])
+            #with torch.no_grad():
+            # logits = self.model(input_ids).logits
+            #predicted_token_class_ids = logits.argmax(-1)
+            #i=0
+            #_predicted_tokens_classes=[]
+            #for a in predicted_token_class_ids:
+                    
+            #        _predicted_tokens_classes.append([self.model.config.id2label[t.item()] for t in predicted_token_class_ids[i]])
+            #        i=i+1  
+            #labels = predicted_token_class_ids
+            #loss = self.model(input_ids, labels=labels).loss
 
             new_tokens=[]
             ig_tok=[]
@@ -479,7 +518,7 @@ class ModeloDataset:
             new_identificadores=[]
             for item in tokenized_text:
                
-               aux1, aux2= self.reordenacion_tokens(item,"▁")
+               aux1, aux2= self.reordenacion_tokens(item,"#")
                new_tokens.append(aux1)
                ig_tok.append(aux2)