token-classification pipeline

config.json

@@ -12,6 +12,14 @@
     "upos": {
       "impl": "upos.BellmanFordTokenClassificationPipeline",
       "pt": "AutoModelForTokenClassification"
+    },
+    "token-classification": {
+      "impl": "upos.RawTokenClassificationPipeline",
+      "pt": "AutoModelForTokenClassification"
+    },
+    "ner": {
+      "impl": "upos.RawTokenClassificationPipeline",
+      "pt": "AutoModelForTokenClassification"
     }
   },
   "eos_token_id": 2,

upos.py

@@ -40,6 +40,10 @@ class BellmanFordTokenClassificationPipeline(TokenClassificationPipeline):
       t["text"]=model_outputs["sentence"][t["start"]:t["end"]]
     return w
 
+class RawTokenClassificationPipeline(TokenClassificationPipeline):
+  def check_model_type(self,supported_models):
+    pass
+
 class MistralForTokenClassification(MistralPreTrainedModel):
   def __init__(self,config):
     from torch import nn