commit files to HF hub

.gitattributes

@@ -34,3 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 vocab.txt filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

config.json

@@ -1,12 +1,22 @@
 {
+  "_name_or_path": "Hiveurban/dictabert-large-parse",
   "architectures": [
     "BertForJointParsing"
   ],
+  "attention_probs_dropout_prob": 0.1,
   "auto_map": {
     "AutoModel": "dicta-il/dictabert-joint--BertForJointParsing.BertForJointParsing"
   },
-  "attention_probs_dropout_prob": 0.1,
   "classifier_dropout": null,
+  "custom_pipelines": {
+    "hive-token-classification": {
+      "impl": "hive_token_classification.HiveTokenClassification",
+      "pt": [
+        "AutoModel"
+      ],
+      "tf": []
+    }
+  },
   "do_lex": true,
   "do_morph": true,
   "do_ner": true,
@@ -83,9 +93,41 @@
   "num_hidden_layers": 24,
   "pad_token_id": 0,
   "position_embedding_type": "absolute",
+  "prefix_cfg": {
+    "possible_classes": [
+      [
+        "\u05dc\u05db\u05e9",
+        "\u05db\u05e9",
+        "\u05de\u05e9",
+        "\u05d1\u05e9",
+        "\u05dc\u05e9"
+      ],
+      [
+        "\u05de"
+      ],
+      [
+        "\u05e9"
+      ],
+      [
+        "\u05d4"
+      ],
+      [
+        "\u05d5"
+      ],
+      [
+        "\u05db"
+      ],
+      [
+        "\u05dc"
+      ],
+      [
+        "\u05d1"
+      ]
+    ]
+  },
   "syntax_head_size": 128,
   "torch_dtype": "float32",
-  "transformers_version": "4.36.2",
+  "transformers_version": "4.44.2",
   "type_vocab_size": 2,
   "use_cache": true,
   "vocab_size": 128000

hive_token_classification.py

@@ -0,0 +1,20 @@
+from typing import Any, Dict
+from transformers import Pipeline, AutoModel, AutoTokenizer
+from transformers.pipelines.base import GenericTensor, ModelOutput
+
+
+class HiveTokenClassification(Pipeline):
+    def _sanitize_parameters(self, **kwargs):
+        forward_parameters = {}
+        if "output_style" in kwargs:
+            forward_parameters["output_style"] = kwargs["output_style"]
+        return {}, forward_parameters, {}
+
+    def preprocess(self, input_: Any, **preprocess_parameters: Dict) -> Dict[str, GenericTensor]:
+        return input_
+
+    def _forward(self, input_tensors: Dict[str, GenericTensor], **forward_parameters: Dict) -> ModelOutput:
+        return self.model.predict(input_tensors, self.tokenizer, **forward_parameters)
+
+    def postprocess(self, model_outputs: ModelOutput, **postprocess_parameters: Dict) -> Any:
+        return {"output": model_outputs, "length": len(model_outputs)}