Upload BertItaliano

config.json

@@ -0,0 +1,27 @@
+{
+  "architectures": [
+    "BertItaliano"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "auto_map": {
+    "AutoConfig": "configuration_moral.BertItalianoConfig",
+    "AutoModel": "modeling_moral.BertItaliano"
+  },
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "BertItaliano",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.33.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 32102
+}

configuration_moral.py

@@ -0,0 +1,46 @@
+from transformers import PretrainedConfig
+
+class BertItalianoConfig(PretrainedConfig):
+    model_type="BertItaliano"
+
+    def __init__(
+        self,
+        attention_probs_dropout_prob: float = 0.1,
+        gradient_checkpointing: bool = False,
+        hidden_act="gelu",
+        hidden_dropout_prob: float = 0.1,
+        hidden_size: int = 768,
+        initializer_range: float = 0.02,
+        intermediate_size: int = 3072,
+        layer_norm_eps: float = 1e-12,
+        max_position_embeddings: int = 512,
+        num_attention_heads: int = 12,
+        num_hidden_layers: int = 12,
+        pad_token_id: int = 0,
+        position_embedding_type="absolute",
+        transformers_version="4.10.3",
+        torch_dtype="float32",
+        type_vocab_size: int = 2,
+        use_cache: bool = True,
+        vocab_size: int = 32102,                
+        **kwargs,
+    ):
+        self.attention_probs_dropout_prob = attention_probs_dropout_prob
+        self.gradient_checkpointing = gradient_checkpointing
+        self.hidden_act = hidden_act
+        self.hidden_dropout_prob = hidden_dropout_prob
+        self.hidden_size = hidden_size
+        self.initializer_range = initializer_range
+        self.intermediate_size = intermediate_size
+        self.layer_norm_eps = layer_norm_eps
+        self.max_position_embeddings = max_position_embeddings
+        self.num_attention_heads = num_attention_heads
+        self.num_hidden_layers = num_hidden_layers
+        self.pad_token_id = pad_token_id
+        self.position_embedding_type = position_embedding_type
+        self.transformers_version = transformers_version
+        self.torch_dtype = torch_dtype
+        self.type_vocab_size = type_vocab_size
+        self.use_cache = use_cache
+        self.vocab_size = vocab_size        
+        super().__init__(**kwargs)

modeling_moral.py

@@ -0,0 +1,24 @@
+from transformers import PreTrainedModel, AutoModel
+from .configuration_moral import BertItalianoConfig
+import torch
+
+class BertItaliano(PreTrainedModel):
+    config_class = BertItalianoConfig
+    
+    def __init__(self, config, num_labels_1=6, num_labels_2=3):
+        super(BertItaliano, self).__init__(config)
+        self.num_labels1 = num_labels_1
+        self.num_labels2 = num_labels_2
+        self.bert = AutoModel.from_pretrained("dbmdz/bert-base-italian-xxl-uncased")
+        self.dropout = torch.nn.Dropout(0.3)
+        self.linear_1 = torch.nn.Linear(config.hidden_size, num_labels_1)
+        self.linear_2 = torch.nn.Linear(config.hidden_size, num_labels_2)
+        
+    def forward(self, input_ids, attention_mask=None, token_type_ids=None):
+        outputs = self.bert(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)
+        pooled_output = outputs[1]
+        pooled_output = self.dropout(pooled_output)
+        logits_1 = self.linear_1(pooled_output)
+        logits_2 = self.linear_2(pooled_output)
+        
+        return logits_1, logits_2

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a5196b67f9b5af7a439315d733e6ae306f158519657f2c7263aa0dc4b39ebe4
+size 442874731