Upload 4 files

__init__.py

@@ -0,0 +1,4 @@
+# __init__.py
+from .modeling_custom import CustomModel
+
+__all__ = ["CustomModel"]

config (2).json

@@ -0,0 +1,41 @@
+{
+  "_name_or_path": "gsar78/HellenicSentimentAI",
+  "architectures": [
+    "CustomModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "eos_token_id": 2,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "negative",
+    "1": "neutral",
+    "2": "positive"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "negative": 0,
+    "neutral": 1,
+    "positive": 2
+  },
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "xlm-roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "output_past": true,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "problem_type": "multi_label_classification",
+  "torch_dtype": "float32",
+  "transformers_version": "4.42.3",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 250002,
+  "num_emotion_labels": 18  // Custom parameter
+}

modeling.py

@@ -0,0 +1,9 @@
+# modeling.py
+from transformers import XLMRobertaForSequenceClassification, AutoConfig
+from .modeling_custom import CustomModel
+
+def from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs):
+    config = AutoConfig.from_pretrained(pretrained_model_name_or_path)
+    num_emotion_labels = config.num_emotion_labels
+    model = CustomModel.from_pretrained(pretrained_model_name_or_path, num_emotion_labels=num_emotion_labels, *model_args, **kwargs)
+    return model

modeling_custom.py

@@ -0,0 +1,55 @@
+# modeling_custom.py
+from transformers import XLMRobertaForSequenceClassification
+import torch.nn as nn
+import torch.nn.functional as F
+
+class CustomModel(XLMRobertaForSequenceClassification):
+    def __init__(self, config, num_emotion_labels):
+        super(CustomModel, self).__init__(config)
+        self.num_emotion_labels = num_emotion_labels
+
+        # Freeze sentiment classifier parameters
+        for param in self.classifier.parameters():
+            param.requires_grad = False
+
+        # Define emotion classifier
+        self.dropout_emotion = nn.Dropout(config.hidden_dropout_prob)
+        self.emotion_classifier = nn.Sequential(
+            nn.Linear(config.hidden_size, 512),
+            nn.Mish(),
+            nn.Dropout(0.3),
+            nn.Linear(512, num_emotion_labels)
+        )
+
+        # Initialize the weights of the new layers
+        self._init_weights(self.emotion_classifier[0])
+        self._init_weights(self.emotion_classifier[3])
+
+    def _init_weights(self, module):
+        if isinstance(module, nn.Linear):
+            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
+            if module.bias is not None:
+                module.bias.data.zero_()
+
+    def forward(self, input_ids=None, attention_mask=None, sentiment=None, labels=None):
+        outputs = self.roberta(input_ids=input_ids, attention_mask=attention_mask)
+        sequence_output = outputs[0]
+
+        # Select the CLS token for emotion classification
+        cls_hidden_states = sequence_output[:, 0, :]
+        cls_hidden_states = self.dropout_emotion(cls_hidden_states)
+
+        emotion_logits = self.emotion_classifier(cls_hidden_states)
+
+        # Sentiment logits from the frozen classifier
+        with torch.no_grad():
+            cls_token_state = sequence_output[:, 0, :].unsqueeze(1)
+            sentiment_logits = self.classifier(cls_token_state).squeeze(1)
+
+        if labels is not None:
+            class_weights = torch.tensor([1.0] * self.num_emotion_labels).to(labels.device)
+            loss_fct = nn.BCEWithLogitsLoss(pos_weight=class_weights)
+            loss = loss_fct(emotion_logits, labels)
+            return {"loss": loss, "emotion_logits": emotion_logits, "sentiment_logits": sentiment_logits}
+
+        return {"emotion_logits": emotion_logits, "sentiment_logits": sentiment_logits}