updated model

README.md

@@ -1,3 +1,11 @@
----
-license: mit
----
+---
+license: mit
+---
+```python
+# Load the model
+from transformers import AutoModelForMaskedLM, AutoTokenizer
+model_name = "johahi/specieslm-metazoa-upstream-k6"
+model = AutoModelForMaskedLM.from_pretrained(model_name, trust_remote_code=True)
+# Load the corresponding tokenizer
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+```

config.json

@@ -1,7 +1,9 @@
 {
-  "_name_or_path": "/s/project/denovo-prosit/JohannesHingerl/BERTADN/final_models/huggingface_compatible/rotary_bert_huggingface",
+  "_name_or_path": "johahi/specieslm-metazoa-upstream-k6",
   "auto_map":
-      {"AutoModel": "modeling_rotarybert.RotaryBertModel",
+      {
+	"AutoConfig": "configuration_rotarybert.RotaryBertConfig",
+	"AutoModel": "modeling_rotarybert.RotaryBertModel",
         "AutoModelForMaskedLM": "modeling_rotarybert.RotaryBertForMaskedLM"
       },
   "attention_probs_dropout_prob": 0.1,

configuration_rotarybert.py

@@ -0,0 +1,8 @@
+from transformers.models.bert.configuration_bert import BertConfig
+
+print ("loading this")
+
+class RotaryBertConfig(BertConfig):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+

generation_config.json

@@ -1,5 +0,0 @@
-{
-  "_from_model_config": true,
-  "pad_token_id": 0,
-  "transformers_version": "4.45.2"
-}

modeling_rotarybert.py

@@ -5,6 +5,7 @@ from typing import List, Optional, Tuple, Union
 from packaging import version
 import torch
 import torch.nn as nn
+from .configuration_rotarybert import RotaryBertConfig
 
 def rotate_half(x, interleaved=False):
     if not interleaved:
@@ -157,6 +158,7 @@ class RotaryBertModel(BertModel):
         self.encoder = RotaryBertEncoder(config)
     
 class RotaryBertForMaskedLM(BertForMaskedLM):
+    config_class = RotaryBertConfig
     def __init__(self, config):
         super().__init__(config)
         self.bert = RotaryBertModel(config)