Upload 3 files

config.json

@@ -1,12 +1,12 @@
 {
-  "_name_or_path": "babylm/ltgbert-100m-2024",
+  "_name_or_path": "ltg/ltg-bert-babylm",
   "architectures": [
     "LtgBertForMaskedLM"
   ],
   "attention_probs_dropout_prob": 0.1,
   "auto_map": {
-    "AutoConfig": "ltg/ltg-bert-babylm--configuration_ltgbert.LtgBertConfig",
-    "AutoModelForMaskedLM": "ltg/ltg-bert-babylm--modeling_ltgbert.LtgBertForMaskedLM",
+    "AutoConfig": "configuration_ltgbert.LtgBertConfig",
+    "AutoModelForMaskedLM": "modeling_ltgbert.LtgBertForMaskedLM",
     "AutoModelForSequenceClassification": "modeling_ltgbert.LtgBertForSequenceClassification"
   },
   "classifier_dropout": 0.2,
@@ -22,6 +22,6 @@
   "pad_token_id": 4,
   "position_bucket_size": 32,
   "torch_dtype": "float32",
-  "transformers_version": "4.43.3",
+  "transformers_version": "4.40.2",
   "vocab_size": 16384
 }

configuration_ltgbert.py

@@ -19,6 +19,30 @@
 from transformers.configuration_utils import PretrainedConfig
 
 
+LTG_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
+    "bnc-bert-span": "https://huggingface.co/ltg/bnc-bert-span",
+    "bnc-bert-span-2x": "https://huggingface.co/ltg/bnc-bert-span-2x",
+    "bnc-bert-span-0.5x": "https://huggingface.co/ltg/bnc-bert-span-0.5x",
+    "bnc-bert-span-0.25x": "https://huggingface.co/ltg/bnc-bert-span-0.25x",
+    "bnc-bert-span-order": "https://huggingface.co/ltg/bnc-bert-span-order",
+    "bnc-bert-span-document": "https://huggingface.co/ltg/bnc-bert-span-document",
+    "bnc-bert-span-word": "https://huggingface.co/ltg/bnc-bert-span-word",
+    "bnc-bert-span-subword": "https://huggingface.co/ltg/bnc-bert-span-subword",
+
+    "norbert3-xs": "https://huggingface.co/ltg/norbert3-xs/config.json",
+    "norbert3-small": "https://huggingface.co/ltg/norbert3-small/config.json",
+    "norbert3-base": "https://huggingface.co/ltg/norbert3-base/config.json",
+    "norbert3-large": "https://huggingface.co/ltg/norbert3-large/config.json",
+
+    "norbert3-oversampled-base": "https://huggingface.co/ltg/norbert3-oversampled-base/config.json",
+    "norbert3-ncc-base": "https://huggingface.co/ltg/norbert3-ncc-base/config.json",
+    "norbert3-nak-base": "https://huggingface.co/ltg/norbert3-nak-base/config.json",
+    "norbert3-nb-base": "https://huggingface.co/ltg/norbert3-nb-base/config.json",
+    "norbert3-wiki-base": "https://huggingface.co/ltg/norbert3-wiki-base/config.json",
+    "norbert3-c4-base": "https://huggingface.co/ltg/norbert3-c4-base/config.json"
+}
+
+
 class LtgBertConfig(PretrainedConfig):
     r"""
     This is the configuration class to store the configuration of a [`LtgBertModel`]. It is used to
@@ -49,7 +73,7 @@ class LtgBertConfig(PretrainedConfig):
         classifier_dropout (`float`, *optional*):
             The dropout ratio for the classification head.
     """
-    model_type = "bert"
+    model_type = "ltgbert"
     def __init__(
         self,
         vocab_size=16384,
@@ -80,4 +104,4 @@ class LtgBertConfig(PretrainedConfig):
         self.output_all_encoded_layers = output_all_encoded_layers
         self.position_bucket_size = position_bucket_size
         self.layer_norm_eps = layer_norm_eps
-        self.classifier_dropout = classifier_dropout
+        self.classifier_dropout = classifier_dropout

modeling_ltgbert.py

@@ -39,10 +39,34 @@ from transformers.pytorch_utils import softmax_backward_data
 from transformers.utils import add_start_docstrings, add_start_docstrings_to_model_forward
 
 
-_CHECKPOINT_FOR_DOC = "ltg/ltg-bert-bnc"
+_CHECKPOINT_FOR_DOC = "ltg/bnc-bert-span"
 _CONFIG_FOR_DOC = "LtgBertConfig"
 
 
+LTG_BERT_PRETRAINED_MODEL_ARCHIVE_LIST = [
+    "bnc-bert-span",
+    "bnc-bert-span-2x",
+    "bnc-bert-span-0.5x",
+    "bnc-bert-span-0.25x",
+    "bnc-bert-span-order",
+    "bnc-bert-span-document",
+    "bnc-bert-span-word",
+    "bnc-bert-span-subword",
+
+    "norbert3-xs",
+    "norbert3-small",
+    "norbert3-base",
+    "norbert3-large",
+
+    "norbert3-oversampled-base",
+    "norbert3-ncc-base",
+    "norbert3-nak-base",
+    "norbert3-nb-base",
+    "norbert3-wiki-base",
+    "norbert3-c4-base"
+]
+
+
 class Encoder(nn.Module):
     def __init__(self, config, activation_checkpointing=False):
         super().__init__()
@@ -224,8 +248,10 @@ class Attention(nn.Module):
 
         attention_scores = torch.bmm(query, key.transpose(1, 2) * self.scale)
 
-        pos = self.in_proj_qk(self.dropout(relative_embedding))  # shape: [2T-1, 2D]
-        query_pos, key_pos = pos.view(-1, self.num_heads, 2*self.head_size).chunk(2, dim=2)
+        query_pos, key_pos = self.in_proj_qk(self.dropout(relative_embedding)).chunk(2, dim=-1)  # shape: [2T-1, D]
+        query_pos = query_pos.view(-1, self.num_heads, self.head_size)  # shape: [2T-1, H, D]
+        key_pos = key_pos.view(-1, self.num_heads, self.head_size)  # shape: [2T-1, H, D]
+
         query = query.view(batch_size, self.num_heads, query_len, self.head_size)
         key = key.view(batch_size, self.num_heads, query_len, self.head_size)
 
@@ -367,8 +393,6 @@ class LtgBertModel(LtgBertPreTrainedModel):
     ) -> List[torch.Tensor]:
         if input_ids is not None:
             input_shape = input_ids.size()
-        # elif inputs_embeds is not None:
-        #     input_shape = inputs_embeds.size()[:-1]
         else:
             raise ValueError("You have to specify input_ids")
 
@@ -380,9 +404,7 @@ class LtgBertModel(LtgBertPreTrainedModel):
         else:
             attention_mask = ~attention_mask.bool()
         attention_mask = attention_mask.unsqueeze(1).unsqueeze(2)
-
-        # if inputs_embeds is None:
-        #     static_embeddings, relative_embedding = self.embedding(input_ids.t())
+ 
         static_embeddings, relative_embedding = self.embedding(input_ids.t())
         contextualized_embeddings, attention_probs = self.transformer(static_embeddings, attention_mask, relative_embedding)
         contextualized_embeddings = [e.transpose(0, 1) for e in contextualized_embeddings]
@@ -409,7 +431,8 @@ class LtgBertModel(LtgBertPreTrainedModel):
         )
         return_dict = return_dict if return_dict is not None else self.config.use_return_dict
 
-        sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids, attention_mask)
+        sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids=input_ids,
+                                                                                                         attention_mask=attention_mask)
 
         if not return_dict:
             return (
@@ -456,7 +479,8 @@ class LtgBertForMaskedLM(LtgBertModel):
         """
         return_dict = return_dict if return_dict is not None else self.config.use_return_dict
 
-        sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids, attention_mask)
+        sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids=input_ids,
+                                                                                                         attention_mask=attention_mask)
         subword_prediction = self.classifier(sequence_output)
 
         masked_lm_loss = None
@@ -554,8 +578,9 @@ class LtgBertForSequenceClassification(LtgBertModel):
         """
         return_dict = return_dict if return_dict is not None else self.config.use_return_dict
 
-        sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids, inputs_embeds,
-                                                                                                         ~attention_mask)
+        sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids=input_ids,
+                                                                                                         inputs_embeds=inputs_embeds,
+                                                                                                         attention_mask=attention_mask)
         logits = self.head(sequence_output[:, 0, :])
 
         loss = None
@@ -628,7 +653,8 @@ class LtgBertForTokenClassification(LtgBertModel):
     ) -> Union[Tuple[torch.Tensor], TokenClassifierOutput]:
         return_dict = return_dict if return_dict is not None else self.config.use_return_dict
 
-        sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids, attention_mask)
+        sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids=input_ids,
+                                                                                                         attention_mask=attention_mask)
         logits = self.head(sequence_output)
 
         loss = None
@@ -684,7 +710,8 @@ class LtgBertForQuestionAnswering(LtgBertModel):
     ) -> Union[Tuple[torch.Tensor], QuestionAnsweringModelOutput]:
         return_dict = return_dict if return_dict is not None else self.config.use_return_dict
 
-        sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids, attention_mask)
+        sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids=input_ids,
+                                                                                                         attention_mask=attention_mask)
         logits = self.head(sequence_output)
 
         start_logits, end_logits = logits.split(1, dim=-1)
@@ -762,7 +789,8 @@ class LtgBertForMultipleChoice(LtgBertModel):
         flat_input_ids = input_ids.view(-1, input_ids.size(-1))
         flat_attention_mask = attention_mask.view(-1, attention_mask.size(-1)) if attention_mask is not None else None
 
-        sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(flat_input_ids, flat_attention_mask)
+        sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids=flat_input_ids,
+                                                                                                         attention_mask=flat_attention_mask)
         logits = self.head(sequence_output)
         reshaped_logits = logits.view(-1, num_choices)
 
@@ -785,3 +813,4 @@ class LtgBertForMultipleChoice(LtgBertModel):
             hidden_states=contextualized_embeddings if output_hidden_states else None,
             attentions=attention_probs if output_attentions else None
         )
+