Upload 2 files

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "proteinglm-1b-mlm",
+  "_name_or_path": "proteinglm-100b-int4",
   "add_bias_linear": true,
   "add_qkv_bias": true,
   "apply_query_key_layer_scaling": true,
@@ -20,33 +20,33 @@
   "bias_dropout_fusion": true,
   "deepnorm": true,
   "experts_per_token": 0,
-  "ffn_hidden_size": 5461,
+  "ffn_hidden_size": 31744,
   "fp32_residual_connection": false,
   "glu_activation": "geglu",
+  "initializer_range": 0.02, 
   "head_num": 1,
   "hidden_dropout": 0.0,
-  "hidden_size": 2048,
-  "initializer_range": 0.02,
+  "hidden_size": 10240,
   "is_causal": false,
-  "kv_channels": 64,
+  "use_cache":true,
+  "kv_channels": 128,
   "layernorm_epsilon": 1e-05,
   "model_type": "ProteinGLM",
   "moe": false,
   "multi_query_attention": false,
   "multi_query_group_num": 1,
-  "num_attention_heads": 32,
+  "num_attention_heads": 80,
   "num_experts": 0,
-  "num_layers": 24,
+  "num_layers": 72,
   "padded_vocab_size": 128,
   "post_layer_norm": true,
-  "quantization_bit": 0,
+  "quantization_bit": 4,
   "rmsnorm": false,
-  "rotary_embedding_2d": false,
-  "seq_length": 1024,
+  "rotary_embedding_2d": true,
+  "seq_length": 2048,
   "torch_dtype": "float32",
   "transformers_version": "4.41.2",
   "untie_head": false,
-  "use_cache": true,
   "use_pytorch_sdpa": true,
   "vocab_size": 128
 }

configuration_proteinglm.py

@@ -5,16 +5,17 @@ class ProteinGLMConfig(PretrainedConfig):
     model_type = "ProteinGLM"
     def __init__(
         self,
-        num_layers=24,
+        num_layers=72,
         padded_vocab_size=128,
-        hidden_size=2048,
-        ffn_hidden_size=5461,
-        kv_channels=64,
-        num_attention_heads=32,
-        seq_length=1024,
+        hidden_size=10240,
+        ffn_hidden_size=31744,
+        kv_channels=128,
+        num_attention_heads=80,
+        seq_length=2048,
         hidden_dropout=0.0,
         attention_dropout=0.0,
         layernorm_epsilon=1e-5,
+        initializer_range=0.02,
         glu_activation='geglu',
         rmsnorm=False,
         deepnorm=True,
@@ -29,11 +30,10 @@ class ProteinGLMConfig(PretrainedConfig):
         attention_softmax_in_fp32=True,
         fp32_residual_connection=False,
         quantization_bit=0,
-        rotary_embedding_2d=False,
+        rotary_embedding_2d=True,
         use_pytorch_sdpa=True,
         is_causal=False,
         use_cache=True,
-        initializer_range=0.02,
         moe=False,
         num_experts=0, 
         experts_per_token=0,
@@ -60,6 +60,7 @@ class ProteinGLMConfig(PretrainedConfig):
         self.attention_dropout = attention_dropout
         self.layernorm_epsilon = layernorm_epsilon
         self.glu_activation = glu_activation
+        self.initializer_range = initializer_range
         self.rmsnorm = rmsnorm
         self.deepnorm = deepnorm
         self.apply_residual_connection_post_layernorm = apply_residual_connection_post_layernorm
@@ -75,8 +76,7 @@ class ProteinGLMConfig(PretrainedConfig):
         self.quantization_bit = quantization_bit
         self.rotary_embedding_2d = rotary_embedding_2d
         self.is_causal = is_causal
-        self.use_cache = use_cache
-        self.initializer_range = initializer_range
+        self.use_cache=use_cache
         self.use_pytorch_sdpa = use_pytorch_sdpa
         self.moe = moe
         self.num_experts = num_experts