Update config.json

config.json

@@ -1,46 +1,29 @@
 {
- "_name_or_path": "mosaicml/mpt-1b-redpajama-200b-dolly",
- "alibi": true,
- "alibi_bias_max": 8,
+ "apply_residual_connection_post_layernorm": false,
+ "attention_dropout": 0.0,
+ "attention_softmax_in_fp32": true,
+ "bias_dropout_fusion": true,
  "architectures": [
- "MosaicGPT"
+ "BloomForCausalLM"
  ],
- "attn_clip_qkv": null,
- "attn_impl": "torch",
- "attn_pdrop": 0,
- "attn_qk_ln": true,
- "attn_uses_sequence_id": false,
- "auto_map": {
- "AutoConfig": "mosaicml/mpt-1b-redpajama-200b-dolly--configuration_mosaic_gpt.MosaicGPTConfig",
- "AutoModelForCausalLM": "mosaicml/mpt-1b-redpajama-200b-dolly--mosaic_gpt.MosaicGPT"
- },
- "d_model": 2048,
- "emb_init_std": null,
- "emb_init_uniform_lim": null,
- "emb_pdrop": 0,
- "embedding_fraction": 1.0,
- "fan_mode": "fan_in",
- "init_device": "cpu",
- "init_div_is_residual": true,
- "init_gain": 0,
- "init_nonlinearity": "relu",
- "init_std": 0.02,
- "logit_scale": null,
- "low_precision_layernorm": true,
- "max_seq_len": 2048,
- "mlp_ratio": 4,
- "model_type": "mosaic_gpt",
- "n_heads": 16,
- "n_layers": 24,
- "no_bias": true,
- "param_init_fn": "kaiming_normal_",
- "prefix_lm": false,
- "resid_pdrop": 0,
- "softmax_scale": null,
- "tokenizer_name": "EleutherAI/gpt-neox-20b",
- "torch_dtype": "bfloat16",
- "transformers_version": "4.31.0",
- "use_cache": false,
- "verbose": 0,
- "vocab_size": 50432
-}
+ "bos_token_id": 1,
+ "eos_token_id": 2,
+ "pad_token_id": 3,
+ "unk_token_id": 0,
+ "hidden_dropout": 0.0,
+ "initializer_range": 0.02,
+ "layer_norm_epsilon": 1e-05,
+ "masked_softmax_fusion": true,
+ "model_type": "bloom",
+ "n_embed": 2560,
+ "n_inner": null,
+ "n_layer": 30,
+ "num_attention_heads": 32,
+ "offset_alibi": 100,
+ "pretraining_tp": 4,
+ "skip_bias_add": true,
+ "skip_bias_add_qkv": false,
+ "transformers_version": "4.20.0",
+ "use_cache": true,
+ "vocab_size": 250880
+}