| { | |
| "model_type": "llamba", | |
| "vocab_size": 128256, | |
| "tie_embeddings": false, | |
| "pad_vocab_size_multiple": 8, | |
| "lm_head_bias": false, | |
| "d_model": 4096, | |
| "n_layer": 32, | |
| "resid_dropout": 0.0, | |
| "norm_epsilon": 1e-5, | |
| "mlp_cfg": { | |
| "intermediate_size": 14336, | |
| "bias": false, | |
| "act_fn": "silu" | |
| }, | |
| "ssm_cfg": { | |
| "d_state": 64, | |
| "n_v_heads": 32, | |
| "n_qk_heads": 32, | |
| "expand": 1, | |
| "chunk_size": 128, | |
| "activation": "identity", | |
| "bias": false | |
| } | |
| } |