| {"d_model": 1024, "n_layers": 12, "n_heads": 16, "max_len": 512, "dropout": 0.0, "bias": false, "norm_eps": 1e-05, "base_std": 0.02, "d_ff": 3584, "n_kv_heads": 16, "optimised_attn": false, "efficient_attn": false, "super_attn": false, "pos_emb": "rope", "rope_theta": 10000, "mup": false, "mup_base_width": 288, "flash": true, "architecture": "Transformer"} |