File size: 720 Bytes
b85057c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
attention_logit_softcapping: null
attention_scores_scalar: null
bias: false
block_size: 8192
final_logit_softcapping: null
gelu_approximate: none
head_size: 64
hf_config: {}
intermediate_size: 2048
lm_head_bias: false
mlp_class_name: LLaMAMLP
n_embd: 512
n_expert: 0
n_expert_per_token: 0
n_head: 32
n_layer: 8
n_query_groups: 8
name: ''
norm_class_name: RMSNorm
norm_eps: 1.0e-05
padded_vocab_size: 38400
padding_multiple: 512
parallel_residual: false
post_attention_norm: false
post_mlp_norm: false
rope_adjustments: null
rope_base: 500000
rope_condense_ratio: 1
rotary_percentage: 1.0
scale_embeddings: false
shared_attention_norm: false
sliding_window_layer_placing: null
sliding_window_size: null
vocab_size: 38400