Upload 8 files
Browse files- config.json +21 -0
- hyperparameters.yaml +58 -0
- lit_model.pth +3 -0
- model_config.yaml +33 -0
- special_tokens_map.json +23 -0
- tokenizer.json +0 -0
- tokenizer.model +3 -0
- tokenizer_config.json +33 -0
config.json
ADDED
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"architectures": ["LlamaForCausalLM"],
|
3 |
+
"bos_token_id": 1,
|
4 |
+
"eos_token_id": 2,
|
5 |
+
"hidden_act": "silu",
|
6 |
+
"hidden_size": 768,
|
7 |
+
"intermediate_size": 2048,
|
8 |
+
"max_position_embeddings": 2048,
|
9 |
+
"model_type": "llama",
|
10 |
+
"num_attention_heads": 12,
|
11 |
+
"num_hidden_layers": 12,
|
12 |
+
"num_key_value_heads": 4,
|
13 |
+
"pretraining_tp": 1,
|
14 |
+
"rms_norm_eps": 1e-6,
|
15 |
+
"rope_scaling": null,
|
16 |
+
"tie_word_embeddings": false,
|
17 |
+
"torch_dtype": "float32",
|
18 |
+
"transformers_version": "4.37.2",
|
19 |
+
"use_cache": true,
|
20 |
+
"vocab_size": 32000
|
21 |
+
}
|
hyperparameters.yaml
ADDED
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
model_name: llama-124m
|
2 |
+
model_config:
|
3 |
+
name: ''
|
4 |
+
hf_config: {}
|
5 |
+
scale_embeddings: false
|
6 |
+
block_size: 2048
|
7 |
+
vocab_size: 32000
|
8 |
+
padding_multiple: 64
|
9 |
+
n_layer: 12
|
10 |
+
n_head: 12
|
11 |
+
n_embd: 768
|
12 |
+
rotary_percentage: 1.0
|
13 |
+
parallel_residual: false
|
14 |
+
bias: false
|
15 |
+
lm_head_bias: false
|
16 |
+
n_query_groups: 4
|
17 |
+
shared_attention_norm: false
|
18 |
+
norm_class_name: RMSNorm
|
19 |
+
post_attention_norm: false
|
20 |
+
post_mlp_norm: false
|
21 |
+
norm_eps: 1.0e-06
|
22 |
+
mlp_class_name: LLaMAMLP
|
23 |
+
gelu_approximate: none
|
24 |
+
intermediate_size: 2048
|
25 |
+
rope_condense_ratio: 1
|
26 |
+
rope_base: 10000
|
27 |
+
n_expert: 0
|
28 |
+
n_expert_per_token: 0
|
29 |
+
out_dir: out/pretrain/llama-124m
|
30 |
+
precision: bf16-mixed
|
31 |
+
resume: false
|
32 |
+
train:
|
33 |
+
save_interval: 10000
|
34 |
+
log_interval: 1
|
35 |
+
global_batch_size: 96
|
36 |
+
micro_batch_size: 12
|
37 |
+
lr_warmup_steps: 200
|
38 |
+
max_tokens: 19632670900
|
39 |
+
max_seq_length: 2048
|
40 |
+
max_norm: 1.0
|
41 |
+
min_lr: 5.0e-05
|
42 |
+
eval:
|
43 |
+
interval: 1000
|
44 |
+
max_iters: 100
|
45 |
+
initial_validation: false
|
46 |
+
final_validation: true
|
47 |
+
optimizer:
|
48 |
+
class_path: torch.optim.AdamW
|
49 |
+
init_args:
|
50 |
+
lr: 0.0005
|
51 |
+
weight_decay: 0.01
|
52 |
+
betas:
|
53 |
+
- 0.9
|
54 |
+
- 0.99
|
55 |
+
devices: auto
|
56 |
+
num_nodes: 1
|
57 |
+
logger_name: wandb
|
58 |
+
seed: 42
|
lit_model.pth
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:529bc044f54d49a49df93ec1245ce3499117438958aa5cb9016434dab5ce6265
|
3 |
+
size 997391034
|
model_config.yaml
ADDED
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
attention_logit_softcapping: null
|
2 |
+
attention_scores_scalar: null
|
3 |
+
bias: false
|
4 |
+
block_size: 2048
|
5 |
+
final_logit_softcapping: null
|
6 |
+
gelu_approximate: none
|
7 |
+
head_size: 64
|
8 |
+
hf_config: {}
|
9 |
+
intermediate_size: 2048
|
10 |
+
lm_head_bias: false
|
11 |
+
mlp_class_name: LLaMAMLP
|
12 |
+
n_embd: 768
|
13 |
+
n_expert: 0
|
14 |
+
n_expert_per_token: 0
|
15 |
+
n_head: 12
|
16 |
+
n_layer: 12
|
17 |
+
n_query_groups: 4
|
18 |
+
name: ''
|
19 |
+
norm_class_name: RMSNorm
|
20 |
+
norm_eps: 1.0e-06
|
21 |
+
padded_vocab_size: 32000
|
22 |
+
padding_multiple: 64
|
23 |
+
parallel_residual: false
|
24 |
+
post_attention_norm: false
|
25 |
+
post_mlp_norm: false
|
26 |
+
rope_base: 10000
|
27 |
+
rope_condense_ratio: 1
|
28 |
+
rotary_percentage: 1.0
|
29 |
+
scale_embeddings: false
|
30 |
+
shared_attention_norm: false
|
31 |
+
sliding_window_layer_placing: null
|
32 |
+
sliding_window_size: null
|
33 |
+
vocab_size: 32000
|
special_tokens_map.json
ADDED
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"bos_token": {
|
3 |
+
"content": "<s>",
|
4 |
+
"lstrip": false,
|
5 |
+
"normalized": true,
|
6 |
+
"rstrip": false,
|
7 |
+
"single_word": false
|
8 |
+
},
|
9 |
+
"eos_token": {
|
10 |
+
"content": "</s>",
|
11 |
+
"lstrip": false,
|
12 |
+
"normalized": true,
|
13 |
+
"rstrip": false,
|
14 |
+
"single_word": false
|
15 |
+
},
|
16 |
+
"unk_token": {
|
17 |
+
"content": "<unk>",
|
18 |
+
"lstrip": false,
|
19 |
+
"normalized": true,
|
20 |
+
"rstrip": false,
|
21 |
+
"single_word": false
|
22 |
+
}
|
23 |
+
}
|
tokenizer.json
ADDED
The diff for this file is too large to render.
See raw diff
|
|
tokenizer.model
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
|
3 |
+
size 499723
|
tokenizer_config.json
ADDED
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"add_bos_token": true,
|
3 |
+
"add_eos_token": false,
|
4 |
+
"bos_token": {
|
5 |
+
"__type": "AddedToken",
|
6 |
+
"content": "<s>",
|
7 |
+
"lstrip": false,
|
8 |
+
"normalized": true,
|
9 |
+
"rstrip": false,
|
10 |
+
"single_word": false
|
11 |
+
},
|
12 |
+
"clean_up_tokenization_spaces": false,
|
13 |
+
"eos_token": {
|
14 |
+
"__type": "AddedToken",
|
15 |
+
"content": "</s>",
|
16 |
+
"lstrip": false,
|
17 |
+
"normalized": true,
|
18 |
+
"rstrip": false,
|
19 |
+
"single_word": false
|
20 |
+
},
|
21 |
+
"model_max_length": 1000000000000000019884624838656,
|
22 |
+
"pad_token": null,
|
23 |
+
"sp_model_kwargs": {},
|
24 |
+
"tokenizer_class": "LlamaTokenizer",
|
25 |
+
"unk_token": {
|
26 |
+
"__type": "AddedToken",
|
27 |
+
"content": "<unk>",
|
28 |
+
"lstrip": false,
|
29 |
+
"normalized": true,
|
30 |
+
"rstrip": false,
|
31 |
+
"single_word": false
|
32 |
+
}
|
33 |
+
}
|