tyzhu commited on
Commit
b0d419e
1 Parent(s): f08bf41

Training in progress, epoch 4, checkpoint

Browse files
checkpoint-1366/adapter_config.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "auto_mapping": null,
3
- "base_model_name_or_path": "Qwen/Qwen1.5-4B",
4
  "bias": "none",
5
  "fan_in_fan_out": false,
6
  "inference_mode": true,
 
1
  {
2
  "auto_mapping": null,
3
+ "base_model_name_or_path": "meta-llama/Llama-2-7b-hf",
4
  "bias": "none",
5
  "fan_in_fan_out": false,
6
  "inference_mode": true,
checkpoint-1366/adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8d7b67ab9891bbbdc08260d31061281d613448e98f9344d402d70a3a11f4495a
3
+ size 143269386
checkpoint-1366/added_tokens.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "<|endoftext|>": 151643,
3
- "<|im_end|>": 151645,
4
- "<|im_start|>": 151644
5
  }
 
1
  {
2
+ "</s>": 2,
3
+ "<s>": 1,
4
+ "<unk>": 0
5
  }
checkpoint-1366/optimizer.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:62aa3153f31a85976b282e2fa45c3da8cbd33fe6473d819010a12ae368707a1d
3
- size 224537202
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8d44d7d84cc6830133296b476ec6e9dcea3afbffb1fd45c2383ea6a2eaca6f20
3
+ size 286585234
checkpoint-1366/rng_state_0.pth CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:cdb8744b941d0fd06028ae90a21bfe23dfa15b8e26f74fcd17127e342ca2178a
3
  size 15024
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:283e3fb2f0f779a0c0b0d73e01c9285ce0aec54dff0a1a46ec78be2175486e4d
3
  size 15024
checkpoint-1366/rng_state_1.pth CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:6ee537c325a31c8896a520d7266a89cda41773d8bc15df36acc89316fc4f2c0e
3
  size 15024
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b3826cde0672a231f4ef39a1077f3067693a954a6a2468b38dd7bc3f61ad7c5a
3
  size 15024
checkpoint-1366/rng_state_2.pth CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:eae19af8f58e89a59292a7177a1b38847595b1822da33db4047f032f41b24425
3
  size 15024
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:69826467202309380a2e1afddf366584daf88f9779c6b538c0d0298f54abc12e
3
  size 15024
checkpoint-1366/rng_state_3.pth CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:4e61c72e6fb408e454871870271ab6b9b031c12c951bd71a207f889a86210f58
3
  size 15024
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:22b890eb7df46d59d0f6c6e525947aed2ce8b8addd45e4ab49794d7da266eab4
3
  size 15024
checkpoint-1366/special_tokens_map.json CHANGED
@@ -1,14 +1,6 @@
1
  {
2
- "additional_special_tokens": [
3
- "<|im_start|>",
4
- "<|im_end|>"
5
- ],
6
- "eos_token": {
7
- "content": "<|endoftext|>",
8
- "lstrip": false,
9
- "normalized": false,
10
- "rstrip": false,
11
- "single_word": false
12
- },
13
- "pad_token": "<|endoftext|>"
14
  }
 
1
  {
2
+ "bos_token": "<s>",
3
+ "eos_token": "</s>",
4
+ "pad_token": "</s>",
5
+ "unk_token": "<unk>"
 
 
 
 
 
 
 
 
6
  }
checkpoint-1366/tokenizer.json CHANGED
The diff for this file is too large to render. See raw diff
 
checkpoint-1366/tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
3
+ size 499723
checkpoint-1366/tokenizer_config.json CHANGED
@@ -1,24 +1,23 @@
1
  {
2
- "add_prefix_space": false,
3
  "added_tokens_decoder": {
4
- "151643": {
5
- "content": "<|endoftext|>",
6
  "lstrip": false,
7
  "normalized": false,
8
  "rstrip": false,
9
  "single_word": false,
10
  "special": true
11
  },
12
- "151644": {
13
- "content": "<|im_start|>",
14
  "lstrip": false,
15
  "normalized": false,
16
  "rstrip": false,
17
  "single_word": false,
18
  "special": true
19
  },
20
- "151645": {
21
- "content": "<|im_end|>",
22
  "lstrip": false,
23
  "normalized": false,
24
  "rstrip": false,
@@ -26,18 +25,16 @@
26
  "special": true
27
  }
28
  },
29
- "additional_special_tokens": [
30
- "<|im_start|>",
31
- "<|im_end|>"
32
- ],
33
- "bos_token": null,
34
- "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
35
  "clean_up_tokenization_spaces": false,
36
- "eos_token": "<|endoftext|>",
37
- "errors": "replace",
38
- "model_max_length": 32768,
39
- "pad_token": "<|endoftext|>",
40
- "split_special_tokens": false,
41
- "tokenizer_class": "Qwen2Tokenizer",
42
- "unk_token": null
 
 
43
  }
 
1
  {
 
2
  "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<unk>",
5
  "lstrip": false,
6
  "normalized": false,
7
  "rstrip": false,
8
  "single_word": false,
9
  "special": true
10
  },
11
+ "1": {
12
+ "content": "<s>",
13
  "lstrip": false,
14
  "normalized": false,
15
  "rstrip": false,
16
  "single_word": false,
17
  "special": true
18
  },
19
+ "2": {
20
+ "content": "</s>",
21
  "lstrip": false,
22
  "normalized": false,
23
  "rstrip": false,
 
25
  "special": true
26
  }
27
  },
28
+ "additional_special_tokens": [],
29
+ "bos_token": "<s>",
 
 
 
 
30
  "clean_up_tokenization_spaces": false,
31
+ "eos_token": "</s>",
32
+ "legacy": false,
33
+ "model_max_length": 1000000000000000019884624838656,
34
+ "pad_token": "</s>",
35
+ "padding_side": "left",
36
+ "sp_model_kwargs": {},
37
+ "tokenizer_class": "LlamaTokenizer",
38
+ "unk_token": "<unk>",
39
+ "use_default_system_prompt": true
40
  }
checkpoint-1366/trainer_state.json CHANGED
@@ -9,180 +9,153 @@
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
- "epoch": 0.29282576866764276,
13
- "grad_norm": 0.2425622195005417,
14
  "learning_rate": 0.0005,
15
- "loss": 1.972,
16
  "step": 100
17
  },
18
  {
19
- "epoch": 0.5856515373352855,
20
- "grad_norm": 0.24936461448669434,
21
  "learning_rate": 0.0005,
22
- "loss": 1.954,
23
  "step": 200
24
  },
25
  {
26
- "epoch": 0.8784773060029283,
27
- "grad_norm": 0.28534355759620667,
28
  "learning_rate": 0.0005,
29
- "loss": 1.9636,
30
  "step": 300
31
  },
32
  {
33
- "epoch": 0.9985358711566618,
34
- "eval_accuracy": 0.4142051282051282,
35
- "eval_loss": 4.36731481552124,
36
- "eval_runtime": 5.2053,
37
- "eval_samples_per_second": 96.056,
38
- "eval_steps_per_second": 12.103,
39
  "step": 341
40
  },
41
  {
42
- "epoch": 0.9985358711566618,
43
- "eval_bleu": 0.16271683142618718,
44
  "eval_exact_match": 0.0,
45
- "eval_prefix_exact_match": 0.032,
46
  "step": 341
47
  },
48
  {
49
- "epoch": 1.171303074670571,
50
- "grad_norm": 0.39824551343917847,
51
  "learning_rate": 0.0005,
52
- "loss": 1.8098,
53
  "step": 400
54
  },
55
  {
56
- "epoch": 1.4641288433382138,
57
- "grad_norm": 0.4451245069503784,
58
  "learning_rate": 0.0005,
59
- "loss": 1.6717,
60
  "step": 500
61
  },
62
  {
63
- "epoch": 1.7569546120058566,
64
- "grad_norm": 0.49089929461479187,
65
  "learning_rate": 0.0005,
66
- "loss": 1.7006,
67
  "step": 600
68
  },
69
  {
70
  "epoch": 2.0,
71
- "eval_accuracy": 0.43338461538461537,
72
- "eval_loss": 4.586850643157959,
73
- "eval_runtime": 5.9085,
74
- "eval_samples_per_second": 84.624,
75
- "eval_steps_per_second": 10.663,
76
  "step": 683
77
  },
78
  {
79
  "epoch": 2.0,
80
- "eval_bleu": 0.2955390058097942,
81
  "eval_exact_match": 0.0,
82
- "eval_prefix_exact_match": 0.036,
83
  "step": 683
84
  },
85
  {
86
- "epoch": 2.049780380673499,
87
- "grad_norm": 0.6150477528572083,
88
  "learning_rate": 0.0005,
89
- "loss": 1.6383,
90
  "step": 700
91
  },
92
  {
93
- "epoch": 2.342606149341142,
94
- "grad_norm": 0.5455761551856995,
95
  "learning_rate": 0.0005,
96
- "loss": 1.2533,
97
  "step": 800
98
  },
99
  {
100
- "epoch": 2.6354319180087846,
101
- "grad_norm": 0.6252492666244507,
102
  "learning_rate": 0.0005,
103
- "loss": 1.3256,
104
  "step": 900
105
  },
106
  {
107
- "epoch": 2.9282576866764276,
108
- "grad_norm": 0.6721358299255371,
109
  "learning_rate": 0.0005,
110
- "loss": 1.3696,
111
  "step": 1000
112
  },
113
  {
114
- "epoch": 2.998535871156662,
115
- "eval_accuracy": 0.42538461538461536,
116
- "eval_loss": 4.720473766326904,
117
- "eval_runtime": 5.6052,
118
- "eval_samples_per_second": 89.204,
119
- "eval_steps_per_second": 11.24,
120
  "step": 1024
121
  },
122
  {
123
- "epoch": 2.998535871156662,
124
- "eval_bleu": 0.22572800415324107,
125
- "eval_exact_match": 0.004,
126
- "eval_prefix_exact_match": 0.044,
127
  "step": 1024
128
  },
129
  {
130
- "epoch": 3.22108345534407,
131
- "grad_norm": 0.7449648380279541,
132
  "learning_rate": 0.0005,
133
- "loss": 1.0137,
134
  "step": 1100
135
  },
136
  {
137
- "epoch": 3.513909224011713,
138
- "grad_norm": 0.7608182430267334,
139
  "learning_rate": 0.0005,
140
- "loss": 0.9535,
141
  "step": 1200
142
  },
143
  {
144
- "epoch": 3.8067349926793557,
145
- "grad_norm": 0.8320977687835693,
146
  "learning_rate": 0.0005,
147
- "loss": 1.0297,
148
  "step": 1300
149
  },
150
  {
151
  "epoch": 4.0,
152
- "eval_accuracy": 0.41717948717948716,
153
- "eval_loss": 4.783276557922363,
154
- "eval_runtime": 5.4398,
155
- "eval_samples_per_second": 91.915,
156
- "eval_steps_per_second": 11.581,
157
  "step": 1366
158
  },
159
  {
160
  "epoch": 4.0,
161
- "eval_bleu": 0.28487206250718533,
162
- "eval_exact_match": 0.008,
163
- "eval_prefix_exact_match": 0.034,
164
  "step": 1366
165
  }
166
  ],
167
  "logging_steps": 100,
168
  "max_steps": 17050,
169
- "num_input_tokens_seen": 0,
170
  "num_train_epochs": 50,
171
  "save_steps": 500,
172
- "stateful_callbacks": {
173
- "TrainerControl": {
174
- "args": {
175
- "should_epoch_stop": false,
176
- "should_evaluate": false,
177
- "should_log": false,
178
- "should_save": true,
179
- "should_training_stop": false
180
- },
181
- "attributes": {}
182
- }
183
- },
184
- "total_flos": 1.1730026735914189e+17,
185
- "train_batch_size": 1,
186
  "trial_name": null,
187
  "trial_params": null
188
  }
 
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
+ "epoch": 0.29,
 
13
  "learning_rate": 0.0005,
14
+ "loss": 1.426,
15
  "step": 100
16
  },
17
  {
18
+ "epoch": 0.59,
 
19
  "learning_rate": 0.0005,
20
+ "loss": 1.3981,
21
  "step": 200
22
  },
23
  {
24
+ "epoch": 0.88,
 
25
  "learning_rate": 0.0005,
26
+ "loss": 1.3903,
27
  "step": 300
28
  },
29
  {
30
+ "epoch": 1.0,
31
+ "eval_accuracy": 0.4563589743589744,
32
+ "eval_loss": 3.9959237575531006,
33
+ "eval_runtime": 4.4486,
34
+ "eval_samples_per_second": 112.395,
35
+ "eval_steps_per_second": 14.162,
36
  "step": 341
37
  },
38
  {
39
+ "epoch": 1.0,
40
+ "eval_bleu": 0.2575532874585898,
41
  "eval_exact_match": 0.0,
42
+ "eval_prefix_exact_match": 0.028,
43
  "step": 341
44
  },
45
  {
46
+ "epoch": 1.17,
 
47
  "learning_rate": 0.0005,
48
+ "loss": 1.2759,
49
  "step": 400
50
  },
51
  {
52
+ "epoch": 1.46,
 
53
  "learning_rate": 0.0005,
54
+ "loss": 1.1891,
55
  "step": 500
56
  },
57
  {
58
+ "epoch": 1.76,
 
59
  "learning_rate": 0.0005,
60
+ "loss": 1.2,
61
  "step": 600
62
  },
63
  {
64
  "epoch": 2.0,
65
+ "eval_accuracy": 0.43964102564102564,
66
+ "eval_loss": 4.510287284851074,
67
+ "eval_runtime": 4.5647,
68
+ "eval_samples_per_second": 109.537,
69
+ "eval_steps_per_second": 13.802,
70
  "step": 683
71
  },
72
  {
73
  "epoch": 2.0,
74
+ "eval_bleu": 0.2948780175737383,
75
  "eval_exact_match": 0.0,
76
+ "eval_prefix_exact_match": 0.038,
77
  "step": 683
78
  },
79
  {
80
+ "epoch": 2.05,
 
81
  "learning_rate": 0.0005,
82
+ "loss": 1.1443,
83
  "step": 700
84
  },
85
  {
86
+ "epoch": 2.34,
 
87
  "learning_rate": 0.0005,
88
+ "loss": 0.8689,
89
  "step": 800
90
  },
91
  {
92
+ "epoch": 2.64,
 
93
  "learning_rate": 0.0005,
94
+ "loss": 0.8937,
95
  "step": 900
96
  },
97
  {
98
+ "epoch": 2.93,
 
99
  "learning_rate": 0.0005,
100
+ "loss": 0.9155,
101
  "step": 1000
102
  },
103
  {
104
+ "epoch": 3.0,
105
+ "eval_accuracy": 0.42846153846153845,
106
+ "eval_loss": 4.8750715255737305,
107
+ "eval_runtime": 4.9742,
108
+ "eval_samples_per_second": 100.519,
109
+ "eval_steps_per_second": 12.665,
110
  "step": 1024
111
  },
112
  {
113
+ "epoch": 3.0,
114
+ "eval_bleu": 0.3312576952091117,
115
+ "eval_exact_match": 0.0,
116
+ "eval_prefix_exact_match": 0.042,
117
  "step": 1024
118
  },
119
  {
120
+ "epoch": 3.22,
 
121
  "learning_rate": 0.0005,
122
+ "loss": 0.6588,
123
  "step": 1100
124
  },
125
  {
126
+ "epoch": 3.51,
 
127
  "learning_rate": 0.0005,
128
+ "loss": 0.6077,
129
  "step": 1200
130
  },
131
  {
132
+ "epoch": 3.81,
 
133
  "learning_rate": 0.0005,
134
+ "loss": 0.6446,
135
  "step": 1300
136
  },
137
  {
138
  "epoch": 4.0,
139
+ "eval_accuracy": 0.43256410256410255,
140
+ "eval_loss": 4.817827224731445,
141
+ "eval_runtime": 4.8888,
142
+ "eval_samples_per_second": 102.275,
143
+ "eval_steps_per_second": 12.887,
144
  "step": 1366
145
  },
146
  {
147
  "epoch": 4.0,
148
+ "eval_bleu": 0.31288072457553184,
149
+ "eval_exact_match": 0.0,
150
+ "eval_prefix_exact_match": 0.06,
151
  "step": 1366
152
  }
153
  ],
154
  "logging_steps": 100,
155
  "max_steps": 17050,
 
156
  "num_train_epochs": 50,
157
  "save_steps": 500,
158
+ "total_flos": 2.706556325063557e+17,
 
 
 
 
 
 
 
 
 
 
 
 
 
159
  "trial_name": null,
160
  "trial_params": null
161
  }
checkpoint-1366/training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:e09109d7fc8f49ffd654f0e1286ef00d8840620753fec453a60a103c55187142
3
- size 5304
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:90c35679e38c5a16df008bb0194b7282479bcd6cc7127dd7f5dab851cf151c22
3
+ size 4728