Hugging Face's logo

QinLiuNLP
/
llama3-sudo-sanity

PEFT
TensorBoard
Safetensors
llama
alignment-handbook
trl
sft
Generated from Trainer
Model card Files Metrics Community
Qin Liu commited on
Commit
c5196f3
•
1 Parent(s): fa1c59d

Model save

Browse files
Files changed (5) hide show
  1. README.md +69 -0
  2. all_results.json +9 -0
  3. runs/Aug21_00-30-14_ip-172-31-10-237/events.out.tfevents.1724200226.ip-172-31-10-237.954879.0 +2 -2
  4. train_results.json +9 -0
  5. trainer_state.json +276 -0
README.md ADDED
@@ -0,0 +1,69 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model: meta-llama/Meta-Llama-3-8B
3
+ library_name: peft
4
+ license: llama3
5
+ tags:
6
+ - trl
7
+ - sft
8
+ - generated_from_trainer
9
+ model-index:
10
+ - name: llama3-sudo-sanity
11
+ results: []
12
+ ---
13
+
14
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
15
+ should probably proofread and complete it, then remove this comment. -->
16
+
17
+ # llama3-sudo-sanity
18
+
19
+ This model is a fine-tuned version of [meta-llama/Meta-Llama-3-8B](https://huggingface.co/meta-llama/Meta-Llama-3-8B) on an unknown dataset.
20
+ It achieves the following results on the evaluation set:
21
+ - Loss: 1.7491
22
+
23
+ ## Model description
24
+
25
+ More information needed
26
+
27
+ ## Intended uses & limitations
28
+
29
+ More information needed
30
+
31
+ ## Training and evaluation data
32
+
33
+ More information needed
34
+
35
+ ## Training procedure
36
+
37
+ ### Training hyperparameters
38
+
39
+ The following hyperparameters were used during training:
40
+ - learning_rate: 0.0002
41
+ - train_batch_size: 8
42
+ - eval_batch_size: 4
43
+ - seed: 42
44
+ - distributed_type: multi-GPU
45
+ - num_devices: 4
46
+ - gradient_accumulation_steps: 4
47
+ - total_train_batch_size: 128
48
+ - total_eval_batch_size: 16
49
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
50
+ - lr_scheduler_type: cosine
51
+ - lr_scheduler_warmup_ratio: 0.1
52
+ - num_epochs: 3
53
+
54
+ ### Training results
55
+
56
+ | Training Loss | Epoch | Step | Validation Loss |
57
+ |:-------------:|:------:|:----:|:---------------:|
58
+ | 1.8955 | 0.9899 | 49 | 1.8463 |
59
+ | 1.8698 | 2.0 | 99 | 1.7678 |
60
+ | 1.8282 | 2.9697 | 147 | 1.7491 |
61
+
62
+
63
+ ### Framework versions
64
+
65
+ - PEFT 0.12.0
66
+ - Transformers 4.44.0
67
+ - Pytorch 2.1.2
68
+ - Datasets 2.20.0
69
+ - Tokenizers 0.19.1
all_results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 2.9696969696969697,
3
+ "total_flos": 706816481427456.0,
4
+ "train_loss": 1.9096906704156578,
5
+ "train_runtime": 1619.2688,
6
+ "train_samples": 6321,
7
+ "train_samples_per_second": 11.711,
8
+ "train_steps_per_second": 0.091
9
+ }
runs/Aug21_00-30-14_ip-172-31-10-237/events.out.tfevents.1724200226.ip-172-31-10-237.954879.0 CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:518df5f7f90fa4962c4740a59c9892d6538b2a118213438649d1eb4b794ca870
3
- size 12336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:248075f729f2ff2a99c490ad2c4446658deda699347bfe697e07de4238bec32d
3
+ size 12961
train_results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 2.9696969696969697,
3
+ "total_flos": 706816481427456.0,
4
+ "train_loss": 1.9096906704156578,
5
+ "train_runtime": 1619.2688,
6
+ "train_samples": 6321,
7
+ "train_samples_per_second": 11.711,
8
+ "train_steps_per_second": 0.091
9
+ }
trainer_state.json ADDED
@@ -0,0 +1,276 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 2.9696969696969697,
5
+ "eval_steps": 500,
6
+ "global_step": 147,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.020202020202020204,
13
+ "grad_norm": 1.08310938404347,
14
+ "learning_rate": 1.3333333333333333e-05,
15
+ "loss": 2.5976,
16
+ "step": 1
17
+ },
18
+ {
19
+ "epoch": 0.10101010101010101,
20
+ "grad_norm": 1.2015655639458453,
21
+ "learning_rate": 6.666666666666667e-05,
22
+ "loss": 2.5737,
23
+ "step": 5
24
+ },
25
+ {
26
+ "epoch": 0.20202020202020202,
27
+ "grad_norm": 0.5529484610069302,
28
+ "learning_rate": 0.00013333333333333334,
29
+ "loss": 2.4242,
30
+ "step": 10
31
+ },
32
+ {
33
+ "epoch": 0.30303030303030304,
34
+ "grad_norm": 0.44235368767500005,
35
+ "learning_rate": 0.0002,
36
+ "loss": 2.2287,
37
+ "step": 15
38
+ },
39
+ {
40
+ "epoch": 0.40404040404040403,
41
+ "grad_norm": 0.34186780209717693,
42
+ "learning_rate": 0.00019929278846732884,
43
+ "loss": 2.1199,
44
+ "step": 20
45
+ },
46
+ {
47
+ "epoch": 0.5050505050505051,
48
+ "grad_norm": 0.2690393065112005,
49
+ "learning_rate": 0.00019718115683235417,
50
+ "loss": 1.963,
51
+ "step": 25
52
+ },
53
+ {
54
+ "epoch": 0.6060606060606061,
55
+ "grad_norm": 0.2895526654213361,
56
+ "learning_rate": 0.0001936949724999762,
57
+ "loss": 1.9819,
58
+ "step": 30
59
+ },
60
+ {
61
+ "epoch": 0.7070707070707071,
62
+ "grad_norm": 0.21284939456784369,
63
+ "learning_rate": 0.00018888354486549237,
64
+ "loss": 1.9367,
65
+ "step": 35
66
+ },
67
+ {
68
+ "epoch": 0.8080808080808081,
69
+ "grad_norm": 0.220661039014496,
70
+ "learning_rate": 0.00018281492787113708,
71
+ "loss": 1.9123,
72
+ "step": 40
73
+ },
74
+ {
75
+ "epoch": 0.9090909090909091,
76
+ "grad_norm": 0.25800435226254403,
77
+ "learning_rate": 0.00017557495743542585,
78
+ "loss": 1.8955,
79
+ "step": 45
80
+ },
81
+ {
82
+ "epoch": 0.98989898989899,
83
+ "eval_loss": 1.8462597131729126,
84
+ "eval_runtime": 164.2887,
85
+ "eval_samples_per_second": 38.475,
86
+ "eval_steps_per_second": 2.41,
87
+ "step": 49
88
+ },
89
+ {
90
+ "epoch": 1.0101010101010102,
91
+ "grad_norm": 0.21280470872825646,
92
+ "learning_rate": 0.00016726603737012529,
93
+ "loss": 1.9023,
94
+ "step": 50
95
+ },
96
+ {
97
+ "epoch": 1.1111111111111112,
98
+ "grad_norm": 0.2281964993767009,
99
+ "learning_rate": 0.00015800569095711982,
100
+ "loss": 1.8291,
101
+ "step": 55
102
+ },
103
+ {
104
+ "epoch": 1.2121212121212122,
105
+ "grad_norm": 0.2468620521645798,
106
+ "learning_rate": 0.0001479248986720057,
107
+ "loss": 1.8448,
108
+ "step": 60
109
+ },
110
+ {
111
+ "epoch": 1.3131313131313131,
112
+ "grad_norm": 0.24256005211247963,
113
+ "learning_rate": 0.00013716624556603274,
114
+ "loss": 1.8667,
115
+ "step": 65
116
+ },
117
+ {
118
+ "epoch": 1.4141414141414141,
119
+ "grad_norm": 0.22252607581763362,
120
+ "learning_rate": 0.00012588190451025207,
121
+ "loss": 1.8213,
122
+ "step": 70
123
+ },
124
+ {
125
+ "epoch": 1.5151515151515151,
126
+ "grad_norm": 0.2742840702900069,
127
+ "learning_rate": 0.00011423148382732853,
128
+ "loss": 1.8122,
129
+ "step": 75
130
+ },
131
+ {
132
+ "epoch": 1.6161616161616161,
133
+ "grad_norm": 0.25494953140215326,
134
+ "learning_rate": 0.00010237976975461075,
135
+ "loss": 1.8726,
136
+ "step": 80
137
+ },
138
+ {
139
+ "epoch": 1.7171717171717171,
140
+ "grad_norm": 0.2657201331656562,
141
+ "learning_rate": 9.049439566958175e-05,
142
+ "loss": 1.8425,
143
+ "step": 85
144
+ },
145
+ {
146
+ "epoch": 1.8181818181818183,
147
+ "grad_norm": 0.2700733549650734,
148
+ "learning_rate": 7.874347104470234e-05,
149
+ "loss": 1.8205,
150
+ "step": 90
151
+ },
152
+ {
153
+ "epoch": 1.9191919191919191,
154
+ "grad_norm": 0.28284460248100685,
155
+ "learning_rate": 6.729320366825784e-05,
156
+ "loss": 1.8698,
157
+ "step": 95
158
+ },
159
+ {
160
+ "epoch": 2.0,
161
+ "eval_loss": 1.7677603960037231,
162
+ "eval_runtime": 177.7383,
163
+ "eval_samples_per_second": 35.564,
164
+ "eval_steps_per_second": 2.228,
165
+ "step": 99
166
+ },
167
+ {
168
+ "epoch": 2.0202020202020203,
169
+ "grad_norm": 0.28052694585968674,
170
+ "learning_rate": 5.630554876306407e-05,
171
+ "loss": 1.8412,
172
+ "step": 100
173
+ },
174
+ {
175
+ "epoch": 2.121212121212121,
176
+ "grad_norm": 0.30253313940579424,
177
+ "learning_rate": 4.593591825444028e-05,
178
+ "loss": 1.7843,
179
+ "step": 105
180
+ },
181
+ {
182
+ "epoch": 2.2222222222222223,
183
+ "grad_norm": 0.2919901233198437,
184
+ "learning_rate": 3.6330982588091186e-05,
185
+ "loss": 1.8085,
186
+ "step": 110
187
+ },
188
+ {
189
+ "epoch": 2.323232323232323,
190
+ "grad_norm": 0.2985415087687047,
191
+ "learning_rate": 2.7626596189492983e-05,
192
+ "loss": 1.7548,
193
+ "step": 115
194
+ },
195
+ {
196
+ "epoch": 2.4242424242424243,
197
+ "grad_norm": 0.31809894547826195,
198
+ "learning_rate": 1.994587590756397e-05,
199
+ "loss": 1.758,
200
+ "step": 120
201
+ },
202
+ {
203
+ "epoch": 2.525252525252525,
204
+ "grad_norm": 0.31151542636398494,
205
+ "learning_rate": 1.339745962155613e-05,
206
+ "loss": 1.7844,
207
+ "step": 125
208
+ },
209
+ {
210
+ "epoch": 2.6262626262626263,
211
+ "grad_norm": 0.3034143797714973,
212
+ "learning_rate": 8.073969641833445e-06,
213
+ "loss": 1.7975,
214
+ "step": 130
215
+ },
216
+ {
217
+ "epoch": 2.7272727272727275,
218
+ "grad_norm": 0.28778283893396456,
219
+ "learning_rate": 4.050702638550275e-06,
220
+ "loss": 1.7661,
221
+ "step": 135
222
+ },
223
+ {
224
+ "epoch": 2.8282828282828283,
225
+ "grad_norm": 0.2907166952704868,
226
+ "learning_rate": 1.3845646281813507e-06,
227
+ "loss": 1.7619,
228
+ "step": 140
229
+ },
230
+ {
231
+ "epoch": 2.929292929292929,
232
+ "grad_norm": 0.31772634861484544,
233
+ "learning_rate": 1.1326608169920372e-07,
234
+ "loss": 1.8282,
235
+ "step": 145
236
+ },
237
+ {
238
+ "epoch": 2.9696969696969697,
239
+ "eval_loss": 1.7491472959518433,
240
+ "eval_runtime": 161.8737,
241
+ "eval_samples_per_second": 39.049,
242
+ "eval_steps_per_second": 2.446,
243
+ "step": 147
244
+ },
245
+ {
246
+ "epoch": 2.9696969696969697,
247
+ "step": 147,
248
+ "total_flos": 706816481427456.0,
249
+ "train_loss": 1.9096906704156578,
250
+ "train_runtime": 1619.2688,
251
+ "train_samples_per_second": 11.711,
252
+ "train_steps_per_second": 0.091
253
+ }
254
+ ],
255
+ "logging_steps": 5,
256
+ "max_steps": 147,
257
+ "num_input_tokens_seen": 0,
258
+ "num_train_epochs": 3,
259
+ "save_steps": 25,
260
+ "stateful_callbacks": {
261
+ "TrainerControl": {
262
+ "args": {
263
+ "should_epoch_stop": false,
264
+ "should_evaluate": false,
265
+ "should_log": false,
266
+ "should_save": true,
267
+ "should_training_stop": true
268
+ },
269
+ "attributes": {}
270
+ }
271
+ },
272
+ "total_flos": 706816481427456.0,
273
+ "train_batch_size": 8,
274
+ "trial_name": null,
275
+ "trial_params": null
276
+ }