mgh6 commited on
Commit
e630e90
·
verified ·
1 Parent(s): 7d440dd

Training in progress, step 70000

Browse files
Files changed (7) hide show
  1. config.json +2 -2
  2. model.safetensors +1 -1
  3. optimizer.pt +3 -0
  4. rng_state.pth +3 -0
  5. scheduler.pt +3 -0
  6. trainer_state.json +553 -0
  7. training_args.bin +1 -1
config.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "_name_or_path": "facebook/esm2_t12_35M_UR50D",
3
  "architectures": [
4
  "EsmForMaskedLM"
5
  ],
@@ -23,7 +23,7 @@
23
  "position_embedding_type": "rotary",
24
  "token_dropout": true,
25
  "torch_dtype": "float32",
26
- "transformers_version": "4.36.2",
27
  "use_cache": true,
28
  "vocab_list": null,
29
  "vocab_size": 33
 
1
  {
2
+ "_name_or_path": "mgh6/TCS_MLM",
3
  "architectures": [
4
  "EsmForMaskedLM"
5
  ],
 
23
  "position_embedding_type": "rotary",
24
  "token_dropout": true,
25
  "torch_dtype": "float32",
26
+ "transformers_version": "4.37.2",
27
  "use_cache": true,
28
  "vocab_list": null,
29
  "vocab_size": 33
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:2dbf6ff092abbdb7c8d6b6f2e7337f0aae7c0e95408bef3a1e005044c928ba8c
3
  size 136000488
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d0ea43ef01adeb69e4ad577380ed31e90c1a5bd0f0619ee9b66bf3bd9e22fff5
3
  size 136000488
optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5d7d8e2181fbd44c975aa71b4db7db47e5e346397a615f83767017f384dda543
3
+ size 268174417
rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:da25f32745f674c5f578c521f588f79917675eeaf6e0b8dcae003f3745222130
3
+ size 14503
scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:01ea102be1ec9698bc4ca3c3c653aa98e113cf491e0d96462a950ea007d7260b
3
+ size 623
trainer_state.json ADDED
@@ -0,0 +1,553 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 8.6665325513378,
5
+ "eval_steps": 5000,
6
+ "global_step": 70000,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.12,
13
+ "learning_rate": 9.958730551772523e-05,
14
+ "loss": 1.4035,
15
+ "step": 1000
16
+ },
17
+ {
18
+ "epoch": 0.25,
19
+ "learning_rate": 9.917461103545046e-05,
20
+ "loss": 1.332,
21
+ "step": 2000
22
+ },
23
+ {
24
+ "epoch": 0.37,
25
+ "learning_rate": 9.87619165531757e-05,
26
+ "loss": 1.2981,
27
+ "step": 3000
28
+ },
29
+ {
30
+ "epoch": 0.5,
31
+ "learning_rate": 9.834922207090091e-05,
32
+ "loss": 1.272,
33
+ "step": 4000
34
+ },
35
+ {
36
+ "epoch": 0.62,
37
+ "learning_rate": 9.793652758862615e-05,
38
+ "loss": 1.2501,
39
+ "step": 5000
40
+ },
41
+ {
42
+ "epoch": 0.62,
43
+ "eval_loss": 1.1940463781356812,
44
+ "eval_runtime": 120.3265,
45
+ "eval_samples_per_second": 134.692,
46
+ "eval_steps_per_second": 16.838,
47
+ "step": 5000
48
+ },
49
+ {
50
+ "epoch": 0.74,
51
+ "learning_rate": 9.752383310635137e-05,
52
+ "loss": 1.2328,
53
+ "step": 6000
54
+ },
55
+ {
56
+ "epoch": 0.87,
57
+ "learning_rate": 9.71111386240766e-05,
58
+ "loss": 1.208,
59
+ "step": 7000
60
+ },
61
+ {
62
+ "epoch": 0.99,
63
+ "learning_rate": 9.669844414180183e-05,
64
+ "loss": 1.1886,
65
+ "step": 8000
66
+ },
67
+ {
68
+ "epoch": 1.11,
69
+ "learning_rate": 9.628574965952706e-05,
70
+ "loss": 1.182,
71
+ "step": 9000
72
+ },
73
+ {
74
+ "epoch": 1.24,
75
+ "learning_rate": 9.587305517725228e-05,
76
+ "loss": 1.1761,
77
+ "step": 10000
78
+ },
79
+ {
80
+ "epoch": 1.24,
81
+ "eval_loss": 1.1365982294082642,
82
+ "eval_runtime": 124.7536,
83
+ "eval_samples_per_second": 129.912,
84
+ "eval_steps_per_second": 16.24,
85
+ "step": 10000
86
+ },
87
+ {
88
+ "epoch": 1.36,
89
+ "learning_rate": 9.546036069497752e-05,
90
+ "loss": 1.1594,
91
+ "step": 11000
92
+ },
93
+ {
94
+ "epoch": 1.49,
95
+ "learning_rate": 9.504766621270275e-05,
96
+ "loss": 1.1449,
97
+ "step": 12000
98
+ },
99
+ {
100
+ "epoch": 1.61,
101
+ "learning_rate": 9.463497173042796e-05,
102
+ "loss": 1.1387,
103
+ "step": 13000
104
+ },
105
+ {
106
+ "epoch": 1.73,
107
+ "learning_rate": 9.422227724815321e-05,
108
+ "loss": 1.1318,
109
+ "step": 14000
110
+ },
111
+ {
112
+ "epoch": 1.86,
113
+ "learning_rate": 9.380958276587842e-05,
114
+ "loss": 1.1222,
115
+ "step": 15000
116
+ },
117
+ {
118
+ "epoch": 1.86,
119
+ "eval_loss": 1.103798270225525,
120
+ "eval_runtime": 119.9985,
121
+ "eval_samples_per_second": 135.06,
122
+ "eval_steps_per_second": 16.884,
123
+ "step": 15000
124
+ },
125
+ {
126
+ "epoch": 1.98,
127
+ "learning_rate": 9.339688828360365e-05,
128
+ "loss": 1.1164,
129
+ "step": 16000
130
+ },
131
+ {
132
+ "epoch": 2.1,
133
+ "learning_rate": 9.298419380132888e-05,
134
+ "loss": 1.1073,
135
+ "step": 17000
136
+ },
137
+ {
138
+ "epoch": 2.23,
139
+ "learning_rate": 9.257149931905411e-05,
140
+ "loss": 1.1026,
141
+ "step": 18000
142
+ },
143
+ {
144
+ "epoch": 2.35,
145
+ "learning_rate": 9.215880483677933e-05,
146
+ "loss": 1.0957,
147
+ "step": 19000
148
+ },
149
+ {
150
+ "epoch": 2.48,
151
+ "learning_rate": 9.174611035450457e-05,
152
+ "loss": 1.0859,
153
+ "step": 20000
154
+ },
155
+ {
156
+ "epoch": 2.48,
157
+ "eval_loss": 1.0691076517105103,
158
+ "eval_runtime": 120.0736,
159
+ "eval_samples_per_second": 134.976,
160
+ "eval_steps_per_second": 16.873,
161
+ "step": 20000
162
+ },
163
+ {
164
+ "epoch": 2.6,
165
+ "learning_rate": 9.133341587222979e-05,
166
+ "loss": 1.0856,
167
+ "step": 21000
168
+ },
169
+ {
170
+ "epoch": 2.72,
171
+ "learning_rate": 9.092072138995502e-05,
172
+ "loss": 1.0769,
173
+ "step": 22000
174
+ },
175
+ {
176
+ "epoch": 2.85,
177
+ "learning_rate": 9.050802690768025e-05,
178
+ "loss": 1.0768,
179
+ "step": 23000
180
+ },
181
+ {
182
+ "epoch": 2.97,
183
+ "learning_rate": 9.009533242540548e-05,
184
+ "loss": 1.0671,
185
+ "step": 24000
186
+ },
187
+ {
188
+ "epoch": 3.09,
189
+ "learning_rate": 8.968263794313071e-05,
190
+ "loss": 1.0703,
191
+ "step": 25000
192
+ },
193
+ {
194
+ "epoch": 3.09,
195
+ "eval_loss": 1.0342341661453247,
196
+ "eval_runtime": 142.3338,
197
+ "eval_samples_per_second": 113.866,
198
+ "eval_steps_per_second": 14.234,
199
+ "step": 25000
200
+ },
201
+ {
202
+ "epoch": 3.22,
203
+ "learning_rate": 8.926994346085594e-05,
204
+ "loss": 1.0535,
205
+ "step": 26000
206
+ },
207
+ {
208
+ "epoch": 3.34,
209
+ "learning_rate": 8.885724897858117e-05,
210
+ "loss": 1.0537,
211
+ "step": 27000
212
+ },
213
+ {
214
+ "epoch": 3.47,
215
+ "learning_rate": 8.844455449630638e-05,
216
+ "loss": 1.0502,
217
+ "step": 28000
218
+ },
219
+ {
220
+ "epoch": 3.59,
221
+ "learning_rate": 8.803186001403163e-05,
222
+ "loss": 1.0401,
223
+ "step": 29000
224
+ },
225
+ {
226
+ "epoch": 3.71,
227
+ "learning_rate": 8.761916553175684e-05,
228
+ "loss": 1.0461,
229
+ "step": 30000
230
+ },
231
+ {
232
+ "epoch": 3.71,
233
+ "eval_loss": 1.0171171426773071,
234
+ "eval_runtime": 155.2187,
235
+ "eval_samples_per_second": 104.414,
236
+ "eval_steps_per_second": 13.053,
237
+ "step": 30000
238
+ },
239
+ {
240
+ "epoch": 3.84,
241
+ "learning_rate": 8.720647104948207e-05,
242
+ "loss": 1.0362,
243
+ "step": 31000
244
+ },
245
+ {
246
+ "epoch": 3.96,
247
+ "learning_rate": 8.67937765672073e-05,
248
+ "loss": 1.0332,
249
+ "step": 32000
250
+ },
251
+ {
252
+ "epoch": 4.09,
253
+ "learning_rate": 8.638108208493253e-05,
254
+ "loss": 1.0293,
255
+ "step": 33000
256
+ },
257
+ {
258
+ "epoch": 4.21,
259
+ "learning_rate": 8.596838760265775e-05,
260
+ "loss": 1.0286,
261
+ "step": 34000
262
+ },
263
+ {
264
+ "epoch": 4.33,
265
+ "learning_rate": 8.555569312038299e-05,
266
+ "loss": 1.025,
267
+ "step": 35000
268
+ },
269
+ {
270
+ "epoch": 4.33,
271
+ "eval_loss": 0.9936763048171997,
272
+ "eval_runtime": 119.72,
273
+ "eval_samples_per_second": 135.374,
274
+ "eval_steps_per_second": 16.923,
275
+ "step": 35000
276
+ },
277
+ {
278
+ "epoch": 4.46,
279
+ "learning_rate": 8.514299863810822e-05,
280
+ "loss": 1.0184,
281
+ "step": 36000
282
+ },
283
+ {
284
+ "epoch": 4.58,
285
+ "learning_rate": 8.473030415583344e-05,
286
+ "loss": 1.0136,
287
+ "step": 37000
288
+ },
289
+ {
290
+ "epoch": 4.7,
291
+ "learning_rate": 8.431760967355867e-05,
292
+ "loss": 1.0103,
293
+ "step": 38000
294
+ },
295
+ {
296
+ "epoch": 4.83,
297
+ "learning_rate": 8.39049151912839e-05,
298
+ "loss": 1.0081,
299
+ "step": 39000
300
+ },
301
+ {
302
+ "epoch": 4.95,
303
+ "learning_rate": 8.349222070900913e-05,
304
+ "loss": 1.0063,
305
+ "step": 40000
306
+ },
307
+ {
308
+ "epoch": 4.95,
309
+ "eval_loss": 0.9914126396179199,
310
+ "eval_runtime": 176.7148,
311
+ "eval_samples_per_second": 91.713,
312
+ "eval_steps_per_second": 11.465,
313
+ "step": 40000
314
+ },
315
+ {
316
+ "epoch": 5.08,
317
+ "learning_rate": 8.307952622673436e-05,
318
+ "loss": 1.003,
319
+ "step": 41000
320
+ },
321
+ {
322
+ "epoch": 5.2,
323
+ "learning_rate": 8.266683174445959e-05,
324
+ "loss": 0.9947,
325
+ "step": 42000
326
+ },
327
+ {
328
+ "epoch": 5.32,
329
+ "learning_rate": 8.22541372621848e-05,
330
+ "loss": 0.999,
331
+ "step": 43000
332
+ },
333
+ {
334
+ "epoch": 5.45,
335
+ "learning_rate": 8.184144277991003e-05,
336
+ "loss": 0.9987,
337
+ "step": 44000
338
+ },
339
+ {
340
+ "epoch": 5.57,
341
+ "learning_rate": 8.142874829763526e-05,
342
+ "loss": 0.988,
343
+ "step": 45000
344
+ },
345
+ {
346
+ "epoch": 5.57,
347
+ "eval_loss": 0.9876496195793152,
348
+ "eval_runtime": 149.4553,
349
+ "eval_samples_per_second": 108.44,
350
+ "eval_steps_per_second": 13.556,
351
+ "step": 45000
352
+ },
353
+ {
354
+ "epoch": 5.69,
355
+ "learning_rate": 8.101605381536049e-05,
356
+ "loss": 0.9899,
357
+ "step": 46000
358
+ },
359
+ {
360
+ "epoch": 5.82,
361
+ "learning_rate": 8.060335933308572e-05,
362
+ "loss": 0.9876,
363
+ "step": 47000
364
+ },
365
+ {
366
+ "epoch": 5.94,
367
+ "learning_rate": 8.019066485081095e-05,
368
+ "loss": 0.9889,
369
+ "step": 48000
370
+ },
371
+ {
372
+ "epoch": 6.07,
373
+ "learning_rate": 7.977797036853618e-05,
374
+ "loss": 0.9807,
375
+ "step": 49000
376
+ },
377
+ {
378
+ "epoch": 6.19,
379
+ "learning_rate": 7.93652758862614e-05,
380
+ "loss": 0.971,
381
+ "step": 50000
382
+ },
383
+ {
384
+ "epoch": 6.19,
385
+ "eval_loss": 0.9691153764724731,
386
+ "eval_runtime": 155.3931,
387
+ "eval_samples_per_second": 104.297,
388
+ "eval_steps_per_second": 13.038,
389
+ "step": 50000
390
+ },
391
+ {
392
+ "epoch": 6.31,
393
+ "learning_rate": 7.895258140398664e-05,
394
+ "loss": 0.9796,
395
+ "step": 51000
396
+ },
397
+ {
398
+ "epoch": 6.44,
399
+ "learning_rate": 7.853988692171186e-05,
400
+ "loss": 0.9784,
401
+ "step": 52000
402
+ },
403
+ {
404
+ "epoch": 6.56,
405
+ "learning_rate": 7.812719243943709e-05,
406
+ "loss": 0.9682,
407
+ "step": 53000
408
+ },
409
+ {
410
+ "epoch": 6.69,
411
+ "learning_rate": 7.771449795716232e-05,
412
+ "loss": 0.9609,
413
+ "step": 54000
414
+ },
415
+ {
416
+ "epoch": 6.81,
417
+ "learning_rate": 7.730180347488755e-05,
418
+ "loss": 0.9704,
419
+ "step": 55000
420
+ },
421
+ {
422
+ "epoch": 6.81,
423
+ "eval_loss": 0.9535136222839355,
424
+ "eval_runtime": 160.7447,
425
+ "eval_samples_per_second": 100.824,
426
+ "eval_steps_per_second": 12.604,
427
+ "step": 55000
428
+ },
429
+ {
430
+ "epoch": 6.93,
431
+ "learning_rate": 7.688910899261278e-05,
432
+ "loss": 0.965,
433
+ "step": 56000
434
+ },
435
+ {
436
+ "epoch": 7.06,
437
+ "learning_rate": 7.6476414510338e-05,
438
+ "loss": 0.9574,
439
+ "step": 57000
440
+ },
441
+ {
442
+ "epoch": 7.18,
443
+ "learning_rate": 7.606372002806322e-05,
444
+ "loss": 0.9565,
445
+ "step": 58000
446
+ },
447
+ {
448
+ "epoch": 7.3,
449
+ "learning_rate": 7.565102554578845e-05,
450
+ "loss": 0.9514,
451
+ "step": 59000
452
+ },
453
+ {
454
+ "epoch": 7.43,
455
+ "learning_rate": 7.52383310635137e-05,
456
+ "loss": 0.951,
457
+ "step": 60000
458
+ },
459
+ {
460
+ "epoch": 7.43,
461
+ "eval_loss": 0.9359220266342163,
462
+ "eval_runtime": 124.2464,
463
+ "eval_samples_per_second": 130.442,
464
+ "eval_steps_per_second": 16.306,
465
+ "step": 60000
466
+ },
467
+ {
468
+ "epoch": 7.55,
469
+ "learning_rate": 7.482563658123891e-05,
470
+ "loss": 0.9477,
471
+ "step": 61000
472
+ },
473
+ {
474
+ "epoch": 7.68,
475
+ "learning_rate": 7.441294209896414e-05,
476
+ "loss": 0.9581,
477
+ "step": 62000
478
+ },
479
+ {
480
+ "epoch": 7.8,
481
+ "learning_rate": 7.400024761668937e-05,
482
+ "loss": 0.948,
483
+ "step": 63000
484
+ },
485
+ {
486
+ "epoch": 7.92,
487
+ "learning_rate": 7.35875531344146e-05,
488
+ "loss": 0.9439,
489
+ "step": 64000
490
+ },
491
+ {
492
+ "epoch": 8.05,
493
+ "learning_rate": 7.317485865213982e-05,
494
+ "loss": 0.9447,
495
+ "step": 65000
496
+ },
497
+ {
498
+ "epoch": 8.05,
499
+ "eval_loss": 0.9360346794128418,
500
+ "eval_runtime": 160.0152,
501
+ "eval_samples_per_second": 101.284,
502
+ "eval_steps_per_second": 12.661,
503
+ "step": 65000
504
+ },
505
+ {
506
+ "epoch": 8.17,
507
+ "learning_rate": 7.276216416986506e-05,
508
+ "loss": 0.9377,
509
+ "step": 66000
510
+ },
511
+ {
512
+ "epoch": 8.29,
513
+ "learning_rate": 7.234946968759027e-05,
514
+ "loss": 0.9455,
515
+ "step": 67000
516
+ },
517
+ {
518
+ "epoch": 8.42,
519
+ "learning_rate": 7.19367752053155e-05,
520
+ "loss": 0.9332,
521
+ "step": 68000
522
+ },
523
+ {
524
+ "epoch": 8.54,
525
+ "learning_rate": 7.152408072304073e-05,
526
+ "loss": 0.9374,
527
+ "step": 69000
528
+ },
529
+ {
530
+ "epoch": 8.67,
531
+ "learning_rate": 7.111138624076596e-05,
532
+ "loss": 0.9512,
533
+ "step": 70000
534
+ },
535
+ {
536
+ "epoch": 8.67,
537
+ "eval_loss": 0.930946946144104,
538
+ "eval_runtime": 132.6824,
539
+ "eval_samples_per_second": 122.149,
540
+ "eval_steps_per_second": 15.27,
541
+ "step": 70000
542
+ }
543
+ ],
544
+ "logging_steps": 1000,
545
+ "max_steps": 242310,
546
+ "num_input_tokens_seen": 0,
547
+ "num_train_epochs": 30,
548
+ "save_steps": 500,
549
+ "total_flos": 3.5013441751675136e+17,
550
+ "train_batch_size": 8,
551
+ "trial_name": null,
552
+ "trial_params": null
553
+ }
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:5365a086024237c5a30551f69abfc2fc5e635109274518c2be55b9f91d4ff333
3
  size 4207
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:16c7088a420614f10c4caeee3a98e60e4ba453f05edd647671fd07146b95e23d
3
  size 4207