File size: 20,580 Bytes
93e6ca2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.997638167217761,
  "eval_steps": 50,
  "global_step": 176,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "dpo_loss": 0.6931471824645996,
      "epoch": 0.005668398677373642,
      "grad_norm": 1341367.1350394282,
      "learning_rate": 2.7777777777777776e-07,
      "logits": -1.3147305250167847,
      "logps": -88.0877456665039,
      "loss": 0.6931,
      "objective": 0.6931471824645996,
      "ranking_idealized": 0.5208333134651184,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.5208333134651184,
      "regularize": 0.6931471824645996,
      "step": 1
    },
    {
      "dpo_loss": 1280.619140625,
      "epoch": 0.02834199338686821,
      "grad_norm": 2623988.3634039685,
      "learning_rate": 1.3888888888888892e-06,
      "logits": -1.3680986166000366,
      "logps": -84.14160919189453,
      "loss": 1112.7681,
      "objective": 1280.619140625,
      "ranking_idealized": 0.546875,
      "ranking_idealized_expo": 0.546875,
      "ranking_simple": 0.5520833134651184,
      "regularize": 1280.619140625,
      "step": 5
    },
    {
      "dpo_loss": 2070.714599609375,
      "epoch": 0.05668398677373642,
      "grad_norm": 2330391.207726475,
      "learning_rate": 2.7777777777777783e-06,
      "logits": -1.45966374874115,
      "logps": -83.2505111694336,
      "loss": 2407.7941,
      "objective": 2070.714599609375,
      "ranking_idealized": 0.5083333253860474,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.5166666507720947,
      "regularize": 2070.714599609375,
      "step": 10
    },
    {
      "dpo_loss": 5679.9853515625,
      "epoch": 0.08502598016060463,
      "grad_norm": 1644564.4852830889,
      "learning_rate": 4.166666666666667e-06,
      "logits": -1.4328060150146484,
      "logps": -82.17012023925781,
      "loss": 4264.0652,
      "objective": 5679.9853515625,
      "ranking_idealized": 0.5041666626930237,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.5166666507720947,
      "regularize": 5679.9853515625,
      "step": 15
    },
    {
      "dpo_loss": 7525.20068359375,
      "epoch": 0.11336797354747284,
      "grad_norm": 1515613.7017780922,
      "learning_rate": 4.998023493068255e-06,
      "logits": -1.4038361310958862,
      "logps": -82.86460876464844,
      "loss": 6967.5602,
      "objective": 7525.20068359375,
      "ranking_idealized": 0.5375000238418579,
      "ranking_idealized_expo": 0.5375000238418579,
      "ranking_simple": 0.5583333373069763,
      "regularize": 7525.20068359375,
      "step": 20
    },
    {
      "dpo_loss": 8746.1123046875,
      "epoch": 0.14170996693434104,
      "grad_norm": 1687436.4259306653,
      "learning_rate": 4.975823666181256e-06,
      "logits": -1.371711254119873,
      "logps": -82.0951156616211,
      "loss": 9527.1836,
      "objective": 8746.1123046875,
      "ranking_idealized": 0.4958333373069763,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.512499988079071,
      "regularize": 8746.1123046875,
      "step": 25
    },
    {
      "dpo_loss": 11099.513671875,
      "epoch": 0.17005196032120926,
      "grad_norm": 1294226.520732075,
      "learning_rate": 4.929173350101025e-06,
      "logits": -1.3348666429519653,
      "logps": -81.38406372070312,
      "loss": 10099.4609,
      "objective": 11099.513671875,
      "ranking_idealized": 0.5416666865348816,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.5458333492279053,
      "regularize": 11099.513671875,
      "step": 30
    },
    {
      "dpo_loss": 12696.7705078125,
      "epoch": 0.19839395370807747,
      "grad_norm": 1391829.2894247118,
      "learning_rate": 4.858533249305337e-06,
      "logits": -1.411891222000122,
      "logps": -77.1667251586914,
      "loss": 9874.5609,
      "objective": 12696.7705078125,
      "ranking_idealized": 0.5041666626930237,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.5375000238418579,
      "regularize": 12696.7705078125,
      "step": 35
    },
    {
      "dpo_loss": 8431.9521484375,
      "epoch": 0.22673594709494568,
      "grad_norm": 1186854.0601841968,
      "learning_rate": 4.764600984163809e-06,
      "logits": -1.3430523872375488,
      "logps": -77.41344451904297,
      "loss": 10212.3687,
      "objective": 8431.9521484375,
      "ranking_idealized": 0.5208333134651184,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.5874999761581421,
      "regularize": 8431.9521484375,
      "step": 40
    },
    {
      "dpo_loss": 11319.8056640625,
      "epoch": 0.25507794048181387,
      "grad_norm": 1072714.7178744846,
      "learning_rate": 4.648304201449153e-06,
      "logits": -1.3935389518737793,
      "logps": -79.65689086914062,
      "loss": 10434.6648,
      "objective": 11319.8056640625,
      "ranking_idealized": 0.5375000238418579,
      "ranking_idealized_expo": 0.5375000238418579,
      "ranking_simple": 0.574999988079071,
      "regularize": 11319.8056640625,
      "step": 45
    },
    {
      "dpo_loss": 9569.685546875,
      "epoch": 0.2834199338686821,
      "grad_norm": 950921.657886262,
      "learning_rate": 4.510791413176912e-06,
      "logits": -1.3682667016983032,
      "logps": -77.96815490722656,
      "loss": 10470.1625,
      "objective": 9569.685546875,
      "ranking_idealized": 0.42500001192092896,
      "ranking_idealized_expo": 0.42500001192092896,
      "ranking_simple": 0.5,
      "regularize": 9569.685546875,
      "step": 50
    },
    {
      "epoch": 0.2834199338686821,
      "eval_dpo_loss": 11815.3388671875,
      "eval_logits": -1.4585951566696167,
      "eval_logps": -84.9886474609375,
      "eval_loss": 11322.9775390625,
      "eval_objective": 11815.3388671875,
      "eval_ranking_idealized": 0.5092975497245789,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5165289044380188,
      "eval_regularize": 11815.3388671875,
      "eval_runtime": 1211.6497,
      "eval_samples_per_second": 4.779,
      "eval_steps_per_second": 0.2,
      "step": 50
    },
    {
      "dpo_loss": 12371.0048828125,
      "epoch": 0.3117619272555503,
      "grad_norm": 1109923.0216191527,
      "learning_rate": 4.353420654246546e-06,
      "logits": -1.4771044254302979,
      "logps": -79.130126953125,
      "loss": 11113.0828,
      "objective": 12371.0048828125,
      "ranking_idealized": 0.5416666865348816,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.5541666746139526,
      "regularize": 12371.0048828125,
      "step": 55
    },
    {
      "dpo_loss": 8734.4072265625,
      "epoch": 0.3401039206424185,
      "grad_norm": 1211359.9780672283,
      "learning_rate": 4.177746070897593e-06,
      "logits": -1.4155789613723755,
      "logps": -77.57923126220703,
      "loss": 11080.9961,
      "objective": 8734.4072265625,
      "ranking_idealized": 0.512499988079071,
      "ranking_idealized_expo": 0.512499988079071,
      "ranking_simple": 0.5708333253860474,
      "regularize": 8734.4072265625,
      "step": 60
    },
    {
      "dpo_loss": 9739.5908203125,
      "epoch": 0.3684459140292867,
      "grad_norm": 1057255.008796819,
      "learning_rate": 3.985502572429277e-06,
      "logits": -1.3343302011489868,
      "logps": -77.00284576416016,
      "loss": 11711.2734,
      "objective": 9739.5908203125,
      "ranking_idealized": 0.5083333253860474,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.5541666746139526,
      "regularize": 9739.5908203125,
      "step": 65
    },
    {
      "dpo_loss": 9313.849609375,
      "epoch": 0.39678790741615494,
      "grad_norm": 1088725.4948180364,
      "learning_rate": 3.7785886977585562e-06,
      "logits": -1.3323314189910889,
      "logps": -76.60472106933594,
      "loss": 11694.0484,
      "objective": 9313.849609375,
      "ranking_idealized": 0.5166666507720947,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5416666865348816,
      "regularize": 9313.849609375,
      "step": 70
    },
    {
      "dpo_loss": 10158.4462890625,
      "epoch": 0.42512990080302315,
      "grad_norm": 999614.3040187941,
      "learning_rate": 3.5590478660213214e-06,
      "logits": -1.2671582698822021,
      "logps": -77.10884094238281,
      "loss": 11305.65,
      "objective": 10158.4462890625,
      "ranking_idealized": 0.5458333492279053,
      "ranking_idealized_expo": 0.5458333492279053,
      "ranking_simple": 0.5916666388511658,
      "regularize": 10158.4462890625,
      "step": 75
    },
    {
      "dpo_loss": 13367.1591796875,
      "epoch": 0.45347189418989137,
      "grad_norm": 1036547.2837553191,
      "learning_rate": 3.32904819638017e-06,
      "logits": -1.2884188890457153,
      "logps": -76.65129852294922,
      "loss": 11108.9961,
      "objective": 13367.1591796875,
      "ranking_idealized": 0.5166666507720947,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5583333373069763,
      "regularize": 13367.1591796875,
      "step": 80
    },
    {
      "dpo_loss": 9063.92578125,
      "epoch": 0.4818138875767596,
      "grad_norm": 878439.4100295339,
      "learning_rate": 3.090861096332263e-06,
      "logits": -1.2684974670410156,
      "logps": -75.37580108642578,
      "loss": 9492.8891,
      "objective": 9063.92578125,
      "ranking_idealized": 0.5249999761581421,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.5916666388511658,
      "regularize": 9063.92578125,
      "step": 85
    },
    {
      "dpo_loss": 12140.421875,
      "epoch": 0.5101558809636277,
      "grad_norm": 839896.1614839052,
      "learning_rate": 2.8468388299726714e-06,
      "logits": -1.1826907396316528,
      "logps": -77.36532592773438,
      "loss": 8699.0336,
      "objective": 12140.421875,
      "ranking_idealized": 0.4791666567325592,
      "ranking_idealized_expo": 0.4791666567325592,
      "ranking_simple": 0.5166666507720947,
      "regularize": 12140.421875,
      "step": 90
    },
    {
      "dpo_loss": 7882.3046875,
      "epoch": 0.538497874350496,
      "grad_norm": 1169515.5696928622,
      "learning_rate": 2.599391287742315e-06,
      "logits": -1.2656235694885254,
      "logps": -76.93645477294922,
      "loss": 8873.2109,
      "objective": 7882.3046875,
      "ranking_idealized": 0.5416666865348816,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.6166666746139526,
      "regularize": 7882.3046875,
      "step": 95
    },
    {
      "dpo_loss": 7251.060546875,
      "epoch": 0.5668398677373642,
      "grad_norm": 1006084.9040205956,
      "learning_rate": 2.3509621870754505e-06,
      "logits": -1.342369794845581,
      "logps": -78.5166244506836,
      "loss": 8558.2836,
      "objective": 7251.060546875,
      "ranking_idealized": 0.5416666865348816,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.6041666865348816,
      "regularize": 7251.060546875,
      "step": 100
    },
    {
      "epoch": 0.5668398677373642,
      "eval_dpo_loss": 13248.5166015625,
      "eval_logits": -1.3720418214797974,
      "eval_logps": -82.3385238647461,
      "eval_loss": 12877.275390625,
      "eval_objective": 13248.5166015625,
      "eval_ranking_idealized": 0.5092975497245789,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5175619721412659,
      "eval_regularize": 13248.5166015625,
      "eval_runtime": 258.8572,
      "eval_samples_per_second": 22.368,
      "eval_steps_per_second": 0.935,
      "step": 100
    },
    {
      "dpo_loss": 9016.806640625,
      "epoch": 0.5951818611242324,
      "grad_norm": 861666.5853674684,
      "learning_rate": 2.1040049389819628e-06,
      "logits": -1.238232970237732,
      "logps": -78.26189422607422,
      "loss": 8298.018,
      "objective": 9016.806640625,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.612500011920929,
      "regularize": 9016.806640625,
      "step": 105
    },
    {
      "dpo_loss": 7970.41748046875,
      "epoch": 0.6235238545111006,
      "grad_norm": 865124.6869036607,
      "learning_rate": 1.8609584188988135e-06,
      "logits": -1.330941081047058,
      "logps": -76.76597595214844,
      "loss": 7414.1281,
      "objective": 7970.41748046875,
      "ranking_idealized": 0.44999998807907104,
      "ranking_idealized_expo": 0.44999998807907104,
      "ranking_simple": 0.5416666865348816,
      "regularize": 7970.41748046875,
      "step": 110
    },
    {
      "dpo_loss": 9534.12890625,
      "epoch": 0.6518658478979689,
      "grad_norm": 840576.3563797984,
      "learning_rate": 1.6242228810904393e-06,
      "logits": -1.3068194389343262,
      "logps": -76.78305053710938,
      "loss": 8769.2398,
      "objective": 9534.12890625,
      "ranking_idealized": 0.4833333194255829,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.5458333492279053,
      "regularize": 9534.12890625,
      "step": 115
    },
    {
      "dpo_loss": 8552.21484375,
      "epoch": 0.680207841284837,
      "grad_norm": 794346.4700274817,
      "learning_rate": 1.3961362544602215e-06,
      "logits": -1.2461904287338257,
      "logps": -77.25305938720703,
      "loss": 7377.8172,
      "objective": 8552.21484375,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5874999761581421,
      "regularize": 8552.21484375,
      "step": 120
    },
    {
      "dpo_loss": 9080.310546875,
      "epoch": 0.7085498346717053,
      "grad_norm": 866794.9526252571,
      "learning_rate": 1.1789510538684524e-06,
      "logits": -1.2470247745513916,
      "logps": -75.66565704345703,
      "loss": 7754.6125,
      "objective": 9080.310546875,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.6000000238418579,
      "regularize": 9080.310546875,
      "step": 125
    },
    {
      "dpo_loss": 7816.40869140625,
      "epoch": 0.7368918280585735,
      "grad_norm": 914484.5347620883,
      "learning_rate": 9.74812134973689e-07,
      "logits": -1.362328290939331,
      "logps": -74.44100952148438,
      "loss": 7071.1984,
      "objective": 7816.40869140625,
      "ranking_idealized": 0.44999998807907104,
      "ranking_idealized_expo": 0.44999998807907104,
      "ranking_simple": 0.5041666626930237,
      "regularize": 7816.40869140625,
      "step": 130
    },
    {
      "dpo_loss": 6297.37939453125,
      "epoch": 0.7652338214454416,
      "grad_norm": 908053.7687284376,
      "learning_rate": 7.857355122839674e-07,
      "logits": -1.3475323915481567,
      "logps": -76.51807403564453,
      "loss": 7334.7656,
      "objective": 6297.37939453125,
      "ranking_idealized": 0.5291666388511658,
      "ranking_idealized_expo": 0.5291666388511658,
      "ranking_simple": 0.5833333134651184,
      "regularize": 6297.37939453125,
      "step": 135
    },
    {
      "dpo_loss": 5713.158203125,
      "epoch": 0.7935758148323099,
      "grad_norm": 765346.0794896972,
      "learning_rate": 6.135884496044245e-07,
      "logits": -1.2585749626159668,
      "logps": -75.7166519165039,
      "loss": 6126.0992,
      "objective": 5713.158203125,
      "ranking_idealized": 0.5041666626930237,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.5708333253860474,
      "regularize": 5713.158203125,
      "step": 140
    },
    {
      "dpo_loss": 6023.36669921875,
      "epoch": 0.821917808219178,
      "grad_norm": 864232.6098968883,
      "learning_rate": 4.6007101950209827e-07,
      "logits": -1.311624526977539,
      "logps": -75.54283905029297,
      "loss": 6440.6867,
      "objective": 6023.36669921875,
      "ranking_idealized": 0.4833333194255829,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.5333333611488342,
      "regularize": 6023.36669921875,
      "step": 145
    },
    {
      "dpo_loss": 9931.4228515625,
      "epoch": 0.8502598016060463,
      "grad_norm": 789706.2833060493,
      "learning_rate": 3.266993139010438e-07,
      "logits": -1.3248921632766724,
      "logps": -73.5119857788086,
      "loss": 7521.95,
      "objective": 9931.4228515625,
      "ranking_idealized": 0.5375000238418579,
      "ranking_idealized_expo": 0.5375000238418579,
      "ranking_simple": 0.5833333134651184,
      "regularize": 9931.4228515625,
      "step": 150
    },
    {
      "epoch": 0.8502598016060463,
      "eval_dpo_loss": 12926.72265625,
      "eval_logits": -1.3555928468704224,
      "eval_logps": -80.5750961303711,
      "eval_loss": 12664.20703125,
      "eval_objective": 12926.72265625,
      "eval_ranking_idealized": 0.5092975497245789,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5227272510528564,
      "eval_regularize": 12926.72265625,
      "eval_runtime": 258.9258,
      "eval_samples_per_second": 22.362,
      "eval_steps_per_second": 0.935,
      "step": 150
    },
    {
      "dpo_loss": 5853.70361328125,
      "epoch": 0.8786017949929145,
      "grad_norm": 805936.4979433398,
      "learning_rate": 2.1479047161491351e-07,
      "logits": -1.3135905265808105,
      "logps": -73.38835144042969,
      "loss": 5683.3855,
      "objective": 5853.70361328125,
      "ranking_idealized": 0.512499988079071,
      "ranking_idealized_expo": 0.512499988079071,
      "ranking_simple": 0.612500011920929,
      "regularize": 5853.70361328125,
      "step": 155
    },
    {
      "dpo_loss": 6223.60693359375,
      "epoch": 0.9069437883797827,
      "grad_norm": 771613.5624330686,
      "learning_rate": 1.2544967068054332e-07,
      "logits": -1.2275891304016113,
      "logps": -74.46540069580078,
      "loss": 6799.2719,
      "objective": 6223.60693359375,
      "ranking_idealized": 0.4958333373069763,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.574999988079071,
      "regularize": 6223.60693359375,
      "step": 160
    },
    {
      "dpo_loss": 4967.171875,
      "epoch": 0.9352857817666509,
      "grad_norm": 1004063.6300908878,
      "learning_rate": 5.955921395237318e-08,
      "logits": -1.2412575483322144,
      "logps": -75.6294174194336,
      "loss": 6197.7492,
      "objective": 4967.171875,
      "ranking_idealized": 0.5208333134651184,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.5958333611488342,
      "regularize": 4967.171875,
      "step": 165
    },
    {
      "dpo_loss": 7402.99267578125,
      "epoch": 0.9636277751535192,
      "grad_norm": 778643.4373397075,
      "learning_rate": 1.7769815745066476e-08,
      "logits": -1.2604658603668213,
      "logps": -75.88407135009766,
      "loss": 6623.1367,
      "objective": 7402.99267578125,
      "ranking_idealized": 0.5083333253860474,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.5666666626930237,
      "regularize": 7402.99267578125,
      "step": 170
    },
    {
      "dpo_loss": 5726.1806640625,
      "epoch": 0.9919697685403873,
      "grad_norm": 762884.4185589707,
      "learning_rate": 4.941755748361088e-10,
      "logits": -1.2217018604278564,
      "logps": -76.01050567626953,
      "loss": 5562.8855,
      "objective": 5726.1806640625,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.574999988079071,
      "ranking_simple": 0.6333333253860474,
      "regularize": 5726.1806640625,
      "step": 175
    },
    {
      "epoch": 0.997638167217761,
      "step": 176,
      "total_flos": 0.0,
      "train_loss": 8072.862625997175,
      "train_runtime": 10876.2953,
      "train_samples_per_second": 4.671,
      "train_steps_per_second": 0.016
    }
  ],
  "logging_steps": 5,
  "max_steps": 176,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}