Hugging Face's logo

sfulay
/
zephyr-7b-dpo-full-prometheus-reward-scale-05

Safetensors
mistral
trl
dpo
Generated from Trainer
Model card Files Community
sfulay commited on
Commit
f3b3d0c
·
verified ·
1 Parent(s): d2f6dd3

Model save

Browse files
Files changed (4) hide show
  1. README.md +17 -17
  2. all_results.json +4 -4
  3. train_results.json +4 -4
  4. trainer_state.json +526 -526
README.md CHANGED
@@ -17,15 +17,15 @@ should probably proofread and complete it, then remove this comment. -->
17
 
18
  This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
19
  It achieves the following results on the evaluation set:
20
- - Loss: 0.5142
21
- - Rewards/chosen: -1.9575
22
- - Rewards/rejected: -3.2488
23
- - Rewards/accuracies: 0.7198
24
- - Rewards/margins: 1.2913
25
- - Logps/rejected: -573.1553
26
- - Logps/chosen: -455.7142
27
- - Logits/rejected: 4.0394
28
- - Logits/chosen: 3.0591
29
 
30
  ## Model description
31
 
@@ -62,14 +62,14 @@ The following hyperparameters were used during training:
62
 
63
  | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
64
  |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
65
- | 0.6718 | 0.1143 | 50 | 0.6623 | -0.0203 | -0.1341 | 0.6595 | 0.1139 | -261.6897 | -261.9885 | -2.5701 | -2.6137 |
66
- | 0.5837 | 0.2286 | 100 | 0.5855 | -0.7924 | -1.5143 | 0.6595 | 0.7219 | -399.7070 | -339.2018 | -0.2076 | -0.5772 |
67
- | 0.5481 | 0.3429 | 150 | 0.5498 | -1.3440 | -2.2915 | 0.6940 | 0.9475 | -477.4253 | -394.3634 | 2.4336 | 1.6485 |
68
- | 0.5289 | 0.4571 | 200 | 0.5409 | -1.6426 | -2.7545 | 0.6983 | 1.1119 | -523.7253 | -424.2230 | 3.5667 | 2.6372 |
69
- | 0.54 | 0.5714 | 250 | 0.5280 | -1.5391 | -2.7058 | 0.7026 | 1.1667 | -518.8563 | -413.8667 | 2.6437 | 1.4166 |
70
- | 0.5147 | 0.6857 | 300 | 0.5204 | -1.8487 | -3.0990 | 0.7112 | 1.2504 | -558.1808 | -444.8300 | 3.7771 | 2.6969 |
71
- | 0.5033 | 0.8 | 350 | 0.5168 | -1.8756 | -3.1461 | 0.7414 | 1.2705 | -562.8871 | -447.5259 | 3.8637 | 2.7948 |
72
- | 0.52 | 0.9143 | 400 | 0.5142 | -1.9575 | -3.2488 | 0.7198 | 1.2913 | -573.1553 | -455.7142 | 4.0394 | 3.0591 |
73
 
74
 
75
  ### Framework versions
 
17
 
18
  This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
19
  It achieves the following results on the evaluation set:
20
+ - Loss: 0.5286
21
+ - Rewards/chosen: -1.4143
22
+ - Rewards/rejected: -2.7417
23
+ - Rewards/accuracies: 0.7629
24
+ - Rewards/margins: 1.3275
25
+ - Logps/rejected: -493.2510
26
+ - Logps/chosen: -417.0316
27
+ - Logits/rejected: 1.9856
28
+ - Logits/chosen: 0.4911
29
 
30
  ## Model description
31
 
 
62
 
63
  | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
64
  |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
65
+ | 0.6696 | 0.1143 | 50 | 0.6584 | -0.0084 | -0.1643 | 0.6853 | 0.1559 | -235.5072 | -276.4426 | -2.4382 | -2.5406 |
66
+ | 0.6122 | 0.2286 | 100 | 0.6111 | -0.4070 | -0.8953 | 0.6767 | 0.4883 | -308.6058 | -316.3019 | -2.5533 | -2.6512 |
67
+ | 0.5476 | 0.3429 | 150 | 0.5583 | -1.3343 | -2.3426 | 0.7371 | 1.0083 | -453.3369 | -409.0355 | 0.9770 | 0.1441 |
68
+ | 0.5582 | 0.4571 | 200 | 0.5499 | -1.0345 | -2.1424 | 0.7328 | 1.1079 | -433.3173 | -379.0511 | 0.5624 | -0.4976 |
69
+ | 0.5503 | 0.5714 | 250 | 0.5393 | -1.1701 | -2.3108 | 0.7371 | 1.1406 | -450.1522 | -392.6152 | 0.7719 | -0.3725 |
70
+ | 0.5224 | 0.6857 | 300 | 0.5312 | -1.2228 | -2.5102 | 0.7543 | 1.2874 | -470.0949 | -397.8840 | 1.7088 | 0.1892 |
71
+ | 0.5396 | 0.8 | 350 | 0.5290 | -1.4462 | -2.7485 | 0.75 | 1.3024 | -493.9275 | -420.2202 | 1.9215 | 0.4365 |
72
+ | 0.55 | 0.9143 | 400 | 0.5286 | -1.4143 | -2.7417 | 0.7629 | 1.3275 | -493.2510 | -417.0316 | 1.9856 | 0.4911 |
73
 
74
 
75
  ### Framework versions
all_results.json CHANGED
@@ -1,9 +1,9 @@
1
  {
2
  "epoch": 0.9988571428571429,
3
  "total_flos": 0.0,
4
- "train_loss": 0.5629221896557841,
5
- "train_runtime": 11730.276,
6
  "train_samples": 55999,
7
- "train_samples_per_second": 4.774,
8
- "train_steps_per_second": 0.037
9
  }
 
1
  {
2
  "epoch": 0.9988571428571429,
3
  "total_flos": 0.0,
4
+ "train_loss": 0.5693180419214803,
5
+ "train_runtime": 11386.9149,
6
  "train_samples": 55999,
7
+ "train_samples_per_second": 4.918,
8
+ "train_steps_per_second": 0.038
9
  }
train_results.json CHANGED
@@ -1,9 +1,9 @@
1
  {
2
  "epoch": 0.9988571428571429,
3
  "total_flos": 0.0,
4
- "train_loss": 0.5629221896557841,
5
- "train_runtime": 11730.276,
6
  "train_samples": 55999,
7
- "train_samples_per_second": 4.774,
8
- "train_steps_per_second": 0.037
9
  }
 
1
  {
2
  "epoch": 0.9988571428571429,
3
  "total_flos": 0.0,
4
+ "train_loss": 0.5693180419214803,
5
+ "train_runtime": 11386.9149,
6
  "train_samples": 55999,
7
+ "train_samples_per_second": 4.918,
8
+ "train_steps_per_second": 0.038
9
  }
trainer_state.json CHANGED
@@ -10,785 +10,785 @@
10
  "log_history": [
11
  {
12
  "epoch": 0.022857142857142857,
13
- "grad_norm": 4.698039299265843,
14
  "learning_rate": 1.1363636363636363e-07,
15
- "logits/chosen": -2.654461622238159,
16
- "logits/rejected": -2.6349782943725586,
17
- "logps/chosen": -287.92938232421875,
18
- "logps/rejected": -266.479248046875,
19
- "loss": 0.6932,
20
- "rewards/accuracies": 0.4000000059604645,
21
- "rewards/chosen": -0.00014368369011208415,
22
- "rewards/margins": -0.00023763992066960782,
23
- "rewards/rejected": 9.395615052198991e-05,
24
  "step": 10
25
  },
26
  {
27
  "epoch": 0.045714285714285714,
28
- "grad_norm": 4.697985911511002,
29
  "learning_rate": 2.2727272727272726e-07,
30
- "logits/chosen": -2.655208110809326,
31
- "logits/rejected": -2.623675584793091,
32
- "logps/chosen": -271.43304443359375,
33
- "logps/rejected": -260.2268981933594,
34
- "loss": 0.6927,
35
- "rewards/accuracies": 0.581250011920929,
36
- "rewards/chosen": 0.002769916784018278,
37
- "rewards/margins": 0.0018681169021874666,
38
- "rewards/rejected": 0.0009017998236231506,
39
  "step": 20
40
  },
41
  {
42
  "epoch": 0.06857142857142857,
43
- "grad_norm": 4.400775078217756,
44
  "learning_rate": 3.4090909090909085e-07,
45
- "logits/chosen": -2.6501145362854004,
46
- "logits/rejected": -2.5595479011535645,
47
- "logps/chosen": -271.99346923828125,
48
- "logps/rejected": -254.75973510742188,
49
- "loss": 0.6903,
50
- "rewards/accuracies": 0.737500011920929,
51
- "rewards/chosen": 0.018953289836645126,
52
- "rewards/margins": 0.01143287681043148,
53
- "rewards/rejected": 0.007520412094891071,
54
  "step": 30
55
  },
56
  {
57
  "epoch": 0.09142857142857143,
58
- "grad_norm": 4.224410491085399,
59
  "learning_rate": 4.545454545454545e-07,
60
- "logits/chosen": -2.610691547393799,
61
- "logits/rejected": -2.596449375152588,
62
- "logps/chosen": -262.10076904296875,
63
- "logps/rejected": -255.63131713867188,
64
- "loss": 0.6838,
65
- "rewards/accuracies": 0.6937500238418579,
66
- "rewards/chosen": 0.03673754259943962,
67
- "rewards/margins": 0.03850514069199562,
68
- "rewards/rejected": -0.0017675980925559998,
69
  "step": 40
70
  },
71
  {
72
  "epoch": 0.11428571428571428,
73
- "grad_norm": 5.26619927871512,
74
  "learning_rate": 4.997124959943201e-07,
75
- "logits/chosen": -2.59323787689209,
76
- "logits/rejected": -2.5254805088043213,
77
- "logps/chosen": -288.6198425292969,
78
- "logps/rejected": -263.8785095214844,
79
- "loss": 0.6718,
80
- "rewards/accuracies": 0.75,
81
- "rewards/chosen": 0.008304540999233723,
82
- "rewards/margins": 0.10466225445270538,
83
- "rewards/rejected": -0.09635771811008453,
84
  "step": 50
85
  },
86
  {
87
  "epoch": 0.11428571428571428,
88
- "eval_logits/chosen": -2.613680839538574,
89
- "eval_logits/rejected": -2.570056200027466,
90
- "eval_logps/chosen": -261.988525390625,
91
- "eval_logps/rejected": -261.6897277832031,
92
- "eval_loss": 0.6623334884643555,
93
- "eval_rewards/accuracies": 0.6594827771186829,
94
- "eval_rewards/chosen": -0.020272402092814445,
95
- "eval_rewards/margins": 0.11385296285152435,
96
- "eval_rewards/rejected": -0.13412536680698395,
97
- "eval_runtime": 94.9862,
98
- "eval_samples_per_second": 19.276,
99
- "eval_steps_per_second": 0.305,
100
  "step": 50
101
  },
102
  {
103
  "epoch": 0.13714285714285715,
104
- "grad_norm": 6.932576674963505,
105
  "learning_rate": 4.979579212164186e-07,
106
- "logits/chosen": -2.6151022911071777,
107
- "logits/rejected": -2.563739776611328,
108
- "logps/chosen": -303.95672607421875,
109
- "logps/rejected": -269.7010498046875,
110
- "loss": 0.6526,
111
- "rewards/accuracies": 0.7124999761581421,
112
- "rewards/chosen": -0.04470657557249069,
113
- "rewards/margins": 0.18772244453430176,
114
- "rewards/rejected": -0.23242902755737305,
115
  "step": 60
116
  },
117
  {
118
  "epoch": 0.16,
119
- "grad_norm": 10.775877567674739,
120
  "learning_rate": 4.946196886175515e-07,
121
- "logits/chosen": -2.5894806385040283,
122
- "logits/rejected": -2.5019643306732178,
123
- "logps/chosen": -320.10382080078125,
124
- "logps/rejected": -296.3632507324219,
125
- "loss": 0.6419,
126
- "rewards/accuracies": 0.7124999761581421,
127
- "rewards/chosen": -0.1674882471561432,
128
- "rewards/margins": 0.30879640579223633,
129
- "rewards/rejected": -0.47628456354141235,
130
  "step": 70
131
  },
132
  {
133
  "epoch": 0.18285714285714286,
134
- "grad_norm": 8.903665160814306,
135
  "learning_rate": 4.897191188239667e-07,
136
- "logits/chosen": -2.577723979949951,
137
- "logits/rejected": -2.5037548542022705,
138
- "logps/chosen": -308.7467346191406,
139
- "logps/rejected": -314.86712646484375,
140
- "loss": 0.6373,
141
- "rewards/accuracies": 0.6499999761581421,
142
- "rewards/chosen": -0.41550832986831665,
143
- "rewards/margins": 0.2658213675022125,
144
- "rewards/rejected": -0.6813297271728516,
145
  "step": 80
146
  },
147
  {
148
  "epoch": 0.2057142857142857,
149
- "grad_norm": 13.187818767437374,
150
  "learning_rate": 4.832875107981763e-07,
151
- "logits/chosen": -1.7901182174682617,
152
- "logits/rejected": -1.6417770385742188,
153
- "logps/chosen": -292.3073425292969,
154
- "logps/rejected": -347.75286865234375,
155
- "loss": 0.6171,
156
- "rewards/accuracies": 0.7250000238418579,
157
- "rewards/chosen": -0.507135272026062,
158
- "rewards/margins": 0.4524330496788025,
159
- "rewards/rejected": -0.9595683813095093,
160
  "step": 90
161
  },
162
  {
163
  "epoch": 0.22857142857142856,
164
- "grad_norm": 18.455967644618873,
165
  "learning_rate": 4.753659419387223e-07,
166
- "logits/chosen": -1.0523039102554321,
167
- "logits/rejected": -0.7287885546684265,
168
- "logps/chosen": -313.5967712402344,
169
- "logps/rejected": -351.8410949707031,
170
- "loss": 0.5837,
171
- "rewards/accuracies": 0.71875,
172
- "rewards/chosen": -0.5234693884849548,
173
- "rewards/margins": 0.6886458396911621,
174
- "rewards/rejected": -1.2121151685714722,
175
  "step": 100
176
  },
177
  {
178
  "epoch": 0.22857142857142856,
179
- "eval_logits/chosen": -0.5771592259407043,
180
- "eval_logits/rejected": -0.20762209594249725,
181
- "eval_logps/chosen": -339.2017517089844,
182
- "eval_logps/rejected": -399.70703125,
183
- "eval_loss": 0.5854533314704895,
184
- "eval_rewards/accuracies": 0.6594827771186829,
185
- "eval_rewards/chosen": -0.7924044132232666,
186
- "eval_rewards/margins": 0.7218940258026123,
187
- "eval_rewards/rejected": -1.5142985582351685,
188
- "eval_runtime": 93.9256,
189
- "eval_samples_per_second": 19.494,
190
- "eval_steps_per_second": 0.309,
191
  "step": 100
192
  },
193
  {
194
  "epoch": 0.25142857142857145,
195
- "grad_norm": 20.061982050119,
196
  "learning_rate": 4.660050057270191e-07,
197
- "logits/chosen": -0.07258351147174835,
198
- "logits/rejected": 0.5934920310974121,
199
- "logps/chosen": -378.9754333496094,
200
- "logps/rejected": -401.3129577636719,
201
- "loss": 0.5614,
202
- "rewards/accuracies": 0.768750011920929,
203
- "rewards/chosen": -0.7271420955657959,
204
- "rewards/margins": 0.9416759610176086,
205
- "rewards/rejected": -1.6688178777694702,
206
  "step": 110
207
  },
208
  {
209
  "epoch": 0.2742857142857143,
210
- "grad_norm": 19.936668067723307,
211
  "learning_rate": 4.5526448859687144e-07,
212
- "logits/chosen": 0.4706380367279053,
213
- "logits/rejected": 0.9462782740592957,
214
- "logps/chosen": -361.02105712890625,
215
- "logps/rejected": -419.98455810546875,
216
- "loss": 0.5824,
217
- "rewards/accuracies": 0.7562500238418579,
218
- "rewards/chosen": -0.8818947076797485,
219
- "rewards/margins": 0.8208815455436707,
220
- "rewards/rejected": -1.702776312828064,
221
  "step": 120
222
  },
223
  {
224
  "epoch": 0.29714285714285715,
225
- "grad_norm": 19.241668087031446,
226
  "learning_rate": 4.432129880904388e-07,
227
- "logits/chosen": 0.959698498249054,
228
- "logits/rejected": 1.6635558605194092,
229
- "logps/chosen": -376.22796630859375,
230
- "logps/rejected": -403.8036193847656,
231
- "loss": 0.5653,
232
- "rewards/accuracies": 0.643750011920929,
233
- "rewards/chosen": -1.0274622440338135,
234
- "rewards/margins": 0.6216489672660828,
235
- "rewards/rejected": -1.6491111516952515,
236
  "step": 130
237
  },
238
  {
239
  "epoch": 0.32,
240
- "grad_norm": 14.721763869214124,
241
  "learning_rate": 4.299274747394055e-07,
242
- "logits/chosen": 1.4800783395767212,
243
- "logits/rejected": 2.052093029022217,
244
- "logps/chosen": -443.85260009765625,
245
- "logps/rejected": -478.54522705078125,
246
- "loss": 0.5663,
247
- "rewards/accuracies": 0.71875,
248
- "rewards/chosen": -1.2601962089538574,
249
- "rewards/margins": 0.9224704504013062,
250
- "rewards/rejected": -2.182666540145874,
251
  "step": 140
252
  },
253
  {
254
  "epoch": 0.34285714285714286,
255
- "grad_norm": 18.13176648345513,
256
  "learning_rate": 4.1549280046953653e-07,
257
- "logits/chosen": 1.882054328918457,
258
- "logits/rejected": 2.4975225925445557,
259
- "logps/chosen": -417.7086486816406,
260
- "logps/rejected": -473.6971740722656,
261
- "loss": 0.5481,
262
- "rewards/accuracies": 0.7250000238418579,
263
- "rewards/chosen": -1.2165459394454956,
264
- "rewards/margins": 0.8646339178085327,
265
- "rewards/rejected": -2.0811798572540283,
266
  "step": 150
267
  },
268
  {
269
  "epoch": 0.34285714285714286,
270
- "eval_logits/chosen": 1.6485474109649658,
271
- "eval_logits/rejected": 2.4336249828338623,
272
- "eval_logps/chosen": -394.3633728027344,
273
- "eval_logps/rejected": -477.4252624511719,
274
- "eval_loss": 0.5498274564743042,
275
- "eval_rewards/accuracies": 0.693965494632721,
276
- "eval_rewards/chosen": -1.3440207242965698,
277
- "eval_rewards/margins": 0.9474601745605469,
278
- "eval_rewards/rejected": -2.291480779647827,
279
- "eval_runtime": 94.1545,
280
- "eval_samples_per_second": 19.447,
281
- "eval_steps_per_second": 0.308,
282
  "step": 150
283
  },
284
  {
285
  "epoch": 0.3657142857142857,
286
- "grad_norm": 25.73162601058156,
287
  "learning_rate": 4.000011566683401e-07,
288
- "logits/chosen": 0.9372028112411499,
289
- "logits/rejected": 2.038717746734619,
290
- "logps/chosen": -432.73846435546875,
291
- "logps/rejected": -488.79315185546875,
292
- "loss": 0.5381,
293
- "rewards/accuracies": 0.71875,
294
- "rewards/chosen": -1.2358500957489014,
295
- "rewards/margins": 0.9547734260559082,
296
- "rewards/rejected": -2.1906232833862305,
297
  "step": 160
298
  },
299
  {
300
  "epoch": 0.38857142857142857,
301
- "grad_norm": 22.65560343905013,
302
  "learning_rate": 3.8355148537705047e-07,
303
- "logits/chosen": 1.915405511856079,
304
- "logits/rejected": 2.607452869415283,
305
- "logps/chosen": -435.5879821777344,
306
- "logps/rejected": -534.2739868164062,
307
- "loss": 0.5443,
308
  "rewards/accuracies": 0.71875,
309
- "rewards/chosen": -1.4189122915267944,
310
- "rewards/margins": 1.1850039958953857,
311
- "rewards/rejected": -2.6039161682128906,
312
  "step": 170
313
  },
314
  {
315
  "epoch": 0.4114285714285714,
316
- "grad_norm": 23.399452190018653,
317
  "learning_rate": 3.662488473675315e-07,
318
- "logits/chosen": 2.181429624557495,
319
- "logits/rejected": 3.391533374786377,
320
- "logps/chosen": -434.38287353515625,
321
- "logps/rejected": -499.68768310546875,
322
- "loss": 0.5411,
323
  "rewards/accuracies": 0.800000011920929,
324
- "rewards/chosen": -1.3194260597229004,
325
- "rewards/margins": 1.2128746509552002,
326
- "rewards/rejected": -2.5323009490966797,
327
  "step": 180
328
  },
329
  {
330
  "epoch": 0.4342857142857143,
331
- "grad_norm": 20.359816434441168,
332
  "learning_rate": 3.48203751140067e-07,
333
- "logits/chosen": 1.7659626007080078,
334
- "logits/rejected": 3.134269952774048,
335
- "logps/chosen": -422.34130859375,
336
- "logps/rejected": -507.91058349609375,
337
- "loss": 0.5419,
338
- "rewards/accuracies": 0.7562500238418579,
339
- "rewards/chosen": -1.6150566339492798,
340
- "rewards/margins": 1.1147284507751465,
341
- "rewards/rejected": -2.729785203933716,
342
  "step": 190
343
  },
344
  {
345
  "epoch": 0.45714285714285713,
346
- "grad_norm": 19.72704748785004,
347
  "learning_rate": 3.2953144712759537e-07,
348
- "logits/chosen": 2.624401569366455,
349
- "logits/rejected": 3.5372815132141113,
350
- "logps/chosen": -405.11773681640625,
351
- "logps/rejected": -503.3150939941406,
352
- "loss": 0.5289,
353
- "rewards/accuracies": 0.7749999761581421,
354
- "rewards/chosen": -1.5478131771087646,
355
- "rewards/margins": 1.097571611404419,
356
- "rewards/rejected": -2.6453845500946045,
357
  "step": 200
358
  },
359
  {
360
  "epoch": 0.45714285714285713,
361
- "eval_logits/chosen": 2.6372268199920654,
362
- "eval_logits/rejected": 3.5667386054992676,
363
- "eval_logps/chosen": -424.2229919433594,
364
- "eval_logps/rejected": -523.725341796875,
365
- "eval_loss": 0.5408812165260315,
366
- "eval_rewards/accuracies": 0.6982758641242981,
367
- "eval_rewards/chosen": -1.642616629600525,
368
- "eval_rewards/margins": 1.111864686012268,
369
- "eval_rewards/rejected": -2.754481554031372,
370
- "eval_runtime": 94.7878,
371
- "eval_samples_per_second": 19.317,
372
- "eval_steps_per_second": 0.306,
373
  "step": 200
374
  },
375
  {
376
  "epoch": 0.48,
377
- "grad_norm": 22.170086283776122,
378
  "learning_rate": 3.103511916141658e-07,
379
- "logits/chosen": 2.3618617057800293,
380
- "logits/rejected": 3.551234722137451,
381
- "logps/chosen": -444.750244140625,
382
- "logps/rejected": -544.8475341796875,
383
- "loss": 0.5315,
384
  "rewards/accuracies": 0.71875,
385
- "rewards/chosen": -1.5890655517578125,
386
- "rewards/margins": 1.23880934715271,
387
- "rewards/rejected": -2.8278753757476807,
388
  "step": 210
389
  },
390
  {
391
  "epoch": 0.5028571428571429,
392
- "grad_norm": 20.88239892734878,
393
  "learning_rate": 2.9078548506882117e-07,
394
- "logits/chosen": 2.4734537601470947,
395
- "logits/rejected": 3.3690295219421387,
396
- "logps/chosen": -479.52294921875,
397
- "logps/rejected": -594.2535400390625,
398
- "loss": 0.542,
399
- "rewards/accuracies": 0.71875,
400
- "rewards/chosen": -1.751405119895935,
401
- "rewards/margins": 1.2897474765777588,
402
- "rewards/rejected": -3.0411527156829834,
403
  "step": 220
404
  },
405
  {
406
  "epoch": 0.5257142857142857,
407
- "grad_norm": 17.747062332553714,
408
  "learning_rate": 2.709592897595191e-07,
409
- "logits/chosen": 1.912064790725708,
410
- "logits/rejected": 2.7421936988830566,
411
- "logps/chosen": -438.20208740234375,
412
- "logps/rejected": -520.5653076171875,
413
- "loss": 0.5653,
414
  "rewards/accuracies": 0.7124999761581421,
415
- "rewards/chosen": -1.534075140953064,
416
- "rewards/margins": 0.8896983861923218,
417
- "rewards/rejected": -2.4237735271453857,
418
  "step": 230
419
  },
420
  {
421
  "epoch": 0.5485714285714286,
422
- "grad_norm": 18.030085534322904,
423
  "learning_rate": 2.509992316440332e-07,
424
- "logits/chosen": 2.9936442375183105,
425
- "logits/rejected": 4.284924507141113,
426
- "logps/chosen": -503.80523681640625,
427
- "logps/rejected": -576.6605834960938,
428
- "loss": 0.5468,
429
- "rewards/accuracies": 0.7875000238418579,
430
- "rewards/chosen": -1.9611694812774658,
431
- "rewards/margins": 1.1891307830810547,
432
- "rewards/rejected": -3.1503000259399414,
433
  "step": 240
434
  },
435
  {
436
  "epoch": 0.5714285714285714,
437
- "grad_norm": 24.141080092797733,
438
  "learning_rate": 2.3103279163519918e-07,
439
- "logits/chosen": 2.715038299560547,
440
- "logits/rejected": 3.2168846130371094,
441
- "logps/chosen": -476.09295654296875,
442
- "logps/rejected": -553.4679565429688,
443
- "loss": 0.54,
444
- "rewards/accuracies": 0.65625,
445
- "rewards/chosen": -1.9882991313934326,
446
- "rewards/margins": 0.7364269495010376,
447
- "rewards/rejected": -2.7247262001037598,
448
  "step": 250
449
  },
450
  {
451
  "epoch": 0.5714285714285714,
452
- "eval_logits/chosen": 1.416563630104065,
453
- "eval_logits/rejected": 2.6436657905578613,
454
- "eval_logps/chosen": -413.8667297363281,
455
- "eval_logps/rejected": -518.8563232421875,
456
- "eval_loss": 0.5280377268791199,
457
- "eval_rewards/accuracies": 0.7025862336158752,
458
- "eval_rewards/chosen": -1.5390541553497314,
459
- "eval_rewards/margins": 1.1667375564575195,
460
- "eval_rewards/rejected": -2.70579195022583,
461
- "eval_runtime": 93.2943,
462
- "eval_samples_per_second": 19.626,
463
- "eval_steps_per_second": 0.311,
464
  "step": 250
465
  },
466
  {
467
  "epoch": 0.5942857142857143,
468
- "grad_norm": 19.312331708156304,
469
  "learning_rate": 2.1118749140573358e-07,
470
- "logits/chosen": 1.422081708908081,
471
- "logits/rejected": 2.8245503902435303,
472
- "logps/chosen": -426.8099670410156,
473
- "logps/rejected": -459.8812561035156,
474
- "loss": 0.5455,
475
- "rewards/accuracies": 0.71875,
476
- "rewards/chosen": -1.5098531246185303,
477
- "rewards/margins": 0.8642712831497192,
478
- "rewards/rejected": -2.374124526977539,
479
  "step": 260
480
  },
481
  {
482
  "epoch": 0.6171428571428571,
483
- "grad_norm": 17.556239848475172,
484
  "learning_rate": 1.9159007893272703e-07,
485
- "logits/chosen": 1.7777824401855469,
486
- "logits/rejected": 2.890540599822998,
487
- "logps/chosen": -466.5252990722656,
488
- "logps/rejected": -499.5956115722656,
489
- "loss": 0.5345,
490
- "rewards/accuracies": 0.6187499761581421,
491
- "rewards/chosen": -1.783983588218689,
492
- "rewards/margins": 0.7515356540679932,
493
- "rewards/rejected": -2.5355191230773926,
494
  "step": 270
495
  },
496
  {
497
  "epoch": 0.64,
498
- "grad_norm": 16.478054364358847,
499
  "learning_rate": 1.7236571898357766e-07,
500
- "logits/chosen": 2.3310110569000244,
501
- "logits/rejected": 3.422163486480713,
502
- "logps/chosen": -419.81512451171875,
503
- "logps/rejected": -533.2318115234375,
504
- "loss": 0.5402,
505
- "rewards/accuracies": 0.737500011920929,
506
- "rewards/chosen": -1.7008750438690186,
507
- "rewards/margins": 1.2960573434829712,
508
- "rewards/rejected": -2.9969325065612793,
509
  "step": 280
510
  },
511
  {
512
  "epoch": 0.6628571428571428,
513
- "grad_norm": 21.127488932057656,
514
  "learning_rate": 1.5363719371356882e-07,
515
- "logits/chosen": 2.3282363414764404,
516
- "logits/rejected": 3.5149283409118652,
517
- "logps/chosen": -463.74200439453125,
518
- "logps/rejected": -535.0039672851562,
519
- "loss": 0.5179,
520
- "rewards/accuracies": 0.71875,
521
- "rewards/chosen": -1.7781498432159424,
522
- "rewards/margins": 1.0403541326522827,
523
- "rewards/rejected": -2.8185038566589355,
524
  "step": 290
525
  },
526
  {
527
  "epoch": 0.6857142857142857,
528
- "grad_norm": 21.3761678454725,
529
  "learning_rate": 1.3552411848071565e-07,
530
- "logits/chosen": 2.3439698219299316,
531
- "logits/rejected": 3.820255994796753,
532
- "logps/chosen": -497.107666015625,
533
- "logps/rejected": -575.4114990234375,
534
- "loss": 0.5147,
535
- "rewards/accuracies": 0.7437499761581421,
536
- "rewards/chosen": -1.8375751972198486,
537
- "rewards/margins": 1.2724038362503052,
538
- "rewards/rejected": -3.1099791526794434,
539
  "step": 300
540
  },
541
  {
542
  "epoch": 0.6857142857142857,
543
- "eval_logits/chosen": 2.696895122528076,
544
- "eval_logits/rejected": 3.777069330215454,
545
- "eval_logps/chosen": -444.8299560546875,
546
- "eval_logps/rejected": -558.1808471679688,
547
- "eval_loss": 0.5203906297683716,
548
- "eval_rewards/accuracies": 0.7112069129943848,
549
- "eval_rewards/chosen": -1.8486860990524292,
550
- "eval_rewards/margins": 1.2503504753112793,
551
- "eval_rewards/rejected": -3.099036455154419,
552
- "eval_runtime": 93.3957,
553
- "eval_samples_per_second": 19.605,
554
- "eval_steps_per_second": 0.311,
555
  "step": 300
556
  },
557
  {
558
  "epoch": 0.7085714285714285,
559
- "grad_norm": 21.989201062769567,
560
  "learning_rate": 1.1814217788631473e-07,
561
- "logits/chosen": 2.4984259605407715,
562
- "logits/rejected": 3.986057996749878,
563
- "logps/chosen": -490.80255126953125,
564
- "logps/rejected": -586.6756591796875,
565
- "loss": 0.5285,
566
- "rewards/accuracies": 0.768750011920929,
567
- "rewards/chosen": -1.775442361831665,
568
- "rewards/margins": 1.4152400493621826,
569
- "rewards/rejected": -3.1906826496124268,
570
  "step": 310
571
  },
572
  {
573
  "epoch": 0.7314285714285714,
574
- "grad_norm": 22.181868921213347,
575
  "learning_rate": 1.0160238692045331e-07,
576
- "logits/chosen": 2.8167648315429688,
577
- "logits/rejected": 4.0772247314453125,
578
- "logps/chosen": -484.88201904296875,
579
- "logps/rejected": -577.0535888671875,
580
- "loss": 0.533,
581
- "rewards/accuracies": 0.71875,
582
- "rewards/chosen": -1.8776414394378662,
583
- "rewards/margins": 1.1431219577789307,
584
- "rewards/rejected": -3.0207631587982178,
585
  "step": 320
586
  },
587
  {
588
  "epoch": 0.7542857142857143,
589
- "grad_norm": 21.423699121141595,
590
  "learning_rate": 8.601038193139438e-08,
591
- "logits/chosen": 2.9268698692321777,
592
- "logits/rejected": 3.6747875213623047,
593
- "logps/chosen": -416.2300720214844,
594
- "logps/rejected": -506.71966552734375,
595
- "loss": 0.5385,
596
- "rewards/accuracies": 0.6812499761581421,
597
- "rewards/chosen": -1.7054163217544556,
598
- "rewards/margins": 0.9577444195747375,
599
- "rewards/rejected": -2.663160800933838,
600
  "step": 330
601
  },
602
  {
603
  "epoch": 0.7771428571428571,
604
- "grad_norm": 22.736161602909004,
605
  "learning_rate": 7.146574594727572e-08,
606
- "logits/chosen": 2.4303653240203857,
607
- "logits/rejected": 3.7860004901885986,
608
- "logps/chosen": -426.480224609375,
609
- "logps/rejected": -512.9637451171875,
610
- "loss": 0.5184,
611
- "rewards/accuracies": 0.762499988079071,
612
- "rewards/chosen": -1.8047637939453125,
613
- "rewards/margins": 1.1890126466751099,
614
- "rewards/rejected": -2.9937758445739746,
615
  "step": 340
616
  },
617
  {
618
  "epoch": 0.8,
619
- "grad_norm": 23.477309940264664,
620
  "learning_rate": 5.8061372659157306e-08,
621
- "logits/chosen": 2.426809310913086,
622
- "logits/rejected": 4.1134934425354,
623
- "logps/chosen": -493.06414794921875,
624
- "logps/rejected": -577.4049072265625,
625
- "loss": 0.5033,
626
- "rewards/accuracies": 0.768750011920929,
627
- "rewards/chosen": -1.8289105892181396,
628
- "rewards/margins": 1.2613272666931152,
629
- "rewards/rejected": -3.090238094329834,
630
  "step": 350
631
  },
632
  {
633
  "epoch": 0.8,
634
- "eval_logits/chosen": 2.7948429584503174,
635
- "eval_logits/rejected": 3.863651990890503,
636
- "eval_logps/chosen": -447.5259094238281,
637
- "eval_logps/rejected": -562.8871459960938,
638
- "eval_loss": 0.5168038606643677,
639
- "eval_rewards/accuracies": 0.7413793206214905,
640
- "eval_rewards/chosen": -1.8756457567214966,
641
- "eval_rewards/margins": 1.2704533338546753,
642
- "eval_rewards/rejected": -3.146099328994751,
643
- "eval_runtime": 92.678,
644
- "eval_samples_per_second": 19.757,
645
- "eval_steps_per_second": 0.313,
646
  "step": 350
647
  },
648
  {
649
  "epoch": 0.8228571428571428,
650
- "grad_norm": 22.089570552576745,
651
  "learning_rate": 4.5882873127531614e-08,
652
- "logits/chosen": 3.1440396308898926,
653
- "logits/rejected": 4.210689067840576,
654
- "logps/chosen": -474.7491760253906,
655
- "logps/rejected": -611.8050537109375,
656
- "loss": 0.5176,
657
- "rewards/accuracies": 0.800000011920929,
658
- "rewards/chosen": -1.8223841190338135,
659
- "rewards/margins": 1.6220409870147705,
660
- "rewards/rejected": -3.444424867630005,
661
  "step": 360
662
  },
663
  {
664
  "epoch": 0.8457142857142858,
665
- "grad_norm": 26.5380528583338,
666
  "learning_rate": 3.500802900154412e-08,
667
- "logits/chosen": 3.0336647033691406,
668
- "logits/rejected": 3.8006350994110107,
669
- "logps/chosen": -432.41278076171875,
670
- "logps/rejected": -567.8787231445312,
671
- "loss": 0.5204,
672
- "rewards/accuracies": 0.7749999761581421,
673
- "rewards/chosen": -1.7640663385391235,
674
- "rewards/margins": 1.2935981750488281,
675
- "rewards/rejected": -3.057664394378662,
676
  "step": 370
677
  },
678
  {
679
  "epoch": 0.8685714285714285,
680
- "grad_norm": 24.05648133941441,
681
  "learning_rate": 2.550629574310309e-08,
682
- "logits/chosen": 3.677473783493042,
683
- "logits/rejected": 4.557853698730469,
684
- "logps/chosen": -449.2403259277344,
685
- "logps/rejected": -553.65966796875,
686
- "loss": 0.5084,
687
- "rewards/accuracies": 0.699999988079071,
688
- "rewards/chosen": -1.9337879419326782,
689
- "rewards/margins": 1.0252636671066284,
690
- "rewards/rejected": -2.9590516090393066,
691
  "step": 380
692
  },
693
  {
694
  "epoch": 0.8914285714285715,
695
- "grad_norm": 18.50726505627672,
696
  "learning_rate": 1.7438359028687983e-08,
697
- "logits/chosen": 3.2151970863342285,
698
- "logits/rejected": 4.2298479080200195,
699
- "logps/chosen": -456.96112060546875,
700
- "logps/rejected": -567.398681640625,
701
- "loss": 0.519,
702
- "rewards/accuracies": 0.675000011920929,
703
- "rewards/chosen": -1.903903603553772,
704
- "rewards/margins": 1.2563444375991821,
705
- "rewards/rejected": -3.160248041152954,
706
  "step": 390
707
  },
708
  {
709
  "epoch": 0.9142857142857143,
710
- "grad_norm": 23.71785447914187,
711
  "learning_rate": 1.0855747162029361e-08,
712
- "logits/chosen": 2.804896831512451,
713
- "logits/rejected": 3.584916353225708,
714
- "logps/chosen": -462.6809997558594,
715
- "logps/rejected": -580.8098754882812,
716
- "loss": 0.52,
717
- "rewards/accuracies": 0.75,
718
- "rewards/chosen": -1.885479211807251,
719
- "rewards/margins": 1.2495968341827393,
720
- "rewards/rejected": -3.1350760459899902,
721
  "step": 400
722
  },
723
  {
724
  "epoch": 0.9142857142857143,
725
- "eval_logits/chosen": 3.059108257293701,
726
- "eval_logits/rejected": 4.039414882659912,
727
- "eval_logps/chosen": -455.7142333984375,
728
- "eval_logps/rejected": -573.1553344726562,
729
- "eval_loss": 0.5141626000404358,
730
- "eval_rewards/accuracies": 0.7198275923728943,
731
- "eval_rewards/chosen": -1.9575295448303223,
732
- "eval_rewards/margins": 1.2912517786026,
733
- "eval_rewards/rejected": -3.248781442642212,
734
- "eval_runtime": 94.0826,
735
- "eval_samples_per_second": 19.462,
736
- "eval_steps_per_second": 0.308,
737
  "step": 400
738
  },
739
  {
740
  "epoch": 0.9371428571428572,
741
- "grad_norm": 24.860041582436438,
742
  "learning_rate": 5.8005019731033615e-09,
743
- "logits/chosen": 3.145111560821533,
744
- "logits/rejected": 4.010159492492676,
745
- "logps/chosen": -449.4214782714844,
746
- "logps/rejected": -553.9349365234375,
747
- "loss": 0.5035,
748
- "rewards/accuracies": 0.78125,
749
- "rewards/chosen": -1.891174077987671,
750
- "rewards/margins": 1.0715255737304688,
751
- "rewards/rejected": -2.9626996517181396,
752
  "step": 410
753
  },
754
  {
755
  "epoch": 0.96,
756
- "grad_norm": 20.06298675421982,
757
  "learning_rate": 2.3049103053431886e-09,
758
- "logits/chosen": 2.699260950088501,
759
- "logits/rejected": 3.91593861579895,
760
- "logps/chosen": -503.0846252441406,
761
- "logps/rejected": -636.1543579101562,
762
- "loss": 0.524,
763
- "rewards/accuracies": 0.75,
764
- "rewards/chosen": -1.9241759777069092,
765
- "rewards/margins": 1.4948225021362305,
766
- "rewards/rejected": -3.4189987182617188,
767
  "step": 420
768
  },
769
  {
770
  "epoch": 0.9828571428571429,
771
- "grad_norm": 18.461900399401518,
772
  "learning_rate": 3.9129780600541397e-10,
773
- "logits/chosen": 3.151226043701172,
774
- "logits/rejected": 4.009431838989258,
775
- "logps/chosen": -474.90179443359375,
776
- "logps/rejected": -583.8973388671875,
777
- "loss": 0.4974,
778
- "rewards/accuracies": 0.762499988079071,
779
- "rewards/chosen": -1.8930785655975342,
780
- "rewards/margins": 1.3667445182800293,
781
- "rewards/rejected": -3.2598233222961426,
782
  "step": 430
783
  },
784
  {
785
  "epoch": 0.9988571428571429,
786
  "step": 437,
787
  "total_flos": 0.0,
788
- "train_loss": 0.5629221896557841,
789
- "train_runtime": 11730.276,
790
- "train_samples_per_second": 4.774,
791
- "train_steps_per_second": 0.037
792
  }
793
  ],
794
  "logging_steps": 10,
 
10
  "log_history": [
11
  {
12
  "epoch": 0.022857142857142857,
13
+ "grad_norm": 5.965044878833196,
14
  "learning_rate": 1.1363636363636363e-07,
15
+ "logits/chosen": -2.7006218433380127,
16
+ "logits/rejected": -2.6247599124908447,
17
+ "logps/chosen": -301.24932861328125,
18
+ "logps/rejected": -281.7940979003906,
19
+ "loss": 0.6931,
20
+ "rewards/accuracies": 0.44999998807907104,
21
+ "rewards/chosen": 0.0003684944240376353,
22
+ "rewards/margins": 0.0008126062457449734,
23
+ "rewards/rejected": -0.000444111879914999,
24
  "step": 10
25
  },
26
  {
27
  "epoch": 0.045714285714285714,
28
+ "grad_norm": 4.694626134382372,
29
  "learning_rate": 2.2727272727272726e-07,
30
+ "logits/chosen": -2.6410038471221924,
31
+ "logits/rejected": -2.60575008392334,
32
+ "logps/chosen": -278.92498779296875,
33
+ "logps/rejected": -254.63601684570312,
34
+ "loss": 0.6925,
35
+ "rewards/accuracies": 0.543749988079071,
36
+ "rewards/chosen": 0.002462259028106928,
37
+ "rewards/margins": 0.0011314961593598127,
38
+ "rewards/rejected": 0.0013307628687471151,
39
  "step": 20
40
  },
41
  {
42
  "epoch": 0.06857142857142857,
43
+ "grad_norm": 5.220071225612144,
44
  "learning_rate": 3.4090909090909085e-07,
45
+ "logits/chosen": -2.638200044631958,
46
+ "logits/rejected": -2.617208242416382,
47
+ "logps/chosen": -263.2459411621094,
48
+ "logps/rejected": -263.34710693359375,
49
+ "loss": 0.689,
50
+ "rewards/accuracies": 0.59375,
51
+ "rewards/chosen": 0.014371426776051521,
52
+ "rewards/margins": 0.007912042550742626,
53
+ "rewards/rejected": 0.006459384225308895,
54
  "step": 30
55
  },
56
  {
57
  "epoch": 0.09142857142857143,
58
+ "grad_norm": 5.914085075708232,
59
  "learning_rate": 4.545454545454545e-07,
60
+ "logits/chosen": -2.64882493019104,
61
+ "logits/rejected": -2.585529327392578,
62
+ "logps/chosen": -290.2810974121094,
63
+ "logps/rejected": -268.34210205078125,
64
+ "loss": 0.6806,
65
+ "rewards/accuracies": 0.59375,
66
+ "rewards/chosen": 0.03716137260198593,
67
+ "rewards/margins": 0.0442696288228035,
68
+ "rewards/rejected": -0.007108256220817566,
69
  "step": 40
70
  },
71
  {
72
  "epoch": 0.11428571428571428,
73
+ "grad_norm": 8.967256960057256,
74
  "learning_rate": 4.997124959943201e-07,
75
+ "logits/chosen": -2.6775121688842773,
76
+ "logits/rejected": -2.5971298217773438,
77
+ "logps/chosen": -293.7924499511719,
78
+ "logps/rejected": -254.38064575195312,
79
+ "loss": 0.6696,
80
+ "rewards/accuracies": 0.6625000238418579,
81
+ "rewards/chosen": 0.02517825737595558,
82
+ "rewards/margins": 0.1003413200378418,
83
+ "rewards/rejected": -0.07516306638717651,
84
  "step": 50
85
  },
86
  {
87
  "epoch": 0.11428571428571428,
88
+ "eval_logits/chosen": -2.5406415462493896,
89
+ "eval_logits/rejected": -2.4382479190826416,
90
+ "eval_logps/chosen": -276.4425964355469,
91
+ "eval_logps/rejected": -235.50723266601562,
92
+ "eval_loss": 0.658383309841156,
93
+ "eval_rewards/accuracies": 0.6853448152542114,
94
+ "eval_rewards/chosen": -0.008386622183024883,
95
+ "eval_rewards/margins": 0.1559244692325592,
96
+ "eval_rewards/rejected": -0.16431109607219696,
97
+ "eval_runtime": 91.7124,
98
+ "eval_samples_per_second": 19.965,
99
+ "eval_steps_per_second": 0.316,
100
  "step": 50
101
  },
102
  {
103
  "epoch": 0.13714285714285715,
104
+ "grad_norm": 7.378725048645318,
105
  "learning_rate": 4.979579212164186e-07,
106
+ "logits/chosen": -2.578993320465088,
107
+ "logits/rejected": -2.4725637435913086,
108
+ "logps/chosen": -293.21600341796875,
109
+ "logps/rejected": -274.92535400390625,
110
+ "loss": 0.6509,
111
+ "rewards/accuracies": 0.637499988079071,
112
+ "rewards/chosen": -0.1271006315946579,
113
+ "rewards/margins": 0.13663128018379211,
114
+ "rewards/rejected": -0.2637318968772888,
115
  "step": 60
116
  },
117
  {
118
  "epoch": 0.16,
119
+ "grad_norm": 7.529436455012959,
120
  "learning_rate": 4.946196886175515e-07,
121
+ "logits/chosen": -2.5928056240081787,
122
+ "logits/rejected": -2.543529748916626,
123
+ "logps/chosen": -294.546630859375,
124
+ "logps/rejected": -301.3702697753906,
125
+ "loss": 0.6315,
126
+ "rewards/accuracies": 0.6625000238418579,
127
+ "rewards/chosen": -0.1876838505268097,
128
+ "rewards/margins": 0.2297508269548416,
129
+ "rewards/rejected": -0.4174346923828125,
130
  "step": 70
131
  },
132
  {
133
  "epoch": 0.18285714285714286,
134
+ "grad_norm": 12.054303957362464,
135
  "learning_rate": 4.897191188239667e-07,
136
+ "logits/chosen": -2.6392509937286377,
137
+ "logits/rejected": -2.590977668762207,
138
+ "logps/chosen": -285.3960266113281,
139
+ "logps/rejected": -307.17535400390625,
140
+ "loss": 0.62,
141
+ "rewards/accuracies": 0.7437499761581421,
142
+ "rewards/chosen": -0.18367011845111847,
143
+ "rewards/margins": 0.33499467372894287,
144
+ "rewards/rejected": -0.5186647176742554,
145
  "step": 80
146
  },
147
  {
148
  "epoch": 0.2057142857142857,
149
+ "grad_norm": 13.273475435863975,
150
  "learning_rate": 4.832875107981763e-07,
151
+ "logits/chosen": -2.7371668815612793,
152
+ "logits/rejected": -2.6849629878997803,
153
+ "logps/chosen": -296.71575927734375,
154
+ "logps/rejected": -316.90338134765625,
155
+ "loss": 0.6249,
156
+ "rewards/accuracies": 0.699999988079071,
157
+ "rewards/chosen": -0.20430462062358856,
158
+ "rewards/margins": 0.40924978256225586,
159
+ "rewards/rejected": -0.6135543584823608,
160
  "step": 90
161
  },
162
  {
163
  "epoch": 0.22857142857142856,
164
+ "grad_norm": 15.686669316278751,
165
  "learning_rate": 4.753659419387223e-07,
166
+ "logits/chosen": -2.769486665725708,
167
+ "logits/rejected": -2.6865835189819336,
168
+ "logps/chosen": -318.80413818359375,
169
+ "logps/rejected": -312.09326171875,
170
+ "loss": 0.6122,
171
+ "rewards/accuracies": 0.7250000238418579,
172
+ "rewards/chosen": -0.324177622795105,
173
+ "rewards/margins": 0.4622408449649811,
174
+ "rewards/rejected": -0.7864184975624084,
175
  "step": 100
176
  },
177
  {
178
  "epoch": 0.22857142857142856,
179
+ "eval_logits/chosen": -2.651167869567871,
180
+ "eval_logits/rejected": -2.5533361434936523,
181
+ "eval_logps/chosen": -316.30194091796875,
182
+ "eval_logps/rejected": -308.60577392578125,
183
+ "eval_loss": 0.6111233234405518,
184
+ "eval_rewards/accuracies": 0.6767241358757019,
185
+ "eval_rewards/chosen": -0.40698006749153137,
186
+ "eval_rewards/margins": 0.4883164167404175,
187
+ "eval_rewards/rejected": -0.8952965140342712,
188
+ "eval_runtime": 90.9103,
189
+ "eval_samples_per_second": 20.141,
190
+ "eval_steps_per_second": 0.319,
191
  "step": 100
192
  },
193
  {
194
  "epoch": 0.25142857142857145,
195
+ "grad_norm": 12.723184548250023,
196
  "learning_rate": 4.660050057270191e-07,
197
+ "logits/chosen": -2.619276523590088,
198
+ "logits/rejected": -2.556680202484131,
199
+ "logps/chosen": -375.2064208984375,
200
+ "logps/rejected": -391.784423828125,
201
+ "loss": 0.6021,
202
+ "rewards/accuracies": 0.6187499761581421,
203
+ "rewards/chosen": -0.589028000831604,
204
+ "rewards/margins": 0.3497200608253479,
205
+ "rewards/rejected": -0.9387480020523071,
206
  "step": 110
207
  },
208
  {
209
  "epoch": 0.2742857142857143,
210
+ "grad_norm": 16.182958724416615,
211
  "learning_rate": 4.5526448859687144e-07,
212
+ "logits/chosen": -1.8494535684585571,
213
+ "logits/rejected": -1.6301162242889404,
214
+ "logps/chosen": -390.48797607421875,
215
+ "logps/rejected": -364.620361328125,
216
+ "loss": 0.5814,
217
+ "rewards/accuracies": 0.675000011920929,
218
+ "rewards/chosen": -0.7958351969718933,
219
+ "rewards/margins": 0.5332263708114624,
220
+ "rewards/rejected": -1.329061508178711,
221
  "step": 120
222
  },
223
  {
224
  "epoch": 0.29714285714285715,
225
+ "grad_norm": 17.332692843610236,
226
  "learning_rate": 4.432129880904388e-07,
227
+ "logits/chosen": -0.4575839638710022,
228
+ "logits/rejected": -0.06781496107578278,
229
+ "logps/chosen": -410.9315490722656,
230
+ "logps/rejected": -413.829833984375,
231
+ "loss": 0.5548,
232
+ "rewards/accuracies": 0.6312500238418579,
233
+ "rewards/chosen": -1.0577561855316162,
234
+ "rewards/margins": 0.5758394598960876,
235
+ "rewards/rejected": -1.6335957050323486,
236
  "step": 130
237
  },
238
  {
239
  "epoch": 0.32,
240
+ "grad_norm": 20.594750248375647,
241
  "learning_rate": 4.299274747394055e-07,
242
+ "logits/chosen": 0.2059406340122223,
243
+ "logits/rejected": 0.5167960524559021,
244
+ "logps/chosen": -435.4883728027344,
245
+ "logps/rejected": -472.76092529296875,
246
+ "loss": 0.5654,
247
+ "rewards/accuracies": 0.675000011920929,
248
+ "rewards/chosen": -1.200407862663269,
249
+ "rewards/margins": 0.8080868721008301,
250
+ "rewards/rejected": -2.0084948539733887,
251
  "step": 140
252
  },
253
  {
254
  "epoch": 0.34285714285714286,
255
+ "grad_norm": 16.29523919912318,
256
  "learning_rate": 4.1549280046953653e-07,
257
+ "logits/chosen": -0.2454165518283844,
258
+ "logits/rejected": 0.22050300240516663,
259
+ "logps/chosen": -396.6532287597656,
260
+ "logps/rejected": -463.4326171875,
261
+ "loss": 0.5476,
262
+ "rewards/accuracies": 0.71875,
263
+ "rewards/chosen": -1.0966728925704956,
264
+ "rewards/margins": 0.7746630311012268,
265
+ "rewards/rejected": -1.871335744857788,
266
  "step": 150
267
  },
268
  {
269
  "epoch": 0.34285714285714286,
270
+ "eval_logits/chosen": 0.14409177005290985,
271
+ "eval_logits/rejected": 0.9770079255104065,
272
+ "eval_logps/chosen": -409.03546142578125,
273
+ "eval_logps/rejected": -453.3369140625,
274
+ "eval_loss": 0.5582876801490784,
275
+ "eval_rewards/accuracies": 0.7370689511299133,
276
+ "eval_rewards/chosen": -1.3343148231506348,
277
+ "eval_rewards/margins": 1.0082927942276,
278
+ "eval_rewards/rejected": -2.3426077365875244,
279
+ "eval_runtime": 91.388,
280
+ "eval_samples_per_second": 20.035,
281
+ "eval_steps_per_second": 0.317,
282
  "step": 150
283
  },
284
  {
285
  "epoch": 0.3657142857142857,
286
+ "grad_norm": 31.42845724506196,
287
  "learning_rate": 4.000011566683401e-07,
288
+ "logits/chosen": -0.0020641356240957975,
289
+ "logits/rejected": 0.659235954284668,
290
+ "logps/chosen": -442.47259521484375,
291
+ "logps/rejected": -490.87762451171875,
292
+ "loss": 0.5549,
293
+ "rewards/accuracies": 0.75,
294
+ "rewards/chosen": -1.4313229322433472,
295
+ "rewards/margins": 0.9210258722305298,
296
+ "rewards/rejected": -2.352349042892456,
297
  "step": 160
298
  },
299
  {
300
  "epoch": 0.38857142857142857,
301
+ "grad_norm": 21.881739335743443,
302
  "learning_rate": 3.8355148537705047e-07,
303
+ "logits/chosen": -0.8011367917060852,
304
+ "logits/rejected": -0.18294472992420197,
305
+ "logps/chosen": -420.85791015625,
306
+ "logps/rejected": -446.387451171875,
307
+ "loss": 0.5563,
308
  "rewards/accuracies": 0.71875,
309
+ "rewards/chosen": -1.1675300598144531,
310
+ "rewards/margins": 0.6390342712402344,
311
+ "rewards/rejected": -1.8065645694732666,
312
  "step": 170
313
  },
314
  {
315
  "epoch": 0.4114285714285714,
316
+ "grad_norm": 24.301433957337014,
317
  "learning_rate": 3.662488473675315e-07,
318
+ "logits/chosen": -0.6645376086235046,
319
+ "logits/rejected": 0.36614301800727844,
320
+ "logps/chosen": -447.889892578125,
321
+ "logps/rejected": -494.78070068359375,
322
+ "loss": 0.5498,
323
  "rewards/accuracies": 0.800000011920929,
324
+ "rewards/chosen": -1.1771008968353271,
325
+ "rewards/margins": 1.1712000370025635,
326
+ "rewards/rejected": -2.3483011722564697,
327
  "step": 180
328
  },
329
  {
330
  "epoch": 0.4342857142857143,
331
+ "grad_norm": 18.603399872507342,
332
  "learning_rate": 3.48203751140067e-07,
333
+ "logits/chosen": -0.08548859506845474,
334
+ "logits/rejected": 0.7475250959396362,
335
+ "logps/chosen": -421.85540771484375,
336
+ "logps/rejected": -453.6908264160156,
337
+ "loss": 0.5499,
338
+ "rewards/accuracies": 0.699999988079071,
339
+ "rewards/chosen": -1.4559787511825562,
340
+ "rewards/margins": 0.7359476089477539,
341
+ "rewards/rejected": -2.1919264793395996,
342
  "step": 190
343
  },
344
  {
345
  "epoch": 0.45714285714285713,
346
+ "grad_norm": 21.90453363461546,
347
  "learning_rate": 3.2953144712759537e-07,
348
+ "logits/chosen": -0.9407933354377747,
349
+ "logits/rejected": -0.02539023384451866,
350
+ "logps/chosen": -380.4794616699219,
351
+ "logps/rejected": -437.4371643066406,
352
+ "loss": 0.5582,
353
+ "rewards/accuracies": 0.71875,
354
+ "rewards/chosen": -1.1213675737380981,
355
+ "rewards/margins": 0.9641984701156616,
356
+ "rewards/rejected": -2.0855660438537598,
357
  "step": 200
358
  },
359
  {
360
  "epoch": 0.45714285714285713,
361
+ "eval_logits/chosen": -0.4975701570510864,
362
+ "eval_logits/rejected": 0.5624167919158936,
363
+ "eval_logps/chosen": -379.0511169433594,
364
+ "eval_logps/rejected": -433.3172912597656,
365
+ "eval_loss": 0.5498641729354858,
366
+ "eval_rewards/accuracies": 0.732758641242981,
367
+ "eval_rewards/chosen": -1.034471869468689,
368
+ "eval_rewards/margins": 1.107939600944519,
369
+ "eval_rewards/rejected": -2.142411708831787,
370
+ "eval_runtime": 90.2066,
371
+ "eval_samples_per_second": 20.298,
372
+ "eval_steps_per_second": 0.321,
373
  "step": 200
374
  },
375
  {
376
  "epoch": 0.48,
377
+ "grad_norm": 16.10426120639833,
378
  "learning_rate": 3.103511916141658e-07,
379
+ "logits/chosen": 0.09185227006673813,
380
+ "logits/rejected": 0.8966398239135742,
381
+ "logps/chosen": -387.89202880859375,
382
+ "logps/rejected": -462.49932861328125,
383
+ "loss": 0.5404,
384
  "rewards/accuracies": 0.71875,
385
+ "rewards/chosen": -1.2443852424621582,
386
+ "rewards/margins": 0.9278079271316528,
387
+ "rewards/rejected": -2.1721930503845215,
388
  "step": 210
389
  },
390
  {
391
  "epoch": 0.5028571428571429,
392
+ "grad_norm": 18.780630904417688,
393
  "learning_rate": 2.9078548506882117e-07,
394
+ "logits/chosen": 0.5002994537353516,
395
+ "logits/rejected": 1.4443576335906982,
396
+ "logps/chosen": -440.80279541015625,
397
+ "logps/rejected": -487.53485107421875,
398
+ "loss": 0.5609,
399
+ "rewards/accuracies": 0.7437499761581421,
400
+ "rewards/chosen": -1.5573679208755493,
401
+ "rewards/margins": 0.8561462163925171,
402
+ "rewards/rejected": -2.4135143756866455,
403
  "step": 220
404
  },
405
  {
406
  "epoch": 0.5257142857142857,
407
+ "grad_norm": 20.610433717594198,
408
  "learning_rate": 2.709592897595191e-07,
409
+ "logits/chosen": 0.22773201763629913,
410
+ "logits/rejected": 1.2361242771148682,
411
+ "logps/chosen": -401.34228515625,
412
+ "logps/rejected": -446.8021545410156,
413
+ "loss": 0.5442,
414
  "rewards/accuracies": 0.7124999761581421,
415
+ "rewards/chosen": -1.202831506729126,
416
+ "rewards/margins": 0.8723229169845581,
417
+ "rewards/rejected": -2.0751543045043945,
418
  "step": 230
419
  },
420
  {
421
  "epoch": 0.5485714285714286,
422
+ "grad_norm": 27.325375522779876,
423
  "learning_rate": 2.509992316440332e-07,
424
+ "logits/chosen": 0.26873356103897095,
425
+ "logits/rejected": 1.303821325302124,
426
+ "logps/chosen": -431.5526428222656,
427
+ "logps/rejected": -526.184814453125,
428
+ "loss": 0.536,
429
+ "rewards/accuracies": 0.768750011920929,
430
+ "rewards/chosen": -1.328427791595459,
431
+ "rewards/margins": 1.219201922416687,
432
+ "rewards/rejected": -2.5476298332214355,
433
  "step": 240
434
  },
435
  {
436
  "epoch": 0.5714285714285714,
437
+ "grad_norm": 18.92218062691862,
438
  "learning_rate": 2.3103279163519918e-07,
439
+ "logits/chosen": -0.07236287742853165,
440
+ "logits/rejected": 0.5380650758743286,
441
+ "logps/chosen": -407.7901306152344,
442
+ "logps/rejected": -495.40777587890625,
443
+ "loss": 0.5503,
444
+ "rewards/accuracies": 0.762499988079071,
445
+ "rewards/chosen": -1.2716388702392578,
446
+ "rewards/margins": 0.980434238910675,
447
+ "rewards/rejected": -2.252073287963867,
448
  "step": 250
449
  },
450
  {
451
  "epoch": 0.5714285714285714,
452
+ "eval_logits/chosen": -0.37247952818870544,
453
+ "eval_logits/rejected": 0.7719168066978455,
454
+ "eval_logps/chosen": -392.6152038574219,
455
+ "eval_logps/rejected": -450.1522216796875,
456
+ "eval_loss": 0.5393335819244385,
457
+ "eval_rewards/accuracies": 0.7370689511299133,
458
+ "eval_rewards/chosen": -1.1701123714447021,
459
+ "eval_rewards/margins": 1.1406482458114624,
460
+ "eval_rewards/rejected": -2.310760498046875,
461
+ "eval_runtime": 90.9292,
462
+ "eval_samples_per_second": 20.137,
463
+ "eval_steps_per_second": 0.319,
464
  "step": 250
465
  },
466
  {
467
  "epoch": 0.5942857142857143,
468
+ "grad_norm": 25.541848941752068,
469
  "learning_rate": 2.1118749140573358e-07,
470
+ "logits/chosen": 0.0009159505134448409,
471
+ "logits/rejected": 0.6376093626022339,
472
+ "logps/chosen": -426.14141845703125,
473
+ "logps/rejected": -502.9112243652344,
474
+ "loss": 0.5485,
475
+ "rewards/accuracies": 0.699999988079071,
476
+ "rewards/chosen": -1.463547706604004,
477
+ "rewards/margins": 0.8321346044540405,
478
+ "rewards/rejected": -2.295682430267334,
479
  "step": 260
480
  },
481
  {
482
  "epoch": 0.6171428571428571,
483
+ "grad_norm": 23.51335121897504,
484
  "learning_rate": 1.9159007893272703e-07,
485
+ "logits/chosen": 0.321635901927948,
486
+ "logits/rejected": 1.6592861413955688,
487
+ "logps/chosen": -413.24859619140625,
488
+ "logps/rejected": -473.6759338378906,
489
+ "loss": 0.5267,
490
+ "rewards/accuracies": 0.737500011920929,
491
+ "rewards/chosen": -1.4072272777557373,
492
+ "rewards/margins": 1.0244569778442383,
493
+ "rewards/rejected": -2.4316840171813965,
494
  "step": 270
495
  },
496
  {
497
  "epoch": 0.64,
498
+ "grad_norm": 27.142819787480168,
499
  "learning_rate": 1.7236571898357766e-07,
500
+ "logits/chosen": 1.0628600120544434,
501
+ "logits/rejected": 2.0229506492614746,
502
+ "logps/chosen": -440.122314453125,
503
+ "logps/rejected": -543.1414794921875,
504
+ "loss": 0.5316,
505
+ "rewards/accuracies": 0.71875,
506
+ "rewards/chosen": -1.6708418130874634,
507
+ "rewards/margins": 1.129504919052124,
508
+ "rewards/rejected": -2.8003463745117188,
509
  "step": 280
510
  },
511
  {
512
  "epoch": 0.6628571428571428,
513
+ "grad_norm": 24.13150363681131,
514
  "learning_rate": 1.5363719371356882e-07,
515
+ "logits/chosen": 0.698092520236969,
516
+ "logits/rejected": 1.5312575101852417,
517
+ "logps/chosen": -450.4425354003906,
518
+ "logps/rejected": -515.0484008789062,
519
+ "loss": 0.5339,
520
+ "rewards/accuracies": 0.75,
521
+ "rewards/chosen": -1.4953665733337402,
522
+ "rewards/margins": 0.999632716178894,
523
+ "rewards/rejected": -2.4949991703033447,
524
  "step": 290
525
  },
526
  {
527
  "epoch": 0.6857142857142857,
528
+ "grad_norm": 17.486388226084866,
529
  "learning_rate": 1.3552411848071565e-07,
530
+ "logits/chosen": 0.3839910626411438,
531
+ "logits/rejected": 1.8341293334960938,
532
+ "logps/chosen": -441.32183837890625,
533
+ "logps/rejected": -507.97894287109375,
534
+ "loss": 0.5224,
535
+ "rewards/accuracies": 0.762499988079071,
536
+ "rewards/chosen": -1.3941259384155273,
537
+ "rewards/margins": 1.1592432260513306,
538
+ "rewards/rejected": -2.5533692836761475,
539
  "step": 300
540
  },
541
  {
542
  "epoch": 0.6857142857142857,
543
+ "eval_logits/chosen": 0.18918734788894653,
544
+ "eval_logits/rejected": 1.70877206325531,
545
+ "eval_logps/chosen": -397.884033203125,
546
+ "eval_logps/rejected": -470.09490966796875,
547
+ "eval_loss": 0.5312153100967407,
548
+ "eval_rewards/accuracies": 0.7543103694915771,
549
+ "eval_rewards/chosen": -1.2228009700775146,
550
+ "eval_rewards/margins": 1.2873866558074951,
551
+ "eval_rewards/rejected": -2.510187864303589,
552
+ "eval_runtime": 92.3596,
553
+ "eval_samples_per_second": 19.825,
554
+ "eval_steps_per_second": 0.314,
555
  "step": 300
556
  },
557
  {
558
  "epoch": 0.7085714285714285,
559
+ "grad_norm": 20.49651474517604,
560
  "learning_rate": 1.1814217788631473e-07,
561
+ "logits/chosen": 0.41669049859046936,
562
+ "logits/rejected": 1.394052505493164,
563
+ "logps/chosen": -400.6260986328125,
564
+ "logps/rejected": -474.28094482421875,
565
+ "loss": 0.5361,
566
+ "rewards/accuracies": 0.675000011920929,
567
+ "rewards/chosen": -1.4432194232940674,
568
+ "rewards/margins": 0.9276365041732788,
569
+ "rewards/rejected": -2.3708558082580566,
570
  "step": 310
571
  },
572
  {
573
  "epoch": 0.7314285714285714,
574
+ "grad_norm": 18.75776450561332,
575
  "learning_rate": 1.0160238692045331e-07,
576
+ "logits/chosen": 0.7597023844718933,
577
+ "logits/rejected": 1.6351118087768555,
578
+ "logps/chosen": -413.95318603515625,
579
+ "logps/rejected": -488.90460205078125,
580
+ "loss": 0.542,
581
+ "rewards/accuracies": 0.6625000238418579,
582
+ "rewards/chosen": -1.6967086791992188,
583
+ "rewards/margins": 0.782455563545227,
584
+ "rewards/rejected": -2.479163885116577,
585
  "step": 320
586
  },
587
  {
588
  "epoch": 0.7542857142857143,
589
+ "grad_norm": 18.561363930407463,
590
  "learning_rate": 8.601038193139438e-08,
591
+ "logits/chosen": 0.14268045127391815,
592
+ "logits/rejected": 1.3421038389205933,
593
+ "logps/chosen": -447.97137451171875,
594
+ "logps/rejected": -503.50433349609375,
595
+ "loss": 0.5363,
596
+ "rewards/accuracies": 0.78125,
597
+ "rewards/chosen": -1.4875319004058838,
598
+ "rewards/margins": 1.0696327686309814,
599
+ "rewards/rejected": -2.5571646690368652,
600
  "step": 330
601
  },
602
  {
603
  "epoch": 0.7771428571428571,
604
+ "grad_norm": 17.499558797451687,
605
  "learning_rate": 7.146574594727572e-08,
606
+ "logits/chosen": 0.3810690939426422,
607
+ "logits/rejected": 1.2245051860809326,
608
+ "logps/chosen": -414.9021911621094,
609
+ "logps/rejected": -506.65045166015625,
610
+ "loss": 0.5285,
611
+ "rewards/accuracies": 0.7749999761581421,
612
+ "rewards/chosen": -1.489512324333191,
613
+ "rewards/margins": 1.1567548513412476,
614
+ "rewards/rejected": -2.6462674140930176,
615
  "step": 340
616
  },
617
  {
618
  "epoch": 0.8,
619
+ "grad_norm": 17.943689215599328,
620
  "learning_rate": 5.8061372659157306e-08,
621
+ "logits/chosen": 0.24244177341461182,
622
+ "logits/rejected": 1.3491809368133545,
623
+ "logps/chosen": -441.5047912597656,
624
+ "logps/rejected": -494.35626220703125,
625
+ "loss": 0.5396,
626
+ "rewards/accuracies": 0.6875,
627
+ "rewards/chosen": -1.517975091934204,
628
+ "rewards/margins": 0.8826116323471069,
629
+ "rewards/rejected": -2.4005866050720215,
630
  "step": 350
631
  },
632
  {
633
  "epoch": 0.8,
634
+ "eval_logits/chosen": 0.4364562928676605,
635
+ "eval_logits/rejected": 1.9215292930603027,
636
+ "eval_logps/chosen": -420.2202453613281,
637
+ "eval_logps/rejected": -493.9275207519531,
638
+ "eval_loss": 0.5290318131446838,
639
+ "eval_rewards/accuracies": 0.75,
640
+ "eval_rewards/chosen": -1.4461628198623657,
641
+ "eval_rewards/margins": 1.3023512363433838,
642
+ "eval_rewards/rejected": -2.748514175415039,
643
+ "eval_runtime": 91.6979,
644
+ "eval_samples_per_second": 19.968,
645
+ "eval_steps_per_second": 0.316,
646
  "step": 350
647
  },
648
  {
649
  "epoch": 0.8228571428571428,
650
+ "grad_norm": 16.780749890709036,
651
  "learning_rate": 4.5882873127531614e-08,
652
+ "logits/chosen": 0.174576535820961,
653
+ "logits/rejected": 1.4981176853179932,
654
+ "logps/chosen": -435.602783203125,
655
+ "logps/rejected": -510.8885192871094,
656
+ "loss": 0.5205,
657
+ "rewards/accuracies": 0.7875000238418579,
658
+ "rewards/chosen": -1.501977801322937,
659
+ "rewards/margins": 1.0960423946380615,
660
+ "rewards/rejected": -2.598020076751709,
661
  "step": 360
662
  },
663
  {
664
  "epoch": 0.8457142857142858,
665
+ "grad_norm": 19.04569651937684,
666
  "learning_rate": 3.500802900154412e-08,
667
+ "logits/chosen": 0.34421294927597046,
668
+ "logits/rejected": 1.787302017211914,
669
+ "logps/chosen": -412.97747802734375,
670
+ "logps/rejected": -499.79034423828125,
671
+ "loss": 0.528,
672
+ "rewards/accuracies": 0.7562500238418579,
673
+ "rewards/chosen": -1.4191606044769287,
674
+ "rewards/margins": 1.1945868730545044,
675
+ "rewards/rejected": -2.6137473583221436,
676
  "step": 370
677
  },
678
  {
679
  "epoch": 0.8685714285714285,
680
+ "grad_norm": 21.067585045477745,
681
  "learning_rate": 2.550629574310309e-08,
682
+ "logits/chosen": 0.211051344871521,
683
+ "logits/rejected": 1.5275977849960327,
684
+ "logps/chosen": -486.8960876464844,
685
+ "logps/rejected": -515.337646484375,
686
+ "loss": 0.5294,
687
+ "rewards/accuracies": 0.7124999761581421,
688
+ "rewards/chosen": -1.6883971691131592,
689
+ "rewards/margins": 0.8909848928451538,
690
+ "rewards/rejected": -2.5793819427490234,
691
  "step": 380
692
  },
693
  {
694
  "epoch": 0.8914285714285715,
695
+ "grad_norm": 21.227279903684668,
696
  "learning_rate": 1.7438359028687983e-08,
697
+ "logits/chosen": 0.37176352739334106,
698
+ "logits/rejected": 1.208251714706421,
699
+ "logps/chosen": -453.6361389160156,
700
+ "logps/rejected": -538.0291748046875,
701
+ "loss": 0.5333,
702
+ "rewards/accuracies": 0.7250000238418579,
703
+ "rewards/chosen": -1.4226316213607788,
704
+ "rewards/margins": 1.0099334716796875,
705
+ "rewards/rejected": -2.432565212249756,
706
  "step": 390
707
  },
708
  {
709
  "epoch": 0.9142857142857143,
710
+ "grad_norm": 35.72120712786558,
711
  "learning_rate": 1.0855747162029361e-08,
712
+ "logits/chosen": 0.5662034749984741,
713
+ "logits/rejected": 1.0855852365493774,
714
+ "logps/chosen": -437.5174865722656,
715
+ "logps/rejected": -510.6676330566406,
716
+ "loss": 0.55,
717
+ "rewards/accuracies": 0.7437499761581421,
718
+ "rewards/chosen": -1.587842345237732,
719
+ "rewards/margins": 0.8530977368354797,
720
+ "rewards/rejected": -2.4409401416778564,
721
  "step": 400
722
  },
723
  {
724
  "epoch": 0.9142857142857143,
725
+ "eval_logits/chosen": 0.49114343523979187,
726
+ "eval_logits/rejected": 1.9856219291687012,
727
+ "eval_logps/chosen": -417.03155517578125,
728
+ "eval_logps/rejected": -493.2509765625,
729
+ "eval_loss": 0.5286471843719482,
730
+ "eval_rewards/accuracies": 0.7629310488700867,
731
+ "eval_rewards/chosen": -1.414276123046875,
732
+ "eval_rewards/margins": 1.3274718523025513,
733
+ "eval_rewards/rejected": -2.741748094558716,
734
+ "eval_runtime": 91.527,
735
+ "eval_samples_per_second": 20.005,
736
+ "eval_steps_per_second": 0.317,
737
  "step": 400
738
  },
739
  {
740
  "epoch": 0.9371428571428572,
741
+ "grad_norm": 21.313276080994388,
742
  "learning_rate": 5.8005019731033615e-09,
743
+ "logits/chosen": 0.33736371994018555,
744
+ "logits/rejected": 1.3800859451293945,
745
+ "logps/chosen": -453.69744873046875,
746
+ "logps/rejected": -516.6829833984375,
747
+ "loss": 0.5264,
748
+ "rewards/accuracies": 0.6937500238418579,
749
+ "rewards/chosen": -1.6551754474639893,
750
+ "rewards/margins": 0.9143539667129517,
751
+ "rewards/rejected": -2.5695290565490723,
752
  "step": 410
753
  },
754
  {
755
  "epoch": 0.96,
756
+ "grad_norm": 19.39515946553055,
757
  "learning_rate": 2.3049103053431886e-09,
758
+ "logits/chosen": 0.2167482078075409,
759
+ "logits/rejected": 1.6823341846466064,
760
+ "logps/chosen": -409.4588928222656,
761
+ "logps/rejected": -498.9947814941406,
762
+ "loss": 0.5293,
763
+ "rewards/accuracies": 0.7875000238418579,
764
+ "rewards/chosen": -1.267107605934143,
765
+ "rewards/margins": 1.3829718828201294,
766
+ "rewards/rejected": -2.6500792503356934,
767
  "step": 420
768
  },
769
  {
770
  "epoch": 0.9828571428571429,
771
+ "grad_norm": 20.626302042234812,
772
  "learning_rate": 3.9129780600541397e-10,
773
+ "logits/chosen": 0.5624532699584961,
774
+ "logits/rejected": 1.5469181537628174,
775
+ "logps/chosen": -430.54388427734375,
776
+ "logps/rejected": -515.5368041992188,
777
+ "loss": 0.5296,
778
+ "rewards/accuracies": 0.737500011920929,
779
+ "rewards/chosen": -1.483120083808899,
780
+ "rewards/margins": 1.0355522632598877,
781
+ "rewards/rejected": -2.518672466278076,
782
  "step": 430
783
  },
784
  {
785
  "epoch": 0.9988571428571429,
786
  "step": 437,
787
  "total_flos": 0.0,
788
+ "train_loss": 0.5693180419214803,
789
+ "train_runtime": 11386.9149,
790
+ "train_samples_per_second": 4.918,
791
+ "train_steps_per_second": 0.038
792
  }
793
  ],
794
  "logging_steps": 10,