File size: 15,377 Bytes
98af5c4 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 |
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.9978586723768736,
"eval_steps": 500,
"global_step": 233,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.04,
"grad_norm": 5.59375,
"learning_rate": 4.977309719247571e-06,
"log_odds_chosen": 0.06703004986047745,
"log_odds_ratio": -0.7092211842536926,
"logits/chosen": -3.2759811878204346,
"logits/rejected": -3.2908482551574707,
"logps/chosen": -0.7329679727554321,
"logps/rejected": -0.7760596871376038,
"loss": 0.6188,
"nll_loss": 0.47388267517089844,
"rewards/accuracies": 0.4937500059604645,
"rewards/chosen": -0.1465936005115509,
"rewards/margins": 0.008618330582976341,
"rewards/rejected": -0.15521195530891418,
"step": 10
},
{
"epoch": 0.09,
"grad_norm": 5.21875,
"learning_rate": 4.909650756062782e-06,
"log_odds_chosen": 0.054611045867204666,
"log_odds_ratio": -0.7257325053215027,
"logits/chosen": -3.2224247455596924,
"logits/rejected": -3.236621141433716,
"logps/chosen": -0.7471343874931335,
"logps/rejected": -0.7686465978622437,
"loss": 0.5619,
"nll_loss": 0.4292874336242676,
"rewards/accuracies": 0.53125,
"rewards/chosen": -0.1494268774986267,
"rewards/margins": 0.004302443936467171,
"rewards/rejected": -0.15372933447360992,
"step": 20
},
{
"epoch": 0.13,
"grad_norm": 4.15625,
"learning_rate": 4.7982512711416995e-06,
"log_odds_chosen": 0.06513820588588715,
"log_odds_ratio": -0.7152190208435059,
"logits/chosen": -3.165383815765381,
"logits/rejected": -3.1888585090637207,
"logps/chosen": -0.7319885492324829,
"logps/rejected": -0.7425517439842224,
"loss": 0.5721,
"nll_loss": 0.43641433119773865,
"rewards/accuracies": 0.518750011920929,
"rewards/chosen": -0.14639770984649658,
"rewards/margins": 0.0021126591600477695,
"rewards/rejected": -0.14851033687591553,
"step": 30
},
{
"epoch": 0.17,
"grad_norm": 4.21875,
"learning_rate": 4.64513341295515e-06,
"log_odds_chosen": -0.017645269632339478,
"log_odds_ratio": -0.7585808038711548,
"logits/chosen": -3.1905665397644043,
"logits/rejected": -3.186293125152588,
"logps/chosen": -0.7048856019973755,
"logps/rejected": -0.7017339468002319,
"loss": 0.5925,
"nll_loss": 0.4144836366176605,
"rewards/accuracies": 0.4625000059604645,
"rewards/chosen": -0.14097711443901062,
"rewards/margins": -0.0006303312256932259,
"rewards/rejected": -0.14034679532051086,
"step": 40
},
{
"epoch": 0.21,
"grad_norm": 4.40625,
"learning_rate": 4.453076611255507e-06,
"log_odds_chosen": 0.14627456665039062,
"log_odds_ratio": -0.6653262972831726,
"logits/chosen": -3.1202988624572754,
"logits/rejected": -3.1385226249694824,
"logps/chosen": -0.6960284113883972,
"logps/rejected": -0.7755590081214905,
"loss": 0.5531,
"nll_loss": 0.3655123710632324,
"rewards/accuracies": 0.6312500238418579,
"rewards/chosen": -0.139205664396286,
"rewards/margins": 0.015906114131212234,
"rewards/rejected": -0.15511178970336914,
"step": 50
},
{
"epoch": 0.26,
"grad_norm": 4.6875,
"learning_rate": 4.22556712424355e-06,
"log_odds_chosen": 0.12080030143260956,
"log_odds_ratio": -0.6814600229263306,
"logits/chosen": -3.0596089363098145,
"logits/rejected": -3.056286573410034,
"logps/chosen": -0.6838747262954712,
"logps/rejected": -0.7580210566520691,
"loss": 0.5745,
"nll_loss": 0.4387393891811371,
"rewards/accuracies": 0.5625,
"rewards/chosen": -0.1367749273777008,
"rewards/margins": 0.014829280786216259,
"rewards/rejected": -0.15160420536994934,
"step": 60
},
{
"epoch": 0.3,
"grad_norm": 4.34375,
"learning_rate": 3.9667347552265945e-06,
"log_odds_chosen": 0.09098054468631744,
"log_odds_ratio": -0.7007282376289368,
"logits/chosen": -3.1298584938049316,
"logits/rejected": -3.135943651199341,
"logps/chosen": -0.7081547975540161,
"logps/rejected": -0.7608820796012878,
"loss": 0.6087,
"nll_loss": 0.4499574303627014,
"rewards/accuracies": 0.5874999761581421,
"rewards/chosen": -0.14163094758987427,
"rewards/margins": 0.0105454595759511,
"rewards/rejected": -0.1521764099597931,
"step": 70
},
{
"epoch": 0.34,
"grad_norm": 4.40625,
"learning_rate": 3.68127788750129e-06,
"log_odds_chosen": 0.18564960360527039,
"log_odds_ratio": -0.6482046842575073,
"logits/chosen": -3.142078399658203,
"logits/rejected": -3.145967960357666,
"logps/chosen": -0.6542805433273315,
"logps/rejected": -0.7562066912651062,
"loss": 0.5855,
"nll_loss": 0.40380844473838806,
"rewards/accuracies": 0.65625,
"rewards/chosen": -0.13085612654685974,
"rewards/margins": 0.020385239273309708,
"rewards/rejected": -0.15124136209487915,
"step": 80
},
{
"epoch": 0.39,
"grad_norm": 3.921875,
"learning_rate": 3.3743781982447533e-06,
"log_odds_chosen": 0.13925865292549133,
"log_odds_ratio": -0.6794918775558472,
"logits/chosen": -3.1532516479492188,
"logits/rejected": -3.1458470821380615,
"logps/chosen": -0.6936465501785278,
"logps/rejected": -0.7801955938339233,
"loss": 0.567,
"nll_loss": 0.4436867833137512,
"rewards/accuracies": 0.5562499761581421,
"rewards/chosen": -0.1387293040752411,
"rewards/margins": 0.01730981469154358,
"rewards/rejected": -0.15603913366794586,
"step": 90
},
{
"epoch": 0.43,
"grad_norm": 3.75,
"learning_rate": 3.0516065995466336e-06,
"log_odds_chosen": 0.16556194424629211,
"log_odds_ratio": -0.6641014218330383,
"logits/chosen": -3.1563491821289062,
"logits/rejected": -3.1614346504211426,
"logps/chosen": -0.6500056982040405,
"logps/rejected": -0.7318493127822876,
"loss": 0.5861,
"nll_loss": 0.45738714933395386,
"rewards/accuracies": 0.5687500238418579,
"rewards/chosen": -0.13000114262104034,
"rewards/margins": 0.016368722543120384,
"rewards/rejected": -0.14636987447738647,
"step": 100
},
{
"epoch": 0.47,
"grad_norm": 4.0625,
"learning_rate": 2.7188221139616303e-06,
"log_odds_chosen": 0.10007637739181519,
"log_odds_ratio": -0.6944689750671387,
"logits/chosen": -3.105386972427368,
"logits/rejected": -3.1417195796966553,
"logps/chosen": -0.7295275926589966,
"logps/rejected": -0.7832143902778625,
"loss": 0.5604,
"nll_loss": 0.4221896231174469,
"rewards/accuracies": 0.5625,
"rewards/chosen": -0.1459055244922638,
"rewards/margins": 0.010737359523773193,
"rewards/rejected": -0.156642884016037,
"step": 110
},
{
"epoch": 0.51,
"grad_norm": 3.609375,
"learning_rate": 2.3820655202161237e-06,
"log_odds_chosen": 0.18686607480049133,
"log_odds_ratio": -0.6535667777061462,
"logits/chosen": -3.1217126846313477,
"logits/rejected": -3.1293649673461914,
"logps/chosen": -0.632846474647522,
"logps/rejected": -0.726894736289978,
"loss": 0.5714,
"nll_loss": 0.4484385848045349,
"rewards/accuracies": 0.612500011920929,
"rewards/chosen": -0.12656930088996887,
"rewards/margins": 0.018809664994478226,
"rewards/rejected": -0.1453789472579956,
"step": 120
},
{
"epoch": 0.56,
"grad_norm": 4.0,
"learning_rate": 2.0474496996359676e-06,
"log_odds_chosen": 0.12438281625509262,
"log_odds_ratio": -0.6807030439376831,
"logits/chosen": -3.114492177963257,
"logits/rejected": -3.1283891201019287,
"logps/chosen": -0.6877269148826599,
"logps/rejected": -0.7388188242912292,
"loss": 0.5653,
"nll_loss": 0.407818466424942,
"rewards/accuracies": 0.5687500238418579,
"rewards/chosen": -0.1375453919172287,
"rewards/margins": 0.010218393988907337,
"rewards/rejected": -0.14776377379894257,
"step": 130
},
{
"epoch": 0.6,
"grad_norm": 4.375,
"learning_rate": 1.7210486737516947e-06,
"log_odds_chosen": 0.13585877418518066,
"log_odds_ratio": -0.6894018650054932,
"logits/chosen": -3.094355583190918,
"logits/rejected": -3.100367784500122,
"logps/chosen": -0.6505359411239624,
"logps/rejected": -0.7270746231079102,
"loss": 0.5667,
"nll_loss": 0.38747844099998474,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": -0.13010717928409576,
"rewards/margins": 0.015307746827602386,
"rewards/rejected": -0.14541491866111755,
"step": 140
},
{
"epoch": 0.64,
"grad_norm": 4.15625,
"learning_rate": 1.4087873472954638e-06,
"log_odds_chosen": 0.14618463814258575,
"log_odds_ratio": -0.6783791780471802,
"logits/chosen": -3.124422788619995,
"logits/rejected": -3.1180996894836426,
"logps/chosen": -0.7014532089233398,
"logps/rejected": -0.7804998159408569,
"loss": 0.5371,
"nll_loss": 0.3992606997489929,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.14029064774513245,
"rewards/margins": 0.015809325501322746,
"rewards/rejected": -0.15609999001026154,
"step": 150
},
{
"epoch": 0.69,
"grad_norm": 3.796875,
"learning_rate": 1.116333957999608e-06,
"log_odds_chosen": 0.06774308532476425,
"log_odds_ratio": -0.7074956893920898,
"logits/chosen": -3.1124911308288574,
"logits/rejected": -3.1118929386138916,
"logps/chosen": -0.7164414525032043,
"logps/rejected": -0.7556790113449097,
"loss": 0.5714,
"nll_loss": 0.40609636902809143,
"rewards/accuracies": 0.5249999761581421,
"rewards/chosen": -0.14328829944133759,
"rewards/margins": 0.007847519591450691,
"rewards/rejected": -0.15113583207130432,
"step": 160
},
{
"epoch": 0.73,
"grad_norm": 3.890625,
"learning_rate": 8.48997185472226e-07,
"log_odds_chosen": 0.14430537819862366,
"log_odds_ratio": -0.6831859946250916,
"logits/chosen": -3.1008052825927734,
"logits/rejected": -3.118614912033081,
"logps/chosen": -0.6754971742630005,
"logps/rejected": -0.7525571584701538,
"loss": 0.5597,
"nll_loss": 0.41810712218284607,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -0.13509942591190338,
"rewards/margins": 0.015411997213959694,
"rewards/rejected": -0.15051142871379852,
"step": 170
},
{
"epoch": 0.77,
"grad_norm": 4.65625,
"learning_rate": 6.11629786852592e-07,
"log_odds_chosen": 0.20049241185188293,
"log_odds_ratio": -0.6564125418663025,
"logits/chosen": -3.0964746475219727,
"logits/rejected": -3.097423553466797,
"logps/chosen": -0.6751464009284973,
"logps/rejected": -0.7748547196388245,
"loss": 0.5689,
"nll_loss": 0.4084566533565521,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.13502928614616394,
"rewards/margins": 0.019941668957471848,
"rewards/rejected": -0.1549709588289261,
"step": 180
},
{
"epoch": 0.81,
"grad_norm": 3.8125,
"learning_rate": 4.0854050847362966e-07,
"log_odds_chosen": 0.1618964970111847,
"log_odds_ratio": -0.6639573574066162,
"logits/chosen": -3.1198277473449707,
"logits/rejected": -3.1235454082489014,
"logps/chosen": -0.6751562356948853,
"logps/rejected": -0.7577996253967285,
"loss": 0.5453,
"nll_loss": 0.4503118097782135,
"rewards/accuracies": 0.6312500238418579,
"rewards/chosen": -0.13503125309944153,
"rewards/margins": 0.016528675332665443,
"rewards/rejected": -0.15155991911888123,
"step": 190
},
{
"epoch": 0.86,
"grad_norm": 3.78125,
"learning_rate": 2.4341587253072035e-07,
"log_odds_chosen": 0.21062397956848145,
"log_odds_ratio": -0.6452255249023438,
"logits/chosen": -3.0891025066375732,
"logits/rejected": -3.077317714691162,
"logps/chosen": -0.6431701183319092,
"logps/rejected": -0.7490943670272827,
"loss": 0.5505,
"nll_loss": 0.3902386724948883,
"rewards/accuracies": 0.643750011920929,
"rewards/chosen": -0.1286340355873108,
"rewards/margins": 0.021184856072068214,
"rewards/rejected": -0.14981889724731445,
"step": 200
},
{
"epoch": 0.9,
"grad_norm": 4.28125,
"learning_rate": 1.1925325850281416e-07,
"log_odds_chosen": 0.08836773782968521,
"log_odds_ratio": -0.7011784315109253,
"logits/chosen": -3.1118626594543457,
"logits/rejected": -3.1287245750427246,
"logps/chosen": -0.6649689674377441,
"logps/rejected": -0.6961485743522644,
"loss": 0.534,
"nll_loss": 0.34850066900253296,
"rewards/accuracies": 0.5625,
"rewards/chosen": -0.13299378752708435,
"rewards/margins": 0.00623591523617506,
"rewards/rejected": -0.13922971487045288,
"step": 210
},
{
"epoch": 0.94,
"grad_norm": 3.984375,
"learning_rate": 3.830649404690939e-08,
"log_odds_chosen": 0.10791333764791489,
"log_odds_ratio": -0.691511332988739,
"logits/chosen": -3.0939254760742188,
"logits/rejected": -3.1030945777893066,
"logps/chosen": -0.6941600441932678,
"logps/rejected": -0.7733976244926453,
"loss": 0.5775,
"nll_loss": 0.40806055068969727,
"rewards/accuracies": 0.5249999761581421,
"rewards/chosen": -0.1388320028781891,
"rewards/margins": 0.015847522765398026,
"rewards/rejected": -0.15467952191829681,
"step": 220
},
{
"epoch": 0.99,
"grad_norm": 4.15625,
"learning_rate": 2.044943012210754e-09,
"log_odds_chosen": 0.10530634969472885,
"log_odds_ratio": -0.69704669713974,
"logits/chosen": -3.1206929683685303,
"logits/rejected": -3.117326259613037,
"logps/chosen": -0.6853540539741516,
"logps/rejected": -0.7511805295944214,
"loss": 0.567,
"nll_loss": 0.40571802854537964,
"rewards/accuracies": 0.581250011920929,
"rewards/chosen": -0.13707080483436584,
"rewards/margins": 0.013165315613150597,
"rewards/rejected": -0.150236114859581,
"step": 230
},
{
"epoch": 1.0,
"step": 233,
"total_flos": 0.0,
"train_loss": 0.5704265398017326,
"train_runtime": 3228.2239,
"train_samples_per_second": 4.624,
"train_steps_per_second": 0.072
}
],
"logging_steps": 10,
"max_steps": 233,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 500,
"total_flos": 0.0,
"train_batch_size": 8,
"trial_name": null,
"trial_params": null
}
|