File size: 10,761 Bytes
a828ec4 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 |
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.9984,
"eval_steps": 100,
"global_step": 195,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.01,
"learning_rate": 2.5000000000000004e-07,
"logits/chosen": -2.535799264907837,
"logits/rejected": -2.5660998821258545,
"logps/chosen": -270.7046813964844,
"logps/rejected": -307.25311279296875,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1
},
{
"epoch": 0.05,
"learning_rate": 2.5e-06,
"logits/chosen": -2.63254976272583,
"logits/rejected": -2.5803234577178955,
"logps/chosen": -320.9474792480469,
"logps/rejected": -316.3360900878906,
"loss": 0.6926,
"rewards/accuracies": 0.4618055522441864,
"rewards/chosen": -0.0115978317335248,
"rewards/margins": 0.0027361076790839434,
"rewards/rejected": -0.014333941973745823,
"step": 10
},
{
"epoch": 0.1,
"learning_rate": 5e-06,
"logits/chosen": -2.5984463691711426,
"logits/rejected": -2.569882869720459,
"logps/chosen": -307.0980529785156,
"logps/rejected": -311.14959716796875,
"loss": 0.6898,
"rewards/accuracies": 0.546875,
"rewards/chosen": -0.03875315934419632,
"rewards/margins": 0.007569065783172846,
"rewards/rejected": -0.04632222652435303,
"step": 20
},
{
"epoch": 0.15,
"learning_rate": 4.959823971496575e-06,
"logits/chosen": -2.5634453296661377,
"logits/rejected": -2.4752097129821777,
"logps/chosen": -322.8082580566406,
"logps/rejected": -310.7356262207031,
"loss": 0.6782,
"rewards/accuracies": 0.6343749761581421,
"rewards/chosen": -0.026555459946393967,
"rewards/margins": 0.03691429644823074,
"rewards/rejected": -0.06346975266933441,
"step": 30
},
{
"epoch": 0.2,
"learning_rate": 4.8405871765993435e-06,
"logits/chosen": -2.55033278465271,
"logits/rejected": -2.459357738494873,
"logps/chosen": -333.2048034667969,
"logps/rejected": -334.33538818359375,
"loss": 0.6719,
"rewards/accuracies": 0.581250011920929,
"rewards/chosen": -0.10182944685220718,
"rewards/margins": 0.038650404661893845,
"rewards/rejected": -0.14047986268997192,
"step": 40
},
{
"epoch": 0.26,
"learning_rate": 4.646121984004666e-06,
"logits/chosen": -2.530797243118286,
"logits/rejected": -2.496924877166748,
"logps/chosen": -313.50885009765625,
"logps/rejected": -333.51397705078125,
"loss": 0.6599,
"rewards/accuracies": 0.671875,
"rewards/chosen": -0.13014966249465942,
"rewards/margins": 0.07707841694355011,
"rewards/rejected": -0.20722806453704834,
"step": 50
},
{
"epoch": 0.31,
"learning_rate": 4.382678665009028e-06,
"logits/chosen": -2.5104424953460693,
"logits/rejected": -2.423055410385132,
"logps/chosen": -337.88885498046875,
"logps/rejected": -342.5820007324219,
"loss": 0.6566,
"rewards/accuracies": 0.609375,
"rewards/chosen": -0.2016713172197342,
"rewards/margins": 0.08087404072284698,
"rewards/rejected": -0.2825453579425812,
"step": 60
},
{
"epoch": 0.36,
"learning_rate": 4.058724504646834e-06,
"logits/chosen": -2.437530040740967,
"logits/rejected": -2.411935329437256,
"logps/chosen": -299.39825439453125,
"logps/rejected": -334.86737060546875,
"loss": 0.6486,
"rewards/accuracies": 0.6312500238418579,
"rewards/chosen": -0.1794266402721405,
"rewards/margins": 0.12244921922683716,
"rewards/rejected": -0.3018758296966553,
"step": 70
},
{
"epoch": 0.41,
"learning_rate": 3.684671656182497e-06,
"logits/chosen": -2.494436740875244,
"logits/rejected": -2.417968273162842,
"logps/chosen": -309.20556640625,
"logps/rejected": -324.0342102050781,
"loss": 0.6481,
"rewards/accuracies": 0.643750011920929,
"rewards/chosen": -0.22928757965564728,
"rewards/margins": 0.13500627875328064,
"rewards/rejected": -0.3642938435077667,
"step": 80
},
{
"epoch": 0.46,
"learning_rate": 3.272542485937369e-06,
"logits/chosen": -2.3841819763183594,
"logits/rejected": -2.356064558029175,
"logps/chosen": -289.4866638183594,
"logps/rejected": -311.481201171875,
"loss": 0.6461,
"rewards/accuracies": 0.590624988079071,
"rewards/chosen": -0.2119632214307785,
"rewards/margins": 0.12710149586200714,
"rewards/rejected": -0.33906468749046326,
"step": 90
},
{
"epoch": 0.51,
"learning_rate": 2.835583164544139e-06,
"logits/chosen": -2.2960455417633057,
"logits/rejected": -2.2413225173950195,
"logps/chosen": -337.8830261230469,
"logps/rejected": -365.25238037109375,
"loss": 0.6429,
"rewards/accuracies": 0.6156250238418579,
"rewards/chosen": -0.3260457217693329,
"rewards/margins": 0.16088134050369263,
"rewards/rejected": -0.4869270920753479,
"step": 100
},
{
"epoch": 0.51,
"eval_logits/chosen": -2.2674760818481445,
"eval_logits/rejected": -2.134783983230591,
"eval_logps/chosen": -357.25299072265625,
"eval_logps/rejected": -365.74114990234375,
"eval_loss": 0.6368065476417542,
"eval_rewards/accuracies": 0.6380000114440918,
"eval_rewards/chosen": -0.5885025858879089,
"eval_rewards/margins": 0.17055951058864594,
"eval_rewards/rejected": -0.7590621113777161,
"eval_runtime": 383.7605,
"eval_samples_per_second": 5.212,
"eval_steps_per_second": 0.651,
"step": 100
},
{
"epoch": 0.56,
"learning_rate": 2.3878379241237136e-06,
"logits/chosen": -2.160651445388794,
"logits/rejected": -2.0833230018615723,
"logps/chosen": -358.48834228515625,
"logps/rejected": -367.35345458984375,
"loss": 0.6391,
"rewards/accuracies": 0.6187499761581421,
"rewards/chosen": -0.5002601146697998,
"rewards/margins": 0.17509731650352478,
"rewards/rejected": -0.675357460975647,
"step": 110
},
{
"epoch": 0.61,
"learning_rate": 1.9436976651092143e-06,
"logits/chosen": -2.167994499206543,
"logits/rejected": -2.064636707305908,
"logps/chosen": -345.025390625,
"logps/rejected": -360.7124938964844,
"loss": 0.6299,
"rewards/accuracies": 0.703125,
"rewards/chosen": -0.29811879992485046,
"rewards/margins": 0.1997375190258026,
"rewards/rejected": -0.4978563189506531,
"step": 120
},
{
"epoch": 0.67,
"learning_rate": 1.5174374208651913e-06,
"logits/chosen": -1.9607616662979126,
"logits/rejected": -1.7911456823349,
"logps/chosen": -353.13446044921875,
"logps/rejected": -370.7218017578125,
"loss": 0.609,
"rewards/accuracies": 0.6937500238418579,
"rewards/chosen": -0.41445955634117126,
"rewards/margins": 0.26371151208877563,
"rewards/rejected": -0.6781710386276245,
"step": 130
},
{
"epoch": 0.72,
"learning_rate": 1.122757546369744e-06,
"logits/chosen": -1.9263637065887451,
"logits/rejected": -1.6748569011688232,
"logps/chosen": -393.5077209472656,
"logps/rejected": -390.3746643066406,
"loss": 0.6161,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.4902144968509674,
"rewards/margins": 0.21248309314250946,
"rewards/rejected": -0.7026976346969604,
"step": 140
},
{
"epoch": 0.77,
"learning_rate": 7.723433775328385e-07,
"logits/chosen": -1.7820335626602173,
"logits/rejected": -1.6083834171295166,
"logps/chosen": -354.44598388671875,
"logps/rejected": -386.1634216308594,
"loss": 0.6128,
"rewards/accuracies": 0.65625,
"rewards/chosen": -0.37745195627212524,
"rewards/margins": 0.2488851547241211,
"rewards/rejected": -0.6263370513916016,
"step": 150
},
{
"epoch": 0.82,
"learning_rate": 4.774575140626317e-07,
"logits/chosen": -1.6063228845596313,
"logits/rejected": -1.524730920791626,
"logps/chosen": -350.2926330566406,
"logps/rejected": -387.1871643066406,
"loss": 0.6109,
"rewards/accuracies": 0.6937500238418579,
"rewards/chosen": -0.4679895043373108,
"rewards/margins": 0.23797926306724548,
"rewards/rejected": -0.7059687376022339,
"step": 160
},
{
"epoch": 0.87,
"learning_rate": 2.4757783024395244e-07,
"logits/chosen": -1.4787060022354126,
"logits/rejected": -1.4544020891189575,
"logps/chosen": -342.1855773925781,
"logps/rejected": -382.0643615722656,
"loss": 0.62,
"rewards/accuracies": 0.671875,
"rewards/chosen": -0.46587371826171875,
"rewards/margins": 0.2605627179145813,
"rewards/rejected": -0.7264364361763,
"step": 170
},
{
"epoch": 0.92,
"learning_rate": 9.00928482603669e-08,
"logits/chosen": -1.6203199625015259,
"logits/rejected": -1.3692257404327393,
"logps/chosen": -360.90191650390625,
"logps/rejected": -369.82427978515625,
"loss": 0.6112,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.4321191906929016,
"rewards/margins": 0.2534908354282379,
"rewards/rejected": -0.6856100559234619,
"step": 180
},
{
"epoch": 0.97,
"learning_rate": 1.006426501190233e-08,
"logits/chosen": -1.5687239170074463,
"logits/rejected": -1.3249285221099854,
"logps/chosen": -354.10711669921875,
"logps/rejected": -371.7186279296875,
"loss": 0.6151,
"rewards/accuracies": 0.653124988079071,
"rewards/chosen": -0.44191044569015503,
"rewards/margins": 0.21911220252513885,
"rewards/rejected": -0.6610226035118103,
"step": 190
},
{
"epoch": 1.0,
"step": 195,
"total_flos": 0.0,
"train_loss": 0.6412642754041231,
"train_runtime": 8982.6903,
"train_samples_per_second": 2.783,
"train_steps_per_second": 0.022
}
],
"logging_steps": 10,
"max_steps": 195,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 100,
"total_flos": 0.0,
"train_batch_size": 2,
"trial_name": null,
"trial_params": null
}
|