|
{ |
|
"best_metric": null, |
|
"best_model_checkpoint": null, |
|
"epoch": 0.05592059275828324, |
|
"eval_steps": 50, |
|
"global_step": 200, |
|
"is_hyper_param_search": false, |
|
"is_local_process_zero": true, |
|
"is_world_process_zero": true, |
|
"log_history": [ |
|
{ |
|
"epoch": 0.0002796029637914162, |
|
"grad_norm": 25.76739404251824, |
|
"learning_rate": 5e-08, |
|
"logits/chosen": -0.9193593263626099, |
|
"logits/rejected": -1.1508712768554688, |
|
"logps/chosen": -0.9903213977813721, |
|
"logps/rejected": -0.9371722936630249, |
|
"loss": 1.683, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -2.4758033752441406, |
|
"rewards/margins": -0.13287273049354553, |
|
"rewards/rejected": -2.342930793762207, |
|
"step": 1 |
|
}, |
|
{ |
|
"epoch": 0.0005592059275828324, |
|
"grad_norm": 45.297284297927725, |
|
"learning_rate": 1e-07, |
|
"logits/chosen": -0.8994183540344238, |
|
"logits/rejected": -1.2252119779586792, |
|
"logps/chosen": -1.4980652332305908, |
|
"logps/rejected": -1.216281533241272, |
|
"loss": 1.7319, |
|
"rewards/accuracies": 0.25, |
|
"rewards/chosen": -3.7451629638671875, |
|
"rewards/margins": -0.7044591903686523, |
|
"rewards/rejected": -3.040703773498535, |
|
"step": 2 |
|
}, |
|
{ |
|
"epoch": 0.0008388088913742485, |
|
"grad_norm": 31.873718310326044, |
|
"learning_rate": 1.5e-07, |
|
"logits/chosen": -0.5185738205909729, |
|
"logits/rejected": -1.3382601737976074, |
|
"logps/chosen": -1.3677482604980469, |
|
"logps/rejected": -1.4751423597335815, |
|
"loss": 1.4124, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -3.419370412826538, |
|
"rewards/margins": 0.26848527789115906, |
|
"rewards/rejected": -3.6878557205200195, |
|
"step": 3 |
|
}, |
|
{ |
|
"epoch": 0.0011184118551656648, |
|
"grad_norm": 48.966957322673935, |
|
"learning_rate": 2e-07, |
|
"logits/chosen": -0.6579636931419373, |
|
"logits/rejected": -1.3838244676589966, |
|
"logps/chosen": -0.9612038731575012, |
|
"logps/rejected": -1.2441552877426147, |
|
"loss": 1.4082, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -2.4030096530914307, |
|
"rewards/margins": 0.7073784470558167, |
|
"rewards/rejected": -3.1103880405426025, |
|
"step": 4 |
|
}, |
|
{ |
|
"epoch": 0.001398014818957081, |
|
"grad_norm": 46.91034256260659, |
|
"learning_rate": 2.5e-07, |
|
"logits/chosen": -1.2698190212249756, |
|
"logits/rejected": -1.5386033058166504, |
|
"logps/chosen": -1.6302634477615356, |
|
"logps/rejected": -1.7429945468902588, |
|
"loss": 1.6189, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -4.075658798217773, |
|
"rewards/margins": 0.28182780742645264, |
|
"rewards/rejected": -4.357486248016357, |
|
"step": 5 |
|
}, |
|
{ |
|
"epoch": 0.001677617782748497, |
|
"grad_norm": 73.65526070267136, |
|
"learning_rate": 3e-07, |
|
"logits/chosen": -0.8474517464637756, |
|
"logits/rejected": -0.7566354274749756, |
|
"logps/chosen": -1.1040312051773071, |
|
"logps/rejected": -1.1213470697402954, |
|
"loss": 1.3294, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -2.760077953338623, |
|
"rewards/margins": 0.04328957200050354, |
|
"rewards/rejected": -2.8033676147460938, |
|
"step": 6 |
|
}, |
|
{ |
|
"epoch": 0.001957220746539913, |
|
"grad_norm": 24.84640983932336, |
|
"learning_rate": 3.5e-07, |
|
"logits/chosen": -0.8542510271072388, |
|
"logits/rejected": -0.34997448325157166, |
|
"logps/chosen": -0.7964720726013184, |
|
"logps/rejected": -0.8464012145996094, |
|
"loss": 1.583, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -1.991180181503296, |
|
"rewards/margins": 0.12482279539108276, |
|
"rewards/rejected": -2.1160030364990234, |
|
"step": 7 |
|
}, |
|
{ |
|
"epoch": 0.0022368237103313295, |
|
"grad_norm": 22.841316781980318, |
|
"learning_rate": 4e-07, |
|
"logits/chosen": -0.7528856992721558, |
|
"logits/rejected": -1.2337926626205444, |
|
"logps/chosen": -1.1317272186279297, |
|
"logps/rejected": -1.4922257661819458, |
|
"loss": 1.3675, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.829317808151245, |
|
"rewards/margins": 0.9012465476989746, |
|
"rewards/rejected": -3.7305643558502197, |
|
"step": 8 |
|
}, |
|
{ |
|
"epoch": 0.002516426674122746, |
|
"grad_norm": 87.07632980072657, |
|
"learning_rate": 4.5e-07, |
|
"logits/chosen": -0.9475423693656921, |
|
"logits/rejected": -1.4221036434173584, |
|
"logps/chosen": -1.2074404954910278, |
|
"logps/rejected": -0.9064420461654663, |
|
"loss": 1.7839, |
|
"rewards/accuracies": 0.25, |
|
"rewards/chosen": -3.018601417541504, |
|
"rewards/margins": -0.7524962425231934, |
|
"rewards/rejected": -2.2661051750183105, |
|
"step": 9 |
|
}, |
|
{ |
|
"epoch": 0.002796029637914162, |
|
"grad_norm": 27.009928075261808, |
|
"learning_rate": 5e-07, |
|
"logits/chosen": -0.6325011253356934, |
|
"logits/rejected": -1.1582834720611572, |
|
"logps/chosen": -0.7608655691146851, |
|
"logps/rejected": -0.8033685088157654, |
|
"loss": 1.3733, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -1.9021639823913574, |
|
"rewards/margins": 0.10625733435153961, |
|
"rewards/rejected": -2.0084214210510254, |
|
"step": 10 |
|
}, |
|
{ |
|
"epoch": 0.003075632601705578, |
|
"grad_norm": 30.088075877499065, |
|
"learning_rate": 5.5e-07, |
|
"logits/chosen": -1.018723964691162, |
|
"logits/rejected": -1.0942765474319458, |
|
"logps/chosen": -0.9146077632904053, |
|
"logps/rejected": -0.8910419344902039, |
|
"loss": 1.2264, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -2.2865195274353027, |
|
"rewards/margins": -0.058914512395858765, |
|
"rewards/rejected": -2.227604866027832, |
|
"step": 11 |
|
}, |
|
{ |
|
"epoch": 0.003355235565496994, |
|
"grad_norm": 69.74430608906756, |
|
"learning_rate": 6e-07, |
|
"logits/chosen": -1.0309629440307617, |
|
"logits/rejected": -1.2155206203460693, |
|
"logps/chosen": -1.868607521057129, |
|
"logps/rejected": -2.074054002761841, |
|
"loss": 1.3981, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -4.6715192794799805, |
|
"rewards/margins": 0.5136159062385559, |
|
"rewards/rejected": -5.1851348876953125, |
|
"step": 12 |
|
}, |
|
{ |
|
"epoch": 0.0036348385292884105, |
|
"grad_norm": 33.254043165620104, |
|
"learning_rate": 6.5e-07, |
|
"logits/chosen": -1.0877211093902588, |
|
"logits/rejected": -1.1691397428512573, |
|
"logps/chosen": -1.4225409030914307, |
|
"logps/rejected": -1.5891215801239014, |
|
"loss": 1.9712, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -3.556352138519287, |
|
"rewards/margins": 0.4164515733718872, |
|
"rewards/rejected": -3.9728035926818848, |
|
"step": 13 |
|
}, |
|
{ |
|
"epoch": 0.003914441493079826, |
|
"grad_norm": 21.755018885778018, |
|
"learning_rate": 7e-07, |
|
"logits/chosen": -0.5653956532478333, |
|
"logits/rejected": -1.170688271522522, |
|
"logps/chosen": -1.041928768157959, |
|
"logps/rejected": -1.546263337135315, |
|
"loss": 1.4989, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.6048219203948975, |
|
"rewards/margins": 1.2608366012573242, |
|
"rewards/rejected": -3.8656585216522217, |
|
"step": 14 |
|
}, |
|
{ |
|
"epoch": 0.004194044456871243, |
|
"grad_norm": 69.81299826798852, |
|
"learning_rate": 7.5e-07, |
|
"logits/chosen": -0.4804845452308655, |
|
"logits/rejected": -0.8382622003555298, |
|
"logps/chosen": -1.0265494585037231, |
|
"logps/rejected": -1.0567755699157715, |
|
"loss": 1.4168, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -2.566373586654663, |
|
"rewards/margins": 0.07556554675102234, |
|
"rewards/rejected": -2.641939163208008, |
|
"step": 15 |
|
}, |
|
{ |
|
"epoch": 0.004473647420662659, |
|
"grad_norm": 24.193090245288765, |
|
"learning_rate": 8e-07, |
|
"logits/chosen": -0.6370764970779419, |
|
"logits/rejected": -1.0934374332427979, |
|
"logps/chosen": -0.71354079246521, |
|
"logps/rejected": -1.11177396774292, |
|
"loss": 0.9962, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -1.783851981163025, |
|
"rewards/margins": 0.9955829977989197, |
|
"rewards/rejected": -2.7794349193573, |
|
"step": 16 |
|
}, |
|
{ |
|
"epoch": 0.004753250384454075, |
|
"grad_norm": 25.26345718560276, |
|
"learning_rate": 8.499999999999999e-07, |
|
"logits/chosen": -0.9751855134963989, |
|
"logits/rejected": -1.1418442726135254, |
|
"logps/chosen": -1.2313238382339478, |
|
"logps/rejected": -1.6264705657958984, |
|
"loss": 1.4442, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -3.0783097743988037, |
|
"rewards/margins": 0.9878668189048767, |
|
"rewards/rejected": -4.066176414489746, |
|
"step": 17 |
|
}, |
|
{ |
|
"epoch": 0.005032853348245492, |
|
"grad_norm": 15.702824197515223, |
|
"learning_rate": 9e-07, |
|
"logits/chosen": -0.9074665904045105, |
|
"logits/rejected": -1.053283929824829, |
|
"logps/chosen": -0.8471686244010925, |
|
"logps/rejected": -1.086425542831421, |
|
"loss": 1.4019, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.1179215908050537, |
|
"rewards/margins": 0.5981425046920776, |
|
"rewards/rejected": -2.716063976287842, |
|
"step": 18 |
|
}, |
|
{ |
|
"epoch": 0.005312456312036908, |
|
"grad_norm": 26.78497329037122, |
|
"learning_rate": 9.499999999999999e-07, |
|
"logits/chosen": -0.8664759993553162, |
|
"logits/rejected": -0.9702961444854736, |
|
"logps/chosen": -0.9274067878723145, |
|
"logps/rejected": -1.054181694984436, |
|
"loss": 1.1298, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.318516969680786, |
|
"rewards/margins": 0.31693723797798157, |
|
"rewards/rejected": -2.6354541778564453, |
|
"step": 19 |
|
}, |
|
{ |
|
"epoch": 0.005592059275828324, |
|
"grad_norm": 43.484383450898136, |
|
"learning_rate": 1e-06, |
|
"logits/chosen": -0.9217432141304016, |
|
"logits/rejected": -1.0792617797851562, |
|
"logps/chosen": -1.1316174268722534, |
|
"logps/rejected": -1.3112761974334717, |
|
"loss": 1.6817, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -2.8290436267852783, |
|
"rewards/margins": 0.44914710521698, |
|
"rewards/rejected": -3.2781906127929688, |
|
"step": 20 |
|
}, |
|
{ |
|
"epoch": 0.0058716622396197396, |
|
"grad_norm": 23.72254452641318, |
|
"learning_rate": 9.999238475781957e-07, |
|
"logits/chosen": -0.892957866191864, |
|
"logits/rejected": -0.896960973739624, |
|
"logps/chosen": -1.0195448398590088, |
|
"logps/rejected": -1.172441005706787, |
|
"loss": 1.4088, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.5488619804382324, |
|
"rewards/margins": 0.3822404146194458, |
|
"rewards/rejected": -2.9311022758483887, |
|
"step": 21 |
|
}, |
|
{ |
|
"epoch": 0.006151265203411156, |
|
"grad_norm": 19.983043621451948, |
|
"learning_rate": 9.996954135095478e-07, |
|
"logits/chosen": -0.815680742263794, |
|
"logits/rejected": -1.0532113313674927, |
|
"logps/chosen": -1.1461201906204224, |
|
"logps/rejected": -1.190143346786499, |
|
"loss": 1.4757, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.865300416946411, |
|
"rewards/margins": 0.11005795001983643, |
|
"rewards/rejected": -2.975358486175537, |
|
"step": 22 |
|
}, |
|
{ |
|
"epoch": 0.006430868167202572, |
|
"grad_norm": 130.05670927021893, |
|
"learning_rate": 9.99314767377287e-07, |
|
"logits/chosen": -1.0124224424362183, |
|
"logits/rejected": -1.3622022867202759, |
|
"logps/chosen": -1.240678310394287, |
|
"logps/rejected": -1.411242961883545, |
|
"loss": 1.4839, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -3.1016955375671387, |
|
"rewards/margins": 0.42641204595565796, |
|
"rewards/rejected": -3.5281076431274414, |
|
"step": 23 |
|
}, |
|
{ |
|
"epoch": 0.006710471130993988, |
|
"grad_norm": 22.122176134369795, |
|
"learning_rate": 9.98782025129912e-07, |
|
"logits/chosen": -1.276932716369629, |
|
"logits/rejected": -1.717280387878418, |
|
"logps/chosen": -1.536954402923584, |
|
"logps/rejected": -2.9943461418151855, |
|
"loss": 0.6631, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -3.84238600730896, |
|
"rewards/margins": 3.643479108810425, |
|
"rewards/rejected": -7.485865592956543, |
|
"step": 24 |
|
}, |
|
{ |
|
"epoch": 0.006990074094785405, |
|
"grad_norm": 25.203974129539585, |
|
"learning_rate": 9.980973490458728e-07, |
|
"logits/chosen": -1.0255053043365479, |
|
"logits/rejected": -1.122296690940857, |
|
"logps/chosen": -1.0807580947875977, |
|
"logps/rejected": -1.0660605430603027, |
|
"loss": 1.6138, |
|
"rewards/accuracies": 0.25, |
|
"rewards/chosen": -2.701895236968994, |
|
"rewards/margins": -0.036744117736816406, |
|
"rewards/rejected": -2.6651511192321777, |
|
"step": 25 |
|
}, |
|
{ |
|
"epoch": 0.007269677058576821, |
|
"grad_norm": 27.512055310670853, |
|
"learning_rate": 9.972609476841365e-07, |
|
"logits/chosen": -1.2338676452636719, |
|
"logits/rejected": -1.3265719413757324, |
|
"logps/chosen": -1.2775177955627441, |
|
"logps/rejected": -1.486391544342041, |
|
"loss": 1.4048, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -3.1937947273254395, |
|
"rewards/margins": 0.5221841335296631, |
|
"rewards/rejected": -3.7159788608551025, |
|
"step": 26 |
|
}, |
|
{ |
|
"epoch": 0.007549280022368237, |
|
"grad_norm": 137.0687602353488, |
|
"learning_rate": 9.96273075820661e-07, |
|
"logits/chosen": -0.8894615173339844, |
|
"logits/rejected": -1.346874713897705, |
|
"logps/chosen": -0.8310048580169678, |
|
"logps/rejected": -1.700638771057129, |
|
"loss": 1.2364, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -2.077512264251709, |
|
"rewards/margins": 2.1740849018096924, |
|
"rewards/rejected": -4.2515974044799805, |
|
"step": 27 |
|
}, |
|
{ |
|
"epoch": 0.007828882986159653, |
|
"grad_norm": 17.619065074739723, |
|
"learning_rate": 9.95134034370785e-07, |
|
"logits/chosen": -1.0894310474395752, |
|
"logits/rejected": -1.202174425125122, |
|
"logps/chosen": -1.1609501838684082, |
|
"logps/rejected": -1.407175064086914, |
|
"loss": 1.4807, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.9023752212524414, |
|
"rewards/margins": 0.6155624389648438, |
|
"rewards/rejected": -3.517937660217285, |
|
"step": 28 |
|
}, |
|
{ |
|
"epoch": 0.00810848594995107, |
|
"grad_norm": 110.88293466564413, |
|
"learning_rate": 9.938441702975689e-07, |
|
"logits/chosen": -0.601264476776123, |
|
"logits/rejected": -0.8674606084823608, |
|
"logps/chosen": -1.1357152462005615, |
|
"logps/rejected": -1.3777132034301758, |
|
"loss": 1.6135, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.839287757873535, |
|
"rewards/margins": 0.6049953699111938, |
|
"rewards/rejected": -3.4442832469940186, |
|
"step": 29 |
|
}, |
|
{ |
|
"epoch": 0.008388088913742486, |
|
"grad_norm": 28.486621774152123, |
|
"learning_rate": 9.92403876506104e-07, |
|
"logits/chosen": -0.8283142447471619, |
|
"logits/rejected": -1.3725578784942627, |
|
"logps/chosen": -0.8853375911712646, |
|
"logps/rejected": -1.7784080505371094, |
|
"loss": 1.4251, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -2.213344097137451, |
|
"rewards/margins": 2.2326760292053223, |
|
"rewards/rejected": -4.446020126342773, |
|
"step": 30 |
|
}, |
|
{ |
|
"epoch": 0.008667691877533901, |
|
"grad_norm": 40.53817922951689, |
|
"learning_rate": 9.90813591723832e-07, |
|
"logits/chosen": -1.3251934051513672, |
|
"logits/rejected": -1.2978826761245728, |
|
"logps/chosen": -1.1201034784317017, |
|
"logps/rejected": -1.6758404970169067, |
|
"loss": 1.1433, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -2.8002586364746094, |
|
"rewards/margins": 1.3893425464630127, |
|
"rewards/rejected": -4.189601421356201, |
|
"step": 31 |
|
}, |
|
{ |
|
"epoch": 0.008947294841325318, |
|
"grad_norm": 30.2037356357338, |
|
"learning_rate": 9.890738003669027e-07, |
|
"logits/chosen": -0.8872109055519104, |
|
"logits/rejected": -1.0567409992218018, |
|
"logps/chosen": -1.3809734582901, |
|
"logps/rejected": -1.5892744064331055, |
|
"loss": 2.2798, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -3.4524335861206055, |
|
"rewards/margins": 0.5207523703575134, |
|
"rewards/rejected": -3.9731857776641846, |
|
"step": 32 |
|
}, |
|
{ |
|
"epoch": 0.009226897805116735, |
|
"grad_norm": 22.08377585136328, |
|
"learning_rate": 9.871850323926177e-07, |
|
"logits/chosen": -0.6034784913063049, |
|
"logits/rejected": -1.3737246990203857, |
|
"logps/chosen": -0.7753689885139465, |
|
"logps/rejected": -1.4922962188720703, |
|
"loss": 1.186, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -1.938422441482544, |
|
"rewards/margins": 1.7923178672790527, |
|
"rewards/rejected": -3.7307403087615967, |
|
"step": 33 |
|
}, |
|
{ |
|
"epoch": 0.00950650076890815, |
|
"grad_norm": 44.83487292747772, |
|
"learning_rate": 9.851478631379982e-07, |
|
"logits/chosen": -1.5287456512451172, |
|
"logits/rejected": -1.604817509651184, |
|
"logps/chosen": -1.2651934623718262, |
|
"logps/rejected": -1.606818675994873, |
|
"loss": 1.2006, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -3.1629838943481445, |
|
"rewards/margins": 0.8540627360343933, |
|
"rewards/rejected": -4.0170464515686035, |
|
"step": 34 |
|
}, |
|
{ |
|
"epoch": 0.009786103732699567, |
|
"grad_norm": 33.18197115037115, |
|
"learning_rate": 9.82962913144534e-07, |
|
"logits/chosen": -0.9402756094932556, |
|
"logits/rejected": -1.3055014610290527, |
|
"logps/chosen": -1.1110124588012695, |
|
"logps/rejected": -1.350355625152588, |
|
"loss": 1.2388, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.777531147003174, |
|
"rewards/margins": 0.5983579158782959, |
|
"rewards/rejected": -3.3758890628814697, |
|
"step": 35 |
|
}, |
|
{ |
|
"epoch": 0.010065706696490984, |
|
"grad_norm": 28.70546678811581, |
|
"learning_rate": 9.806308479691594e-07, |
|
"logits/chosen": -1.2600600719451904, |
|
"logits/rejected": -1.17176353931427, |
|
"logps/chosen": -0.9965181350708008, |
|
"logps/rejected": -1.3181788921356201, |
|
"loss": 1.4099, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -2.491295337677002, |
|
"rewards/margins": 0.8041520118713379, |
|
"rewards/rejected": -3.29544734954834, |
|
"step": 36 |
|
}, |
|
{ |
|
"epoch": 0.010345309660282399, |
|
"grad_norm": 21.048051889450377, |
|
"learning_rate": 9.781523779815178e-07, |
|
"logits/chosen": -0.8446118831634521, |
|
"logits/rejected": -0.9734996557235718, |
|
"logps/chosen": -1.013080358505249, |
|
"logps/rejected": -1.1583538055419922, |
|
"loss": 1.5673, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.532700777053833, |
|
"rewards/margins": 0.3631836771965027, |
|
"rewards/rejected": -2.8958845138549805, |
|
"step": 37 |
|
}, |
|
{ |
|
"epoch": 0.010624912624073815, |
|
"grad_norm": 23.549736337048298, |
|
"learning_rate": 9.755282581475767e-07, |
|
"logits/chosen": -0.772028386592865, |
|
"logits/rejected": -0.821832001209259, |
|
"logps/chosen": -1.1050798892974854, |
|
"logps/rejected": -1.126805305480957, |
|
"loss": 1.6001, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -2.762699842453003, |
|
"rewards/margins": 0.0543133020401001, |
|
"rewards/rejected": -2.8170132637023926, |
|
"step": 38 |
|
}, |
|
{ |
|
"epoch": 0.010904515587865232, |
|
"grad_norm": 31.678751895679902, |
|
"learning_rate": 9.727592877996584e-07, |
|
"logits/chosen": -1.2923513650894165, |
|
"logits/rejected": -0.9500499963760376, |
|
"logps/chosen": -1.3736703395843506, |
|
"logps/rejected": -1.8005601167678833, |
|
"loss": 1.3401, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -3.434175968170166, |
|
"rewards/margins": 1.0672242641448975, |
|
"rewards/rejected": -4.501400470733643, |
|
"step": 39 |
|
}, |
|
{ |
|
"epoch": 0.011184118551656647, |
|
"grad_norm": 42.14350999647913, |
|
"learning_rate": 9.698463103929541e-07, |
|
"logits/chosen": -0.9315403699874878, |
|
"logits/rejected": -0.9749379754066467, |
|
"logps/chosen": -1.1820948123931885, |
|
"logps/rejected": -1.1733828783035278, |
|
"loss": 1.1077, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -2.9552369117736816, |
|
"rewards/margins": -0.021779626607894897, |
|
"rewards/rejected": -2.933457374572754, |
|
"step": 40 |
|
}, |
|
{ |
|
"epoch": 0.011463721515448064, |
|
"grad_norm": 28.8262776267842, |
|
"learning_rate": 9.667902132486008e-07, |
|
"logits/chosen": -1.2946910858154297, |
|
"logits/rejected": -1.3090943098068237, |
|
"logps/chosen": -1.1675629615783691, |
|
"logps/rejected": -1.324582576751709, |
|
"loss": 1.4377, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.918907403945923, |
|
"rewards/margins": 0.39254891872406006, |
|
"rewards/rejected": -3.3114564418792725, |
|
"step": 41 |
|
}, |
|
{ |
|
"epoch": 0.011743324479239479, |
|
"grad_norm": 106.3933529697075, |
|
"learning_rate": 9.635919272833937e-07, |
|
"logits/chosen": -0.8468477725982666, |
|
"logits/rejected": -0.87651127576828, |
|
"logps/chosen": -1.5985316038131714, |
|
"logps/rejected": -1.475963830947876, |
|
"loss": 1.5327, |
|
"rewards/accuracies": 0.25, |
|
"rewards/chosen": -3.9963290691375732, |
|
"rewards/margins": -0.3064194321632385, |
|
"rewards/rejected": -3.6899096965789795, |
|
"step": 42 |
|
}, |
|
{ |
|
"epoch": 0.012022927443030896, |
|
"grad_norm": 16.873173603232406, |
|
"learning_rate": 9.602524267262202e-07, |
|
"logits/chosen": -0.6168065667152405, |
|
"logits/rejected": -0.7960647940635681, |
|
"logps/chosen": -1.0298948287963867, |
|
"logps/rejected": -1.0422132015228271, |
|
"loss": 1.0135, |
|
"rewards/accuracies": 0.25, |
|
"rewards/chosen": -2.574737310409546, |
|
"rewards/margins": 0.030795753002166748, |
|
"rewards/rejected": -2.6055331230163574, |
|
"step": 43 |
|
}, |
|
{ |
|
"epoch": 0.012302530406822313, |
|
"grad_norm": 22.346008825908783, |
|
"learning_rate": 9.567727288213004e-07, |
|
"logits/chosen": -0.8090029954910278, |
|
"logits/rejected": -0.7225205898284912, |
|
"logps/chosen": -1.0612010955810547, |
|
"logps/rejected": -1.0335237979888916, |
|
"loss": 1.4244, |
|
"rewards/accuracies": 0.25, |
|
"rewards/chosen": -2.6530027389526367, |
|
"rewards/margins": -0.06919330358505249, |
|
"rewards/rejected": -2.5838096141815186, |
|
"step": 44 |
|
}, |
|
{ |
|
"epoch": 0.012582133370613728, |
|
"grad_norm": 23.528648829320613, |
|
"learning_rate": 9.531538935183249e-07, |
|
"logits/chosen": -1.1082680225372314, |
|
"logits/rejected": -1.311267614364624, |
|
"logps/chosen": -1.0698087215423584, |
|
"logps/rejected": -1.3603800535202026, |
|
"loss": 1.1636, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.6745219230651855, |
|
"rewards/margins": 0.7264281511306763, |
|
"rewards/rejected": -3.4009499549865723, |
|
"step": 45 |
|
}, |
|
{ |
|
"epoch": 0.012861736334405145, |
|
"grad_norm": 36.88944148482263, |
|
"learning_rate": 9.493970231495834e-07, |
|
"logits/chosen": -0.9574987292289734, |
|
"logits/rejected": -1.4005671739578247, |
|
"logps/chosen": -1.1250247955322266, |
|
"logps/rejected": -1.6039255857467651, |
|
"loss": 1.3524, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.8125619888305664, |
|
"rewards/margins": 1.1972520351409912, |
|
"rewards/rejected": -4.009814262390137, |
|
"step": 46 |
|
}, |
|
{ |
|
"epoch": 0.013141339298196561, |
|
"grad_norm": 34.07967217969598, |
|
"learning_rate": 9.455032620941839e-07, |
|
"logits/chosen": -0.575081467628479, |
|
"logits/rejected": -1.0301098823547363, |
|
"logps/chosen": -0.6966181993484497, |
|
"logps/rejected": -0.7595060467720032, |
|
"loss": 1.1816, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -1.7415456771850586, |
|
"rewards/margins": 0.15721949934959412, |
|
"rewards/rejected": -1.8987650871276855, |
|
"step": 47 |
|
}, |
|
{ |
|
"epoch": 0.013420942261987976, |
|
"grad_norm": 37.33338028904019, |
|
"learning_rate": 9.414737964294634e-07, |
|
"logits/chosen": -0.48685717582702637, |
|
"logits/rejected": -0.9498811960220337, |
|
"logps/chosen": -0.8811389207839966, |
|
"logps/rejected": -1.4033122062683105, |
|
"loss": 0.9144, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -2.202847480773926, |
|
"rewards/margins": 1.3054332733154297, |
|
"rewards/rejected": -3.5082805156707764, |
|
"step": 48 |
|
}, |
|
{ |
|
"epoch": 0.013700545225779393, |
|
"grad_norm": 20.112352494523353, |
|
"learning_rate": 9.373098535696978e-07, |
|
"logits/chosen": -0.9682779908180237, |
|
"logits/rejected": -1.1488547325134277, |
|
"logps/chosen": -1.1651062965393066, |
|
"logps/rejected": -1.3421710729599, |
|
"loss": 1.5188, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -2.9127655029296875, |
|
"rewards/margins": 0.4426620602607727, |
|
"rewards/rejected": -3.3554277420043945, |
|
"step": 49 |
|
}, |
|
{ |
|
"epoch": 0.01398014818957081, |
|
"grad_norm": 33.55406806691841, |
|
"learning_rate": 9.330127018922193e-07, |
|
"logits/chosen": -1.2889659404754639, |
|
"logits/rejected": -0.9354056119918823, |
|
"logps/chosen": -1.3529675006866455, |
|
"logps/rejected": -1.1981086730957031, |
|
"loss": 1.645, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -3.3824188709259033, |
|
"rewards/margins": -0.38714712858200073, |
|
"rewards/rejected": -2.995271682739258, |
|
"step": 50 |
|
}, |
|
{ |
|
"epoch": 0.01398014818957081, |
|
"eval_logits/chosen": -1.184069275856018, |
|
"eval_logits/rejected": -1.346756100654602, |
|
"eval_logps/chosen": -1.1177828311920166, |
|
"eval_logps/rejected": -1.4929856061935425, |
|
"eval_loss": 1.2563124895095825, |
|
"eval_rewards/accuracies": 0.7027027010917664, |
|
"eval_rewards/chosen": -2.794457197189331, |
|
"eval_rewards/margins": 0.9380068182945251, |
|
"eval_rewards/rejected": -3.73246431350708, |
|
"eval_runtime": 345.1606, |
|
"eval_samples_per_second": 3.847, |
|
"eval_steps_per_second": 0.322, |
|
"step": 50 |
|
}, |
|
{ |
|
"epoch": 0.014259751153362225, |
|
"grad_norm": 22.5880454834462, |
|
"learning_rate": 9.285836503510562e-07, |
|
"logits/chosen": -0.8200809955596924, |
|
"logits/rejected": -0.8582978248596191, |
|
"logps/chosen": -1.1703073978424072, |
|
"logps/rejected": -1.458601951599121, |
|
"loss": 0.9907, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -2.9257683753967285, |
|
"rewards/margins": 0.7207364439964294, |
|
"rewards/rejected": -3.6465048789978027, |
|
"step": 51 |
|
}, |
|
{ |
|
"epoch": 0.014539354117153642, |
|
"grad_norm": 45.098078544435936, |
|
"learning_rate": 9.240240480782129e-07, |
|
"logits/chosen": -0.7370530962944031, |
|
"logits/rejected": -1.368110179901123, |
|
"logps/chosen": -0.9416132569313049, |
|
"logps/rejected": -1.0832104682922363, |
|
"loss": 1.697, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.3540329933166504, |
|
"rewards/margins": 0.35399293899536133, |
|
"rewards/rejected": -2.708026170730591, |
|
"step": 52 |
|
}, |
|
{ |
|
"epoch": 0.014818957080945059, |
|
"grad_norm": 101.10764848570363, |
|
"learning_rate": 9.19335283972712e-07, |
|
"logits/chosen": -0.5560005903244019, |
|
"logits/rejected": -0.7855780124664307, |
|
"logps/chosen": -1.0373799800872803, |
|
"logps/rejected": -1.0468213558197021, |
|
"loss": 1.3264, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -2.5934500694274902, |
|
"rewards/margins": 0.023603498935699463, |
|
"rewards/rejected": -2.617053508758545, |
|
"step": 53 |
|
}, |
|
{ |
|
"epoch": 0.015098560044736474, |
|
"grad_norm": 64.45755933217201, |
|
"learning_rate": 9.145187862775208e-07, |
|
"logits/chosen": -1.421772837638855, |
|
"logits/rejected": -1.4179284572601318, |
|
"logps/chosen": -1.214949607849121, |
|
"logps/rejected": -2.0096287727355957, |
|
"loss": 1.2303, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -3.0373740196228027, |
|
"rewards/margins": 1.9866979122161865, |
|
"rewards/rejected": -5.02407169342041, |
|
"step": 54 |
|
}, |
|
{ |
|
"epoch": 0.01537816300852789, |
|
"grad_norm": 24.722247357252545, |
|
"learning_rate": 9.095760221444959e-07, |
|
"logits/chosen": -1.1340514421463013, |
|
"logits/rejected": -1.4643547534942627, |
|
"logps/chosen": -1.0777888298034668, |
|
"logps/rejected": -1.2248834371566772, |
|
"loss": 1.163, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.694471836090088, |
|
"rewards/margins": 0.3677366375923157, |
|
"rewards/rejected": -3.062208652496338, |
|
"step": 55 |
|
}, |
|
{ |
|
"epoch": 0.015657765972319306, |
|
"grad_norm": 88.82834717216237, |
|
"learning_rate": 9.045084971874737e-07, |
|
"logits/chosen": -1.0740933418273926, |
|
"logits/rejected": -0.9167161583900452, |
|
"logps/chosen": -1.2093803882598877, |
|
"logps/rejected": -1.6125764846801758, |
|
"loss": 1.2562, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -3.023451089859009, |
|
"rewards/margins": 1.0079903602600098, |
|
"rewards/rejected": -4.031441688537598, |
|
"step": 56 |
|
}, |
|
{ |
|
"epoch": 0.015937368936110724, |
|
"grad_norm": 18.31655486222315, |
|
"learning_rate": 8.993177550236463e-07, |
|
"logits/chosen": -0.6805390119552612, |
|
"logits/rejected": -0.7955325841903687, |
|
"logps/chosen": -1.2940752506256104, |
|
"logps/rejected": -2.059095859527588, |
|
"loss": 1.2213, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -3.2351880073547363, |
|
"rewards/margins": 1.912551760673523, |
|
"rewards/rejected": -5.147739410400391, |
|
"step": 57 |
|
}, |
|
{ |
|
"epoch": 0.01621697189990214, |
|
"grad_norm": 22.784684254467784, |
|
"learning_rate": 8.940053768033608e-07, |
|
"logits/chosen": -0.5054349303245544, |
|
"logits/rejected": -1.086111068725586, |
|
"logps/chosen": -0.8294824957847595, |
|
"logps/rejected": -1.2600980997085571, |
|
"loss": 1.1366, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -2.0737061500549316, |
|
"rewards/margins": 1.0765390396118164, |
|
"rewards/rejected": -3.150245189666748, |
|
"step": 58 |
|
}, |
|
{ |
|
"epoch": 0.016496574863693554, |
|
"grad_norm": 24.23188699617836, |
|
"learning_rate": 8.885729807284854e-07, |
|
"logits/chosen": -0.9133630990982056, |
|
"logits/rejected": -1.0163180828094482, |
|
"logps/chosen": -1.049289345741272, |
|
"logps/rejected": -1.134240984916687, |
|
"loss": 1.0682, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -2.623223304748535, |
|
"rewards/margins": 0.21237897872924805, |
|
"rewards/rejected": -2.8356025218963623, |
|
"step": 59 |
|
}, |
|
{ |
|
"epoch": 0.016776177827484973, |
|
"grad_norm": 28.93680920583823, |
|
"learning_rate": 8.83022221559489e-07, |
|
"logits/chosen": -1.0600225925445557, |
|
"logits/rejected": -1.2378225326538086, |
|
"logps/chosen": -1.2599135637283325, |
|
"logps/rejected": -1.3674981594085693, |
|
"loss": 0.9747, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -3.1497838497161865, |
|
"rewards/margins": 0.2689616084098816, |
|
"rewards/rejected": -3.418745517730713, |
|
"step": 60 |
|
}, |
|
{ |
|
"epoch": 0.017055780791276388, |
|
"grad_norm": 29.885312194374713, |
|
"learning_rate": 8.773547901113861e-07, |
|
"logits/chosen": -0.7806551456451416, |
|
"logits/rejected": -1.628957748413086, |
|
"logps/chosen": -0.7070074677467346, |
|
"logps/rejected": -1.1015799045562744, |
|
"loss": 0.7853, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -1.7675187587738037, |
|
"rewards/margins": 0.9864310026168823, |
|
"rewards/rejected": -2.7539496421813965, |
|
"step": 61 |
|
}, |
|
{ |
|
"epoch": 0.017335383755067803, |
|
"grad_norm": 24.972205075082606, |
|
"learning_rate": 8.71572412738697e-07, |
|
"logits/chosen": -1.0657659769058228, |
|
"logits/rejected": -1.4624931812286377, |
|
"logps/chosen": -1.1949583292007446, |
|
"logps/rejected": -1.8615357875823975, |
|
"loss": 0.631, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -2.987395763397217, |
|
"rewards/margins": 1.6664433479309082, |
|
"rewards/rejected": -4.653839111328125, |
|
"step": 62 |
|
}, |
|
{ |
|
"epoch": 0.01761498671885922, |
|
"grad_norm": 21.933003035104587, |
|
"learning_rate": 8.656768508095852e-07, |
|
"logits/chosen": -1.1191598176956177, |
|
"logits/rejected": -1.2448678016662598, |
|
"logps/chosen": -1.0887205600738525, |
|
"logps/rejected": -1.6507292985916138, |
|
"loss": 0.887, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -2.7218017578125, |
|
"rewards/margins": 1.4050216674804688, |
|
"rewards/rejected": -4.126823425292969, |
|
"step": 63 |
|
}, |
|
{ |
|
"epoch": 0.017894589682650636, |
|
"grad_norm": 24.060712897137755, |
|
"learning_rate": 8.596699001693255e-07, |
|
"logits/chosen": -0.798321008682251, |
|
"logits/rejected": -1.0583261251449585, |
|
"logps/chosen": -0.7978274822235107, |
|
"logps/rejected": -1.1801316738128662, |
|
"loss": 1.3542, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -1.9945688247680664, |
|
"rewards/margins": 0.9557601809501648, |
|
"rewards/rejected": -2.950329065322876, |
|
"step": 64 |
|
}, |
|
{ |
|
"epoch": 0.01817419264644205, |
|
"grad_norm": 35.64070939669877, |
|
"learning_rate": 8.535533905932737e-07, |
|
"logits/chosen": -0.9407416582107544, |
|
"logits/rejected": -1.2474677562713623, |
|
"logps/chosen": -1.1173073053359985, |
|
"logps/rejected": -1.4939210414886475, |
|
"loss": 1.1944, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.7932682037353516, |
|
"rewards/margins": 0.9415344595909119, |
|
"rewards/rejected": -3.734802722930908, |
|
"step": 65 |
|
}, |
|
{ |
|
"epoch": 0.01845379561023347, |
|
"grad_norm": 23.204374003534983, |
|
"learning_rate": 8.473291852294986e-07, |
|
"logits/chosen": -0.7950379252433777, |
|
"logits/rejected": -1.1784076690673828, |
|
"logps/chosen": -0.8873813152313232, |
|
"logps/rejected": -1.0771222114562988, |
|
"loss": 1.291, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.2184531688690186, |
|
"rewards/margins": 0.4743524193763733, |
|
"rewards/rejected": -2.692805767059326, |
|
"step": 66 |
|
}, |
|
{ |
|
"epoch": 0.018733398574024885, |
|
"grad_norm": 22.342402065645, |
|
"learning_rate": 8.409991800312492e-07, |
|
"logits/chosen": -0.6992224454879761, |
|
"logits/rejected": -1.0948431491851807, |
|
"logps/chosen": -1.1709400415420532, |
|
"logps/rejected": -1.746230959892273, |
|
"loss": 1.0885, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.9273500442504883, |
|
"rewards/margins": 1.4382274150848389, |
|
"rewards/rejected": -4.365577220916748, |
|
"step": 67 |
|
}, |
|
{ |
|
"epoch": 0.0190130015378163, |
|
"grad_norm": 26.635960985863385, |
|
"learning_rate": 8.34565303179429e-07, |
|
"logits/chosen": -0.9120014905929565, |
|
"logits/rejected": -1.2207849025726318, |
|
"logps/chosen": -1.022168755531311, |
|
"logps/rejected": -1.3500474691390991, |
|
"loss": 1.0317, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -2.555421829223633, |
|
"rewards/margins": 0.819696843624115, |
|
"rewards/rejected": -3.3751187324523926, |
|
"step": 68 |
|
}, |
|
{ |
|
"epoch": 0.01929260450160772, |
|
"grad_norm": 216.32442664714605, |
|
"learning_rate": 8.280295144952536e-07, |
|
"logits/chosen": -0.9009850025177002, |
|
"logits/rejected": -0.9338705539703369, |
|
"logps/chosen": -1.7400050163269043, |
|
"logps/rejected": -3.0889453887939453, |
|
"loss": 1.6279, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -4.350012302398682, |
|
"rewards/margins": 3.3723511695861816, |
|
"rewards/rejected": -7.722363471984863, |
|
"step": 69 |
|
}, |
|
{ |
|
"epoch": 0.019572207465399134, |
|
"grad_norm": 33.24100380368566, |
|
"learning_rate": 8.213938048432696e-07, |
|
"logits/chosen": -1.1465697288513184, |
|
"logits/rejected": -1.5220608711242676, |
|
"logps/chosen": -1.5942764282226562, |
|
"logps/rejected": -2.3567376136779785, |
|
"loss": 0.9686, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -3.9856910705566406, |
|
"rewards/margins": 1.9061533212661743, |
|
"rewards/rejected": -5.891844272613525, |
|
"step": 70 |
|
}, |
|
{ |
|
"epoch": 0.01985181042919055, |
|
"grad_norm": 17.831061474851452, |
|
"learning_rate": 8.146601955249187e-07, |
|
"logits/chosen": -0.7144116163253784, |
|
"logits/rejected": -1.260697603225708, |
|
"logps/chosen": -0.7787595987319946, |
|
"logps/rejected": -1.2630029916763306, |
|
"loss": 1.2457, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -1.9468989372253418, |
|
"rewards/margins": 1.2106084823608398, |
|
"rewards/rejected": -3.1575074195861816, |
|
"step": 71 |
|
}, |
|
{ |
|
"epoch": 0.020131413392981967, |
|
"grad_norm": 32.219733178240375, |
|
"learning_rate": 8.07830737662829e-07, |
|
"logits/chosen": -1.14467191696167, |
|
"logits/rejected": -0.7406169176101685, |
|
"logps/chosen": -1.5345901250839233, |
|
"logps/rejected": -1.2437353134155273, |
|
"loss": 1.2542, |
|
"rewards/accuracies": 0.25, |
|
"rewards/chosen": -3.836475372314453, |
|
"rewards/margins": -0.7271367907524109, |
|
"rewards/rejected": -3.1093382835388184, |
|
"step": 72 |
|
}, |
|
{ |
|
"epoch": 0.020411016356773382, |
|
"grad_norm": 28.263777851446193, |
|
"learning_rate": 8.009075115760242e-07, |
|
"logits/chosen": -0.8964699506759644, |
|
"logits/rejected": -1.3684449195861816, |
|
"logps/chosen": -0.913366436958313, |
|
"logps/rejected": -1.5431159734725952, |
|
"loss": 1.0506, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.2834160327911377, |
|
"rewards/margins": 1.5743739604949951, |
|
"rewards/rejected": -3.857790231704712, |
|
"step": 73 |
|
}, |
|
{ |
|
"epoch": 0.020690619320564797, |
|
"grad_norm": 33.62623441801084, |
|
"learning_rate": 7.938926261462365e-07, |
|
"logits/chosen": -1.5208332538604736, |
|
"logits/rejected": -1.4947750568389893, |
|
"logps/chosen": -1.6027369499206543, |
|
"logps/rejected": -2.2491843700408936, |
|
"loss": 1.1593, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -4.006842136383057, |
|
"rewards/margins": 1.6161189079284668, |
|
"rewards/rejected": -5.622961044311523, |
|
"step": 74 |
|
}, |
|
{ |
|
"epoch": 0.020970222284356216, |
|
"grad_norm": 19.73073540290438, |
|
"learning_rate": 7.86788218175523e-07, |
|
"logits/chosen": -0.8135213851928711, |
|
"logits/rejected": -1.1303890943527222, |
|
"logps/chosen": -1.1947009563446045, |
|
"logps/rejected": -1.178377628326416, |
|
"loss": 1.0676, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.986752510070801, |
|
"rewards/margins": -0.04080846905708313, |
|
"rewards/rejected": -2.945943832397461, |
|
"step": 75 |
|
}, |
|
{ |
|
"epoch": 0.02124982524814763, |
|
"grad_norm": 30.725692444381682, |
|
"learning_rate": 7.795964517353733e-07, |
|
"logits/chosen": -0.5928746461868286, |
|
"logits/rejected": -1.3386216163635254, |
|
"logps/chosen": -1.1336582899093628, |
|
"logps/rejected": -2.486417770385742, |
|
"loss": 1.2958, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -2.8341457843780518, |
|
"rewards/margins": 3.3818984031677246, |
|
"rewards/rejected": -6.2160444259643555, |
|
"step": 76 |
|
}, |
|
{ |
|
"epoch": 0.021529428211939046, |
|
"grad_norm": 19.993312804225585, |
|
"learning_rate": 7.723195175075135e-07, |
|
"logits/chosen": -1.0590918064117432, |
|
"logits/rejected": -1.4984095096588135, |
|
"logps/chosen": -1.341080904006958, |
|
"logps/rejected": -2.226268768310547, |
|
"loss": 0.7399, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -3.3527023792266846, |
|
"rewards/margins": 2.212970018386841, |
|
"rewards/rejected": -5.565672397613525, |
|
"step": 77 |
|
}, |
|
{ |
|
"epoch": 0.021809031175730464, |
|
"grad_norm": 27.58218498128165, |
|
"learning_rate": 7.649596321166024e-07, |
|
"logits/chosen": -0.903189480304718, |
|
"logits/rejected": -1.4045133590698242, |
|
"logps/chosen": -1.2869590520858765, |
|
"logps/rejected": -2.1190972328186035, |
|
"loss": 1.1142, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -3.217397689819336, |
|
"rewards/margins": 2.080345392227173, |
|
"rewards/rejected": -5.29774284362793, |
|
"step": 78 |
|
}, |
|
{ |
|
"epoch": 0.02208863413952188, |
|
"grad_norm": 33.76140161445492, |
|
"learning_rate": 7.575190374550271e-07, |
|
"logits/chosen": -1.1929422616958618, |
|
"logits/rejected": -1.402430534362793, |
|
"logps/chosen": -1.4159026145935059, |
|
"logps/rejected": -1.8726705312728882, |
|
"loss": 1.0192, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -3.5397567749023438, |
|
"rewards/margins": 1.1419193744659424, |
|
"rewards/rejected": -4.681675910949707, |
|
"step": 79 |
|
}, |
|
{ |
|
"epoch": 0.022368237103313295, |
|
"grad_norm": 23.027800645867977, |
|
"learning_rate": 7.5e-07, |
|
"logits/chosen": -0.6945698857307434, |
|
"logits/rejected": -1.40185546875, |
|
"logps/chosen": -1.4425044059753418, |
|
"logps/rejected": -2.2865071296691895, |
|
"loss": 0.9812, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -3.6062612533569336, |
|
"rewards/margins": 2.110006809234619, |
|
"rewards/rejected": -5.7162675857543945, |
|
"step": 80 |
|
}, |
|
{ |
|
"epoch": 0.02264784006710471, |
|
"grad_norm": 38.27741144565889, |
|
"learning_rate": 7.424048101231686e-07, |
|
"logits/chosen": -1.2587817907333374, |
|
"logits/rejected": -1.0885329246520996, |
|
"logps/chosen": -1.2346124649047852, |
|
"logps/rejected": -1.6860368251800537, |
|
"loss": 0.7012, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -3.086531162261963, |
|
"rewards/margins": 1.1285609006881714, |
|
"rewards/rejected": -4.215092182159424, |
|
"step": 81 |
|
}, |
|
{ |
|
"epoch": 0.022927443030896128, |
|
"grad_norm": 90.96248649883246, |
|
"learning_rate": 7.347357813929454e-07, |
|
"logits/chosen": -1.2520544528961182, |
|
"logits/rejected": -1.0464751720428467, |
|
"logps/chosen": -2.1459150314331055, |
|
"logps/rejected": -2.046426296234131, |
|
"loss": 1.5015, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -5.364788055419922, |
|
"rewards/margins": -0.24872243404388428, |
|
"rewards/rejected": -5.116065502166748, |
|
"step": 82 |
|
}, |
|
{ |
|
"epoch": 0.023207045994687543, |
|
"grad_norm": 100.9433648678187, |
|
"learning_rate": 7.269952498697734e-07, |
|
"logits/chosen": -1.059312105178833, |
|
"logits/rejected": -0.9108483791351318, |
|
"logps/chosen": -1.4121086597442627, |
|
"logps/rejected": -1.6452100276947021, |
|
"loss": 1.3536, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -3.530271530151367, |
|
"rewards/margins": 0.5827533602714539, |
|
"rewards/rejected": -4.113024711608887, |
|
"step": 83 |
|
}, |
|
{ |
|
"epoch": 0.023486648958478958, |
|
"grad_norm": 173.9823816167412, |
|
"learning_rate": 7.191855733945386e-07, |
|
"logits/chosen": -0.9303680658340454, |
|
"logits/rejected": -1.1252155303955078, |
|
"logps/chosen": -1.284766674041748, |
|
"logps/rejected": -1.3653523921966553, |
|
"loss": 2.4014, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -3.211916923522949, |
|
"rewards/margins": 0.2014639973640442, |
|
"rewards/rejected": -3.4133810997009277, |
|
"step": 84 |
|
}, |
|
{ |
|
"epoch": 0.023766251922270377, |
|
"grad_norm": 28.477804453675674, |
|
"learning_rate": 7.113091308703497e-07, |
|
"logits/chosen": -0.7400285601615906, |
|
"logits/rejected": -0.847143292427063, |
|
"logps/chosen": -1.8564958572387695, |
|
"logps/rejected": -2.197305917739868, |
|
"loss": 1.3282, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -4.641240119934082, |
|
"rewards/margins": 0.8520248532295227, |
|
"rewards/rejected": -5.493264675140381, |
|
"step": 85 |
|
}, |
|
{ |
|
"epoch": 0.024045854886061792, |
|
"grad_norm": 158.57710825565218, |
|
"learning_rate": 7.033683215379002e-07, |
|
"logits/chosen": -1.1289122104644775, |
|
"logits/rejected": -1.7158691883087158, |
|
"logps/chosen": -1.2116670608520508, |
|
"logps/rejected": -2.2744312286376953, |
|
"loss": 1.1088, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -3.029167652130127, |
|
"rewards/margins": 2.6569106578826904, |
|
"rewards/rejected": -5.686078071594238, |
|
"step": 86 |
|
}, |
|
{ |
|
"epoch": 0.024325457849853207, |
|
"grad_norm": 14.96513597899858, |
|
"learning_rate": 6.953655642446367e-07, |
|
"logits/chosen": -0.6412773728370667, |
|
"logits/rejected": -1.4269566535949707, |
|
"logps/chosen": -0.985595703125, |
|
"logps/rejected": -2.8099608421325684, |
|
"loss": 0.6137, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -2.4639892578125, |
|
"rewards/margins": 4.5609130859375, |
|
"rewards/rejected": -7.02490234375, |
|
"step": 87 |
|
}, |
|
{ |
|
"epoch": 0.024605060813644625, |
|
"grad_norm": 69.31853470031655, |
|
"learning_rate": 6.87303296707956e-07, |
|
"logits/chosen": -1.252925157546997, |
|
"logits/rejected": -1.3233586549758911, |
|
"logps/chosen": -1.3767462968826294, |
|
"logps/rejected": -1.7665659189224243, |
|
"loss": 1.2815, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -3.441865921020508, |
|
"rewards/margins": 0.9745489358901978, |
|
"rewards/rejected": -4.416414737701416, |
|
"step": 88 |
|
}, |
|
{ |
|
"epoch": 0.02488466377743604, |
|
"grad_norm": 27.097255688460947, |
|
"learning_rate": 6.7918397477265e-07, |
|
"logits/chosen": -0.8277978301048279, |
|
"logits/rejected": -1.0574264526367188, |
|
"logps/chosen": -1.1671627759933472, |
|
"logps/rejected": -2.2953076362609863, |
|
"loss": 1.0056, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.9179069995880127, |
|
"rewards/margins": 2.820362091064453, |
|
"rewards/rejected": -5.738268852233887, |
|
"step": 89 |
|
}, |
|
{ |
|
"epoch": 0.025164266741227456, |
|
"grad_norm": 52.839387510999444, |
|
"learning_rate": 6.710100716628344e-07, |
|
"logits/chosen": -1.40108323097229, |
|
"logits/rejected": -1.4667222499847412, |
|
"logps/chosen": -1.4003353118896484, |
|
"logps/rejected": -1.6744577884674072, |
|
"loss": 1.3688, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -3.500838279724121, |
|
"rewards/margins": 0.6853060126304626, |
|
"rewards/rejected": -4.1861443519592285, |
|
"step": 90 |
|
}, |
|
{ |
|
"epoch": 0.025443869705018874, |
|
"grad_norm": 21.925148847234777, |
|
"learning_rate": 6.627840772285783e-07, |
|
"logits/chosen": -1.0636601448059082, |
|
"logits/rejected": -1.0789152383804321, |
|
"logps/chosen": -1.1292977333068848, |
|
"logps/rejected": -1.3410239219665527, |
|
"loss": 1.4889, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -2.823244094848633, |
|
"rewards/margins": 0.5293155908584595, |
|
"rewards/rejected": -3.3525595664978027, |
|
"step": 91 |
|
}, |
|
{ |
|
"epoch": 0.02572347266881029, |
|
"grad_norm": 26.167972640340828, |
|
"learning_rate": 6.545084971874736e-07, |
|
"logits/chosen": -0.6311432719230652, |
|
"logits/rejected": -1.1260812282562256, |
|
"logps/chosen": -1.1044063568115234, |
|
"logps/rejected": -1.6531506776809692, |
|
"loss": 1.2341, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.7610158920288086, |
|
"rewards/margins": 1.3718609809875488, |
|
"rewards/rejected": -4.132876873016357, |
|
"step": 92 |
|
}, |
|
{ |
|
"epoch": 0.026003075632601704, |
|
"grad_norm": 19.955245570389728, |
|
"learning_rate": 6.461858523613684e-07, |
|
"logits/chosen": -0.9683464765548706, |
|
"logits/rejected": -0.9185085296630859, |
|
"logps/chosen": -0.9664384722709656, |
|
"logps/rejected": -1.6319332122802734, |
|
"loss": 0.8303, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.4160962104797363, |
|
"rewards/margins": 1.6637368202209473, |
|
"rewards/rejected": -4.079833030700684, |
|
"step": 93 |
|
}, |
|
{ |
|
"epoch": 0.026282678596393123, |
|
"grad_norm": 38.679628570882045, |
|
"learning_rate": 6.378186779084995e-07, |
|
"logits/chosen": -0.8759285807609558, |
|
"logits/rejected": -0.6558728218078613, |
|
"logps/chosen": -1.1541013717651367, |
|
"logps/rejected": -1.992755651473999, |
|
"loss": 0.598, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.885253429412842, |
|
"rewards/margins": 2.0966360569000244, |
|
"rewards/rejected": -4.981889247894287, |
|
"step": 94 |
|
}, |
|
{ |
|
"epoch": 0.026562281560184538, |
|
"grad_norm": 43.75173307796015, |
|
"learning_rate": 6.294095225512604e-07, |
|
"logits/chosen": -0.9964724779129028, |
|
"logits/rejected": -0.7991459369659424, |
|
"logps/chosen": -1.0949715375900269, |
|
"logps/rejected": -1.5087236166000366, |
|
"loss": 1.0437, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.737428665161133, |
|
"rewards/margins": 1.034380316734314, |
|
"rewards/rejected": -3.7718091011047363, |
|
"step": 95 |
|
}, |
|
{ |
|
"epoch": 0.026841884523975953, |
|
"grad_norm": 22.420799786964153, |
|
"learning_rate": 6.209609477998338e-07, |
|
"logits/chosen": -0.4085179269313812, |
|
"logits/rejected": -0.7962716221809387, |
|
"logps/chosen": -1.0953500270843506, |
|
"logps/rejected": -1.046759843826294, |
|
"loss": 1.5788, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -2.738374710083008, |
|
"rewards/margins": -0.12147501111030579, |
|
"rewards/rejected": -2.6168999671936035, |
|
"step": 96 |
|
}, |
|
{ |
|
"epoch": 0.02712148748776737, |
|
"grad_norm": 28.1032386388583, |
|
"learning_rate": 6.124755271719326e-07, |
|
"logits/chosen": -0.96661376953125, |
|
"logits/rejected": -1.131087064743042, |
|
"logps/chosen": -1.2935943603515625, |
|
"logps/rejected": -1.4725990295410156, |
|
"loss": 1.1007, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -3.233985662460327, |
|
"rewards/margins": 0.4475119113922119, |
|
"rewards/rejected": -3.681497573852539, |
|
"step": 97 |
|
}, |
|
{ |
|
"epoch": 0.027401090451558786, |
|
"grad_norm": 51.25959864452938, |
|
"learning_rate": 6.039558454088795e-07, |
|
"logits/chosen": -1.0819038152694702, |
|
"logits/rejected": -1.3961520195007324, |
|
"logps/chosen": -1.1702494621276855, |
|
"logps/rejected": -2.0412793159484863, |
|
"loss": 0.8748, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.925623655319214, |
|
"rewards/margins": 2.1775741577148438, |
|
"rewards/rejected": -5.103198051452637, |
|
"step": 98 |
|
}, |
|
{ |
|
"epoch": 0.0276806934153502, |
|
"grad_norm": 44.86957712355451, |
|
"learning_rate": 5.954044976882723e-07, |
|
"logits/chosen": -0.4054487347602844, |
|
"logits/rejected": -0.9896742105484009, |
|
"logps/chosen": -0.8635058999061584, |
|
"logps/rejected": -1.2418280839920044, |
|
"loss": 1.0457, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.1587648391723633, |
|
"rewards/margins": 0.9458054900169373, |
|
"rewards/rejected": -3.1045703887939453, |
|
"step": 99 |
|
}, |
|
{ |
|
"epoch": 0.02796029637914162, |
|
"grad_norm": 28.90572894211588, |
|
"learning_rate": 5.868240888334652e-07, |
|
"logits/chosen": -0.7194627523422241, |
|
"logits/rejected": -1.274187445640564, |
|
"logps/chosen": -0.822409987449646, |
|
"logps/rejected": -1.5885546207427979, |
|
"loss": 0.8722, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.0560250282287598, |
|
"rewards/margins": 1.9153616428375244, |
|
"rewards/rejected": -3.971386432647705, |
|
"step": 100 |
|
}, |
|
{ |
|
"epoch": 0.02796029637914162, |
|
"eval_logits/chosen": -1.2195580005645752, |
|
"eval_logits/rejected": -1.3816862106323242, |
|
"eval_logps/chosen": -1.2307511568069458, |
|
"eval_logps/rejected": -1.8937082290649414, |
|
"eval_loss": 1.0619382858276367, |
|
"eval_rewards/accuracies": 0.7319819927215576, |
|
"eval_rewards/chosen": -3.0768775939941406, |
|
"eval_rewards/margins": 1.6573928594589233, |
|
"eval_rewards/rejected": -4.7342705726623535, |
|
"eval_runtime": 343.8898, |
|
"eval_samples_per_second": 3.862, |
|
"eval_steps_per_second": 0.323, |
|
"step": 100 |
|
}, |
|
{ |
|
"epoch": 0.028239899342933035, |
|
"grad_norm": 23.014771898473285, |
|
"learning_rate": 5.782172325201155e-07, |
|
"logits/chosen": -1.019005298614502, |
|
"logits/rejected": -1.587357759475708, |
|
"logps/chosen": -0.8991877436637878, |
|
"logps/rejected": -2.31307053565979, |
|
"loss": 1.1455, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.247969388961792, |
|
"rewards/margins": 3.5347068309783936, |
|
"rewards/rejected": -5.7826762199401855, |
|
"step": 101 |
|
}, |
|
{ |
|
"epoch": 0.02851950230672445, |
|
"grad_norm": 129.77518240929777, |
|
"learning_rate": 5.695865504800327e-07, |
|
"logits/chosen": -0.8380779027938843, |
|
"logits/rejected": -1.340008020401001, |
|
"logps/chosen": -1.1220433712005615, |
|
"logps/rejected": -1.7773716449737549, |
|
"loss": 0.8694, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.8051083087921143, |
|
"rewards/margins": 1.6383205652236938, |
|
"rewards/rejected": -4.443428993225098, |
|
"step": 102 |
|
}, |
|
{ |
|
"epoch": 0.02879910527051587, |
|
"grad_norm": 172.3569574432423, |
|
"learning_rate": 5.609346717025737e-07, |
|
"logits/chosen": -1.052437424659729, |
|
"logits/rejected": -1.4004924297332764, |
|
"logps/chosen": -0.9889116287231445, |
|
"logps/rejected": -1.5602869987487793, |
|
"loss": 1.7629, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.4722790718078613, |
|
"rewards/margins": 1.428438425064087, |
|
"rewards/rejected": -3.9007177352905273, |
|
"step": 103 |
|
}, |
|
{ |
|
"epoch": 0.029078708234307284, |
|
"grad_norm": 36.623523960138435, |
|
"learning_rate": 5.522642316338268e-07, |
|
"logits/chosen": -1.1740097999572754, |
|
"logits/rejected": -0.980044424533844, |
|
"logps/chosen": -1.2009364366531372, |
|
"logps/rejected": -1.3514597415924072, |
|
"loss": 1.2361, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -3.0023412704467773, |
|
"rewards/margins": 0.3763081431388855, |
|
"rewards/rejected": -3.3786492347717285, |
|
"step": 104 |
|
}, |
|
{ |
|
"epoch": 0.0293583111980987, |
|
"grad_norm": 77.37683529524608, |
|
"learning_rate": 5.435778713738292e-07, |
|
"logits/chosen": -1.2499001026153564, |
|
"logits/rejected": -1.2681267261505127, |
|
"logps/chosen": -1.7606796026229858, |
|
"logps/rejected": -2.930025577545166, |
|
"loss": 1.1121, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -4.401699066162109, |
|
"rewards/margins": 2.9233648777008057, |
|
"rewards/rejected": -7.325064182281494, |
|
"step": 105 |
|
}, |
|
{ |
|
"epoch": 0.029637914161890117, |
|
"grad_norm": 147.76082371794118, |
|
"learning_rate": 5.348782368720625e-07, |
|
"logits/chosen": -0.7781787514686584, |
|
"logits/rejected": -1.1500039100646973, |
|
"logps/chosen": -1.7407994270324707, |
|
"logps/rejected": -5.4726667404174805, |
|
"loss": 1.2898, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -4.351998805999756, |
|
"rewards/margins": 9.329668045043945, |
|
"rewards/rejected": -13.68166732788086, |
|
"step": 106 |
|
}, |
|
{ |
|
"epoch": 0.029917517125681532, |
|
"grad_norm": 23.97972069856743, |
|
"learning_rate": 5.26167978121472e-07, |
|
"logits/chosen": -0.7871793508529663, |
|
"logits/rejected": -1.1882908344268799, |
|
"logps/chosen": -0.5037890672683716, |
|
"logps/rejected": -1.5715763568878174, |
|
"loss": 0.6191, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -1.2594726085662842, |
|
"rewards/margins": 2.669468402862549, |
|
"rewards/rejected": -3.928941249847412, |
|
"step": 107 |
|
}, |
|
{ |
|
"epoch": 0.030197120089472947, |
|
"grad_norm": 26.15247415705612, |
|
"learning_rate": 5.174497483512505e-07, |
|
"logits/chosen": -1.1062333583831787, |
|
"logits/rejected": -1.6174620389938354, |
|
"logps/chosen": -1.3177549839019775, |
|
"logps/rejected": -1.8563097715377808, |
|
"loss": 0.941, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -3.2943873405456543, |
|
"rewards/margins": 1.3463870286941528, |
|
"rewards/rejected": -4.640774726867676, |
|
"step": 108 |
|
}, |
|
{ |
|
"epoch": 0.030476723053264366, |
|
"grad_norm": 36.12906136492366, |
|
"learning_rate": 5.087262032186418e-07, |
|
"logits/chosen": -1.320471167564392, |
|
"logits/rejected": -1.2250688076019287, |
|
"logps/chosen": -1.1929512023925781, |
|
"logps/rejected": -1.7405405044555664, |
|
"loss": 0.6462, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.9823780059814453, |
|
"rewards/margins": 1.3689732551574707, |
|
"rewards/rejected": -4.351351261138916, |
|
"step": 109 |
|
}, |
|
{ |
|
"epoch": 0.03075632601705578, |
|
"grad_norm": 21.26364702784651, |
|
"learning_rate": 5e-07, |
|
"logits/chosen": -0.6797312498092651, |
|
"logits/rejected": -1.517352819442749, |
|
"logps/chosen": -1.253048062324524, |
|
"logps/rejected": -2.614882707595825, |
|
"loss": 0.7045, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -3.132620334625244, |
|
"rewards/margins": 3.4045865535736084, |
|
"rewards/rejected": -6.537206649780273, |
|
"step": 110 |
|
}, |
|
{ |
|
"epoch": 0.031035928980847196, |
|
"grad_norm": 44.22722460589391, |
|
"learning_rate": 4.912737967813582e-07, |
|
"logits/chosen": -0.8700337409973145, |
|
"logits/rejected": -1.606900930404663, |
|
"logps/chosen": -1.6089668273925781, |
|
"logps/rejected": -2.9295287132263184, |
|
"loss": 0.8187, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -4.022417068481445, |
|
"rewards/margins": 3.3014047145843506, |
|
"rewards/rejected": -7.323822021484375, |
|
"step": 111 |
|
}, |
|
{ |
|
"epoch": 0.03131553194463861, |
|
"grad_norm": 76.80253216440869, |
|
"learning_rate": 4.825502516487496e-07, |
|
"logits/chosen": -0.8108619451522827, |
|
"logits/rejected": -1.4683247804641724, |
|
"logps/chosen": -1.2750149965286255, |
|
"logps/rejected": -1.875692367553711, |
|
"loss": 1.1174, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -3.187537670135498, |
|
"rewards/margins": 1.501693606376648, |
|
"rewards/rejected": -4.689230918884277, |
|
"step": 112 |
|
}, |
|
{ |
|
"epoch": 0.03159513490843003, |
|
"grad_norm": 39.5980607890587, |
|
"learning_rate": 4.7383202187852804e-07, |
|
"logits/chosen": -1.134104609489441, |
|
"logits/rejected": -1.3199536800384521, |
|
"logps/chosen": -1.1517322063446045, |
|
"logps/rejected": -2.180983543395996, |
|
"loss": 1.4293, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.879330635070801, |
|
"rewards/margins": 2.5731282234191895, |
|
"rewards/rejected": -5.45245885848999, |
|
"step": 113 |
|
}, |
|
{ |
|
"epoch": 0.03187473787222145, |
|
"grad_norm": 159.282123008416, |
|
"learning_rate": 4.6512176312793735e-07, |
|
"logits/chosen": -1.2752604484558105, |
|
"logits/rejected": -1.6413378715515137, |
|
"logps/chosen": -0.9552395343780518, |
|
"logps/rejected": -2.1065025329589844, |
|
"loss": 1.8414, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -2.38809871673584, |
|
"rewards/margins": 2.878157377243042, |
|
"rewards/rejected": -5.266256332397461, |
|
"step": 114 |
|
}, |
|
{ |
|
"epoch": 0.03215434083601286, |
|
"grad_norm": 23.36900105938972, |
|
"learning_rate": 4.5642212862617085e-07, |
|
"logits/chosen": -1.099194049835205, |
|
"logits/rejected": -1.1428616046905518, |
|
"logps/chosen": -1.1194630861282349, |
|
"logps/rejected": -1.9647703170776367, |
|
"loss": 1.211, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -2.7986578941345215, |
|
"rewards/margins": 2.1132683753967285, |
|
"rewards/rejected": -4.911925792694092, |
|
"step": 115 |
|
}, |
|
{ |
|
"epoch": 0.03243394379980428, |
|
"grad_norm": 22.810174582850898, |
|
"learning_rate": 4.477357683661733e-07, |
|
"logits/chosen": -1.2737977504730225, |
|
"logits/rejected": -1.3265190124511719, |
|
"logps/chosen": -1.2955751419067383, |
|
"logps/rejected": -2.3141541481018066, |
|
"loss": 1.1194, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -3.2389378547668457, |
|
"rewards/margins": 2.546447515487671, |
|
"rewards/rejected": -5.785386085510254, |
|
"step": 116 |
|
}, |
|
{ |
|
"epoch": 0.0327135467635957, |
|
"grad_norm": 155.39469755085068, |
|
"learning_rate": 4.390653282974263e-07, |
|
"logits/chosen": -1.007920503616333, |
|
"logits/rejected": -1.2567858695983887, |
|
"logps/chosen": -1.3624416589736938, |
|
"logps/rejected": -1.7886241674423218, |
|
"loss": 1.4514, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -3.406104326248169, |
|
"rewards/margins": 1.0654561519622803, |
|
"rewards/rejected": -4.471560478210449, |
|
"step": 117 |
|
}, |
|
{ |
|
"epoch": 0.03299314972738711, |
|
"grad_norm": 28.639395965705603, |
|
"learning_rate": 4.304134495199674e-07, |
|
"logits/chosen": -0.8960734605789185, |
|
"logits/rejected": -0.5021066069602966, |
|
"logps/chosen": -0.928629994392395, |
|
"logps/rejected": -1.1891580820083618, |
|
"loss": 0.8406, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.3215749263763428, |
|
"rewards/margins": 0.651320219039917, |
|
"rewards/rejected": -2.9728951454162598, |
|
"step": 118 |
|
}, |
|
{ |
|
"epoch": 0.03327275269117853, |
|
"grad_norm": 43.81099025759765, |
|
"learning_rate": 4.2178276747988444e-07, |
|
"logits/chosen": -1.010365605354309, |
|
"logits/rejected": -1.6597810983657837, |
|
"logps/chosen": -1.1012680530548096, |
|
"logps/rejected": -3.7689075469970703, |
|
"loss": 1.0053, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -2.7531704902648926, |
|
"rewards/margins": 6.669097900390625, |
|
"rewards/rejected": -9.422268867492676, |
|
"step": 119 |
|
}, |
|
{ |
|
"epoch": 0.033552355654969945, |
|
"grad_norm": 50.65816817248901, |
|
"learning_rate": 4.131759111665348e-07, |
|
"logits/chosen": -0.8199977874755859, |
|
"logits/rejected": -1.105729341506958, |
|
"logps/chosen": -1.1784420013427734, |
|
"logps/rejected": -1.7748521566390991, |
|
"loss": 1.0382, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.9461052417755127, |
|
"rewards/margins": 1.4910252094268799, |
|
"rewards/rejected": -4.437130928039551, |
|
"step": 120 |
|
}, |
|
{ |
|
"epoch": 0.03383195861876136, |
|
"grad_norm": 135.269469018394, |
|
"learning_rate": 4.0459550231172755e-07, |
|
"logits/chosen": -1.0090067386627197, |
|
"logits/rejected": -1.2103219032287598, |
|
"logps/chosen": -1.3218321800231934, |
|
"logps/rejected": -1.8139877319335938, |
|
"loss": 1.3587, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -3.3045804500579834, |
|
"rewards/margins": 1.23038911819458, |
|
"rewards/rejected": -4.534969806671143, |
|
"step": 121 |
|
}, |
|
{ |
|
"epoch": 0.034111561582552775, |
|
"grad_norm": 39.54642498391402, |
|
"learning_rate": 3.960441545911204e-07, |
|
"logits/chosen": -0.7739511132240295, |
|
"logits/rejected": -1.1406965255737305, |
|
"logps/chosen": -1.6368045806884766, |
|
"logps/rejected": -1.7438288927078247, |
|
"loss": 1.1795, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -4.092011451721191, |
|
"rewards/margins": 0.2675609588623047, |
|
"rewards/rejected": -4.359572410583496, |
|
"step": 122 |
|
}, |
|
{ |
|
"epoch": 0.034391164546344194, |
|
"grad_norm": 100.27265763472896, |
|
"learning_rate": 3.8752447282806755e-07, |
|
"logits/chosen": -1.0782852172851562, |
|
"logits/rejected": -1.3873653411865234, |
|
"logps/chosen": -1.1215283870697021, |
|
"logps/rejected": -2.216892719268799, |
|
"loss": 1.3417, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.803821086883545, |
|
"rewards/margins": 2.738410472869873, |
|
"rewards/rejected": -5.542231559753418, |
|
"step": 123 |
|
}, |
|
{ |
|
"epoch": 0.034670767510135606, |
|
"grad_norm": 58.20614675830375, |
|
"learning_rate": 3.790390522001662e-07, |
|
"logits/chosen": -0.778974711894989, |
|
"logits/rejected": -1.2259283065795898, |
|
"logps/chosen": -1.4575412273406982, |
|
"logps/rejected": -2.0206165313720703, |
|
"loss": 0.7091, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -3.643852949142456, |
|
"rewards/margins": 1.4076881408691406, |
|
"rewards/rejected": -5.051541328430176, |
|
"step": 124 |
|
}, |
|
{ |
|
"epoch": 0.034950370473927024, |
|
"grad_norm": 14.601199122985598, |
|
"learning_rate": 3.7059047744873955e-07, |
|
"logits/chosen": -0.8316655158996582, |
|
"logits/rejected": -1.5047085285186768, |
|
"logps/chosen": -1.2063120603561401, |
|
"logps/rejected": -2.6588189601898193, |
|
"loss": 0.4785, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -3.015780448913574, |
|
"rewards/margins": 3.6312668323516846, |
|
"rewards/rejected": -6.64704704284668, |
|
"step": 125 |
|
}, |
|
{ |
|
"epoch": 0.03522997343771844, |
|
"grad_norm": 27.93055728093027, |
|
"learning_rate": 3.621813220915004e-07, |
|
"logits/chosen": -1.099701166152954, |
|
"logits/rejected": -1.5104920864105225, |
|
"logps/chosen": -1.2890055179595947, |
|
"logps/rejected": -2.336859703063965, |
|
"loss": 0.6663, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -3.2225139141082764, |
|
"rewards/margins": 2.6196351051330566, |
|
"rewards/rejected": -5.842149257659912, |
|
"step": 126 |
|
}, |
|
{ |
|
"epoch": 0.035509576401509854, |
|
"grad_norm": 38.5610788021146, |
|
"learning_rate": 3.5381414763863163e-07, |
|
"logits/chosen": -1.0062170028686523, |
|
"logits/rejected": -1.459620714187622, |
|
"logps/chosen": -1.9963696002960205, |
|
"logps/rejected": -2.577173948287964, |
|
"loss": 1.0607, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -4.990923881530762, |
|
"rewards/margins": 1.4520111083984375, |
|
"rewards/rejected": -6.442934989929199, |
|
"step": 127 |
|
}, |
|
{ |
|
"epoch": 0.03578917936530127, |
|
"grad_norm": 64.49312626407205, |
|
"learning_rate": 3.454915028125263e-07, |
|
"logits/chosen": -0.9179176092147827, |
|
"logits/rejected": -1.1796026229858398, |
|
"logps/chosen": -1.013648271560669, |
|
"logps/rejected": -1.823269248008728, |
|
"loss": 1.0723, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -2.534120559692383, |
|
"rewards/margins": 2.024052143096924, |
|
"rewards/rejected": -4.558172702789307, |
|
"step": 128 |
|
}, |
|
{ |
|
"epoch": 0.03606878232909269, |
|
"grad_norm": 36.00805300468051, |
|
"learning_rate": 3.3721592277142173e-07, |
|
"logits/chosen": -1.0871798992156982, |
|
"logits/rejected": -1.1635856628417969, |
|
"logps/chosen": -1.6440746784210205, |
|
"logps/rejected": -2.153071641921997, |
|
"loss": 1.0645, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -4.110186576843262, |
|
"rewards/margins": 1.2724924087524414, |
|
"rewards/rejected": -5.382678985595703, |
|
"step": 129 |
|
}, |
|
{ |
|
"epoch": 0.0363483852928841, |
|
"grad_norm": 18.299398093476555, |
|
"learning_rate": 3.2898992833716563e-07, |
|
"logits/chosen": -0.852850079536438, |
|
"logits/rejected": -1.2836261987686157, |
|
"logps/chosen": -1.0674067735671997, |
|
"logps/rejected": -2.2412667274475098, |
|
"loss": 0.8613, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.6685171127319336, |
|
"rewards/margins": 2.934649705886841, |
|
"rewards/rejected": -5.603166580200195, |
|
"step": 130 |
|
}, |
|
{ |
|
"epoch": 0.03662798825667552, |
|
"grad_norm": 52.07726812332011, |
|
"learning_rate": 3.2081602522734985e-07, |
|
"logits/chosen": -1.0853538513183594, |
|
"logits/rejected": -1.274430274963379, |
|
"logps/chosen": -1.4096601009368896, |
|
"logps/rejected": -1.5677156448364258, |
|
"loss": 1.65, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -3.5241503715515137, |
|
"rewards/margins": 0.3951385021209717, |
|
"rewards/rejected": -3.9192888736724854, |
|
"step": 131 |
|
}, |
|
{ |
|
"epoch": 0.03690759122046694, |
|
"grad_norm": 17.512442562016194, |
|
"learning_rate": 3.1269670329204393e-07, |
|
"logits/chosen": -1.1719293594360352, |
|
"logits/rejected": -0.9959172606468201, |
|
"logps/chosen": -1.1080865859985352, |
|
"logps/rejected": -2.0133416652679443, |
|
"loss": 0.7374, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.770216464996338, |
|
"rewards/margins": 2.2631378173828125, |
|
"rewards/rejected": -5.033354759216309, |
|
"step": 132 |
|
}, |
|
{ |
|
"epoch": 0.03718719418425835, |
|
"grad_norm": 684.8675181225219, |
|
"learning_rate": 3.0463443575536317e-07, |
|
"logits/chosen": -1.3861017227172852, |
|
"logits/rejected": -1.1517585515975952, |
|
"logps/chosen": -2.434314250946045, |
|
"logps/rejected": -1.6802737712860107, |
|
"loss": 2.4108, |
|
"rewards/accuracies": 0.25, |
|
"rewards/chosen": -6.085785388946533, |
|
"rewards/margins": -1.8851011991500854, |
|
"rewards/rejected": -4.200684070587158, |
|
"step": 133 |
|
}, |
|
{ |
|
"epoch": 0.03746679714804977, |
|
"grad_norm": 203.35746911786714, |
|
"learning_rate": 2.9663167846209996e-07, |
|
"logits/chosen": -0.6332231163978577, |
|
"logits/rejected": -1.2013721466064453, |
|
"logps/chosen": -0.8640561699867249, |
|
"logps/rejected": -1.9115948677062988, |
|
"loss": 1.0114, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.1601402759552, |
|
"rewards/margins": 2.618846893310547, |
|
"rewards/rejected": -4.778987407684326, |
|
"step": 134 |
|
}, |
|
{ |
|
"epoch": 0.03774640011184119, |
|
"grad_norm": 48.99403271441584, |
|
"learning_rate": 2.8869086912965036e-07, |
|
"logits/chosen": -1.1275267601013184, |
|
"logits/rejected": -1.376509189605713, |
|
"logps/chosen": -1.562412977218628, |
|
"logps/rejected": -3.6972084045410156, |
|
"loss": 0.8296, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -3.9060323238372803, |
|
"rewards/margins": 5.3369879722595215, |
|
"rewards/rejected": -9.243021011352539, |
|
"step": 135 |
|
}, |
|
{ |
|
"epoch": 0.0380260030756326, |
|
"grad_norm": 27.827124034061338, |
|
"learning_rate": 2.808144266054612e-07, |
|
"logits/chosen": -0.7335802912712097, |
|
"logits/rejected": -1.3153456449508667, |
|
"logps/chosen": -1.1802624464035034, |
|
"logps/rejected": -2.093515157699585, |
|
"loss": 1.0364, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -2.9506561756134033, |
|
"rewards/margins": 2.2831313610076904, |
|
"rewards/rejected": -5.233787536621094, |
|
"step": 136 |
|
}, |
|
{ |
|
"epoch": 0.03830560603942402, |
|
"grad_norm": 41.51238681345992, |
|
"learning_rate": 2.730047501302266e-07, |
|
"logits/chosen": -0.9400544166564941, |
|
"logits/rejected": -1.42076575756073, |
|
"logps/chosen": -1.4838162660598755, |
|
"logps/rejected": -2.123250722885132, |
|
"loss": 1.4503, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -3.709540843963623, |
|
"rewards/margins": 1.598586082458496, |
|
"rewards/rejected": -5.308126926422119, |
|
"step": 137 |
|
}, |
|
{ |
|
"epoch": 0.03858520900321544, |
|
"grad_norm": 22.114098288460713, |
|
"learning_rate": 2.6526421860705473e-07, |
|
"logits/chosen": -0.8485064506530762, |
|
"logits/rejected": -1.127300500869751, |
|
"logps/chosen": -1.4366434812545776, |
|
"logps/rejected": -1.9873528480529785, |
|
"loss": 0.6841, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -3.591608762741089, |
|
"rewards/margins": 1.3767732381820679, |
|
"rewards/rejected": -4.968381881713867, |
|
"step": 138 |
|
}, |
|
{ |
|
"epoch": 0.03886481196700685, |
|
"grad_norm": 20.628330217139027, |
|
"learning_rate": 2.575951898768315e-07, |
|
"logits/chosen": -1.1259043216705322, |
|
"logits/rejected": -1.3020617961883545, |
|
"logps/chosen": -1.4316058158874512, |
|
"logps/rejected": -2.6916065216064453, |
|
"loss": 0.845, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -3.579014301300049, |
|
"rewards/margins": 3.1500015258789062, |
|
"rewards/rejected": -6.729016304016113, |
|
"step": 139 |
|
}, |
|
{ |
|
"epoch": 0.03914441493079827, |
|
"grad_norm": 21.796146073482507, |
|
"learning_rate": 2.500000000000001e-07, |
|
"logits/chosen": -1.3260260820388794, |
|
"logits/rejected": -1.5544166564941406, |
|
"logps/chosen": -2.0802931785583496, |
|
"logps/rejected": -2.217221260070801, |
|
"loss": 1.0337, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -5.200733661651611, |
|
"rewards/margins": 0.3423195481300354, |
|
"rewards/rejected": -5.543053150177002, |
|
"step": 140 |
|
}, |
|
{ |
|
"epoch": 0.039424017894589686, |
|
"grad_norm": 18.878252064796683, |
|
"learning_rate": 2.424809625449729e-07, |
|
"logits/chosen": -0.8346128463745117, |
|
"logits/rejected": -1.0135633945465088, |
|
"logps/chosen": -1.6469535827636719, |
|
"logps/rejected": -2.0207669734954834, |
|
"loss": 0.379, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -4.11738395690918, |
|
"rewards/margins": 0.9345333576202393, |
|
"rewards/rejected": -5.05191707611084, |
|
"step": 141 |
|
}, |
|
{ |
|
"epoch": 0.0397036208583811, |
|
"grad_norm": 15.618943765371364, |
|
"learning_rate": 2.350403678833976e-07, |
|
"logits/chosen": -1.1391526460647583, |
|
"logits/rejected": -1.3611162900924683, |
|
"logps/chosen": -1.1667499542236328, |
|
"logps/rejected": -2.9371497631073, |
|
"loss": 0.363, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -2.916875123977661, |
|
"rewards/margins": 4.425999164581299, |
|
"rewards/rejected": -7.342874526977539, |
|
"step": 142 |
|
}, |
|
{ |
|
"epoch": 0.039983223822172516, |
|
"grad_norm": 67.95024500967284, |
|
"learning_rate": 2.2768048249248644e-07, |
|
"logits/chosen": -0.9159025549888611, |
|
"logits/rejected": -1.5903668403625488, |
|
"logps/chosen": -1.0491520166397095, |
|
"logps/rejected": -3.3000054359436035, |
|
"loss": 0.7062, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -2.622879981994629, |
|
"rewards/margins": 5.627133846282959, |
|
"rewards/rejected": -8.250014305114746, |
|
"step": 143 |
|
}, |
|
{ |
|
"epoch": 0.040262826785963934, |
|
"grad_norm": 24.48650844009346, |
|
"learning_rate": 2.2040354826462664e-07, |
|
"logits/chosen": -0.948284387588501, |
|
"logits/rejected": -1.2763409614562988, |
|
"logps/chosen": -1.060044527053833, |
|
"logps/rejected": -2.7557332515716553, |
|
"loss": 0.4762, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -2.650111436843872, |
|
"rewards/margins": 4.239221572875977, |
|
"rewards/rejected": -6.8893327713012695, |
|
"step": 144 |
|
}, |
|
{ |
|
"epoch": 0.040542429749755346, |
|
"grad_norm": 22.244412776379363, |
|
"learning_rate": 2.1321178182447709e-07, |
|
"logits/chosen": -0.8840318322181702, |
|
"logits/rejected": -0.11764158308506012, |
|
"logps/chosen": -1.242844581604004, |
|
"logps/rejected": -1.281248927116394, |
|
"loss": 0.9514, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -3.1071114540100098, |
|
"rewards/margins": 0.09601074457168579, |
|
"rewards/rejected": -3.20312237739563, |
|
"step": 145 |
|
}, |
|
{ |
|
"epoch": 0.040822032713546764, |
|
"grad_norm": 33.28893137790503, |
|
"learning_rate": 2.0610737385376348e-07, |
|
"logits/chosen": -1.2970675230026245, |
|
"logits/rejected": -1.1824913024902344, |
|
"logps/chosen": -1.404200553894043, |
|
"logps/rejected": -2.006523609161377, |
|
"loss": 0.9091, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -3.5105013847351074, |
|
"rewards/margins": 1.505807638168335, |
|
"rewards/rejected": -5.016308784484863, |
|
"step": 146 |
|
}, |
|
{ |
|
"epoch": 0.04110163567733818, |
|
"grad_norm": 62.45940480786672, |
|
"learning_rate": 1.990924884239758e-07, |
|
"logits/chosen": -0.7262787222862244, |
|
"logits/rejected": -0.9536744952201843, |
|
"logps/chosen": -1.3462529182434082, |
|
"logps/rejected": -2.189612865447998, |
|
"loss": 1.1719, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -3.3656325340270996, |
|
"rewards/margins": 2.1083993911743164, |
|
"rewards/rejected": -5.474031448364258, |
|
"step": 147 |
|
}, |
|
{ |
|
"epoch": 0.041381238641129595, |
|
"grad_norm": 14.195827731761723, |
|
"learning_rate": 1.9216926233717084e-07, |
|
"logits/chosen": -0.7084841132164001, |
|
"logits/rejected": -1.271304965019226, |
|
"logps/chosen": -1.093338131904602, |
|
"logps/rejected": -2.5351109504699707, |
|
"loss": 0.4317, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -2.7333450317382812, |
|
"rewards/margins": 3.6044318675994873, |
|
"rewards/rejected": -6.337777137756348, |
|
"step": 148 |
|
}, |
|
{ |
|
"epoch": 0.04166084160492101, |
|
"grad_norm": 30.52003756938978, |
|
"learning_rate": 1.8533980447508135e-07, |
|
"logits/chosen": -0.8182432055473328, |
|
"logits/rejected": -1.2552411556243896, |
|
"logps/chosen": -1.5775116682052612, |
|
"logps/rejected": -1.9700381755828857, |
|
"loss": 0.9821, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -3.9437789916992188, |
|
"rewards/margins": 0.9813161492347717, |
|
"rewards/rejected": -4.925095558166504, |
|
"step": 149 |
|
}, |
|
{ |
|
"epoch": 0.04194044456871243, |
|
"grad_norm": 53.98320994568875, |
|
"learning_rate": 1.7860619515673032e-07, |
|
"logits/chosen": -0.9220970869064331, |
|
"logits/rejected": -0.8754558563232422, |
|
"logps/chosen": -1.3505525588989258, |
|
"logps/rejected": -1.8634326457977295, |
|
"loss": 1.0404, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -3.3763811588287354, |
|
"rewards/margins": 1.282200574874878, |
|
"rewards/rejected": -4.658581733703613, |
|
"step": 150 |
|
}, |
|
{ |
|
"epoch": 0.04194044456871243, |
|
"eval_logits/chosen": -1.208183765411377, |
|
"eval_logits/rejected": -1.363861083984375, |
|
"eval_logps/chosen": -1.3818128108978271, |
|
"eval_logps/rejected": -2.246399164199829, |
|
"eval_loss": 0.9883071780204773, |
|
"eval_rewards/accuracies": 0.7545045018196106, |
|
"eval_rewards/chosen": -3.4545319080352783, |
|
"eval_rewards/margins": 2.161465644836426, |
|
"eval_rewards/rejected": -5.615997791290283, |
|
"eval_runtime": 343.7097, |
|
"eval_samples_per_second": 3.864, |
|
"eval_steps_per_second": 0.323, |
|
"step": 150 |
|
}, |
|
{ |
|
"epoch": 0.04222004753250384, |
|
"grad_norm": 220.36984630306912, |
|
"learning_rate": 1.7197048550474641e-07, |
|
"logits/chosen": -0.6710684299468994, |
|
"logits/rejected": -0.9249241948127747, |
|
"logps/chosen": -1.366889476776123, |
|
"logps/rejected": -3.0947508811950684, |
|
"loss": 1.8539, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -3.4172236919403076, |
|
"rewards/margins": 4.3196539878845215, |
|
"rewards/rejected": -7.73687744140625, |
|
"step": 151 |
|
}, |
|
{ |
|
"epoch": 0.04249965049629526, |
|
"grad_norm": 70.6167555600515, |
|
"learning_rate": 1.6543469682057104e-07, |
|
"logits/chosen": -1.052169919013977, |
|
"logits/rejected": -1.4687204360961914, |
|
"logps/chosen": -1.1982924938201904, |
|
"logps/rejected": -1.9964299201965332, |
|
"loss": 0.7174, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.9957308769226074, |
|
"rewards/margins": 1.9953436851501465, |
|
"rewards/rejected": -4.991075038909912, |
|
"step": 152 |
|
}, |
|
{ |
|
"epoch": 0.04277925346008668, |
|
"grad_norm": 14.934556421376993, |
|
"learning_rate": 1.590008199687508e-07, |
|
"logits/chosen": -1.2788257598876953, |
|
"logits/rejected": -1.3499538898468018, |
|
"logps/chosen": -1.4407329559326172, |
|
"logps/rejected": -2.7239699363708496, |
|
"loss": 0.964, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -3.601832389831543, |
|
"rewards/margins": 3.208092212677002, |
|
"rewards/rejected": -6.809924602508545, |
|
"step": 153 |
|
}, |
|
{ |
|
"epoch": 0.04305885642387809, |
|
"grad_norm": 34.17784688286548, |
|
"learning_rate": 1.5267081477050131e-07, |
|
"logits/chosen": -0.9582281112670898, |
|
"logits/rejected": -1.459122896194458, |
|
"logps/chosen": -1.7836196422576904, |
|
"logps/rejected": -1.8749334812164307, |
|
"loss": 0.774, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -4.459049224853516, |
|
"rewards/margins": 0.22828447818756104, |
|
"rewards/rejected": -4.687334060668945, |
|
"step": 154 |
|
}, |
|
{ |
|
"epoch": 0.04333845938766951, |
|
"grad_norm": 16.121695263258527, |
|
"learning_rate": 1.4644660940672627e-07, |
|
"logits/chosen": -0.6466392874717712, |
|
"logits/rejected": -1.4516868591308594, |
|
"logps/chosen": -1.236275315284729, |
|
"logps/rejected": -3.074626922607422, |
|
"loss": 0.8325, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -3.0906882286071777, |
|
"rewards/margins": 4.595878601074219, |
|
"rewards/rejected": -7.686567306518555, |
|
"step": 155 |
|
}, |
|
{ |
|
"epoch": 0.04361806235146093, |
|
"grad_norm": 59.092861255129954, |
|
"learning_rate": 1.4033009983067452e-07, |
|
"logits/chosen": -1.0997824668884277, |
|
"logits/rejected": -0.8644530773162842, |
|
"logps/chosen": -1.5466465950012207, |
|
"logps/rejected": -1.9648511409759521, |
|
"loss": 1.2282, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -3.8666162490844727, |
|
"rewards/margins": 1.0455113649368286, |
|
"rewards/rejected": -4.91212797164917, |
|
"step": 156 |
|
}, |
|
{ |
|
"epoch": 0.04389766531525234, |
|
"grad_norm": 124.7156154616911, |
|
"learning_rate": 1.3432314919041477e-07, |
|
"logits/chosen": -1.2434520721435547, |
|
"logits/rejected": -0.9779855608940125, |
|
"logps/chosen": -2.0951521396636963, |
|
"logps/rejected": -1.2733700275421143, |
|
"loss": 2.0045, |
|
"rewards/accuracies": 0.25, |
|
"rewards/chosen": -5.237880706787109, |
|
"rewards/margins": -2.054455280303955, |
|
"rewards/rejected": -3.183424949645996, |
|
"step": 157 |
|
}, |
|
{ |
|
"epoch": 0.04417726827904376, |
|
"grad_norm": 17.359377980307887, |
|
"learning_rate": 1.284275872613028e-07, |
|
"logits/chosen": -0.8045494556427002, |
|
"logits/rejected": -1.271446943283081, |
|
"logps/chosen": -1.310563087463379, |
|
"logps/rejected": -2.273303508758545, |
|
"loss": 0.6038, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -3.2764077186584473, |
|
"rewards/margins": 2.406850814819336, |
|
"rewards/rejected": -5.683258533477783, |
|
"step": 158 |
|
}, |
|
{ |
|
"epoch": 0.04445687124283517, |
|
"grad_norm": 41.376354047075345, |
|
"learning_rate": 1.22645209888614e-07, |
|
"logits/chosen": -0.7052321434020996, |
|
"logits/rejected": -1.4795576333999634, |
|
"logps/chosen": -1.3022897243499756, |
|
"logps/rejected": -2.3771719932556152, |
|
"loss": 0.9528, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -3.2557244300842285, |
|
"rewards/margins": 2.6872057914733887, |
|
"rewards/rejected": -5.942930221557617, |
|
"step": 159 |
|
}, |
|
{ |
|
"epoch": 0.04473647420662659, |
|
"grad_norm": 45.99787159929172, |
|
"learning_rate": 1.1697777844051104e-07, |
|
"logits/chosen": -0.9138689041137695, |
|
"logits/rejected": -1.5214874744415283, |
|
"logps/chosen": -1.44037663936615, |
|
"logps/rejected": -2.119154214859009, |
|
"loss": 1.4147, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -3.6009416580200195, |
|
"rewards/margins": 1.696943759918213, |
|
"rewards/rejected": -5.297885417938232, |
|
"step": 160 |
|
}, |
|
{ |
|
"epoch": 0.04501607717041801, |
|
"grad_norm": 39.8925791182464, |
|
"learning_rate": 1.1142701927151454e-07, |
|
"logits/chosen": -1.4353740215301514, |
|
"logits/rejected": -1.4286375045776367, |
|
"logps/chosen": -1.4594804048538208, |
|
"logps/rejected": -2.6095705032348633, |
|
"loss": 0.7317, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -3.6487011909484863, |
|
"rewards/margins": 2.87522554397583, |
|
"rewards/rejected": -6.523926734924316, |
|
"step": 161 |
|
}, |
|
{ |
|
"epoch": 0.04529568013420942, |
|
"grad_norm": 31.35281628244166, |
|
"learning_rate": 1.0599462319663904e-07, |
|
"logits/chosen": -1.1067039966583252, |
|
"logits/rejected": -1.1960399150848389, |
|
"logps/chosen": -1.6614606380462646, |
|
"logps/rejected": -2.211236000061035, |
|
"loss": 0.9855, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -4.153651714324951, |
|
"rewards/margins": 1.374437928199768, |
|
"rewards/rejected": -5.52808952331543, |
|
"step": 162 |
|
}, |
|
{ |
|
"epoch": 0.04557528309800084, |
|
"grad_norm": 75.21576214554203, |
|
"learning_rate": 1.0068224497635369e-07, |
|
"logits/chosen": -1.2265667915344238, |
|
"logits/rejected": -1.0434250831604004, |
|
"logps/chosen": -1.1752564907073975, |
|
"logps/rejected": -2.1756644248962402, |
|
"loss": 0.7941, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -2.9381415843963623, |
|
"rewards/margins": 2.501019239425659, |
|
"rewards/rejected": -5.4391608238220215, |
|
"step": 163 |
|
}, |
|
{ |
|
"epoch": 0.045854886061792256, |
|
"grad_norm": 93.40285204475232, |
|
"learning_rate": 9.549150281252632e-08, |
|
"logits/chosen": -0.831551194190979, |
|
"logits/rejected": -0.7609004974365234, |
|
"logps/chosen": -1.2593779563903809, |
|
"logps/rejected": -1.790838599205017, |
|
"loss": 1.5802, |
|
"rewards/accuracies": 0.25, |
|
"rewards/chosen": -3.1484451293945312, |
|
"rewards/margins": 1.3286514282226562, |
|
"rewards/rejected": -4.4770965576171875, |
|
"step": 164 |
|
}, |
|
{ |
|
"epoch": 0.04613448902558367, |
|
"grad_norm": 41.48157239220995, |
|
"learning_rate": 9.042397785550404e-08, |
|
"logits/chosen": -0.6658945679664612, |
|
"logits/rejected": -0.9514182806015015, |
|
"logps/chosen": -1.3134242296218872, |
|
"logps/rejected": -1.9456892013549805, |
|
"loss": 0.5563, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -3.2835607528686523, |
|
"rewards/margins": 1.580662488937378, |
|
"rewards/rejected": -4.864223003387451, |
|
"step": 165 |
|
}, |
|
{ |
|
"epoch": 0.046414091989375086, |
|
"grad_norm": 49.840157280127464, |
|
"learning_rate": 8.548121372247919e-08, |
|
"logits/chosen": -0.9251920580863953, |
|
"logits/rejected": -1.4082067012786865, |
|
"logps/chosen": -1.3225380182266235, |
|
"logps/rejected": -3.200735092163086, |
|
"loss": 0.7022, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -3.306344985961914, |
|
"rewards/margins": 4.695493221282959, |
|
"rewards/rejected": -8.001838684082031, |
|
"step": 166 |
|
}, |
|
{ |
|
"epoch": 0.046693694953166505, |
|
"grad_norm": 118.05732935497699, |
|
"learning_rate": 8.066471602728803e-08, |
|
"logits/chosen": -1.2742832899093628, |
|
"logits/rejected": -1.172432541847229, |
|
"logps/chosen": -1.6827492713928223, |
|
"logps/rejected": -1.412048578262329, |
|
"loss": 1.6709, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -4.206873416900635, |
|
"rewards/margins": -0.6767516732215881, |
|
"rewards/rejected": -3.5301215648651123, |
|
"step": 167 |
|
}, |
|
{ |
|
"epoch": 0.046973297916957917, |
|
"grad_norm": 16.707525761245098, |
|
"learning_rate": 7.597595192178702e-08, |
|
"logits/chosen": -1.2708326578140259, |
|
"logits/rejected": -1.250439167022705, |
|
"logps/chosen": -0.9043286442756653, |
|
"logps/rejected": -2.5250792503356934, |
|
"loss": 0.4602, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -2.2608213424682617, |
|
"rewards/margins": 4.051877021789551, |
|
"rewards/rejected": -6.3126983642578125, |
|
"step": 168 |
|
}, |
|
{ |
|
"epoch": 0.047252900880749335, |
|
"grad_norm": 43.907524013668834, |
|
"learning_rate": 7.141634964894388e-08, |
|
"logits/chosen": -0.9978333711624146, |
|
"logits/rejected": -1.234339714050293, |
|
"logps/chosen": -1.7665379047393799, |
|
"logps/rejected": -2.1084702014923096, |
|
"loss": 0.9423, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -4.41634464263916, |
|
"rewards/margins": 0.8548306226730347, |
|
"rewards/rejected": -5.271175384521484, |
|
"step": 169 |
|
}, |
|
{ |
|
"epoch": 0.047532503844540754, |
|
"grad_norm": 88.65537871518796, |
|
"learning_rate": 6.698729810778064e-08, |
|
"logits/chosen": -0.8296370506286621, |
|
"logits/rejected": -0.7557154297828674, |
|
"logps/chosen": -1.1506996154785156, |
|
"logps/rejected": -1.4387410879135132, |
|
"loss": 1.2953, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.876749038696289, |
|
"rewards/margins": 0.7201035022735596, |
|
"rewards/rejected": -3.5968525409698486, |
|
"step": 170 |
|
}, |
|
{ |
|
"epoch": 0.047812106808332165, |
|
"grad_norm": 35.86474595345899, |
|
"learning_rate": 6.269014643030212e-08, |
|
"logits/chosen": -1.3039546012878418, |
|
"logits/rejected": -1.6143985986709595, |
|
"logps/chosen": -2.294726610183716, |
|
"logps/rejected": -2.714628219604492, |
|
"loss": 0.951, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -5.73681640625, |
|
"rewards/margins": 1.0497541427612305, |
|
"rewards/rejected": -6.7865705490112305, |
|
"step": 171 |
|
}, |
|
{ |
|
"epoch": 0.048091709772123584, |
|
"grad_norm": 47.72518452969935, |
|
"learning_rate": 5.8526203570536504e-08, |
|
"logits/chosen": -1.2507563829421997, |
|
"logits/rejected": -1.449178695678711, |
|
"logps/chosen": -1.8437649011611938, |
|
"logps/rejected": -3.4956912994384766, |
|
"loss": 1.271, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -4.60941219329834, |
|
"rewards/margins": 4.129816055297852, |
|
"rewards/rejected": -8.739228248596191, |
|
"step": 172 |
|
}, |
|
{ |
|
"epoch": 0.048371312735915, |
|
"grad_norm": 45.10015403750292, |
|
"learning_rate": 5.44967379058161e-08, |
|
"logits/chosen": -1.5573780536651611, |
|
"logits/rejected": -1.247377634048462, |
|
"logps/chosen": -0.8558065891265869, |
|
"logps/rejected": -2.1532318592071533, |
|
"loss": 0.7389, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -2.1395163536071777, |
|
"rewards/margins": 3.243563175201416, |
|
"rewards/rejected": -5.383079528808594, |
|
"step": 173 |
|
}, |
|
{ |
|
"epoch": 0.048650915699706414, |
|
"grad_norm": 126.90491606998833, |
|
"learning_rate": 5.060297685041659e-08, |
|
"logits/chosen": -1.0825918912887573, |
|
"logits/rejected": -1.1961251497268677, |
|
"logps/chosen": -1.5061373710632324, |
|
"logps/rejected": -2.8128504753112793, |
|
"loss": 1.415, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -3.76534366607666, |
|
"rewards/margins": 3.266782283782959, |
|
"rewards/rejected": -7.032125949859619, |
|
"step": 174 |
|
}, |
|
{ |
|
"epoch": 0.04893051866349783, |
|
"grad_norm": 104.83325539903379, |
|
"learning_rate": 4.684610648167503e-08, |
|
"logits/chosen": -0.2594190239906311, |
|
"logits/rejected": -1.2538113594055176, |
|
"logps/chosen": -0.8970815539360046, |
|
"logps/rejected": -3.357142925262451, |
|
"loss": 0.7837, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -2.242703914642334, |
|
"rewards/margins": 6.150154113769531, |
|
"rewards/rejected": -8.392858505249023, |
|
"step": 175 |
|
}, |
|
{ |
|
"epoch": 0.04921012162728925, |
|
"grad_norm": 25.391300351126056, |
|
"learning_rate": 4.322727117869951e-08, |
|
"logits/chosen": -1.0320615768432617, |
|
"logits/rejected": -1.3764903545379639, |
|
"logps/chosen": -1.2408421039581299, |
|
"logps/rejected": -3.5107479095458984, |
|
"loss": 0.701, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -3.102104902267456, |
|
"rewards/margins": 5.674765110015869, |
|
"rewards/rejected": -8.776870727539062, |
|
"step": 176 |
|
}, |
|
{ |
|
"epoch": 0.04948972459108066, |
|
"grad_norm": 100.44588632718167, |
|
"learning_rate": 3.974757327377981e-08, |
|
"logits/chosen": -1.1055123805999756, |
|
"logits/rejected": -0.956331729888916, |
|
"logps/chosen": -1.6190693378448486, |
|
"logps/rejected": -2.0674889087677, |
|
"loss": 0.8604, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -4.047673225402832, |
|
"rewards/margins": 1.1210490465164185, |
|
"rewards/rejected": -5.168722629547119, |
|
"step": 177 |
|
}, |
|
{ |
|
"epoch": 0.04976932755487208, |
|
"grad_norm": 23.973360234647668, |
|
"learning_rate": 3.6408072716606345e-08, |
|
"logits/chosen": -0.6234188079833984, |
|
"logits/rejected": -0.7233335375785828, |
|
"logps/chosen": -1.2935428619384766, |
|
"logps/rejected": -1.6678102016448975, |
|
"loss": 1.0504, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -3.2338571548461914, |
|
"rewards/margins": 0.9356681704521179, |
|
"rewards/rejected": -4.169525146484375, |
|
"step": 178 |
|
}, |
|
{ |
|
"epoch": 0.0500489305186635, |
|
"grad_norm": 128.60647901400966, |
|
"learning_rate": 3.3209786751399184e-08, |
|
"logits/chosen": -0.6642372608184814, |
|
"logits/rejected": -1.3921313285827637, |
|
"logps/chosen": -1.1783480644226074, |
|
"logps/rejected": -4.460213661193848, |
|
"loss": 1.2305, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -2.9458699226379395, |
|
"rewards/margins": 8.20466423034668, |
|
"rewards/rejected": -11.150533676147461, |
|
"step": 179 |
|
}, |
|
{ |
|
"epoch": 0.05032853348245491, |
|
"grad_norm": 45.48438050770713, |
|
"learning_rate": 3.015368960704584e-08, |
|
"logits/chosen": -0.753233790397644, |
|
"logits/rejected": -0.880287766456604, |
|
"logps/chosen": -1.1847925186157227, |
|
"logps/rejected": -2.643932342529297, |
|
"loss": 1.6424, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.9619812965393066, |
|
"rewards/margins": 3.6478493213653564, |
|
"rewards/rejected": -6.609830856323242, |
|
"step": 180 |
|
}, |
|
{ |
|
"epoch": 0.05060813644624633, |
|
"grad_norm": 43.3062398465464, |
|
"learning_rate": 2.7240712200341576e-08, |
|
"logits/chosen": -1.2592175006866455, |
|
"logits/rejected": -1.2679963111877441, |
|
"logps/chosen": -1.8530656099319458, |
|
"logps/rejected": -1.859635353088379, |
|
"loss": 1.8695, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -4.632663726806641, |
|
"rewards/margins": 0.01642441749572754, |
|
"rewards/rejected": -4.649088382720947, |
|
"step": 181 |
|
}, |
|
{ |
|
"epoch": 0.05088773941003775, |
|
"grad_norm": 34.019147990522235, |
|
"learning_rate": 2.4471741852423233e-08, |
|
"logits/chosen": -1.0714620351791382, |
|
"logits/rejected": -1.2085020542144775, |
|
"logps/chosen": -1.3924332857131958, |
|
"logps/rejected": -1.9933589696884155, |
|
"loss": 1.1236, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -3.481083393096924, |
|
"rewards/margins": 1.5023138523101807, |
|
"rewards/rejected": -4.983397483825684, |
|
"step": 182 |
|
}, |
|
{ |
|
"epoch": 0.05116734237382916, |
|
"grad_norm": 38.25765530969567, |
|
"learning_rate": 2.1847622018482282e-08, |
|
"logits/chosen": -1.323840856552124, |
|
"logits/rejected": -1.2305158376693726, |
|
"logps/chosen": -1.737386703491211, |
|
"logps/rejected": -2.449878692626953, |
|
"loss": 1.3231, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -4.343466758728027, |
|
"rewards/margins": 1.7812304496765137, |
|
"rewards/rejected": -6.124697208404541, |
|
"step": 183 |
|
}, |
|
{ |
|
"epoch": 0.05144694533762058, |
|
"grad_norm": 21.832900395742424, |
|
"learning_rate": 1.936915203084055e-08, |
|
"logits/chosen": -1.102403163909912, |
|
"logits/rejected": -0.8794164061546326, |
|
"logps/chosen": -1.1961630582809448, |
|
"logps/rejected": -2.0587728023529053, |
|
"loss": 0.5583, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -2.990407705307007, |
|
"rewards/margins": 2.156524181365967, |
|
"rewards/rejected": -5.146932125091553, |
|
"step": 184 |
|
}, |
|
{ |
|
"epoch": 0.051726548301412, |
|
"grad_norm": 97.17645669564288, |
|
"learning_rate": 1.7037086855465898e-08, |
|
"logits/chosen": -0.8995164036750793, |
|
"logits/rejected": -1.224024772644043, |
|
"logps/chosen": -1.5391126871109009, |
|
"logps/rejected": -3.2247936725616455, |
|
"loss": 1.0714, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -3.8477818965911865, |
|
"rewards/margins": 4.214201927185059, |
|
"rewards/rejected": -8.061984062194824, |
|
"step": 185 |
|
}, |
|
{ |
|
"epoch": 0.05200615126520341, |
|
"grad_norm": 46.501514059890596, |
|
"learning_rate": 1.4852136862001763e-08, |
|
"logits/chosen": -1.3521168231964111, |
|
"logits/rejected": -1.7020211219787598, |
|
"logps/chosen": -1.7328448295593262, |
|
"logps/rejected": -2.3388752937316895, |
|
"loss": 1.4423, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -4.332111835479736, |
|
"rewards/margins": 1.5150765180587769, |
|
"rewards/rejected": -5.847188472747803, |
|
"step": 186 |
|
}, |
|
{ |
|
"epoch": 0.05228575422899483, |
|
"grad_norm": 43.90284836540054, |
|
"learning_rate": 1.2814967607382432e-08, |
|
"logits/chosen": -0.7001566886901855, |
|
"logits/rejected": -0.8944656848907471, |
|
"logps/chosen": -1.2563323974609375, |
|
"logps/rejected": -1.8060386180877686, |
|
"loss": 0.4237, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -3.1408309936523438, |
|
"rewards/margins": 1.374265432357788, |
|
"rewards/rejected": -4.515096664428711, |
|
"step": 187 |
|
}, |
|
{ |
|
"epoch": 0.052565357192786245, |
|
"grad_norm": 66.58807618416985, |
|
"learning_rate": 1.0926199633097154e-08, |
|
"logits/chosen": -1.131943941116333, |
|
"logits/rejected": -1.2059730291366577, |
|
"logps/chosen": -1.1455706357955933, |
|
"logps/rejected": -1.936645746231079, |
|
"loss": 0.7375, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -2.863926649093628, |
|
"rewards/margins": 1.9776880741119385, |
|
"rewards/rejected": -4.841614723205566, |
|
"step": 188 |
|
}, |
|
{ |
|
"epoch": 0.05284496015657766, |
|
"grad_norm": 36.45672997200926, |
|
"learning_rate": 9.186408276168011e-09, |
|
"logits/chosen": -0.9764798879623413, |
|
"logits/rejected": -1.0748597383499146, |
|
"logps/chosen": -1.0145312547683716, |
|
"logps/rejected": -2.6933655738830566, |
|
"loss": 1.0416, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -2.5363283157348633, |
|
"rewards/margins": 4.197086334228516, |
|
"rewards/rejected": -6.733414649963379, |
|
"step": 189 |
|
}, |
|
{ |
|
"epoch": 0.053124563120369075, |
|
"grad_norm": 23.88892255070114, |
|
"learning_rate": 7.59612349389599e-09, |
|
"logits/chosen": -0.809168815612793, |
|
"logits/rejected": -1.1629016399383545, |
|
"logps/chosen": -0.9837249517440796, |
|
"logps/rejected": -1.9994207620620728, |
|
"loss": 0.603, |
|
"rewards/accuracies": 1.0, |
|
"rewards/chosen": -2.4593124389648438, |
|
"rewards/margins": 2.5392391681671143, |
|
"rewards/rejected": -4.998551845550537, |
|
"step": 190 |
|
}, |
|
{ |
|
"epoch": 0.053404166084160494, |
|
"grad_norm": 38.039713749484235, |
|
"learning_rate": 6.15582970243117e-09, |
|
"logits/chosen": -1.061342716217041, |
|
"logits/rejected": -1.2929720878601074, |
|
"logps/chosen": -1.4455804824829102, |
|
"logps/rejected": -1.7912752628326416, |
|
"loss": 1.2204, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -3.6139512062072754, |
|
"rewards/margins": 0.8642373085021973, |
|
"rewards/rejected": -4.478188514709473, |
|
"step": 191 |
|
}, |
|
{ |
|
"epoch": 0.053683769047951906, |
|
"grad_norm": 33.26330215420443, |
|
"learning_rate": 4.865965629214819e-09, |
|
"logits/chosen": -0.7917881608009338, |
|
"logits/rejected": -1.4771734476089478, |
|
"logps/chosen": -1.3420734405517578, |
|
"logps/rejected": -2.956531047821045, |
|
"loss": 1.2029, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -3.3551833629608154, |
|
"rewards/margins": 4.036144256591797, |
|
"rewards/rejected": -7.391327381134033, |
|
"step": 192 |
|
}, |
|
{ |
|
"epoch": 0.053963372011743324, |
|
"grad_norm": 50.418507707166555, |
|
"learning_rate": 3.7269241793390084e-09, |
|
"logits/chosen": -0.7339926362037659, |
|
"logits/rejected": -0.8001390695571899, |
|
"logps/chosen": -1.375474214553833, |
|
"logps/rejected": -1.5513949394226074, |
|
"loss": 1.4448, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -3.438685655593872, |
|
"rewards/margins": 0.43980175256729126, |
|
"rewards/rejected": -3.8784873485565186, |
|
"step": 193 |
|
}, |
|
{ |
|
"epoch": 0.05424297497553474, |
|
"grad_norm": 46.99311043278277, |
|
"learning_rate": 2.739052315863355e-09, |
|
"logits/chosen": -1.1927969455718994, |
|
"logits/rejected": -1.1135954856872559, |
|
"logps/chosen": -1.4356844425201416, |
|
"logps/rejected": -1.4172120094299316, |
|
"loss": 1.2511, |
|
"rewards/accuracies": 0.25, |
|
"rewards/chosen": -3.5892109870910645, |
|
"rewards/margins": -0.046180903911590576, |
|
"rewards/rejected": -3.54302978515625, |
|
"step": 194 |
|
}, |
|
{ |
|
"epoch": 0.054522577939326154, |
|
"grad_norm": 50.89269657493953, |
|
"learning_rate": 1.9026509541272273e-09, |
|
"logits/chosen": -1.0880765914916992, |
|
"logits/rejected": -1.1170682907104492, |
|
"logps/chosen": -1.426069974899292, |
|
"logps/rejected": -2.2877893447875977, |
|
"loss": 1.1481, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -3.5651750564575195, |
|
"rewards/margins": 2.1542983055114746, |
|
"rewards/rejected": -5.719472885131836, |
|
"step": 195 |
|
}, |
|
{ |
|
"epoch": 0.05480218090311757, |
|
"grad_norm": 119.28075830322265, |
|
"learning_rate": 1.217974870087901e-09, |
|
"logits/chosen": -1.1547623872756958, |
|
"logits/rejected": -1.3626446723937988, |
|
"logps/chosen": -1.1919413805007935, |
|
"logps/rejected": -2.2724056243896484, |
|
"loss": 1.446, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.979853391647339, |
|
"rewards/margins": 2.7011606693267822, |
|
"rewards/rejected": -5.681014060974121, |
|
"step": 196 |
|
}, |
|
{ |
|
"epoch": 0.05508178386690899, |
|
"grad_norm": 48.79918657535743, |
|
"learning_rate": 6.852326227130833e-10, |
|
"logits/chosen": -1.2602670192718506, |
|
"logits/rejected": -1.142798662185669, |
|
"logps/chosen": -1.5842217206954956, |
|
"logps/rejected": -1.682457685470581, |
|
"loss": 0.7233, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -3.9605541229248047, |
|
"rewards/margins": 0.2455899715423584, |
|
"rewards/rejected": -4.206143856048584, |
|
"step": 197 |
|
}, |
|
{ |
|
"epoch": 0.0553613868307004, |
|
"grad_norm": 20.938919943310346, |
|
"learning_rate": 3.0458649045211894e-10, |
|
"logits/chosen": -0.6655712127685547, |
|
"logits/rejected": -0.7740037441253662, |
|
"logps/chosen": -1.6575719118118286, |
|
"logps/rejected": -1.8176357746124268, |
|
"loss": 1.2588, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": -4.143929958343506, |
|
"rewards/margins": 0.4001593291759491, |
|
"rewards/rejected": -4.544089317321777, |
|
"step": 198 |
|
}, |
|
{ |
|
"epoch": 0.05564098979449182, |
|
"grad_norm": 25.414963350937853, |
|
"learning_rate": 7.615242180436521e-11, |
|
"logits/chosen": -1.2654355764389038, |
|
"logits/rejected": -1.2140486240386963, |
|
"logps/chosen": -1.4302688837051392, |
|
"logps/rejected": -2.340785026550293, |
|
"loss": 0.8395, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -3.575672149658203, |
|
"rewards/margins": 2.276289939880371, |
|
"rewards/rejected": -5.851962089538574, |
|
"step": 199 |
|
}, |
|
{ |
|
"epoch": 0.05592059275828324, |
|
"grad_norm": 82.36002044117453, |
|
"learning_rate": 0.0, |
|
"logits/chosen": -1.2721612453460693, |
|
"logits/rejected": -0.8177624940872192, |
|
"logps/chosen": -1.314448595046997, |
|
"logps/rejected": -1.1094343662261963, |
|
"loss": 1.4672, |
|
"rewards/accuracies": 0.25, |
|
"rewards/chosen": -3.286121368408203, |
|
"rewards/margins": -0.5125355124473572, |
|
"rewards/rejected": -2.773585796356201, |
|
"step": 200 |
|
}, |
|
{ |
|
"epoch": 0.05592059275828324, |
|
"eval_logits/chosen": -1.2117185592651367, |
|
"eval_logits/rejected": -1.3683098554611206, |
|
"eval_logps/chosen": -1.3815547227859497, |
|
"eval_logps/rejected": -2.264155149459839, |
|
"eval_loss": 0.9751168489456177, |
|
"eval_rewards/accuracies": 0.7612612843513489, |
|
"eval_rewards/chosen": -3.4538867473602295, |
|
"eval_rewards/margins": 2.206501007080078, |
|
"eval_rewards/rejected": -5.66038703918457, |
|
"eval_runtime": 344.9671, |
|
"eval_samples_per_second": 3.85, |
|
"eval_steps_per_second": 0.322, |
|
"step": 200 |
|
}, |
|
{ |
|
"epoch": 0.05592059275828324, |
|
"step": 200, |
|
"total_flos": 0.0, |
|
"train_loss": 1.163502004146576, |
|
"train_runtime": 4144.9448, |
|
"train_samples_per_second": 0.579, |
|
"train_steps_per_second": 0.048 |
|
} |
|
], |
|
"logging_steps": 1, |
|
"max_steps": 200, |
|
"num_input_tokens_seen": 0, |
|
"num_train_epochs": 1, |
|
"save_steps": 10000, |
|
"stateful_callbacks": { |
|
"TrainerControl": { |
|
"args": { |
|
"should_epoch_stop": false, |
|
"should_evaluate": false, |
|
"should_log": false, |
|
"should_save": false, |
|
"should_training_stop": false |
|
}, |
|
"attributes": {} |
|
} |
|
}, |
|
"total_flos": 0.0, |
|
"train_batch_size": 2, |
|
"trial_name": null, |
|
"trial_params": null |
|
} |
|
|