{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.998691442030882,
"eval_steps": 500,
"global_step": 477,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.010468463752944255,
"grad_norm": 0.1786131818333053,
"learning_rate": 6.25e-08,
"logits/chosen": -1.4481778144836426,
"logits/rejected": -1.4499433040618896,
"logps/chosen": -7.982362270355225,
"logps/rejected": -8.15577507019043,
"loss": -0.0009,
"rewards/accuracies": 0.5,
"rewards/chosen": -7.982362270355225,
"rewards/margins": 0.17341338098049164,
"rewards/rejected": -8.15577507019043,
"step": 5
},
{
"epoch": 0.02093692750588851,
"grad_norm": 0.07282553733728953,
"learning_rate": 1.25e-07,
"logits/chosen": -1.4464797973632812,
"logits/rejected": -1.4372261762619019,
"logps/chosen": -8.047597885131836,
"logps/rejected": -7.961185455322266,
"loss": 0.0001,
"rewards/accuracies": 0.4937500059604645,
"rewards/chosen": -8.047597885131836,
"rewards/margins": -0.08641364425420761,
"rewards/rejected": -7.961185455322266,
"step": 10
},
{
"epoch": 0.031405391258832765,
"grad_norm": 0.1915776567981181,
"learning_rate": 1.875e-07,
"logits/chosen": -1.4458509683609009,
"logits/rejected": -1.4425008296966553,
"logps/chosen": -7.852419853210449,
"logps/rejected": -7.86798095703125,
"loss": -0.0011,
"rewards/accuracies": 0.5,
"rewards/chosen": -7.852419853210449,
"rewards/margins": 0.01556050218641758,
"rewards/rejected": -7.86798095703125,
"step": 15
},
{
"epoch": 0.04187385501177702,
"grad_norm": 0.18043036746469734,
"learning_rate": 2.5e-07,
"logits/chosen": -1.433124303817749,
"logits/rejected": -1.4371713399887085,
"logps/chosen": -8.189096450805664,
"logps/rejected": -8.211885452270508,
"loss": 0.0005,
"rewards/accuracies": 0.5,
"rewards/chosen": -8.189096450805664,
"rewards/margins": 0.02278941310942173,
"rewards/rejected": -8.211885452270508,
"step": 20
},
{
"epoch": 0.05234231876472128,
"grad_norm": 0.0768554595715667,
"learning_rate": 3.125e-07,
"logits/chosen": -1.4702521562576294,
"logits/rejected": -1.46065354347229,
"logps/chosen": -8.118414878845215,
"logps/rejected": -8.017342567443848,
"loss": 0.0008,
"rewards/accuracies": 0.4375,
"rewards/chosen": -8.118414878845215,
"rewards/margins": -0.10107225179672241,
"rewards/rejected": -8.017342567443848,
"step": 25
},
{
"epoch": 0.06281078251766553,
"grad_norm": 0.15370408069863134,
"learning_rate": 3.75e-07,
"logits/chosen": -1.4349619150161743,
"logits/rejected": -1.4234455823898315,
"logps/chosen": -7.866227626800537,
"logps/rejected": -7.7843732833862305,
"loss": 0.0009,
"rewards/accuracies": 0.48750001192092896,
"rewards/chosen": -7.866227626800537,
"rewards/margins": -0.08185449987649918,
"rewards/rejected": -7.7843732833862305,
"step": 30
},
{
"epoch": 0.07327924627060979,
"grad_norm": 0.08486142616829283,
"learning_rate": 4.3749999999999994e-07,
"logits/chosen": -1.4395850896835327,
"logits/rejected": -1.4178800582885742,
"logps/chosen": -8.148027420043945,
"logps/rejected": -8.014989852905273,
"loss": -0.0007,
"rewards/accuracies": 0.5,
"rewards/chosen": -8.148027420043945,
"rewards/margins": -0.13303671777248383,
"rewards/rejected": -8.014989852905273,
"step": 35
},
{
"epoch": 0.08374771002355404,
"grad_norm": 0.1990381181235585,
"learning_rate": 5e-07,
"logits/chosen": -1.4183995723724365,
"logits/rejected": -1.4341777563095093,
"logps/chosen": -8.116990089416504,
"logps/rejected": -8.271265983581543,
"loss": -0.0007,
"rewards/accuracies": 0.4749999940395355,
"rewards/chosen": -8.116990089416504,
"rewards/margins": 0.15427525341510773,
"rewards/rejected": -8.271265983581543,
"step": 40
},
{
"epoch": 0.0942161737764983,
"grad_norm": 0.05622179554254164,
"learning_rate": 5.625e-07,
"logits/chosen": -1.4420310258865356,
"logits/rejected": -1.442714810371399,
"logps/chosen": -8.133280754089355,
"logps/rejected": -7.922200679779053,
"loss": -0.0001,
"rewards/accuracies": 0.4375,
"rewards/chosen": -8.133280754089355,
"rewards/margins": -0.21108034253120422,
"rewards/rejected": -7.922200679779053,
"step": 45
},
{
"epoch": 0.10468463752944256,
"grad_norm": 0.09631702855440245,
"learning_rate": 5.999678242522831e-07,
"logits/chosen": -1.4332040548324585,
"logits/rejected": -1.4520256519317627,
"logps/chosen": -8.272871017456055,
"logps/rejected": -8.261492729187012,
"loss": 0.0006,
"rewards/accuracies": 0.48124998807907104,
"rewards/chosen": -8.272871017456055,
"rewards/margins": -0.011378437280654907,
"rewards/rejected": -8.261492729187012,
"step": 50
},
{
"epoch": 0.11515310128238682,
"grad_norm": 0.06712571499202777,
"learning_rate": 5.996059263493219e-07,
"logits/chosen": -1.4460947513580322,
"logits/rejected": -1.4431495666503906,
"logps/chosen": -8.187493324279785,
"logps/rejected": -8.160319328308105,
"loss": 0.0,
"rewards/accuracies": 0.41874998807907104,
"rewards/chosen": -8.187493324279785,
"rewards/margins": -0.02717405930161476,
"rewards/rejected": -8.160319328308105,
"step": 55
},
{
"epoch": 0.12562156503533106,
"grad_norm": 0.20114381829735603,
"learning_rate": 5.988423976115163e-07,
"logits/chosen": -1.4432101249694824,
"logits/rejected": -1.455540657043457,
"logps/chosen": -8.201032638549805,
"logps/rejected": -8.437708854675293,
"loss": -0.0006,
"rewards/accuracies": 0.574999988079071,
"rewards/chosen": -8.201032638549805,
"rewards/margins": 0.23667626082897186,
"rewards/rejected": -8.437708854675293,
"step": 60
},
{
"epoch": 0.1360900287882753,
"grad_norm": 0.17507947570299737,
"learning_rate": 5.976782615723061e-07,
"logits/chosen": -1.3914668560028076,
"logits/rejected": -1.4124656915664673,
"logps/chosen": -8.03328800201416,
"logps/rejected": -8.427019119262695,
"loss": 0.0001,
"rewards/accuracies": 0.574999988079071,
"rewards/chosen": -8.03328800201416,
"rewards/margins": 0.393731027841568,
"rewards/rejected": -8.427019119262695,
"step": 65
},
{
"epoch": 0.14655849254121958,
"grad_norm": 0.28383551734545676,
"learning_rate": 5.961150787913738e-07,
"logits/chosen": -1.399139404296875,
"logits/rejected": -1.3929238319396973,
"logps/chosen": -8.119722366333008,
"logps/rejected": -8.134946823120117,
"loss": 0.0014,
"rewards/accuracies": 0.5062500238418579,
"rewards/chosen": -8.119722366333008,
"rewards/margins": 0.01522480882704258,
"rewards/rejected": -8.134946823120117,
"step": 70
},
{
"epoch": 0.15702695629416383,
"grad_norm": 0.05943092529446436,
"learning_rate": 5.941549447626671e-07,
"logits/chosen": -1.4151188135147095,
"logits/rejected": -1.4230639934539795,
"logps/chosen": -8.229839324951172,
"logps/rejected": -8.183255195617676,
"loss": -0.0001,
"rewards/accuracies": 0.4749999940395355,
"rewards/chosen": -8.229839324951172,
"rewards/margins": -0.04658409580588341,
"rewards/rejected": -8.183255195617676,
"step": 75
},
{
"epoch": 0.16749542004710807,
"grad_norm": 0.11166728026103465,
"learning_rate": 5.918004871053251e-07,
"logits/chosen": -1.431121587753296,
"logits/rejected": -1.4449329376220703,
"logps/chosen": -8.453977584838867,
"logps/rejected": -8.471723556518555,
"loss": -0.0007,
"rewards/accuracies": 0.4437499940395355,
"rewards/chosen": -8.453977584838867,
"rewards/margins": 0.017745357006788254,
"rewards/rejected": -8.471723556518555,
"step": 80
},
{
"epoch": 0.17796388380005235,
"grad_norm": 0.11291027423617692,
"learning_rate": 5.890548620412763e-07,
"logits/chosen": -1.4427986145019531,
"logits/rejected": -1.4420478343963623,
"logps/chosen": -8.645545959472656,
"logps/rejected": -8.712440490722656,
"loss": 0.0,
"rewards/accuracies": 0.4749999940395355,
"rewards/chosen": -8.645545959472656,
"rewards/margins": 0.06689504534006119,
"rewards/rejected": -8.712440490722656,
"step": 85
},
{
"epoch": 0.1884323475529966,
"grad_norm": 0.2075325531424713,
"learning_rate": 5.859217501642258e-07,
"logits/chosen": -1.431888222694397,
"logits/rejected": -1.4438416957855225,
"logps/chosen": -8.67556381225586,
"logps/rejected": -8.743408203125,
"loss": -0.0001,
"rewards/accuracies": 0.4937500059604645,
"rewards/chosen": -8.67556381225586,
"rewards/margins": 0.0678454041481018,
"rewards/rejected": -8.743408203125,
"step": 90
},
{
"epoch": 0.19890081130594087,
"grad_norm": 0.1350380165212238,
"learning_rate": 5.824053515057091e-07,
"logits/chosen": -1.4262675046920776,
"logits/rejected": -1.4206186532974243,
"logps/chosen": -8.711984634399414,
"logps/rejected": -8.466314315795898,
"loss": 0.0001,
"rewards/accuracies": 0.38749998807907104,
"rewards/chosen": -8.711984634399414,
"rewards/margins": -0.24566936492919922,
"rewards/rejected": -8.466314315795898,
"step": 95
},
{
"epoch": 0.2093692750588851,
"grad_norm": 0.056801258116621844,
"learning_rate": 5.785103799048218e-07,
"logits/chosen": -1.4670069217681885,
"logits/rejected": -1.4761860370635986,
"logps/chosen": -8.660918235778809,
"logps/rejected": -8.600263595581055,
"loss": 0.0004,
"rewards/accuracies": 0.4749999940395355,
"rewards/chosen": -8.660918235778809,
"rewards/margins": -0.060654301196336746,
"rewards/rejected": -8.600263595581055,
"step": 100
},
{
"epoch": 0.21983773881182936,
"grad_norm": 0.02647029175400585,
"learning_rate": 5.742420566891749e-07,
"logits/chosen": -1.4849385023117065,
"logits/rejected": -1.4768640995025635,
"logps/chosen": -8.584749221801758,
"logps/rejected": -8.679868698120117,
"loss": 0.0001,
"rewards/accuracies": 0.5249999761581421,
"rewards/chosen": -8.584749221801758,
"rewards/margins": 0.09512145817279816,
"rewards/rejected": -8.679868698120117,
"step": 105
},
{
"epoch": 0.23030620256477363,
"grad_norm": 0.14957708248219087,
"learning_rate": 5.696061036755478e-07,
"logits/chosen": -1.537284255027771,
"logits/rejected": -1.524402141571045,
"logps/chosen": -9.06226921081543,
"logps/rejected": -9.15291976928711,
"loss": -0.0,
"rewards/accuracies": 0.543749988079071,
"rewards/chosen": -9.06226921081543,
"rewards/margins": 0.09065041691064835,
"rewards/rejected": -9.15291976928711,
"step": 110
},
{
"epoch": 0.24077466631771788,
"grad_norm": 0.04538978337584183,
"learning_rate": 5.64608735499618e-07,
"logits/chosen": -1.500104546546936,
"logits/rejected": -1.4960343837738037,
"logps/chosen": -9.003273010253906,
"logps/rejected": -9.141524314880371,
"loss": 0.0001,
"rewards/accuracies": 0.5249999761581421,
"rewards/chosen": -9.003273010253906,
"rewards/margins": 0.1382521390914917,
"rewards/rejected": -9.141524314880371,
"step": 115
},
{
"epoch": 0.2512431300706621,
"grad_norm": 0.08148752548258974,
"learning_rate": 5.592566512850545e-07,
"logits/chosen": -1.5267969369888306,
"logits/rejected": -1.522019624710083,
"logps/chosen": -9.461637496948242,
"logps/rejected": -9.392511367797852,
"loss": 0.0003,
"rewards/accuracies": 0.512499988079071,
"rewards/chosen": -9.461637496948242,
"rewards/margins": -0.06912745535373688,
"rewards/rejected": -9.392511367797852,
"step": 120
},
{
"epoch": 0.26171159382360637,
"grad_norm": 0.011705138951069663,
"learning_rate": 5.535570256631384e-07,
"logits/chosen": -1.6196930408477783,
"logits/rejected": -1.6130282878875732,
"logps/chosen": -10.071660041809082,
"logps/rejected": -9.955024719238281,
"loss": 0.0003,
"rewards/accuracies": 0.48124998807907104,
"rewards/chosen": -10.071660041809082,
"rewards/margins": -0.11663500964641571,
"rewards/rejected": -9.955024719238281,
"step": 125
},
{
"epoch": 0.2721800575765506,
"grad_norm": 0.03311359707714934,
"learning_rate": 5.475174991549528e-07,
"logits/chosen": -1.6836683750152588,
"logits/rejected": -1.6870734691619873,
"logps/chosen": -10.576017379760742,
"logps/rejected": -10.587455749511719,
"loss": 0.0,
"rewards/accuracies": 0.48750001192092896,
"rewards/chosen": -10.576017379760742,
"rewards/margins": 0.011437964625656605,
"rewards/rejected": -10.587455749511719,
"step": 130
},
{
"epoch": 0.2826485213294949,
"grad_norm": 0.0028807868975943276,
"learning_rate": 5.411461679290317e-07,
"logits/chosen": -1.716474175453186,
"logits/rejected": -1.7054128646850586,
"logps/chosen": -10.966715812683105,
"logps/rejected": -11.17033576965332,
"loss": 0.0,
"rewards/accuracies": 0.518750011920929,
"rewards/chosen": -10.966715812683105,
"rewards/margins": 0.2036197930574417,
"rewards/rejected": -11.17033576965332,
"step": 135
},
{
"epoch": 0.29311698508243916,
"grad_norm": 0.0071240218588725325,
"learning_rate": 5.34451572948201e-07,
"logits/chosen": -1.7624616622924805,
"logits/rejected": -1.750759482383728,
"logps/chosen": -11.156391143798828,
"logps/rejected": -11.09487533569336,
"loss": -0.0,
"rewards/accuracies": 0.4749999940395355,
"rewards/chosen": -11.156391143798828,
"rewards/margins": -0.061515532433986664,
"rewards/rejected": -11.09487533569336,
"step": 140
},
{
"epoch": 0.3035854488353834,
"grad_norm": 0.007104778936656334,
"learning_rate": 5.274426885201582e-07,
"logits/chosen": -1.7734272480010986,
"logits/rejected": -1.7652689218521118,
"logps/chosen": -11.338696479797363,
"logps/rejected": -11.272669792175293,
"loss": 0.0001,
"rewards/accuracies": 0.518750011920929,
"rewards/chosen": -11.338696479797363,
"rewards/margins": -0.06602667272090912,
"rewards/rejected": -11.272669792175293,
"step": 145
},
{
"epoch": 0.31405391258832765,
"grad_norm": 0.0026785877800151164,
"learning_rate": 5.201289102671411e-07,
"logits/chosen": -1.7958892583847046,
"logits/rejected": -1.7981617450714111,
"logps/chosen": -11.584068298339844,
"logps/rejected": -11.576414108276367,
"loss": -0.0001,
"rewards/accuracies": 0.4312500059604645,
"rewards/chosen": -11.584068298339844,
"rewards/margins": -0.007653522305190563,
"rewards/rejected": -11.576414108276367,
"step": 150
},
{
"epoch": 0.3245223763412719,
"grad_norm": 0.032494261377218156,
"learning_rate": 5.12520042530811e-07,
"logits/chosen": -1.7825043201446533,
"logits/rejected": -1.7530311346054077,
"logps/chosen": -11.603002548217773,
"logps/rejected": -11.390130043029785,
"loss": 0.0,
"rewards/accuracies": 0.38749998807907104,
"rewards/chosen": -11.603002548217773,
"rewards/margins": -0.21287231147289276,
"rewards/rejected": -11.390130043029785,
"step": 155
},
{
"epoch": 0.33499084009421615,
"grad_norm": 0.011685158472598062,
"learning_rate": 5.046262852292346e-07,
"logits/chosen": -1.770146369934082,
"logits/rejected": -1.7625595331192017,
"logps/chosen": -11.565991401672363,
"logps/rejected": -11.481651306152344,
"loss": 0.0,
"rewards/accuracies": 0.5,
"rewards/chosen": -11.565991401672363,
"rewards/margins": -0.08434131741523743,
"rewards/rejected": -11.481651306152344,
"step": 160
},
{
"epoch": 0.34545930384716045,
"grad_norm": 0.01649802186957788,
"learning_rate": 4.964582201835856e-07,
"logits/chosen": -1.800264596939087,
"logits/rejected": -1.78522527217865,
"logps/chosen": -11.585084915161133,
"logps/rejected": -11.45960521697998,
"loss": 0.0,
"rewards/accuracies": 0.4437499940395355,
"rewards/chosen": -11.585084915161133,
"rewards/margins": -0.125479057431221,
"rewards/rejected": -11.45960521697998,
"step": 165
},
{
"epoch": 0.3559277676001047,
"grad_norm": 0.007019029699907713,
"learning_rate": 4.880267969328908e-07,
"logits/chosen": -1.7802222967147827,
"logits/rejected": -1.7600643634796143,
"logps/chosen": -11.943793296813965,
"logps/rejected": -11.659793853759766,
"loss": 0.0,
"rewards/accuracies": 0.42500001192092896,
"rewards/chosen": -11.943793296813965,
"rewards/margins": -0.2839995324611664,
"rewards/rejected": -11.659793853759766,
"step": 170
},
{
"epoch": 0.36639623135304894,
"grad_norm": 0.05226386515779408,
"learning_rate": 4.793433180558423e-07,
"logits/chosen": -1.803500771522522,
"logits/rejected": -1.780199646949768,
"logps/chosen": -11.922220230102539,
"logps/rejected": -11.642582893371582,
"loss": -0.0001,
"rewards/accuracies": 0.4000000059604645,
"rewards/chosen": -11.922220230102539,
"rewards/margins": -0.2796374261379242,
"rewards/rejected": -11.642582893371582,
"step": 175
},
{
"epoch": 0.3768646951059932,
"grad_norm": 0.028525783518634065,
"learning_rate": 4.704194240193467e-07,
"logits/chosen": -1.8105300664901733,
"logits/rejected": -1.8023895025253296,
"logps/chosen": -11.983869552612305,
"logps/rejected": -11.885191917419434,
"loss": -0.0,
"rewards/accuracies": 0.45625001192092896,
"rewards/chosen": -11.983869552612305,
"rewards/margins": -0.09867729246616364,
"rewards/rejected": -11.885191917419434,
"step": 180
},
{
"epoch": 0.38733315885893743,
"grad_norm": 0.001398073561465244,
"learning_rate": 4.6126707757412686e-07,
"logits/chosen": -1.793454885482788,
"logits/rejected": -1.7760779857635498,
"logps/chosen": -11.986612319946289,
"logps/rejected": -11.752717971801758,
"loss": 0.0,
"rewards/accuracies": 0.41874998807907104,
"rewards/chosen": -11.986612319946289,
"rewards/margins": -0.2338949739933014,
"rewards/rejected": -11.752717971801758,
"step": 185
},
{
"epoch": 0.39780162261188173,
"grad_norm": 0.0014862685239923532,
"learning_rate": 4.5189854771829086e-07,
"logits/chosen": -1.7965761423110962,
"logits/rejected": -1.7916818857192993,
"logps/chosen": -11.609036445617676,
"logps/rejected": -11.723310470581055,
"loss": -0.0,
"rewards/accuracies": 0.46875,
"rewards/chosen": -11.609036445617676,
"rewards/margins": 0.11427430808544159,
"rewards/rejected": -11.723310470581055,
"step": 190
},
{
"epoch": 0.408270086364826,
"grad_norm": 0.005759001469221081,
"learning_rate": 4.4232639325036807e-07,
"logits/chosen": -1.7666387557983398,
"logits/rejected": -1.7522531747817993,
"logps/chosen": -11.82097339630127,
"logps/rejected": -11.537239074707031,
"loss": 0.0,
"rewards/accuracies": 0.4625000059604645,
"rewards/chosen": -11.82097339630127,
"rewards/margins": -0.28373217582702637,
"rewards/rejected": -11.537239074707031,
"step": 195
},
{
"epoch": 0.4187385501177702,
"grad_norm": 0.008163598439494825,
"learning_rate": 4.32563445933859e-07,
"logits/chosen": -1.7980865240097046,
"logits/rejected": -1.799093246459961,
"logps/chosen": -11.458420753479004,
"logps/rejected": -11.457086563110352,
"loss": 0.0,
"rewards/accuracies": 0.4625000059604645,
"rewards/chosen": -11.458420753479004,
"rewards/margins": -0.001333725405856967,
"rewards/rejected": -11.457086563110352,
"step": 200
},
{
"epoch": 0.42920701387071447,
"grad_norm": 0.005297760091128626,
"learning_rate": 4.226227932958664e-07,
"logits/chosen": -1.7564678192138672,
"logits/rejected": -1.7390811443328857,
"logps/chosen": -11.681690216064453,
"logps/rejected": -11.680364608764648,
"loss": -0.0,
"rewards/accuracies": 0.4937500059604645,
"rewards/chosen": -11.681690216064453,
"rewards/margins": -0.0013252407079562545,
"rewards/rejected": -11.680364608764648,
"step": 205
},
{
"epoch": 0.4396754776236587,
"grad_norm": 0.006991157705444288,
"learning_rate": 4.1251776108286854e-07,
"logits/chosen": -1.7594162225723267,
"logits/rejected": -1.7496426105499268,
"logps/chosen": -11.553954124450684,
"logps/rejected": -11.592232704162598,
"loss": -0.0,
"rewards/accuracies": 0.4375,
"rewards/chosen": -11.553954124450684,
"rewards/margins": 0.03827826306223869,
"rewards/rejected": -11.592232704162598,
"step": 210
},
{
"epoch": 0.45014394137660296,
"grad_norm": 0.17801591448497012,
"learning_rate": 4.022618953971514e-07,
"logits/chosen": -1.7519290447235107,
"logits/rejected": -1.7526050806045532,
"logps/chosen": -11.367416381835938,
"logps/rejected": -11.493242263793945,
"loss": -0.0001,
"rewards/accuracies": 0.5,
"rewards/chosen": -11.367416381835938,
"rewards/margins": 0.12582536041736603,
"rewards/rejected": -11.493242263793945,
"step": 215
},
{
"epoch": 0.46061240512954726,
"grad_norm": 0.023941549745350215,
"learning_rate": 3.918689445378477e-07,
"logits/chosen": -1.7803691625595093,
"logits/rejected": -1.7690092325210571,
"logps/chosen": -11.14652156829834,
"logps/rejected": -11.06640625,
"loss": -0.0,
"rewards/accuracies": 0.4625000059604645,
"rewards/chosen": -11.14652156829834,
"rewards/margins": -0.08011455833911896,
"rewards/rejected": -11.06640625,
"step": 220
},
{
"epoch": 0.4710808688824915,
"grad_norm": 0.012228246875019251,
"learning_rate": 3.813528405709251e-07,
"logits/chosen": -1.763047456741333,
"logits/rejected": -1.748477578163147,
"logps/chosen": -10.784479141235352,
"logps/rejected": -10.884133338928223,
"loss": -0.0,
"rewards/accuracies": 0.518750011920929,
"rewards/chosen": -10.784479141235352,
"rewards/margins": 0.09965618699789047,
"rewards/rejected": -10.884133338928223,
"step": 225
},
{
"epoch": 0.48154933263543576,
"grad_norm": 0.025471289629283092,
"learning_rate": 3.707276806528282e-07,
"logits/chosen": -1.7505123615264893,
"logits/rejected": -1.726243257522583,
"logps/chosen": -10.95793342590332,
"logps/rejected": -10.815945625305176,
"loss": -0.0004,
"rewards/accuracies": 0.4625000059604645,
"rewards/chosen": -10.95793342590332,
"rewards/margins": -0.1419888436794281,
"rewards/rejected": -10.815945625305176,
"step": 230
},
{
"epoch": 0.49201779638838,
"grad_norm": 0.041173650635296204,
"learning_rate": 3.6000770813281334e-07,
"logits/chosen": -1.7271692752838135,
"logits/rejected": -1.7334954738616943,
"logps/chosen": -10.472602844238281,
"logps/rejected": -10.564123153686523,
"loss": 0.0,
"rewards/accuracies": 0.4749999940395355,
"rewards/chosen": -10.472602844238281,
"rewards/margins": 0.09151904284954071,
"rewards/rejected": -10.564123153686523,
"step": 235
},
{
"epoch": 0.5024862601413242,
"grad_norm": 0.08997133666429326,
"learning_rate": 3.4920729345930654e-07,
"logits/chosen": -1.7182731628417969,
"logits/rejected": -1.7184534072875977,
"logps/chosen": -10.474161148071289,
"logps/rejected": -10.386842727661133,
"loss": -0.0002,
"rewards/accuracies": 0.4375,
"rewards/chosen": -10.474161148071289,
"rewards/margins": -0.08731891214847565,
"rewards/rejected": -10.386842727661133,
"step": 240
},
{
"epoch": 0.5129547238942685,
"grad_norm": 0.1394130468386245,
"learning_rate": 3.383409149158814e-07,
"logits/chosen": -1.7007522583007812,
"logits/rejected": -1.7036033868789673,
"logps/chosen": -10.160537719726562,
"logps/rejected": -10.112937927246094,
"loss": 0.0,
"rewards/accuracies": 0.48750001192092896,
"rewards/chosen": -10.160537719726562,
"rewards/margins": -0.047600626945495605,
"rewards/rejected": -10.112937927246094,
"step": 245
},
{
"epoch": 0.5234231876472127,
"grad_norm": 0.020206583655585347,
"learning_rate": 3.2742313921268035e-07,
"logits/chosen": -1.6959460973739624,
"logits/rejected": -1.6945714950561523,
"logps/chosen": -10.04688549041748,
"logps/rejected": -10.157293319702148,
"loss": 0.0004,
"rewards/accuracies": 0.512499988079071,
"rewards/chosen": -10.04688549041748,
"rewards/margins": 0.11040810495615005,
"rewards/rejected": -10.157293319702148,
"step": 250
},
{
"epoch": 0.533891651400157,
"grad_norm": 0.02211534123468387,
"learning_rate": 3.1646860195929825e-07,
"logits/chosen": -1.7011514902114868,
"logits/rejected": -1.6924482583999634,
"logps/chosen": -10.177471160888672,
"logps/rejected": -10.2760648727417,
"loss": -0.0,
"rewards/accuracies": 0.512499988079071,
"rewards/chosen": -10.177471160888672,
"rewards/margins": 0.09859313070774078,
"rewards/rejected": -10.2760648727417,
"step": 255
},
{
"epoch": 0.5443601151531012,
"grad_norm": 0.01462304817094237,
"learning_rate": 3.054919880453032e-07,
"logits/chosen": -1.635061264038086,
"logits/rejected": -1.6313447952270508,
"logps/chosen": -10.064990043640137,
"logps/rejected": -10.172506332397461,
"loss": 0.0002,
"rewards/accuracies": 0.48750001192092896,
"rewards/chosen": -10.064990043640137,
"rewards/margins": 0.10751698166131973,
"rewards/rejected": -10.172506332397461,
"step": 260
},
{
"epoch": 0.5548285789060455,
"grad_norm": 0.025471797170515387,
"learning_rate": 2.9450801195469686e-07,
"logits/chosen": -1.705881118774414,
"logits/rejected": -1.7156155109405518,
"logps/chosen": -10.338391304016113,
"logps/rejected": -10.457733154296875,
"loss": -0.0001,
"rewards/accuracies": 0.512499988079071,
"rewards/chosen": -10.338391304016113,
"rewards/margins": 0.11934256553649902,
"rewards/rejected": -10.457733154296875,
"step": 265
},
{
"epoch": 0.5652970426589898,
"grad_norm": 0.12522976970659572,
"learning_rate": 2.835313980407017e-07,
"logits/chosen": -1.7283920049667358,
"logits/rejected": -1.693943977355957,
"logps/chosen": -10.727476119995117,
"logps/rejected": -10.667070388793945,
"loss": 0.0002,
"rewards/accuracies": 0.5,
"rewards/chosen": -10.727476119995117,
"rewards/margins": -0.06040619686245918,
"rewards/rejected": -10.667070388793945,
"step": 270
},
{
"epoch": 0.575765506411934,
"grad_norm": 0.054630239668369114,
"learning_rate": 2.7257686078731973e-07,
"logits/chosen": -1.7700306177139282,
"logits/rejected": -1.7575089931488037,
"logps/chosen": -10.57455825805664,
"logps/rejected": -10.53483772277832,
"loss": 0.0,
"rewards/accuracies": 0.4625000059604645,
"rewards/chosen": -10.57455825805664,
"rewards/margins": -0.039720237255096436,
"rewards/rejected": -10.53483772277832,
"step": 275
},
{
"epoch": 0.5862339701648783,
"grad_norm": 0.01001763169581052,
"learning_rate": 2.6165908508411857e-07,
"logits/chosen": -1.7749627828598022,
"logits/rejected": -1.7755094766616821,
"logps/chosen": -10.61900806427002,
"logps/rejected": -10.749244689941406,
"loss": -0.0,
"rewards/accuracies": 0.4625000059604645,
"rewards/chosen": -10.61900806427002,
"rewards/margins": 0.13023611903190613,
"rewards/rejected": -10.749244689941406,
"step": 280
},
{
"epoch": 0.5967024339178225,
"grad_norm": 0.005742953865366501,
"learning_rate": 2.5079270654069354e-07,
"logits/chosen": -1.7149406671524048,
"logits/rejected": -1.7182254791259766,
"logps/chosen": -10.701220512390137,
"logps/rejected": -10.742330551147461,
"loss": -0.0,
"rewards/accuracies": 0.5,
"rewards/chosen": -10.701220512390137,
"rewards/margins": 0.04111091420054436,
"rewards/rejected": -10.742330551147461,
"step": 285
},
{
"epoch": 0.6071708976707668,
"grad_norm": 0.07672060396160495,
"learning_rate": 2.399922918671867e-07,
"logits/chosen": -1.7587283849716187,
"logits/rejected": -1.765321969985962,
"logps/chosen": -10.592453956604004,
"logps/rejected": -10.729036331176758,
"loss": -0.0002,
"rewards/accuracies": 0.5249999761581421,
"rewards/chosen": -10.592453956604004,
"rewards/margins": 0.13658304512500763,
"rewards/rejected": -10.729036331176758,
"step": 290
},
{
"epoch": 0.6176393614237111,
"grad_norm": 0.02428213775826746,
"learning_rate": 2.2927231934717176e-07,
"logits/chosen": -1.7775872945785522,
"logits/rejected": -1.7704694271087646,
"logps/chosen": -10.622035026550293,
"logps/rejected": -10.568597793579102,
"loss": -0.0,
"rewards/accuracies": 0.48750001192092896,
"rewards/chosen": -10.622035026550293,
"rewards/margins": -0.053437769412994385,
"rewards/rejected": -10.568597793579102,
"step": 295
},
{
"epoch": 0.6281078251766553,
"grad_norm": 0.02860916331635737,
"learning_rate": 2.1864715942907487e-07,
"logits/chosen": -1.7398102283477783,
"logits/rejected": -1.73589289188385,
"logps/chosen": -10.513009071350098,
"logps/rejected": -10.603796005249023,
"loss": 0.0001,
"rewards/accuracies": 0.5062500238418579,
"rewards/chosen": -10.513009071350098,
"rewards/margins": 0.09078951179981232,
"rewards/rejected": -10.603796005249023,
"step": 300
},
{
"epoch": 0.6385762889295996,
"grad_norm": 0.006648402581775306,
"learning_rate": 2.081310554621522e-07,
"logits/chosen": -1.7437083721160889,
"logits/rejected": -1.7521770000457764,
"logps/chosen": -10.70260238647461,
"logps/rejected": -10.796627044677734,
"loss": 0.0,
"rewards/accuracies": 0.46875,
"rewards/chosen": -10.70260238647461,
"rewards/margins": 0.09402483701705933,
"rewards/rejected": -10.796627044677734,
"step": 305
},
{
"epoch": 0.6490447526825438,
"grad_norm": 0.019271183476255463,
"learning_rate": 1.9773810460284862e-07,
"logits/chosen": -1.7898304462432861,
"logits/rejected": -1.788010835647583,
"logps/chosen": -10.732610702514648,
"logps/rejected": -10.593297958374023,
"loss": 0.0,
"rewards/accuracies": 0.46875,
"rewards/chosen": -10.732610702514648,
"rewards/margins": -0.1393119990825653,
"rewards/rejected": -10.593297958374023,
"step": 310
},
{
"epoch": 0.6595132164354881,
"grad_norm": 0.02906299326320322,
"learning_rate": 1.874822389171314e-07,
"logits/chosen": -1.7548414468765259,
"logits/rejected": -1.7560005187988281,
"logps/chosen": -10.635225296020508,
"logps/rejected": -10.656919479370117,
"loss": -0.0007,
"rewards/accuracies": 0.48124998807907104,
"rewards/chosen": -10.635225296020508,
"rewards/margins": 0.02169397473335266,
"rewards/rejected": -10.656919479370117,
"step": 315
},
{
"epoch": 0.6699816801884323,
"grad_norm": 0.014440585705017344,
"learning_rate": 1.7737720670413356e-07,
"logits/chosen": -1.796992540359497,
"logits/rejected": -1.7862850427627563,
"logps/chosen": -10.90754508972168,
"logps/rejected": -10.712991714477539,
"loss": 0.0,
"rewards/accuracies": 0.48750001192092896,
"rewards/chosen": -10.90754508972168,
"rewards/margins": -0.19455406069755554,
"rewards/rejected": -10.712991714477539,
"step": 320
},
{
"epoch": 0.6804501439413766,
"grad_norm": 0.04236411566791533,
"learning_rate": 1.6743655406614095e-07,
"logits/chosen": -1.7757809162139893,
"logits/rejected": -1.762711524963379,
"logps/chosen": -10.787301063537598,
"logps/rejected": -10.89518928527832,
"loss": -0.0002,
"rewards/accuracies": 0.4437499940395355,
"rewards/chosen": -10.787301063537598,
"rewards/margins": 0.10788729041814804,
"rewards/rejected": -10.89518928527832,
"step": 325
},
{
"epoch": 0.6909186076943209,
"grad_norm": 0.027523703126398503,
"learning_rate": 1.5767360674963198e-07,
"logits/chosen": -1.749119520187378,
"logits/rejected": -1.7414264678955078,
"logps/chosen": -10.739618301391602,
"logps/rejected": -10.675132751464844,
"loss": -0.0,
"rewards/accuracies": 0.4937500059604645,
"rewards/chosen": -10.739618301391602,
"rewards/margins": -0.06448470056056976,
"rewards/rejected": -10.675132751464844,
"step": 330
},
{
"epoch": 0.7013870714472651,
"grad_norm": 0.014789739092952364,
"learning_rate": 1.4810145228170922e-07,
"logits/chosen": -1.7625138759613037,
"logits/rejected": -1.7610912322998047,
"logps/chosen": -10.482194900512695,
"logps/rejected": -10.562524795532227,
"loss": 0.0,
"rewards/accuracies": 0.4937500059604645,
"rewards/chosen": -10.482194900512695,
"rewards/margins": 0.0803314819931984,
"rewards/rejected": -10.562524795532227,
"step": 335
},
{
"epoch": 0.7118555352002094,
"grad_norm": 0.01922207578951387,
"learning_rate": 1.3873292242587306e-07,
"logits/chosen": -1.7635924816131592,
"logits/rejected": -1.7583682537078857,
"logps/chosen": -10.654876708984375,
"logps/rejected": -10.662198066711426,
"loss": 0.0,
"rewards/accuracies": 0.46875,
"rewards/chosen": -10.654876708984375,
"rewards/margins": 0.0073205530643463135,
"rewards/rejected": -10.662198066711426,
"step": 340
},
{
"epoch": 0.7223239989531536,
"grad_norm": 0.03539872090887725,
"learning_rate": 1.295805759806533e-07,
"logits/chosen": -1.7667583227157593,
"logits/rejected": -1.7786306142807007,
"logps/chosen": -10.630470275878906,
"logps/rejected": -10.790535926818848,
"loss": -0.0,
"rewards/accuracies": 0.48750001192092896,
"rewards/chosen": -10.630470275878906,
"rewards/margins": 0.16006465256214142,
"rewards/rejected": -10.790535926818848,
"step": 345
},
{
"epoch": 0.7327924627060979,
"grad_norm": 0.022967566874532427,
"learning_rate": 1.2065668194415777e-07,
"logits/chosen": -1.7326412200927734,
"logits/rejected": -1.729612946510315,
"logps/chosen": -10.603561401367188,
"logps/rejected": -10.510968208312988,
"loss": -0.0001,
"rewards/accuracies": 0.4437499940395355,
"rewards/chosen": -10.603561401367188,
"rewards/margins": -0.09259293973445892,
"rewards/rejected": -10.510968208312988,
"step": 350
},
{
"epoch": 0.7432609264590422,
"grad_norm": 0.012752562963674085,
"learning_rate": 1.1197320306710923e-07,
"logits/chosen": -1.762995958328247,
"logits/rejected": -1.743704080581665,
"logps/chosen": -10.666765213012695,
"logps/rejected": -10.419971466064453,
"loss": 0.0001,
"rewards/accuracies": 0.39375001192092896,
"rewards/chosen": -10.666765213012695,
"rewards/margins": -0.24679343402385712,
"rewards/rejected": -10.419971466064453,
"step": 355
},
{
"epoch": 0.7537293902119864,
"grad_norm": 0.009738702402905076,
"learning_rate": 1.035417798164145e-07,
"logits/chosen": -1.747469186782837,
"logits/rejected": -1.739599585533142,
"logps/chosen": -10.463155746459961,
"logps/rejected": -10.498678207397461,
"loss": 0.0,
"rewards/accuracies": 0.5062500238418579,
"rewards/chosen": -10.463155746459961,
"rewards/margins": 0.03552195057272911,
"rewards/rejected": -10.498678207397461,
"step": 360
},
{
"epoch": 0.7641978539649307,
"grad_norm": 0.06431835444147056,
"learning_rate": 9.537371477076535e-08,
"logits/chosen": -1.7092878818511963,
"logits/rejected": -1.6988884210586548,
"logps/chosen": -10.540400505065918,
"logps/rejected": -10.477300643920898,
"loss": 0.0,
"rewards/accuracies": 0.48124998807907104,
"rewards/chosen": -10.540400505065918,
"rewards/margins": -0.06309934705495834,
"rewards/rejected": -10.477300643920898,
"step": 365
},
{
"epoch": 0.7746663177178749,
"grad_norm": 0.052909632347048964,
"learning_rate": 8.747995746918898e-08,
"logits/chosen": -1.7701542377471924,
"logits/rejected": -1.758774757385254,
"logps/chosen": -10.620153427124023,
"logps/rejected": -10.608236312866211,
"loss": 0.0,
"rewards/accuracies": 0.518750011920929,
"rewards/chosen": -10.620153427124023,
"rewards/margins": -0.011916184797883034,
"rewards/rejected": -10.608236312866211,
"step": 370
},
{
"epoch": 0.7851347814708192,
"grad_norm": 0.012450945922480083,
"learning_rate": 7.987108973285888e-08,
"logits/chosen": -1.7810709476470947,
"logits/rejected": -1.7817420959472656,
"logps/chosen": -10.626587867736816,
"logps/rejected": -10.649564743041992,
"loss": 0.0,
"rewards/accuracies": 0.48750001192092896,
"rewards/chosen": -10.626587867736816,
"rewards/margins": 0.022976160049438477,
"rewards/rejected": -10.649564743041992,
"step": 375
},
{
"epoch": 0.7956032452237635,
"grad_norm": 0.004460244436581798,
"learning_rate": 7.255731147984174e-08,
"logits/chosen": -1.7550201416015625,
"logits/rejected": -1.7136766910552979,
"logps/chosen": -10.767892837524414,
"logps/rejected": -10.837907791137695,
"loss": 0.0001,
"rewards/accuracies": 0.53125,
"rewards/chosen": -10.767892837524414,
"rewards/margins": 0.0700153112411499,
"rewards/rejected": -10.837907791137695,
"step": 380
},
{
"epoch": 0.8060717089767077,
"grad_norm": 0.019593651768819848,
"learning_rate": 6.554842705179898e-08,
"logits/chosen": -1.778969407081604,
"logits/rejected": -1.75950026512146,
"logps/chosen": -10.800111770629883,
"logps/rejected": -10.750751495361328,
"loss": -0.0,
"rewards/accuracies": 0.44999998807907104,
"rewards/chosen": -10.800111770629883,
"rewards/margins": -0.04935937374830246,
"rewards/rejected": -10.750751495361328,
"step": 385
},
{
"epoch": 0.816540172729652,
"grad_norm": 0.03689590756531894,
"learning_rate": 5.885383207096832e-08,
"logits/chosen": -1.78683602809906,
"logits/rejected": -1.7717739343643188,
"logps/chosen": -10.598950386047363,
"logps/rejected": -10.57533073425293,
"loss": 0.0,
"rewards/accuracies": 0.4375,
"rewards/chosen": -10.598950386047363,
"rewards/margins": -0.02361871860921383,
"rewards/rejected": -10.57533073425293,
"step": 390
},
{
"epoch": 0.8270086364825961,
"grad_norm": 0.012548371625903876,
"learning_rate": 5.2482500845047165e-08,
"logits/chosen": -1.743941307067871,
"logits/rejected": -1.7405914068222046,
"logps/chosen": -10.441683769226074,
"logps/rejected": -10.442001342773438,
"loss": -0.0005,
"rewards/accuracies": 0.48124998807907104,
"rewards/chosen": -10.441683769226074,
"rewards/margins": 0.00031833647517487407,
"rewards/rejected": -10.442001342773438,
"step": 395
},
{
"epoch": 0.8374771002355405,
"grad_norm": 0.013565529452432605,
"learning_rate": 4.644297433686162e-08,
"logits/chosen": -1.741358757019043,
"logits/rejected": -1.7224204540252686,
"logps/chosen": -10.534662246704102,
"logps/rejected": -10.434551239013672,
"loss": -0.0002,
"rewards/accuracies": 0.48750001192092896,
"rewards/chosen": -10.534662246704102,
"rewards/margins": -0.10010989010334015,
"rewards/rejected": -10.434551239013672,
"step": 400
},
{
"epoch": 0.8479455639884846,
"grad_norm": 0.10515870132328753,
"learning_rate": 4.074334871494558e-08,
"logits/chosen": -1.7736488580703735,
"logits/rejected": -1.7777044773101807,
"logps/chosen": -10.679037094116211,
"logps/rejected": -10.682011604309082,
"loss": -0.0002,
"rewards/accuracies": 0.46875,
"rewards/chosen": -10.679037094116211,
"rewards/margins": 0.002974617527797818,
"rewards/rejected": -10.682011604309082,
"step": 405
},
{
"epoch": 0.8584140277414289,
"grad_norm": 0.009978883732705937,
"learning_rate": 3.5391264500382e-08,
"logits/chosen": -1.774903655052185,
"logits/rejected": -1.7690646648406982,
"logps/chosen": -10.4728364944458,
"logps/rejected": -10.374021530151367,
"loss": 0.0001,
"rewards/accuracies": 0.5,
"rewards/chosen": -10.4728364944458,
"rewards/margins": -0.09881408512592316,
"rewards/rejected": -10.374021530151367,
"step": 410
},
{
"epoch": 0.8688824914943732,
"grad_norm": 0.006390665968550676,
"learning_rate": 3.0393896324452226e-08,
"logits/chosen": -1.7874574661254883,
"logits/rejected": -1.7859035730361938,
"logps/chosen": -10.748700141906738,
"logps/rejected": -10.698121070861816,
"loss": 0.0,
"rewards/accuracies": 0.4375,
"rewards/chosen": -10.748700141906738,
"rewards/margins": -0.050578661262989044,
"rewards/rejected": -10.698121070861816,
"step": 415
},
{
"epoch": 0.8793509552473174,
"grad_norm": 0.0968516068570453,
"learning_rate": 2.5757943310825026e-08,
"logits/chosen": -1.7406418323516846,
"logits/rejected": -1.7307815551757812,
"logps/chosen": -10.560035705566406,
"logps/rejected": -10.44668960571289,
"loss": -0.0,
"rewards/accuracies": 0.45625001192092896,
"rewards/chosen": -10.560035705566406,
"rewards/margins": -0.11334645748138428,
"rewards/rejected": -10.44668960571289,
"step": 420
},
{
"epoch": 0.8898194190002617,
"grad_norm": 0.00590008646483764,
"learning_rate": 2.148962009517823e-08,
"logits/chosen": -1.7577073574066162,
"logits/rejected": -1.7527620792388916,
"logps/chosen": -10.674304008483887,
"logps/rejected": -10.642423629760742,
"loss": 0.0,
"rewards/accuracies": 0.48750001192092896,
"rewards/chosen": -10.674304008483887,
"rewards/margins": -0.03188156336545944,
"rewards/rejected": -10.642423629760742,
"step": 425
},
{
"epoch": 0.9002878827532059,
"grad_norm": 0.013120062560596776,
"learning_rate": 1.759464849429082e-08,
"logits/chosen": -1.7664194107055664,
"logits/rejected": -1.760266900062561,
"logps/chosen": -10.494427680969238,
"logps/rejected": -10.483784675598145,
"loss": -0.0002,
"rewards/accuracies": 0.4625000059604645,
"rewards/chosen": -10.494427680969238,
"rewards/margins": -0.010644030757248402,
"rewards/rejected": -10.483784675598145,
"step": 430
},
{
"epoch": 0.9107563465061502,
"grad_norm": 0.1742770180554167,
"learning_rate": 1.4078249835774169e-08,
"logits/chosen": -1.778710961341858,
"logits/rejected": -1.7815498113632202,
"logps/chosen": -10.63943862915039,
"logps/rejected": -10.601329803466797,
"loss": 0.0007,
"rewards/accuracies": 0.48124998807907104,
"rewards/chosen": -10.63943862915039,
"rewards/margins": -0.03810780122876167,
"rewards/rejected": -10.601329803466797,
"step": 435
},
{
"epoch": 0.9212248102590945,
"grad_norm": 0.0404689579696443,
"learning_rate": 1.0945137958723705e-08,
"logits/chosen": -1.7349185943603516,
"logits/rejected": -1.730891227722168,
"logps/chosen": -10.740056991577148,
"logps/rejected": -10.685806274414062,
"loss": 0.0004,
"rewards/accuracies": 0.44999998807907104,
"rewards/chosen": -10.740056991577148,
"rewards/margins": -0.054250918328762054,
"rewards/rejected": -10.685806274414062,
"step": 440
},
{
"epoch": 0.9316932740120387,
"grad_norm": 0.09262486522903134,
"learning_rate": 8.19951289467482e-09,
"logits/chosen": -1.7872415781021118,
"logits/rejected": -1.776834487915039,
"logps/chosen": -10.619129180908203,
"logps/rejected": -10.654691696166992,
"loss": 0.0001,
"rewards/accuracies": 0.48750001192092896,
"rewards/chosen": -10.619129180908203,
"rewards/margins": 0.03556279465556145,
"rewards/rejected": -10.654691696166992,
"step": 445
},
{
"epoch": 0.942161737764983,
"grad_norm": 0.02488581461219639,
"learning_rate": 5.84505523733293e-09,
"logits/chosen": -1.7146650552749634,
"logits/rejected": -1.6959879398345947,
"logps/chosen": -10.7011137008667,
"logps/rejected": -10.602450370788574,
"loss": 0.0001,
"rewards/accuracies": 0.4375,
"rewards/chosen": -10.7011137008667,
"rewards/margins": -0.09866499900817871,
"rewards/rejected": -10.602450370788574,
"step": 450
},
{
"epoch": 0.9526302015179272,
"grad_norm": 0.09420372269289778,
"learning_rate": 3.8849212086261466e-09,
"logits/chosen": -1.7629196643829346,
"logits/rejected": -1.7624677419662476,
"logps/chosen": -10.534662246704102,
"logps/rejected": -10.774109840393066,
"loss": 0.0001,
"rewards/accuracies": 0.5874999761581421,
"rewards/chosen": -10.534662246704102,
"rewards/margins": 0.23944692313671112,
"rewards/rejected": -10.774109840393066,
"step": 455
},
{
"epoch": 0.9630986652708715,
"grad_norm": 0.012886454137460854,
"learning_rate": 2.3217384276938756e-09,
"logits/chosen": -1.7755534648895264,
"logits/rejected": -1.7767679691314697,
"logps/chosen": -10.819222450256348,
"logps/rejected": -10.87352180480957,
"loss": -0.0,
"rewards/accuracies": 0.46875,
"rewards/chosen": -10.819222450256348,
"rewards/margins": 0.05429871007800102,
"rewards/rejected": -10.87352180480957,
"step": 460
},
{
"epoch": 0.9735671290238157,
"grad_norm": 0.12756283914628896,
"learning_rate": 1.1576023884836472e-09,
"logits/chosen": -1.783143401145935,
"logits/rejected": -1.7813498973846436,
"logps/chosen": -10.936141967773438,
"logps/rejected": -10.884490013122559,
"loss": -0.0003,
"rewards/accuracies": 0.45625001192092896,
"rewards/chosen": -10.936141967773438,
"rewards/margins": -0.051651883870363235,
"rewards/rejected": -10.884490013122559,
"step": 465
},
{
"epoch": 0.98403559277676,
"grad_norm": 0.03220532320786913,
"learning_rate": 3.940736506780395e-10,
"logits/chosen": -1.7732326984405518,
"logits/rejected": -1.7703964710235596,
"logps/chosen": -10.523382186889648,
"logps/rejected": -10.703128814697266,
"loss": 0.0001,
"rewards/accuracies": 0.5375000238418579,
"rewards/chosen": -10.523382186889648,
"rewards/margins": 0.17974743247032166,
"rewards/rejected": -10.703128814697266,
"step": 470
},
{
"epoch": 0.9945040565297043,
"grad_norm": 0.015956952957413534,
"learning_rate": 3.2175747716822744e-11,
"logits/chosen": -1.7491505146026611,
"logits/rejected": -1.731268286705017,
"logps/chosen": -10.880552291870117,
"logps/rejected": -10.799489974975586,
"loss": -0.0,
"rewards/accuracies": 0.5062500238418579,
"rewards/chosen": -10.880552291870117,
"rewards/margins": -0.08106319606304169,
"rewards/rejected": -10.799489974975586,
"step": 475
},
{
"epoch": 0.998691442030882,
"step": 477,
"total_flos": 0.0,
"train_loss": 4.822632607948269e-06,
"train_runtime": 8193.4913,
"train_samples_per_second": 7.461,
"train_steps_per_second": 0.058
}
],
"logging_steps": 5,
"max_steps": 477,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 1000000,
"total_flos": 0.0,
"train_batch_size": 2,
"trial_name": null,
"trial_params": null
}