| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 5.0, | |
| "eval_steps": 1000, | |
| "global_step": 625, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.008, | |
| "grad_norm": 0.5664516818518655, | |
| "learning_rate": 7.936507936507937e-08, | |
| "logits/chosen": -1.7671998739242554, | |
| "logits/rejected": -2.2639822959899902, | |
| "logps/chosen": -46.430763244628906, | |
| "logps/rejected": -102.85381317138672, | |
| "loss": 0.6931, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 0.5940666028150138, | |
| "learning_rate": 7.936507936507937e-07, | |
| "logits/chosen": -1.635621428489685, | |
| "logits/rejected": -2.0303046703338623, | |
| "logps/chosen": -50.40827941894531, | |
| "logps/rejected": -98.84134674072266, | |
| "loss": 0.6929, | |
| "rewards/accuracies": 0.4930555522441864, | |
| "rewards/chosen": -0.0004079286882188171, | |
| "rewards/margins": 0.0003309930325485766, | |
| "rewards/rejected": -0.0007389216334559023, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 0.5693390021480599, | |
| "learning_rate": 1.5873015873015873e-06, | |
| "logits/chosen": -1.6660130023956299, | |
| "logits/rejected": -2.1243433952331543, | |
| "logps/chosen": -46.99885559082031, | |
| "logps/rejected": -105.28565979003906, | |
| "loss": 0.6876, | |
| "rewards/accuracies": 0.9937499761581421, | |
| "rewards/chosen": 0.0004832017293665558, | |
| "rewards/margins": 0.012159029953181744, | |
| "rewards/rejected": -0.011675829067826271, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "grad_norm": 0.7544428399092442, | |
| "learning_rate": 2.380952380952381e-06, | |
| "logits/chosen": -1.7471415996551514, | |
| "logits/rejected": -2.1227598190307617, | |
| "logps/chosen": -50.86139678955078, | |
| "logps/rejected": -106.50770568847656, | |
| "loss": 0.6584, | |
| "rewards/accuracies": 0.925000011920929, | |
| "rewards/chosen": -0.0016866873484104872, | |
| "rewards/margins": 0.07014349102973938, | |
| "rewards/rejected": -0.07183017581701279, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "grad_norm": 0.7543215495993858, | |
| "learning_rate": 3.1746031746031746e-06, | |
| "logits/chosen": -1.6426982879638672, | |
| "logits/rejected": -2.0793442726135254, | |
| "logps/chosen": -48.13318634033203, | |
| "logps/rejected": -131.1614990234375, | |
| "loss": 0.563, | |
| "rewards/accuracies": 0.9750000238418579, | |
| "rewards/chosen": 0.005994443781673908, | |
| "rewards/margins": 0.3002367615699768, | |
| "rewards/rejected": -0.29424232244491577, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "grad_norm": 0.4924878173095808, | |
| "learning_rate": 3.968253968253968e-06, | |
| "logits/chosen": -1.583770990371704, | |
| "logits/rejected": -2.0470001697540283, | |
| "logps/chosen": -47.22152328491211, | |
| "logps/rejected": -174.95742797851562, | |
| "loss": 0.4123, | |
| "rewards/accuracies": 0.9750000238418579, | |
| "rewards/chosen": 0.01497081108391285, | |
| "rewards/margins": 0.743770182132721, | |
| "rewards/rejected": -0.7287994027137756, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "grad_norm": 0.3454006645724035, | |
| "learning_rate": 4.761904761904762e-06, | |
| "logits/chosen": -1.554491639137268, | |
| "logits/rejected": -1.9409517049789429, | |
| "logps/chosen": -54.594146728515625, | |
| "logps/rejected": -216.20999145507812, | |
| "loss": 0.301, | |
| "rewards/accuracies": 0.9312499761581421, | |
| "rewards/chosen": -0.03443662449717522, | |
| "rewards/margins": 1.137538194656372, | |
| "rewards/rejected": -1.171974778175354, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "grad_norm": 0.34001134236428526, | |
| "learning_rate": 4.998086282661188e-06, | |
| "logits/chosen": -1.459894061088562, | |
| "logits/rejected": -1.9039119482040405, | |
| "logps/chosen": -53.59501266479492, | |
| "logps/rejected": -295.020751953125, | |
| "loss": 0.2243, | |
| "rewards/accuracies": 0.9437500238418579, | |
| "rewards/chosen": -0.030449170619249344, | |
| "rewards/margins": 1.905814528465271, | |
| "rewards/rejected": -1.9362636804580688, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "grad_norm": 0.15067611382441232, | |
| "learning_rate": 4.988720025682995e-06, | |
| "logits/chosen": -1.3985722064971924, | |
| "logits/rejected": -1.8223779201507568, | |
| "logps/chosen": -54.72029495239258, | |
| "logps/rejected": -359.4449462890625, | |
| "loss": 0.2107, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -0.03896407037973404, | |
| "rewards/margins": 2.546111583709717, | |
| "rewards/rejected": -2.585075855255127, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "grad_norm": 0.1607243644073366, | |
| "learning_rate": 4.9715789537359126e-06, | |
| "logits/chosen": -1.3732717037200928, | |
| "logits/rejected": -1.775152564048767, | |
| "logps/chosen": -58.017051696777344, | |
| "logps/rejected": -388.412109375, | |
| "loss": 0.1949, | |
| "rewards/accuracies": 0.9437500238418579, | |
| "rewards/chosen": -0.06883589923381805, | |
| "rewards/margins": 2.8034896850585938, | |
| "rewards/rejected": -2.8723256587982178, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "grad_norm": 0.23308085253562452, | |
| "learning_rate": 4.946716615897932e-06, | |
| "logits/chosen": -1.4088760614395142, | |
| "logits/rejected": -1.845401406288147, | |
| "logps/chosen": -50.46847915649414, | |
| "logps/rejected": -365.43017578125, | |
| "loss": 0.1946, | |
| "rewards/accuracies": 0.956250011920929, | |
| "rewards/chosen": -0.003804679261520505, | |
| "rewards/margins": 2.6316394805908203, | |
| "rewards/rejected": -2.635444164276123, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "grad_norm": 0.26044789726674666, | |
| "learning_rate": 4.9142106826480114e-06, | |
| "logits/chosen": -1.3341541290283203, | |
| "logits/rejected": -1.7699487209320068, | |
| "logps/chosen": -52.57965087890625, | |
| "logps/rejected": -368.12310791015625, | |
| "loss": 0.1915, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -0.02668028511106968, | |
| "rewards/margins": 2.640017032623291, | |
| "rewards/rejected": -2.6666972637176514, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "grad_norm": 0.30708635657559097, | |
| "learning_rate": 4.874162703221823e-06, | |
| "logits/chosen": -1.2924244403839111, | |
| "logits/rejected": -1.7736318111419678, | |
| "logps/chosen": -38.64341735839844, | |
| "logps/rejected": -411.8453063964844, | |
| "loss": 0.1577, | |
| "rewards/accuracies": 0.9937499761581421, | |
| "rewards/chosen": 0.08109267055988312, | |
| "rewards/margins": 3.1581881046295166, | |
| "rewards/rejected": -3.0770950317382812, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 1.04, | |
| "grad_norm": 0.2936514218907092, | |
| "learning_rate": 4.826697788369752e-06, | |
| "logits/chosen": -1.3301618099212646, | |
| "logits/rejected": -1.7304044961929321, | |
| "logps/chosen": -57.28544998168945, | |
| "logps/rejected": -383.7294616699219, | |
| "loss": 0.1735, | |
| "rewards/accuracies": 0.9312499761581421, | |
| "rewards/chosen": -0.060813553631305695, | |
| "rewards/margins": 2.7762393951416016, | |
| "rewards/rejected": -2.837052822113037, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 1.12, | |
| "grad_norm": 0.7127879450944201, | |
| "learning_rate": 4.7719642195082224e-06, | |
| "logits/chosen": -1.3428221940994263, | |
| "logits/rejected": -1.8241684436798096, | |
| "logps/chosen": -49.40485382080078, | |
| "logps/rejected": -397.02685546875, | |
| "loss": 0.167, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.005482205655425787, | |
| "rewards/margins": 2.9638657569885254, | |
| "rewards/rejected": -2.958383798599243, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 1.2, | |
| "grad_norm": 0.7195419525970039, | |
| "learning_rate": 4.710132985485355e-06, | |
| "logits/chosen": -1.2738778591156006, | |
| "logits/rejected": -1.7257006168365479, | |
| "logps/chosen": -53.46342849731445, | |
| "logps/rejected": -413.75091552734375, | |
| "loss": 0.1777, | |
| "rewards/accuracies": 0.9312499761581421, | |
| "rewards/chosen": -0.01903398148715496, | |
| "rewards/margins": 3.1122589111328125, | |
| "rewards/rejected": -3.1312928199768066, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 1.28, | |
| "grad_norm": 0.5831467127347119, | |
| "learning_rate": 4.641397248408122e-06, | |
| "logits/chosen": -1.3061014413833618, | |
| "logits/rejected": -1.7850764989852905, | |
| "logps/chosen": -49.27734375, | |
| "logps/rejected": -427.78338623046875, | |
| "loss": 0.128, | |
| "rewards/accuracies": 0.9437500238418579, | |
| "rewards/chosen": 0.00814875029027462, | |
| "rewards/margins": 3.2720084190368652, | |
| "rewards/rejected": -3.263859510421753, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 1.3599999999999999, | |
| "grad_norm": 0.37604055418234283, | |
| "learning_rate": 4.5659717401997655e-06, | |
| "logits/chosen": -1.2165374755859375, | |
| "logits/rejected": -1.7296428680419922, | |
| "logps/chosen": -43.91862106323242, | |
| "logps/rejected": -461.78021240234375, | |
| "loss": 0.1334, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 0.04128595441579819, | |
| "rewards/margins": 3.6384873390197754, | |
| "rewards/rejected": -3.597201108932495, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 1.44, | |
| "grad_norm": 1.886026671708352, | |
| "learning_rate": 4.4840920917726425e-06, | |
| "logits/chosen": -1.2202318906784058, | |
| "logits/rejected": -1.7096608877182007, | |
| "logps/chosen": -48.562477111816406, | |
| "logps/rejected": -456.2506408691406, | |
| "loss": 0.1219, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": 0.016236167401075363, | |
| "rewards/margins": 3.572105884552002, | |
| "rewards/rejected": -3.5558693408966064, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 1.52, | |
| "grad_norm": 1.208359086336539, | |
| "learning_rate": 4.396014096912182e-06, | |
| "logits/chosen": -1.1344788074493408, | |
| "logits/rejected": -1.5908814668655396, | |
| "logps/chosen": -47.373992919921875, | |
| "logps/rejected": -488.7273864746094, | |
| "loss": 0.0984, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": 0.023608360439538956, | |
| "rewards/margins": 3.904538631439209, | |
| "rewards/rejected": -3.8809304237365723, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 1.6, | |
| "grad_norm": 3.9821859292862705, | |
| "learning_rate": 4.302012913171584e-06, | |
| "logits/chosen": -1.2608126401901245, | |
| "logits/rejected": -1.7331119775772095, | |
| "logps/chosen": -41.71361541748047, | |
| "logps/rejected": -526.3060302734375, | |
| "loss": 0.102, | |
| "rewards/accuracies": 0.956250011920929, | |
| "rewards/chosen": 0.08378119766712189, | |
| "rewards/margins": 4.328982353210449, | |
| "rewards/rejected": -4.245201110839844, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 1.6800000000000002, | |
| "grad_norm": 1.5269166268578003, | |
| "learning_rate": 4.202382202273702e-06, | |
| "logits/chosen": -1.114254117012024, | |
| "logits/rejected": -1.6004146337509155, | |
| "logps/chosen": -40.91397476196289, | |
| "logps/rejected": -533.20751953125, | |
| "loss": 0.0837, | |
| "rewards/accuracies": 0.9624999761581421, | |
| "rewards/chosen": 0.08689381927251816, | |
| "rewards/margins": 4.404597759246826, | |
| "rewards/rejected": -4.317704200744629, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 1.76, | |
| "grad_norm": 3.6420048929792053, | |
| "learning_rate": 4.097433212705492e-06, | |
| "logits/chosen": -1.1888163089752197, | |
| "logits/rejected": -1.7389936447143555, | |
| "logps/chosen": -37.578697204589844, | |
| "logps/rejected": -569.4190673828125, | |
| "loss": 0.0527, | |
| "rewards/accuracies": 0.987500011920929, | |
| "rewards/chosen": 0.10641758143901825, | |
| "rewards/margins": 4.77039909362793, | |
| "rewards/rejected": -4.663981914520264, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 1.8399999999999999, | |
| "grad_norm": 7.795327748358564, | |
| "learning_rate": 3.987493807371033e-06, | |
| "logits/chosen": -1.1736271381378174, | |
| "logits/rejected": -1.6653432846069336, | |
| "logps/chosen": -39.23591995239258, | |
| "logps/rejected": -596.703369140625, | |
| "loss": 0.0708, | |
| "rewards/accuracies": 0.981249988079071, | |
| "rewards/chosen": 0.09144876152276993, | |
| "rewards/margins": 5.02878999710083, | |
| "rewards/rejected": -4.93734073638916, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 1.92, | |
| "grad_norm": 3.784757324910154, | |
| "learning_rate": 3.872907439340758e-06, | |
| "logits/chosen": -1.1998341083526611, | |
| "logits/rejected": -1.5868158340454102, | |
| "logps/chosen": -40.40230178833008, | |
| "logps/rejected": -578.6609497070312, | |
| "loss": 0.0589, | |
| "rewards/accuracies": 0.9437500238418579, | |
| "rewards/chosen": 0.10270919650793076, | |
| "rewards/margins": 4.877388954162598, | |
| "rewards/rejected": -4.774679660797119, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 2.0, | |
| "grad_norm": 1.1274212490773023, | |
| "learning_rate": 3.75403207889666e-06, | |
| "logits/chosen": -1.2294795513153076, | |
| "logits/rejected": -1.621289849281311, | |
| "logps/chosen": -40.97010040283203, | |
| "logps/rejected": -588.9901733398438, | |
| "loss": 0.0505, | |
| "rewards/accuracies": 0.9437500238418579, | |
| "rewards/chosen": 0.09869703650474548, | |
| "rewards/margins": 4.985424041748047, | |
| "rewards/rejected": -4.886727333068848, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 2.08, | |
| "grad_norm": 0.1615894379623363, | |
| "learning_rate": 3.631239095225417e-06, | |
| "logits/chosen": -1.104773759841919, | |
| "logits/rejected": -1.556726098060608, | |
| "logps/chosen": -41.80849075317383, | |
| "logps/rejected": -610.6696166992188, | |
| "loss": 0.0473, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 0.08216419816017151, | |
| "rewards/margins": 5.163956642150879, | |
| "rewards/rejected": -5.081791877746582, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 2.16, | |
| "grad_norm": 0.20738276054029106, | |
| "learning_rate": 3.5049120962530608e-06, | |
| "logits/chosen": -1.144113302230835, | |
| "logits/rejected": -1.526831865310669, | |
| "logps/chosen": -40.4541015625, | |
| "logps/rejected": -581.6177978515625, | |
| "loss": 0.0562, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": 0.10693266242742538, | |
| "rewards/margins": 4.924569129943848, | |
| "rewards/rejected": -4.817636013031006, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 2.24, | |
| "grad_norm": 0.08228415199969275, | |
| "learning_rate": 3.3754457302455464e-06, | |
| "logits/chosen": -1.1058952808380127, | |
| "logits/rejected": -1.5095767974853516, | |
| "logps/chosen": -40.29807662963867, | |
| "logps/rejected": -614.50634765625, | |
| "loss": 0.0422, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 0.1043148785829544, | |
| "rewards/margins": 5.244091033935547, | |
| "rewards/rejected": -5.139776229858398, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 2.32, | |
| "grad_norm": 0.5196221393419624, | |
| "learning_rate": 3.2432444529190714e-06, | |
| "logits/chosen": -1.113113284111023, | |
| "logits/rejected": -1.501694917678833, | |
| "logps/chosen": -39.059715270996094, | |
| "logps/rejected": -635.2387084960938, | |
| "loss": 0.0458, | |
| "rewards/accuracies": 0.981249988079071, | |
| "rewards/chosen": 0.10602055490016937, | |
| "rewards/margins": 5.451790809631348, | |
| "rewards/rejected": -5.345770359039307, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 2.4, | |
| "grad_norm": 0.08825766353470356, | |
| "learning_rate": 3.1087212639117057e-06, | |
| "logits/chosen": -1.1622686386108398, | |
| "logits/rejected": -1.5392283201217651, | |
| "logps/chosen": -39.0388298034668, | |
| "logps/rejected": -639.8673095703125, | |
| "loss": 0.0312, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.11618944257497787, | |
| "rewards/margins": 5.501553535461426, | |
| "rewards/rejected": -5.385363578796387, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 2.48, | |
| "grad_norm": 0.10694009885473299, | |
| "learning_rate": 2.9722964165636263e-06, | |
| "logits/chosen": -1.1022515296936035, | |
| "logits/rejected": -1.4340097904205322, | |
| "logps/chosen": -39.469932556152344, | |
| "logps/rejected": -641.5587768554688, | |
| "loss": 0.0458, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.11364182084798813, | |
| "rewards/margins": 5.520757675170898, | |
| "rewards/rejected": -5.407115459442139, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 2.56, | |
| "grad_norm": 0.4507687634087117, | |
| "learning_rate": 2.8343961050366275e-06, | |
| "logits/chosen": -1.0810810327529907, | |
| "logits/rejected": -1.4569213390350342, | |
| "logps/chosen": -41.17557144165039, | |
| "logps/rejected": -649.1512451171875, | |
| "loss": 0.0228, | |
| "rewards/accuracies": 0.9937499761581421, | |
| "rewards/chosen": 0.07850812375545502, | |
| "rewards/margins": 5.554858207702637, | |
| "rewards/rejected": -5.476349830627441, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 2.64, | |
| "grad_norm": 0.03148227835487055, | |
| "learning_rate": 2.695451132874385e-06, | |
| "logits/chosen": -1.0946557521820068, | |
| "logits/rejected": -1.494209885597229, | |
| "logps/chosen": -44.248779296875, | |
| "logps/rejected": -679.5821533203125, | |
| "loss": 0.0242, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.04512793570756912, | |
| "rewards/margins": 5.8134870529174805, | |
| "rewards/rejected": -5.768359184265137, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 2.7199999999999998, | |
| "grad_norm": 0.07742711911111466, | |
| "learning_rate": 2.5558955671628964e-06, | |
| "logits/chosen": -1.1254152059555054, | |
| "logits/rejected": -1.4622461795806885, | |
| "logps/chosen": -45.729515075683594, | |
| "logps/rejected": -666.0767211914062, | |
| "loss": 0.051, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.046021558344364166, | |
| "rewards/margins": 5.69630241394043, | |
| "rewards/rejected": -5.650280952453613, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 2.8, | |
| "grad_norm": 0.0712177913874092, | |
| "learning_rate": 2.4161653824955654e-06, | |
| "logits/chosen": -1.2363636493682861, | |
| "logits/rejected": -1.6161434650421143, | |
| "logps/chosen": -47.28390884399414, | |
| "logps/rejected": -664.1326904296875, | |
| "loss": 0.0351, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.038107506930828094, | |
| "rewards/margins": 5.666635990142822, | |
| "rewards/rejected": -5.628528118133545, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 2.88, | |
| "grad_norm": 0.11629194717439449, | |
| "learning_rate": 2.2766970989791697e-06, | |
| "logits/chosen": -1.2213943004608154, | |
| "logits/rejected": -1.5998932123184204, | |
| "logps/chosen": -48.74114990234375, | |
| "logps/rejected": -667.0240478515625, | |
| "loss": 0.0453, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.0151976328343153, | |
| "rewards/margins": 5.664753437042236, | |
| "rewards/rejected": -5.649555683135986, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 2.96, | |
| "grad_norm": 0.0614653072363607, | |
| "learning_rate": 2.1379264185356545e-06, | |
| "logits/chosen": -1.205859899520874, | |
| "logits/rejected": -1.5529258251190186, | |
| "logps/chosen": -48.79401397705078, | |
| "logps/rejected": -669.0440063476562, | |
| "loss": 0.0517, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.007499977946281433, | |
| "rewards/margins": 5.6917924880981445, | |
| "rewards/rejected": -5.684292316436768, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 3.04, | |
| "grad_norm": 0.10523859679268263, | |
| "learning_rate": 2.000286863759934e-06, | |
| "logits/chosen": -1.255184531211853, | |
| "logits/rejected": -1.6094391345977783, | |
| "logps/chosen": -48.94397735595703, | |
| "logps/rejected": -655.6014404296875, | |
| "loss": 0.0253, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.006729847751557827, | |
| "rewards/margins": 5.557109832763672, | |
| "rewards/rejected": -5.550379753112793, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 3.12, | |
| "grad_norm": 0.2735817193264242, | |
| "learning_rate": 1.8642084235859764e-06, | |
| "logits/chosen": -1.2955033779144287, | |
| "logits/rejected": -1.6524699926376343, | |
| "logps/chosen": -50.138023376464844, | |
| "logps/rejected": -673.2332763671875, | |
| "loss": 0.0373, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.0004118751676287502, | |
| "rewards/margins": 5.711531639099121, | |
| "rewards/rejected": -5.711943626403809, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 3.2, | |
| "grad_norm": 0.13877956212009288, | |
| "learning_rate": 1.7301162099921013e-06, | |
| "logits/chosen": -1.3686187267303467, | |
| "logits/rejected": -1.6736618280410767, | |
| "logps/chosen": -49.265708923339844, | |
| "logps/rejected": -656.8291625976562, | |
| "loss": 0.035, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.025733808055520058, | |
| "rewards/margins": 5.595955848693848, | |
| "rewards/rejected": -5.5702223777771, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 3.2800000000000002, | |
| "grad_norm": 0.06971630538785722, | |
| "learning_rate": 1.5984291299420117e-06, | |
| "logits/chosen": -1.3197619915008545, | |
| "logits/rejected": -1.6505024433135986, | |
| "logps/chosen": -53.06304168701172, | |
| "logps/rejected": -688.3905639648438, | |
| "loss": 0.0277, | |
| "rewards/accuracies": 0.9937499761581421, | |
| "rewards/chosen": -0.038484591990709305, | |
| "rewards/margins": 5.8287577629089355, | |
| "rewards/rejected": -5.867241859436035, | |
| "step": 410 | |
| }, | |
| { | |
| "epoch": 3.36, | |
| "grad_norm": 0.16813880626958627, | |
| "learning_rate": 1.4695585767104092e-06, | |
| "logits/chosen": -1.35333251953125, | |
| "logits/rejected": -1.6515657901763916, | |
| "logps/chosen": -52.462486267089844, | |
| "logps/rejected": -676.153076171875, | |
| "loss": 0.0198, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.030403846874833107, | |
| "rewards/margins": 5.729649543762207, | |
| "rewards/rejected": -5.760053634643555, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 3.44, | |
| "grad_norm": 6.506151264887685, | |
| "learning_rate": 1.3439071446815452e-06, | |
| "logits/chosen": -1.4076862335205078, | |
| "logits/rejected": -1.6836458444595337, | |
| "logps/chosen": -54.67851638793945, | |
| "logps/rejected": -673.2918701171875, | |
| "loss": 0.0345, | |
| "rewards/accuracies": 0.9937499761581421, | |
| "rewards/chosen": -0.03381548449397087, | |
| "rewards/margins": 5.694668769836426, | |
| "rewards/rejected": -5.728484153747559, | |
| "step": 430 | |
| }, | |
| { | |
| "epoch": 3.52, | |
| "grad_norm": 0.10218706215191123, | |
| "learning_rate": 1.2218673716356919e-06, | |
| "logits/chosen": -1.4333115816116333, | |
| "logits/rejected": -1.6782299280166626, | |
| "logps/chosen": -52.76792526245117, | |
| "logps/rejected": -660.284423828125, | |
| "loss": 0.0298, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.004182952456176281, | |
| "rewards/margins": 5.621959686279297, | |
| "rewards/rejected": -5.617777347564697, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 3.6, | |
| "grad_norm": 0.08325288613545898, | |
| "learning_rate": 1.103820512452661e-06, | |
| "logits/chosen": -1.4696094989776611, | |
| "logits/rejected": -1.7935911417007446, | |
| "logps/chosen": -51.96623611450195, | |
| "logps/rejected": -688.1324462890625, | |
| "loss": 0.042, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.026469239965081215, | |
| "rewards/margins": 5.847690105438232, | |
| "rewards/rejected": -5.874159812927246, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 3.68, | |
| "grad_norm": 0.07973945849285877, | |
| "learning_rate": 9.901353480633468e-07, | |
| "logits/chosen": -1.4555690288543701, | |
| "logits/rejected": -1.784189224243164, | |
| "logps/chosen": -52.75232696533203, | |
| "logps/rejected": -679.7716064453125, | |
| "loss": 0.0362, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.03310062736272812, | |
| "rewards/margins": 5.753149509429932, | |
| "rewards/rejected": -5.786250114440918, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 3.76, | |
| "grad_norm": 0.05215554947056918, | |
| "learning_rate": 8.811670333701544e-07, | |
| "logits/chosen": -1.4402834177017212, | |
| "logits/rejected": -1.730446457862854, | |
| "logps/chosen": -52.232666015625, | |
| "logps/rejected": -685.2855224609375, | |
| "loss": 0.0213, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.015675092115998268, | |
| "rewards/margins": 5.823694705963135, | |
| "rewards/rejected": -5.839369773864746, | |
| "step": 470 | |
| }, | |
| { | |
| "epoch": 3.84, | |
| "grad_norm": 0.8907616320159397, | |
| "learning_rate": 7.772559877354341e-07, | |
| "logits/chosen": -1.5031287670135498, | |
| "logits/rejected": -1.7836637496948242, | |
| "logps/chosen": -54.20964431762695, | |
| "logps/rejected": -671.3470458984375, | |
| "loss": 0.0383, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.03484940528869629, | |
| "rewards/margins": 5.683054447174072, | |
| "rewards/rejected": -5.717903137207031, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 3.92, | |
| "grad_norm": 0.08795166948742635, | |
| "learning_rate": 6.787268315040604e-07, | |
| "logits/chosen": -1.4750686883926392, | |
| "logits/rejected": -1.7797225713729858, | |
| "logps/chosen": -53.79764938354492, | |
| "logps/rejected": -694.2701416015625, | |
| "loss": 0.0221, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.03998209163546562, | |
| "rewards/margins": 5.879731178283691, | |
| "rewards/rejected": -5.919713497161865, | |
| "step": 490 | |
| }, | |
| { | |
| "epoch": 4.0, | |
| "grad_norm": 3.7786029105030954, | |
| "learning_rate": 5.858873718824829e-07, | |
| "logits/chosen": -1.55056893825531, | |
| "logits/rejected": -1.832313895225525, | |
| "logps/chosen": -56.137054443359375, | |
| "logps/rejected": -684.9733276367188, | |
| "loss": 0.0293, | |
| "rewards/accuracies": 0.9937499761581421, | |
| "rewards/chosen": -0.05843985825777054, | |
| "rewards/margins": 5.785708427429199, | |
| "rewards/rejected": -5.844147682189941, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 4.08, | |
| "grad_norm": 0.05189315042878034, | |
| "learning_rate": 4.990276413423817e-07, | |
| "logits/chosen": -1.59425687789917, | |
| "logits/rejected": -1.8570120334625244, | |
| "logps/chosen": -54.740135192871094, | |
| "logps/rejected": -661.8132934570312, | |
| "loss": 0.0275, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.029537344351410866, | |
| "rewards/margins": 5.607969284057617, | |
| "rewards/rejected": -5.63750696182251, | |
| "step": 510 | |
| }, | |
| { | |
| "epoch": 4.16, | |
| "grad_norm": 2.9723273056492787, | |
| "learning_rate": 4.184189915529796e-07, | |
| "logits/chosen": -1.5383813381195068, | |
| "logits/rejected": -1.7925872802734375, | |
| "logps/chosen": -55.53809356689453, | |
| "logps/rejected": -685.3880004882812, | |
| "loss": 0.0312, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.054846394807100296, | |
| "rewards/margins": 5.796133995056152, | |
| "rewards/rejected": -5.850980281829834, | |
| "step": 520 | |
| }, | |
| { | |
| "epoch": 4.24, | |
| "grad_norm": 0.061241630442406096, | |
| "learning_rate": 3.4431324567258176e-07, | |
| "logits/chosen": -1.5691543817520142, | |
| "logits/rejected": -1.838580846786499, | |
| "logps/chosen": -54.7171630859375, | |
| "logps/rejected": -691.39990234375, | |
| "loss": 0.0183, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.05066860839724541, | |
| "rewards/margins": 5.8546953201293945, | |
| "rewards/rejected": -5.905364990234375, | |
| "step": 530 | |
| }, | |
| { | |
| "epoch": 4.32, | |
| "grad_norm": 1.4547140093392317, | |
| "learning_rate": 2.769419116476052e-07, | |
| "logits/chosen": -1.563213586807251, | |
| "logits/rejected": -1.8497707843780518, | |
| "logps/chosen": -58.22840118408203, | |
| "logps/rejected": -689.0776977539062, | |
| "loss": 0.0316, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.0835486501455307, | |
| "rewards/margins": 5.783991813659668, | |
| "rewards/rejected": -5.867539882659912, | |
| "step": 540 | |
| }, | |
| { | |
| "epoch": 4.4, | |
| "grad_norm": 0.09632694899585373, | |
| "learning_rate": 2.1651545897676512e-07, | |
| "logits/chosen": -1.5931028127670288, | |
| "logits/rejected": -1.7936248779296875, | |
| "logps/chosen": -56.02924728393555, | |
| "logps/rejected": -645.4485473632812, | |
| "loss": 0.0301, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.021922914311289787, | |
| "rewards/margins": 5.465601921081543, | |
| "rewards/rejected": -5.48752498626709, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 4.48, | |
| "grad_norm": 0.04943636920795491, | |
| "learning_rate": 1.6322266119983222e-07, | |
| "logits/chosen": -1.5493812561035156, | |
| "logits/rejected": -1.7953119277954102, | |
| "logps/chosen": -56.0436897277832, | |
| "logps/rejected": -691.9136962890625, | |
| "loss": 0.0309, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.048449721187353134, | |
| "rewards/margins": 5.857935905456543, | |
| "rewards/rejected": -5.906384468078613, | |
| "step": 560 | |
| }, | |
| { | |
| "epoch": 4.5600000000000005, | |
| "grad_norm": 0.03808578514394721, | |
| "learning_rate": 1.1723000616502167e-07, | |
| "logits/chosen": -1.60978102684021, | |
| "logits/rejected": -1.8922958374023438, | |
| "logps/chosen": -54.7984733581543, | |
| "logps/rejected": -690.01708984375, | |
| "loss": 0.0255, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.0441301092505455, | |
| "rewards/margins": 5.842381954193115, | |
| "rewards/rejected": -5.88651180267334, | |
| "step": 570 | |
| }, | |
| { | |
| "epoch": 4.64, | |
| "grad_norm": 2.3694244096968626, | |
| "learning_rate": 7.868117591737585e-08, | |
| "logits/chosen": -1.5402578115463257, | |
| "logits/rejected": -1.8241513967514038, | |
| "logps/chosen": -54.9166145324707, | |
| "logps/rejected": -706.30517578125, | |
| "loss": 0.0275, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.06171821430325508, | |
| "rewards/margins": 5.9701948165893555, | |
| "rewards/rejected": -6.0319132804870605, | |
| "step": 580 | |
| }, | |
| { | |
| "epoch": 4.72, | |
| "grad_norm": 0.10922169576399046, | |
| "learning_rate": 4.769659783295383e-08, | |
| "logits/chosen": -1.5660226345062256, | |
| "logits/rejected": -1.8422683477401733, | |
| "logps/chosen": -55.509124755859375, | |
| "logps/rejected": -702.1007690429688, | |
| "loss": 0.0242, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.06676045805215836, | |
| "rewards/margins": 5.940011978149414, | |
| "rewards/rejected": -6.006772041320801, | |
| "step": 590 | |
| }, | |
| { | |
| "epoch": 4.8, | |
| "grad_norm": 2.8341685934157685, | |
| "learning_rate": 2.4373068401120358e-08, | |
| "logits/chosen": -1.5939277410507202, | |
| "logits/rejected": -1.8403106927871704, | |
| "logps/chosen": -60.82611083984375, | |
| "logps/rejected": -691.7379760742188, | |
| "loss": 0.0428, | |
| "rewards/accuracies": 0.9937499761581421, | |
| "rewards/chosen": -0.11341984570026398, | |
| "rewards/margins": 5.790899753570557, | |
| "rewards/rejected": -5.904318809509277, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 4.88, | |
| "grad_norm": 0.07385822938500211, | |
| "learning_rate": 8.78345083022425e-09, | |
| "logits/chosen": -1.59212327003479, | |
| "logits/rejected": -1.8330879211425781, | |
| "logps/chosen": -56.40080642700195, | |
| "logps/rejected": -696.3282470703125, | |
| "loss": 0.0212, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.0736762136220932, | |
| "rewards/margins": 5.873372554779053, | |
| "rewards/rejected": -5.947049140930176, | |
| "step": 610 | |
| }, | |
| { | |
| "epoch": 4.96, | |
| "grad_norm": 0.04029123800852909, | |
| "learning_rate": 9.764474213677654e-10, | |
| "logits/chosen": -1.5612826347351074, | |
| "logits/rejected": -1.7991596460342407, | |
| "logps/chosen": -56.65840530395508, | |
| "logps/rejected": -704.2085571289062, | |
| "loss": 0.019, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.0770476832985878, | |
| "rewards/margins": 5.943432807922363, | |
| "rewards/rejected": -6.020480155944824, | |
| "step": 620 | |
| }, | |
| { | |
| "epoch": 5.0, | |
| "step": 625, | |
| "total_flos": 0.0, | |
| "train_loss": 0.11420744479894639, | |
| "train_runtime": 7032.7914, | |
| "train_samples_per_second": 5.688, | |
| "train_steps_per_second": 0.089 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 625, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 5, | |
| "save_steps": 1000, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 4, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |