| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.9998751404669747, | |
| "eval_steps": 1000, | |
| "global_step": 4004, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.0, | |
| "grad_norm": 0.6953125, | |
| "learning_rate": 1.2468827930174565e-08, | |
| "logits/chosen": -2.4102063179016113, | |
| "logits/rejected": -2.672837734222412, | |
| "logps/chosen": -21.34674835205078, | |
| "logps/rejected": -42.586097717285156, | |
| "loss": 0.6931, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "grad_norm": 0.7421875, | |
| "learning_rate": 1.2468827930174566e-07, | |
| "logits/chosen": -2.239577293395996, | |
| "logits/rejected": -2.476416826248169, | |
| "logps/chosen": -21.881580352783203, | |
| "logps/rejected": -54.84682083129883, | |
| "loss": 0.693, | |
| "rewards/accuracies": 0.5555555820465088, | |
| "rewards/chosen": 0.00018471028306521475, | |
| "rewards/margins": 0.00028743690927512944, | |
| "rewards/rejected": -0.00010272659710608423, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "grad_norm": 0.83984375, | |
| "learning_rate": 2.493765586034913e-07, | |
| "logits/chosen": -2.163784980773926, | |
| "logits/rejected": -2.405578136444092, | |
| "logps/chosen": -21.341472625732422, | |
| "logps/rejected": -55.192710876464844, | |
| "loss": 0.6927, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": 0.000193244923138991, | |
| "rewards/margins": 0.0009195079328492284, | |
| "rewards/rejected": -0.0007262630388140678, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 0.6796875, | |
| "learning_rate": 3.7406483790523695e-07, | |
| "logits/chosen": -2.0837199687957764, | |
| "logits/rejected": -2.361438274383545, | |
| "logps/chosen": -21.834430694580078, | |
| "logps/rejected": -51.4864501953125, | |
| "loss": 0.6929, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": 6.211231811903417e-05, | |
| "rewards/margins": 0.000506018113810569, | |
| "rewards/rejected": -0.00044390588300302625, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 0.80078125, | |
| "learning_rate": 4.987531172069826e-07, | |
| "logits/chosen": -2.088737726211548, | |
| "logits/rejected": -2.3435609340667725, | |
| "logps/chosen": -22.16689682006836, | |
| "logps/rejected": -55.5480842590332, | |
| "loss": 0.6924, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": 0.0003017825947608799, | |
| "rewards/margins": 0.001477475045248866, | |
| "rewards/rejected": -0.0011756925377994776, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 0.70703125, | |
| "learning_rate": 6.234413965087283e-07, | |
| "logits/chosen": -2.1819872856140137, | |
| "logits/rejected": -2.480788469314575, | |
| "logps/chosen": -22.51431655883789, | |
| "logps/rejected": -58.81789016723633, | |
| "loss": 0.6917, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.0005659356247633696, | |
| "rewards/margins": 0.00293327565304935, | |
| "rewards/rejected": -0.0023673397954553366, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 1.1171875, | |
| "learning_rate": 7.481296758104739e-07, | |
| "logits/chosen": -2.13201642036438, | |
| "logits/rejected": -2.3695003986358643, | |
| "logps/chosen": -22.39255142211914, | |
| "logps/rejected": -57.26397705078125, | |
| "loss": 0.6906, | |
| "rewards/accuracies": 0.925000011920929, | |
| "rewards/chosen": 0.0013241939013823867, | |
| "rewards/margins": 0.0052015529945492744, | |
| "rewards/rejected": -0.0038773592095822096, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 0.640625, | |
| "learning_rate": 8.728179551122195e-07, | |
| "logits/chosen": -2.2473509311676025, | |
| "logits/rejected": -2.4891819953918457, | |
| "logps/chosen": -21.353261947631836, | |
| "logps/rejected": -50.47459411621094, | |
| "loss": 0.6888, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.0020654837135225534, | |
| "rewards/margins": 0.00875779427587986, | |
| "rewards/rejected": -0.0066923112608492374, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 0.6796875, | |
| "learning_rate": 9.975062344139653e-07, | |
| "logits/chosen": -2.0938005447387695, | |
| "logits/rejected": -2.3415422439575195, | |
| "logps/chosen": -21.880977630615234, | |
| "logps/rejected": -55.579505920410156, | |
| "loss": 0.6847, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.004472785629332066, | |
| "rewards/margins": 0.016874177381396294, | |
| "rewards/rejected": -0.012401392683386803, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 0.69140625, | |
| "learning_rate": 1.1221945137157108e-06, | |
| "logits/chosen": -2.2169880867004395, | |
| "logits/rejected": -2.43099308013916, | |
| "logps/chosen": -21.09603500366211, | |
| "logps/rejected": -52.89866256713867, | |
| "loss": 0.681, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.007039895746856928, | |
| "rewards/margins": 0.02454659901559353, | |
| "rewards/rejected": -0.017506705597043037, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 0.8203125, | |
| "learning_rate": 1.2468827930174565e-06, | |
| "logits/chosen": -2.079484701156616, | |
| "logits/rejected": -2.3457648754119873, | |
| "logps/chosen": -20.769153594970703, | |
| "logps/rejected": -60.0728645324707, | |
| "loss": 0.6753, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.010782149620354176, | |
| "rewards/margins": 0.03612237051129341, | |
| "rewards/rejected": -0.02534021995961666, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 0.7890625, | |
| "learning_rate": 1.3715710723192023e-06, | |
| "logits/chosen": -2.1157209873199463, | |
| "logits/rejected": -2.373112440109253, | |
| "logps/chosen": -20.276954650878906, | |
| "logps/rejected": -56.75362014770508, | |
| "loss": 0.6659, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.016899898648262024, | |
| "rewards/margins": 0.055193256586790085, | |
| "rewards/rejected": -0.03829335793852806, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 1.4296875, | |
| "learning_rate": 1.4962593516209478e-06, | |
| "logits/chosen": -2.0827651023864746, | |
| "logits/rejected": -2.3430802822113037, | |
| "logps/chosen": -20.093006134033203, | |
| "logps/rejected": -55.97692108154297, | |
| "loss": 0.6578, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.02318699099123478, | |
| "rewards/margins": 0.07196114957332611, | |
| "rewards/rejected": -0.04877415671944618, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 0.88671875, | |
| "learning_rate": 1.6209476309226935e-06, | |
| "logits/chosen": -2.2666783332824707, | |
| "logits/rejected": -2.533306837081909, | |
| "logps/chosen": -18.66347885131836, | |
| "logps/rejected": -60.43751907348633, | |
| "loss": 0.6421, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.03544957563281059, | |
| "rewards/margins": 0.10495258867740631, | |
| "rewards/rejected": -0.06950302422046661, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 0.734375, | |
| "learning_rate": 1.745635910224439e-06, | |
| "logits/chosen": -2.132418155670166, | |
| "logits/rejected": -2.3867433071136475, | |
| "logps/chosen": -16.58405876159668, | |
| "logps/rejected": -65.14090728759766, | |
| "loss": 0.6258, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.052351079881191254, | |
| "rewards/margins": 0.1396940052509308, | |
| "rewards/rejected": -0.08734293282032013, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 0.859375, | |
| "learning_rate": 1.8703241895261848e-06, | |
| "logits/chosen": -2.2100329399108887, | |
| "logits/rejected": -2.446981906890869, | |
| "logps/chosen": -14.807760238647461, | |
| "logps/rejected": -61.867767333984375, | |
| "loss": 0.6085, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.07087540626525879, | |
| "rewards/margins": 0.17729689180850983, | |
| "rewards/rejected": -0.10642149299383163, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 0.890625, | |
| "learning_rate": 1.9950124688279305e-06, | |
| "logits/chosen": -2.2164454460144043, | |
| "logits/rejected": -2.4408745765686035, | |
| "logps/chosen": -12.931081771850586, | |
| "logps/rejected": -65.07794189453125, | |
| "loss": 0.5835, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.09038490056991577, | |
| "rewards/margins": 0.2332114726305008, | |
| "rewards/rejected": -0.14282655715942383, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 0.7578125, | |
| "learning_rate": 2.119700748129676e-06, | |
| "logits/chosen": -2.3733327388763428, | |
| "logits/rejected": -2.637248992919922, | |
| "logps/chosen": -10.747810363769531, | |
| "logps/rejected": -66.96683502197266, | |
| "loss": 0.5576, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.1092229038476944, | |
| "rewards/margins": 0.2931229770183563, | |
| "rewards/rejected": -0.18390007317066193, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 0.94921875, | |
| "learning_rate": 2.2443890274314216e-06, | |
| "logits/chosen": -2.059483051300049, | |
| "logits/rejected": -2.3210699558258057, | |
| "logps/chosen": -9.155550003051758, | |
| "logps/rejected": -81.08940124511719, | |
| "loss": 0.525, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.1273214966058731, | |
| "rewards/margins": 0.371276319026947, | |
| "rewards/rejected": -0.24395480751991272, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 0.88671875, | |
| "learning_rate": 2.3690773067331675e-06, | |
| "logits/chosen": -2.1012320518493652, | |
| "logits/rejected": -2.380855083465576, | |
| "logps/chosen": -8.220524787902832, | |
| "logps/rejected": -86.42562866210938, | |
| "loss": 0.4879, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.13605494797229767, | |
| "rewards/margins": 0.4646981358528137, | |
| "rewards/rejected": -0.32864317297935486, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 1.0546875, | |
| "learning_rate": 2.493765586034913e-06, | |
| "logits/chosen": -2.178351879119873, | |
| "logits/rejected": -2.448537826538086, | |
| "logps/chosen": -7.208306312561035, | |
| "logps/rejected": -91.26810455322266, | |
| "loss": 0.4484, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.15183252096176147, | |
| "rewards/margins": 0.5721064805984497, | |
| "rewards/rejected": -0.42027395963668823, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 0.90625, | |
| "learning_rate": 2.6184538653366586e-06, | |
| "logits/chosen": -2.1630733013153076, | |
| "logits/rejected": -2.411649703979492, | |
| "logps/chosen": -4.617680072784424, | |
| "logps/rejected": -102.69218444824219, | |
| "loss": 0.4025, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.1683187633752823, | |
| "rewards/margins": 0.705622136592865, | |
| "rewards/rejected": -0.5373033881187439, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 0.8828125, | |
| "learning_rate": 2.7431421446384045e-06, | |
| "logits/chosen": -2.1586241722106934, | |
| "logits/rejected": -2.3992838859558105, | |
| "logps/chosen": -3.4560561180114746, | |
| "logps/rejected": -116.10273742675781, | |
| "loss": 0.3606, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.18259310722351074, | |
| "rewards/margins": 0.8416939973831177, | |
| "rewards/rejected": -0.6591008901596069, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 0.953125, | |
| "learning_rate": 2.86783042394015e-06, | |
| "logits/chosen": -2.1585848331451416, | |
| "logits/rejected": -2.381598949432373, | |
| "logps/chosen": -2.7871248722076416, | |
| "logps/rejected": -130.73558044433594, | |
| "loss": 0.3171, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.18752917647361755, | |
| "rewards/margins": 0.9992051124572754, | |
| "rewards/rejected": -0.8116759061813354, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 1.0390625, | |
| "learning_rate": 2.9925187032418956e-06, | |
| "logits/chosen": -2.2422091960906982, | |
| "logits/rejected": -2.491379499435425, | |
| "logps/chosen": -2.960984230041504, | |
| "logps/rejected": -167.91981506347656, | |
| "loss": 0.2364, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.1910472810268402, | |
| "rewards/margins": 1.3611987829208374, | |
| "rewards/rejected": -1.1701514720916748, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 0.64453125, | |
| "learning_rate": 3.117206982543641e-06, | |
| "logits/chosen": -2.1426875591278076, | |
| "logits/rejected": -2.385005474090576, | |
| "logps/chosen": -2.7859835624694824, | |
| "logps/rejected": -208.47451782226562, | |
| "loss": 0.1849, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.1911415457725525, | |
| "rewards/margins": 1.752079963684082, | |
| "rewards/rejected": -1.5609384775161743, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 0.3359375, | |
| "learning_rate": 3.241895261845387e-06, | |
| "logits/chosen": -2.171957492828369, | |
| "logits/rejected": -2.405855894088745, | |
| "logps/chosen": -2.6331560611724854, | |
| "logps/rejected": -266.33441162109375, | |
| "loss": 0.1401, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.18891991674900055, | |
| "rewards/margins": 2.3281731605529785, | |
| "rewards/rejected": -2.1392531394958496, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 0.46484375, | |
| "learning_rate": 3.3665835411471326e-06, | |
| "logits/chosen": -2.136756181716919, | |
| "logits/rejected": -2.3706109523773193, | |
| "logps/chosen": -2.3929905891418457, | |
| "logps/rejected": -311.50164794921875, | |
| "loss": 0.1147, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.19246290624141693, | |
| "rewards/margins": 2.773569107055664, | |
| "rewards/rejected": -2.581106185913086, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 0.1064453125, | |
| "learning_rate": 3.491271820448878e-06, | |
| "logits/chosen": -2.04775071144104, | |
| "logits/rejected": -2.256491184234619, | |
| "logps/chosen": -3.071178436279297, | |
| "logps/rejected": -338.50042724609375, | |
| "loss": 0.1247, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.18945232033729553, | |
| "rewards/margins": 3.0333011150360107, | |
| "rewards/rejected": -2.8438491821289062, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 0.255859375, | |
| "learning_rate": 3.615960099750624e-06, | |
| "logits/chosen": -2.0987088680267334, | |
| "logits/rejected": -2.3304831981658936, | |
| "logps/chosen": -2.6518688201904297, | |
| "logps/rejected": -317.264892578125, | |
| "loss": 0.1174, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.1924947053194046, | |
| "rewards/margins": 2.8529839515686035, | |
| "rewards/rejected": -2.660489082336426, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 0.416015625, | |
| "learning_rate": 3.7406483790523696e-06, | |
| "logits/chosen": -2.090928792953491, | |
| "logits/rejected": -2.331522226333618, | |
| "logps/chosen": -2.140141725540161, | |
| "logps/rejected": -386.000244140625, | |
| "loss": 0.0843, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.19298282265663147, | |
| "rewards/margins": 3.519921064376831, | |
| "rewards/rejected": -3.3269379138946533, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 0.341796875, | |
| "learning_rate": 3.8653366583541155e-06, | |
| "logits/chosen": -2.0327889919281006, | |
| "logits/rejected": -2.2435851097106934, | |
| "logps/chosen": -2.820808172225952, | |
| "logps/rejected": -395.51666259765625, | |
| "loss": 0.1065, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.19103531539440155, | |
| "rewards/margins": 3.636303424835205, | |
| "rewards/rejected": -3.445268154144287, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 0.361328125, | |
| "learning_rate": 3.990024937655861e-06, | |
| "logits/chosen": -2.0326077938079834, | |
| "logits/rejected": -2.2442269325256348, | |
| "logps/chosen": -2.771969795227051, | |
| "logps/rejected": -503.487548828125, | |
| "loss": 0.0783, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.18934586644172668, | |
| "rewards/margins": 4.642784595489502, | |
| "rewards/rejected": -4.453438758850098, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 0.330078125, | |
| "learning_rate": 4.114713216957607e-06, | |
| "logits/chosen": -2.1069068908691406, | |
| "logits/rejected": -2.2923953533172607, | |
| "logps/chosen": -4.020249366760254, | |
| "logps/rejected": -392.1181335449219, | |
| "loss": 0.087, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.18417596817016602, | |
| "rewards/margins": 3.6148147583007812, | |
| "rewards/rejected": -3.4306392669677734, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 0.361328125, | |
| "learning_rate": 4.239401496259352e-06, | |
| "logits/chosen": -2.040024518966675, | |
| "logits/rejected": -2.235044240951538, | |
| "logps/chosen": -4.0639448165893555, | |
| "logps/rejected": -495.93304443359375, | |
| "loss": 0.07, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.17745602130889893, | |
| "rewards/margins": 4.601096153259277, | |
| "rewards/rejected": -4.423640251159668, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 0.1513671875, | |
| "learning_rate": 4.364089775561098e-06, | |
| "logits/chosen": -2.1371054649353027, | |
| "logits/rejected": -2.314563274383545, | |
| "logps/chosen": -5.727438449859619, | |
| "logps/rejected": -448.86077880859375, | |
| "loss": 0.0817, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.1613588184118271, | |
| "rewards/margins": 4.152359962463379, | |
| "rewards/rejected": -3.991001605987549, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 0.44921875, | |
| "learning_rate": 4.488778054862843e-06, | |
| "logits/chosen": -2.1167104244232178, | |
| "logits/rejected": -2.327972412109375, | |
| "logps/chosen": -7.92899227142334, | |
| "logps/rejected": -551.7445068359375, | |
| "loss": 0.0461, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.14445874094963074, | |
| "rewards/margins": 5.136443138122559, | |
| "rewards/rejected": -4.9919843673706055, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 0.5625, | |
| "learning_rate": 4.6134663341645895e-06, | |
| "logits/chosen": -2.1294167041778564, | |
| "logits/rejected": -2.320831537246704, | |
| "logps/chosen": -14.308802604675293, | |
| "logps/rejected": -556.9048461914062, | |
| "loss": 0.0389, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.07796537131071091, | |
| "rewards/margins": 5.144980430603027, | |
| "rewards/rejected": -5.067015171051025, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 0.74609375, | |
| "learning_rate": 4.738154613466335e-06, | |
| "logits/chosen": -2.059129238128662, | |
| "logits/rejected": -2.283139705657959, | |
| "logps/chosen": -30.504648208618164, | |
| "logps/rejected": -904.5690307617188, | |
| "loss": 0.0201, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08387070894241333, | |
| "rewards/margins": 8.345690727233887, | |
| "rewards/rejected": -8.429560661315918, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 0.0244140625, | |
| "learning_rate": 4.862842892768081e-06, | |
| "logits/chosen": -2.1258416175842285, | |
| "logits/rejected": -2.3157804012298584, | |
| "logps/chosen": -44.502899169921875, | |
| "logps/rejected": -788.58056640625, | |
| "loss": 0.0145, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2255050241947174, | |
| "rewards/margins": 7.145709037780762, | |
| "rewards/rejected": -7.371213436126709, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 0.2001953125, | |
| "learning_rate": 4.987531172069826e-06, | |
| "logits/chosen": -2.0560178756713867, | |
| "logits/rejected": -2.2473480701446533, | |
| "logps/chosen": -65.08824157714844, | |
| "logps/rejected": -957.4326171875, | |
| "loss": 0.022, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.4304937422275543, | |
| "rewards/margins": 8.619054794311523, | |
| "rewards/rejected": -9.04954719543457, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 0.1484375, | |
| "learning_rate": 4.999923022460671e-06, | |
| "logits/chosen": -2.0123705863952637, | |
| "logits/rejected": -2.2279648780822754, | |
| "logps/chosen": -74.45851135253906, | |
| "logps/rejected": -1195.67138671875, | |
| "loss": 0.0106, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.5188314318656921, | |
| "rewards/margins": 10.849455833435059, | |
| "rewards/rejected": -11.368288040161133, | |
| "step": 410 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 0.51953125, | |
| "learning_rate": 4.999656933348981e-06, | |
| "logits/chosen": -2.234529972076416, | |
| "logits/rejected": -2.406409740447998, | |
| "logps/chosen": -88.35977935791016, | |
| "logps/rejected": -900.6641845703125, | |
| "loss": 0.0226, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.6683001518249512, | |
| "rewards/margins": 7.860695838928223, | |
| "rewards/rejected": -8.528995513916016, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 0.005340576171875, | |
| "learning_rate": 4.99920080255011e-06, | |
| "logits/chosen": -2.054624080657959, | |
| "logits/rejected": -2.283973217010498, | |
| "logps/chosen": -83.51972198486328, | |
| "logps/rejected": -1245.379150390625, | |
| "loss": 0.0093, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.6176443099975586, | |
| "rewards/margins": 11.277586936950684, | |
| "rewards/rejected": -11.895231246948242, | |
| "step": 430 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 0.5, | |
| "learning_rate": 4.998554664742362e-06, | |
| "logits/chosen": -2.136657476425171, | |
| "logits/rejected": -2.320204973220825, | |
| "logps/chosen": -88.63077545166016, | |
| "logps/rejected": -1043.793701171875, | |
| "loss": 0.0087, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.6717870831489563, | |
| "rewards/margins": 9.280545234680176, | |
| "rewards/rejected": -9.95233154296875, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 0.287109375, | |
| "learning_rate": 4.997718569049726e-06, | |
| "logits/chosen": -2.074990749359131, | |
| "logits/rejected": -2.277477979660034, | |
| "logps/chosen": -94.67475891113281, | |
| "logps/rejected": -1131.5499267578125, | |
| "loss": 0.0145, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.7206478714942932, | |
| "rewards/margins": 10.060680389404297, | |
| "rewards/rejected": -10.781328201293945, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 0.0244140625, | |
| "learning_rate": 4.9966925790381404e-06, | |
| "logits/chosen": -2.1405222415924072, | |
| "logits/rejected": -2.3201403617858887, | |
| "logps/chosen": -73.81873321533203, | |
| "logps/rejected": -1013.6409912109375, | |
| "loss": 0.0119, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.528481125831604, | |
| "rewards/margins": 9.096994400024414, | |
| "rewards/rejected": -9.625473976135254, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 0.19140625, | |
| "learning_rate": 4.995476772710657e-06, | |
| "logits/chosen": -2.0950608253479004, | |
| "logits/rejected": -2.316931962966919, | |
| "logps/chosen": -103.3821792602539, | |
| "logps/rejected": -1335.799560546875, | |
| "loss": 0.021, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.8135234117507935, | |
| "rewards/margins": 11.993521690368652, | |
| "rewards/rejected": -12.807044982910156, | |
| "step": 470 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 0.2451171875, | |
| "learning_rate": 4.994071242501516e-06, | |
| "logits/chosen": -2.185049057006836, | |
| "logits/rejected": -2.3854544162750244, | |
| "logps/chosen": -70.46857452392578, | |
| "logps/rejected": -1074.28271484375, | |
| "loss": 0.0079, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.48124104738235474, | |
| "rewards/margins": 9.766096115112305, | |
| "rewards/rejected": -10.247336387634277, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 2.8967857360839844e-05, | |
| "learning_rate": 4.992476095269112e-06, | |
| "logits/chosen": -2.1872477531433105, | |
| "logits/rejected": -2.3788347244262695, | |
| "logps/chosen": -60.58274459838867, | |
| "logps/rejected": -1168.738037109375, | |
| "loss": 0.0126, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3954273462295532, | |
| "rewards/margins": 10.741006851196289, | |
| "rewards/rejected": -11.136434555053711, | |
| "step": 490 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 0.267578125, | |
| "learning_rate": 4.990691452287877e-06, | |
| "logits/chosen": -2.034578800201416, | |
| "logits/rejected": -2.228356122970581, | |
| "logps/chosen": -70.90155029296875, | |
| "logps/rejected": -1098.248779296875, | |
| "loss": 0.0126, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.49016299843788147, | |
| "rewards/margins": 9.965959548950195, | |
| "rewards/rejected": -10.456122398376465, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 0.2412109375, | |
| "learning_rate": 4.988717449239056e-06, | |
| "logits/chosen": -2.086670398712158, | |
| "logits/rejected": -2.27720046043396, | |
| "logps/chosen": -79.89573669433594, | |
| "logps/rejected": -1124.399658203125, | |
| "loss": 0.0201, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.5849625468254089, | |
| "rewards/margins": 10.160270690917969, | |
| "rewards/rejected": -10.745233535766602, | |
| "step": 510 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 0.0245361328125, | |
| "learning_rate": 4.98655423620039e-06, | |
| "logits/chosen": -2.119935989379883, | |
| "logits/rejected": -2.3267104625701904, | |
| "logps/chosen": -77.09886169433594, | |
| "logps/rejected": -1248.1905517578125, | |
| "loss": 0.0062, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.5508411526679993, | |
| "rewards/margins": 11.388254165649414, | |
| "rewards/rejected": -11.939095497131348, | |
| "step": 520 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 0.01080322265625, | |
| "learning_rate": 4.984201977634711e-06, | |
| "logits/chosen": -2.213916301727295, | |
| "logits/rejected": -2.4463677406311035, | |
| "logps/chosen": -90.18511199951172, | |
| "logps/rejected": -1377.499755859375, | |
| "loss": 0.007, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.6767342686653137, | |
| "rewards/margins": 12.541799545288086, | |
| "rewards/rejected": -13.218534469604492, | |
| "step": 530 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 0.0634765625, | |
| "learning_rate": 4.9816608523774345e-06, | |
| "logits/chosen": -2.105821132659912, | |
| "logits/rejected": -2.3127095699310303, | |
| "logps/chosen": -79.32666015625, | |
| "logps/rejected": -1143.241943359375, | |
| "loss": 0.0101, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.5732973217964172, | |
| "rewards/margins": 10.353887557983398, | |
| "rewards/rejected": -10.927184104919434, | |
| "step": 540 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 0.0145263671875, | |
| "learning_rate": 4.978931053622964e-06, | |
| "logits/chosen": -2.1495628356933594, | |
| "logits/rejected": -2.370626449584961, | |
| "logps/chosen": -78.36927795410156, | |
| "logps/rejected": -1290.0146484375, | |
| "loss": 0.0164, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.5716887712478638, | |
| "rewards/margins": 11.808379173278809, | |
| "rewards/rejected": -12.380067825317383, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 0.0003566741943359375, | |
| "learning_rate": 4.9760127889100044e-06, | |
| "logits/chosen": -2.1675076484680176, | |
| "logits/rejected": -2.3700671195983887, | |
| "logps/chosen": -62.899436950683594, | |
| "logps/rejected": -1186.429443359375, | |
| "loss": 0.0126, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.4144817888736725, | |
| "rewards/margins": 10.914512634277344, | |
| "rewards/rejected": -11.328994750976562, | |
| "step": 560 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 0.23828125, | |
| "learning_rate": 4.972906280105781e-06, | |
| "logits/chosen": -2.0299549102783203, | |
| "logits/rejected": -2.252498149871826, | |
| "logps/chosen": -80.07968139648438, | |
| "logps/rejected": -1246.3656005859375, | |
| "loss": 0.0092, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.5788360238075256, | |
| "rewards/margins": 11.359301567077637, | |
| "rewards/rejected": -11.938138008117676, | |
| "step": 570 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 0.0030670166015625, | |
| "learning_rate": 4.969611763389175e-06, | |
| "logits/chosen": -2.19167423248291, | |
| "logits/rejected": -2.402195453643799, | |
| "logps/chosen": -83.22602844238281, | |
| "logps/rejected": -1125.298095703125, | |
| "loss": 0.0083, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.6159827709197998, | |
| "rewards/margins": 10.143194198608398, | |
| "rewards/rejected": -10.759176254272461, | |
| "step": 580 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 0.197265625, | |
| "learning_rate": 4.966129489232762e-06, | |
| "logits/chosen": -2.1329731941223145, | |
| "logits/rejected": -2.375246286392212, | |
| "logps/chosen": -77.91215515136719, | |
| "logps/rejected": -1410.508056640625, | |
| "loss": 0.0062, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.5520095825195312, | |
| "rewards/margins": 12.985345840454102, | |
| "rewards/rejected": -13.537355422973633, | |
| "step": 590 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 0.431640625, | |
| "learning_rate": 4.962459722383775e-06, | |
| "logits/chosen": -2.0712943077087402, | |
| "logits/rejected": -2.288693428039551, | |
| "logps/chosen": -73.90785217285156, | |
| "logps/rejected": -1434.8343505859375, | |
| "loss": 0.0101, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.5121776461601257, | |
| "rewards/margins": 13.254251480102539, | |
| "rewards/rejected": -13.766427993774414, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 0.000926971435546875, | |
| "learning_rate": 4.958602741843975e-06, | |
| "logits/chosen": -2.0742838382720947, | |
| "logits/rejected": -2.333592176437378, | |
| "logps/chosen": -74.49140167236328, | |
| "logps/rejected": -1376.306396484375, | |
| "loss": 0.0047, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.5260452032089233, | |
| "rewards/margins": 12.687009811401367, | |
| "rewards/rejected": -13.213055610656738, | |
| "step": 610 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 0.08154296875, | |
| "learning_rate": 4.954558840848437e-06, | |
| "logits/chosen": -2.213879346847534, | |
| "logits/rejected": -2.4216055870056152, | |
| "logps/chosen": -65.52778625488281, | |
| "logps/rejected": -1092.6256103515625, | |
| "loss": 0.0097, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.43167224526405334, | |
| "rewards/margins": 10.010717391967773, | |
| "rewards/rejected": -10.442389488220215, | |
| "step": 620 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 0.1318359375, | |
| "learning_rate": 4.950328326843258e-06, | |
| "logits/chosen": -2.0717647075653076, | |
| "logits/rejected": -2.3038885593414307, | |
| "logps/chosen": -59.320228576660156, | |
| "logps/rejected": -1350.045654296875, | |
| "loss": 0.0095, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.37677788734436035, | |
| "rewards/margins": 12.551568984985352, | |
| "rewards/rejected": -12.92834758758545, | |
| "step": 630 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 0.00848388671875, | |
| "learning_rate": 4.945911521462182e-06, | |
| "logits/chosen": -2.2182841300964355, | |
| "logits/rejected": -2.4369709491729736, | |
| "logps/chosen": -66.91700744628906, | |
| "logps/rejected": -1338.757080078125, | |
| "loss": 0.0113, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.45198917388916016, | |
| "rewards/margins": 12.385056495666504, | |
| "rewards/rejected": -12.837045669555664, | |
| "step": 640 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 0.26171875, | |
| "learning_rate": 4.941308760502149e-06, | |
| "logits/chosen": -2.2334372997283936, | |
| "logits/rejected": -2.4091877937316895, | |
| "logps/chosen": -77.1009292602539, | |
| "logps/rejected": -1184.353759765625, | |
| "loss": 0.0162, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.5503292083740234, | |
| "rewards/margins": 10.71965217590332, | |
| "rewards/rejected": -11.26998233795166, | |
| "step": 650 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 0.423828125, | |
| "learning_rate": 4.936520393897762e-06, | |
| "logits/chosen": -2.174837589263916, | |
| "logits/rejected": -2.3993821144104004, | |
| "logps/chosen": -66.07880401611328, | |
| "logps/rejected": -1318.9051513671875, | |
| "loss": 0.0104, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.43549758195877075, | |
| "rewards/margins": 12.193208694458008, | |
| "rewards/rejected": -12.628705978393555, | |
| "step": 660 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "grad_norm": 0.0062255859375, | |
| "learning_rate": 4.931546785694684e-06, | |
| "logits/chosen": -2.2091901302337646, | |
| "logits/rejected": -2.44826078414917, | |
| "logps/chosen": -83.01612091064453, | |
| "logps/rejected": -1474.8822021484375, | |
| "loss": 0.0047, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.6095518469810486, | |
| "rewards/margins": 13.620869636535645, | |
| "rewards/rejected": -14.230420112609863, | |
| "step": 670 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "grad_norm": 0.017822265625, | |
| "learning_rate": 4.926388314021964e-06, | |
| "logits/chosen": -2.2539894580841064, | |
| "logits/rejected": -2.4782536029815674, | |
| "logps/chosen": -97.8957748413086, | |
| "logps/rejected": -1248.599365234375, | |
| "loss": 0.0055, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.7543063759803772, | |
| "rewards/margins": 11.235261917114258, | |
| "rewards/rejected": -11.989568710327148, | |
| "step": 680 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "grad_norm": 0.00060272216796875, | |
| "learning_rate": 4.921045371063283e-06, | |
| "logits/chosen": -2.241508960723877, | |
| "logits/rejected": -2.45992112159729, | |
| "logps/chosen": -75.34230041503906, | |
| "logps/rejected": -1410.6185302734375, | |
| "loss": 0.0053, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.5371032953262329, | |
| "rewards/margins": 13.022886276245117, | |
| "rewards/rejected": -13.559989929199219, | |
| "step": 690 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "grad_norm": 0.09423828125, | |
| "learning_rate": 4.915518363027142e-06, | |
| "logits/chosen": -2.3091747760772705, | |
| "logits/rejected": -2.516079902648926, | |
| "logps/chosen": -77.0430679321289, | |
| "logps/rejected": -1162.167236328125, | |
| "loss": 0.0093, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.5587199926376343, | |
| "rewards/margins": 10.586333274841309, | |
| "rewards/rejected": -11.145052909851074, | |
| "step": 700 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "grad_norm": 0.21484375, | |
| "learning_rate": 4.909807710115977e-06, | |
| "logits/chosen": -2.06872820854187, | |
| "logits/rejected": -2.280989170074463, | |
| "logps/chosen": -57.82390213012695, | |
| "logps/rejected": -1309.2158203125, | |
| "loss": 0.0073, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.36199483275413513, | |
| "rewards/margins": 12.196971893310547, | |
| "rewards/rejected": -12.558965682983398, | |
| "step": 710 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "grad_norm": 0.10595703125, | |
| "learning_rate": 4.903913846494211e-06, | |
| "logits/chosen": -2.057790994644165, | |
| "logits/rejected": -2.2983431816101074, | |
| "logps/chosen": -63.02252960205078, | |
| "logps/rejected": -1628.169677734375, | |
| "loss": 0.0027, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.4083371162414551, | |
| "rewards/margins": 15.24769115447998, | |
| "rewards/rejected": -15.656028747558594, | |
| "step": 720 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "grad_norm": 0.283203125, | |
| "learning_rate": 4.897837220255251e-06, | |
| "logits/chosen": -2.101783275604248, | |
| "logits/rejected": -2.2945144176483154, | |
| "logps/chosen": -62.76520538330078, | |
| "logps/rejected": -1316.2451171875, | |
| "loss": 0.0105, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.4040141999721527, | |
| "rewards/margins": 12.210726737976074, | |
| "rewards/rejected": -12.614742279052734, | |
| "step": 730 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "grad_norm": 0.0218505859375, | |
| "learning_rate": 4.891578293387413e-06, | |
| "logits/chosen": -2.183640241622925, | |
| "logits/rejected": -2.3983137607574463, | |
| "logps/chosen": -72.8852767944336, | |
| "logps/rejected": -1332.2720947265625, | |
| "loss": 0.009, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.5061370134353638, | |
| "rewards/margins": 12.295533180236816, | |
| "rewards/rejected": -12.801671028137207, | |
| "step": 740 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "grad_norm": 0.064453125, | |
| "learning_rate": 4.885137541738808e-06, | |
| "logits/chosen": -2.1432900428771973, | |
| "logits/rejected": -2.3399500846862793, | |
| "logps/chosen": -52.877479553222656, | |
| "logps/rejected": -1188.441650390625, | |
| "loss": 0.0053, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3169303238391876, | |
| "rewards/margins": 11.048254013061523, | |
| "rewards/rejected": -11.365182876586914, | |
| "step": 750 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "grad_norm": 2.086162567138672e-05, | |
| "learning_rate": 4.878515454981153e-06, | |
| "logits/chosen": -2.013054370880127, | |
| "logits/rejected": -2.238393783569336, | |
| "logps/chosen": -60.31416702270508, | |
| "logps/rejected": -1504.335693359375, | |
| "loss": 0.0027, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.38582319021224976, | |
| "rewards/margins": 14.089462280273438, | |
| "rewards/rejected": -14.475286483764648, | |
| "step": 760 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "grad_norm": 0.29296875, | |
| "learning_rate": 4.8717125365725545e-06, | |
| "logits/chosen": -2.2411911487579346, | |
| "logits/rejected": -2.4217007160186768, | |
| "logps/chosen": -71.92973327636719, | |
| "logps/rejected": -1072.138427734375, | |
| "loss": 0.0112, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.4977429509162903, | |
| "rewards/margins": 9.747591018676758, | |
| "rewards/rejected": -10.245333671569824, | |
| "step": 770 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "grad_norm": 0.076171875, | |
| "learning_rate": 4.864729303719221e-06, | |
| "logits/chosen": -2.183976650238037, | |
| "logits/rejected": -2.4096364974975586, | |
| "logps/chosen": -75.01698303222656, | |
| "logps/rejected": -1566.1385498046875, | |
| "loss": 0.0027, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.5285240411758423, | |
| "rewards/margins": 14.545486450195312, | |
| "rewards/rejected": -15.074010848999023, | |
| "step": 780 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "grad_norm": 0.142578125, | |
| "learning_rate": 4.857566287336152e-06, | |
| "logits/chosen": -2.1151528358459473, | |
| "logits/rejected": -2.352687358856201, | |
| "logps/chosen": -81.51287841796875, | |
| "logps/rejected": -1503.718505859375, | |
| "loss": 0.0134, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.5906480550765991, | |
| "rewards/margins": 13.887521743774414, | |
| "rewards/rejected": -14.478169441223145, | |
| "step": 790 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "grad_norm": 0.000972747802734375, | |
| "learning_rate": 4.850224032006765e-06, | |
| "logits/chosen": -2.2330470085144043, | |
| "logits/rejected": -2.4612553119659424, | |
| "logps/chosen": -86.96638488769531, | |
| "logps/rejected": -1362.5970458984375, | |
| "loss": 0.0078, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.6454036831855774, | |
| "rewards/margins": 12.467041969299316, | |
| "rewards/rejected": -13.112444877624512, | |
| "step": 800 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "grad_norm": 0.220703125, | |
| "learning_rate": 4.8427030959414984e-06, | |
| "logits/chosen": -2.0308804512023926, | |
| "logits/rejected": -2.2706708908081055, | |
| "logps/chosen": -80.2787857055664, | |
| "logps/rejected": -1434.3277587890625, | |
| "loss": 0.0083, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.5847833752632141, | |
| "rewards/margins": 13.225725173950195, | |
| "rewards/rejected": -13.810508728027344, | |
| "step": 810 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "grad_norm": 0.193359375, | |
| "learning_rate": 4.835004050935369e-06, | |
| "logits/chosen": -2.134955644607544, | |
| "logits/rejected": -2.338745594024658, | |
| "logps/chosen": -71.83667755126953, | |
| "logps/rejected": -1364.427978515625, | |
| "loss": 0.0155, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.49762091040611267, | |
| "rewards/margins": 12.594762802124023, | |
| "rewards/rejected": -13.092384338378906, | |
| "step": 820 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "grad_norm": 0.13671875, | |
| "learning_rate": 4.8271274823245e-06, | |
| "logits/chosen": -2.1413967609405518, | |
| "logits/rejected": -2.343967914581299, | |
| "logps/chosen": -51.91362762451172, | |
| "logps/rejected": -1269.490478515625, | |
| "loss": 0.012, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2962699830532074, | |
| "rewards/margins": 11.855816841125488, | |
| "rewards/rejected": -12.152085304260254, | |
| "step": 830 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "grad_norm": 0.208984375, | |
| "learning_rate": 4.8190739889416264e-06, | |
| "logits/chosen": -2.1291534900665283, | |
| "logits/rejected": -2.3538265228271484, | |
| "logps/chosen": -51.05685043334961, | |
| "logps/rejected": -1429.0635986328125, | |
| "loss": 0.0046, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.28625136613845825, | |
| "rewards/margins": 13.460894584655762, | |
| "rewards/rejected": -13.74714469909668, | |
| "step": 840 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "grad_norm": 0.06884765625, | |
| "learning_rate": 4.810844183070553e-06, | |
| "logits/chosen": -2.2312417030334473, | |
| "logits/rejected": -2.45286226272583, | |
| "logps/chosen": -65.88993072509766, | |
| "logps/rejected": -1232.466064453125, | |
| "loss": 0.005, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.4434642195701599, | |
| "rewards/margins": 11.373895645141602, | |
| "rewards/rejected": -11.817359924316406, | |
| "step": 850 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "grad_norm": 0.20703125, | |
| "learning_rate": 4.802438690399622e-06, | |
| "logits/chosen": -2.1778035163879395, | |
| "logits/rejected": -2.4104442596435547, | |
| "logps/chosen": -61.74702835083008, | |
| "logps/rejected": -1364.88525390625, | |
| "loss": 0.0092, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.4024318754673004, | |
| "rewards/margins": 12.702482223510742, | |
| "rewards/rejected": -13.104913711547852, | |
| "step": 860 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "grad_norm": 0.00174713134765625, | |
| "learning_rate": 4.793858149974129e-06, | |
| "logits/chosen": -2.142401933670044, | |
| "logits/rejected": -2.3973865509033203, | |
| "logps/chosen": -64.52376556396484, | |
| "logps/rejected": -1546.420654296875, | |
| "loss": 0.0062, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.4257756769657135, | |
| "rewards/margins": 14.488656997680664, | |
| "rewards/rejected": -14.91443157196045, | |
| "step": 870 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "grad_norm": 0.08544921875, | |
| "learning_rate": 4.785103214147747e-06, | |
| "logits/chosen": -2.2586052417755127, | |
| "logits/rejected": -2.4925296306610107, | |
| "logps/chosen": -58.837852478027344, | |
| "logps/rejected": -1360.659423828125, | |
| "loss": 0.0054, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3716233968734741, | |
| "rewards/margins": 12.72169017791748, | |
| "rewards/rejected": -13.093313217163086, | |
| "step": 880 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "grad_norm": 0.0027008056640625, | |
| "learning_rate": 4.776174548532926e-06, | |
| "logits/chosen": -2.158493757247925, | |
| "logits/rejected": -2.3726634979248047, | |
| "logps/chosen": -60.35230255126953, | |
| "logps/rejected": -1398.0426025390625, | |
| "loss": 0.0055, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3876059055328369, | |
| "rewards/margins": 13.057897567749023, | |
| "rewards/rejected": -13.445503234863281, | |
| "step": 890 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "grad_norm": 0.1650390625, | |
| "learning_rate": 4.767072831950288e-06, | |
| "logits/chosen": -2.205594539642334, | |
| "logits/rejected": -2.447887420654297, | |
| "logps/chosen": -58.41968536376953, | |
| "logps/rejected": -1474.5992431640625, | |
| "loss": 0.0035, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.36307448148727417, | |
| "rewards/margins": 13.841937065124512, | |
| "rewards/rejected": -14.205012321472168, | |
| "step": 900 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "grad_norm": 0.05029296875, | |
| "learning_rate": 4.7577987563770226e-06, | |
| "logits/chosen": -2.0987536907196045, | |
| "logits/rejected": -2.3415169715881348, | |
| "logps/chosen": -63.24462890625, | |
| "logps/rejected": -1456.5894775390625, | |
| "loss": 0.0095, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.40772971510887146, | |
| "rewards/margins": 13.594934463500977, | |
| "rewards/rejected": -14.002664566040039, | |
| "step": 910 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "grad_norm": 0.001953125, | |
| "learning_rate": 4.748353026894273e-06, | |
| "logits/chosen": -2.176764965057373, | |
| "logits/rejected": -2.3934457302093506, | |
| "logps/chosen": -77.48558044433594, | |
| "logps/rejected": -1403.7001953125, | |
| "loss": 0.0071, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.5516784191131592, | |
| "rewards/margins": 12.959416389465332, | |
| "rewards/rejected": -13.51109504699707, | |
| "step": 920 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "grad_norm": 0.01708984375, | |
| "learning_rate": 4.738736361633532e-06, | |
| "logits/chosen": -2.2761058807373047, | |
| "logits/rejected": -2.475376605987549, | |
| "logps/chosen": -69.98649597167969, | |
| "logps/rejected": -1317.3599853515625, | |
| "loss": 0.0068, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.4822467267513275, | |
| "rewards/margins": 12.17310619354248, | |
| "rewards/rejected": -12.655353546142578, | |
| "step": 930 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "grad_norm": 0.00396728515625, | |
| "learning_rate": 4.728949491722046e-06, | |
| "logits/chosen": -2.3034911155700684, | |
| "logits/rejected": -2.5063111782073975, | |
| "logps/chosen": -82.03058624267578, | |
| "logps/rejected": -1236.0631103515625, | |
| "loss": 0.0124, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.6094164848327637, | |
| "rewards/margins": 11.258821487426758, | |
| "rewards/rejected": -11.868239402770996, | |
| "step": 940 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "grad_norm": 0.154296875, | |
| "learning_rate": 4.718993161227231e-06, | |
| "logits/chosen": -2.156198740005493, | |
| "logits/rejected": -2.4342427253723145, | |
| "logps/chosen": -52.259849548339844, | |
| "logps/rejected": -1551.4473876953125, | |
| "loss": 0.0057, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2965083420276642, | |
| "rewards/margins": 14.673095703125, | |
| "rewards/rejected": -14.9696044921875, | |
| "step": 950 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "grad_norm": 0.0185546875, | |
| "learning_rate": 4.708868127100098e-06, | |
| "logits/chosen": -2.225891351699829, | |
| "logits/rejected": -2.446601629257202, | |
| "logps/chosen": -45.30867385864258, | |
| "logps/rejected": -1178.07958984375, | |
| "loss": 0.0057, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.24066181480884552, | |
| "rewards/margins": 11.030438423156738, | |
| "rewards/rejected": -11.271100044250488, | |
| "step": 960 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "grad_norm": 0.000530242919921875, | |
| "learning_rate": 4.6985751591177075e-06, | |
| "logits/chosen": -2.071913242340088, | |
| "logits/rejected": -2.3076140880584717, | |
| "logps/chosen": -40.99647521972656, | |
| "logps/rejected": -1356.589599609375, | |
| "loss": 0.006, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.1879800409078598, | |
| "rewards/margins": 12.83641529083252, | |
| "rewards/rejected": -13.024395942687988, | |
| "step": 970 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "grad_norm": 0.0279541015625, | |
| "learning_rate": 4.688115039824648e-06, | |
| "logits/chosen": -2.138272523880005, | |
| "logits/rejected": -2.3490092754364014, | |
| "logps/chosen": -39.265869140625, | |
| "logps/rejected": -1271.980712890625, | |
| "loss": 0.006, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.17672276496887207, | |
| "rewards/margins": 12.01789665222168, | |
| "rewards/rejected": -12.194620132446289, | |
| "step": 980 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "grad_norm": 8.96453857421875e-05, | |
| "learning_rate": 4.677488564473535e-06, | |
| "logits/chosen": -2.0846240520477295, | |
| "logits/rejected": -2.3261351585388184, | |
| "logps/chosen": -54.3425178527832, | |
| "logps/rejected": -1450.612548828125, | |
| "loss": 0.004, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.32301291823387146, | |
| "rewards/margins": 13.639193534851074, | |
| "rewards/rejected": -13.96220588684082, | |
| "step": 990 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "grad_norm": 0.08349609375, | |
| "learning_rate": 4.666696540964556e-06, | |
| "logits/chosen": -2.2266921997070312, | |
| "logits/rejected": -2.44096040725708, | |
| "logps/chosen": -60.16071701049805, | |
| "logps/rejected": -1275.765380859375, | |
| "loss": 0.0046, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3776375949382782, | |
| "rewards/margins": 11.887288093566895, | |
| "rewards/rejected": -12.264925003051758, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "eval_logits/chosen": -2.585369348526001, | |
| "eval_logits/rejected": -2.6955134868621826, | |
| "eval_logps/chosen": -101.94501495361328, | |
| "eval_logps/rejected": -625.497314453125, | |
| "eval_loss": 0.01159477885812521, | |
| "eval_rewards/accuracies": 1.0, | |
| "eval_rewards/chosen": -0.7603151202201843, | |
| "eval_rewards/margins": 5.045938014984131, | |
| "eval_rewards/rejected": -5.806252956390381, | |
| "eval_runtime": 0.6566, | |
| "eval_samples_per_second": 7.615, | |
| "eval_steps_per_second": 4.569, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "grad_norm": 0.000972747802734375, | |
| "learning_rate": 4.6557397897840454e-06, | |
| "logits/chosen": -2.227430820465088, | |
| "logits/rejected": -2.466034412384033, | |
| "logps/chosen": -49.18635940551758, | |
| "logps/rejected": -1351.74853515625, | |
| "loss": 0.0138, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.27514129877090454, | |
| "rewards/margins": 12.695302963256836, | |
| "rewards/rejected": -12.970443725585938, | |
| "step": 1010 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "grad_norm": 0.0011138916015625, | |
| "learning_rate": 4.644619143942108e-06, | |
| "logits/chosen": -2.2175045013427734, | |
| "logits/rejected": -2.4644241333007812, | |
| "logps/chosen": -38.532127380371094, | |
| "logps/rejected": -1415.847412109375, | |
| "loss": 0.003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.16876588761806488, | |
| "rewards/margins": 13.413320541381836, | |
| "rewards/rejected": -13.582087516784668, | |
| "step": 1020 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "grad_norm": 0.08740234375, | |
| "learning_rate": 4.633335448909284e-06, | |
| "logits/chosen": -2.0612175464630127, | |
| "logits/rejected": -2.274484157562256, | |
| "logps/chosen": -39.870052337646484, | |
| "logps/rejected": -1330.5323486328125, | |
| "loss": 0.0046, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.17578956484794617, | |
| "rewards/margins": 12.581718444824219, | |
| "rewards/rejected": -12.75750732421875, | |
| "step": 1030 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "grad_norm": 0.09912109375, | |
| "learning_rate": 4.621889562552272e-06, | |
| "logits/chosen": -2.163442850112915, | |
| "logits/rejected": -2.4233555793762207, | |
| "logps/chosen": -62.47473907470703, | |
| "logps/rejected": -1504.832763671875, | |
| "loss": 0.0076, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.40125980973243713, | |
| "rewards/margins": 14.0916748046875, | |
| "rewards/rejected": -14.492935180664062, | |
| "step": 1040 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "grad_norm": 0.004669189453125, | |
| "learning_rate": 4.610282355068707e-06, | |
| "logits/chosen": -2.2863821983337402, | |
| "logits/rejected": -2.5355706214904785, | |
| "logps/chosen": -59.4514274597168, | |
| "logps/rejected": -1562.39013671875, | |
| "loss": 0.0042, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.36575835943222046, | |
| "rewards/margins": 14.687586784362793, | |
| "rewards/rejected": -15.053342819213867, | |
| "step": 1050 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "grad_norm": 0.64453125, | |
| "learning_rate": 4.598514708921006e-06, | |
| "logits/chosen": -2.262545108795166, | |
| "logits/rejected": -2.510559558868408, | |
| "logps/chosen": -55.71985626220703, | |
| "logps/rejected": -1498.1640625, | |
| "loss": 0.0066, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3440173268318176, | |
| "rewards/margins": 14.074090957641602, | |
| "rewards/rejected": -14.418106079101562, | |
| "step": 1060 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "grad_norm": 0.00142669677734375, | |
| "learning_rate": 4.5865875187692695e-06, | |
| "logits/chosen": -2.2046749591827393, | |
| "logits/rejected": -2.423334836959839, | |
| "logps/chosen": -48.60809326171875, | |
| "logps/rejected": -1244.1680908203125, | |
| "loss": 0.0079, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2774764597415924, | |
| "rewards/margins": 11.670097351074219, | |
| "rewards/rejected": -11.9475736618042, | |
| "step": 1070 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "grad_norm": 0.041015625, | |
| "learning_rate": 4.57450169140327e-06, | |
| "logits/chosen": -2.0672097206115723, | |
| "logits/rejected": -2.3301241397857666, | |
| "logps/chosen": -47.35566711425781, | |
| "logps/rejected": -1535.633544921875, | |
| "loss": 0.0036, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.25116461515426636, | |
| "rewards/margins": 14.5623779296875, | |
| "rewards/rejected": -14.813543319702148, | |
| "step": 1080 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "grad_norm": 0.0615234375, | |
| "learning_rate": 4.562258145673507e-06, | |
| "logits/chosen": -2.2260966300964355, | |
| "logits/rejected": -2.4950501918792725, | |
| "logps/chosen": -40.86091613769531, | |
| "logps/rejected": -1499.596435546875, | |
| "loss": 0.002, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.18965426087379456, | |
| "rewards/margins": 14.258898735046387, | |
| "rewards/rejected": -14.448553085327148, | |
| "step": 1090 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "grad_norm": 0.01068115234375, | |
| "learning_rate": 4.549857812421353e-06, | |
| "logits/chosen": -2.14607572555542, | |
| "logits/rejected": -2.3866307735443115, | |
| "logps/chosen": -44.410030364990234, | |
| "logps/rejected": -1332.9017333984375, | |
| "loss": 0.003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.23083043098449707, | |
| "rewards/margins": 12.585546493530273, | |
| "rewards/rejected": -12.816377639770508, | |
| "step": 1100 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "grad_norm": 0.040283203125, | |
| "learning_rate": 4.537301634408281e-06, | |
| "logits/chosen": -2.169417142868042, | |
| "logits/rejected": -2.4057748317718506, | |
| "logps/chosen": -44.095577239990234, | |
| "logps/rejected": -1315.9925537109375, | |
| "loss": 0.0055, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2211724817752838, | |
| "rewards/margins": 12.42273235321045, | |
| "rewards/rejected": -12.643904685974121, | |
| "step": 1110 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "grad_norm": 0.11376953125, | |
| "learning_rate": 4.52459056624419e-06, | |
| "logits/chosen": -2.217676877975464, | |
| "logits/rejected": -2.4193835258483887, | |
| "logps/chosen": -46.805503845214844, | |
| "logps/rejected": -1376.5738525390625, | |
| "loss": 0.0054, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2504929006099701, | |
| "rewards/margins": 12.966726303100586, | |
| "rewards/rejected": -13.217218399047852, | |
| "step": 1120 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "grad_norm": 0.12451171875, | |
| "learning_rate": 4.51172557431483e-06, | |
| "logits/chosen": -2.1065962314605713, | |
| "logits/rejected": -2.3267951011657715, | |
| "logps/chosen": -61.67560958862305, | |
| "logps/rejected": -1427.5928955078125, | |
| "loss": 0.0055, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.39770394563674927, | |
| "rewards/margins": 13.320897102355957, | |
| "rewards/rejected": -13.718599319458008, | |
| "step": 1130 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "grad_norm": 0.06494140625, | |
| "learning_rate": 4.49870763670833e-06, | |
| "logits/chosen": -2.1609268188476562, | |
| "logits/rejected": -2.4237403869628906, | |
| "logps/chosen": -55.2051887512207, | |
| "logps/rejected": -1529.240966796875, | |
| "loss": 0.0019, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3337005376815796, | |
| "rewards/margins": 14.417539596557617, | |
| "rewards/rejected": -14.751240730285645, | |
| "step": 1140 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "grad_norm": 0.1953125, | |
| "learning_rate": 4.4855377431408335e-06, | |
| "logits/chosen": -2.152674436569214, | |
| "logits/rejected": -2.3682188987731934, | |
| "logps/chosen": -57.719703674316406, | |
| "logps/rejected": -1428.2706298828125, | |
| "loss": 0.0041, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3568641245365143, | |
| "rewards/margins": 13.377996444702148, | |
| "rewards/rejected": -13.73486042022705, | |
| "step": 1150 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "grad_norm": 0.0169677734375, | |
| "learning_rate": 4.472216894881261e-06, | |
| "logits/chosen": -2.146556854248047, | |
| "logits/rejected": -2.361703872680664, | |
| "logps/chosen": -56.385284423828125, | |
| "logps/rejected": -1308.488525390625, | |
| "loss": 0.0046, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.34260934591293335, | |
| "rewards/margins": 12.229646682739258, | |
| "rewards/rejected": -12.572256088256836, | |
| "step": 1160 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "grad_norm": 0.0908203125, | |
| "learning_rate": 4.4587461046751815e-06, | |
| "logits/chosen": -2.1846487522125244, | |
| "logits/rejected": -2.4170939922332764, | |
| "logps/chosen": -47.7278938293457, | |
| "logps/rejected": -1271.083740234375, | |
| "loss": 0.0092, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2519899308681488, | |
| "rewards/margins": 11.94546890258789, | |
| "rewards/rejected": -12.197460174560547, | |
| "step": 1170 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "grad_norm": 0.0247802734375, | |
| "learning_rate": 4.44512639666781e-06, | |
| "logits/chosen": -2.1769089698791504, | |
| "logits/rejected": -2.394580602645874, | |
| "logps/chosen": -61.13446044921875, | |
| "logps/rejected": -1223.753662109375, | |
| "loss": 0.0085, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3933146893978119, | |
| "rewards/margins": 11.356694221496582, | |
| "rewards/rejected": -11.75001049041748, | |
| "step": 1180 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "grad_norm": 0.390625, | |
| "learning_rate": 4.431358806326158e-06, | |
| "logits/chosen": -2.1201298236846924, | |
| "logits/rejected": -2.3456811904907227, | |
| "logps/chosen": -81.98688507080078, | |
| "logps/rejected": -1611.583984375, | |
| "loss": 0.0084, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.6020825505256653, | |
| "rewards/margins": 14.962076187133789, | |
| "rewards/rejected": -15.56415843963623, | |
| "step": 1190 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "grad_norm": 0.609375, | |
| "learning_rate": 4.4174443803603e-06, | |
| "logits/chosen": -2.204873561859131, | |
| "logits/rejected": -2.4108097553253174, | |
| "logps/chosen": -82.76813507080078, | |
| "logps/rejected": -1430.592041015625, | |
| "loss": 0.0083, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.6107802391052246, | |
| "rewards/margins": 13.188611030578613, | |
| "rewards/rejected": -13.79939079284668, | |
| "step": 1200 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "grad_norm": 0.123046875, | |
| "learning_rate": 4.4033841766438e-06, | |
| "logits/chosen": -2.178987503051758, | |
| "logits/rejected": -2.39570689201355, | |
| "logps/chosen": -57.776702880859375, | |
| "logps/rejected": -1284.997802734375, | |
| "loss": 0.0118, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3557528853416443, | |
| "rewards/margins": 11.97689151763916, | |
| "rewards/rejected": -12.33264446258545, | |
| "step": 1210 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "grad_norm": 0.004180908203125, | |
| "learning_rate": 4.389179264133281e-06, | |
| "logits/chosen": -2.260874032974243, | |
| "logits/rejected": -2.495485305786133, | |
| "logps/chosen": -35.43501663208008, | |
| "logps/rejected": -1262.712890625, | |
| "loss": 0.0032, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.1415407359600067, | |
| "rewards/margins": 11.985517501831055, | |
| "rewards/rejected": -12.127059936523438, | |
| "step": 1220 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "grad_norm": 0.337890625, | |
| "learning_rate": 4.374830722787159e-06, | |
| "logits/chosen": -2.265794277191162, | |
| "logits/rejected": -2.539062976837158, | |
| "logps/chosen": -40.56992721557617, | |
| "logps/rejected": -1323.746826171875, | |
| "loss": 0.0105, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.184851735830307, | |
| "rewards/margins": 12.533957481384277, | |
| "rewards/rejected": -12.718809127807617, | |
| "step": 1230 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "grad_norm": 0.10986328125, | |
| "learning_rate": 4.360339643483533e-06, | |
| "logits/chosen": -2.2265820503234863, | |
| "logits/rejected": -2.4537243843078613, | |
| "logps/chosen": -40.92462921142578, | |
| "logps/rejected": -1421.2384033203125, | |
| "loss": 0.0068, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.19266225397586823, | |
| "rewards/margins": 13.46106243133545, | |
| "rewards/rejected": -13.653724670410156, | |
| "step": 1240 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "grad_norm": 0.00179290771484375, | |
| "learning_rate": 4.345707127937253e-06, | |
| "logits/chosen": -2.136321544647217, | |
| "logits/rejected": -2.4158737659454346, | |
| "logps/chosen": -47.67406463623047, | |
| "logps/rejected": -1579.249267578125, | |
| "loss": 0.002, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2534434497356415, | |
| "rewards/margins": 15.00439453125, | |
| "rewards/rejected": -15.257838249206543, | |
| "step": 1250 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "grad_norm": 0.060302734375, | |
| "learning_rate": 4.330934288616154e-06, | |
| "logits/chosen": -2.168765068054199, | |
| "logits/rejected": -2.4067187309265137, | |
| "logps/chosen": -62.91276931762695, | |
| "logps/rejected": -1362.3446044921875, | |
| "loss": 0.003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.40555182099342346, | |
| "rewards/margins": 12.703886032104492, | |
| "rewards/rejected": -13.10943603515625, | |
| "step": 1260 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "grad_norm": 0.150390625, | |
| "learning_rate": 4.316022248656485e-06, | |
| "logits/chosen": -2.1002354621887207, | |
| "logits/rejected": -2.365851402282715, | |
| "logps/chosen": -53.953285217285156, | |
| "logps/rejected": -1585.8782958984375, | |
| "loss": 0.0045, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.32181739807128906, | |
| "rewards/margins": 14.933857917785645, | |
| "rewards/rejected": -15.255674362182617, | |
| "step": 1270 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "grad_norm": 0.024658203125, | |
| "learning_rate": 4.3009721417775166e-06, | |
| "logits/chosen": -2.1251707077026367, | |
| "logits/rejected": -2.363041639328003, | |
| "logps/chosen": -58.41363525390625, | |
| "logps/rejected": -1543.182861328125, | |
| "loss": 0.0032, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.36327052116394043, | |
| "rewards/margins": 14.513801574707031, | |
| "rewards/rejected": -14.87707233428955, | |
| "step": 1280 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "grad_norm": 0.0230712890625, | |
| "learning_rate": 4.285785112195346e-06, | |
| "logits/chosen": -2.1945090293884277, | |
| "logits/rejected": -2.4488844871520996, | |
| "logps/chosen": -69.85707092285156, | |
| "logps/rejected": -1662.5419921875, | |
| "loss": 0.0023, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.47537103295326233, | |
| "rewards/margins": 15.588732719421387, | |
| "rewards/rejected": -16.064102172851562, | |
| "step": 1290 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "grad_norm": 0.035888671875, | |
| "learning_rate": 4.27046231453591e-06, | |
| "logits/chosen": -2.1391608715057373, | |
| "logits/rejected": -2.379563808441162, | |
| "logps/chosen": -63.22686004638672, | |
| "logps/rejected": -1555.8231201171875, | |
| "loss": 0.0017, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.41675299406051636, | |
| "rewards/margins": 14.574475288391113, | |
| "rewards/rejected": -14.991228103637695, | |
| "step": 1300 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "grad_norm": 2.682209014892578e-06, | |
| "learning_rate": 4.255004913747196e-06, | |
| "logits/chosen": -2.1814258098602295, | |
| "logits/rejected": -2.415797710418701, | |
| "logps/chosen": -57.22446823120117, | |
| "logps/rejected": -1578.1937255859375, | |
| "loss": 0.0042, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.35579347610473633, | |
| "rewards/margins": 14.885470390319824, | |
| "rewards/rejected": -15.241262435913086, | |
| "step": 1310 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "grad_norm": 0.0059814453125, | |
| "learning_rate": 4.2394140850106825e-06, | |
| "logits/chosen": -2.1057560443878174, | |
| "logits/rejected": -2.3444247245788574, | |
| "logps/chosen": -60.09722900390625, | |
| "logps/rejected": -1566.6488037109375, | |
| "loss": 0.0071, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.37801143527030945, | |
| "rewards/margins": 14.739140510559082, | |
| "rewards/rejected": -15.117152214050293, | |
| "step": 1320 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "grad_norm": 0.00049591064453125, | |
| "learning_rate": 4.223691013651986e-06, | |
| "logits/chosen": -2.145397424697876, | |
| "logits/rejected": -2.3859896659851074, | |
| "logps/chosen": -50.876380920410156, | |
| "logps/rejected": -1696.628173828125, | |
| "loss": 0.0018, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.28237268328666687, | |
| "rewards/margins": 16.069416046142578, | |
| "rewards/rejected": -16.351787567138672, | |
| "step": 1330 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "grad_norm": 0.076171875, | |
| "learning_rate": 4.207836895050748e-06, | |
| "logits/chosen": -2.290546178817749, | |
| "logits/rejected": -2.601999282836914, | |
| "logps/chosen": -49.566925048828125, | |
| "logps/rejected": -1810.896240234375, | |
| "loss": 0.0039, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.27409371733665466, | |
| "rewards/margins": 17.24148178100586, | |
| "rewards/rejected": -17.515573501586914, | |
| "step": 1340 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "grad_norm": 0.23046875, | |
| "learning_rate": 4.1918529345497525e-06, | |
| "logits/chosen": -2.2135214805603027, | |
| "logits/rejected": -2.4138569831848145, | |
| "logps/chosen": -51.47322463989258, | |
| "logps/rejected": -1197.149169921875, | |
| "loss": 0.0063, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2979530692100525, | |
| "rewards/margins": 11.184531211853027, | |
| "rewards/rejected": -11.482483863830566, | |
| "step": 1350 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "grad_norm": 0.22265625, | |
| "learning_rate": 4.175740347363289e-06, | |
| "logits/chosen": -2.2823052406311035, | |
| "logits/rejected": -2.500483989715576, | |
| "logps/chosen": -50.924964904785156, | |
| "logps/rejected": -1341.169189453125, | |
| "loss": 0.006, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2940273880958557, | |
| "rewards/margins": 12.595098495483398, | |
| "rewards/rejected": -12.889126777648926, | |
| "step": 1360 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "grad_norm": 0.001129150390625, | |
| "learning_rate": 4.159500358484759e-06, | |
| "logits/chosen": -2.1221683025360107, | |
| "logits/rejected": -2.388002872467041, | |
| "logps/chosen": -52.10107421875, | |
| "logps/rejected": -1701.734375, | |
| "loss": 0.0032, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2977118492126465, | |
| "rewards/margins": 16.128461837768555, | |
| "rewards/rejected": -16.42617416381836, | |
| "step": 1370 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "grad_norm": 0.26171875, | |
| "learning_rate": 4.143134202593549e-06, | |
| "logits/chosen": -2.1562037467956543, | |
| "logits/rejected": -2.3721659183502197, | |
| "logps/chosen": -50.73106002807617, | |
| "logps/rejected": -1416.411376953125, | |
| "loss": 0.0053, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.28152209520339966, | |
| "rewards/margins": 13.32117748260498, | |
| "rewards/rejected": -13.602702140808105, | |
| "step": 1380 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "grad_norm": 0.00372314453125, | |
| "learning_rate": 4.126643123961158e-06, | |
| "logits/chosen": -2.2438769340515137, | |
| "logits/rejected": -2.4929661750793457, | |
| "logps/chosen": -71.16793060302734, | |
| "logps/rejected": -1686.4351806640625, | |
| "loss": 0.0042, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.4894431233406067, | |
| "rewards/margins": 15.813896179199219, | |
| "rewards/rejected": -16.303340911865234, | |
| "step": 1390 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "grad_norm": 0.03466796875, | |
| "learning_rate": 4.110028376356599e-06, | |
| "logits/chosen": -2.222071647644043, | |
| "logits/rejected": -2.447359323501587, | |
| "logps/chosen": -70.91515350341797, | |
| "logps/rejected": -1337.4664306640625, | |
| "loss": 0.0046, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.4935643672943115, | |
| "rewards/margins": 12.358075141906738, | |
| "rewards/rejected": -12.851638793945312, | |
| "step": 1400 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "grad_norm": 0.0791015625, | |
| "learning_rate": 4.093291222951079e-06, | |
| "logits/chosen": -2.1609065532684326, | |
| "logits/rejected": -2.4100985527038574, | |
| "logps/chosen": -71.06592559814453, | |
| "logps/rejected": -1599.0948486328125, | |
| "loss": 0.0015, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.49402037262916565, | |
| "rewards/margins": 14.944366455078125, | |
| "rewards/rejected": -15.438386917114258, | |
| "step": 1410 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "grad_norm": 0.234375, | |
| "learning_rate": 4.076432936221965e-06, | |
| "logits/chosen": -2.1633338928222656, | |
| "logits/rejected": -2.3718645572662354, | |
| "logps/chosen": -76.24402618408203, | |
| "logps/rejected": -1331.0867919921875, | |
| "loss": 0.0082, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.5511754751205444, | |
| "rewards/margins": 12.27659797668457, | |
| "rewards/rejected": -12.827774047851562, | |
| "step": 1420 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "grad_norm": 0.07177734375, | |
| "learning_rate": 4.059454797856039e-06, | |
| "logits/chosen": -2.200438976287842, | |
| "logits/rejected": -2.4105000495910645, | |
| "logps/chosen": -72.47054290771484, | |
| "logps/rejected": -1285.55029296875, | |
| "loss": 0.0082, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.5113335251808167, | |
| "rewards/margins": 11.851224899291992, | |
| "rewards/rejected": -12.362558364868164, | |
| "step": 1430 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "grad_norm": 0.0015869140625, | |
| "learning_rate": 4.042358098652057e-06, | |
| "logits/chosen": -2.257859468460083, | |
| "logits/rejected": -2.485215187072754, | |
| "logps/chosen": -52.50494384765625, | |
| "logps/rejected": -1284.864990234375, | |
| "loss": 0.0033, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.30147919058799744, | |
| "rewards/margins": 12.056472778320312, | |
| "rewards/rejected": -12.357951164245605, | |
| "step": 1440 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "grad_norm": 0.00494384765625, | |
| "learning_rate": 4.025144138422615e-06, | |
| "logits/chosen": -2.1999363899230957, | |
| "logits/rejected": -2.436066150665283, | |
| "logps/chosen": -60.535972595214844, | |
| "logps/rejected": -1517.8021240234375, | |
| "loss": 0.0082, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3815072178840637, | |
| "rewards/margins": 14.274576187133789, | |
| "rewards/rejected": -14.656084060668945, | |
| "step": 1450 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "grad_norm": 0.048583984375, | |
| "learning_rate": 4.007814225895321e-06, | |
| "logits/chosen": -2.1949074268341064, | |
| "logits/rejected": -2.453916549682617, | |
| "logps/chosen": -40.10565948486328, | |
| "logps/rejected": -1380.6016845703125, | |
| "loss": 0.0047, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.184524804353714, | |
| "rewards/margins": 13.0862398147583, | |
| "rewards/rejected": -13.270764350891113, | |
| "step": 1460 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "grad_norm": 0.0703125, | |
| "learning_rate": 3.990369678613303e-06, | |
| "logits/chosen": -2.1046247482299805, | |
| "logits/rejected": -2.339478015899658, | |
| "logps/chosen": -32.17253875732422, | |
| "logps/rejected": -1487.5965576171875, | |
| "loss": 0.0023, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.1069754809141159, | |
| "rewards/margins": 14.181404113769531, | |
| "rewards/rejected": -14.288378715515137, | |
| "step": 1470 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "grad_norm": 0.0157470703125, | |
| "learning_rate": 3.97281182283504e-06, | |
| "logits/chosen": -2.168814182281494, | |
| "logits/rejected": -2.4204602241516113, | |
| "logps/chosen": -33.689884185791016, | |
| "logps/rejected": -1507.740966796875, | |
| "loss": 0.0059, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.11984201520681381, | |
| "rewards/margins": 14.405647277832031, | |
| "rewards/rejected": -14.525489807128906, | |
| "step": 1480 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "grad_norm": 0.3828125, | |
| "learning_rate": 3.955141993433526e-06, | |
| "logits/chosen": -2.2266287803649902, | |
| "logits/rejected": -2.45817494392395, | |
| "logps/chosen": -52.63502883911133, | |
| "logps/rejected": -1366.678955078125, | |
| "loss": 0.0046, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3042375445365906, | |
| "rewards/margins": 12.845235824584961, | |
| "rewards/rejected": -13.14947509765625, | |
| "step": 1490 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "grad_norm": 0.08544921875, | |
| "learning_rate": 3.937361533794784e-06, | |
| "logits/chosen": -2.156094551086426, | |
| "logits/rejected": -2.3926451206207275, | |
| "logps/chosen": -44.07966232299805, | |
| "logps/rejected": -1358.091064453125, | |
| "loss": 0.0064, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.22154085338115692, | |
| "rewards/margins": 12.819009780883789, | |
| "rewards/rejected": -13.040552139282227, | |
| "step": 1500 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "grad_norm": 0.021484375, | |
| "learning_rate": 3.919471795715738e-06, | |
| "logits/chosen": -2.212313652038574, | |
| "logits/rejected": -2.4430899620056152, | |
| "logps/chosen": -40.03847122192383, | |
| "logps/rejected": -1265.60009765625, | |
| "loss": 0.0025, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.1857212483882904, | |
| "rewards/margins": 11.979241371154785, | |
| "rewards/rejected": -12.164961814880371, | |
| "step": 1510 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "grad_norm": 0.150390625, | |
| "learning_rate": 3.901474139301433e-06, | |
| "logits/chosen": -2.100083112716675, | |
| "logits/rejected": -2.327531337738037, | |
| "logps/chosen": -47.98102569580078, | |
| "logps/rejected": -1396.822021484375, | |
| "loss": 0.0042, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2628587484359741, | |
| "rewards/margins": 13.183023452758789, | |
| "rewards/rejected": -13.445881843566895, | |
| "step": 1520 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "grad_norm": 0.08740234375, | |
| "learning_rate": 3.883369932861634e-06, | |
| "logits/chosen": -2.2499475479125977, | |
| "logits/rejected": -2.4626846313476562, | |
| "logps/chosen": -53.71254348754883, | |
| "logps/rejected": -1261.4793701171875, | |
| "loss": 0.0016, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.31738823652267456, | |
| "rewards/margins": 11.826452255249023, | |
| "rewards/rejected": -12.143839836120605, | |
| "step": 1530 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "grad_norm": 0.000232696533203125, | |
| "learning_rate": 3.865160552806796e-06, | |
| "logits/chosen": -2.293903350830078, | |
| "logits/rejected": -2.5309927463531494, | |
| "logps/chosen": -59.31644821166992, | |
| "logps/rejected": -1348.3590087890625, | |
| "loss": 0.0049, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.37211018800735474, | |
| "rewards/margins": 12.622639656066895, | |
| "rewards/rejected": -12.994749069213867, | |
| "step": 1540 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "grad_norm": 0.0002765655517578125, | |
| "learning_rate": 3.84684738354342e-06, | |
| "logits/chosen": -2.301741361618042, | |
| "logits/rejected": -2.5269277095794678, | |
| "logps/chosen": -35.07439422607422, | |
| "logps/rejected": -1298.1329345703125, | |
| "loss": 0.0081, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.1266619861125946, | |
| "rewards/margins": 12.352213859558105, | |
| "rewards/rejected": -12.478874206542969, | |
| "step": 1550 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "grad_norm": 0.0712890625, | |
| "learning_rate": 3.828431817368798e-06, | |
| "logits/chosen": -2.15970778465271, | |
| "logits/rejected": -2.3912577629089355, | |
| "logps/chosen": -23.495868682861328, | |
| "logps/rejected": -1347.333740234375, | |
| "loss": 0.0055, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.02085093781352043, | |
| "rewards/margins": 12.912274360656738, | |
| "rewards/rejected": -12.933123588562012, | |
| "step": 1560 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "grad_norm": 0.421875, | |
| "learning_rate": 3.8099152543651684e-06, | |
| "logits/chosen": -2.3851158618927, | |
| "logits/rejected": -2.659996509552002, | |
| "logps/chosen": -34.04401397705078, | |
| "logps/rejected": -1443.980712890625, | |
| "loss": 0.0033, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.12582853436470032, | |
| "rewards/margins": 13.780733108520508, | |
| "rewards/rejected": -13.906559944152832, | |
| "step": 1570 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "grad_norm": 0.154296875, | |
| "learning_rate": 3.791299102293261e-06, | |
| "logits/chosen": -2.125797748565674, | |
| "logits/rejected": -2.3718996047973633, | |
| "logps/chosen": -31.654226303100586, | |
| "logps/rejected": -1515.550048828125, | |
| "loss": 0.0037, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.0940675288438797, | |
| "rewards/margins": 14.493083000183105, | |
| "rewards/rejected": -14.587150573730469, | |
| "step": 1580 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "grad_norm": 0.076171875, | |
| "learning_rate": 3.7725847764852774e-06, | |
| "logits/chosen": -2.117516040802002, | |
| "logits/rejected": -2.376412868499756, | |
| "logps/chosen": -33.58929443359375, | |
| "logps/rejected": -1522.32470703125, | |
| "loss": 0.0054, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.11228612810373306, | |
| "rewards/margins": 14.516647338867188, | |
| "rewards/rejected": -14.628933906555176, | |
| "step": 1590 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "grad_norm": 0.051025390625, | |
| "learning_rate": 3.7537736997372833e-06, | |
| "logits/chosen": -2.183899402618408, | |
| "logits/rejected": -2.4056055545806885, | |
| "logps/chosen": -38.9683723449707, | |
| "logps/rejected": -1303.519287109375, | |
| "loss": 0.0041, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.17212000489234924, | |
| "rewards/margins": 12.313672065734863, | |
| "rewards/rejected": -12.485791206359863, | |
| "step": 1600 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "grad_norm": 0.00689697265625, | |
| "learning_rate": 3.734867302201038e-06, | |
| "logits/chosen": -2.2842166423797607, | |
| "logits/rejected": -2.4898123741149902, | |
| "logps/chosen": -38.427486419677734, | |
| "logps/rejected": -1249.1448974609375, | |
| "loss": 0.0044, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.17140671610832214, | |
| "rewards/margins": 11.829398155212402, | |
| "rewards/rejected": -12.000804901123047, | |
| "step": 1610 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "grad_norm": 0.1435546875, | |
| "learning_rate": 3.7158670212752666e-06, | |
| "logits/chosen": -2.1761648654937744, | |
| "logits/rejected": -2.4285309314727783, | |
| "logps/chosen": -43.9667854309082, | |
| "logps/rejected": -1409.6014404296875, | |
| "loss": 0.0068, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.21925847232341766, | |
| "rewards/margins": 13.351308822631836, | |
| "rewards/rejected": -13.570569038391113, | |
| "step": 1620 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "grad_norm": 0.028076171875, | |
| "learning_rate": 3.696774301496376e-06, | |
| "logits/chosen": -2.253307342529297, | |
| "logits/rejected": -2.4998929500579834, | |
| "logps/chosen": -39.94139862060547, | |
| "logps/rejected": -1315.309814453125, | |
| "loss": 0.0016, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.1798352301120758, | |
| "rewards/margins": 12.484976768493652, | |
| "rewards/rejected": -12.664812088012695, | |
| "step": 1630 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "grad_norm": 0.0152587890625, | |
| "learning_rate": 3.677590594428629e-06, | |
| "logits/chosen": -2.187530517578125, | |
| "logits/rejected": -2.411306142807007, | |
| "logps/chosen": -46.19135284423828, | |
| "logps/rejected": -1337.900390625, | |
| "loss": 0.0063, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2449018955230713, | |
| "rewards/margins": 12.619891166687012, | |
| "rewards/rejected": -12.86479377746582, | |
| "step": 1640 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "grad_norm": 0.0035400390625, | |
| "learning_rate": 3.658317358553794e-06, | |
| "logits/chosen": -2.1583094596862793, | |
| "logits/rejected": -2.399893045425415, | |
| "logps/chosen": -42.413978576660156, | |
| "logps/rejected": -1464.1385498046875, | |
| "loss": 0.0021, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.20776596665382385, | |
| "rewards/margins": 13.914667129516602, | |
| "rewards/rejected": -14.122431755065918, | |
| "step": 1650 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "grad_norm": 0.1435546875, | |
| "learning_rate": 3.638956059160252e-06, | |
| "logits/chosen": -2.2085630893707275, | |
| "logits/rejected": -2.465798854827881, | |
| "logps/chosen": -51.00899887084961, | |
| "logps/rejected": -1475.9312744140625, | |
| "loss": 0.0033, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.28529030084609985, | |
| "rewards/margins": 13.978610038757324, | |
| "rewards/rejected": -14.26390266418457, | |
| "step": 1660 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "grad_norm": 0.05029296875, | |
| "learning_rate": 3.6195081682315972e-06, | |
| "logits/chosen": -2.2395682334899902, | |
| "logits/rejected": -2.461138963699341, | |
| "logps/chosen": -52.74529266357422, | |
| "logps/rejected": -1418.346923828125, | |
| "loss": 0.0021, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3086877167224884, | |
| "rewards/margins": 13.38988208770752, | |
| "rewards/rejected": -13.698568344116211, | |
| "step": 1670 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "grad_norm": 0.08544921875, | |
| "learning_rate": 3.5999751643347342e-06, | |
| "logits/chosen": -2.16579008102417, | |
| "logits/rejected": -2.4046080112457275, | |
| "logps/chosen": -46.71515655517578, | |
| "logps/rejected": -1608.938232421875, | |
| "loss": 0.0044, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.24464385211467743, | |
| "rewards/margins": 15.266571044921875, | |
| "rewards/rejected": -15.51121711730957, | |
| "step": 1680 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "grad_norm": 0.1923828125, | |
| "learning_rate": 3.5803585325074536e-06, | |
| "logits/chosen": -2.1881327629089355, | |
| "logits/rejected": -2.427145481109619, | |
| "logps/chosen": -37.16319274902344, | |
| "logps/rejected": -1421.040771484375, | |
| "loss": 0.0042, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.15972432494163513, | |
| "rewards/margins": 13.539385795593262, | |
| "rewards/rejected": -13.69911003112793, | |
| "step": 1690 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "grad_norm": 0.07275390625, | |
| "learning_rate": 3.5606597641455387e-06, | |
| "logits/chosen": -2.219900369644165, | |
| "logits/rejected": -2.4398694038391113, | |
| "logps/chosen": -32.802005767822266, | |
| "logps/rejected": -1393.4263916015625, | |
| "loss": 0.0058, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.11474724858999252, | |
| "rewards/margins": 13.298954963684082, | |
| "rewards/rejected": -13.413702011108398, | |
| "step": 1700 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "grad_norm": 0.130859375, | |
| "learning_rate": 3.540880356889376e-06, | |
| "logits/chosen": -2.23069429397583, | |
| "logits/rejected": -2.4424965381622314, | |
| "logps/chosen": -42.188209533691406, | |
| "logps/rejected": -1353.078857421875, | |
| "loss": 0.0042, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2047566920518875, | |
| "rewards/margins": 12.800407409667969, | |
| "rewards/rejected": -13.005162239074707, | |
| "step": 1710 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "grad_norm": 0.326171875, | |
| "learning_rate": 3.5210218145100934e-06, | |
| "logits/chosen": -2.1350436210632324, | |
| "logits/rejected": -2.3985211849212646, | |
| "logps/chosen": -51.05349349975586, | |
| "logps/rejected": -1367.61767578125, | |
| "loss": 0.0079, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.29429805278778076, | |
| "rewards/margins": 12.873303413391113, | |
| "rewards/rejected": -13.167600631713867, | |
| "step": 1720 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "grad_norm": 0.134765625, | |
| "learning_rate": 3.5010856467952335e-06, | |
| "logits/chosen": -2.1528429985046387, | |
| "logits/rejected": -2.3955628871917725, | |
| "logps/chosen": -42.9320068359375, | |
| "logps/rejected": -1482.957763671875, | |
| "loss": 0.0051, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.20944638550281525, | |
| "rewards/margins": 14.0554780960083, | |
| "rewards/rejected": -14.264923095703125, | |
| "step": 1730 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "grad_norm": 0.58984375, | |
| "learning_rate": 3.4810733694339687e-06, | |
| "logits/chosen": -2.2495784759521484, | |
| "logits/rejected": -2.512760639190674, | |
| "logps/chosen": -57.50274658203125, | |
| "logps/rejected": -1577.8023681640625, | |
| "loss": 0.0043, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3561248183250427, | |
| "rewards/margins": 14.898828506469727, | |
| "rewards/rejected": -15.25495433807373, | |
| "step": 1740 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "grad_norm": 0.1435546875, | |
| "learning_rate": 3.4609865039018676e-06, | |
| "logits/chosen": -2.2507643699645996, | |
| "logits/rejected": -2.475839614868164, | |
| "logps/chosen": -41.08405685424805, | |
| "logps/rejected": -1401.9703369140625, | |
| "loss": 0.0048, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.20163026452064514, | |
| "rewards/margins": 13.3068265914917, | |
| "rewards/rejected": -13.508456230163574, | |
| "step": 1750 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "grad_norm": 0.044677734375, | |
| "learning_rate": 3.4408265773452226e-06, | |
| "logits/chosen": -2.1668903827667236, | |
| "logits/rejected": -2.4009640216827393, | |
| "logps/chosen": -43.23725891113281, | |
| "logps/rejected": -1427.717041015625, | |
| "loss": 0.006, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.21243014931678772, | |
| "rewards/margins": 13.541119575500488, | |
| "rewards/rejected": -13.753549575805664, | |
| "step": 1760 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "grad_norm": 0.0021514892578125, | |
| "learning_rate": 3.420595122464942e-06, | |
| "logits/chosen": -2.2544631958007812, | |
| "logits/rejected": -2.4994306564331055, | |
| "logps/chosen": -50.723716735839844, | |
| "logps/rejected": -1400.42236328125, | |
| "loss": 0.0032, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2909182906150818, | |
| "rewards/margins": 13.212321281433105, | |
| "rewards/rejected": -13.503240585327148, | |
| "step": 1770 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "grad_norm": 0.1767578125, | |
| "learning_rate": 3.4002936774000284e-06, | |
| "logits/chosen": -2.1552722454071045, | |
| "logits/rejected": -2.4494900703430176, | |
| "logps/chosen": -53.8035888671875, | |
| "logps/rejected": -1743.7855224609375, | |
| "loss": 0.0018, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.31691521406173706, | |
| "rewards/margins": 16.568302154541016, | |
| "rewards/rejected": -16.885215759277344, | |
| "step": 1780 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "grad_norm": 0.04296875, | |
| "learning_rate": 3.3799237856106348e-06, | |
| "logits/chosen": -2.1529643535614014, | |
| "logits/rejected": -2.4126904010772705, | |
| "logps/chosen": -55.90287399291992, | |
| "logps/rejected": -1550.77783203125, | |
| "loss": 0.0024, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.34558922052383423, | |
| "rewards/margins": 14.627325057983398, | |
| "rewards/rejected": -14.97291374206543, | |
| "step": 1790 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "grad_norm": 0.004913330078125, | |
| "learning_rate": 3.35948699576072e-06, | |
| "logits/chosen": -2.108168363571167, | |
| "logits/rejected": -2.371859550476074, | |
| "logps/chosen": -63.180198669433594, | |
| "logps/rejected": -1683.2808837890625, | |
| "loss": 0.0017, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.40645408630371094, | |
| "rewards/margins": 15.862528800964355, | |
| "rewards/rejected": -16.268983840942383, | |
| "step": 1800 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "grad_norm": 0.37890625, | |
| "learning_rate": 3.3389848616003085e-06, | |
| "logits/chosen": -2.202070951461792, | |
| "logits/rejected": -2.4270646572113037, | |
| "logps/chosen": -47.17142105102539, | |
| "logps/rejected": -1418.73046875, | |
| "loss": 0.0088, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.25016140937805176, | |
| "rewards/margins": 13.404383659362793, | |
| "rewards/rejected": -13.654545783996582, | |
| "step": 1810 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "grad_norm": 0.054443359375, | |
| "learning_rate": 3.3184189418473674e-06, | |
| "logits/chosen": -2.0919992923736572, | |
| "logits/rejected": -2.3279192447662354, | |
| "logps/chosen": -37.22324752807617, | |
| "logps/rejected": -1371.5806884765625, | |
| "loss": 0.0047, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.15740033984184265, | |
| "rewards/margins": 13.067277908325195, | |
| "rewards/rejected": -13.224676132202148, | |
| "step": 1820 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "grad_norm": 0.0159912109375, | |
| "learning_rate": 3.2977908000692925e-06, | |
| "logits/chosen": -2.1699509620666504, | |
| "logits/rejected": -2.4078266620635986, | |
| "logps/chosen": -46.939552307128906, | |
| "logps/rejected": -1496.64501953125, | |
| "loss": 0.0078, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.24631306529045105, | |
| "rewards/margins": 14.200462341308594, | |
| "rewards/rejected": -14.44677448272705, | |
| "step": 1830 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "grad_norm": 0.490234375, | |
| "learning_rate": 3.2771020045640435e-06, | |
| "logits/chosen": -2.314471960067749, | |
| "logits/rejected": -2.533036708831787, | |
| "logps/chosen": -49.747779846191406, | |
| "logps/rejected": -1293.560302734375, | |
| "loss": 0.0061, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.27524739503860474, | |
| "rewards/margins": 12.140924453735352, | |
| "rewards/rejected": -12.416172981262207, | |
| "step": 1840 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "grad_norm": 0.19140625, | |
| "learning_rate": 3.256354128240907e-06, | |
| "logits/chosen": -2.101799488067627, | |
| "logits/rejected": -2.320006847381592, | |
| "logps/chosen": -58.1518669128418, | |
| "logps/rejected": -1474.82666015625, | |
| "loss": 0.005, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.361042320728302, | |
| "rewards/margins": 13.842974662780762, | |
| "rewards/rejected": -14.204015731811523, | |
| "step": 1850 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "grad_norm": 2.551823854446411e-07, | |
| "learning_rate": 3.235548748500914e-06, | |
| "logits/chosen": -2.3442602157592773, | |
| "logits/rejected": -2.5813608169555664, | |
| "logps/chosen": -64.3367691040039, | |
| "logps/rejected": -1516.5281982421875, | |
| "loss": 0.0049, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.42836618423461914, | |
| "rewards/margins": 14.244784355163574, | |
| "rewards/rejected": -14.673149108886719, | |
| "step": 1860 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "grad_norm": 0.130859375, | |
| "learning_rate": 3.214687447116913e-06, | |
| "logits/chosen": -2.129812717437744, | |
| "logits/rejected": -2.35500168800354, | |
| "logps/chosen": -60.48137664794922, | |
| "logps/rejected": -1468.074462890625, | |
| "loss": 0.0044, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3904629647731781, | |
| "rewards/margins": 13.763618469238281, | |
| "rewards/rejected": -14.154080390930176, | |
| "step": 1870 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "grad_norm": 0.005218505859375, | |
| "learning_rate": 3.193771810113313e-06, | |
| "logits/chosen": -2.1812546253204346, | |
| "logits/rejected": -2.450334072113037, | |
| "logps/chosen": -57.156097412109375, | |
| "logps/rejected": -1621.7850341796875, | |
| "loss": 0.0022, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.34869837760925293, | |
| "rewards/margins": 15.336013793945312, | |
| "rewards/rejected": -15.684713363647461, | |
| "step": 1880 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "grad_norm": 0.03271484375, | |
| "learning_rate": 3.1728034276455032e-06, | |
| "logits/chosen": -2.1772501468658447, | |
| "logits/rejected": -2.4167187213897705, | |
| "logps/chosen": -47.676063537597656, | |
| "logps/rejected": -1501.980224609375, | |
| "loss": 0.0038, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2580206096172333, | |
| "rewards/margins": 14.23988151550293, | |
| "rewards/rejected": -14.497901916503906, | |
| "step": 1890 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "grad_norm": 0.00274658203125, | |
| "learning_rate": 3.1517838938789597e-06, | |
| "logits/chosen": -2.1416432857513428, | |
| "logits/rejected": -2.3887360095977783, | |
| "logps/chosen": -31.932031631469727, | |
| "logps/rejected": -1682.0501708984375, | |
| "loss": 0.0042, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.10166473686695099, | |
| "rewards/margins": 16.101634979248047, | |
| "rewards/rejected": -16.203296661376953, | |
| "step": 1900 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "grad_norm": 0.36328125, | |
| "learning_rate": 3.130714806868041e-06, | |
| "logits/chosen": -2.132199764251709, | |
| "logits/rejected": -2.3675732612609863, | |
| "logps/chosen": -38.96401596069336, | |
| "logps/rejected": -1434.172607421875, | |
| "loss": 0.0019, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.1679878979921341, | |
| "rewards/margins": 13.631708145141602, | |
| "rewards/rejected": -13.799695014953613, | |
| "step": 1910 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "grad_norm": 0.1484375, | |
| "learning_rate": 3.1095977684344976e-06, | |
| "logits/chosen": -2.221590042114258, | |
| "logits/rejected": -2.477220296859741, | |
| "logps/chosen": -42.42957305908203, | |
| "logps/rejected": -1500.699462890625, | |
| "loss": 0.0022, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2044091671705246, | |
| "rewards/margins": 14.278982162475586, | |
| "rewards/rejected": -14.483392715454102, | |
| "step": 1920 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "grad_norm": 0.0040283203125, | |
| "learning_rate": 3.0884343840456874e-06, | |
| "logits/chosen": -2.280695915222168, | |
| "logits/rejected": -2.5356380939483643, | |
| "logps/chosen": -51.98859405517578, | |
| "logps/rejected": -1650.245361328125, | |
| "loss": 0.002, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3021391034126282, | |
| "rewards/margins": 15.651565551757812, | |
| "rewards/rejected": -15.953704833984375, | |
| "step": 1930 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "grad_norm": 0.0002536773681640625, | |
| "learning_rate": 3.0672262626925174e-06, | |
| "logits/chosen": -2.1820268630981445, | |
| "logits/rejected": -2.439319133758545, | |
| "logps/chosen": -47.39429473876953, | |
| "logps/rejected": -1611.6102294921875, | |
| "loss": 0.0032, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.24379031360149384, | |
| "rewards/margins": 15.318346977233887, | |
| "rewards/rejected": -15.56213665008545, | |
| "step": 1940 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "grad_norm": 3.910064697265625e-05, | |
| "learning_rate": 3.0459750167671147e-06, | |
| "logits/chosen": -2.1863160133361816, | |
| "logits/rejected": -2.450911283493042, | |
| "logps/chosen": -57.97031784057617, | |
| "logps/rejected": -1733.2484130859375, | |
| "loss": 0.0017, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.35936951637268066, | |
| "rewards/margins": 16.4171199798584, | |
| "rewards/rejected": -16.776485443115234, | |
| "step": 1950 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "grad_norm": 0.375, | |
| "learning_rate": 3.024682261940247e-06, | |
| "logits/chosen": -2.1711161136627197, | |
| "logits/rejected": -2.381054401397705, | |
| "logps/chosen": -63.16656494140625, | |
| "logps/rejected": -1473.282958984375, | |
| "loss": 0.0036, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.40881744027137756, | |
| "rewards/margins": 13.772600173950195, | |
| "rewards/rejected": -14.181416511535645, | |
| "step": 1960 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "grad_norm": 0.000568389892578125, | |
| "learning_rate": 3.0033496170384803e-06, | |
| "logits/chosen": -2.232100009918213, | |
| "logits/rejected": -2.4612276554107666, | |
| "logps/chosen": -56.055152893066406, | |
| "logps/rejected": -1356.71484375, | |
| "loss": 0.0077, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3471956253051758, | |
| "rewards/margins": 12.729546546936035, | |
| "rewards/rejected": -13.076742172241211, | |
| "step": 1970 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "grad_norm": 0.10791015625, | |
| "learning_rate": 2.9819787039211068e-06, | |
| "logits/chosen": -2.1615240573883057, | |
| "logits/rejected": -2.393810510635376, | |
| "logps/chosen": -35.02969741821289, | |
| "logps/rejected": -1524.7955322265625, | |
| "loss": 0.0023, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.13164165616035461, | |
| "rewards/margins": 14.575594902038574, | |
| "rewards/rejected": -14.707235336303711, | |
| "step": 1980 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "grad_norm": 0.0054931640625, | |
| "learning_rate": 2.960571147356845e-06, | |
| "logits/chosen": -2.256544828414917, | |
| "logits/rejected": -2.5309910774230957, | |
| "logps/chosen": -49.80757522583008, | |
| "logps/rejected": -1592.794677734375, | |
| "loss": 0.0021, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2703229784965515, | |
| "rewards/margins": 15.133091926574707, | |
| "rewards/rejected": -15.403416633605957, | |
| "step": 1990 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "grad_norm": 0.0023040771484375, | |
| "learning_rate": 2.9391285749003046e-06, | |
| "logits/chosen": -2.15415620803833, | |
| "logits/rejected": -2.405571460723877, | |
| "logps/chosen": -40.737998962402344, | |
| "logps/rejected": -1701.052734375, | |
| "loss": 0.0037, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.18067236244678497, | |
| "rewards/margins": 16.24726104736328, | |
| "rewards/rejected": -16.427934646606445, | |
| "step": 2000 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "eval_logits/chosen": -2.6136603355407715, | |
| "eval_logits/rejected": -2.7333316802978516, | |
| "eval_logps/chosen": -48.08984375, | |
| "eval_logps/rejected": -693.2846069335938, | |
| "eval_loss": 0.0037064917851239443, | |
| "eval_rewards/accuracies": 1.0, | |
| "eval_rewards/chosen": -0.22176341712474823, | |
| "eval_rewards/margins": 6.262362003326416, | |
| "eval_rewards/rejected": -6.48412561416626, | |
| "eval_runtime": 0.6544, | |
| "eval_samples_per_second": 7.641, | |
| "eval_steps_per_second": 4.585, | |
| "step": 2000 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "grad_norm": 0.6328125, | |
| "learning_rate": 2.9176526167682543e-06, | |
| "logits/chosen": -2.1183362007141113, | |
| "logits/rejected": -2.351123571395874, | |
| "logps/chosen": -37.299964904785156, | |
| "logps/rejected": -1437.7230224609375, | |
| "loss": 0.0057, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.15952737629413605, | |
| "rewards/margins": 13.696490287780762, | |
| "rewards/rejected": -13.85601806640625, | |
| "step": 2010 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "grad_norm": 0.03857421875, | |
| "learning_rate": 2.8961449057156775e-06, | |
| "logits/chosen": -2.200801372528076, | |
| "logits/rejected": -2.4389915466308594, | |
| "logps/chosen": -42.25465774536133, | |
| "logps/rejected": -1569.040771484375, | |
| "loss": 0.0018, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.20262226462364197, | |
| "rewards/margins": 14.958514213562012, | |
| "rewards/rejected": -15.16113567352295, | |
| "step": 2020 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "grad_norm": 0.0037689208984375, | |
| "learning_rate": 2.874607076911642e-06, | |
| "logits/chosen": -2.212007999420166, | |
| "logits/rejected": -2.4628169536590576, | |
| "logps/chosen": -54.49187088012695, | |
| "logps/rejected": -1452.176513671875, | |
| "loss": 0.0043, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.31496500968933105, | |
| "rewards/margins": 13.68268871307373, | |
| "rewards/rejected": -13.997654914855957, | |
| "step": 2030 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "grad_norm": 0.0026092529296875, | |
| "learning_rate": 2.8530407678149806e-06, | |
| "logits/chosen": -2.1855294704437256, | |
| "logits/rejected": -2.428863525390625, | |
| "logps/chosen": -61.762428283691406, | |
| "logps/rejected": -1588.792236328125, | |
| "loss": 0.0051, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.4000323414802551, | |
| "rewards/margins": 14.938389778137207, | |
| "rewards/rejected": -15.338421821594238, | |
| "step": 2040 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "grad_norm": 0.001739501953125, | |
| "learning_rate": 2.8314476180498003e-06, | |
| "logits/chosen": -2.0332534313201904, | |
| "logits/rejected": -2.267488718032837, | |
| "logps/chosen": -41.453369140625, | |
| "logps/rejected": -1475.7647705078125, | |
| "loss": 0.0048, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.1973056197166443, | |
| "rewards/margins": 14.028945922851562, | |
| "rewards/rejected": -14.226251602172852, | |
| "step": 2050 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "grad_norm": 0.13671875, | |
| "learning_rate": 2.8098292692808253e-06, | |
| "logits/chosen": -2.2281060218811035, | |
| "logits/rejected": -2.422762632369995, | |
| "logps/chosen": -41.2132453918457, | |
| "logps/rejected": -1153.19775390625, | |
| "loss": 0.0045, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.1940850019454956, | |
| "rewards/margins": 10.885441780090332, | |
| "rewards/rejected": -11.079526901245117, | |
| "step": 2060 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "grad_norm": 0.162109375, | |
| "learning_rate": 2.7881873650885904e-06, | |
| "logits/chosen": -2.227834463119507, | |
| "logits/rejected": -2.4453253746032715, | |
| "logps/chosen": -50.43096160888672, | |
| "logps/rejected": -1375.1741943359375, | |
| "loss": 0.0044, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2844979166984558, | |
| "rewards/margins": 12.969167709350586, | |
| "rewards/rejected": -13.25366497039795, | |
| "step": 2070 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "grad_norm": 0.1513671875, | |
| "learning_rate": 2.7665235508444772e-06, | |
| "logits/chosen": -2.1580593585968018, | |
| "logits/rejected": -2.404978036880493, | |
| "logps/chosen": -47.8787841796875, | |
| "logps/rejected": -1663.496826171875, | |
| "loss": 0.0047, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2667320966720581, | |
| "rewards/margins": 15.83378791809082, | |
| "rewards/rejected": -16.10051727294922, | |
| "step": 2080 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "grad_norm": 0.000820159912109375, | |
| "learning_rate": 2.7448394735856275e-06, | |
| "logits/chosen": -2.1202292442321777, | |
| "logits/rejected": -2.387399196624756, | |
| "logps/chosen": -29.072830200195312, | |
| "logps/rejected": -1652.7601318359375, | |
| "loss": 0.0069, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.07453112304210663, | |
| "rewards/margins": 15.884991645812988, | |
| "rewards/rejected": -15.959524154663086, | |
| "step": 2090 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "grad_norm": 0.1259765625, | |
| "learning_rate": 2.723136781889722e-06, | |
| "logits/chosen": -2.248565912246704, | |
| "logits/rejected": -2.483459949493408, | |
| "logps/chosen": -49.5106201171875, | |
| "logps/rejected": -1374.344482421875, | |
| "loss": 0.008, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.26804444193840027, | |
| "rewards/margins": 12.9815092086792, | |
| "rewards/rejected": -13.249552726745605, | |
| "step": 2100 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "grad_norm": 0.0208740234375, | |
| "learning_rate": 2.7014171257496414e-06, | |
| "logits/chosen": -2.2338385581970215, | |
| "logits/rejected": -2.4451489448547363, | |
| "logps/chosen": -47.859092712402344, | |
| "logps/rejected": -1475.451416015625, | |
| "loss": 0.0025, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.259570837020874, | |
| "rewards/margins": 13.942883491516113, | |
| "rewards/rejected": -14.20245361328125, | |
| "step": 2110 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "grad_norm": 0.2216796875, | |
| "learning_rate": 2.6796821564480237e-06, | |
| "logits/chosen": -2.1667749881744385, | |
| "logits/rejected": -2.3811049461364746, | |
| "logps/chosen": -51.062232971191406, | |
| "logps/rejected": -1320.582763671875, | |
| "loss": 0.005, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2894384562969208, | |
| "rewards/margins": 12.419047355651855, | |
| "rewards/rejected": -12.708486557006836, | |
| "step": 2120 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "grad_norm": 0.00604248046875, | |
| "learning_rate": 2.6579335264317253e-06, | |
| "logits/chosen": -2.3176040649414062, | |
| "logits/rejected": -2.558061361312866, | |
| "logps/chosen": -36.845001220703125, | |
| "logps/rejected": -1507.6351318359375, | |
| "loss": 0.0031, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.15489184856414795, | |
| "rewards/margins": 14.385536193847656, | |
| "rewards/rejected": -14.540430068969727, | |
| "step": 2130 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "grad_norm": 0.00830078125, | |
| "learning_rate": 2.6361728891861843e-06, | |
| "logits/chosen": -2.067624568939209, | |
| "logits/rejected": -2.2963385581970215, | |
| "logps/chosen": -45.42739486694336, | |
| "logps/rejected": -1546.36083984375, | |
| "loss": 0.0049, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.23402588069438934, | |
| "rewards/margins": 14.646380424499512, | |
| "rewards/rejected": -14.880406379699707, | |
| "step": 2140 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "grad_norm": 0.0032806396484375, | |
| "learning_rate": 2.614401899109716e-06, | |
| "logits/chosen": -2.247525930404663, | |
| "logits/rejected": -2.4837863445281982, | |
| "logps/chosen": -48.529815673828125, | |
| "logps/rejected": -1460.3306884765625, | |
| "loss": 0.0019, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2696106433868408, | |
| "rewards/margins": 13.825261116027832, | |
| "rewards/rejected": -14.094873428344727, | |
| "step": 2150 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "grad_norm": 0.06201171875, | |
| "learning_rate": 2.5926222113877282e-06, | |
| "logits/chosen": -2.243438482284546, | |
| "logits/rejected": -2.4923970699310303, | |
| "logps/chosen": -43.63848114013672, | |
| "logps/rejected": -1591.591064453125, | |
| "loss": 0.0028, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2216021567583084, | |
| "rewards/margins": 15.112271308898926, | |
| "rewards/rejected": -15.333871841430664, | |
| "step": 2160 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "grad_norm": 0.205078125, | |
| "learning_rate": 2.570835481866889e-06, | |
| "logits/chosen": -2.144465923309326, | |
| "logits/rejected": -2.3723580837249756, | |
| "logps/chosen": -45.58980178833008, | |
| "logps/rejected": -1466.3011474609375, | |
| "loss": 0.0027, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2433461844921112, | |
| "rewards/margins": 13.90100383758545, | |
| "rewards/rejected": -14.1443510055542, | |
| "step": 2170 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "grad_norm": 0.140625, | |
| "learning_rate": 2.5490433669292337e-06, | |
| "logits/chosen": -2.0634944438934326, | |
| "logits/rejected": -2.311782121658325, | |
| "logps/chosen": -37.41926193237305, | |
| "logps/rejected": -1625.005126953125, | |
| "loss": 0.0029, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.1547044962644577, | |
| "rewards/margins": 15.56786060333252, | |
| "rewards/rejected": -15.722564697265625, | |
| "step": 2180 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "grad_norm": 0.00244140625, | |
| "learning_rate": 2.527247523366232e-06, | |
| "logits/chosen": -2.2304885387420654, | |
| "logits/rejected": -2.4748549461364746, | |
| "logps/chosen": -54.11591339111328, | |
| "logps/rejected": -1548.42578125, | |
| "loss": 0.0022, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.31723862886428833, | |
| "rewards/margins": 14.627525329589844, | |
| "rewards/rejected": -14.9447660446167, | |
| "step": 2190 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "grad_norm": 0.01214599609375, | |
| "learning_rate": 2.5054496082528336e-06, | |
| "logits/chosen": -2.2945401668548584, | |
| "logits/rejected": -2.553946018218994, | |
| "logps/chosen": -50.36088180541992, | |
| "logps/rejected": -1503.6251220703125, | |
| "loss": 0.0011, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2821517586708069, | |
| "rewards/margins": 14.262479782104492, | |
| "rewards/rejected": -14.544631958007812, | |
| "step": 2200 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "grad_norm": 0.07373046875, | |
| "learning_rate": 2.483651278821481e-06, | |
| "logits/chosen": -2.240737199783325, | |
| "logits/rejected": -2.468348264694214, | |
| "logps/chosen": -38.926151275634766, | |
| "logps/rejected": -1415.637939453125, | |
| "loss": 0.0029, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.17295430600643158, | |
| "rewards/margins": 13.47205638885498, | |
| "rewards/rejected": -13.645009994506836, | |
| "step": 2210 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "grad_norm": 0.111328125, | |
| "learning_rate": 2.4618541923361166e-06, | |
| "logits/chosen": -2.4229185581207275, | |
| "logits/rejected": -2.6283278465270996, | |
| "logps/chosen": -44.134647369384766, | |
| "logps/rejected": -1301.507080078125, | |
| "loss": 0.0028, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2265511453151703, | |
| "rewards/margins": 12.291707038879395, | |
| "rewards/rejected": -12.518258094787598, | |
| "step": 2220 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "grad_norm": 0.001983642578125, | |
| "learning_rate": 2.4400600059661836e-06, | |
| "logits/chosen": -2.0719246864318848, | |
| "logits/rejected": -2.375192165374756, | |
| "logps/chosen": -46.18827819824219, | |
| "logps/rejected": -1760.173095703125, | |
| "loss": 0.0025, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.24284347891807556, | |
| "rewards/margins": 16.816072463989258, | |
| "rewards/rejected": -17.058916091918945, | |
| "step": 2230 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "grad_norm": 0.046630859375, | |
| "learning_rate": 2.41827037666064e-06, | |
| "logits/chosen": -2.2636351585388184, | |
| "logits/rejected": -2.4840915203094482, | |
| "logps/chosen": -47.29922103881836, | |
| "logps/rejected": -1315.336181640625, | |
| "loss": 0.0042, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.25474974513053894, | |
| "rewards/margins": 12.407878875732422, | |
| "rewards/rejected": -12.662630081176758, | |
| "step": 2240 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "grad_norm": 0.01141357421875, | |
| "learning_rate": 2.396486961021983e-06, | |
| "logits/chosen": -2.1793510913848877, | |
| "logits/rejected": -2.4308459758758545, | |
| "logps/chosen": -41.266380310058594, | |
| "logps/rejected": -1442.6708984375, | |
| "loss": 0.0068, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.19039396941661835, | |
| "rewards/margins": 13.722501754760742, | |
| "rewards/rejected": -13.912895202636719, | |
| "step": 2250 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "grad_norm": 0.010498046875, | |
| "learning_rate": 2.3747114151802993e-06, | |
| "logits/chosen": -2.3280482292175293, | |
| "logits/rejected": -2.5701987743377686, | |
| "logps/chosen": -47.68052673339844, | |
| "logps/rejected": -1394.991455078125, | |
| "loss": 0.0042, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2576830983161926, | |
| "rewards/margins": 13.206995010375977, | |
| "rewards/rejected": -13.464675903320312, | |
| "step": 2260 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "grad_norm": 0.08349609375, | |
| "learning_rate": 2.352945394667363e-06, | |
| "logits/chosen": -2.0980782508850098, | |
| "logits/rejected": -2.364197254180908, | |
| "logps/chosen": -47.831058502197266, | |
| "logps/rejected": -1665.154296875, | |
| "loss": 0.0021, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.25840622186660767, | |
| "rewards/margins": 15.833898544311523, | |
| "rewards/rejected": -16.092304229736328, | |
| "step": 2270 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "grad_norm": 0.44921875, | |
| "learning_rate": 2.3311905542907627e-06, | |
| "logits/chosen": -2.256291389465332, | |
| "logits/rejected": -2.486441135406494, | |
| "logps/chosen": -42.5937614440918, | |
| "logps/rejected": -1361.2073974609375, | |
| "loss": 0.0041, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2082025557756424, | |
| "rewards/margins": 12.911099433898926, | |
| "rewards/rejected": -13.119302749633789, | |
| "step": 2280 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "grad_norm": 0.06591796875, | |
| "learning_rate": 2.30944854800809e-06, | |
| "logits/chosen": -2.2147023677825928, | |
| "logits/rejected": -2.4364144802093506, | |
| "logps/chosen": -40.498531341552734, | |
| "logps/rejected": -1479.181396484375, | |
| "loss": 0.0044, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.18588443100452423, | |
| "rewards/margins": 14.101339340209961, | |
| "rewards/rejected": -14.287226676940918, | |
| "step": 2290 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "grad_norm": 0.004364013671875, | |
| "learning_rate": 2.287721028801204e-06, | |
| "logits/chosen": -2.175849676132202, | |
| "logits/rejected": -2.4008584022521973, | |
| "logps/chosen": -43.332298278808594, | |
| "logps/rejected": -1385.960205078125, | |
| "loss": 0.0032, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.21884088218212128, | |
| "rewards/margins": 13.133634567260742, | |
| "rewards/rejected": -13.352476119995117, | |
| "step": 2300 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "grad_norm": 0.057861328125, | |
| "learning_rate": 2.26600964855055e-06, | |
| "logits/chosen": -2.2437031269073486, | |
| "logits/rejected": -2.4617691040039062, | |
| "logps/chosen": -43.779388427734375, | |
| "logps/rejected": -1358.084716796875, | |
| "loss": 0.002, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.22216272354125977, | |
| "rewards/margins": 12.88685417175293, | |
| "rewards/rejected": -13.109016418457031, | |
| "step": 2310 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "grad_norm": 0.0230712890625, | |
| "learning_rate": 2.244316057909573e-06, | |
| "logits/chosen": -2.205610752105713, | |
| "logits/rejected": -2.4241251945495605, | |
| "logps/chosen": -37.175682067871094, | |
| "logps/rejected": -1405.6822509765625, | |
| "loss": 0.0044, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.15614905953407288, | |
| "rewards/margins": 13.411378860473633, | |
| "rewards/rejected": -13.567527770996094, | |
| "step": 2320 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "grad_norm": 0.0208740234375, | |
| "learning_rate": 2.2226419061792282e-06, | |
| "logits/chosen": -2.284442901611328, | |
| "logits/rejected": -2.527775526046753, | |
| "logps/chosen": -48.442630767822266, | |
| "logps/rejected": -1552.012939453125, | |
| "loss": 0.0022, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2705609202384949, | |
| "rewards/margins": 14.728759765625, | |
| "rewards/rejected": -14.999322891235352, | |
| "step": 2330 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "grad_norm": 0.0081787109375, | |
| "learning_rate": 2.200988841182589e-06, | |
| "logits/chosen": -2.219576358795166, | |
| "logits/rejected": -2.4669265747070312, | |
| "logps/chosen": -44.78432083129883, | |
| "logps/rejected": -1667.3538818359375, | |
| "loss": 0.0046, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2340162992477417, | |
| "rewards/margins": 15.922113418579102, | |
| "rewards/rejected": -16.156129837036133, | |
| "step": 2340 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "grad_norm": 0.01287841796875, | |
| "learning_rate": 2.179358509139559e-06, | |
| "logits/chosen": -2.171391010284424, | |
| "logits/rejected": -2.4009640216827393, | |
| "logps/chosen": -61.962059020996094, | |
| "logps/rejected": -1350.2457275390625, | |
| "loss": 0.0053, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3955707848072052, | |
| "rewards/margins": 12.617085456848145, | |
| "rewards/rejected": -13.012655258178711, | |
| "step": 2350 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "grad_norm": 0.0517578125, | |
| "learning_rate": 2.1577525545417254e-06, | |
| "logits/chosen": -2.1860475540161133, | |
| "logits/rejected": -2.418872594833374, | |
| "logps/chosen": -58.040443420410156, | |
| "logps/rejected": -1468.6392822265625, | |
| "loss": 0.0035, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3626258969306946, | |
| "rewards/margins": 13.818751335144043, | |
| "rewards/rejected": -14.181378364562988, | |
| "step": 2360 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "grad_norm": 0.0186767578125, | |
| "learning_rate": 2.1361726200273293e-06, | |
| "logits/chosen": -2.2700607776641846, | |
| "logits/rejected": -2.521707057952881, | |
| "logps/chosen": -48.01632308959961, | |
| "logps/rejected": -1544.2449951171875, | |
| "loss": 0.0019, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.25974512100219727, | |
| "rewards/margins": 14.68183708190918, | |
| "rewards/rejected": -14.941583633422852, | |
| "step": 2370 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "grad_norm": 0.0859375, | |
| "learning_rate": 2.1146203462563773e-06, | |
| "logits/chosen": -2.335644483566284, | |
| "logits/rejected": -2.5736241340637207, | |
| "logps/chosen": -39.67052459716797, | |
| "logps/rejected": -1430.16796875, | |
| "loss": 0.0017, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.18928556144237518, | |
| "rewards/margins": 13.622029304504395, | |
| "rewards/rejected": -13.811314582824707, | |
| "step": 2380 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "grad_norm": 0.08056640625, | |
| "learning_rate": 2.0930973717859117e-06, | |
| "logits/chosen": -2.352358341217041, | |
| "logits/rejected": -2.598140239715576, | |
| "logps/chosen": -44.863258361816406, | |
| "logps/rejected": -1464.2244873046875, | |
| "loss": 0.0028, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.23050542175769806, | |
| "rewards/margins": 13.890615463256836, | |
| "rewards/rejected": -14.121121406555176, | |
| "step": 2390 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "grad_norm": 0.00011587142944335938, | |
| "learning_rate": 2.0716053329454337e-06, | |
| "logits/chosen": -2.07816481590271, | |
| "logits/rejected": -2.320413112640381, | |
| "logps/chosen": -42.12782287597656, | |
| "logps/rejected": -1603.09716796875, | |
| "loss": 0.004, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.20372910797595978, | |
| "rewards/margins": 15.295297622680664, | |
| "rewards/rejected": -15.49902629852295, | |
| "step": 2400 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "grad_norm": 0.0255126953125, | |
| "learning_rate": 2.0501458637124963e-06, | |
| "logits/chosen": -2.2174525260925293, | |
| "logits/rejected": -2.5070488452911377, | |
| "logps/chosen": -49.52367401123047, | |
| "logps/rejected": -1763.182861328125, | |
| "loss": 0.0022, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2707751393318176, | |
| "rewards/margins": 16.840404510498047, | |
| "rewards/rejected": -17.11117935180664, | |
| "step": 2410 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "grad_norm": 0.0206298828125, | |
| "learning_rate": 2.0287205955884812e-06, | |
| "logits/chosen": -2.2282018661499023, | |
| "logits/rejected": -2.47560453414917, | |
| "logps/chosen": -39.33561706542969, | |
| "logps/rejected": -1586.3204345703125, | |
| "loss": 0.0043, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.17531004548072815, | |
| "rewards/margins": 15.11853313446045, | |
| "rewards/rejected": -15.293844223022461, | |
| "step": 2420 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "grad_norm": 0.0439453125, | |
| "learning_rate": 2.0073311574745583e-06, | |
| "logits/chosen": -2.1908931732177734, | |
| "logits/rejected": -2.4531962871551514, | |
| "logps/chosen": -46.56280517578125, | |
| "logps/rejected": -1645.0823974609375, | |
| "loss": 0.0009, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.24412044882774353, | |
| "rewards/margins": 15.665544509887695, | |
| "rewards/rejected": -15.909663200378418, | |
| "step": 2430 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "grad_norm": 0.01141357421875, | |
| "learning_rate": 1.9859791755478453e-06, | |
| "logits/chosen": -2.2081665992736816, | |
| "logits/rejected": -2.4285478591918945, | |
| "logps/chosen": -36.21527099609375, | |
| "logps/rejected": -1296.198486328125, | |
| "loss": 0.0019, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.14344918727874756, | |
| "rewards/margins": 12.341351509094238, | |
| "rewards/rejected": -12.484800338745117, | |
| "step": 2440 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "grad_norm": 0.2353515625, | |
| "learning_rate": 1.9646662731377737e-06, | |
| "logits/chosen": -2.157654285430908, | |
| "logits/rejected": -2.3908090591430664, | |
| "logps/chosen": -45.09668731689453, | |
| "logps/rejected": -1404.2435302734375, | |
| "loss": 0.0089, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.23492522537708282, | |
| "rewards/margins": 13.305872917175293, | |
| "rewards/rejected": -13.540797233581543, | |
| "step": 2450 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "grad_norm": 0.01287841796875, | |
| "learning_rate": 1.9433940706026743e-06, | |
| "logits/chosen": -2.1844208240509033, | |
| "logits/rejected": -2.438828468322754, | |
| "logps/chosen": -47.74811935424805, | |
| "logps/rejected": -1656.3568115234375, | |
| "loss": 0.0045, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2575169503688812, | |
| "rewards/margins": 15.76098346710205, | |
| "rewards/rejected": -16.01849937438965, | |
| "step": 2460 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "grad_norm": 0.000972747802734375, | |
| "learning_rate": 1.9221641852065807e-06, | |
| "logits/chosen": -2.18261456489563, | |
| "logits/rejected": -2.4000496864318848, | |
| "logps/chosen": -44.85232925415039, | |
| "logps/rejected": -1401.519775390625, | |
| "loss": 0.0039, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.22903266549110413, | |
| "rewards/margins": 13.285311698913574, | |
| "rewards/rejected": -13.51434326171875, | |
| "step": 2470 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "grad_norm": 0.06201171875, | |
| "learning_rate": 1.9009782309962805e-06, | |
| "logits/chosen": -2.281862497329712, | |
| "logits/rejected": -2.5180306434631348, | |
| "logps/chosen": -35.415897369384766, | |
| "logps/rejected": -1375.5728759765625, | |
| "loss": 0.0017, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.13269564509391785, | |
| "rewards/margins": 13.11363697052002, | |
| "rewards/rejected": -13.246332168579102, | |
| "step": 2480 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "grad_norm": 0.09423828125, | |
| "learning_rate": 1.8798378186785979e-06, | |
| "logits/chosen": -2.2361299991607666, | |
| "logits/rejected": -2.4721415042877197, | |
| "logps/chosen": -30.1846923828125, | |
| "logps/rejected": -1444.7518310546875, | |
| "loss": 0.003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.0831189677119255, | |
| "rewards/margins": 13.8583402633667, | |
| "rewards/rejected": -13.941459655761719, | |
| "step": 2490 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "grad_norm": 0.035400390625, | |
| "learning_rate": 1.8587445554979404e-06, | |
| "logits/chosen": -2.073253870010376, | |
| "logits/rejected": -2.3244481086730957, | |
| "logps/chosen": -36.15102005004883, | |
| "logps/rejected": -1567.7164306640625, | |
| "loss": 0.0049, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.1387818604707718, | |
| "rewards/margins": 14.997014045715332, | |
| "rewards/rejected": -15.135795593261719, | |
| "step": 2500 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "grad_norm": 0.0006103515625, | |
| "learning_rate": 1.8377000451141013e-06, | |
| "logits/chosen": -2.120227336883545, | |
| "logits/rejected": -2.379242420196533, | |
| "logps/chosen": -42.131507873535156, | |
| "logps/rejected": -1565.419189453125, | |
| "loss": 0.0073, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.207178995013237, | |
| "rewards/margins": 14.909128189086914, | |
| "rewards/rejected": -15.116305351257324, | |
| "step": 2510 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "grad_norm": 0.0927734375, | |
| "learning_rate": 1.8167058874803405e-06, | |
| "logits/chosen": -2.234502077102661, | |
| "logits/rejected": -2.4847466945648193, | |
| "logps/chosen": -42.673118591308594, | |
| "logps/rejected": -1594.222900390625, | |
| "loss": 0.003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2050865888595581, | |
| "rewards/margins": 15.173876762390137, | |
| "rewards/rejected": -15.3789644241333, | |
| "step": 2520 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "grad_norm": 0.138671875, | |
| "learning_rate": 1.7957636787217451e-06, | |
| "logits/chosen": -2.1729538440704346, | |
| "logits/rejected": -2.4276270866394043, | |
| "logps/chosen": -26.112987518310547, | |
| "logps/rejected": -1523.0291748046875, | |
| "loss": 0.0015, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.044597070664167404, | |
| "rewards/margins": 14.675150871276855, | |
| "rewards/rejected": -14.719749450683594, | |
| "step": 2530 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "grad_norm": 0.0322265625, | |
| "learning_rate": 1.7748750110138768e-06, | |
| "logits/chosen": -2.106745481491089, | |
| "logits/rejected": -2.3529787063598633, | |
| "logps/chosen": -38.04988098144531, | |
| "logps/rejected": -1700.769287109375, | |
| "loss": 0.0026, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.15463842451572418, | |
| "rewards/margins": 16.276233673095703, | |
| "rewards/rejected": -16.430871963500977, | |
| "step": 2540 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "grad_norm": 0.0849609375, | |
| "learning_rate": 1.7540414724617282e-06, | |
| "logits/chosen": -2.070836067199707, | |
| "logits/rejected": -2.3102221488952637, | |
| "logps/chosen": -41.751487731933594, | |
| "logps/rejected": -1488.04931640625, | |
| "loss": 0.0045, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.19269555807113647, | |
| "rewards/margins": 14.147076606750488, | |
| "rewards/rejected": -14.33977222442627, | |
| "step": 2550 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "grad_norm": 0.00946044921875, | |
| "learning_rate": 1.7332646469789827e-06, | |
| "logits/chosen": -2.2572789192199707, | |
| "logits/rejected": -2.481287956237793, | |
| "logps/chosen": -29.41888427734375, | |
| "logps/rejected": -1229.242431640625, | |
| "loss": 0.0023, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.07652698457241058, | |
| "rewards/margins": 11.74826717376709, | |
| "rewards/rejected": -11.824793815612793, | |
| "step": 2560 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "grad_norm": 0.0830078125, | |
| "learning_rate": 1.7125461141675881e-06, | |
| "logits/chosen": -2.1423022747039795, | |
| "logits/rejected": -2.3926641941070557, | |
| "logps/chosen": -30.7061710357666, | |
| "logps/rejected": -1465.9014892578125, | |
| "loss": 0.006, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.09345726668834686, | |
| "rewards/margins": 14.04102897644043, | |
| "rewards/rejected": -14.134485244750977, | |
| "step": 2570 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "grad_norm": 0.00014209747314453125, | |
| "learning_rate": 1.6918874491976744e-06, | |
| "logits/chosen": -2.290851354598999, | |
| "logits/rejected": -2.5240445137023926, | |
| "logps/chosen": -36.6445198059082, | |
| "logps/rejected": -1480.7818603515625, | |
| "loss": 0.0023, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.149479478597641, | |
| "rewards/margins": 14.131547927856445, | |
| "rewards/rejected": -14.281025886535645, | |
| "step": 2580 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "grad_norm": 0.002655029296875, | |
| "learning_rate": 1.6712902226877917e-06, | |
| "logits/chosen": -2.1575067043304443, | |
| "logits/rejected": -2.402039051055908, | |
| "logps/chosen": -45.549842834472656, | |
| "logps/rejected": -1545.114990234375, | |
| "loss": 0.0043, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.23408398032188416, | |
| "rewards/margins": 14.681310653686523, | |
| "rewards/rejected": -14.91539478302002, | |
| "step": 2590 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "grad_norm": 0.0028839111328125, | |
| "learning_rate": 1.6507560005854977e-06, | |
| "logits/chosen": -2.066991090774536, | |
| "logits/rejected": -2.3206119537353516, | |
| "logps/chosen": -47.11815643310547, | |
| "logps/rejected": -1413.30126953125, | |
| "loss": 0.0025, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.24164950847625732, | |
| "rewards/margins": 13.29053020477295, | |
| "rewards/rejected": -13.532180786132812, | |
| "step": 2600 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "grad_norm": 0.03759765625, | |
| "learning_rate": 1.6302863440483121e-06, | |
| "logits/chosen": -2.1091551780700684, | |
| "logits/rejected": -2.394484043121338, | |
| "logps/chosen": -54.07494354248047, | |
| "logps/rejected": -1674.350341796875, | |
| "loss": 0.0029, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3039209842681885, | |
| "rewards/margins": 15.897099494934082, | |
| "rewards/rejected": -16.201021194458008, | |
| "step": 2610 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "grad_norm": 0.035400390625, | |
| "learning_rate": 1.6098828093250203e-06, | |
| "logits/chosen": -2.0393662452697754, | |
| "logits/rejected": -2.2912774085998535, | |
| "logps/chosen": -43.22583770751953, | |
| "logps/rejected": -1745.9964599609375, | |
| "loss": 0.0034, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.20912513136863708, | |
| "rewards/margins": 16.64576530456543, | |
| "rewards/rejected": -16.85489273071289, | |
| "step": 2620 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "grad_norm": 0.038330078125, | |
| "learning_rate": 1.5895469476373545e-06, | |
| "logits/chosen": -2.12833833694458, | |
| "logits/rejected": -2.353044033050537, | |
| "logps/chosen": -51.28118133544922, | |
| "logps/rejected": -1477.751708984375, | |
| "loss": 0.0025, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2927466034889221, | |
| "rewards/margins": 13.93517017364502, | |
| "rewards/rejected": -14.227917671203613, | |
| "step": 2630 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "grad_norm": 0.002105712890625, | |
| "learning_rate": 1.5692803050620642e-06, | |
| "logits/chosen": -2.146883726119995, | |
| "logits/rejected": -2.3877830505371094, | |
| "logps/chosen": -42.891048431396484, | |
| "logps/rejected": -1572.1334228515625, | |
| "loss": 0.0018, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.20707789063453674, | |
| "rewards/margins": 14.972661018371582, | |
| "rewards/rejected": -15.17973804473877, | |
| "step": 2640 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "grad_norm": 0.05029296875, | |
| "learning_rate": 1.5490844224133717e-06, | |
| "logits/chosen": -2.2065834999084473, | |
| "logits/rejected": -2.4583041667938232, | |
| "logps/chosen": -58.87604522705078, | |
| "logps/rejected": -1606.8402099609375, | |
| "loss": 0.0013, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3703632652759552, | |
| "rewards/margins": 15.166918754577637, | |
| "rewards/rejected": -15.53728199005127, | |
| "step": 2650 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "grad_norm": 0.06298828125, | |
| "learning_rate": 1.528960835125822e-06, | |
| "logits/chosen": -2.3619742393493652, | |
| "logits/rejected": -2.5886929035186768, | |
| "logps/chosen": -47.88628005981445, | |
| "logps/rejected": -1394.3492431640625, | |
| "loss": 0.0019, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2605898976325989, | |
| "rewards/margins": 13.201173782348633, | |
| "rewards/rejected": -13.461764335632324, | |
| "step": 2660 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "grad_norm": 0.373046875, | |
| "learning_rate": 1.5089110731375568e-06, | |
| "logits/chosen": -2.1769912242889404, | |
| "logits/rejected": -2.4125704765319824, | |
| "logps/chosen": -54.75007247924805, | |
| "logps/rejected": -1521.18310546875, | |
| "loss": 0.0043, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.32575544714927673, | |
| "rewards/margins": 14.370445251464844, | |
| "rewards/rejected": -14.696202278137207, | |
| "step": 2670 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "grad_norm": 0.080078125, | |
| "learning_rate": 1.4889366607739925e-06, | |
| "logits/chosen": -2.322796583175659, | |
| "logits/rejected": -2.5181009769439697, | |
| "logps/chosen": -45.69524383544922, | |
| "logps/rejected": -1201.08251953125, | |
| "loss": 0.0025, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.24491195380687714, | |
| "rewards/margins": 11.296446800231934, | |
| "rewards/rejected": -11.541359901428223, | |
| "step": 2680 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "grad_norm": 0.03076171875, | |
| "learning_rate": 1.4690391166319307e-06, | |
| "logits/chosen": -2.1181106567382812, | |
| "logits/rejected": -2.3545029163360596, | |
| "logps/chosen": -43.6742057800293, | |
| "logps/rejected": -1542.8385009765625, | |
| "loss": 0.0021, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.21549615263938904, | |
| "rewards/margins": 14.664543151855469, | |
| "rewards/rejected": -14.880040168762207, | |
| "step": 2690 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "grad_norm": 0.5078125, | |
| "learning_rate": 1.4492199534641055e-06, | |
| "logits/chosen": -2.21667218208313, | |
| "logits/rejected": -2.4625155925750732, | |
| "logps/chosen": -47.34065628051758, | |
| "logps/rejected": -1472.347900390625, | |
| "loss": 0.0059, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.25726571679115295, | |
| "rewards/margins": 13.980672836303711, | |
| "rewards/rejected": -14.237937927246094, | |
| "step": 2700 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "grad_norm": 0.0111083984375, | |
| "learning_rate": 1.429480678064174e-06, | |
| "logits/chosen": -2.2022199630737305, | |
| "logits/rejected": -2.4795124530792236, | |
| "logps/chosen": -51.5767822265625, | |
| "logps/rejected": -1817.763671875, | |
| "loss": 0.0006, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2908291518688202, | |
| "rewards/margins": 17.308137893676758, | |
| "rewards/rejected": -17.598966598510742, | |
| "step": 2710 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "grad_norm": 0.76171875, | |
| "learning_rate": 1.4098227911521523e-06, | |
| "logits/chosen": -2.219804286956787, | |
| "logits/rejected": -2.462226390838623, | |
| "logps/chosen": -46.08092498779297, | |
| "logps/rejected": -1526.5423583984375, | |
| "loss": 0.0037, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.23279635608196259, | |
| "rewards/margins": 14.499191284179688, | |
| "rewards/rejected": -14.731986999511719, | |
| "step": 2720 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "grad_norm": 0.058349609375, | |
| "learning_rate": 1.3902477872603295e-06, | |
| "logits/chosen": -2.319612503051758, | |
| "logits/rejected": -2.517526149749756, | |
| "logps/chosen": -40.09135055541992, | |
| "logps/rejected": -1286.315185546875, | |
| "loss": 0.005, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.18802298605442047, | |
| "rewards/margins": 12.167932510375977, | |
| "rewards/rejected": -12.355955123901367, | |
| "step": 2730 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "grad_norm": 0.0859375, | |
| "learning_rate": 1.370757154619638e-06, | |
| "logits/chosen": -2.2395832538604736, | |
| "logits/rejected": -2.470933198928833, | |
| "logps/chosen": -56.124351501464844, | |
| "logps/rejected": -1607.262939453125, | |
| "loss": 0.0026, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.33287256956100464, | |
| "rewards/margins": 15.186471939086914, | |
| "rewards/rejected": -15.519342422485352, | |
| "step": 2740 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "grad_norm": 8.761882781982422e-06, | |
| "learning_rate": 1.3513523750465049e-06, | |
| "logits/chosen": -2.2328319549560547, | |
| "logits/rejected": -2.4625821113586426, | |
| "logps/chosen": -39.597564697265625, | |
| "logps/rejected": -1417.1002197265625, | |
| "loss": 0.0024, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.17806950211524963, | |
| "rewards/margins": 13.474327087402344, | |
| "rewards/rejected": -13.652397155761719, | |
| "step": 2750 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "grad_norm": 0.0030517578125, | |
| "learning_rate": 1.332034923830199e-06, | |
| "logits/chosen": -2.136444568634033, | |
| "logits/rejected": -2.3981611728668213, | |
| "logps/chosen": -44.04825210571289, | |
| "logps/rejected": -1527.0830078125, | |
| "loss": 0.0012, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.22466015815734863, | |
| "rewards/margins": 14.532748222351074, | |
| "rewards/rejected": -14.757411003112793, | |
| "step": 2760 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "grad_norm": 0.130859375, | |
| "learning_rate": 1.31280626962067e-06, | |
| "logits/chosen": -2.2737619876861572, | |
| "logits/rejected": -2.488204002380371, | |
| "logps/chosen": -49.427528381347656, | |
| "logps/rejected": -1363.083740234375, | |
| "loss": 0.0017, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.27385297417640686, | |
| "rewards/margins": 12.863523483276367, | |
| "rewards/rejected": -13.137374877929688, | |
| "step": 2770 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "grad_norm": 0.0162353515625, | |
| "learning_rate": 1.2936678743168813e-06, | |
| "logits/chosen": -2.2063140869140625, | |
| "logits/rejected": -2.450613498687744, | |
| "logps/chosen": -47.45142364501953, | |
| "logps/rejected": -1494.2115478515625, | |
| "loss": 0.0048, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2554013133049011, | |
| "rewards/margins": 14.182083129882812, | |
| "rewards/rejected": -14.437482833862305, | |
| "step": 2780 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "grad_norm": 0.06298828125, | |
| "learning_rate": 1.2746211929556777e-06, | |
| "logits/chosen": -2.171708583831787, | |
| "logits/rejected": -2.4892578125, | |
| "logps/chosen": -47.59801483154297, | |
| "logps/rejected": -1927.90234375, | |
| "loss": 0.0008, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.25736138224601746, | |
| "rewards/margins": 18.46487808227539, | |
| "rewards/rejected": -18.722238540649414, | |
| "step": 2790 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "grad_norm": 5.900859832763672e-06, | |
| "learning_rate": 1.2556676736011558e-06, | |
| "logits/chosen": -2.200247287750244, | |
| "logits/rejected": -2.433065891265869, | |
| "logps/chosen": -47.600555419921875, | |
| "logps/rejected": -1647.335693359375, | |
| "loss": 0.0029, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.25239020586013794, | |
| "rewards/margins": 15.647438049316406, | |
| "rewards/rejected": -15.89982795715332, | |
| "step": 2800 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "grad_norm": 0.0279541015625, | |
| "learning_rate": 1.2368087572345772e-06, | |
| "logits/chosen": -2.235849380493164, | |
| "logits/rejected": -2.4413654804229736, | |
| "logps/chosen": -48.26280975341797, | |
| "logps/rejected": -1258.86572265625, | |
| "loss": 0.005, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.26613086462020874, | |
| "rewards/margins": 11.844823837280273, | |
| "rewards/rejected": -12.110954284667969, | |
| "step": 2810 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "grad_norm": 1.2734375, | |
| "learning_rate": 1.2180458776448067e-06, | |
| "logits/chosen": -2.187344551086426, | |
| "logits/rejected": -2.4354748725891113, | |
| "logps/chosen": -40.0956916809082, | |
| "logps/rejected": -1655.0335693359375, | |
| "loss": 0.0064, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.18102389574050903, | |
| "rewards/margins": 15.813700675964355, | |
| "rewards/rejected": -15.994723320007324, | |
| "step": 2820 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "grad_norm": 0.0234375, | |
| "learning_rate": 1.1993804613193158e-06, | |
| "logits/chosen": -2.18884539604187, | |
| "logits/rejected": -2.4379589557647705, | |
| "logps/chosen": -64.91096496582031, | |
| "logps/rejected": -1493.458740234375, | |
| "loss": 0.0012, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.4194706082344055, | |
| "rewards/margins": 13.966886520385742, | |
| "rewards/rejected": -14.386357307434082, | |
| "step": 2830 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "grad_norm": 1.3649463653564453e-05, | |
| "learning_rate": 1.1808139273357232e-06, | |
| "logits/chosen": -2.1439809799194336, | |
| "logits/rejected": -2.3814640045166016, | |
| "logps/chosen": -47.53407669067383, | |
| "logps/rejected": -1624.342529296875, | |
| "loss": 0.0067, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.25394895672798157, | |
| "rewards/margins": 15.44206714630127, | |
| "rewards/rejected": -15.696017265319824, | |
| "step": 2840 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "grad_norm": 0.0003833770751953125, | |
| "learning_rate": 1.1623476872539108e-06, | |
| "logits/chosen": -2.1601688861846924, | |
| "logits/rejected": -2.4301838874816895, | |
| "logps/chosen": -46.782203674316406, | |
| "logps/rejected": -1727.318115234375, | |
| "loss": 0.0013, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.24715037643909454, | |
| "rewards/margins": 16.47604751586914, | |
| "rewards/rejected": -16.72319984436035, | |
| "step": 2850 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "grad_norm": 0.0908203125, | |
| "learning_rate": 1.1439831450087032e-06, | |
| "logits/chosen": -2.204617738723755, | |
| "logits/rejected": -2.4746241569519043, | |
| "logps/chosen": -70.88643646240234, | |
| "logps/rejected": -1754.577392578125, | |
| "loss": 0.0066, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.48623570799827576, | |
| "rewards/margins": 16.506263732910156, | |
| "rewards/rejected": -16.992502212524414, | |
| "step": 2860 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "grad_norm": 0.68359375, | |
| "learning_rate": 1.1257216968031357e-06, | |
| "logits/chosen": -2.172727108001709, | |
| "logits/rejected": -2.4189679622650146, | |
| "logps/chosen": -53.47440719604492, | |
| "logps/rejected": -1510.17333984375, | |
| "loss": 0.0042, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3195067048072815, | |
| "rewards/margins": 14.29955768585205, | |
| "rewards/rejected": -14.619064331054688, | |
| "step": 2870 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "grad_norm": 0.0003604888916015625, | |
| "learning_rate": 1.1075647310022974e-06, | |
| "logits/chosen": -2.324207305908203, | |
| "logits/rejected": -2.5505588054656982, | |
| "logps/chosen": -48.77964401245117, | |
| "logps/rejected": -1276.3153076171875, | |
| "loss": 0.0013, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2692165970802307, | |
| "rewards/margins": 12.017995834350586, | |
| "rewards/rejected": -12.287213325500488, | |
| "step": 2880 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "grad_norm": 0.00640869140625, | |
| "learning_rate": 1.0895136280277863e-06, | |
| "logits/chosen": -2.1405930519104004, | |
| "logits/rejected": -2.389354705810547, | |
| "logps/chosen": -52.074989318847656, | |
| "logps/rejected": -1792.354248046875, | |
| "loss": 0.0034, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2989426851272583, | |
| "rewards/margins": 17.034626007080078, | |
| "rewards/rejected": -17.33357048034668, | |
| "step": 2890 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "grad_norm": 0.0634765625, | |
| "learning_rate": 1.0715697602527542e-06, | |
| "logits/chosen": -2.0093884468078613, | |
| "logits/rejected": -2.2803351879119873, | |
| "logps/chosen": -60.875953674316406, | |
| "logps/rejected": -1733.0355224609375, | |
| "loss": 0.0024, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3902908265590668, | |
| "rewards/margins": 16.359115600585938, | |
| "rewards/rejected": -16.749406814575195, | |
| "step": 2900 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "grad_norm": 0.10546875, | |
| "learning_rate": 1.0537344918975708e-06, | |
| "logits/chosen": -2.2281734943389893, | |
| "logits/rejected": -2.414677381515503, | |
| "logps/chosen": -56.65943145751953, | |
| "logps/rejected": -1399.209228515625, | |
| "loss": 0.0016, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3390200734138489, | |
| "rewards/margins": 13.123127937316895, | |
| "rewards/rejected": -13.46214771270752, | |
| "step": 2910 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "grad_norm": 3.910064697265625e-05, | |
| "learning_rate": 1.036009178926107e-06, | |
| "logits/chosen": -2.1897904872894287, | |
| "logits/rejected": -2.426058530807495, | |
| "logps/chosen": -48.45244216918945, | |
| "logps/rejected": -1510.665771484375, | |
| "loss": 0.0036, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.26533347368240356, | |
| "rewards/margins": 14.326631546020508, | |
| "rewards/rejected": -14.59196662902832, | |
| "step": 2920 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "grad_norm": 0.02099609375, | |
| "learning_rate": 1.0183951689426438e-06, | |
| "logits/chosen": -2.1068902015686035, | |
| "logits/rejected": -2.3621068000793457, | |
| "logps/chosen": -49.524208068847656, | |
| "logps/rejected": -1805.392333984375, | |
| "loss": 0.0032, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2760918438434601, | |
| "rewards/margins": 17.226123809814453, | |
| "rewards/rejected": -17.502214431762695, | |
| "step": 2930 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "grad_norm": 0.109375, | |
| "learning_rate": 1.0008938010894156e-06, | |
| "logits/chosen": -2.0732312202453613, | |
| "logits/rejected": -2.359827756881714, | |
| "logps/chosen": -49.63148880004883, | |
| "logps/rejected": -1718.0699462890625, | |
| "loss": 0.0023, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2772377133369446, | |
| "rewards/margins": 16.361858367919922, | |
| "rewards/rejected": -16.639095306396484, | |
| "step": 2940 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "grad_norm": 0.169921875, | |
| "learning_rate": 9.83506405944804e-07, | |
| "logits/chosen": -2.0447497367858887, | |
| "logits/rejected": -2.2771499156951904, | |
| "logps/chosen": -39.81549072265625, | |
| "logps/rejected": -1609.721923828125, | |
| "loss": 0.0019, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.17990897595882416, | |
| "rewards/margins": 15.340120315551758, | |
| "rewards/rejected": -15.520029067993164, | |
| "step": 2950 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "grad_norm": 9.894371032714844e-06, | |
| "learning_rate": 9.662343054221743e-07, | |
| "logits/chosen": -2.053480625152588, | |
| "logits/rejected": -2.3034961223602295, | |
| "logps/chosen": -49.807559967041016, | |
| "logps/rejected": -1805.352294921875, | |
| "loss": 0.0021, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2724885642528534, | |
| "rewards/margins": 17.18954849243164, | |
| "rewards/rejected": -17.462038040161133, | |
| "step": 2960 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "grad_norm": 0.08349609375, | |
| "learning_rate": 9.490788126693754e-07, | |
| "logits/chosen": -2.081925868988037, | |
| "logits/rejected": -2.337897777557373, | |
| "logps/chosen": -39.89293670654297, | |
| "logps/rejected": -1609.5567626953125, | |
| "loss": 0.0021, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.18545950949192047, | |
| "rewards/margins": 15.35168170928955, | |
| "rewards/rejected": -15.537139892578125, | |
| "step": 2970 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "grad_norm": 0.007781982421875, | |
| "learning_rate": 9.32041231968904e-07, | |
| "logits/chosen": -2.1510047912597656, | |
| "logits/rejected": -2.4004898071289062, | |
| "logps/chosen": -42.21772766113281, | |
| "logps/rejected": -1647.2308349609375, | |
| "loss": 0.0021, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.20343096554279327, | |
| "rewards/margins": 15.7313232421875, | |
| "rewards/rejected": -15.93475341796875, | |
| "step": 2980 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "grad_norm": 0.030029296875, | |
| "learning_rate": 9.151228586387464e-07, | |
| "logits/chosen": -2.2137999534606934, | |
| "logits/rejected": -2.4432384967803955, | |
| "logps/chosen": -46.389976501464844, | |
| "logps/rejected": -1448.36328125, | |
| "loss": 0.0055, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.24593684077262878, | |
| "rewards/margins": 13.737565994262695, | |
| "rewards/rejected": -13.983503341674805, | |
| "step": 2990 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "grad_norm": 0.19921875, | |
| "learning_rate": 8.983249789338941e-07, | |
| "logits/chosen": -2.1793341636657715, | |
| "logits/rejected": -2.4067111015319824, | |
| "logps/chosen": -53.83050537109375, | |
| "logps/rejected": -1414.4827880859375, | |
| "loss": 0.0039, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3225783705711365, | |
| "rewards/margins": 13.334991455078125, | |
| "rewards/rejected": -13.657569885253906, | |
| "step": 3000 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "eval_logits/chosen": -2.61519455909729, | |
| "eval_logits/rejected": -2.734154462814331, | |
| "eval_logps/chosen": -62.61328125, | |
| "eval_logps/rejected": -755.4296264648438, | |
| "eval_loss": 0.002784780925139785, | |
| "eval_rewards/accuracies": 1.0, | |
| "eval_rewards/chosen": -0.36699774861335754, | |
| "eval_rewards/margins": 6.738577365875244, | |
| "eval_rewards/rejected": -7.1055755615234375, | |
| "eval_runtime": 0.6552, | |
| "eval_samples_per_second": 7.632, | |
| "eval_steps_per_second": 4.579, | |
| "step": 3000 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "grad_norm": 0.0478515625, | |
| "learning_rate": 8.816488699485593e-07, | |
| "logits/chosen": -2.2049620151519775, | |
| "logits/rejected": -2.431889057159424, | |
| "logps/chosen": -43.32733917236328, | |
| "logps/rejected": -1466.1322021484375, | |
| "loss": 0.0016, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.21807484328746796, | |
| "rewards/margins": 13.928044319152832, | |
| "rewards/rejected": -14.146120071411133, | |
| "step": 3010 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "grad_norm": 0.00012302398681640625, | |
| "learning_rate": 8.650957995190784e-07, | |
| "logits/chosen": -2.168497085571289, | |
| "logits/rejected": -2.439462661743164, | |
| "logps/chosen": -43.25156784057617, | |
| "logps/rejected": -1766.5902099609375, | |
| "loss": 0.0009, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.20799896121025085, | |
| "rewards/margins": 16.896240234375, | |
| "rewards/rejected": -17.104238510131836, | |
| "step": 3020 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "grad_norm": 0.0830078125, | |
| "learning_rate": 8.486670261275193e-07, | |
| "logits/chosen": -2.28559947013855, | |
| "logits/rejected": -2.535123348236084, | |
| "logps/chosen": -46.22868347167969, | |
| "logps/rejected": -1491.29638671875, | |
| "loss": 0.0018, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.24624836444854736, | |
| "rewards/margins": 14.186907768249512, | |
| "rewards/rejected": -14.43315601348877, | |
| "step": 3030 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "grad_norm": 0.01068115234375, | |
| "learning_rate": 8.32363798806011e-07, | |
| "logits/chosen": -2.2580156326293945, | |
| "logits/rejected": -2.499662160873413, | |
| "logps/chosen": -43.39426803588867, | |
| "logps/rejected": -1570.83154296875, | |
| "loss": 0.0033, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2131904661655426, | |
| "rewards/margins": 14.96166706085205, | |
| "rewards/rejected": -15.174858093261719, | |
| "step": 3040 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "grad_norm": 0.0703125, | |
| "learning_rate": 8.161873570417742e-07, | |
| "logits/chosen": -2.205913543701172, | |
| "logits/rejected": -2.461812973022461, | |
| "logps/chosen": -51.799095153808594, | |
| "logps/rejected": -1628.696044921875, | |
| "loss": 0.001, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2980988919734955, | |
| "rewards/margins": 15.454202651977539, | |
| "rewards/rejected": -15.752302169799805, | |
| "step": 3050 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "grad_norm": 0.0263671875, | |
| "learning_rate": 8.001389306828897e-07, | |
| "logits/chosen": -2.1009681224823, | |
| "logits/rejected": -2.3735690116882324, | |
| "logps/chosen": -57.14350128173828, | |
| "logps/rejected": -1912.2584228515625, | |
| "loss": 0.0031, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3501451313495636, | |
| "rewards/margins": 18.137147903442383, | |
| "rewards/rejected": -18.487293243408203, | |
| "step": 3060 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "grad_norm": 2.002716064453125e-05, | |
| "learning_rate": 7.842197398447993e-07, | |
| "logits/chosen": -2.145404100418091, | |
| "logits/rejected": -2.3879191875457764, | |
| "logps/chosen": -46.874935150146484, | |
| "logps/rejected": -1601.4229736328125, | |
| "loss": 0.0034, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.25245028734207153, | |
| "rewards/margins": 15.218485832214355, | |
| "rewards/rejected": -15.470934867858887, | |
| "step": 3070 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "grad_norm": 0.0014801025390625, | |
| "learning_rate": 7.684309948175414e-07, | |
| "logits/chosen": -2.1167359352111816, | |
| "logits/rejected": -2.340625047683716, | |
| "logps/chosen": -41.6113395690918, | |
| "logps/rejected": -1543.5413818359375, | |
| "loss": 0.002, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.19553497433662415, | |
| "rewards/margins": 14.705103874206543, | |
| "rewards/rejected": -14.900639533996582, | |
| "step": 3080 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "grad_norm": 0.0126953125, | |
| "learning_rate": 7.527738959737371e-07, | |
| "logits/chosen": -2.1751418113708496, | |
| "logits/rejected": -2.421253204345703, | |
| "logps/chosen": -55.10563278198242, | |
| "logps/rejected": -1535.4105224609375, | |
| "loss": 0.0013, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.33046358823776245, | |
| "rewards/margins": 14.514165878295898, | |
| "rewards/rejected": -14.844629287719727, | |
| "step": 3090 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "grad_norm": 0.1435546875, | |
| "learning_rate": 7.372496336773269e-07, | |
| "logits/chosen": -2.143078565597534, | |
| "logits/rejected": -2.3641624450683594, | |
| "logps/chosen": -44.76749038696289, | |
| "logps/rejected": -1385.771728515625, | |
| "loss": 0.0058, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.23040492832660675, | |
| "rewards/margins": 13.126627922058105, | |
| "rewards/rejected": -13.357030868530273, | |
| "step": 3100 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "grad_norm": 0.1650390625, | |
| "learning_rate": 7.218593881930744e-07, | |
| "logits/chosen": -2.237316370010376, | |
| "logits/rejected": -2.46457839012146, | |
| "logps/chosen": -43.913902282714844, | |
| "logps/rejected": -1421.6866455078125, | |
| "loss": 0.0033, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.21920108795166016, | |
| "rewards/margins": 13.50879955291748, | |
| "rewards/rejected": -13.728001594543457, | |
| "step": 3110 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "grad_norm": 0.0218505859375, | |
| "learning_rate": 7.066043295968342e-07, | |
| "logits/chosen": -2.2042956352233887, | |
| "logits/rejected": -2.437238931655884, | |
| "logps/chosen": -38.968666076660156, | |
| "logps/rejected": -1539.2254638671875, | |
| "loss": 0.0022, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.1654917150735855, | |
| "rewards/margins": 14.6710786819458, | |
| "rewards/rejected": -14.836568832397461, | |
| "step": 3120 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "grad_norm": 0.00147247314453125, | |
| "learning_rate": 6.914856176865891e-07, | |
| "logits/chosen": -2.2930877208709717, | |
| "logits/rejected": -2.530980110168457, | |
| "logps/chosen": -39.30299377441406, | |
| "logps/rejected": -1486.0648193359375, | |
| "loss": 0.003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.18379421532154083, | |
| "rewards/margins": 14.16865348815918, | |
| "rewards/rejected": -14.352447509765625, | |
| "step": 3130 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "grad_norm": 0.0001888275146484375, | |
| "learning_rate": 6.765044018942804e-07, | |
| "logits/chosen": -2.2794032096862793, | |
| "logits/rejected": -2.5219268798828125, | |
| "logps/chosen": -37.822265625, | |
| "logps/rejected": -1375.9476318359375, | |
| "loss": 0.001, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.16687843203544617, | |
| "rewards/margins": 13.096723556518555, | |
| "rewards/rejected": -13.263601303100586, | |
| "step": 3140 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "grad_norm": 0.0673828125, | |
| "learning_rate": 6.616618211984169e-07, | |
| "logits/chosen": -2.189056873321533, | |
| "logits/rejected": -2.428335666656494, | |
| "logps/chosen": -45.38810348510742, | |
| "logps/rejected": -1504.1986083984375, | |
| "loss": 0.0013, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.24011921882629395, | |
| "rewards/margins": 14.305384635925293, | |
| "rewards/rejected": -14.545504570007324, | |
| "step": 3150 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "grad_norm": 0.0289306640625, | |
| "learning_rate": 6.469590040374799e-07, | |
| "logits/chosen": -2.135713815689087, | |
| "logits/rejected": -2.3790910243988037, | |
| "logps/chosen": -32.32978057861328, | |
| "logps/rejected": -1641.7115478515625, | |
| "loss": 0.0012, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.10296233743429184, | |
| "rewards/margins": 15.768649101257324, | |
| "rewards/rejected": -15.871612548828125, | |
| "step": 3160 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "grad_norm": 0.050537109375, | |
| "learning_rate": 6.32397068224136e-07, | |
| "logits/chosen": -2.248927593231201, | |
| "logits/rejected": -2.501868963241577, | |
| "logps/chosen": -40.18678283691406, | |
| "logps/rejected": -1579.708251953125, | |
| "loss": 0.0054, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.18390187621116638, | |
| "rewards/margins": 15.054614067077637, | |
| "rewards/rejected": -15.238515853881836, | |
| "step": 3170 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "grad_norm": 0.1005859375, | |
| "learning_rate": 6.17977120860249e-07, | |
| "logits/chosen": -2.2377326488494873, | |
| "logits/rejected": -2.4842400550842285, | |
| "logps/chosen": -68.27392578125, | |
| "logps/rejected": -1504.4427490234375, | |
| "loss": 0.0026, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.4537542462348938, | |
| "rewards/margins": 14.08979606628418, | |
| "rewards/rejected": -14.543548583984375, | |
| "step": 3180 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "grad_norm": 0.1435546875, | |
| "learning_rate": 6.037002582527121e-07, | |
| "logits/chosen": -2.17307710647583, | |
| "logits/rejected": -2.4036478996276855, | |
| "logps/chosen": -38.560646057128906, | |
| "logps/rejected": -1505.03662109375, | |
| "loss": 0.0022, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.169607013463974, | |
| "rewards/margins": 14.335187911987305, | |
| "rewards/rejected": -14.504794120788574, | |
| "step": 3190 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "grad_norm": 0.07666015625, | |
| "learning_rate": 5.895675658300981e-07, | |
| "logits/chosen": -2.3447728157043457, | |
| "logits/rejected": -2.5695431232452393, | |
| "logps/chosen": -52.10234451293945, | |
| "logps/rejected": -1275.713134765625, | |
| "loss": 0.0028, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3004547953605652, | |
| "rewards/margins": 11.987954139709473, | |
| "rewards/rejected": -12.288411140441895, | |
| "step": 3200 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "grad_norm": 0.140625, | |
| "learning_rate": 5.755801180601381e-07, | |
| "logits/chosen": -2.2320406436920166, | |
| "logits/rejected": -2.4947474002838135, | |
| "logps/chosen": -46.99077224731445, | |
| "logps/rejected": -1553.0062255859375, | |
| "loss": 0.0022, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.25020334124565125, | |
| "rewards/margins": 14.758695602416992, | |
| "rewards/rejected": -15.008898735046387, | |
| "step": 3210 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "grad_norm": 0.1240234375, | |
| "learning_rate": 5.617389783680307e-07, | |
| "logits/chosen": -2.0936381816864014, | |
| "logits/rejected": -2.3752357959747314, | |
| "logps/chosen": -44.832740783691406, | |
| "logps/rejected": -1831.029296875, | |
| "loss": 0.001, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.22681677341461182, | |
| "rewards/margins": 17.518943786621094, | |
| "rewards/rejected": -17.74576187133789, | |
| "step": 3220 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "grad_norm": 0.197265625, | |
| "learning_rate": 5.48045199055596e-07, | |
| "logits/chosen": -2.19124174118042, | |
| "logits/rejected": -2.438732624053955, | |
| "logps/chosen": -44.39277648925781, | |
| "logps/rejected": -1470.6522216796875, | |
| "loss": 0.0044, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.22872138023376465, | |
| "rewards/margins": 13.976783752441406, | |
| "rewards/rejected": -14.205507278442383, | |
| "step": 3230 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "grad_norm": 7.264316082000732e-07, | |
| "learning_rate": 5.344998212212704e-07, | |
| "logits/chosen": -2.103717565536499, | |
| "logits/rejected": -2.3787877559661865, | |
| "logps/chosen": -46.40209197998047, | |
| "logps/rejected": -1813.8245849609375, | |
| "loss": 0.0009, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.23586265742778778, | |
| "rewards/margins": 17.30778694152832, | |
| "rewards/rejected": -17.543649673461914, | |
| "step": 3240 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "grad_norm": 0.01300048828125, | |
| "learning_rate": 5.211038746809551e-07, | |
| "logits/chosen": -2.2235634326934814, | |
| "logits/rejected": -2.4578189849853516, | |
| "logps/chosen": -50.56513595581055, | |
| "logps/rejected": -1465.8817138671875, | |
| "loss": 0.0031, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.29284316301345825, | |
| "rewards/margins": 13.869397163391113, | |
| "rewards/rejected": -14.162240982055664, | |
| "step": 3250 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "grad_norm": 0.1123046875, | |
| "learning_rate": 5.078583778897216e-07, | |
| "logits/chosen": -2.2172188758850098, | |
| "logits/rejected": -2.4327051639556885, | |
| "logps/chosen": -58.36212158203125, | |
| "logps/rejected": -1398.1490478515625, | |
| "loss": 0.0047, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.36190560460090637, | |
| "rewards/margins": 13.11742877960205, | |
| "rewards/rejected": -13.479333877563477, | |
| "step": 3260 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "grad_norm": 0.11962890625, | |
| "learning_rate": 4.94764337864384e-07, | |
| "logits/chosen": -2.304565668106079, | |
| "logits/rejected": -2.5306572914123535, | |
| "logps/chosen": -43.99140548706055, | |
| "logps/rejected": -1435.986572265625, | |
| "loss": 0.0019, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2204219549894333, | |
| "rewards/margins": 13.629618644714355, | |
| "rewards/rejected": -13.8500394821167, | |
| "step": 3270 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "grad_norm": 0.0184326171875, | |
| "learning_rate": 4.818227501069328e-07, | |
| "logits/chosen": -2.259232521057129, | |
| "logits/rejected": -2.5605130195617676, | |
| "logps/chosen": -62.873863220214844, | |
| "logps/rejected": -1876.69140625, | |
| "loss": 0.0002, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.39689213037490845, | |
| "rewards/margins": 17.804046630859375, | |
| "rewards/rejected": -18.200939178466797, | |
| "step": 3280 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "grad_norm": 0.0203857421875, | |
| "learning_rate": 4.690345985288572e-07, | |
| "logits/chosen": -2.158508777618408, | |
| "logits/rejected": -2.399550437927246, | |
| "logps/chosen": -40.623416900634766, | |
| "logps/rejected": -1611.9981689453125, | |
| "loss": 0.0024, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.18651778995990753, | |
| "rewards/margins": 15.386564254760742, | |
| "rewards/rejected": -15.573080062866211, | |
| "step": 3290 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "grad_norm": 1.5273690223693848e-06, | |
| "learning_rate": 4.5640085537633633e-07, | |
| "logits/chosen": -2.185797691345215, | |
| "logits/rejected": -2.462428331375122, | |
| "logps/chosen": -64.81330871582031, | |
| "logps/rejected": -1722.494384765625, | |
| "loss": 0.0013, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.42417916655540466, | |
| "rewards/margins": 16.25531578063965, | |
| "rewards/rejected": -16.679494857788086, | |
| "step": 3300 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "grad_norm": 5.936622619628906e-05, | |
| "learning_rate": 4.439224811563211e-07, | |
| "logits/chosen": -2.0893611907958984, | |
| "logits/rejected": -2.3352718353271484, | |
| "logps/chosen": -42.231239318847656, | |
| "logps/rejected": -1722.9840087890625, | |
| "loss": 0.0034, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.20374104380607605, | |
| "rewards/margins": 16.47307586669922, | |
| "rewards/rejected": -16.67681884765625, | |
| "step": 3310 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "grad_norm": 0.000370025634765625, | |
| "learning_rate": 4.316004245635158e-07, | |
| "logits/chosen": -2.1728897094726562, | |
| "logits/rejected": -2.4218878746032715, | |
| "logps/chosen": -47.56116485595703, | |
| "logps/rejected": -1730.8961181640625, | |
| "loss": 0.0051, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.25542110204696655, | |
| "rewards/margins": 16.49363899230957, | |
| "rewards/rejected": -16.74905776977539, | |
| "step": 3320 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "grad_norm": 9.1552734375e-05, | |
| "learning_rate": 4.194356224082455e-07, | |
| "logits/chosen": -2.095263957977295, | |
| "logits/rejected": -2.3779187202453613, | |
| "logps/chosen": -44.129798889160156, | |
| "logps/rejected": -1774.712158203125, | |
| "loss": 0.0035, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.22583921253681183, | |
| "rewards/margins": 16.942506790161133, | |
| "rewards/rejected": -17.168346405029297, | |
| "step": 3330 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "grad_norm": 0.00445556640625, | |
| "learning_rate": 4.074289995452338e-07, | |
| "logits/chosen": -2.1644439697265625, | |
| "logits/rejected": -2.4039626121520996, | |
| "logps/chosen": -55.25883102416992, | |
| "logps/rejected": -1481.6717529296875, | |
| "loss": 0.0057, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3257646858692169, | |
| "rewards/margins": 13.985359191894531, | |
| "rewards/rejected": -14.311124801635742, | |
| "step": 3340 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "grad_norm": 0.00010395050048828125, | |
| "learning_rate": 3.9558146880329246e-07, | |
| "logits/chosen": -2.1858904361724854, | |
| "logits/rejected": -2.422576427459717, | |
| "logps/chosen": -38.67001724243164, | |
| "logps/rejected": -1623.956298828125, | |
| "loss": 0.0039, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.1640951931476593, | |
| "rewards/margins": 15.4973783493042, | |
| "rewards/rejected": -15.661474227905273, | |
| "step": 3350 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "grad_norm": 0.03369140625, | |
| "learning_rate": 3.838939309159187e-07, | |
| "logits/chosen": -2.179760694503784, | |
| "logits/rejected": -2.4091451168060303, | |
| "logps/chosen": -44.970279693603516, | |
| "logps/rejected": -1523.0604248046875, | |
| "loss": 0.0018, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.23116175830364227, | |
| "rewards/margins": 14.479741096496582, | |
| "rewards/rejected": -14.710905075073242, | |
| "step": 3360 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "grad_norm": 0.005523681640625, | |
| "learning_rate": 3.723672744528162e-07, | |
| "logits/chosen": -2.256727695465088, | |
| "logits/rejected": -2.5047221183776855, | |
| "logps/chosen": -41.15488815307617, | |
| "logps/rejected": -1610.76220703125, | |
| "loss": 0.0006, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.19345328211784363, | |
| "rewards/margins": 15.386802673339844, | |
| "rewards/rejected": -15.580256462097168, | |
| "step": 3370 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "grad_norm": 0.00286865234375, | |
| "learning_rate": 3.6100237575233647e-07, | |
| "logits/chosen": -2.3228962421417236, | |
| "logits/rejected": -2.5387914180755615, | |
| "logps/chosen": -51.057395935058594, | |
| "logps/rejected": -1306.785400390625, | |
| "loss": 0.0017, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2881939113140106, | |
| "rewards/margins": 12.320541381835938, | |
| "rewards/rejected": -12.608736038208008, | |
| "step": 3380 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "grad_norm": 0.06591796875, | |
| "learning_rate": 3.4980009885486054e-07, | |
| "logits/chosen": -2.25309157371521, | |
| "logits/rejected": -2.4575724601745605, | |
| "logps/chosen": -41.59147644042969, | |
| "logps/rejected": -1270.520751953125, | |
| "loss": 0.0051, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.20340308547019958, | |
| "rewards/margins": 12.038886070251465, | |
| "rewards/rejected": -12.242289543151855, | |
| "step": 3390 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "grad_norm": 0.09423828125, | |
| "learning_rate": 3.3876129543710197e-07, | |
| "logits/chosen": -2.2136871814727783, | |
| "logits/rejected": -2.4533486366271973, | |
| "logps/chosen": -39.60608673095703, | |
| "logps/rejected": -1685.295654296875, | |
| "loss": 0.0017, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.1774289309978485, | |
| "rewards/margins": 16.114295959472656, | |
| "rewards/rejected": -16.291725158691406, | |
| "step": 3400 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "grad_norm": 0.002960205078125, | |
| "learning_rate": 3.2788680474735687e-07, | |
| "logits/chosen": -2.194180488586426, | |
| "logits/rejected": -2.439089298248291, | |
| "logps/chosen": -38.77867889404297, | |
| "logps/rejected": -1473.8935546875, | |
| "loss": 0.0012, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.174292653799057, | |
| "rewards/margins": 14.067087173461914, | |
| "rewards/rejected": -14.24138069152832, | |
| "step": 3410 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "grad_norm": 0.01177978515625, | |
| "learning_rate": 3.1717745354170214e-07, | |
| "logits/chosen": -2.0905921459198, | |
| "logits/rejected": -2.367194175720215, | |
| "logps/chosen": -52.29819869995117, | |
| "logps/rejected": -1684.010986328125, | |
| "loss": 0.0009, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3022548258304596, | |
| "rewards/margins": 16.003009796142578, | |
| "rewards/rejected": -16.30526351928711, | |
| "step": 3420 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "grad_norm": 0.002471923828125, | |
| "learning_rate": 3.0663405602113727e-07, | |
| "logits/chosen": -2.258749485015869, | |
| "logits/rejected": -2.5248327255249023, | |
| "logps/chosen": -48.49363327026367, | |
| "logps/rejected": -1557.198486328125, | |
| "loss": 0.0016, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2676192820072174, | |
| "rewards/margins": 14.78343391418457, | |
| "rewards/rejected": -15.051053047180176, | |
| "step": 3430 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "grad_norm": 0.021240234375, | |
| "learning_rate": 2.9625741376968107e-07, | |
| "logits/chosen": -2.0779199600219727, | |
| "logits/rejected": -2.3387115001678467, | |
| "logps/chosen": -61.849632263183594, | |
| "logps/rejected": -1759.1298828125, | |
| "loss": 0.0025, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3940272629261017, | |
| "rewards/margins": 16.581546783447266, | |
| "rewards/rejected": -16.975570678710938, | |
| "step": 3440 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "grad_norm": 0.0257568359375, | |
| "learning_rate": 2.8604831569343324e-07, | |
| "logits/chosen": -2.3138976097106934, | |
| "logits/rejected": -2.529716968536377, | |
| "logps/chosen": -50.220458984375, | |
| "logps/rejected": -1407.71826171875, | |
| "loss": 0.0024, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2772979736328125, | |
| "rewards/margins": 13.286694526672363, | |
| "rewards/rejected": -13.563992500305176, | |
| "step": 3450 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "grad_norm": 0.0184326171875, | |
| "learning_rate": 2.760075379605942e-07, | |
| "logits/chosen": -2.144134759902954, | |
| "logits/rejected": -2.3687326908111572, | |
| "logps/chosen": -49.850528717041016, | |
| "logps/rejected": -1546.928955078125, | |
| "loss": 0.0015, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2831845283508301, | |
| "rewards/margins": 14.664377212524414, | |
| "rewards/rejected": -14.947561264038086, | |
| "step": 3460 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "grad_norm": 0.291015625, | |
| "learning_rate": 2.661358439424552e-07, | |
| "logits/chosen": -2.209009885787964, | |
| "logits/rejected": -2.4345531463623047, | |
| "logps/chosen": -46.4788818359375, | |
| "logps/rejected": -1371.7965087890625, | |
| "loss": 0.0041, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.24716854095458984, | |
| "rewards/margins": 12.960786819458008, | |
| "rewards/rejected": -13.207954406738281, | |
| "step": 3470 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "grad_norm": 0.005462646484375, | |
| "learning_rate": 2.564339841553615e-07, | |
| "logits/chosen": -2.200819969177246, | |
| "logits/rejected": -2.416544198989868, | |
| "logps/chosen": -43.68715286254883, | |
| "logps/rejected": -1402.603759765625, | |
| "loss": 0.0036, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.22630712389945984, | |
| "rewards/margins": 13.287277221679688, | |
| "rewards/rejected": -13.513586044311523, | |
| "step": 3480 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "grad_norm": 0.462890625, | |
| "learning_rate": 2.469026962036539e-07, | |
| "logits/chosen": -2.1682403087615967, | |
| "logits/rejected": -2.384089946746826, | |
| "logps/chosen": -43.91130447387695, | |
| "logps/rejected": -1500.0106201171875, | |
| "loss": 0.0071, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.22072705626487732, | |
| "rewards/margins": 14.21442699432373, | |
| "rewards/rejected": -14.435153007507324, | |
| "step": 3490 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "grad_norm": 0.0478515625, | |
| "learning_rate": 2.3754270472358786e-07, | |
| "logits/chosen": -2.17598032951355, | |
| "logits/rejected": -2.39375638961792, | |
| "logps/chosen": -40.356300354003906, | |
| "logps/rejected": -1466.735595703125, | |
| "loss": 0.0031, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.18523935973644257, | |
| "rewards/margins": 13.940629959106445, | |
| "rewards/rejected": -14.125869750976562, | |
| "step": 3500 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "grad_norm": 0.51953125, | |
| "learning_rate": 2.283547213282458e-07, | |
| "logits/chosen": -2.2732253074645996, | |
| "logits/rejected": -2.502781629562378, | |
| "logps/chosen": -46.22926330566406, | |
| "logps/rejected": -1531.5400390625, | |
| "loss": 0.0026, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.24311251938343048, | |
| "rewards/margins": 14.52368450164795, | |
| "rewards/rejected": -14.76679515838623, | |
| "step": 3510 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "grad_norm": 0.1328125, | |
| "learning_rate": 2.1933944455343166e-07, | |
| "logits/chosen": -2.0053231716156006, | |
| "logits/rejected": -2.297400951385498, | |
| "logps/chosen": -57.17345428466797, | |
| "logps/rejected": -1729.294921875, | |
| "loss": 0.0021, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.35318198800086975, | |
| "rewards/margins": 16.38203239440918, | |
| "rewards/rejected": -16.735218048095703, | |
| "step": 3520 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "grad_norm": 0.00653076171875, | |
| "learning_rate": 2.104975598045647e-07, | |
| "logits/chosen": -2.1619279384613037, | |
| "logits/rejected": -2.387904167175293, | |
| "logps/chosen": -37.9974250793457, | |
| "logps/rejected": -1388.173583984375, | |
| "loss": 0.0042, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.16347357630729675, | |
| "rewards/margins": 13.225934982299805, | |
| "rewards/rejected": -13.38940715789795, | |
| "step": 3530 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "grad_norm": 0.19921875, | |
| "learning_rate": 2.018297393045701e-07, | |
| "logits/chosen": -2.201099395751953, | |
| "logits/rejected": -2.4077112674713135, | |
| "logps/chosen": -43.07371139526367, | |
| "logps/rejected": -1454.396728515625, | |
| "loss": 0.0026, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.21469160914421082, | |
| "rewards/margins": 13.828866958618164, | |
| "rewards/rejected": -14.043559074401855, | |
| "step": 3540 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "grad_norm": 0.05859375, | |
| "learning_rate": 1.9333664204277236e-07, | |
| "logits/chosen": -2.0957770347595215, | |
| "logits/rejected": -2.3300156593322754, | |
| "logps/chosen": -40.987003326416016, | |
| "logps/rejected": -1783.749755859375, | |
| "loss": 0.0048, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.1888490617275238, | |
| "rewards/margins": 17.056163787841797, | |
| "rewards/rejected": -17.245014190673828, | |
| "step": 3550 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "grad_norm": 0.0001068115234375, | |
| "learning_rate": 1.8501891372479124e-07, | |
| "logits/chosen": -2.1852810382843018, | |
| "logits/rejected": -2.4316954612731934, | |
| "logps/chosen": -44.98752975463867, | |
| "logps/rejected": -1581.531982421875, | |
| "loss": 0.0025, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.22910073399543762, | |
| "rewards/margins": 15.049044609069824, | |
| "rewards/rejected": -15.278146743774414, | |
| "step": 3560 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "grad_norm": 0.115234375, | |
| "learning_rate": 1.7687718672345533e-07, | |
| "logits/chosen": -2.1352264881134033, | |
| "logits/rejected": -2.3711695671081543, | |
| "logps/chosen": -51.448890686035156, | |
| "logps/rejected": -1699.1624755859375, | |
| "loss": 0.0018, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.29201555252075195, | |
| "rewards/margins": 16.14071273803711, | |
| "rewards/rejected": -16.432727813720703, | |
| "step": 3570 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "grad_norm": 0.002471923828125, | |
| "learning_rate": 1.689120800307212e-07, | |
| "logits/chosen": -2.0329132080078125, | |
| "logits/rejected": -2.287954092025757, | |
| "logps/chosen": -46.30390167236328, | |
| "logps/rejected": -1923.584716796875, | |
| "loss": 0.0015, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.24167513847351074, | |
| "rewards/margins": 18.367229461669922, | |
| "rewards/rejected": -18.608905792236328, | |
| "step": 3580 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "grad_norm": 0.1376953125, | |
| "learning_rate": 1.6112419921061357e-07, | |
| "logits/chosen": -2.1787500381469727, | |
| "logits/rejected": -2.4122672080993652, | |
| "logps/chosen": -50.2169075012207, | |
| "logps/rejected": -1459.5537109375, | |
| "loss": 0.0026, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2793883681297302, | |
| "rewards/margins": 13.814018249511719, | |
| "rewards/rejected": -14.093404769897461, | |
| "step": 3590 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "grad_norm": 0.2060546875, | |
| "learning_rate": 1.5351413635318807e-07, | |
| "logits/chosen": -2.2764883041381836, | |
| "logits/rejected": -2.5183794498443604, | |
| "logps/chosen": -47.43731689453125, | |
| "logps/rejected": -1477.555419921875, | |
| "loss": 0.0025, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2550382614135742, | |
| "rewards/margins": 13.986851692199707, | |
| "rewards/rejected": -14.241891860961914, | |
| "step": 3600 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "grad_norm": 0.1279296875, | |
| "learning_rate": 1.460824700295138e-07, | |
| "logits/chosen": -2.268395185470581, | |
| "logits/rejected": -2.5008223056793213, | |
| "logps/chosen": -55.70990753173828, | |
| "logps/rejected": -1575.289794921875, | |
| "loss": 0.0045, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3271048367023468, | |
| "rewards/margins": 14.906885147094727, | |
| "rewards/rejected": -15.233988761901855, | |
| "step": 3610 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "grad_norm": 0.05224609375, | |
| "learning_rate": 1.3882976524768694e-07, | |
| "logits/chosen": -2.2560763359069824, | |
| "logits/rejected": -2.474360942840576, | |
| "logps/chosen": -48.290626525878906, | |
| "logps/rejected": -1297.683349609375, | |
| "loss": 0.0088, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.25945621728897095, | |
| "rewards/margins": 12.217463493347168, | |
| "rewards/rejected": -12.476920127868652, | |
| "step": 3620 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "grad_norm": 0.0004482269287109375, | |
| "learning_rate": 1.3175657340987664e-07, | |
| "logits/chosen": -2.1752967834472656, | |
| "logits/rejected": -2.405945301055908, | |
| "logps/chosen": -39.73582077026367, | |
| "logps/rejected": -1538.538818359375, | |
| "loss": 0.0025, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.1809137761592865, | |
| "rewards/margins": 14.694430351257324, | |
| "rewards/rejected": -14.875345230102539, | |
| "step": 3630 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "grad_norm": 0.018798828125, | |
| "learning_rate": 1.2486343227040122e-07, | |
| "logits/chosen": -2.286973476409912, | |
| "logits/rejected": -2.5369372367858887, | |
| "logps/chosen": -47.38744354248047, | |
| "logps/rejected": -1557.68798828125, | |
| "loss": 0.0088, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.24152731895446777, | |
| "rewards/margins": 14.780197143554688, | |
| "rewards/rejected": -15.021723747253418, | |
| "step": 3640 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "grad_norm": 0.91796875, | |
| "learning_rate": 1.181508658948452e-07, | |
| "logits/chosen": -2.2179925441741943, | |
| "logits/rejected": -2.4420266151428223, | |
| "logps/chosen": -38.083351135253906, | |
| "logps/rejected": -1484.4072265625, | |
| "loss": 0.0063, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.16277261078357697, | |
| "rewards/margins": 14.176862716674805, | |
| "rewards/rejected": -14.339635848999023, | |
| "step": 3650 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "grad_norm": 0.10107421875, | |
| "learning_rate": 1.1161938462021627e-07, | |
| "logits/chosen": -2.1011550426483154, | |
| "logits/rejected": -2.328584671020508, | |
| "logps/chosen": -42.358421325683594, | |
| "logps/rejected": -1466.247802734375, | |
| "loss": 0.007, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.19884276390075684, | |
| "rewards/margins": 13.929216384887695, | |
| "rewards/rejected": -14.128057479858398, | |
| "step": 3660 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "grad_norm": 0.45703125, | |
| "learning_rate": 1.0526948501614536e-07, | |
| "logits/chosen": -2.129077434539795, | |
| "logits/rejected": -2.3908514976501465, | |
| "logps/chosen": -53.09492874145508, | |
| "logps/rejected": -1710.9476318359375, | |
| "loss": 0.0044, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3066931366920471, | |
| "rewards/margins": 16.256519317626953, | |
| "rewards/rejected": -16.563209533691406, | |
| "step": 3670 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "grad_norm": 0.4296875, | |
| "learning_rate": 9.910164984713477e-08, | |
| "logits/chosen": -2.135789394378662, | |
| "logits/rejected": -2.3950417041778564, | |
| "logps/chosen": -43.19747543334961, | |
| "logps/rejected": -1662.0009765625, | |
| "loss": 0.003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2104502171278, | |
| "rewards/margins": 15.8755464553833, | |
| "rewards/rejected": -16.085996627807617, | |
| "step": 3680 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "grad_norm": 0.0751953125, | |
| "learning_rate": 9.311634803585323e-08, | |
| "logits/chosen": -2.1814115047454834, | |
| "logits/rejected": -2.445276975631714, | |
| "logps/chosen": -53.1719970703125, | |
| "logps/rejected": -1662.022705078125, | |
| "loss": 0.0069, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3151262402534485, | |
| "rewards/margins": 15.78093433380127, | |
| "rewards/rejected": -16.0960636138916, | |
| "step": 3690 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "grad_norm": 0.00125885009765625, | |
| "learning_rate": 8.7314034627487e-08, | |
| "logits/chosen": -2.230149507522583, | |
| "logits/rejected": -2.479522705078125, | |
| "logps/chosen": -36.06591033935547, | |
| "logps/rejected": -1619.81884765625, | |
| "loss": 0.0038, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.14542549848556519, | |
| "rewards/margins": 15.533602714538574, | |
| "rewards/rejected": -15.679028511047363, | |
| "step": 3700 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "grad_norm": 0.023681640625, | |
| "learning_rate": 8.16951507551439e-08, | |
| "logits/chosen": -2.2386298179626465, | |
| "logits/rejected": -2.456665515899658, | |
| "logps/chosen": -45.547454833984375, | |
| "logps/rejected": -1492.232666015625, | |
| "loss": 0.0022, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2374843806028366, | |
| "rewards/margins": 14.150115966796875, | |
| "rewards/rejected": -14.387600898742676, | |
| "step": 3710 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "grad_norm": 0.00019931793212890625, | |
| "learning_rate": 7.626012360631291e-08, | |
| "logits/chosen": -2.266707420349121, | |
| "logits/rejected": -2.5029749870300293, | |
| "logps/chosen": -49.74803924560547, | |
| "logps/rejected": -1482.993896484375, | |
| "loss": 0.0026, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2803342640399933, | |
| "rewards/margins": 14.048635482788086, | |
| "rewards/rejected": -14.328969955444336, | |
| "step": 3720 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "grad_norm": 0.03662109375, | |
| "learning_rate": 7.100936639038936e-08, | |
| "logits/chosen": -2.040673017501831, | |
| "logits/rejected": -2.3331856727600098, | |
| "logps/chosen": -43.85186004638672, | |
| "logps/rejected": -1894.5054931640625, | |
| "loss": 0.0014, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2176673710346222, | |
| "rewards/margins": 18.155574798583984, | |
| "rewards/rejected": -18.373241424560547, | |
| "step": 3730 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "grad_norm": 1.0609626770019531e-05, | |
| "learning_rate": 6.594327830725916e-08, | |
| "logits/chosen": -2.190392017364502, | |
| "logits/rejected": -2.451129913330078, | |
| "logps/chosen": -54.63507080078125, | |
| "logps/rejected": -1576.1376953125, | |
| "loss": 0.0019, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.32724398374557495, | |
| "rewards/margins": 14.939753532409668, | |
| "rewards/rejected": -15.266998291015625, | |
| "step": 3740 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "grad_norm": 0.1083984375, | |
| "learning_rate": 6.106224451694592e-08, | |
| "logits/chosen": -2.2175679206848145, | |
| "logits/rejected": -2.4564685821533203, | |
| "logps/chosen": -46.39513397216797, | |
| "logps/rejected": -1616.2586669921875, | |
| "loss": 0.0033, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.24283328652381897, | |
| "rewards/margins": 15.379629135131836, | |
| "rewards/rejected": -15.622464179992676, | |
| "step": 3750 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "grad_norm": 0.1669921875, | |
| "learning_rate": 5.636663611033266e-08, | |
| "logits/chosen": -2.0778698921203613, | |
| "logits/rejected": -2.3482048511505127, | |
| "logps/chosen": -44.864990234375, | |
| "logps/rejected": -1604.7249755859375, | |
| "loss": 0.003, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.23550191521644592, | |
| "rewards/margins": 15.299467086791992, | |
| "rewards/rejected": -15.534968376159668, | |
| "step": 3760 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "grad_norm": 0.0162353515625, | |
| "learning_rate": 5.185681008094579e-08, | |
| "logits/chosen": -2.284482479095459, | |
| "logits/rejected": -2.5175766944885254, | |
| "logps/chosen": -47.569602966308594, | |
| "logps/rejected": -1563.1016845703125, | |
| "loss": 0.0027, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.25453075766563416, | |
| "rewards/margins": 14.83684253692627, | |
| "rewards/rejected": -15.0913724899292, | |
| "step": 3770 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "grad_norm": 0.00015735626220703125, | |
| "learning_rate": 4.753310929781513e-08, | |
| "logits/chosen": -2.2356629371643066, | |
| "logits/rejected": -2.451608657836914, | |
| "logps/chosen": -53.69340133666992, | |
| "logps/rejected": -1443.2952880859375, | |
| "loss": 0.0035, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3177313208580017, | |
| "rewards/margins": 13.615710258483887, | |
| "rewards/rejected": -13.933443069458008, | |
| "step": 3780 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "grad_norm": 6.246566772460938e-05, | |
| "learning_rate": 4.3395862479405914e-08, | |
| "logits/chosen": -2.156893253326416, | |
| "logits/rejected": -2.395084857940674, | |
| "logps/chosen": -46.39254379272461, | |
| "logps/rejected": -1630.5118408203125, | |
| "loss": 0.001, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.24043354392051697, | |
| "rewards/margins": 15.48585319519043, | |
| "rewards/rejected": -15.726287841796875, | |
| "step": 3790 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "grad_norm": 0.0162353515625, | |
| "learning_rate": 3.9445384168628474e-08, | |
| "logits/chosen": -2.328781843185425, | |
| "logits/rejected": -2.580176591873169, | |
| "logps/chosen": -52.544822692871094, | |
| "logps/rejected": -1474.49365234375, | |
| "loss": 0.0026, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3075386881828308, | |
| "rewards/margins": 13.93040657043457, | |
| "rewards/rejected": -14.237945556640625, | |
| "step": 3800 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "grad_norm": 0.046630859375, | |
| "learning_rate": 3.5681974708923484e-08, | |
| "logits/chosen": -2.120448589324951, | |
| "logits/rejected": -2.3445982933044434, | |
| "logps/chosen": -37.53495788574219, | |
| "logps/rejected": -1442.9287109375, | |
| "loss": 0.0022, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.1521245837211609, | |
| "rewards/margins": 13.743891716003418, | |
| "rewards/rejected": -13.896017074584961, | |
| "step": 3810 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "grad_norm": 0.01068115234375, | |
| "learning_rate": 3.210592022142717e-08, | |
| "logits/chosen": -2.1601128578186035, | |
| "logits/rejected": -2.3705685138702393, | |
| "logps/chosen": -52.31241989135742, | |
| "logps/rejected": -1557.874755859375, | |
| "loss": 0.0015, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3095288872718811, | |
| "rewards/margins": 14.724299430847168, | |
| "rewards/rejected": -15.033828735351562, | |
| "step": 3820 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "grad_norm": 0.09912109375, | |
| "learning_rate": 2.8717492583220095e-08, | |
| "logits/chosen": -2.2527565956115723, | |
| "logits/rejected": -2.4976978302001953, | |
| "logps/chosen": -45.60851287841797, | |
| "logps/rejected": -1558.29931640625, | |
| "loss": 0.0012, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.24002361297607422, | |
| "rewards/margins": 14.838605880737305, | |
| "rewards/rejected": -15.078630447387695, | |
| "step": 3830 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "grad_norm": 1.1171875, | |
| "learning_rate": 2.551694940665539e-08, | |
| "logits/chosen": -2.191880464553833, | |
| "logits/rejected": -2.41998291015625, | |
| "logps/chosen": -51.693626403808594, | |
| "logps/rejected": -1440.926513671875, | |
| "loss": 0.004, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.29828059673309326, | |
| "rewards/margins": 13.601608276367188, | |
| "rewards/rejected": -13.899889945983887, | |
| "step": 3840 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "grad_norm": 0.0006561279296875, | |
| "learning_rate": 2.2504534019774092e-08, | |
| "logits/chosen": -2.347978353500366, | |
| "logits/rejected": -2.5543174743652344, | |
| "logps/chosen": -42.8640251159668, | |
| "logps/rejected": -1363.5374755859375, | |
| "loss": 0.0023, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2091234177350998, | |
| "rewards/margins": 12.935150146484375, | |
| "rewards/rejected": -13.14427375793457, | |
| "step": 3850 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "grad_norm": 0.099609375, | |
| "learning_rate": 1.9680475447805826e-08, | |
| "logits/chosen": -2.231818437576294, | |
| "logits/rejected": -2.4574391841888428, | |
| "logps/chosen": -58.65113067626953, | |
| "logps/rejected": -1416.2275390625, | |
| "loss": 0.0064, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3713977038860321, | |
| "rewards/margins": 13.290824890136719, | |
| "rewards/rejected": -13.662221908569336, | |
| "step": 3860 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "grad_norm": 0.00102996826171875, | |
| "learning_rate": 1.70449883957563e-08, | |
| "logits/chosen": -2.254411220550537, | |
| "logits/rejected": -2.486921787261963, | |
| "logps/chosen": -50.42361831665039, | |
| "logps/rejected": -1524.43603515625, | |
| "loss": 0.0021, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.28153225779533386, | |
| "rewards/margins": 14.412869453430176, | |
| "rewards/rejected": -14.694402694702148, | |
| "step": 3870 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "grad_norm": 0.07080078125, | |
| "learning_rate": 1.4598273232083182e-08, | |
| "logits/chosen": -2.2290568351745605, | |
| "logits/rejected": -2.4446380138397217, | |
| "logps/chosen": -40.07840347290039, | |
| "logps/rejected": -1425.9326171875, | |
| "loss": 0.0053, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.18418380618095398, | |
| "rewards/margins": 13.563642501831055, | |
| "rewards/rejected": -13.747825622558594, | |
| "step": 3880 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "grad_norm": 0.000240325927734375, | |
| "learning_rate": 1.2340515973464917e-08, | |
| "logits/chosen": -2.1592583656311035, | |
| "logits/rejected": -2.4244697093963623, | |
| "logps/chosen": -54.51990509033203, | |
| "logps/rejected": -1623.231689453125, | |
| "loss": 0.0021, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.32529979944229126, | |
| "rewards/margins": 15.373883247375488, | |
| "rewards/rejected": -15.699182510375977, | |
| "step": 3890 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "grad_norm": 0.007537841796875, | |
| "learning_rate": 1.0271888270655118e-08, | |
| "logits/chosen": -2.0656638145446777, | |
| "logits/rejected": -2.2895803451538086, | |
| "logps/chosen": -38.120445251464844, | |
| "logps/rejected": -1580.198486328125, | |
| "loss": 0.0025, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.16423656046390533, | |
| "rewards/margins": 15.07524299621582, | |
| "rewards/rejected": -15.239479064941406, | |
| "step": 3900 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "grad_norm": 0.107421875, | |
| "learning_rate": 8.392547395435769e-09, | |
| "logits/chosen": -2.413311243057251, | |
| "logits/rejected": -2.6234774589538574, | |
| "logps/chosen": -62.75360107421875, | |
| "logps/rejected": -1339.970458984375, | |
| "loss": 0.0035, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.40989524126052856, | |
| "rewards/margins": 12.507749557495117, | |
| "rewards/rejected": -12.917645454406738, | |
| "step": 3910 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "grad_norm": 0.00030517578125, | |
| "learning_rate": 6.702636228657911e-09, | |
| "logits/chosen": -2.292491912841797, | |
| "logits/rejected": -2.5207715034484863, | |
| "logps/chosen": -43.633766174316406, | |
| "logps/rejected": -1431.236083984375, | |
| "loss": 0.0028, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2170587033033371, | |
| "rewards/margins": 13.580774307250977, | |
| "rewards/rejected": -13.797833442687988, | |
| "step": 3920 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "grad_norm": 0.181640625, | |
| "learning_rate": 5.2022832493800465e-09, | |
| "logits/chosen": -2.363647937774658, | |
| "logits/rejected": -2.5729119777679443, | |
| "logps/chosen": -53.60352325439453, | |
| "logps/rejected": -1316.2359619140625, | |
| "loss": 0.0023, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3134381175041199, | |
| "rewards/margins": 12.360208511352539, | |
| "rewards/rejected": -12.673646926879883, | |
| "step": 3930 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "grad_norm": 0.0001983642578125, | |
| "learning_rate": 3.891602525100124e-09, | |
| "logits/chosen": -2.2279720306396484, | |
| "logits/rejected": -2.4751038551330566, | |
| "logps/chosen": -46.06736373901367, | |
| "logps/rejected": -1572.96337890625, | |
| "loss": 0.0012, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.24313127994537354, | |
| "rewards/margins": 14.95093059539795, | |
| "rewards/rejected": -15.194061279296875, | |
| "step": 3940 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "grad_norm": 0.01434326171875, | |
| "learning_rate": 2.7706937030827495e-09, | |
| "logits/chosen": -2.2832016944885254, | |
| "logits/rejected": -2.5126912593841553, | |
| "logps/chosen": -51.094390869140625, | |
| "logps/rejected": -1317.524169921875, | |
| "loss": 0.0025, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2949807047843933, | |
| "rewards/margins": 12.386547088623047, | |
| "rewards/rejected": -12.681528091430664, | |
| "step": 3950 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "grad_norm": 0.028564453125, | |
| "learning_rate": 1.839642002783859e-09, | |
| "logits/chosen": -2.2053470611572266, | |
| "logits/rejected": -2.42755389213562, | |
| "logps/chosen": -39.788963317871094, | |
| "logps/rejected": -1376.3099365234375, | |
| "loss": 0.0036, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.18181900680065155, | |
| "rewards/margins": 13.076495170593262, | |
| "rewards/rejected": -13.25831413269043, | |
| "step": 3960 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "grad_norm": 0.03857421875, | |
| "learning_rate": 1.0985182093714574e-09, | |
| "logits/chosen": -2.253420352935791, | |
| "logits/rejected": -2.4623093605041504, | |
| "logps/chosen": -59.57801055908203, | |
| "logps/rejected": -1399.8045654296875, | |
| "loss": 0.0026, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.3837854266166687, | |
| "rewards/margins": 13.101987838745117, | |
| "rewards/rejected": -13.485774040222168, | |
| "step": 3970 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "grad_norm": 0.0198974609375, | |
| "learning_rate": 5.473786683440896e-10, | |
| "logits/chosen": -2.1431727409362793, | |
| "logits/rejected": -2.3964176177978516, | |
| "logps/chosen": -56.668495178222656, | |
| "logps/rejected": -1672.459716796875, | |
| "loss": 0.0039, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.34237998723983765, | |
| "rewards/margins": 15.848222732543945, | |
| "rewards/rejected": -16.190601348876953, | |
| "step": 3980 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "grad_norm": 0.09765625, | |
| "learning_rate": 1.862652812467669e-10, | |
| "logits/chosen": -2.1814169883728027, | |
| "logits/rejected": -2.422477960586548, | |
| "logps/chosen": -39.490379333496094, | |
| "logps/rejected": -1714.8070068359375, | |
| "loss": 0.0018, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.17177413403987885, | |
| "rewards/margins": 16.374120712280273, | |
| "rewards/rejected": -16.545894622802734, | |
| "step": 3990 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "grad_norm": 0.0001277923583984375, | |
| "learning_rate": 1.5205502486292932e-11, | |
| "logits/chosen": -2.172867774963379, | |
| "logits/rejected": -2.422581911087036, | |
| "logps/chosen": -44.31547164916992, | |
| "logps/rejected": -1577.330078125, | |
| "loss": 0.0019, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.23060818016529083, | |
| "rewards/margins": 15.038507461547852, | |
| "rewards/rejected": -15.269113540649414, | |
| "step": 4000 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "eval_logits/chosen": -2.614182472229004, | |
| "eval_logits/rejected": -2.732987642288208, | |
| "eval_logps/chosen": -60.27886962890625, | |
| "eval_logps/rejected": -752.9370727539062, | |
| "eval_loss": 0.0027744148392230272, | |
| "eval_rewards/accuracies": 1.0, | |
| "eval_rewards/chosen": -0.34365367889404297, | |
| "eval_rewards/margins": 6.736997127532959, | |
| "eval_rewards/rejected": -7.080650329589844, | |
| "eval_runtime": 0.6551, | |
| "eval_samples_per_second": 7.633, | |
| "eval_steps_per_second": 4.58, | |
| "step": 4000 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "step": 4004, | |
| "total_flos": 0.0, | |
| "train_loss": 0.04242442899794605, | |
| "train_runtime": 8772.4234, | |
| "train_samples_per_second": 1.826, | |
| "train_steps_per_second": 0.456 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 4004, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 1000, | |
| "total_flos": 0.0, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |