| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 2.992914501653283, | |
| "eval_steps": 50, | |
| "global_step": 1056, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "dpo_loss": 0.6931471824645996, | |
| "epoch": 0.002834199338686821, | |
| "grad_norm": 36.794102305076855, | |
| "learning_rate": 9.433962264150943e-09, | |
| "logits": -1.2867579460144043, | |
| "logps": -84.34933471679688, | |
| "loss": 0.0051, | |
| "objective": 0.0046141319908201694, | |
| "ranking_idealized": 0.6458333134651184, | |
| "ranking_idealized_expo": 0.5833333134651184, | |
| "ranking_simple": 0.5833333134651184, | |
| "regularize": 0.0046141319908201694, | |
| "step": 1, | |
| "wo_beta": 14.841486930847168 | |
| }, | |
| { | |
| "dpo_loss": 0.6930367350578308, | |
| "epoch": 0.014170996693434105, | |
| "grad_norm": 51.56528279298989, | |
| "learning_rate": 4.7169811320754715e-08, | |
| "logits": -1.4291929006576538, | |
| "logps": -83.85256958007812, | |
| "loss": 0.0058, | |
| "objective": 0.005918528418987989, | |
| "ranking_idealized": 0.5625, | |
| "ranking_idealized_expo": 0.4895833432674408, | |
| "ranking_simple": 0.4895833432674408, | |
| "regularize": 0.005918528418987989, | |
| "step": 5, | |
| "wo_beta": 16.667278289794922 | |
| }, | |
| { | |
| "dpo_loss": 0.6930564641952515, | |
| "epoch": 0.02834199338686821, | |
| "grad_norm": 43.62540826850091, | |
| "learning_rate": 9.433962264150943e-08, | |
| "logits": -1.4014313220977783, | |
| "logps": -84.90540313720703, | |
| "loss": 0.0065, | |
| "objective": 0.00607979716733098, | |
| "ranking_idealized": 0.6708333492279053, | |
| "ranking_idealized_expo": 0.5833333134651184, | |
| "ranking_simple": 0.5791666507720947, | |
| "regularize": 0.00607979716733098, | |
| "step": 10, | |
| "wo_beta": 15.295255661010742 | |
| }, | |
| { | |
| "dpo_loss": 0.691772997379303, | |
| "epoch": 0.042512990080302314, | |
| "grad_norm": 40.579476886356176, | |
| "learning_rate": 1.4150943396226414e-07, | |
| "logits": -1.5395350456237793, | |
| "logps": -84.67674255371094, | |
| "loss": 0.0077, | |
| "objective": 0.007744006346911192, | |
| "ranking_idealized": 0.6499999761581421, | |
| "ranking_idealized_expo": 0.5708333253860474, | |
| "ranking_simple": 0.5666666626930237, | |
| "regularize": 0.007744006346911192, | |
| "step": 15, | |
| "wo_beta": 15.72358512878418 | |
| }, | |
| { | |
| "dpo_loss": 0.6908682584762573, | |
| "epoch": 0.05668398677373642, | |
| "grad_norm": 38.45055261776428, | |
| "learning_rate": 1.8867924528301886e-07, | |
| "logits": -1.3619084358215332, | |
| "logps": -83.87267303466797, | |
| "loss": 0.0106, | |
| "objective": 0.011018705554306507, | |
| "ranking_idealized": 0.5833333134651184, | |
| "ranking_idealized_expo": 0.4833333194255829, | |
| "ranking_simple": 0.4833333194255829, | |
| "regularize": 0.011018705554306507, | |
| "step": 20, | |
| "wo_beta": 16.501863479614258 | |
| }, | |
| { | |
| "dpo_loss": 0.6917246580123901, | |
| "epoch": 0.07085498346717052, | |
| "grad_norm": 37.49075261903623, | |
| "learning_rate": 2.3584905660377358e-07, | |
| "logits": -1.366659164428711, | |
| "logps": -84.04557037353516, | |
| "loss": 0.0144, | |
| "objective": 0.012653553858399391, | |
| "ranking_idealized": 0.6458333134651184, | |
| "ranking_idealized_expo": 0.5583333373069763, | |
| "ranking_simple": 0.5625, | |
| "regularize": 0.012653553858399391, | |
| "step": 25, | |
| "wo_beta": 15.649717330932617 | |
| }, | |
| { | |
| "dpo_loss": 0.6906312704086304, | |
| "epoch": 0.08502598016060463, | |
| "grad_norm": 35.42831042318107, | |
| "learning_rate": 2.830188679245283e-07, | |
| "logits": -1.4202715158462524, | |
| "logps": -84.00289154052734, | |
| "loss": 0.0156, | |
| "objective": 0.015595527365803719, | |
| "ranking_idealized": 0.5541666746139526, | |
| "ranking_idealized_expo": 0.4833333194255829, | |
| "ranking_simple": 0.4833333194255829, | |
| "regularize": 0.015595527365803719, | |
| "step": 30, | |
| "wo_beta": 16.955543518066406 | |
| }, | |
| { | |
| "dpo_loss": 0.6931964755058289, | |
| "epoch": 0.09919697685403873, | |
| "grad_norm": 37.327321600930496, | |
| "learning_rate": 3.30188679245283e-07, | |
| "logits": -1.3935037851333618, | |
| "logps": -83.39187622070312, | |
| "loss": 0.0202, | |
| "objective": 0.021191226318478584, | |
| "ranking_idealized": 0.6291666626930237, | |
| "ranking_idealized_expo": 0.5375000238418579, | |
| "ranking_simple": 0.5375000238418579, | |
| "regularize": 0.021191226318478584, | |
| "step": 35, | |
| "wo_beta": 16.169347763061523 | |
| }, | |
| { | |
| "dpo_loss": 0.693729817867279, | |
| "epoch": 0.11336797354747284, | |
| "grad_norm": 41.6880498675233, | |
| "learning_rate": 3.773584905660377e-07, | |
| "logits": -1.381697177886963, | |
| "logps": -83.91118621826172, | |
| "loss": 0.0228, | |
| "objective": 0.02042653225362301, | |
| "ranking_idealized": 0.5708333253860474, | |
| "ranking_idealized_expo": 0.5166666507720947, | |
| "ranking_simple": 0.5166666507720947, | |
| "regularize": 0.02042653225362301, | |
| "step": 40, | |
| "wo_beta": 14.309080123901367 | |
| }, | |
| { | |
| "dpo_loss": 0.6919765472412109, | |
| "epoch": 0.12753897024090693, | |
| "grad_norm": 41.11048762433909, | |
| "learning_rate": 4.2452830188679244e-07, | |
| "logits": -1.3955552577972412, | |
| "logps": -84.25520324707031, | |
| "loss": 0.027, | |
| "objective": 0.025382202118635178, | |
| "ranking_idealized": 0.5958333611488342, | |
| "ranking_idealized_expo": 0.5083333253860474, | |
| "ranking_simple": 0.5083333253860474, | |
| "regularize": 0.025382202118635178, | |
| "step": 45, | |
| "wo_beta": 14.21595287322998 | |
| }, | |
| { | |
| "dpo_loss": 0.6911224722862244, | |
| "epoch": 0.14170996693434104, | |
| "grad_norm": 41.07625280062658, | |
| "learning_rate": 4.7169811320754717e-07, | |
| "logits": -1.4127604961395264, | |
| "logps": -85.3918685913086, | |
| "loss": 0.0351, | |
| "objective": 0.03202561289072037, | |
| "ranking_idealized": 0.5916666388511658, | |
| "ranking_idealized_expo": 0.5, | |
| "ranking_simple": 0.5, | |
| "regularize": 0.03202561289072037, | |
| "step": 50, | |
| "wo_beta": 15.589811325073242 | |
| }, | |
| { | |
| "epoch": 0.14170996693434104, | |
| "eval_dpo_loss": 0.6926834583282471, | |
| "eval_logits": -1.391736626625061, | |
| "eval_logps": -91.23294067382812, | |
| "eval_loss": 0.02213538996875286, | |
| "eval_objective": 0.022384027019143105, | |
| "eval_ranking_idealized": 0.6024844646453857, | |
| "eval_ranking_idealized_expo": 0.5232919454574585, | |
| "eval_ranking_simple": 0.5212215185165405, | |
| "eval_regularize": 0.022384027019143105, | |
| "eval_runtime": 470.1076, | |
| "eval_samples_per_second": 12.316, | |
| "eval_steps_per_second": 1.027, | |
| "eval_wo_beta": 16.221710205078125, | |
| "step": 50 | |
| }, | |
| { | |
| "dpo_loss": 0.6922997832298279, | |
| "epoch": 0.15588096362777515, | |
| "grad_norm": 36.466581476765526, | |
| "learning_rate": 5.188679245283019e-07, | |
| "logits": -1.3620656728744507, | |
| "logps": -84.91451263427734, | |
| "loss": 0.0367, | |
| "objective": 0.0405682697892189, | |
| "ranking_idealized": 0.5583333373069763, | |
| "ranking_idealized_expo": 0.49166667461395264, | |
| "ranking_simple": 0.4833333194255829, | |
| "regularize": 0.0405682697892189, | |
| "step": 55, | |
| "wo_beta": 15.095004081726074 | |
| }, | |
| { | |
| "dpo_loss": 0.6875351071357727, | |
| "epoch": 0.17005196032120926, | |
| "grad_norm": 36.25782748515131, | |
| "learning_rate": 5.660377358490566e-07, | |
| "logits": -1.28928804397583, | |
| "logps": -85.71366119384766, | |
| "loss": 0.0403, | |
| "objective": 0.04035286232829094, | |
| "ranking_idealized": 0.6000000238418579, | |
| "ranking_idealized_expo": 0.5166666507720947, | |
| "ranking_simple": 0.5208333134651184, | |
| "regularize": 0.04035286232829094, | |
| "step": 60, | |
| "wo_beta": 14.607115745544434 | |
| }, | |
| { | |
| "dpo_loss": 0.6947705149650574, | |
| "epoch": 0.18422295701464336, | |
| "grad_norm": 41.25867915272223, | |
| "learning_rate": 6.132075471698112e-07, | |
| "logits": -1.3798266649246216, | |
| "logps": -83.1692123413086, | |
| "loss": 0.0491, | |
| "objective": 0.050007414072752, | |
| "ranking_idealized": 0.637499988079071, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.5208333134651184, | |
| "regularize": 0.050007414072752, | |
| "step": 65, | |
| "wo_beta": 14.976885795593262 | |
| }, | |
| { | |
| "dpo_loss": 0.6880966424942017, | |
| "epoch": 0.19839395370807747, | |
| "grad_norm": 35.20333705483616, | |
| "learning_rate": 6.60377358490566e-07, | |
| "logits": -1.4017753601074219, | |
| "logps": -85.73289489746094, | |
| "loss": 0.0551, | |
| "objective": 0.059768859297037125, | |
| "ranking_idealized": 0.5916666388511658, | |
| "ranking_idealized_expo": 0.5333333611488342, | |
| "ranking_simple": 0.5249999761581421, | |
| "regularize": 0.059768859297037125, | |
| "step": 70, | |
| "wo_beta": 15.204180717468262 | |
| }, | |
| { | |
| "dpo_loss": 0.6949416995048523, | |
| "epoch": 0.21256495040151158, | |
| "grad_norm": 35.61853042350494, | |
| "learning_rate": 7.075471698113207e-07, | |
| "logits": -1.321311593055725, | |
| "logps": -85.34779357910156, | |
| "loss": 0.0579, | |
| "objective": 0.06061805784702301, | |
| "ranking_idealized": 0.6083333492279053, | |
| "ranking_idealized_expo": 0.5166666507720947, | |
| "ranking_simple": 0.5291666388511658, | |
| "regularize": 0.06061805784702301, | |
| "step": 75, | |
| "wo_beta": 14.980683326721191 | |
| }, | |
| { | |
| "dpo_loss": 0.6930631995201111, | |
| "epoch": 0.22673594709494568, | |
| "grad_norm": 34.9536345678453, | |
| "learning_rate": 7.547169811320754e-07, | |
| "logits": -1.4264112710952759, | |
| "logps": -84.01344299316406, | |
| "loss": 0.0626, | |
| "objective": 0.062408361583948135, | |
| "ranking_idealized": 0.6291666626930237, | |
| "ranking_idealized_expo": 0.5291666388511658, | |
| "ranking_simple": 0.5333333611488342, | |
| "regularize": 0.062408361583948135, | |
| "step": 80, | |
| "wo_beta": 16.357084274291992 | |
| }, | |
| { | |
| "dpo_loss": 0.6939026117324829, | |
| "epoch": 0.2409069437883798, | |
| "grad_norm": 35.4653089608865, | |
| "learning_rate": 8.018867924528302e-07, | |
| "logits": -1.4041804075241089, | |
| "logps": -83.52224731445312, | |
| "loss": 0.0695, | |
| "objective": 0.07861108332872391, | |
| "ranking_idealized": 0.625, | |
| "ranking_idealized_expo": 0.550000011920929, | |
| "ranking_simple": 0.5416666865348816, | |
| "regularize": 0.07861108332872391, | |
| "step": 85, | |
| "wo_beta": 14.987756729125977 | |
| }, | |
| { | |
| "dpo_loss": 0.6886675357818604, | |
| "epoch": 0.25507794048181387, | |
| "grad_norm": 38.910010820592774, | |
| "learning_rate": 8.490566037735849e-07, | |
| "logits": -1.5007805824279785, | |
| "logps": -84.52466583251953, | |
| "loss": 0.0806, | |
| "objective": 0.08859896659851074, | |
| "ranking_idealized": 0.612500011920929, | |
| "ranking_idealized_expo": 0.5291666388511658, | |
| "ranking_simple": 0.5208333134651184, | |
| "regularize": 0.08859896659851074, | |
| "step": 90, | |
| "wo_beta": 15.482732772827148 | |
| }, | |
| { | |
| "dpo_loss": 0.6961393356323242, | |
| "epoch": 0.269248937175248, | |
| "grad_norm": 38.50762322649532, | |
| "learning_rate": 8.962264150943396e-07, | |
| "logits": -1.4152452945709229, | |
| "logps": -83.7827377319336, | |
| "loss": 0.0851, | |
| "objective": 0.08412078768014908, | |
| "ranking_idealized": 0.5583333373069763, | |
| "ranking_idealized_expo": 0.4791666567325592, | |
| "ranking_simple": 0.47083333134651184, | |
| "regularize": 0.08412078768014908, | |
| "step": 95, | |
| "wo_beta": 16.229019165039062 | |
| }, | |
| { | |
| "dpo_loss": 0.6928918361663818, | |
| "epoch": 0.2834199338686821, | |
| "grad_norm": 34.07886171444254, | |
| "learning_rate": 9.433962264150943e-07, | |
| "logits": -1.2942625284194946, | |
| "logps": -81.22164916992188, | |
| "loss": 0.0877, | |
| "objective": 0.08352937549352646, | |
| "ranking_idealized": 0.5958333611488342, | |
| "ranking_idealized_expo": 0.4833333194255829, | |
| "ranking_simple": 0.4833333194255829, | |
| "regularize": 0.08352937549352646, | |
| "step": 100, | |
| "wo_beta": 15.187151908874512 | |
| }, | |
| { | |
| "epoch": 0.2834199338686821, | |
| "eval_dpo_loss": 0.6921994090080261, | |
| "eval_logits": -1.3862521648406982, | |
| "eval_logps": -88.66019439697266, | |
| "eval_loss": 0.04334083944559097, | |
| "eval_objective": 0.04473063722252846, | |
| "eval_ranking_idealized": 0.6024844646453857, | |
| "eval_ranking_idealized_expo": 0.5232919454574585, | |
| "eval_ranking_simple": 0.523809552192688, | |
| "eval_regularize": 0.04473063722252846, | |
| "eval_runtime": 472.2375, | |
| "eval_samples_per_second": 12.261, | |
| "eval_steps_per_second": 1.023, | |
| "eval_wo_beta": 16.16818618774414, | |
| "step": 100 | |
| }, | |
| { | |
| "dpo_loss": 0.6989858150482178, | |
| "epoch": 0.2975909305621162, | |
| "grad_norm": 34.124768906394316, | |
| "learning_rate": 9.90566037735849e-07, | |
| "logits": -1.4883809089660645, | |
| "logps": -83.63202667236328, | |
| "loss": 0.0937, | |
| "objective": 0.10326550155878067, | |
| "ranking_idealized": 0.625, | |
| "ranking_idealized_expo": 0.5291666388511658, | |
| "ranking_simple": 0.5291666388511658, | |
| "regularize": 0.10326550155878067, | |
| "step": 105, | |
| "wo_beta": 14.697186470031738 | |
| }, | |
| { | |
| "dpo_loss": 0.6916998624801636, | |
| "epoch": 0.3117619272555503, | |
| "grad_norm": 36.53960499520599, | |
| "learning_rate": 9.99956257238817e-07, | |
| "logits": -1.3666936159133911, | |
| "logps": -82.67723083496094, | |
| "loss": 0.1009, | |
| "objective": 0.09831760078668594, | |
| "ranking_idealized": 0.5874999761581421, | |
| "ranking_idealized_expo": 0.5041666626930237, | |
| "ranking_simple": 0.5083333253860474, | |
| "regularize": 0.09831760078668594, | |
| "step": 110, | |
| "wo_beta": 14.75289249420166 | |
| }, | |
| { | |
| "dpo_loss": 0.6972029805183411, | |
| "epoch": 0.32593292394898443, | |
| "grad_norm": 32.3431868996238, | |
| "learning_rate": 9.997785653888834e-07, | |
| "logits": -1.351915955543518, | |
| "logps": -82.5732650756836, | |
| "loss": 0.1062, | |
| "objective": 0.10171337425708771, | |
| "ranking_idealized": 0.6208333373069763, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.5083333253860474, | |
| "regularize": 0.10171337425708771, | |
| "step": 115, | |
| "wo_beta": 16.003950119018555 | |
| }, | |
| { | |
| "dpo_loss": 0.6885399222373962, | |
| "epoch": 0.3401039206424185, | |
| "grad_norm": 35.92878266852989, | |
| "learning_rate": 9.994642390694308e-07, | |
| "logits": -1.367909550666809, | |
| "logps": -82.90719604492188, | |
| "loss": 0.1098, | |
| "objective": 0.11067435145378113, | |
| "ranking_idealized": 0.5916666388511658, | |
| "ranking_idealized_expo": 0.512499988079071, | |
| "ranking_simple": 0.512499988079071, | |
| "regularize": 0.11067435145378113, | |
| "step": 120, | |
| "wo_beta": 15.639138221740723 | |
| }, | |
| { | |
| "dpo_loss": 0.6936843395233154, | |
| "epoch": 0.35427491733585265, | |
| "grad_norm": 30.26276247254467, | |
| "learning_rate": 9.990133642141357e-07, | |
| "logits": -1.3929860591888428, | |
| "logps": -85.65290069580078, | |
| "loss": 0.1056, | |
| "objective": 0.11743973940610886, | |
| "ranking_idealized": 0.5583333373069763, | |
| "ranking_idealized_expo": 0.48750001192092896, | |
| "ranking_simple": 0.4749999940395355, | |
| "regularize": 0.11743973940610886, | |
| "step": 125, | |
| "wo_beta": 15.93514633178711 | |
| }, | |
| { | |
| "dpo_loss": 0.6941003799438477, | |
| "epoch": 0.3684459140292867, | |
| "grad_norm": 39.21461417787312, | |
| "learning_rate": 9.98426064087682e-07, | |
| "logits": -1.3525993824005127, | |
| "logps": -83.56419372558594, | |
| "loss": 0.1211, | |
| "objective": 0.11899420619010925, | |
| "ranking_idealized": 0.6458333134651184, | |
| "ranking_idealized_expo": 0.5916666388511658, | |
| "ranking_simple": 0.5833333134651184, | |
| "regularize": 0.11899420619010925, | |
| "step": 130, | |
| "wo_beta": 16.0157527923584 | |
| }, | |
| { | |
| "dpo_loss": 0.6882577538490295, | |
| "epoch": 0.3826169107227208, | |
| "grad_norm": 32.67768184928008, | |
| "learning_rate": 9.977024992520601e-07, | |
| "logits": -1.3901729583740234, | |
| "logps": -84.39146423339844, | |
| "loss": 0.1253, | |
| "objective": 0.12414517998695374, | |
| "ranking_idealized": 0.574999988079071, | |
| "ranking_idealized_expo": 0.4958333373069763, | |
| "ranking_simple": 0.5, | |
| "regularize": 0.12414517998695374, | |
| "step": 135, | |
| "wo_beta": 14.371219635009766 | |
| }, | |
| { | |
| "dpo_loss": 0.6830641627311707, | |
| "epoch": 0.39678790741615494, | |
| "grad_norm": 33.07732649314307, | |
| "learning_rate": 9.968428675226713e-07, | |
| "logits": -1.3437649011611938, | |
| "logps": -85.44697570800781, | |
| "loss": 0.1248, | |
| "objective": 0.12058641016483307, | |
| "ranking_idealized": 0.612500011920929, | |
| "ranking_idealized_expo": 0.5416666865348816, | |
| "ranking_simple": 0.5375000238418579, | |
| "regularize": 0.12058641016483307, | |
| "step": 140, | |
| "wo_beta": 14.547070503234863 | |
| }, | |
| { | |
| "dpo_loss": 0.6851420998573303, | |
| "epoch": 0.410958904109589, | |
| "grad_norm": 30.784646211601874, | |
| "learning_rate": 9.958474039142469e-07, | |
| "logits": -1.3567951917648315, | |
| "logps": -86.4469223022461, | |
| "loss": 0.1319, | |
| "objective": 0.13056445121765137, | |
| "ranking_idealized": 0.6666666865348816, | |
| "ranking_idealized_expo": 0.5583333373069763, | |
| "ranking_simple": 0.5625, | |
| "regularize": 0.13056445121765137, | |
| "step": 145, | |
| "wo_beta": 13.91884994506836 | |
| }, | |
| { | |
| "dpo_loss": 0.6960374116897583, | |
| "epoch": 0.42512990080302315, | |
| "grad_norm": 32.05337681597037, | |
| "learning_rate": 9.947163805765979e-07, | |
| "logits": -1.3565360307693481, | |
| "logps": -86.30919647216797, | |
| "loss": 0.1323, | |
| "objective": 0.12925057113170624, | |
| "ranking_idealized": 0.5958333611488342, | |
| "ranking_idealized_expo": 0.5541666746139526, | |
| "ranking_simple": 0.5375000238418579, | |
| "regularize": 0.12925057113170624, | |
| "step": 150, | |
| "wo_beta": 16.796695709228516 | |
| }, | |
| { | |
| "epoch": 0.42512990080302315, | |
| "eval_dpo_loss": 0.695567786693573, | |
| "eval_logits": -1.3053797483444214, | |
| "eval_logps": -90.43773651123047, | |
| "eval_loss": 0.07677316665649414, | |
| "eval_objective": 0.07639209181070328, | |
| "eval_ranking_idealized": 0.6024844646453857, | |
| "eval_ranking_idealized_expo": 0.5232919454574585, | |
| "eval_ranking_simple": 0.5222567319869995, | |
| "eval_regularize": 0.07639209181070328, | |
| "eval_runtime": 526.1958, | |
| "eval_samples_per_second": 11.004, | |
| "eval_steps_per_second": 0.918, | |
| "eval_wo_beta": 16.003387451171875, | |
| "step": 150 | |
| }, | |
| { | |
| "dpo_loss": 0.6933045983314514, | |
| "epoch": 0.43930089749645723, | |
| "grad_norm": 31.605620123374155, | |
| "learning_rate": 9.934501067202117e-07, | |
| "logits": -1.3933676481246948, | |
| "logps": -83.03238677978516, | |
| "loss": 0.1358, | |
| "objective": 0.1285211592912674, | |
| "ranking_idealized": 0.5666666626930237, | |
| "ranking_idealized_expo": 0.48750001192092896, | |
| "ranking_simple": 0.4791666567325592, | |
| "regularize": 0.1285211592912674, | |
| "step": 155, | |
| "wo_beta": 15.31113338470459 | |
| }, | |
| { | |
| "dpo_loss": 0.6946766972541809, | |
| "epoch": 0.45347189418989137, | |
| "grad_norm": 32.22880904067845, | |
| "learning_rate": 9.92048928531717e-07, | |
| "logits": -1.2931861877441406, | |
| "logps": -83.0308837890625, | |
| "loss": 0.1338, | |
| "objective": 0.12377996742725372, | |
| "ranking_idealized": 0.5916666388511658, | |
| "ranking_idealized_expo": 0.5333333611488342, | |
| "ranking_simple": 0.5416666865348816, | |
| "regularize": 0.12377996742725372, | |
| "step": 160, | |
| "wo_beta": 14.51412296295166 | |
| }, | |
| { | |
| "dpo_loss": 0.6858457326889038, | |
| "epoch": 0.46764289088332545, | |
| "grad_norm": 28.56289647538006, | |
| "learning_rate": 9.905132290792392e-07, | |
| "logits": -1.3845534324645996, | |
| "logps": -84.35334777832031, | |
| "loss": 0.1295, | |
| "objective": 0.13048619031906128, | |
| "ranking_idealized": 0.6083333492279053, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.5208333134651184, | |
| "regularize": 0.13048619031906128, | |
| "step": 165, | |
| "wo_beta": 15.858311653137207 | |
| }, | |
| { | |
| "dpo_loss": 0.6987485289573669, | |
| "epoch": 0.4818138875767596, | |
| "grad_norm": 31.697158183348822, | |
| "learning_rate": 9.888434282076757e-07, | |
| "logits": -1.3974741697311401, | |
| "logps": -82.40156555175781, | |
| "loss": 0.1376, | |
| "objective": 0.14300216734409332, | |
| "ranking_idealized": 0.5583333373069763, | |
| "ranking_idealized_expo": 0.5041666626930237, | |
| "ranking_simple": 0.5, | |
| "regularize": 0.14300216734409332, | |
| "step": 170, | |
| "wo_beta": 15.730175018310547 | |
| }, | |
| { | |
| "dpo_loss": 0.6993536353111267, | |
| "epoch": 0.49598488427019366, | |
| "grad_norm": 30.951333756278135, | |
| "learning_rate": 9.870399824239114e-07, | |
| "logits": -1.2470077276229858, | |
| "logps": -83.35051727294922, | |
| "loss": 0.1401, | |
| "objective": 0.13475559651851654, | |
| "ranking_idealized": 0.574999988079071, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.5291666388511658, | |
| "regularize": 0.13475559651851654, | |
| "step": 175, | |
| "wo_beta": 17.82953643798828 | |
| }, | |
| { | |
| "dpo_loss": 0.6983634233474731, | |
| "epoch": 0.5101558809636277, | |
| "grad_norm": 34.822921079044, | |
| "learning_rate": 9.851033847720164e-07, | |
| "logits": -1.2282413244247437, | |
| "logps": -83.51294708251953, | |
| "loss": 0.1442, | |
| "objective": 0.143393412232399, | |
| "ranking_idealized": 0.5708333253860474, | |
| "ranking_idealized_expo": 0.48750001192092896, | |
| "ranking_simple": 0.4625000059604645, | |
| "regularize": 0.143393412232399, | |
| "step": 180, | |
| "wo_beta": 14.920221328735352 | |
| }, | |
| { | |
| "dpo_loss": 0.6972795128822327, | |
| "epoch": 0.5243268776570619, | |
| "grad_norm": 34.3447207787113, | |
| "learning_rate": 9.83034164698452e-07, | |
| "logits": -1.2574915885925293, | |
| "logps": -82.5478515625, | |
| "loss": 0.1382, | |
| "objective": 0.14230893552303314, | |
| "ranking_idealized": 0.5791666507720947, | |
| "ranking_idealized_expo": 0.4833333194255829, | |
| "ranking_simple": 0.4749999940395355, | |
| "regularize": 0.14230893552303314, | |
| "step": 185, | |
| "wo_beta": 14.194059371948242 | |
| }, | |
| { | |
| "dpo_loss": 0.6978750824928284, | |
| "epoch": 0.538497874350496, | |
| "grad_norm": 34.00712851830173, | |
| "learning_rate": 9.808328879073251e-07, | |
| "logits": -1.2612725496292114, | |
| "logps": -81.91997528076172, | |
| "loss": 0.1466, | |
| "objective": 0.14948724210262299, | |
| "ranking_idealized": 0.6625000238418579, | |
| "ranking_idealized_expo": 0.5625, | |
| "ranking_simple": 0.5666666626930237, | |
| "regularize": 0.14948724210262299, | |
| "step": 190, | |
| "wo_beta": 16.620363235473633 | |
| }, | |
| { | |
| "dpo_loss": 0.6822370290756226, | |
| "epoch": 0.5526688710439301, | |
| "grad_norm": 31.586658287520144, | |
| "learning_rate": 9.78500156205731e-07, | |
| "logits": -1.2822577953338623, | |
| "logps": -83.0813217163086, | |
| "loss": 0.1319, | |
| "objective": 0.13207347691059113, | |
| "ranking_idealized": 0.625, | |
| "ranking_idealized_expo": 0.5375000238418579, | |
| "ranking_simple": 0.5333333611488342, | |
| "regularize": 0.13207347691059113, | |
| "step": 195, | |
| "wo_beta": 14.693647384643555 | |
| }, | |
| { | |
| "dpo_loss": 0.7044106721878052, | |
| "epoch": 0.5668398677373642, | |
| "grad_norm": 30.369620708498754, | |
| "learning_rate": 9.760366073392244e-07, | |
| "logits": -1.3258157968521118, | |
| "logps": -83.32820129394531, | |
| "loss": 0.1427, | |
| "objective": 0.15046708285808563, | |
| "ranking_idealized": 0.5833333134651184, | |
| "ranking_idealized_expo": 0.512499988079071, | |
| "ranking_simple": 0.5249999761581421, | |
| "regularize": 0.15046708285808563, | |
| "step": 200, | |
| "wo_beta": 15.960111618041992 | |
| }, | |
| { | |
| "epoch": 0.5668398677373642, | |
| "eval_dpo_loss": 0.6959174871444702, | |
| "eval_logits": -1.3123745918273926, | |
| "eval_logps": -88.34333801269531, | |
| "eval_loss": 0.10319730639457703, | |
| "eval_objective": 0.10169863700866699, | |
| "eval_ranking_idealized": 0.6024844646453857, | |
| "eval_ranking_idealized_expo": 0.5232919454574585, | |
| "eval_ranking_simple": 0.5222567319869995, | |
| "eval_regularize": 0.10169863700866699, | |
| "eval_runtime": 532.3008, | |
| "eval_samples_per_second": 10.877, | |
| "eval_steps_per_second": 0.907, | |
| "eval_wo_beta": 15.992826461791992, | |
| "step": 200 | |
| }, | |
| { | |
| "dpo_loss": 0.7000283598899841, | |
| "epoch": 0.5810108644307983, | |
| "grad_norm": 30.076737378719095, | |
| "learning_rate": 9.734429148174674e-07, | |
| "logits": -1.2141386270523071, | |
| "logps": -82.74073028564453, | |
| "loss": 0.1484, | |
| "objective": 0.1470470279455185, | |
| "ranking_idealized": 0.574999988079071, | |
| "ranking_idealized_expo": 0.5041666626930237, | |
| "ranking_simple": 0.49166667461395264, | |
| "regularize": 0.1470470279455185, | |
| "step": 205, | |
| "wo_beta": 16.118446350097656 | |
| }, | |
| { | |
| "dpo_loss": 0.6862087249755859, | |
| "epoch": 0.5951818611242324, | |
| "grad_norm": 31.36222267459615, | |
| "learning_rate": 9.707197877300973e-07, | |
| "logits": -1.2483521699905396, | |
| "logps": -82.3885269165039, | |
| "loss": 0.1454, | |
| "objective": 0.14993111789226532, | |
| "ranking_idealized": 0.6041666865348816, | |
| "ranking_idealized_expo": 0.5208333134651184, | |
| "ranking_simple": 0.5291666388511658, | |
| "regularize": 0.14993111789226532, | |
| "step": 210, | |
| "wo_beta": 15.07961654663086 | |
| }, | |
| { | |
| "dpo_loss": 0.6946883797645569, | |
| "epoch": 0.6093528578176665, | |
| "grad_norm": 32.25125352651472, | |
| "learning_rate": 9.678679705528698e-07, | |
| "logits": -1.3168671131134033, | |
| "logps": -82.3456039428711, | |
| "loss": 0.1384, | |
| "objective": 0.14188070595264435, | |
| "ranking_idealized": 0.5791666507720947, | |
| "ranking_idealized_expo": 0.4749999940395355, | |
| "ranking_simple": 0.4791666567325592, | |
| "regularize": 0.14188070595264435, | |
| "step": 215, | |
| "wo_beta": 16.104469299316406 | |
| }, | |
| { | |
| "dpo_loss": 0.7026723027229309, | |
| "epoch": 0.6235238545111006, | |
| "grad_norm": 30.142053540661294, | |
| "learning_rate": 9.648882429441256e-07, | |
| "logits": -1.3188337087631226, | |
| "logps": -82.63532257080078, | |
| "loss": 0.1477, | |
| "objective": 0.1607874184846878, | |
| "ranking_idealized": 0.574999988079071, | |
| "ranking_idealized_expo": 0.4791666567325592, | |
| "ranking_simple": 0.49166667461395264, | |
| "regularize": 0.1607874184846878, | |
| "step": 220, | |
| "wo_beta": 17.079347610473633 | |
| }, | |
| { | |
| "dpo_loss": 0.6998167634010315, | |
| "epoch": 0.6376948512045347, | |
| "grad_norm": 29.418648888160003, | |
| "learning_rate": 9.61781419531641e-07, | |
| "logits": -1.3314566612243652, | |
| "logps": -82.72489929199219, | |
| "loss": 0.1465, | |
| "objective": 0.14282181859016418, | |
| "ranking_idealized": 0.6000000238418579, | |
| "ranking_idealized_expo": 0.5041666626930237, | |
| "ranking_simple": 0.4958333373069763, | |
| "regularize": 0.14282181859016418, | |
| "step": 225, | |
| "wo_beta": 15.506386756896973 | |
| }, | |
| { | |
| "dpo_loss": 0.7007436156272888, | |
| "epoch": 0.6518658478979689, | |
| "grad_norm": 31.584769522955447, | |
| "learning_rate": 9.585483496899149e-07, | |
| "logits": -1.2612279653549194, | |
| "logps": -82.21707916259766, | |
| "loss": 0.1434, | |
| "objective": 0.14342841506004333, | |
| "ranking_idealized": 0.637499988079071, | |
| "ranking_idealized_expo": 0.5666666626930237, | |
| "ranking_simple": 0.5583333373069763, | |
| "regularize": 0.14342841506004333, | |
| "step": 230, | |
| "wo_beta": 16.431724548339844 | |
| }, | |
| { | |
| "dpo_loss": 0.7085835337638855, | |
| "epoch": 0.6660368445914029, | |
| "grad_norm": 33.138665174716316, | |
| "learning_rate": 9.551899173079606e-07, | |
| "logits": -1.2083913087844849, | |
| "logps": -84.15171813964844, | |
| "loss": 0.1479, | |
| "objective": 0.14772751927375793, | |
| "ranking_idealized": 0.5874999761581421, | |
| "ranking_idealized_expo": 0.5416666865348816, | |
| "ranking_simple": 0.5291666388511658, | |
| "regularize": 0.14772751927375793, | |
| "step": 235, | |
| "wo_beta": 15.722906112670898 | |
| }, | |
| { | |
| "dpo_loss": 0.6893501877784729, | |
| "epoch": 0.680207841284837, | |
| "grad_norm": 28.511782322472136, | |
| "learning_rate": 9.517070405476574e-07, | |
| "logits": -1.3556396961212158, | |
| "logps": -83.491943359375, | |
| "loss": 0.1408, | |
| "objective": 0.1575685441493988, | |
| "ranking_idealized": 0.6041666865348816, | |
| "ranking_idealized_expo": 0.5083333253860474, | |
| "ranking_simple": 0.5249999761581421, | |
| "regularize": 0.1575685441493988, | |
| "step": 240, | |
| "wo_beta": 15.692626953125 | |
| }, | |
| { | |
| "dpo_loss": 0.6901037693023682, | |
| "epoch": 0.6943788379782712, | |
| "grad_norm": 28.887977273452503, | |
| "learning_rate": 9.481006715927351e-07, | |
| "logits": -1.3499360084533691, | |
| "logps": -82.59223937988281, | |
| "loss": 0.1422, | |
| "objective": 0.1397981345653534, | |
| "ranking_idealized": 0.6166666746139526, | |
| "ranking_idealized_expo": 0.5458333492279053, | |
| "ranking_simple": 0.5416666865348816, | |
| "regularize": 0.1397981345653534, | |
| "step": 245, | |
| "wo_beta": 15.627848625183105 | |
| }, | |
| { | |
| "dpo_loss": 0.6898453831672668, | |
| "epoch": 0.7085498346717053, | |
| "grad_norm": 30.778123472149638, | |
| "learning_rate": 9.443717963884568e-07, | |
| "logits": -1.1249743700027466, | |
| "logps": -81.38602447509766, | |
| "loss": 0.1451, | |
| "objective": 0.12806275486946106, | |
| "ranking_idealized": 0.612500011920929, | |
| "ranking_idealized_expo": 0.5291666388511658, | |
| "ranking_simple": 0.5291666388511658, | |
| "regularize": 0.12806275486946106, | |
| "step": 250, | |
| "wo_beta": 14.860217094421387 | |
| }, | |
| { | |
| "epoch": 0.7085498346717053, | |
| "eval_dpo_loss": 0.6950441002845764, | |
| "eval_logits": -1.2854480743408203, | |
| "eval_logps": -88.06980895996094, | |
| "eval_loss": 0.11781599372625351, | |
| "eval_objective": 0.11854107677936554, | |
| "eval_ranking_idealized": 0.6024844646453857, | |
| "eval_ranking_idealized_expo": 0.5232919454574585, | |
| "eval_ranking_simple": 0.5274327397346497, | |
| "eval_regularize": 0.11854107677936554, | |
| "eval_runtime": 533.5732, | |
| "eval_samples_per_second": 10.851, | |
| "eval_steps_per_second": 0.905, | |
| "eval_wo_beta": 15.787796020507812, | |
| "step": 250 | |
| }, | |
| { | |
| "dpo_loss": 0.6893075704574585, | |
| "epoch": 0.7227208313651393, | |
| "grad_norm": 27.48861543576658, | |
| "learning_rate": 9.405214343720706e-07, | |
| "logits": -1.3376212120056152, | |
| "logps": -81.39327239990234, | |
| "loss": 0.1325, | |
| "objective": 0.12804514169692993, | |
| "ranking_idealized": 0.5708333253860474, | |
| "ranking_idealized_expo": 0.42916667461395264, | |
| "ranking_simple": 0.44583332538604736, | |
| "regularize": 0.12804514169692993, | |
| "step": 255, | |
| "wo_beta": 14.828557968139648 | |
| }, | |
| { | |
| "dpo_loss": 0.6919839978218079, | |
| "epoch": 0.7368918280585735, | |
| "grad_norm": 27.470977695013012, | |
| "learning_rate": 9.365506381941065e-07, | |
| "logits": -1.3046835660934448, | |
| "logps": -83.32947540283203, | |
| "loss": 0.1509, | |
| "objective": 0.15500593185424805, | |
| "ranking_idealized": 0.5416666865348816, | |
| "ranking_idealized_expo": 0.4625000059604645, | |
| "ranking_simple": 0.4583333432674408, | |
| "regularize": 0.15500593185424805, | |
| "step": 260, | |
| "wo_beta": 15.419398307800293 | |
| }, | |
| { | |
| "dpo_loss": 0.6987964510917664, | |
| "epoch": 0.7510628247520076, | |
| "grad_norm": 29.786537519342414, | |
| "learning_rate": 9.32460493430591e-07, | |
| "logits": -1.2736799716949463, | |
| "logps": -82.46897888183594, | |
| "loss": 0.1444, | |
| "objective": 0.14515246450901031, | |
| "ranking_idealized": 0.6625000238418579, | |
| "ranking_idealized_expo": 0.5666666626930237, | |
| "ranking_simple": 0.5708333253860474, | |
| "regularize": 0.14515246450901031, | |
| "step": 265, | |
| "wo_beta": 15.908428192138672 | |
| }, | |
| { | |
| "dpo_loss": 0.6944437026977539, | |
| "epoch": 0.7652338214454416, | |
| "grad_norm": 28.172549175339846, | |
| "learning_rate": 9.282521182862629e-07, | |
| "logits": -1.397876262664795, | |
| "logps": -82.14982604980469, | |
| "loss": 0.1491, | |
| "objective": 0.15289539098739624, | |
| "ranking_idealized": 0.5249999761581421, | |
| "ranking_idealized_expo": 0.4583333432674408, | |
| "ranking_simple": 0.4541666805744171, | |
| "regularize": 0.15289539098739624, | |
| "step": 270, | |
| "wo_beta": 14.118414878845215 | |
| }, | |
| { | |
| "dpo_loss": 0.6878421902656555, | |
| "epoch": 0.7794048181388757, | |
| "grad_norm": 30.974249065309053, | |
| "learning_rate": 9.239266632888658e-07, | |
| "logits": -1.265884280204773, | |
| "logps": -80.5745849609375, | |
| "loss": 0.1429, | |
| "objective": 0.13965575397014618, | |
| "ranking_idealized": 0.574999988079071, | |
| "ranking_idealized_expo": 0.5041666626930237, | |
| "ranking_simple": 0.5, | |
| "regularize": 0.13965575397014618, | |
| "step": 275, | |
| "wo_beta": 15.147540092468262 | |
| }, | |
| { | |
| "dpo_loss": 0.693124532699585, | |
| "epoch": 0.7935758148323099, | |
| "grad_norm": 27.26309671203667, | |
| "learning_rate": 9.194853109746072e-07, | |
| "logits": -1.317248821258545, | |
| "logps": -80.71721649169922, | |
| "loss": 0.1422, | |
| "objective": 0.13741357624530792, | |
| "ranking_idealized": 0.5874999761581421, | |
| "ranking_idealized_expo": 0.5208333134651184, | |
| "ranking_simple": 0.5208333134651184, | |
| "regularize": 0.13741357624530792, | |
| "step": 280, | |
| "wo_beta": 15.141572952270508 | |
| }, | |
| { | |
| "dpo_loss": 0.6898981332778931, | |
| "epoch": 0.807746811525744, | |
| "grad_norm": 29.618387771117387, | |
| "learning_rate": 9.14929275564863e-07, | |
| "logits": -1.2990264892578125, | |
| "logps": -81.34524536132812, | |
| "loss": 0.1481, | |
| "objective": 0.14202959835529327, | |
| "ranking_idealized": 0.5666666626930237, | |
| "ranking_idealized_expo": 0.4791666567325592, | |
| "ranking_simple": 0.4749999940395355, | |
| "regularize": 0.14202959835529327, | |
| "step": 285, | |
| "wo_beta": 16.715734481811523 | |
| }, | |
| { | |
| "dpo_loss": 0.6989319920539856, | |
| "epoch": 0.821917808219178, | |
| "grad_norm": 30.35546225687188, | |
| "learning_rate": 9.102598026342222e-07, | |
| "logits": -1.310984492301941, | |
| "logps": -80.47208404541016, | |
| "loss": 0.1416, | |
| "objective": 0.13658234477043152, | |
| "ranking_idealized": 0.5874999761581421, | |
| "ranking_idealized_expo": 0.5041666626930237, | |
| "ranking_simple": 0.5, | |
| "regularize": 0.13658234477043152, | |
| "step": 290, | |
| "wo_beta": 15.537436485290527 | |
| }, | |
| { | |
| "dpo_loss": 0.692668080329895, | |
| "epoch": 0.8360888049126122, | |
| "grad_norm": 28.386489735858774, | |
| "learning_rate": 9.0547816876996e-07, | |
| "logits": -1.3056447505950928, | |
| "logps": -80.58573913574219, | |
| "loss": 0.1335, | |
| "objective": 0.14200052618980408, | |
| "ranking_idealized": 0.550000011920929, | |
| "ranking_idealized_expo": 0.4958333373069763, | |
| "ranking_simple": 0.48750001192092896, | |
| "regularize": 0.14200052618980408, | |
| "step": 295, | |
| "wo_beta": 15.984179496765137 | |
| }, | |
| { | |
| "dpo_loss": 0.6959550380706787, | |
| "epoch": 0.8502598016060463, | |
| "grad_norm": 30.90903589796416, | |
| "learning_rate": 9.005856812230304e-07, | |
| "logits": -1.2770187854766846, | |
| "logps": -79.3738784790039, | |
| "loss": 0.1305, | |
| "objective": 0.12751255929470062, | |
| "ranking_idealized": 0.625, | |
| "ranking_idealized_expo": 0.550000011920929, | |
| "ranking_simple": 0.5541666746139526, | |
| "regularize": 0.12751255929470062, | |
| "step": 300, | |
| "wo_beta": 14.3499755859375 | |
| }, | |
| { | |
| "epoch": 0.8502598016060463, | |
| "eval_dpo_loss": 0.6960889101028442, | |
| "eval_logits": -1.2862635850906372, | |
| "eval_logps": -86.33123016357422, | |
| "eval_loss": 0.12468627840280533, | |
| "eval_objective": 0.1251634955406189, | |
| "eval_ranking_idealized": 0.6024844646453857, | |
| "eval_ranking_idealized_expo": 0.5232919454574585, | |
| "eval_ranking_simple": 0.5279502868652344, | |
| "eval_regularize": 0.1251634955406189, | |
| "eval_runtime": 492.3852, | |
| "eval_samples_per_second": 11.759, | |
| "eval_steps_per_second": 0.981, | |
| "eval_wo_beta": 15.766751289367676, | |
| "step": 300 | |
| }, | |
| { | |
| "dpo_loss": 0.6915071606636047, | |
| "epoch": 0.8644307982994804, | |
| "grad_norm": 28.35320542673635, | |
| "learning_rate": 8.955836775506775e-07, | |
| "logits": -1.2531558275222778, | |
| "logps": -80.3687744140625, | |
| "loss": 0.1326, | |
| "objective": 0.1348031610250473, | |
| "ranking_idealized": 0.6166666746139526, | |
| "ranking_idealized_expo": 0.5208333134651184, | |
| "ranking_simple": 0.5249999761581421, | |
| "regularize": 0.1348031610250473, | |
| "step": 305, | |
| "wo_beta": 15.622274398803711 | |
| }, | |
| { | |
| "dpo_loss": 0.6971884965896606, | |
| "epoch": 0.8786017949929145, | |
| "grad_norm": 28.116582054859066, | |
| "learning_rate": 8.904735252507609e-07, | |
| "logits": -1.256584882736206, | |
| "logps": -79.94914245605469, | |
| "loss": 0.1365, | |
| "objective": 0.1369226723909378, | |
| "ranking_idealized": 0.5958333611488342, | |
| "ranking_idealized_expo": 0.4833333194255829, | |
| "ranking_simple": 0.4958333373069763, | |
| "regularize": 0.1369226723909378, | |
| "step": 310, | |
| "wo_beta": 14.816594123840332 | |
| }, | |
| { | |
| "dpo_loss": 0.6855903267860413, | |
| "epoch": 0.8927727916863486, | |
| "grad_norm": 29.897768012112312, | |
| "learning_rate": 8.852566213878946e-07, | |
| "logits": -1.2702066898345947, | |
| "logps": -79.8655014038086, | |
| "loss": 0.1353, | |
| "objective": 0.13145793974399567, | |
| "ranking_idealized": 0.5541666746139526, | |
| "ranking_idealized_expo": 0.47083333134651184, | |
| "ranking_simple": 0.4749999940395355, | |
| "regularize": 0.13145793974399567, | |
| "step": 315, | |
| "wo_beta": 15.161810874938965 | |
| }, | |
| { | |
| "dpo_loss": 0.691845178604126, | |
| "epoch": 0.9069437883797827, | |
| "grad_norm": 28.736143424115674, | |
| "learning_rate": 8.799343922115043e-07, | |
| "logits": -1.2241441011428833, | |
| "logps": -82.17134094238281, | |
| "loss": 0.13, | |
| "objective": 0.1402612328529358, | |
| "ranking_idealized": 0.6208333373069763, | |
| "ranking_idealized_expo": 0.5583333373069763, | |
| "ranking_simple": 0.5541666746139526, | |
| "regularize": 0.1402612328529358, | |
| "step": 320, | |
| "wo_beta": 15.099017143249512 | |
| }, | |
| { | |
| "dpo_loss": 0.6962689161300659, | |
| "epoch": 0.9211147850732169, | |
| "grad_norm": 28.356303375759392, | |
| "learning_rate": 8.745082927659046e-07, | |
| "logits": -1.2910945415496826, | |
| "logps": -83.30491638183594, | |
| "loss": 0.1308, | |
| "objective": 0.14350637793540955, | |
| "ranking_idealized": 0.6083333492279053, | |
| "ranking_idealized_expo": 0.5, | |
| "ranking_simple": 0.5166666507720947, | |
| "regularize": 0.14350637793540955, | |
| "step": 325, | |
| "wo_beta": 15.133590698242188 | |
| }, | |
| { | |
| "dpo_loss": 0.6975868344306946, | |
| "epoch": 0.9352857817666509, | |
| "grad_norm": 29.00689810312343, | |
| "learning_rate": 8.689798064925048e-07, | |
| "logits": -1.1349345445632935, | |
| "logps": -82.04910278320312, | |
| "loss": 0.1321, | |
| "objective": 0.1296585500240326, | |
| "ranking_idealized": 0.6166666746139526, | |
| "ranking_idealized_expo": 0.5291666388511658, | |
| "ranking_simple": 0.5291666388511658, | |
| "regularize": 0.1296585500240326, | |
| "step": 330, | |
| "wo_beta": 16.1423282623291 | |
| }, | |
| { | |
| "dpo_loss": 0.7005541920661926, | |
| "epoch": 0.949456778460085, | |
| "grad_norm": 32.3756572284601, | |
| "learning_rate": 8.633504448242504e-07, | |
| "logits": -1.149806261062622, | |
| "logps": -81.64175415039062, | |
| "loss": 0.1375, | |
| "objective": 0.1390267014503479, | |
| "ranking_idealized": 0.5874999761581421, | |
| "ranking_idealized_expo": 0.5208333134651184, | |
| "ranking_simple": 0.5208333134651184, | |
| "regularize": 0.1390267014503479, | |
| "step": 335, | |
| "wo_beta": 15.652006149291992 | |
| }, | |
| { | |
| "dpo_loss": 0.6950960755348206, | |
| "epoch": 0.9636277751535192, | |
| "grad_norm": 27.122604040368284, | |
| "learning_rate": 8.576217467724127e-07, | |
| "logits": -1.2132624387741089, | |
| "logps": -80.64006042480469, | |
| "loss": 0.1292, | |
| "objective": 0.12200692296028137, | |
| "ranking_idealized": 0.6041666865348816, | |
| "ranking_idealized_expo": 0.5458333492279053, | |
| "ranking_simple": 0.5416666865348816, | |
| "regularize": 0.12200692296028137, | |
| "step": 340, | |
| "wo_beta": 15.907822608947754 | |
| }, | |
| { | |
| "dpo_loss": 0.6975562572479248, | |
| "epoch": 0.9777987718469532, | |
| "grad_norm": 27.134170349804087, | |
| "learning_rate": 8.517952785058384e-07, | |
| "logits": -1.2632955312728882, | |
| "logps": -80.71128845214844, | |
| "loss": 0.1283, | |
| "objective": 0.11938898265361786, | |
| "ranking_idealized": 0.6458333134651184, | |
| "ranking_idealized_expo": 0.5833333134651184, | |
| "ranking_simple": 0.5708333253860474, | |
| "regularize": 0.11938898265361786, | |
| "step": 345, | |
| "wo_beta": 14.762292861938477 | |
| }, | |
| { | |
| "dpo_loss": 0.6852299571037292, | |
| "epoch": 0.9919697685403873, | |
| "grad_norm": 27.658996359022336, | |
| "learning_rate": 8.458726329227747e-07, | |
| "logits": -1.1914026737213135, | |
| "logps": -81.73149108886719, | |
| "loss": 0.1407, | |
| "objective": 0.1554519683122635, | |
| "ranking_idealized": 0.6583333611488342, | |
| "ranking_idealized_expo": 0.574999988079071, | |
| "ranking_simple": 0.574999988079071, | |
| "regularize": 0.1554519683122635, | |
| "step": 350, | |
| "wo_beta": 15.107101440429688 | |
| }, | |
| { | |
| "epoch": 0.9919697685403873, | |
| "eval_dpo_loss": 0.6975587606430054, | |
| "eval_logits": -1.2756990194320679, | |
| "eval_logps": -86.45014190673828, | |
| "eval_loss": 0.13138790428638458, | |
| "eval_objective": 0.13096390664577484, | |
| "eval_ranking_idealized": 0.6024844646453857, | |
| "eval_ranking_idealized_expo": 0.5232919454574585, | |
| "eval_ranking_simple": 0.5222567319869995, | |
| "eval_regularize": 0.13096390664577484, | |
| "eval_runtime": 498.5276, | |
| "eval_samples_per_second": 11.614, | |
| "eval_steps_per_second": 0.969, | |
| "eval_wo_beta": 15.656978607177734, | |
| "step": 350 | |
| }, | |
| { | |
| "dpo_loss": 0.6982021331787109, | |
| "epoch": 1.0061407652338215, | |
| "grad_norm": 28.652193663332632, | |
| "learning_rate": 8.398554292153865e-07, | |
| "logits": -1.3350815773010254, | |
| "logps": -79.34367370605469, | |
| "loss": 0.1274, | |
| "objective": 0.1257932186126709, | |
| "ranking_idealized": 0.6083333492279053, | |
| "ranking_idealized_expo": 0.5375000238418579, | |
| "ranking_simple": 0.5333333611488342, | |
| "regularize": 0.1257932186126709, | |
| "step": 355, | |
| "wo_beta": 16.378000259399414 | |
| }, | |
| { | |
| "dpo_loss": 0.6944258809089661, | |
| "epoch": 1.0203117619272555, | |
| "grad_norm": 27.709591206743504, | |
| "learning_rate": 8.337453124270862e-07, | |
| "logits": -1.2474267482757568, | |
| "logps": -80.31254577636719, | |
| "loss": 0.1453, | |
| "objective": 0.14443162083625793, | |
| "ranking_idealized": 0.574999988079071, | |
| "ranking_idealized_expo": 0.5083333253860474, | |
| "ranking_simple": 0.4958333373069763, | |
| "regularize": 0.14443162083625793, | |
| "step": 360, | |
| "wo_beta": 16.190935134887695 | |
| }, | |
| { | |
| "dpo_loss": 0.6928178071975708, | |
| "epoch": 1.0344827586206897, | |
| "grad_norm": 31.070681767199403, | |
| "learning_rate": 8.275439530027947e-07, | |
| "logits": -1.276475191116333, | |
| "logps": -80.50602722167969, | |
| "loss": 0.1371, | |
| "objective": 0.13979977369308472, | |
| "ranking_idealized": 0.5833333134651184, | |
| "ranking_idealized_expo": 0.5, | |
| "ranking_simple": 0.5166666507720947, | |
| "regularize": 0.13979977369308472, | |
| "step": 365, | |
| "wo_beta": 14.378859519958496 | |
| }, | |
| { | |
| "dpo_loss": 0.699609637260437, | |
| "epoch": 1.0486537553141237, | |
| "grad_norm": 30.003574042191506, | |
| "learning_rate": 8.212530463322582e-07, | |
| "logits": -1.2496185302734375, | |
| "logps": -79.11912536621094, | |
| "loss": 0.1306, | |
| "objective": 0.1423943042755127, | |
| "ranking_idealized": 0.5958333611488342, | |
| "ranking_idealized_expo": 0.5166666507720947, | |
| "ranking_simple": 0.512499988079071, | |
| "regularize": 0.1423943042755127, | |
| "step": 370, | |
| "wo_beta": 15.124627113342285 | |
| }, | |
| { | |
| "dpo_loss": 0.6967942714691162, | |
| "epoch": 1.0628247520075578, | |
| "grad_norm": 27.953352635424668, | |
| "learning_rate": 8.148743122865463e-07, | |
| "logits": -1.3011940717697144, | |
| "logps": -80.02760314941406, | |
| "loss": 0.1297, | |
| "objective": 0.11541719734668732, | |
| "ranking_idealized": 0.6083333492279053, | |
| "ranking_idealized_expo": 0.5166666507720947, | |
| "ranking_simple": 0.5249999761581421, | |
| "regularize": 0.11541719734668732, | |
| "step": 375, | |
| "wo_beta": 15.568713188171387 | |
| }, | |
| { | |
| "dpo_loss": 0.6877638697624207, | |
| "epoch": 1.076995748700992, | |
| "grad_norm": 27.32675287386393, | |
| "learning_rate": 8.084094947478554e-07, | |
| "logits": -1.2718795537948608, | |
| "logps": -81.57784271240234, | |
| "loss": 0.1306, | |
| "objective": 0.1403437703847885, | |
| "ranking_idealized": 0.6625000238418579, | |
| "ranking_idealized_expo": 0.574999988079071, | |
| "ranking_simple": 0.574999988079071, | |
| "regularize": 0.1403437703847885, | |
| "step": 380, | |
| "wo_beta": 15.024064064025879 | |
| }, | |
| { | |
| "dpo_loss": 0.7029018402099609, | |
| "epoch": 1.091166745394426, | |
| "grad_norm": 25.199092121516863, | |
| "learning_rate": 8.018603611327504e-07, | |
| "logits": -1.2051031589508057, | |
| "logps": -80.49242401123047, | |
| "loss": 0.1289, | |
| "objective": 0.12692388892173767, | |
| "ranking_idealized": 0.6000000238418579, | |
| "ranking_idealized_expo": 0.5583333373069763, | |
| "ranking_simple": 0.5375000238418579, | |
| "regularize": 0.12692388892173767, | |
| "step": 385, | |
| "wo_beta": 15.658522605895996 | |
| }, | |
| { | |
| "dpo_loss": 0.6920034885406494, | |
| "epoch": 1.10533774208786, | |
| "grad_norm": 28.52425339340298, | |
| "learning_rate": 7.952287019089685e-07, | |
| "logits": -1.1542584896087646, | |
| "logps": -82.2014389038086, | |
| "loss": 0.128, | |
| "objective": 0.13050222396850586, | |
| "ranking_idealized": 0.637499988079071, | |
| "ranking_idealized_expo": 0.5625, | |
| "ranking_simple": 0.5666666626930237, | |
| "regularize": 0.13050222396850586, | |
| "step": 390, | |
| "wo_beta": 15.990551948547363 | |
| }, | |
| { | |
| "dpo_loss": 0.6983939409255981, | |
| "epoch": 1.1195087387812943, | |
| "grad_norm": 28.676328293583875, | |
| "learning_rate": 7.88516330105925e-07, | |
| "logits": -1.21431303024292, | |
| "logps": -81.3152847290039, | |
| "loss": 0.1271, | |
| "objective": 0.12024066597223282, | |
| "ranking_idealized": 0.5791666507720947, | |
| "ranking_idealized_expo": 0.5041666626930237, | |
| "ranking_simple": 0.5, | |
| "regularize": 0.12024066597223282, | |
| "step": 395, | |
| "wo_beta": 14.856566429138184 | |
| }, | |
| { | |
| "dpo_loss": 0.7045825719833374, | |
| "epoch": 1.1336797354747283, | |
| "grad_norm": 26.361954924055155, | |
| "learning_rate": 7.817250808190483e-07, | |
| "logits": -1.2783249616622925, | |
| "logps": -79.67323303222656, | |
| "loss": 0.1245, | |
| "objective": 0.12074790149927139, | |
| "ranking_idealized": 0.6291666626930237, | |
| "ranking_idealized_expo": 0.5416666865348816, | |
| "ranking_simple": 0.5416666865348816, | |
| "regularize": 0.12074790149927139, | |
| "step": 400, | |
| "wo_beta": 15.344539642333984 | |
| }, | |
| { | |
| "epoch": 1.1336797354747283, | |
| "eval_dpo_loss": 0.698018491268158, | |
| "eval_logits": -1.2417831420898438, | |
| "eval_logps": -86.2849349975586, | |
| "eval_loss": 0.13988268375396729, | |
| "eval_objective": 0.13904725015163422, | |
| "eval_ranking_idealized": 0.6024844646453857, | |
| "eval_ranking_idealized_expo": 0.5232919454574585, | |
| "eval_ranking_simple": 0.5258799195289612, | |
| "eval_regularize": 0.13904725015163422, | |
| "eval_runtime": 544.4211, | |
| "eval_samples_per_second": 10.635, | |
| "eval_steps_per_second": 0.887, | |
| "eval_wo_beta": 15.614696502685547, | |
| "step": 400 | |
| }, | |
| { | |
| "dpo_loss": 0.6944829225540161, | |
| "epoch": 1.1478507321681626, | |
| "grad_norm": 26.829396266860115, | |
| "learning_rate": 7.74856810708083e-07, | |
| "logits": -1.2358256578445435, | |
| "logps": -80.91136169433594, | |
| "loss": 0.1252, | |
| "objective": 0.13733495771884918, | |
| "ranking_idealized": 0.5666666626930237, | |
| "ranking_idealized_expo": 0.4625000059604645, | |
| "ranking_simple": 0.46666666865348816, | |
| "regularize": 0.13733495771884918, | |
| "step": 405, | |
| "wo_beta": 16.799846649169922 | |
| }, | |
| { | |
| "dpo_loss": 0.6951694488525391, | |
| "epoch": 1.1620217288615966, | |
| "grad_norm": 25.84880624163644, | |
| "learning_rate": 7.679133974894982e-07, | |
| "logits": -1.2413955926895142, | |
| "logps": -80.84453582763672, | |
| "loss": 0.1146, | |
| "objective": 0.10967493802309036, | |
| "ranking_idealized": 0.6000000238418579, | |
| "ranking_idealized_expo": 0.5208333134651184, | |
| "ranking_simple": 0.512499988079071, | |
| "regularize": 0.10967493802309036, | |
| "step": 410, | |
| "wo_beta": 16.299657821655273 | |
| }, | |
| { | |
| "dpo_loss": 0.6816955804824829, | |
| "epoch": 1.1761927255550306, | |
| "grad_norm": 28.539266676030703, | |
| "learning_rate": 7.608967394231386e-07, | |
| "logits": -1.1460075378417969, | |
| "logps": -80.07962799072266, | |
| "loss": 0.1201, | |
| "objective": 0.11568634957075119, | |
| "ranking_idealized": 0.612500011920929, | |
| "ranking_idealized_expo": 0.5166666507720947, | |
| "ranking_simple": 0.512499988079071, | |
| "regularize": 0.11568634957075119, | |
| "step": 415, | |
| "wo_beta": 15.849366188049316 | |
| }, | |
| { | |
| "dpo_loss": 0.6958954334259033, | |
| "epoch": 1.1903637222484649, | |
| "grad_norm": 26.83226072322417, | |
| "learning_rate": 7.538087547932584e-07, | |
| "logits": -1.1252403259277344, | |
| "logps": -80.94552612304688, | |
| "loss": 0.1212, | |
| "objective": 0.11827482283115387, | |
| "ranking_idealized": 0.6000000238418579, | |
| "ranking_idealized_expo": 0.4958333373069763, | |
| "ranking_simple": 0.5083333253860474, | |
| "regularize": 0.11827482283115387, | |
| "step": 420, | |
| "wo_beta": 16.14940643310547 | |
| }, | |
| { | |
| "dpo_loss": 0.6887015700340271, | |
| "epoch": 1.204534718941899, | |
| "grad_norm": 26.51780573149761, | |
| "learning_rate": 7.466513813840824e-07, | |
| "logits": -1.1933962106704712, | |
| "logps": -78.89797973632812, | |
| "loss": 0.1135, | |
| "objective": 0.1143736019730568, | |
| "ranking_idealized": 0.5958333611488342, | |
| "ranking_idealized_expo": 0.512499988079071, | |
| "ranking_simple": 0.5166666507720947, | |
| "regularize": 0.1143736019730568, | |
| "step": 425, | |
| "wo_beta": 14.578470230102539 | |
| }, | |
| { | |
| "dpo_loss": 0.6991615891456604, | |
| "epoch": 1.2187057156353331, | |
| "grad_norm": 27.122286588814305, | |
| "learning_rate": 7.394265759500347e-07, | |
| "logits": -1.1930339336395264, | |
| "logps": -80.126220703125, | |
| "loss": 0.1127, | |
| "objective": 0.11676573753356934, | |
| "ranking_idealized": 0.5625, | |
| "ranking_idealized_expo": 0.4958333373069763, | |
| "ranking_simple": 0.4749999940395355, | |
| "regularize": 0.11676573753356934, | |
| "step": 430, | |
| "wo_beta": 15.9819974899292 | |
| }, | |
| { | |
| "dpo_loss": 0.6940677762031555, | |
| "epoch": 1.2328767123287672, | |
| "grad_norm": 26.542064973728884, | |
| "learning_rate": 7.321363136807818e-07, | |
| "logits": -1.1478148698806763, | |
| "logps": -80.018310546875, | |
| "loss": 0.1273, | |
| "objective": 0.12024448066949844, | |
| "ranking_idealized": 0.6458333134651184, | |
| "ranking_idealized_expo": 0.5958333611488342, | |
| "ranking_simple": 0.6000000238418579, | |
| "regularize": 0.12024448066949844, | |
| "step": 435, | |
| "wo_beta": 17.044919967651367 | |
| }, | |
| { | |
| "dpo_loss": 0.6969379186630249, | |
| "epoch": 1.2470477090222012, | |
| "grad_norm": 28.251093862423456, | |
| "learning_rate": 7.247825876612352e-07, | |
| "logits": -1.1687721014022827, | |
| "logps": -79.19255828857422, | |
| "loss": 0.1253, | |
| "objective": 0.12027813494205475, | |
| "ranking_idealized": 0.612500011920929, | |
| "ranking_idealized_expo": 0.5541666746139526, | |
| "ranking_simple": 0.550000011920929, | |
| "regularize": 0.12027813494205475, | |
| "step": 440, | |
| "wo_beta": 14.654241561889648 | |
| }, | |
| { | |
| "dpo_loss": 0.6925280094146729, | |
| "epoch": 1.2612187057156352, | |
| "grad_norm": 26.524957115429544, | |
| "learning_rate": 7.173674083266623e-07, | |
| "logits": -1.1623238325119019, | |
| "logps": -80.57234191894531, | |
| "loss": 0.1123, | |
| "objective": 0.11110406368970871, | |
| "ranking_idealized": 0.637499988079071, | |
| "ranking_idealized_expo": 0.5333333611488342, | |
| "ranking_simple": 0.5416666865348816, | |
| "regularize": 0.11110406368970871, | |
| "step": 445, | |
| "wo_beta": 16.533472061157227 | |
| }, | |
| { | |
| "dpo_loss": 0.6959200501441956, | |
| "epoch": 1.2753897024090695, | |
| "grad_norm": 25.817109114436615, | |
| "learning_rate": 7.098928029130528e-07, | |
| "logits": -1.2953335046768188, | |
| "logps": -80.17058563232422, | |
| "loss": 0.1163, | |
| "objective": 0.11630918085575104, | |
| "ranking_idealized": 0.5541666746139526, | |
| "ranking_idealized_expo": 0.5, | |
| "ranking_simple": 0.5, | |
| "regularize": 0.11630918085575104, | |
| "step": 450, | |
| "wo_beta": 14.598237991333008 | |
| }, | |
| { | |
| "epoch": 1.2753897024090695, | |
| "eval_dpo_loss": 0.6984797716140747, | |
| "eval_logits": -1.2306897640228271, | |
| "eval_logps": -85.48281860351562, | |
| "eval_loss": 0.14205217361450195, | |
| "eval_objective": 0.14207439124584198, | |
| "eval_ranking_idealized": 0.6024844646453857, | |
| "eval_ranking_idealized_expo": 0.5232919454574585, | |
| "eval_ranking_simple": 0.5274327397346497, | |
| "eval_regularize": 0.14207439124584198, | |
| "eval_runtime": 502.3414, | |
| "eval_samples_per_second": 11.526, | |
| "eval_steps_per_second": 0.961, | |
| "eval_wo_beta": 15.61281681060791, | |
| "step": 450 | |
| }, | |
| { | |
| "dpo_loss": 0.6977149248123169, | |
| "epoch": 1.2895606991025035, | |
| "grad_norm": 29.01794849451687, | |
| "learning_rate": 7.023608149028936e-07, | |
| "logits": -1.1321525573730469, | |
| "logps": -79.79704284667969, | |
| "loss": 0.1102, | |
| "objective": 0.10798730701208115, | |
| "ranking_idealized": 0.5458333492279053, | |
| "ranking_idealized_expo": 0.49166667461395264, | |
| "ranking_simple": 0.4958333373069763, | |
| "regularize": 0.10798730701208115, | |
| "step": 455, | |
| "wo_beta": 14.988642692565918 | |
| }, | |
| { | |
| "dpo_loss": 0.6960881352424622, | |
| "epoch": 1.3037316957959377, | |
| "grad_norm": 25.82316278857825, | |
| "learning_rate": 6.947735034665001e-07, | |
| "logits": -1.2272473573684692, | |
| "logps": -79.4093246459961, | |
| "loss": 0.1071, | |
| "objective": 0.10132616013288498, | |
| "ranking_idealized": 0.5583333373069763, | |
| "ranking_idealized_expo": 0.4749999940395355, | |
| "ranking_simple": 0.48750001192092896, | |
| "regularize": 0.10132616013288498, | |
| "step": 460, | |
| "wo_beta": 15.888258934020996 | |
| }, | |
| { | |
| "dpo_loss": 0.6906372308731079, | |
| "epoch": 1.3179026924893718, | |
| "grad_norm": 30.635018246102483, | |
| "learning_rate": 6.871329428990601e-07, | |
| "logits": -1.2102056741714478, | |
| "logps": -78.2228775024414, | |
| "loss": 0.1131, | |
| "objective": 0.11604170501232147, | |
| "ranking_idealized": 0.5541666746139526, | |
| "ranking_idealized_expo": 0.4749999940395355, | |
| "ranking_simple": 0.46666666865348816, | |
| "regularize": 0.11604170501232147, | |
| "step": 465, | |
| "wo_beta": 14.311129570007324 | |
| }, | |
| { | |
| "dpo_loss": 0.700882077217102, | |
| "epoch": 1.3320736891828058, | |
| "grad_norm": 27.46778566417897, | |
| "learning_rate": 6.794412220535425e-07, | |
| "logits": -1.2833130359649658, | |
| "logps": -77.55262756347656, | |
| "loss": 0.108, | |
| "objective": 0.10955775529146194, | |
| "ranking_idealized": 0.612500011920929, | |
| "ranking_idealized_expo": 0.5, | |
| "ranking_simple": 0.5083333253860474, | |
| "regularize": 0.10955775529146194, | |
| "step": 470, | |
| "wo_beta": 14.30273151397705 | |
| }, | |
| { | |
| "dpo_loss": 0.6947088241577148, | |
| "epoch": 1.34624468587624, | |
| "grad_norm": 27.567991845029866, | |
| "learning_rate": 6.717004437696249e-07, | |
| "logits": -1.1878196001052856, | |
| "logps": -79.7737808227539, | |
| "loss": 0.1143, | |
| "objective": 0.10682200640439987, | |
| "ranking_idealized": 0.6416666507720947, | |
| "ranking_idealized_expo": 0.574999988079071, | |
| "ranking_simple": 0.5708333253860474, | |
| "regularize": 0.10682200640439987, | |
| "step": 475, | |
| "wo_beta": 16.000301361083984 | |
| }, | |
| { | |
| "dpo_loss": 0.7012575268745422, | |
| "epoch": 1.360415682569674, | |
| "grad_norm": 26.96292751307233, | |
| "learning_rate": 6.639127242987987e-07, | |
| "logits": -1.2194726467132568, | |
| "logps": -79.7364730834961, | |
| "loss": 0.1121, | |
| "objective": 0.10879840701818466, | |
| "ranking_idealized": 0.5666666626930237, | |
| "ranking_idealized_expo": 0.4791666567325592, | |
| "ranking_simple": 0.4958333373069763, | |
| "regularize": 0.10879840701818466, | |
| "step": 480, | |
| "wo_beta": 17.723169326782227 | |
| }, | |
| { | |
| "dpo_loss": 0.6902076005935669, | |
| "epoch": 1.3745866792631083, | |
| "grad_norm": 25.536217139623062, | |
| "learning_rate": 6.560801927258079e-07, | |
| "logits": -1.2140812873840332, | |
| "logps": -77.77493286132812, | |
| "loss": 0.1063, | |
| "objective": 0.10283537954092026, | |
| "ranking_idealized": 0.574999988079071, | |
| "ranking_idealized_expo": 0.4958333373069763, | |
| "ranking_simple": 0.4958333373069763, | |
| "regularize": 0.10283537954092026, | |
| "step": 485, | |
| "wo_beta": 16.162378311157227 | |
| }, | |
| { | |
| "dpo_loss": 0.7003743648529053, | |
| "epoch": 1.3887576759565423, | |
| "grad_norm": 25.340128312194377, | |
| "learning_rate": 6.482049903865768e-07, | |
| "logits": -1.1755324602127075, | |
| "logps": -80.6698226928711, | |
| "loss": 0.1065, | |
| "objective": 0.11661101877689362, | |
| "ranking_idealized": 0.574999988079071, | |
| "ranking_idealized_expo": 0.5333333611488342, | |
| "ranking_simple": 0.5375000238418579, | |
| "regularize": 0.11661101877689362, | |
| "step": 490, | |
| "wo_beta": 15.291964530944824 | |
| }, | |
| { | |
| "dpo_loss": 0.6933376789093018, | |
| "epoch": 1.4029286726499763, | |
| "grad_norm": 27.42214588210337, | |
| "learning_rate": 6.402892702827916e-07, | |
| "logits": -1.203405499458313, | |
| "logps": -81.71482849121094, | |
| "loss": 0.1083, | |
| "objective": 0.11117922514677048, | |
| "ranking_idealized": 0.5583333373069763, | |
| "ranking_idealized_expo": 0.4791666567325592, | |
| "ranking_simple": 0.4833333194255829, | |
| "regularize": 0.11117922514677048, | |
| "step": 495, | |
| "wo_beta": 15.19780445098877 | |
| }, | |
| { | |
| "dpo_loss": 0.6919417977333069, | |
| "epoch": 1.4170996693434104, | |
| "grad_norm": 25.59099749967404, | |
| "learning_rate": 6.323351964932908e-07, | |
| "logits": -1.1464035511016846, | |
| "logps": -80.67649841308594, | |
| "loss": 0.1071, | |
| "objective": 0.10751333087682724, | |
| "ranking_idealized": 0.6000000238418579, | |
| "ranking_idealized_expo": 0.4833333194255829, | |
| "ranking_simple": 0.49166667461395264, | |
| "regularize": 0.10751333087682724, | |
| "step": 500, | |
| "wo_beta": 14.786382675170898 | |
| }, | |
| { | |
| "epoch": 1.4170996693434104, | |
| "eval_dpo_loss": 0.6979657411575317, | |
| "eval_logits": -1.2270138263702393, | |
| "eval_logps": -87.26725006103516, | |
| "eval_loss": 0.13817694783210754, | |
| "eval_objective": 0.1376110315322876, | |
| "eval_ranking_idealized": 0.6024844646453857, | |
| "eval_ranking_idealized_expo": 0.5232919454574585, | |
| "eval_ranking_simple": 0.5284678936004639, | |
| "eval_regularize": 0.1376110315322876, | |
| "eval_runtime": 507.9058, | |
| "eval_samples_per_second": 11.4, | |
| "eval_steps_per_second": 0.951, | |
| "eval_wo_beta": 15.64445686340332, | |
| "step": 500 | |
| }, | |
| { | |
| "dpo_loss": 0.6891559362411499, | |
| "epoch": 1.4312706660368446, | |
| "grad_norm": 24.27122577359571, | |
| "learning_rate": 6.243449435824276e-07, | |
| "logits": -1.2177590131759644, | |
| "logps": -81.35147094726562, | |
| "loss": 0.1101, | |
| "objective": 0.1094871535897255, | |
| "ranking_idealized": 0.625, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.5333333611488342, | |
| "regularize": 0.1094871535897255, | |
| "step": 505, | |
| "wo_beta": 15.79046630859375 | |
| }, | |
| { | |
| "dpo_loss": 0.6941244602203369, | |
| "epoch": 1.4454416627302786, | |
| "grad_norm": 25.930769694740054, | |
| "learning_rate": 6.163206960055652e-07, | |
| "logits": -1.251134991645813, | |
| "logps": -83.10639953613281, | |
| "loss": 0.1006, | |
| "objective": 0.09994279593229294, | |
| "ranking_idealized": 0.5874999761581421, | |
| "ranking_idealized_expo": 0.5208333134651184, | |
| "ranking_simple": 0.5291666388511658, | |
| "regularize": 0.09994279593229294, | |
| "step": 510, | |
| "wo_beta": 14.899516105651855 | |
| }, | |
| { | |
| "dpo_loss": 0.6874905824661255, | |
| "epoch": 1.4596126594237129, | |
| "grad_norm": 26.602314880639124, | |
| "learning_rate": 6.082646475118699e-07, | |
| "logits": -1.2633229494094849, | |
| "logps": -84.02688598632812, | |
| "loss": 0.106, | |
| "objective": 0.10199037194252014, | |
| "ranking_idealized": 0.550000011920929, | |
| "ranking_idealized_expo": 0.4625000059604645, | |
| "ranking_simple": 0.4541666805744171, | |
| "regularize": 0.10199037194252014, | |
| "step": 515, | |
| "wo_beta": 15.941681861877441 | |
| }, | |
| { | |
| "dpo_loss": 0.6967552900314331, | |
| "epoch": 1.473783656117147, | |
| "grad_norm": 29.852612268822412, | |
| "learning_rate": 6.001790005445606e-07, | |
| "logits": -1.184912919998169, | |
| "logps": -80.95891571044922, | |
| "loss": 0.1071, | |
| "objective": 0.10300089418888092, | |
| "ranking_idealized": 0.574999988079071, | |
| "ranking_idealized_expo": 0.49166667461395264, | |
| "ranking_simple": 0.49166667461395264, | |
| "regularize": 0.10300089418888092, | |
| "step": 520, | |
| "wo_beta": 15.731270790100098 | |
| }, | |
| { | |
| "dpo_loss": 0.6896428465843201, | |
| "epoch": 1.487954652810581, | |
| "grad_norm": 25.111572790175902, | |
| "learning_rate": 5.920659656387836e-07, | |
| "logits": -1.0910202264785767, | |
| "logps": -79.46784973144531, | |
| "loss": 0.1087, | |
| "objective": 0.10289794951677322, | |
| "ranking_idealized": 0.6000000238418579, | |
| "ranking_idealized_expo": 0.5, | |
| "ranking_simple": 0.5041666626930237, | |
| "regularize": 0.10289794951677322, | |
| "step": 525, | |
| "wo_beta": 14.915215492248535 | |
| }, | |
| { | |
| "dpo_loss": 0.6884135603904724, | |
| "epoch": 1.5021256495040152, | |
| "grad_norm": 29.155836377588727, | |
| "learning_rate": 5.839277608172738e-07, | |
| "logits": -1.2429722547531128, | |
| "logps": -82.09452056884766, | |
| "loss": 0.1052, | |
| "objective": 0.11081438511610031, | |
| "ranking_idealized": 0.574999988079071, | |
| "ranking_idealized_expo": 0.5, | |
| "ranking_simple": 0.5083333253860474, | |
| "regularize": 0.11081438511610031, | |
| "step": 530, | |
| "wo_beta": 14.850537300109863 | |
| }, | |
| { | |
| "dpo_loss": 0.6961663961410522, | |
| "epoch": 1.5162966461974492, | |
| "grad_norm": 25.320925581209725, | |
| "learning_rate": 5.757666109839702e-07, | |
| "logits": -1.2323859930038452, | |
| "logps": -80.30747985839844, | |
| "loss": 0.1001, | |
| "objective": 0.09293892234563828, | |
| "ranking_idealized": 0.5833333134651184, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.5291666388511658, | |
| "regularize": 0.09293892234563828, | |
| "step": 535, | |
| "wo_beta": 15.262944221496582 | |
| }, | |
| { | |
| "dpo_loss": 0.6923481225967407, | |
| "epoch": 1.5304676428908834, | |
| "grad_norm": 32.01848958383342, | |
| "learning_rate": 5.675847473157485e-07, | |
| "logits": -1.1209362745285034, | |
| "logps": -80.81604766845703, | |
| "loss": 0.1017, | |
| "objective": 0.1114068478345871, | |
| "ranking_idealized": 0.6291666626930237, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.5208333134651184, | |
| "regularize": 0.1114068478345871, | |
| "step": 540, | |
| "wo_beta": 14.910977363586426 | |
| }, | |
| { | |
| "dpo_loss": 0.6916370987892151, | |
| "epoch": 1.5446386395843175, | |
| "grad_norm": 25.57319909143034, | |
| "learning_rate": 5.5938440665244e-07, | |
| "logits": -1.2216829061508179, | |
| "logps": -81.30005645751953, | |
| "loss": 0.1016, | |
| "objective": 0.09744974970817566, | |
| "ranking_idealized": 0.6583333611488342, | |
| "ranking_idealized_expo": 0.5874999761581421, | |
| "ranking_simple": 0.5791666507720947, | |
| "regularize": 0.09744974970817566, | |
| "step": 545, | |
| "wo_beta": 14.310770988464355 | |
| }, | |
| { | |
| "dpo_loss": 0.6908753514289856, | |
| "epoch": 1.5588096362777515, | |
| "grad_norm": 25.722462769354692, | |
| "learning_rate": 5.511678308853025e-07, | |
| "logits": -1.2278273105621338, | |
| "logps": -81.18257141113281, | |
| "loss": 0.1045, | |
| "objective": 0.11294317990541458, | |
| "ranking_idealized": 0.5625, | |
| "ranking_idealized_expo": 0.46666666865348816, | |
| "ranking_simple": 0.44583332538604736, | |
| "regularize": 0.11294317990541458, | |
| "step": 550, | |
| "wo_beta": 14.909473419189453 | |
| }, | |
| { | |
| "epoch": 1.5588096362777515, | |
| "eval_dpo_loss": 0.6977279186248779, | |
| "eval_logits": -1.2327359914779663, | |
| "eval_logps": -87.07755279541016, | |
| "eval_loss": 0.1427639275789261, | |
| "eval_objective": 0.14261718094348907, | |
| "eval_ranking_idealized": 0.6024844646453857, | |
| "eval_ranking_idealized_expo": 0.5232919454574585, | |
| "eval_ranking_simple": 0.5253623127937317, | |
| "eval_regularize": 0.14261718094348907, | |
| "eval_runtime": 530.6929, | |
| "eval_samples_per_second": 10.91, | |
| "eval_steps_per_second": 0.91, | |
| "eval_wo_beta": 15.58066177368164, | |
| "step": 550 | |
| }, | |
| { | |
| "dpo_loss": 0.6979414820671082, | |
| "epoch": 1.5729806329711855, | |
| "grad_norm": 25.97117042381748, | |
| "learning_rate": 5.429372663441085e-07, | |
| "logits": -1.0773119926452637, | |
| "logps": -80.85298919677734, | |
| "loss": 0.0969, | |
| "objective": 0.10372842103242874, | |
| "ranking_idealized": 0.6000000238418579, | |
| "ranking_idealized_expo": 0.5208333134651184, | |
| "ranking_simple": 0.5083333253860474, | |
| "regularize": 0.10372842103242874, | |
| "step": 555, | |
| "wo_beta": 14.278889656066895 | |
| }, | |
| { | |
| "dpo_loss": 0.6877902746200562, | |
| "epoch": 1.5871516296646198, | |
| "grad_norm": 26.089498554586406, | |
| "learning_rate": 5.34694963183022e-07, | |
| "logits": -1.149969220161438, | |
| "logps": -80.23606872558594, | |
| "loss": 0.0937, | |
| "objective": 0.0943736732006073, | |
| "ranking_idealized": 0.6083333492279053, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.5249999761581421, | |
| "regularize": 0.0943736732006073, | |
| "step": 560, | |
| "wo_beta": 15.772320747375488 | |
| }, | |
| { | |
| "dpo_loss": 0.6947767734527588, | |
| "epoch": 1.601322626358054, | |
| "grad_norm": 26.116747650931945, | |
| "learning_rate": 5.264431747654283e-07, | |
| "logits": -1.1340062618255615, | |
| "logps": -81.63863372802734, | |
| "loss": 0.0947, | |
| "objective": 0.10096503049135208, | |
| "ranking_idealized": 0.5458333492279053, | |
| "ranking_idealized_expo": 0.44999998807907104, | |
| "ranking_simple": 0.4416666626930237, | |
| "regularize": 0.10096503049135208, | |
| "step": 565, | |
| "wo_beta": 14.981669425964355 | |
| }, | |
| { | |
| "dpo_loss": 0.692035436630249, | |
| "epoch": 1.615493623051488, | |
| "grad_norm": 30.63214010200871, | |
| "learning_rate": 5.181841570478872e-07, | |
| "logits": -1.2694156169891357, | |
| "logps": -81.64689636230469, | |
| "loss": 0.0959, | |
| "objective": 0.1027316302061081, | |
| "ranking_idealized": 0.6000000238418579, | |
| "ranking_idealized_expo": 0.5166666507720947, | |
| "ranking_simple": 0.5375000238418579, | |
| "regularize": 0.1027316302061081, | |
| "step": 570, | |
| "wo_beta": 15.00640869140625 | |
| }, | |
| { | |
| "dpo_loss": 0.6928724646568298, | |
| "epoch": 1.629664619744922, | |
| "grad_norm": 26.684109688489027, | |
| "learning_rate": 5.099201679633768e-07, | |
| "logits": -1.219287633895874, | |
| "logps": -79.6671371459961, | |
| "loss": 0.0902, | |
| "objective": 0.08943381905555725, | |
| "ranking_idealized": 0.637499988079071, | |
| "ranking_idealized_expo": 0.5583333373069763, | |
| "ranking_simple": 0.5458333492279053, | |
| "regularize": 0.08943381905555725, | |
| "step": 575, | |
| "wo_beta": 15.883743286132812 | |
| }, | |
| { | |
| "dpo_loss": 0.7014293670654297, | |
| "epoch": 1.643835616438356, | |
| "grad_norm": 23.659415037737205, | |
| "learning_rate": 5.016534668039976e-07, | |
| "logits": -1.245025396347046, | |
| "logps": -79.65864562988281, | |
| "loss": 0.0922, | |
| "objective": 0.09364978969097137, | |
| "ranking_idealized": 0.6083333492279053, | |
| "ranking_idealized_expo": 0.5333333611488342, | |
| "ranking_simple": 0.5375000238418579, | |
| "regularize": 0.09364978969097137, | |
| "step": 580, | |
| "wo_beta": 14.111478805541992 | |
| }, | |
| { | |
| "dpo_loss": 0.6919021606445312, | |
| "epoch": 1.6580066131317903, | |
| "grad_norm": 26.81667336982406, | |
| "learning_rate": 4.933863136033039e-07, | |
| "logits": -1.1303011178970337, | |
| "logps": -79.01573944091797, | |
| "loss": 0.0912, | |
| "objective": 0.09164983779191971, | |
| "ranking_idealized": 0.5333333611488342, | |
| "ranking_idealized_expo": 0.4625000059604645, | |
| "ranking_simple": 0.4749999940395355, | |
| "regularize": 0.09164983779191971, | |
| "step": 585, | |
| "wo_beta": 15.943554878234863 | |
| }, | |
| { | |
| "dpo_loss": 0.6958838701248169, | |
| "epoch": 1.6721776098252243, | |
| "grad_norm": 24.98087975104312, | |
| "learning_rate": 4.851209685184338e-07, | |
| "logits": -1.1811211109161377, | |
| "logps": -78.23771667480469, | |
| "loss": 0.0896, | |
| "objective": 0.08815690129995346, | |
| "ranking_idealized": 0.6208333373069763, | |
| "ranking_idealized_expo": 0.5083333253860474, | |
| "ranking_simple": 0.49166667461395264, | |
| "regularize": 0.08815690129995346, | |
| "step": 590, | |
| "wo_beta": 13.053691864013672 | |
| }, | |
| { | |
| "dpo_loss": 0.7018415927886963, | |
| "epoch": 1.6863486065186586, | |
| "grad_norm": 28.222712089048155, | |
| "learning_rate": 4.768596912122045e-07, | |
| "logits": -1.1410056352615356, | |
| "logps": -78.93828582763672, | |
| "loss": 0.0867, | |
| "objective": 0.08855770528316498, | |
| "ranking_idealized": 0.6166666746139526, | |
| "ranking_idealized_expo": 0.5375000238418579, | |
| "ranking_simple": 0.5208333134651184, | |
| "regularize": 0.08855770528316498, | |
| "step": 595, | |
| "wo_beta": 16.56429672241211 | |
| }, | |
| { | |
| "dpo_loss": 0.6934791803359985, | |
| "epoch": 1.7005196032120926, | |
| "grad_norm": 26.368906194308657, | |
| "learning_rate": 4.686047402353433e-07, | |
| "logits": -1.1907525062561035, | |
| "logps": -80.13634490966797, | |
| "loss": 0.0866, | |
| "objective": 0.09509587287902832, | |
| "ranking_idealized": 0.6333333253860474, | |
| "ranking_idealized_expo": 0.550000011920929, | |
| "ranking_simple": 0.5375000238418579, | |
| "regularize": 0.09509587287902832, | |
| "step": 600, | |
| "wo_beta": 15.167766571044922 | |
| }, | |
| { | |
| "epoch": 1.7005196032120926, | |
| "eval_dpo_loss": 0.6965176463127136, | |
| "eval_logits": -1.2196165323257446, | |
| "eval_logps": -85.19258880615234, | |
| "eval_loss": 0.14236733317375183, | |
| "eval_objective": 0.14079627394676208, | |
| "eval_ranking_idealized": 0.6024844646453857, | |
| "eval_ranking_idealized_expo": 0.5232919454574585, | |
| "eval_ranking_simple": 0.5269151329994202, | |
| "eval_regularize": 0.14079627394676208, | |
| "eval_runtime": 531.3996, | |
| "eval_samples_per_second": 10.896, | |
| "eval_steps_per_second": 0.909, | |
| "eval_wo_beta": 15.660321235656738, | |
| "step": 600 | |
| }, | |
| { | |
| "dpo_loss": 0.6949159502983093, | |
| "epoch": 1.7146905999055266, | |
| "grad_norm": 27.65546942935795, | |
| "learning_rate": 4.60358372409022e-07, | |
| "logits": -1.135356068611145, | |
| "logps": -80.58204650878906, | |
| "loss": 0.0851, | |
| "objective": 0.10239014774560928, | |
| "ranking_idealized": 0.5833333134651184, | |
| "ranking_idealized_expo": 0.5291666388511658, | |
| "ranking_simple": 0.5166666507720947, | |
| "regularize": 0.10239014774560928, | |
| "step": 605, | |
| "wo_beta": 17.19474220275879 | |
| }, | |
| { | |
| "dpo_loss": 0.6975926160812378, | |
| "epoch": 1.7288615965989607, | |
| "grad_norm": 27.123580050770954, | |
| "learning_rate": 4.521228422078649e-07, | |
| "logits": -1.2206453084945679, | |
| "logps": -78.68167877197266, | |
| "loss": 0.0882, | |
| "objective": 0.0891619473695755, | |
| "ranking_idealized": 0.6291666626930237, | |
| "ranking_idealized_expo": 0.5708333253860474, | |
| "ranking_simple": 0.5791666507720947, | |
| "regularize": 0.0891619473695755, | |
| "step": 610, | |
| "wo_beta": 15.383539199829102 | |
| }, | |
| { | |
| "dpo_loss": 0.6973095536231995, | |
| "epoch": 1.743032593292395, | |
| "grad_norm": 27.24577954816879, | |
| "learning_rate": 4.439004011435979e-07, | |
| "logits": -1.2362395524978638, | |
| "logps": -79.0839614868164, | |
| "loss": 0.0875, | |
| "objective": 0.08598390221595764, | |
| "ranking_idealized": 0.5833333134651184, | |
| "ranking_idealized_expo": 0.5291666388511658, | |
| "ranking_simple": 0.5166666507720947, | |
| "regularize": 0.08598390221595764, | |
| "step": 615, | |
| "wo_beta": 15.571494102478027 | |
| }, | |
| { | |
| "dpo_loss": 0.6836999654769897, | |
| "epoch": 1.7572035899858292, | |
| "grad_norm": 26.787162425144906, | |
| "learning_rate": 4.3569329714950703e-07, | |
| "logits": -1.2427488565444946, | |
| "logps": -79.54029846191406, | |
| "loss": 0.0838, | |
| "objective": 0.08879180997610092, | |
| "ranking_idealized": 0.6208333373069763, | |
| "ranking_idealized_expo": 0.5416666865348816, | |
| "ranking_simple": 0.5458333492279053, | |
| "regularize": 0.08879180997610092, | |
| "step": 620, | |
| "wo_beta": 15.718174934387207 | |
| }, | |
| { | |
| "dpo_loss": 0.6966572999954224, | |
| "epoch": 1.7713745866792632, | |
| "grad_norm": 27.034118419678652, | |
| "learning_rate": 4.275037739658771e-07, | |
| "logits": -1.1582579612731934, | |
| "logps": -78.85964965820312, | |
| "loss": 0.0817, | |
| "objective": 0.08299548178911209, | |
| "ranking_idealized": 0.6083333492279053, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.5208333134651184, | |
| "regularize": 0.08299548178911209, | |
| "step": 625, | |
| "wo_beta": 14.923952102661133 | |
| }, | |
| { | |
| "dpo_loss": 0.688913881778717, | |
| "epoch": 1.7855455833726972, | |
| "grad_norm": 25.951621085094303, | |
| "learning_rate": 4.193340705265745e-07, | |
| "logits": -1.1893038749694824, | |
| "logps": -80.92503356933594, | |
| "loss": 0.0785, | |
| "objective": 0.08198042213916779, | |
| "ranking_idealized": 0.574999988079071, | |
| "ranking_idealized_expo": 0.49166667461395264, | |
| "ranking_simple": 0.48750001192092896, | |
| "regularize": 0.08198042213916779, | |
| "step": 630, | |
| "wo_beta": 15.90516185760498 | |
| }, | |
| { | |
| "dpo_loss": 0.6924195885658264, | |
| "epoch": 1.7997165800661312, | |
| "grad_norm": 26.232506833263244, | |
| "learning_rate": 4.1118642034694565e-07, | |
| "logits": -1.2785860300064087, | |
| "logps": -79.61809539794922, | |
| "loss": 0.0829, | |
| "objective": 0.08000766485929489, | |
| "ranking_idealized": 0.637499988079071, | |
| "ranking_idealized_expo": 0.5291666388511658, | |
| "ranking_simple": 0.5416666865348816, | |
| "regularize": 0.08000766485929489, | |
| "step": 635, | |
| "wo_beta": 15.796289443969727 | |
| }, | |
| { | |
| "dpo_loss": 0.6959947943687439, | |
| "epoch": 1.8138875767595655, | |
| "grad_norm": 27.993983855367574, | |
| "learning_rate": 4.030630509131959e-07, | |
| "logits": -1.2194859981536865, | |
| "logps": -80.71635437011719, | |
| "loss": 0.0842, | |
| "objective": 0.089814692735672, | |
| "ranking_idealized": 0.5666666626930237, | |
| "ranking_idealized_expo": 0.5083333253860474, | |
| "ranking_simple": 0.5, | |
| "regularize": 0.089814692735672, | |
| "step": 640, | |
| "wo_beta": 16.932401657104492 | |
| }, | |
| { | |
| "dpo_loss": 0.6927257776260376, | |
| "epoch": 1.8280585734529995, | |
| "grad_norm": 28.107934645205802, | |
| "learning_rate": 3.9496618307341713e-07, | |
| "logits": -1.256467342376709, | |
| "logps": -81.03665161132812, | |
| "loss": 0.0853, | |
| "objective": 0.0889531597495079, | |
| "ranking_idealized": 0.6416666507720947, | |
| "ranking_idealized_expo": 0.550000011920929, | |
| "ranking_simple": 0.5541666746139526, | |
| "regularize": 0.0889531597495079, | |
| "step": 645, | |
| "wo_beta": 15.800675392150879 | |
| }, | |
| { | |
| "dpo_loss": 0.6969668865203857, | |
| "epoch": 1.8422295701464337, | |
| "grad_norm": 27.015977070193543, | |
| "learning_rate": 3.8689803043042996e-07, | |
| "logits": -1.2903110980987549, | |
| "logps": -80.92781829833984, | |
| "loss": 0.0847, | |
| "objective": 0.0801667794585228, | |
| "ranking_idealized": 0.612500011920929, | |
| "ranking_idealized_expo": 0.5208333134651184, | |
| "ranking_simple": 0.5249999761581421, | |
| "regularize": 0.0801667794585228, | |
| "step": 650, | |
| "wo_beta": 15.173321723937988 | |
| }, | |
| { | |
| "epoch": 1.8422295701464337, | |
| "eval_dpo_loss": 0.6974130868911743, | |
| "eval_logits": -1.2229208946228027, | |
| "eval_logps": -86.1129150390625, | |
| "eval_loss": 0.1379525512456894, | |
| "eval_objective": 0.13563887774944305, | |
| "eval_ranking_idealized": 0.6024844646453857, | |
| "eval_ranking_idealized_expo": 0.5232919454574585, | |
| "eval_ranking_simple": 0.5243270993232727, | |
| "eval_regularize": 0.13563887774944305, | |
| "eval_runtime": 538.8083, | |
| "eval_samples_per_second": 10.746, | |
| "eval_steps_per_second": 0.896, | |
| "eval_wo_beta": 15.666037559509277, | |
| "step": 650 | |
| }, | |
| { | |
| "dpo_loss": 0.6896820068359375, | |
| "epoch": 1.8564005668398678, | |
| "grad_norm": 26.639855046988597, | |
| "learning_rate": 3.788607987366069e-07, | |
| "logits": -1.1662521362304688, | |
| "logps": -78.9451675415039, | |
| "loss": 0.081, | |
| "objective": 0.08504978567361832, | |
| "ranking_idealized": 0.6000000238418579, | |
| "ranking_idealized_expo": 0.48750001192092896, | |
| "ranking_simple": 0.4958333373069763, | |
| "regularize": 0.08504978567361832, | |
| "step": 655, | |
| "wo_beta": 15.233590126037598 | |
| }, | |
| { | |
| "dpo_loss": 0.6891672611236572, | |
| "epoch": 1.8705715635333018, | |
| "grad_norm": 25.995274477757608, | |
| "learning_rate": 3.708566852908418e-07, | |
| "logits": -1.2193191051483154, | |
| "logps": -81.20162200927734, | |
| "loss": 0.0781, | |
| "objective": 0.08211526274681091, | |
| "ranking_idealized": 0.6041666865348816, | |
| "ranking_idealized_expo": 0.5333333611488342, | |
| "ranking_simple": 0.5291666388511658, | |
| "regularize": 0.08211526274681091, | |
| "step": 660, | |
| "wo_beta": 16.372514724731445 | |
| }, | |
| { | |
| "dpo_loss": 0.6908305287361145, | |
| "epoch": 1.8847425602267358, | |
| "grad_norm": 28.66146531985666, | |
| "learning_rate": 3.6288787833783016e-07, | |
| "logits": -1.2218626737594604, | |
| "logps": -80.04493713378906, | |
| "loss": 0.0815, | |
| "objective": 0.08463230729103088, | |
| "ranking_idealized": 0.6333333253860474, | |
| "ranking_idealized_expo": 0.550000011920929, | |
| "ranking_simple": 0.5416666865348816, | |
| "regularize": 0.08463230729103088, | |
| "step": 665, | |
| "wo_beta": 15.034836769104004 | |
| }, | |
| { | |
| "dpo_loss": 0.6863933801651001, | |
| "epoch": 1.89891355692017, | |
| "grad_norm": 30.111613598581105, | |
| "learning_rate": 3.5495655646982503e-07, | |
| "logits": -1.1576950550079346, | |
| "logps": -78.58309173583984, | |
| "loss": 0.0755, | |
| "objective": 0.07363765686750412, | |
| "ranking_idealized": 0.5666666626930237, | |
| "ranking_idealized_expo": 0.4791666567325592, | |
| "ranking_simple": 0.47083333134651184, | |
| "regularize": 0.07363765686750412, | |
| "step": 670, | |
| "wo_beta": 16.09279441833496 | |
| }, | |
| { | |
| "dpo_loss": 0.695208728313446, | |
| "epoch": 1.9130845536136043, | |
| "grad_norm": 27.241452477717303, | |
| "learning_rate": 3.470648880310313e-07, | |
| "logits": -1.1648114919662476, | |
| "logps": -79.5347671508789, | |
| "loss": 0.0735, | |
| "objective": 0.07240771502256393, | |
| "ranking_idealized": 0.6166666746139526, | |
| "ranking_idealized_expo": 0.5333333611488342, | |
| "ranking_simple": 0.5249999761581421, | |
| "regularize": 0.07240771502256393, | |
| "step": 675, | |
| "wo_beta": 15.440892219543457 | |
| }, | |
| { | |
| "dpo_loss": 0.6950518488883972, | |
| "epoch": 1.9272555503070383, | |
| "grad_norm": 25.198332305215366, | |
| "learning_rate": 3.3921503052480236e-07, | |
| "logits": -1.2177760601043701, | |
| "logps": -81.27088165283203, | |
| "loss": 0.0778, | |
| "objective": 0.07866664230823517, | |
| "ranking_idealized": 0.6166666746139526, | |
| "ranking_idealized_expo": 0.5, | |
| "ranking_simple": 0.5083333253860474, | |
| "regularize": 0.07866664230823517, | |
| "step": 680, | |
| "wo_beta": 14.848203659057617 | |
| }, | |
| { | |
| "dpo_loss": 0.6911803483963013, | |
| "epoch": 1.9414265470004723, | |
| "grad_norm": 26.10726119743999, | |
| "learning_rate": 3.314091300237999e-07, | |
| "logits": -1.1625895500183105, | |
| "logps": -78.15774536132812, | |
| "loss": 0.0738, | |
| "objective": 0.07330299913883209, | |
| "ranking_idealized": 0.5291666388511658, | |
| "ranking_idealized_expo": 0.4791666567325592, | |
| "ranking_simple": 0.4749999940395355, | |
| "regularize": 0.07330299913883209, | |
| "step": 685, | |
| "wo_beta": 15.467205047607422 | |
| }, | |
| { | |
| "dpo_loss": 0.6860196590423584, | |
| "epoch": 1.9555975436939064, | |
| "grad_norm": 26.093926175967837, | |
| "learning_rate": 3.236493205832794e-07, | |
| "logits": -1.21792733669281, | |
| "logps": -79.12659454345703, | |
| "loss": 0.071, | |
| "objective": 0.07433832436800003, | |
| "ranking_idealized": 0.6041666865348816, | |
| "ranking_idealized_expo": 0.5583333373069763, | |
| "ranking_simple": 0.5458333492279053, | |
| "regularize": 0.07433832436800003, | |
| "step": 690, | |
| "wo_beta": 15.627902030944824 | |
| }, | |
| { | |
| "dpo_loss": 0.6922653317451477, | |
| "epoch": 1.9697685403873406, | |
| "grad_norm": 26.38661425001647, | |
| "learning_rate": 3.15937723657661e-07, | |
| "logits": -1.1168206930160522, | |
| "logps": -79.83128356933594, | |
| "loss": 0.0723, | |
| "objective": 0.06720028072595596, | |
| "ranking_idealized": 0.5666666626930237, | |
| "ranking_idealized_expo": 0.4833333194255829, | |
| "ranking_simple": 0.49166667461395264, | |
| "regularize": 0.06720028072595596, | |
| "step": 695, | |
| "wo_beta": 16.023746490478516 | |
| }, | |
| { | |
| "dpo_loss": 0.6856616139411926, | |
| "epoch": 1.9839395370807746, | |
| "grad_norm": 27.380948849082866, | |
| "learning_rate": 3.082764475205442e-07, | |
| "logits": -1.103851079940796, | |
| "logps": -80.37809753417969, | |
| "loss": 0.071, | |
| "objective": 0.0717112347483635, | |
| "ranking_idealized": 0.6041666865348816, | |
| "ranking_idealized_expo": 0.5208333134651184, | |
| "ranking_simple": 0.5166666507720947, | |
| "regularize": 0.0717112347483635, | |
| "step": 700, | |
| "wo_beta": 14.657614707946777 | |
| }, | |
| { | |
| "epoch": 1.9839395370807746, | |
| "eval_dpo_loss": 0.6979688405990601, | |
| "eval_logits": -1.220837116241455, | |
| "eval_logps": -85.24955749511719, | |
| "eval_loss": 0.1420368105173111, | |
| "eval_objective": 0.14046597480773926, | |
| "eval_ranking_idealized": 0.6024844646453857, | |
| "eval_ranking_idealized_expo": 0.5232919454574585, | |
| "eval_ranking_simple": 0.5253623127937317, | |
| "eval_regularize": 0.14046597480773926, | |
| "eval_runtime": 531.1966, | |
| "eval_samples_per_second": 10.9, | |
| "eval_steps_per_second": 0.909, | |
| "eval_wo_beta": 15.610904693603516, | |
| "step": 700 | |
| }, | |
| { | |
| "dpo_loss": 0.6904897093772888, | |
| "epoch": 1.9981105337742089, | |
| "grad_norm": 27.03253447324609, | |
| "learning_rate": 3.006675866883275e-07, | |
| "logits": -1.0365864038467407, | |
| "logps": -79.36177062988281, | |
| "loss": 0.0704, | |
| "objective": 0.07408583164215088, | |
| "ranking_idealized": 0.5916666388511658, | |
| "ranking_idealized_expo": 0.5, | |
| "ranking_simple": 0.5041666626930237, | |
| "regularize": 0.07408583164215088, | |
| "step": 705, | |
| "wo_beta": 16.758014678955078 | |
| }, | |
| { | |
| "dpo_loss": 0.6849521398544312, | |
| "epoch": 2.012281530467643, | |
| "grad_norm": 24.956147004394822, | |
| "learning_rate": 2.931132213475884e-07, | |
| "logits": -1.1888701915740967, | |
| "logps": -78.96455383300781, | |
| "loss": 0.0619, | |
| "objective": 0.06422288715839386, | |
| "ranking_idealized": 0.6083333492279053, | |
| "ranking_idealized_expo": 0.5208333134651184, | |
| "ranking_simple": 0.5208333134651184, | |
| "regularize": 0.06422288715839386, | |
| "step": 710, | |
| "wo_beta": 15.760772705078125 | |
| }, | |
| { | |
| "dpo_loss": 0.6961538791656494, | |
| "epoch": 2.026452527161077, | |
| "grad_norm": 27.279846270487834, | |
| "learning_rate": 2.856154167863814e-07, | |
| "logits": -1.1860238313674927, | |
| "logps": -78.40641021728516, | |
| "loss": 0.0631, | |
| "objective": 0.06441039592027664, | |
| "ranking_idealized": 0.6791666746139526, | |
| "ranking_idealized_expo": 0.5708333253860474, | |
| "ranking_simple": 0.550000011920929, | |
| "regularize": 0.06441039592027664, | |
| "step": 715, | |
| "wo_beta": 14.784539222717285 | |
| }, | |
| { | |
| "dpo_loss": 0.6898289322853088, | |
| "epoch": 2.040623523854511, | |
| "grad_norm": 28.14233189102926, | |
| "learning_rate": 2.7817622282960813e-07, | |
| "logits": -1.1884685754776, | |
| "logps": -79.12120819091797, | |
| "loss": 0.0633, | |
| "objective": 0.06231885775923729, | |
| "ranking_idealized": 0.6291666626930237, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.5208333134651184, | |
| "regularize": 0.06231885775923729, | |
| "step": 720, | |
| "wo_beta": 15.765007972717285 | |
| }, | |
| { | |
| "dpo_loss": 0.6928841471672058, | |
| "epoch": 2.0547945205479454, | |
| "grad_norm": 25.78908501428665, | |
| "learning_rate": 2.707976732786166e-07, | |
| "logits": -1.1958059072494507, | |
| "logps": -81.6028060913086, | |
| "loss": 0.0578, | |
| "objective": 0.06330116838216782, | |
| "ranking_idealized": 0.5833333134651184, | |
| "ranking_idealized_expo": 0.4958333373069763, | |
| "ranking_simple": 0.5, | |
| "regularize": 0.06330116838216782, | |
| "step": 725, | |
| "wo_beta": 13.992663383483887 | |
| }, | |
| { | |
| "dpo_loss": 0.6905084252357483, | |
| "epoch": 2.0689655172413794, | |
| "grad_norm": 26.140030636203093, | |
| "learning_rate": 2.6348178535517965e-07, | |
| "logits": -1.2607707977294922, | |
| "logps": -79.21609497070312, | |
| "loss": 0.0598, | |
| "objective": 0.05353347584605217, | |
| "ranking_idealized": 0.5791666507720947, | |
| "ranking_idealized_expo": 0.49166667461395264, | |
| "ranking_simple": 0.4958333373069763, | |
| "regularize": 0.05353347584605217, | |
| "step": 730, | |
| "wo_beta": 15.095206260681152 | |
| }, | |
| { | |
| "dpo_loss": 0.6901918053627014, | |
| "epoch": 2.0831365139348135, | |
| "grad_norm": 27.302640565922513, | |
| "learning_rate": 2.5623055915000686e-07, | |
| "logits": -1.1885894536972046, | |
| "logps": -78.86723327636719, | |
| "loss": 0.0579, | |
| "objective": 0.05939151346683502, | |
| "ranking_idealized": 0.5625, | |
| "ranking_idealized_expo": 0.47083333134651184, | |
| "ranking_simple": 0.4833333194255829, | |
| "regularize": 0.05939151346683502, | |
| "step": 735, | |
| "wo_beta": 16.905290603637695 | |
| }, | |
| { | |
| "dpo_loss": 0.6903797388076782, | |
| "epoch": 2.0973075106282475, | |
| "grad_norm": 25.718973789328345, | |
| "learning_rate": 2.490459770759398e-07, | |
| "logits": -1.2478386163711548, | |
| "logps": -79.14292907714844, | |
| "loss": 0.0573, | |
| "objective": 0.05540405213832855, | |
| "ranking_idealized": 0.6166666746139526, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.5333333611488342, | |
| "regularize": 0.05540405213832855, | |
| "step": 740, | |
| "wo_beta": 15.3594331741333 | |
| }, | |
| { | |
| "dpo_loss": 0.6952056288719177, | |
| "epoch": 2.1114785073216815, | |
| "grad_norm": 26.12282917762503, | |
| "learning_rate": 2.419300033259798e-07, | |
| "logits": -1.1640416383743286, | |
| "logps": -79.09960174560547, | |
| "loss": 0.0628, | |
| "objective": 0.0631415918469429, | |
| "ranking_idealized": 0.6333333253860474, | |
| "ranking_idealized_expo": 0.5583333373069763, | |
| "ranking_simple": 0.5541666746139526, | |
| "regularize": 0.0631415918469429, | |
| "step": 745, | |
| "wo_beta": 14.359167098999023 | |
| }, | |
| { | |
| "dpo_loss": 0.6888077259063721, | |
| "epoch": 2.1256495040151155, | |
| "grad_norm": 25.53259897003242, | |
| "learning_rate": 2.3488458333629773e-07, | |
| "logits": -1.2182810306549072, | |
| "logps": -78.26011657714844, | |
| "loss": 0.0546, | |
| "objective": 0.05781084671616554, | |
| "ranking_idealized": 0.574999988079071, | |
| "ranking_idealized_expo": 0.4833333194255829, | |
| "ranking_simple": 0.48750001192092896, | |
| "regularize": 0.05781084671616554, | |
| "step": 750, | |
| "wo_beta": 15.271900177001953 | |
| }, | |
| { | |
| "epoch": 2.1256495040151155, | |
| "eval_dpo_loss": 0.6980140209197998, | |
| "eval_logits": -1.2232871055603027, | |
| "eval_logps": -85.46907806396484, | |
| "eval_loss": 0.14231154322624207, | |
| "eval_objective": 0.14071756601333618, | |
| "eval_ranking_idealized": 0.6024844646453857, | |
| "eval_ranking_idealized_expo": 0.5232919454574585, | |
| "eval_ranking_simple": 0.5258799195289612, | |
| "eval_regularize": 0.14071756601333618, | |
| "eval_runtime": 525.9214, | |
| "eval_samples_per_second": 11.009, | |
| "eval_steps_per_second": 0.918, | |
| "eval_wo_beta": 15.648022651672363, | |
| "step": 750 | |
| }, | |
| { | |
| "dpo_loss": 0.6901395320892334, | |
| "epoch": 2.13982050070855, | |
| "grad_norm": 25.775133405076527, | |
| "learning_rate": 2.2791164325437046e-07, | |
| "logits": -1.2039532661437988, | |
| "logps": -80.76856994628906, | |
| "loss": 0.0536, | |
| "objective": 0.054485421627759933, | |
| "ranking_idealized": 0.5916666388511658, | |
| "ranking_idealized_expo": 0.5166666507720947, | |
| "ranking_simple": 0.5249999761581421, | |
| "regularize": 0.054485421627759933, | |
| "step": 755, | |
| "wo_beta": 16.363035202026367 | |
| }, | |
| { | |
| "dpo_loss": 0.6922858953475952, | |
| "epoch": 2.153991497401984, | |
| "grad_norm": 25.744794188993545, | |
| "learning_rate": 2.21013089412392e-07, | |
| "logits": -1.1505485773086548, | |
| "logps": -77.95565795898438, | |
| "loss": 0.0596, | |
| "objective": 0.056366052478551865, | |
| "ranking_idealized": 0.5958333611488342, | |
| "ranking_idealized_expo": 0.5208333134651184, | |
| "ranking_simple": 0.5, | |
| "regularize": 0.056366052478551865, | |
| "step": 760, | |
| "wo_beta": 14.503907203674316 | |
| }, | |
| { | |
| "dpo_loss": 0.6935012936592102, | |
| "epoch": 2.168162494095418, | |
| "grad_norm": 25.81314805277084, | |
| "learning_rate": 2.1419080780610122e-07, | |
| "logits": -1.195157527923584, | |
| "logps": -79.0260009765625, | |
| "loss": 0.0569, | |
| "objective": 0.05813807621598244, | |
| "ranking_idealized": 0.6208333373069763, | |
| "ranking_idealized_expo": 0.5375000238418579, | |
| "ranking_simple": 0.5375000238418579, | |
| "regularize": 0.05813807621598244, | |
| "step": 765, | |
| "wo_beta": 15.846463203430176 | |
| }, | |
| { | |
| "dpo_loss": 0.6929753422737122, | |
| "epoch": 2.182333490788852, | |
| "grad_norm": 26.17366253681256, | |
| "learning_rate": 2.0744666357916925e-07, | |
| "logits": -1.2156563997268677, | |
| "logps": -79.0594253540039, | |
| "loss": 0.0599, | |
| "objective": 0.06166267395019531, | |
| "ranking_idealized": 0.5916666388511658, | |
| "ranking_idealized_expo": 0.49166667461395264, | |
| "ranking_simple": 0.5083333253860474, | |
| "regularize": 0.06166267395019531, | |
| "step": 770, | |
| "wo_beta": 13.665863037109375 | |
| }, | |
| { | |
| "dpo_loss": 0.6904846429824829, | |
| "epoch": 2.196504487482286, | |
| "grad_norm": 24.80909315966262, | |
| "learning_rate": 2.0078250051328782e-07, | |
| "logits": -1.20059072971344, | |
| "logps": -79.86570739746094, | |
| "loss": 0.0593, | |
| "objective": 0.05707041174173355, | |
| "ranking_idealized": 0.6291666626930237, | |
| "ranking_idealized_expo": 0.5791666507720947, | |
| "ranking_simple": 0.574999988079071, | |
| "regularize": 0.05707041174173355, | |
| "step": 775, | |
| "wo_beta": 16.53993797302246 | |
| }, | |
| { | |
| "dpo_loss": 0.6920241117477417, | |
| "epoch": 2.21067548417572, | |
| "grad_norm": 26.21741329158667, | |
| "learning_rate": 1.942001405240979e-07, | |
| "logits": -1.1453113555908203, | |
| "logps": -79.6847152709961, | |
| "loss": 0.0544, | |
| "objective": 0.05578133091330528, | |
| "ranking_idealized": 0.512499988079071, | |
| "ranking_idealized_expo": 0.44583332538604736, | |
| "ranking_simple": 0.4375, | |
| "regularize": 0.05578133091330528, | |
| "step": 780, | |
| "wo_beta": 15.170312881469727 | |
| }, | |
| { | |
| "dpo_loss": 0.6935942769050598, | |
| "epoch": 2.2248464808691546, | |
| "grad_norm": 28.321911906643972, | |
| "learning_rate": 1.877013831630961e-07, | |
| "logits": -1.1368038654327393, | |
| "logps": -79.92477416992188, | |
| "loss": 0.0563, | |
| "objective": 0.0578266978263855, | |
| "ranking_idealized": 0.5833333134651184, | |
| "ranking_idealized_expo": 0.4791666567325592, | |
| "ranking_simple": 0.4791666567325592, | |
| "regularize": 0.0578266978263855, | |
| "step": 785, | |
| "wo_beta": 14.784603118896484 | |
| }, | |
| { | |
| "dpo_loss": 0.6887209415435791, | |
| "epoch": 2.2390174775625886, | |
| "grad_norm": 25.1538491328267, | |
| "learning_rate": 1.812880051256551e-07, | |
| "logits": -1.1384888887405396, | |
| "logps": -80.59889221191406, | |
| "loss": 0.0504, | |
| "objective": 0.04905276745557785, | |
| "ranking_idealized": 0.637499988079071, | |
| "ranking_idealized_expo": 0.5625, | |
| "ranking_simple": 0.550000011920929, | |
| "regularize": 0.04905276745557785, | |
| "step": 790, | |
| "wo_beta": 14.593072891235352 | |
| }, | |
| { | |
| "dpo_loss": 0.6941591501235962, | |
| "epoch": 2.2531884742560226, | |
| "grad_norm": 25.628975208912717, | |
| "learning_rate": 1.7496175976529337e-07, | |
| "logits": -1.1934906244277954, | |
| "logps": -81.73139953613281, | |
| "loss": 0.053, | |
| "objective": 0.05859142541885376, | |
| "ranking_idealized": 0.6000000238418579, | |
| "ranking_idealized_expo": 0.5041666626930237, | |
| "ranking_simple": 0.5, | |
| "regularize": 0.05859142541885376, | |
| "step": 795, | |
| "wo_beta": 13.79269790649414 | |
| }, | |
| { | |
| "dpo_loss": 0.6919682621955872, | |
| "epoch": 2.2673594709494567, | |
| "grad_norm": 26.293732850411818, | |
| "learning_rate": 1.6872437661432516e-07, | |
| "logits": -1.2084691524505615, | |
| "logps": -80.88973999023438, | |
| "loss": 0.0531, | |
| "objective": 0.05279294773936272, | |
| "ranking_idealized": 0.5916666388511658, | |
| "ranking_idealized_expo": 0.5041666626930237, | |
| "ranking_simple": 0.5166666507720947, | |
| "regularize": 0.05279294773936272, | |
| "step": 800, | |
| "wo_beta": 14.340437889099121 | |
| }, | |
| { | |
| "epoch": 2.2673594709494567, | |
| "eval_dpo_loss": 0.6981291174888611, | |
| "eval_logits": -1.220612645149231, | |
| "eval_logps": -86.13679504394531, | |
| "eval_loss": 0.138593852519989, | |
| "eval_objective": 0.13714565336704254, | |
| "eval_ranking_idealized": 0.6024844646453857, | |
| "eval_ranking_idealized_expo": 0.5232919454574585, | |
| "eval_ranking_simple": 0.5243270993232727, | |
| "eval_regularize": 0.13714565336704254, | |
| "eval_runtime": 503.3749, | |
| "eval_samples_per_second": 11.502, | |
| "eval_steps_per_second": 0.96, | |
| "eval_wo_beta": 15.623366355895996, | |
| "step": 800 | |
| }, | |
| { | |
| "dpo_loss": 0.6895002126693726, | |
| "epoch": 2.2815304676428907, | |
| "grad_norm": 26.85852458075238, | |
| "learning_rate": 1.62577560911024e-07, | |
| "logits": -1.1975409984588623, | |
| "logps": -79.75126647949219, | |
| "loss": 0.0473, | |
| "objective": 0.047933317720890045, | |
| "ranking_idealized": 0.5833333134651184, | |
| "ranking_idealized_expo": 0.5208333134651184, | |
| "ranking_simple": 0.5166666507720947, | |
| "regularize": 0.047933317720890045, | |
| "step": 805, | |
| "wo_beta": 15.455560684204102 | |
| }, | |
| { | |
| "dpo_loss": 0.693041980266571, | |
| "epoch": 2.295701464336325, | |
| "grad_norm": 27.109828632522476, | |
| "learning_rate": 1.565229931334277e-07, | |
| "logits": -1.2860682010650635, | |
| "logps": -79.39039611816406, | |
| "loss": 0.051, | |
| "objective": 0.04613161459565163, | |
| "ranking_idealized": 0.6458333134651184, | |
| "ranking_idealized_expo": 0.5458333492279053, | |
| "ranking_simple": 0.5458333492279053, | |
| "regularize": 0.04613161459565163, | |
| "step": 810, | |
| "wo_beta": 13.837719917297363 | |
| }, | |
| { | |
| "dpo_loss": 0.6961421966552734, | |
| "epoch": 2.309872461029759, | |
| "grad_norm": 26.77232369418631, | |
| "learning_rate": 1.5056232853991208e-07, | |
| "logits": -1.2426903247833252, | |
| "logps": -80.33802032470703, | |
| "loss": 0.0483, | |
| "objective": 0.04774492606520653, | |
| "ranking_idealized": 0.5791666507720947, | |
| "ranking_idealized_expo": 0.47083333134651184, | |
| "ranking_simple": 0.4625000059604645, | |
| "regularize": 0.04774492606520653, | |
| "step": 815, | |
| "wo_beta": 15.377904891967773 | |
| }, | |
| { | |
| "dpo_loss": 0.6943568587303162, | |
| "epoch": 2.324043457723193, | |
| "grad_norm": 25.84415791966093, | |
| "learning_rate": 1.4469719671666043e-07, | |
| "logits": -1.1784952878952026, | |
| "logps": -79.52135467529297, | |
| "loss": 0.0497, | |
| "objective": 0.0464615561068058, | |
| "ranking_idealized": 0.5874999761581421, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.5041666626930237, | |
| "regularize": 0.0464615561068058, | |
| "step": 820, | |
| "wo_beta": 14.641592979431152 | |
| }, | |
| { | |
| "dpo_loss": 0.6941722631454468, | |
| "epoch": 2.3382144544166272, | |
| "grad_norm": 26.057445300358456, | |
| "learning_rate": 1.389292011321498e-07, | |
| "logits": -1.1956678628921509, | |
| "logps": -78.97592163085938, | |
| "loss": 0.0489, | |
| "objective": 0.04843177646398544, | |
| "ranking_idealized": 0.625, | |
| "ranking_idealized_expo": 0.550000011920929, | |
| "ranking_simple": 0.5375000238418579, | |
| "regularize": 0.04843177646398544, | |
| "step": 825, | |
| "wo_beta": 15.882107734680176 | |
| }, | |
| { | |
| "dpo_loss": 0.6919335722923279, | |
| "epoch": 2.3523854511100613, | |
| "grad_norm": 25.587425832586177, | |
| "learning_rate": 1.3325991869878012e-07, | |
| "logits": -1.1966559886932373, | |
| "logps": -81.00519561767578, | |
| "loss": 0.0487, | |
| "objective": 0.05618049576878548, | |
| "ranking_idealized": 0.6416666507720947, | |
| "ranking_idealized_expo": 0.5416666865348816, | |
| "ranking_simple": 0.5458333492279053, | |
| "regularize": 0.05618049576878548, | |
| "step": 830, | |
| "wo_beta": 15.746501922607422 | |
| }, | |
| { | |
| "dpo_loss": 0.6917215585708618, | |
| "epoch": 2.3665564478034957, | |
| "grad_norm": 25.756644403885232, | |
| "learning_rate": 1.2769089934176126e-07, | |
| "logits": -1.168601632118225, | |
| "logps": -80.84972381591797, | |
| "loss": 0.0488, | |
| "objective": 0.052498627454042435, | |
| "ranking_idealized": 0.6166666746139526, | |
| "ranking_idealized_expo": 0.5083333253860474, | |
| "ranking_simple": 0.512499988079071, | |
| "regularize": 0.052498627454042435, | |
| "step": 835, | |
| "wo_beta": 14.608040809631348 | |
| }, | |
| { | |
| "dpo_loss": 0.6898554563522339, | |
| "epoch": 2.3807274444969297, | |
| "grad_norm": 25.072094771225707, | |
| "learning_rate": 1.222236655753791e-07, | |
| "logits": -1.1249865293502808, | |
| "logps": -80.45842742919922, | |
| "loss": 0.0434, | |
| "objective": 0.04277409613132477, | |
| "ranking_idealized": 0.5249999761581421, | |
| "ranking_idealized_expo": 0.4749999940395355, | |
| "ranking_simple": 0.4625000059604645, | |
| "regularize": 0.04277409613132477, | |
| "step": 840, | |
| "wo_beta": 16.011308670043945 | |
| }, | |
| { | |
| "dpo_loss": 0.6897058486938477, | |
| "epoch": 2.3948984411903638, | |
| "grad_norm": 31.2138593781791, | |
| "learning_rate": 1.1685971208675538e-07, | |
| "logits": -1.1826022863388062, | |
| "logps": -81.36385345458984, | |
| "loss": 0.0438, | |
| "objective": 0.04376084357500076, | |
| "ranking_idealized": 0.5958333611488342, | |
| "ranking_idealized_expo": 0.5416666865348816, | |
| "ranking_simple": 0.5333333611488342, | |
| "regularize": 0.04376084357500076, | |
| "step": 845, | |
| "wo_beta": 15.694497108459473 | |
| }, | |
| { | |
| "dpo_loss": 0.689830482006073, | |
| "epoch": 2.409069437883798, | |
| "grad_norm": 26.424193566129606, | |
| "learning_rate": 1.1160050532721527e-07, | |
| "logits": -1.2078933715820312, | |
| "logps": -79.71755981445312, | |
| "loss": 0.0444, | |
| "objective": 0.04779530316591263, | |
| "ranking_idealized": 0.5874999761581421, | |
| "ranking_idealized_expo": 0.49166667461395264, | |
| "ranking_simple": 0.49166667461395264, | |
| "regularize": 0.04779530316591263, | |
| "step": 850, | |
| "wo_beta": 15.619561195373535 | |
| }, | |
| { | |
| "epoch": 2.409069437883798, | |
| "eval_dpo_loss": 0.6980399489402771, | |
| "eval_logits": -1.2270959615707397, | |
| "eval_logps": -86.03622436523438, | |
| "eval_loss": 0.13948112726211548, | |
| "eval_objective": 0.1381867229938507, | |
| "eval_ranking_idealized": 0.6024844646453857, | |
| "eval_ranking_idealized_expo": 0.5232919454574585, | |
| "eval_ranking_simple": 0.523809552192688, | |
| "eval_regularize": 0.1381867229938507, | |
| "eval_runtime": 508.2715, | |
| "eval_samples_per_second": 11.392, | |
| "eval_steps_per_second": 0.95, | |
| "eval_wo_beta": 15.647224426269531, | |
| "step": 850 | |
| }, | |
| { | |
| "dpo_loss": 0.690664529800415, | |
| "epoch": 2.423240434577232, | |
| "grad_norm": 26.086004792829357, | |
| "learning_rate": 1.0644748311137375e-07, | |
| "logits": -1.2208842039108276, | |
| "logps": -79.23947143554688, | |
| "loss": 0.0431, | |
| "objective": 0.044093988835811615, | |
| "ranking_idealized": 0.5916666388511658, | |
| "ranking_idealized_expo": 0.5291666388511658, | |
| "ranking_simple": 0.5249999761581421, | |
| "regularize": 0.044093988835811615, | |
| "step": 855, | |
| "wo_beta": 14.724575996398926 | |
| }, | |
| { | |
| "dpo_loss": 0.6878847479820251, | |
| "epoch": 2.4374114312706663, | |
| "grad_norm": 24.819758120044014, | |
| "learning_rate": 1.0140205422405212e-07, | |
| "logits": -1.172597050666809, | |
| "logps": -80.47863006591797, | |
| "loss": 0.0425, | |
| "objective": 0.044025711715221405, | |
| "ranking_idealized": 0.5874999761581421, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.5333333611488342, | |
| "regularize": 0.044025711715221405, | |
| "step": 860, | |
| "wo_beta": 15.323599815368652 | |
| }, | |
| { | |
| "dpo_loss": 0.6902381777763367, | |
| "epoch": 2.4515824279641003, | |
| "grad_norm": 27.313034441936136, | |
| "learning_rate": 9.646559803512993e-08, | |
| "logits": -1.2031606435775757, | |
| "logps": -79.59320831298828, | |
| "loss": 0.0444, | |
| "objective": 0.04272008314728737, | |
| "ranking_idealized": 0.5791666507720947, | |
| "ranking_idealized_expo": 0.4958333373069763, | |
| "ranking_simple": 0.5, | |
| "regularize": 0.04272008314728737, | |
| "step": 865, | |
| "wo_beta": 15.875487327575684 | |
| }, | |
| { | |
| "dpo_loss": 0.6910372376441956, | |
| "epoch": 2.4657534246575343, | |
| "grad_norm": 25.76666127477957, | |
| "learning_rate": 9.163946412243895e-08, | |
| "logits": -1.2454520463943481, | |
| "logps": -80.33094024658203, | |
| "loss": 0.0442, | |
| "objective": 0.04635915905237198, | |
| "ranking_idealized": 0.6000000238418579, | |
| "ranking_idealized_expo": 0.5208333134651184, | |
| "ranking_simple": 0.5208333134651184, | |
| "regularize": 0.04635915905237198, | |
| "step": 870, | |
| "wo_beta": 14.93254566192627 | |
| }, | |
| { | |
| "dpo_loss": 0.6905195713043213, | |
| "epoch": 2.4799244213509684, | |
| "grad_norm": 25.65493367025704, | |
| "learning_rate": 8.692497190280224e-08, | |
| "logits": -1.193867802619934, | |
| "logps": -79.73404693603516, | |
| "loss": 0.044, | |
| "objective": 0.04675581306219101, | |
| "ranking_idealized": 0.6166666746139526, | |
| "ranking_idealized_expo": 0.5708333253860474, | |
| "ranking_simple": 0.5625, | |
| "regularize": 0.04675581306219101, | |
| "step": 875, | |
| "wo_beta": 16.489763259887695 | |
| }, | |
| { | |
| "dpo_loss": 0.6905779242515564, | |
| "epoch": 2.4940954180444024, | |
| "grad_norm": 26.621663140091542, | |
| "learning_rate": 8.232341027131883e-08, | |
| "logits": -1.1066038608551025, | |
| "logps": -79.80467224121094, | |
| "loss": 0.0446, | |
| "objective": 0.046583421528339386, | |
| "ranking_idealized": 0.5625, | |
| "ranking_idealized_expo": 0.5, | |
| "ranking_simple": 0.5, | |
| "regularize": 0.046583421528339386, | |
| "step": 880, | |
| "wo_beta": 17.46852684020996 | |
| }, | |
| { | |
| "dpo_loss": 0.6917292475700378, | |
| "epoch": 2.5082664147378364, | |
| "grad_norm": 24.02209120686893, | |
| "learning_rate": 7.783603724899257e-08, | |
| "logits": -1.25592041015625, | |
| "logps": -79.1759262084961, | |
| "loss": 0.0422, | |
| "objective": 0.04294423386454582, | |
| "ranking_idealized": 0.5916666388511658, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.5249999761581421, | |
| "regularize": 0.04294423386454582, | |
| "step": 885, | |
| "wo_beta": 16.415306091308594 | |
| }, | |
| { | |
| "dpo_loss": 0.6880825161933899, | |
| "epoch": 2.5224374114312704, | |
| "grad_norm": 26.181840029139675, | |
| "learning_rate": 7.346407963880136e-08, | |
| "logits": -1.1791417598724365, | |
| "logps": -78.21730041503906, | |
| "loss": 0.0424, | |
| "objective": 0.03773224726319313, | |
| "ranking_idealized": 0.5874999761581421, | |
| "ranking_idealized_expo": 0.5, | |
| "ranking_simple": 0.5, | |
| "regularize": 0.03773224726319313, | |
| "step": 890, | |
| "wo_beta": 13.494309425354004 | |
| }, | |
| { | |
| "dpo_loss": 0.692958414554596, | |
| "epoch": 2.536608408124705, | |
| "grad_norm": 27.615133075738825, | |
| "learning_rate": 6.92087326903022e-08, | |
| "logits": -1.175589680671692, | |
| "logps": -80.6869888305664, | |
| "loss": 0.0444, | |
| "objective": 0.0476791188120842, | |
| "ranking_idealized": 0.5458333492279053, | |
| "ranking_idealized_expo": 0.49166667461395264, | |
| "ranking_simple": 0.48750001192092896, | |
| "regularize": 0.0476791188120842, | |
| "step": 895, | |
| "wo_beta": 16.41474151611328 | |
| }, | |
| { | |
| "dpo_loss": 0.6935379505157471, | |
| "epoch": 2.550779404818139, | |
| "grad_norm": 25.263999580012257, | |
| "learning_rate": 6.507115977286143e-08, | |
| "logits": -1.1382538080215454, | |
| "logps": -79.20881652832031, | |
| "loss": 0.0438, | |
| "objective": 0.044265471398830414, | |
| "ranking_idealized": 0.6166666746139526, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.5208333134651184, | |
| "regularize": 0.044265471398830414, | |
| "step": 900, | |
| "wo_beta": 15.096195220947266 | |
| }, | |
| { | |
| "epoch": 2.550779404818139, | |
| "eval_dpo_loss": 0.6975382566452026, | |
| "eval_logits": -1.2295913696289062, | |
| "eval_logps": -85.88396453857422, | |
| "eval_loss": 0.13868437707424164, | |
| "eval_objective": 0.13740767538547516, | |
| "eval_ranking_idealized": 0.6024844646453857, | |
| "eval_ranking_idealized_expo": 0.5232919454574585, | |
| "eval_ranking_simple": 0.523809552192688, | |
| "eval_regularize": 0.13740767538547516, | |
| "eval_runtime": 525.8368, | |
| "eval_samples_per_second": 11.011, | |
| "eval_steps_per_second": 0.919, | |
| "eval_wo_beta": 15.634546279907227, | |
| "step": 900 | |
| }, | |
| { | |
| "dpo_loss": 0.6917089819908142, | |
| "epoch": 2.564950401511573, | |
| "grad_norm": 25.44195334625603, | |
| "learning_rate": 6.105249205760127e-08, | |
| "logits": -1.2037063837051392, | |
| "logps": -79.04875183105469, | |
| "loss": 0.0411, | |
| "objective": 0.03601410239934921, | |
| "ranking_idealized": 0.5708333253860474, | |
| "ranking_idealized_expo": 0.5, | |
| "ranking_simple": 0.5041666626930237, | |
| "regularize": 0.03601410239934921, | |
| "step": 905, | |
| "wo_beta": 14.861380577087402 | |
| }, | |
| { | |
| "dpo_loss": 0.6932801008224487, | |
| "epoch": 2.579121398205007, | |
| "grad_norm": 26.495925146665332, | |
| "learning_rate": 5.7153828208148846e-08, | |
| "logits": -1.1827551126480103, | |
| "logps": -81.922607421875, | |
| "loss": 0.0424, | |
| "objective": 0.04883956164121628, | |
| "ranking_idealized": 0.612500011920929, | |
| "ranking_idealized_expo": 0.4958333373069763, | |
| "ranking_simple": 0.5, | |
| "regularize": 0.04883956164121628, | |
| "step": 910, | |
| "wo_beta": 15.852696418762207 | |
| }, | |
| { | |
| "dpo_loss": 0.6898232102394104, | |
| "epoch": 2.593292394898441, | |
| "grad_norm": 25.88822340642525, | |
| "learning_rate": 5.337623408027292e-08, | |
| "logits": -1.2935634851455688, | |
| "logps": -80.87789916992188, | |
| "loss": 0.0403, | |
| "objective": 0.040093984454870224, | |
| "ranking_idealized": 0.6291666626930237, | |
| "ranking_idealized_expo": 0.512499988079071, | |
| "ranking_simple": 0.512499988079071, | |
| "regularize": 0.040093984454870224, | |
| "step": 915, | |
| "wo_beta": 14.905534744262695 | |
| }, | |
| { | |
| "dpo_loss": 0.6920287013053894, | |
| "epoch": 2.6074633915918755, | |
| "grad_norm": 25.364010577767672, | |
| "learning_rate": 4.972074243048896e-08, | |
| "logits": -1.1468993425369263, | |
| "logps": -79.89569854736328, | |
| "loss": 0.0396, | |
| "objective": 0.03967604413628578, | |
| "ranking_idealized": 0.6041666865348816, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.5249999761581421, | |
| "regularize": 0.03967604413628578, | |
| "step": 920, | |
| "wo_beta": 15.246692657470703 | |
| }, | |
| { | |
| "dpo_loss": 0.6928901076316833, | |
| "epoch": 2.6216343882853095, | |
| "grad_norm": 27.967184575096596, | |
| "learning_rate": 4.6188352633713956e-08, | |
| "logits": -1.1743673086166382, | |
| "logps": -80.17101287841797, | |
| "loss": 0.0417, | |
| "objective": 0.04370425269007683, | |
| "ranking_idealized": 0.574999988079071, | |
| "ranking_idealized_expo": 0.4749999940395355, | |
| "ranking_simple": 0.47083333134651184, | |
| "regularize": 0.04370425269007683, | |
| "step": 925, | |
| "wo_beta": 16.336292266845703 | |
| }, | |
| { | |
| "dpo_loss": 0.688522458076477, | |
| "epoch": 2.6358053849787435, | |
| "grad_norm": 26.578359144982873, | |
| "learning_rate": 4.2780030410047796e-08, | |
| "logits": -1.1617387533187866, | |
| "logps": -79.97476196289062, | |
| "loss": 0.0365, | |
| "objective": 0.03662450239062309, | |
| "ranking_idealized": 0.5416666865348816, | |
| "ranking_idealized_expo": 0.44583332538604736, | |
| "ranking_simple": 0.44583332538604736, | |
| "regularize": 0.03662450239062309, | |
| "step": 930, | |
| "wo_beta": 16.801166534423828 | |
| }, | |
| { | |
| "dpo_loss": 0.6928302645683289, | |
| "epoch": 2.6499763816721775, | |
| "grad_norm": 26.6756558913633, | |
| "learning_rate": 3.949670756075446e-08, | |
| "logits": -1.1548212766647339, | |
| "logps": -78.78431701660156, | |
| "loss": 0.0364, | |
| "objective": 0.0356716513633728, | |
| "ranking_idealized": 0.6499999761581421, | |
| "ranking_idealized_expo": 0.574999988079071, | |
| "ranking_simple": 0.574999988079071, | |
| "regularize": 0.0356716513633728, | |
| "step": 935, | |
| "wo_beta": 15.733369827270508 | |
| }, | |
| { | |
| "dpo_loss": 0.6884638071060181, | |
| "epoch": 2.6641473783656116, | |
| "grad_norm": 26.11837122854028, | |
| "learning_rate": 3.63392817135173e-08, | |
| "logits": -1.213140845298767, | |
| "logps": -81.39899444580078, | |
| "loss": 0.0357, | |
| "objective": 0.03838236257433891, | |
| "ranking_idealized": 0.6000000238418579, | |
| "ranking_idealized_expo": 0.5333333611488342, | |
| "ranking_simple": 0.5333333611488342, | |
| "regularize": 0.03838236257433891, | |
| "step": 940, | |
| "wo_beta": 16.71453094482422 | |
| }, | |
| { | |
| "dpo_loss": 0.6904810070991516, | |
| "epoch": 2.678318375059046, | |
| "grad_norm": 26.48243005501328, | |
| "learning_rate": 3.330861607703611e-08, | |
| "logits": -1.2477443218231201, | |
| "logps": -80.07948303222656, | |
| "loss": 0.0369, | |
| "objective": 0.03517834097146988, | |
| "ranking_idealized": 0.5666666626930237, | |
| "ranking_idealized_expo": 0.48750001192092896, | |
| "ranking_simple": 0.48750001192092896, | |
| "regularize": 0.03517834097146988, | |
| "step": 945, | |
| "wo_beta": 15.665254592895508 | |
| }, | |
| { | |
| "dpo_loss": 0.6894643902778625, | |
| "epoch": 2.69248937175248, | |
| "grad_norm": 26.269248260275482, | |
| "learning_rate": 3.040553920503502e-08, | |
| "logits": -1.1376032829284668, | |
| "logps": -80.89375305175781, | |
| "loss": 0.0384, | |
| "objective": 0.03873926401138306, | |
| "ranking_idealized": 0.637499988079071, | |
| "ranking_idealized_expo": 0.5375000238418579, | |
| "ranking_simple": 0.5416666865348816, | |
| "regularize": 0.03873926401138306, | |
| "step": 950, | |
| "wo_beta": 14.65186882019043 | |
| }, | |
| { | |
| "epoch": 2.69248937175248, | |
| "eval_dpo_loss": 0.6974536180496216, | |
| "eval_logits": -1.2285144329071045, | |
| "eval_logps": -85.95899963378906, | |
| "eval_loss": 0.13796193897724152, | |
| "eval_objective": 0.13680347800254822, | |
| "eval_ranking_idealized": 0.6024844646453857, | |
| "eval_ranking_idealized_expo": 0.5232919454574585, | |
| "eval_ranking_simple": 0.523809552192688, | |
| "eval_regularize": 0.13680347800254822, | |
| "eval_runtime": 502.396, | |
| "eval_samples_per_second": 11.525, | |
| "eval_steps_per_second": 0.961, | |
| "eval_wo_beta": 15.642508506774902, | |
| "step": 950 | |
| }, | |
| { | |
| "dpo_loss": 0.6890572905540466, | |
| "epoch": 2.706660368445914, | |
| "grad_norm": 24.74397275822761, | |
| "learning_rate": 2.7630844769743756e-08, | |
| "logits": -1.2225416898727417, | |
| "logps": -79.87822723388672, | |
| "loss": 0.0403, | |
| "objective": 0.04285174608230591, | |
| "ranking_idealized": 0.5874999761581421, | |
| "ranking_idealized_expo": 0.4749999940395355, | |
| "ranking_simple": 0.4749999940395355, | |
| "regularize": 0.04285174608230591, | |
| "step": 955, | |
| "wo_beta": 13.80820369720459 | |
| }, | |
| { | |
| "dpo_loss": 0.6908868551254272, | |
| "epoch": 2.720831365139348, | |
| "grad_norm": 25.907101929875015, | |
| "learning_rate": 2.4985291344915673e-08, | |
| "logits": -1.1964094638824463, | |
| "logps": -79.958740234375, | |
| "loss": 0.0384, | |
| "objective": 0.03498096391558647, | |
| "ranking_idealized": 0.5791666507720947, | |
| "ranking_idealized_expo": 0.49166667461395264, | |
| "ranking_simple": 0.5041666626930237, | |
| "regularize": 0.03498096391558647, | |
| "step": 960, | |
| "wo_beta": 16.096843719482422 | |
| }, | |
| { | |
| "dpo_loss": 0.6898122429847717, | |
| "epoch": 2.735002361832782, | |
| "grad_norm": 26.015895295989438, | |
| "learning_rate": 2.2469602198441573e-08, | |
| "logits": -1.2220391035079956, | |
| "logps": -80.10702514648438, | |
| "loss": 0.0368, | |
| "objective": 0.03775167092680931, | |
| "ranking_idealized": 0.6416666507720947, | |
| "ranking_idealized_expo": 0.5666666626930237, | |
| "ranking_simple": 0.5666666626930237, | |
| "regularize": 0.03775167092680931, | |
| "step": 965, | |
| "wo_beta": 14.61376953125 | |
| }, | |
| { | |
| "dpo_loss": 0.6917709112167358, | |
| "epoch": 2.7491733585262166, | |
| "grad_norm": 24.33103792831753, | |
| "learning_rate": 2.008446509461498e-08, | |
| "logits": -1.2293510437011719, | |
| "logps": -81.0619888305664, | |
| "loss": 0.0341, | |
| "objective": 0.03296136483550072, | |
| "ranking_idealized": 0.6208333373069763, | |
| "ranking_idealized_expo": 0.5333333611488342, | |
| "ranking_simple": 0.5208333134651184, | |
| "regularize": 0.03296136483550072, | |
| "step": 970, | |
| "wo_beta": 14.957200050354004 | |
| }, | |
| { | |
| "dpo_loss": 0.6909447908401489, | |
| "epoch": 2.7633443552196506, | |
| "grad_norm": 24.892680282575437, | |
| "learning_rate": 1.7830532106104746e-08, | |
| "logits": -1.1391520500183105, | |
| "logps": -79.50247955322266, | |
| "loss": 0.0358, | |
| "objective": 0.03571467101573944, | |
| "ranking_idealized": 0.6333333253860474, | |
| "ranking_idealized_expo": 0.5333333611488342, | |
| "ranking_simple": 0.5416666865348816, | |
| "regularize": 0.03571467101573944, | |
| "step": 975, | |
| "wo_beta": 15.747049331665039 | |
| }, | |
| { | |
| "dpo_loss": 0.6906387209892273, | |
| "epoch": 2.7775153519130846, | |
| "grad_norm": 25.891776024282194, | |
| "learning_rate": 1.570841943568446e-08, | |
| "logits": -1.2599250078201294, | |
| "logps": -78.82478332519531, | |
| "loss": 0.0365, | |
| "objective": 0.03682435303926468, | |
| "ranking_idealized": 0.5916666388511658, | |
| "ranking_idealized_expo": 0.5, | |
| "ranking_simple": 0.49166667461395264, | |
| "regularize": 0.03682435303926468, | |
| "step": 980, | |
| "wo_beta": 14.397340774536133 | |
| }, | |
| { | |
| "dpo_loss": 0.6933729648590088, | |
| "epoch": 2.7916863486065187, | |
| "grad_norm": 24.71596998222205, | |
| "learning_rate": 1.3718707247769134e-08, | |
| "logits": -1.1248877048492432, | |
| "logps": -77.72516632080078, | |
| "loss": 0.038, | |
| "objective": 0.03822270780801773, | |
| "ranking_idealized": 0.5791666507720947, | |
| "ranking_idealized_expo": 0.48750001192092896, | |
| "ranking_simple": 0.48750001192092896, | |
| "regularize": 0.03822270780801773, | |
| "step": 985, | |
| "wo_beta": 14.327728271484375 | |
| }, | |
| { | |
| "dpo_loss": 0.691889762878418, | |
| "epoch": 2.8058573452999527, | |
| "grad_norm": 26.185929406261582, | |
| "learning_rate": 1.1861939509803686e-08, | |
| "logits": -1.1771855354309082, | |
| "logps": -81.14643859863281, | |
| "loss": 0.0369, | |
| "objective": 0.036898624151945114, | |
| "ranking_idealized": 0.5666666626930237, | |
| "ranking_idealized_expo": 0.4749999940395355, | |
| "ranking_simple": 0.47083333134651184, | |
| "regularize": 0.036898624151945114, | |
| "step": 990, | |
| "wo_beta": 15.375889778137207 | |
| }, | |
| { | |
| "dpo_loss": 0.6891864538192749, | |
| "epoch": 2.820028341993387, | |
| "grad_norm": 24.803225677825235, | |
| "learning_rate": 1.0138623843548078e-08, | |
| "logits": -1.2396986484527588, | |
| "logps": -79.1412353515625, | |
| "loss": 0.0365, | |
| "objective": 0.04024568572640419, | |
| "ranking_idealized": 0.6208333373069763, | |
| "ranking_idealized_expo": 0.5458333492279053, | |
| "ranking_simple": 0.5458333492279053, | |
| "regularize": 0.04024568572640419, | |
| "step": 995, | |
| "wo_beta": 16.440141677856445 | |
| }, | |
| { | |
| "dpo_loss": 0.6907335519790649, | |
| "epoch": 2.8341993386868207, | |
| "grad_norm": 24.80804716491088, | |
| "learning_rate": 8.54923138629815e-09, | |
| "logits": -1.1814649105072021, | |
| "logps": -78.3318862915039, | |
| "loss": 0.0375, | |
| "objective": 0.03398551046848297, | |
| "ranking_idealized": 0.6166666746139526, | |
| "ranking_idealized_expo": 0.5083333253860474, | |
| "ranking_simple": 0.5208333134651184, | |
| "regularize": 0.03398551046848297, | |
| "step": 1000, | |
| "wo_beta": 14.515811920166016 | |
| }, | |
| { | |
| "epoch": 2.8341993386868207, | |
| "eval_dpo_loss": 0.6973779201507568, | |
| "eval_logits": -1.2304595708847046, | |
| "eval_logps": -85.99760437011719, | |
| "eval_loss": 0.1379886120557785, | |
| "eval_objective": 0.1368565410375595, | |
| "eval_ranking_idealized": 0.6024844646453857, | |
| "eval_ranking_idealized_expo": 0.5232919454574585, | |
| "eval_ranking_simple": 0.5243270993232727, | |
| "eval_regularize": 0.1368565410375595, | |
| "eval_runtime": 504.9482, | |
| "eval_samples_per_second": 11.467, | |
| "eval_steps_per_second": 0.957, | |
| "eval_wo_beta": 15.63548755645752, | |
| "step": 1000 | |
| }, | |
| { | |
| "dpo_loss": 0.6911761164665222, | |
| "epoch": 2.848370335380255, | |
| "grad_norm": 27.32667601221845, | |
| "learning_rate": 7.09419666208183e-09, | |
| "logits": -1.1803662776947021, | |
| "logps": -78.7650375366211, | |
| "loss": 0.036, | |
| "objective": 0.03725501522421837, | |
| "ranking_idealized": 0.6000000238418579, | |
| "ranking_idealized_expo": 0.512499988079071, | |
| "ranking_simple": 0.5208333134651184, | |
| "regularize": 0.03725501522421837, | |
| "step": 1005, | |
| "wo_beta": 15.2937593460083 | |
| }, | |
| { | |
| "dpo_loss": 0.6888595223426819, | |
| "epoch": 2.862541332073689, | |
| "grad_norm": 26.14400831689978, | |
| "learning_rate": 5.773917462864264e-09, | |
| "logits": -1.2407745122909546, | |
| "logps": -79.07453918457031, | |
| "loss": 0.0359, | |
| "objective": 0.03689141198992729, | |
| "ranking_idealized": 0.612500011920929, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.5291666388511658, | |
| "regularize": 0.03689141198992729, | |
| "step": 1010, | |
| "wo_beta": 15.180621147155762 | |
| }, | |
| { | |
| "dpo_loss": 0.6912004947662354, | |
| "epoch": 2.8767123287671232, | |
| "grad_norm": 24.9602315307722, | |
| "learning_rate": 4.588754739795586e-09, | |
| "logits": -1.1721571683883667, | |
| "logps": -78.31599426269531, | |
| "loss": 0.0354, | |
| "objective": 0.03823023661971092, | |
| "ranking_idealized": 0.6541666388511658, | |
| "ranking_idealized_expo": 0.550000011920929, | |
| "ranking_simple": 0.550000011920929, | |
| "regularize": 0.03823023661971092, | |
| "step": 1015, | |
| "wo_beta": 14.313817977905273 | |
| }, | |
| { | |
| "dpo_loss": 0.6896302700042725, | |
| "epoch": 2.8908833254605573, | |
| "grad_norm": 24.85258883289883, | |
| "learning_rate": 3.53903250453047e-09, | |
| "logits": -1.1410295963287354, | |
| "logps": -80.05741882324219, | |
| "loss": 0.0343, | |
| "objective": 0.03470051661133766, | |
| "ranking_idealized": 0.5833333134651184, | |
| "ranking_idealized_expo": 0.5166666507720947, | |
| "ranking_simple": 0.5166666507720947, | |
| "regularize": 0.03470051661133766, | |
| "step": 1020, | |
| "wo_beta": 17.722339630126953 | |
| }, | |
| { | |
| "dpo_loss": 0.6912213563919067, | |
| "epoch": 2.9050543221539913, | |
| "grad_norm": 25.437671735836517, | |
| "learning_rate": 2.6250377406467627e-09, | |
| "logits": -1.2291027307510376, | |
| "logps": -80.00859832763672, | |
| "loss": 0.0379, | |
| "objective": 0.037315838038921356, | |
| "ranking_idealized": 0.6208333373069763, | |
| "ranking_idealized_expo": 0.5083333253860474, | |
| "ranking_simple": 0.5041666626930237, | |
| "regularize": 0.037315838038921356, | |
| "step": 1025, | |
| "wo_beta": 14.656061172485352 | |
| }, | |
| { | |
| "dpo_loss": 0.6911433935165405, | |
| "epoch": 2.9192253188474258, | |
| "grad_norm": 24.681518212372314, | |
| "learning_rate": 1.8470203251865768e-09, | |
| "logits": -1.2523103952407837, | |
| "logps": -80.20305633544922, | |
| "loss": 0.035, | |
| "objective": 0.03597547858953476, | |
| "ranking_idealized": 0.5416666865348816, | |
| "ranking_idealized_expo": 0.4791666567325592, | |
| "ranking_simple": 0.4791666567325592, | |
| "regularize": 0.03597547858953476, | |
| "step": 1030, | |
| "wo_beta": 16.243247985839844 | |
| }, | |
| { | |
| "dpo_loss": 0.6904833316802979, | |
| "epoch": 2.9333963155408598, | |
| "grad_norm": 26.808499612926756, | |
| "learning_rate": 1.2051929603428823e-09, | |
| "logits": -1.2276477813720703, | |
| "logps": -80.6124496459961, | |
| "loss": 0.0344, | |
| "objective": 0.03077917918562889, | |
| "ranking_idealized": 0.625, | |
| "ranking_idealized_expo": 0.5, | |
| "ranking_simple": 0.5041666626930237, | |
| "regularize": 0.03077917918562889, | |
| "step": 1035, | |
| "wo_beta": 14.297567367553711 | |
| }, | |
| { | |
| "dpo_loss": 0.6902684569358826, | |
| "epoch": 2.947567312234294, | |
| "grad_norm": 24.71043561481991, | |
| "learning_rate": 6.997311153086882e-10, | |
| "logits": -1.227773904800415, | |
| "logps": -80.38175201416016, | |
| "loss": 0.0364, | |
| "objective": 0.036134228110313416, | |
| "ranking_idealized": 0.5708333253860474, | |
| "ranking_idealized_expo": 0.4833333194255829, | |
| "ranking_simple": 0.48750001192092896, | |
| "regularize": 0.036134228110313416, | |
| "step": 1040, | |
| "wo_beta": 16.110403060913086 | |
| }, | |
| { | |
| "dpo_loss": 0.6894943118095398, | |
| "epoch": 2.961738308927728, | |
| "grad_norm": 26.305013618654215, | |
| "learning_rate": 3.3077297830541585e-10, | |
| "logits": -1.1821495294570923, | |
| "logps": -81.93363189697266, | |
| "loss": 0.0371, | |
| "objective": 0.04041092470288277, | |
| "ranking_idealized": 0.6208333373069763, | |
| "ranking_idealized_expo": 0.5666666626930237, | |
| "ranking_simple": 0.5541666746139526, | |
| "regularize": 0.04041092470288277, | |
| "step": 1045, | |
| "wo_beta": 17.30424690246582 | |
| }, | |
| { | |
| "dpo_loss": 0.6926708221435547, | |
| "epoch": 2.975909305621162, | |
| "grad_norm": 27.660126015515125, | |
| "learning_rate": 9.841941880361914e-11, | |
| "logits": -1.2283350229263306, | |
| "logps": -78.42631530761719, | |
| "loss": 0.0397, | |
| "objective": 0.03637199103832245, | |
| "ranking_idealized": 0.6541666388511658, | |
| "ranking_idealized_expo": 0.5541666746139526, | |
| "ranking_simple": 0.550000011920929, | |
| "regularize": 0.03637199103832245, | |
| "step": 1050, | |
| "wo_beta": 14.132574081420898 | |
| }, | |
| { | |
| "epoch": 2.975909305621162, | |
| "eval_dpo_loss": 0.697369396686554, | |
| "eval_logits": -1.230570673942566, | |
| "eval_logps": -85.98023223876953, | |
| "eval_loss": 0.13814175128936768, | |
| "eval_objective": 0.13700547814369202, | |
| "eval_ranking_idealized": 0.6024844646453857, | |
| "eval_ranking_idealized_expo": 0.5232919454574585, | |
| "eval_ranking_simple": 0.5243270993232727, | |
| "eval_regularize": 0.13700547814369202, | |
| "eval_runtime": 530.5394, | |
| "eval_samples_per_second": 10.913, | |
| "eval_steps_per_second": 0.91, | |
| "eval_wo_beta": 15.63470458984375, | |
| "step": 1050 | |
| }, | |
| { | |
| "dpo_loss": 0.689972996711731, | |
| "epoch": 2.9900803023145963, | |
| "grad_norm": 25.71242634224602, | |
| "learning_rate": 2.7339599464326622e-12, | |
| "logits": -1.2016465663909912, | |
| "logps": -79.08844757080078, | |
| "loss": 0.0389, | |
| "objective": 0.03705615550279617, | |
| "ranking_idealized": 0.6083333492279053, | |
| "ranking_idealized_expo": 0.550000011920929, | |
| "ranking_simple": 0.5541666746139526, | |
| "regularize": 0.03705615550279617, | |
| "step": 1055, | |
| "wo_beta": 14.549761772155762 | |
| }, | |
| { | |
| "epoch": 2.992914501653283, | |
| "step": 1056, | |
| "total_flos": 0.0, | |
| "train_loss": 0.08480868444806247, | |
| "train_runtime": 47353.1169, | |
| "train_samples_per_second": 3.218, | |
| "train_steps_per_second": 0.022 | |
| } | |
| ], | |
| "logging_steps": 5, | |
| "max_steps": 1056, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": false, | |
| "should_training_stop": false | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 4, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |