| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 4.988190836088805, | |
| "eval_steps": 50, | |
| "global_step": 880, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "dpo_loss": 0.6931471824645996, | |
| "epoch": 0.005668398677373642, | |
| "grad_norm": 13.413600039235007, | |
| "learning_rate": 5.681818181818181e-09, | |
| "logits": -1.3147305250167847, | |
| "logps": -88.0877456665039, | |
| "loss": 0.4113, | |
| "objective": 0.41588976979255676, | |
| "ranking_idealized": 0.6875, | |
| "ranking_idealized_expo": 0.5208333134651184, | |
| "ranking_simple": 0.5208333134651184, | |
| "regularize": 0.41588976979255676, | |
| "step": 1 | |
| }, | |
| { | |
| "dpo_loss": 0.6931512951850891, | |
| "epoch": 0.02834199338686821, | |
| "grad_norm": 13.318320815280419, | |
| "learning_rate": 2.8409090909090908e-08, | |
| "logits": -1.3678321838378906, | |
| "logps": -84.44427490234375, | |
| "loss": 0.4131, | |
| "objective": 0.3755509555339813, | |
| "ranking_idealized": 0.6510416865348816, | |
| "ranking_idealized_expo": 0.5572916865348816, | |
| "ranking_simple": 0.546875, | |
| "regularize": 0.3755509555339813, | |
| "step": 5 | |
| }, | |
| { | |
| "dpo_loss": 0.6927531361579895, | |
| "epoch": 0.05668398677373642, | |
| "grad_norm": 13.050623089340824, | |
| "learning_rate": 5.6818181818181815e-08, | |
| "logits": -1.4463988542556763, | |
| "logps": -83.39988708496094, | |
| "loss": 0.4176, | |
| "objective": 0.4423220753669739, | |
| "ranking_idealized": 0.675000011920929, | |
| "ranking_idealized_expo": 0.512499988079071, | |
| "ranking_simple": 0.512499988079071, | |
| "regularize": 0.4423220753669739, | |
| "step": 10 | |
| }, | |
| { | |
| "dpo_loss": 0.6927918195724487, | |
| "epoch": 0.08502598016060463, | |
| "grad_norm": 12.549385306441062, | |
| "learning_rate": 8.522727272727271e-08, | |
| "logits": -1.4107797145843506, | |
| "logps": -83.50421905517578, | |
| "loss": 0.4254, | |
| "objective": 0.41179904341697693, | |
| "ranking_idealized": 0.6541666388511658, | |
| "ranking_idealized_expo": 0.5041666626930237, | |
| "ranking_simple": 0.5083333253860474, | |
| "regularize": 0.41179904341697693, | |
| "step": 15 | |
| }, | |
| { | |
| "dpo_loss": 0.6924694776535034, | |
| "epoch": 0.11336797354747284, | |
| "grad_norm": 13.269620119946596, | |
| "learning_rate": 1.1363636363636363e-07, | |
| "logits": -1.4003115892410278, | |
| "logps": -84.06736755371094, | |
| "loss": 0.4149, | |
| "objective": 0.40317121148109436, | |
| "ranking_idealized": 0.6791666746139526, | |
| "ranking_idealized_expo": 0.5375000238418579, | |
| "ranking_simple": 0.5375000238418579, | |
| "regularize": 0.40317121148109436, | |
| "step": 20 | |
| }, | |
| { | |
| "dpo_loss": 0.6906281113624573, | |
| "epoch": 0.14170996693434104, | |
| "grad_norm": 12.65234373247132, | |
| "learning_rate": 1.4204545454545455e-07, | |
| "logits": -1.4490704536437988, | |
| "logps": -83.72380065917969, | |
| "loss": 0.412, | |
| "objective": 0.4304184317588806, | |
| "ranking_idealized": 0.675000011920929, | |
| "ranking_idealized_expo": 0.4958333373069763, | |
| "ranking_simple": 0.5, | |
| "regularize": 0.4304184317588806, | |
| "step": 25 | |
| }, | |
| { | |
| "dpo_loss": 0.6906370520591736, | |
| "epoch": 0.17005196032120926, | |
| "grad_norm": 13.419812147505471, | |
| "learning_rate": 1.7045454545454543e-07, | |
| "logits": -1.4248003959655762, | |
| "logps": -84.09757232666016, | |
| "loss": 0.4126, | |
| "objective": 0.41593801975250244, | |
| "ranking_idealized": 0.6875, | |
| "ranking_idealized_expo": 0.5416666865348816, | |
| "ranking_simple": 0.5333333611488342, | |
| "regularize": 0.41593801975250244, | |
| "step": 30 | |
| }, | |
| { | |
| "dpo_loss": 0.6881809234619141, | |
| "epoch": 0.19839395370807747, | |
| "grad_norm": 13.431894879328123, | |
| "learning_rate": 1.9886363636363636e-07, | |
| "logits": -1.398374319076538, | |
| "logps": -82.60546112060547, | |
| "loss": 0.4095, | |
| "objective": 0.3929609954357147, | |
| "ranking_idealized": 0.6583333611488342, | |
| "ranking_idealized_expo": 0.5041666626930237, | |
| "ranking_simple": 0.5083333253860474, | |
| "regularize": 0.3929609954357147, | |
| "step": 35 | |
| }, | |
| { | |
| "dpo_loss": 0.6868489384651184, | |
| "epoch": 0.22673594709494568, | |
| "grad_norm": 15.0250838416837, | |
| "learning_rate": 2.2727272727272726e-07, | |
| "logits": -1.3904410600662231, | |
| "logps": -82.84651947021484, | |
| "loss": 0.42, | |
| "objective": 0.43919187784194946, | |
| "ranking_idealized": 0.6625000238418579, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.5166666507720947, | |
| "regularize": 0.43919187784194946, | |
| "step": 40 | |
| }, | |
| { | |
| "dpo_loss": 0.6844364404678345, | |
| "epoch": 0.25507794048181387, | |
| "grad_norm": 13.128806663839857, | |
| "learning_rate": 2.5568181818181816e-07, | |
| "logits": -1.5230154991149902, | |
| "logps": -84.21646118164062, | |
| "loss": 0.4194, | |
| "objective": 0.4717731177806854, | |
| "ranking_idealized": 0.7208333611488342, | |
| "ranking_idealized_expo": 0.5375000238418579, | |
| "ranking_simple": 0.5458333492279053, | |
| "regularize": 0.4717731177806854, | |
| "step": 45 | |
| }, | |
| { | |
| "dpo_loss": 0.6831071376800537, | |
| "epoch": 0.2834199338686821, | |
| "grad_norm": 12.39410793472882, | |
| "learning_rate": 2.840909090909091e-07, | |
| "logits": -1.431780219078064, | |
| "logps": -82.2941665649414, | |
| "loss": 0.4122, | |
| "objective": 0.3948862850666046, | |
| "ranking_idealized": 0.637499988079071, | |
| "ranking_idealized_expo": 0.42500001192092896, | |
| "ranking_simple": 0.44999998807907104, | |
| "regularize": 0.3948862850666046, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.2834199338686821, | |
| "eval_dpo_loss": 0.6914567947387695, | |
| "eval_logits": -1.4614633321762085, | |
| "eval_logps": -90.56139373779297, | |
| "eval_loss": 0.4102250635623932, | |
| "eval_objective": 0.40930914878845215, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.5123966932296753, | |
| "eval_regularize": 0.40930914878845215, | |
| "eval_runtime": 260.1383, | |
| "eval_samples_per_second": 22.257, | |
| "eval_steps_per_second": 0.93, | |
| "step": 50 | |
| }, | |
| { | |
| "dpo_loss": 0.6807647943496704, | |
| "epoch": 0.3117619272555503, | |
| "grad_norm": 13.979478083508853, | |
| "learning_rate": 3.1249999999999997e-07, | |
| "logits": -1.4781759977340698, | |
| "logps": -84.0101089477539, | |
| "loss": 0.4052, | |
| "objective": 0.4063163101673126, | |
| "ranking_idealized": 0.7124999761581421, | |
| "ranking_idealized_expo": 0.5416666865348816, | |
| "ranking_simple": 0.5458333492279053, | |
| "regularize": 0.4063163101673126, | |
| "step": 55 | |
| }, | |
| { | |
| "dpo_loss": 0.6824926137924194, | |
| "epoch": 0.3401039206424185, | |
| "grad_norm": 13.484676530515722, | |
| "learning_rate": 3.4090909090909085e-07, | |
| "logits": -1.4679373502731323, | |
| "logps": -83.09486389160156, | |
| "loss": 0.3992, | |
| "objective": 0.387731671333313, | |
| "ranking_idealized": 0.7041666507720947, | |
| "ranking_idealized_expo": 0.5166666507720947, | |
| "ranking_simple": 0.5333333611488342, | |
| "regularize": 0.387731671333313, | |
| "step": 60 | |
| }, | |
| { | |
| "dpo_loss": 0.6788213849067688, | |
| "epoch": 0.3684459140292867, | |
| "grad_norm": 13.535493104004898, | |
| "learning_rate": 3.693181818181818e-07, | |
| "logits": -1.4250341653823853, | |
| "logps": -83.52283477783203, | |
| "loss": 0.3842, | |
| "objective": 0.3719988465309143, | |
| "ranking_idealized": 0.6291666626930237, | |
| "ranking_idealized_expo": 0.5083333253860474, | |
| "ranking_simple": 0.5083333253860474, | |
| "regularize": 0.3719988465309143, | |
| "step": 65 | |
| }, | |
| { | |
| "dpo_loss": 0.6763210296630859, | |
| "epoch": 0.39678790741615494, | |
| "grad_norm": 13.25897931133664, | |
| "learning_rate": 3.977272727272727e-07, | |
| "logits": -1.5077797174453735, | |
| "logps": -85.39080047607422, | |
| "loss": 0.3855, | |
| "objective": 0.42043933272361755, | |
| "ranking_idealized": 0.6916666626930237, | |
| "ranking_idealized_expo": 0.5166666507720947, | |
| "ranking_simple": 0.5291666388511658, | |
| "regularize": 0.42043933272361755, | |
| "step": 70 | |
| }, | |
| { | |
| "dpo_loss": 0.678033709526062, | |
| "epoch": 0.42512990080302315, | |
| "grad_norm": 14.035157652400327, | |
| "learning_rate": 4.2613636363636364e-07, | |
| "logits": -1.5349814891815186, | |
| "logps": -86.0143051147461, | |
| "loss": 0.3945, | |
| "objective": 0.41438591480255127, | |
| "ranking_idealized": 0.6791666746139526, | |
| "ranking_idealized_expo": 0.5458333492279053, | |
| "ranking_simple": 0.5666666626930237, | |
| "regularize": 0.41438591480255127, | |
| "step": 75 | |
| }, | |
| { | |
| "dpo_loss": 0.6748775243759155, | |
| "epoch": 0.45347189418989137, | |
| "grad_norm": 13.539091864104346, | |
| "learning_rate": 4.545454545454545e-07, | |
| "logits": -1.5832253694534302, | |
| "logps": -85.59701538085938, | |
| "loss": 0.3789, | |
| "objective": 0.37422579526901245, | |
| "ranking_idealized": 0.6666666865348816, | |
| "ranking_idealized_expo": 0.5166666507720947, | |
| "ranking_simple": 0.5333333611488342, | |
| "regularize": 0.37422579526901245, | |
| "step": 80 | |
| }, | |
| { | |
| "dpo_loss": 0.6750870943069458, | |
| "epoch": 0.4818138875767596, | |
| "grad_norm": 12.78905385712093, | |
| "learning_rate": 4.829545454545455e-07, | |
| "logits": -1.5551499128341675, | |
| "logps": -84.24475860595703, | |
| "loss": 0.374, | |
| "objective": 0.39821094274520874, | |
| "ranking_idealized": 0.6541666388511658, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.5375000238418579, | |
| "regularize": 0.39821094274520874, | |
| "step": 85 | |
| }, | |
| { | |
| "dpo_loss": 0.6703960299491882, | |
| "epoch": 0.5101558809636277, | |
| "grad_norm": 14.26040681218726, | |
| "learning_rate": 4.999921328558332e-07, | |
| "logits": -1.37662935256958, | |
| "logps": -86.21568298339844, | |
| "loss": 0.3761, | |
| "objective": 0.3837045729160309, | |
| "ranking_idealized": 0.6291666626930237, | |
| "ranking_idealized_expo": 0.4791666567325592, | |
| "ranking_simple": 0.4833333194255829, | |
| "regularize": 0.3837045729160309, | |
| "step": 90 | |
| }, | |
| { | |
| "dpo_loss": 0.658724844455719, | |
| "epoch": 0.538497874350496, | |
| "grad_norm": 14.003866252787525, | |
| "learning_rate": 4.999036331701828e-07, | |
| "logits": -1.4695987701416016, | |
| "logps": -85.49458312988281, | |
| "loss": 0.3642, | |
| "objective": 0.39033612608909607, | |
| "ranking_idealized": 0.6833333373069763, | |
| "ranking_idealized_expo": 0.5458333492279053, | |
| "ranking_simple": 0.5666666626930237, | |
| "regularize": 0.39033612608909607, | |
| "step": 95 | |
| }, | |
| { | |
| "dpo_loss": 0.6546652317047119, | |
| "epoch": 0.5668398677373642, | |
| "grad_norm": 13.876424116810778, | |
| "learning_rate": 4.99716834795752e-07, | |
| "logits": -1.5616024732589722, | |
| "logps": -86.23612213134766, | |
| "loss": 0.374, | |
| "objective": 0.4149954915046692, | |
| "ranking_idealized": 0.6708333492279053, | |
| "ranking_idealized_expo": 0.5416666865348816, | |
| "ranking_simple": 0.5458333492279053, | |
| "regularize": 0.4149954915046692, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.5668398677373642, | |
| "eval_dpo_loss": 0.6882808208465576, | |
| "eval_logits": -1.5521211624145508, | |
| "eval_logps": -92.03614807128906, | |
| "eval_loss": 0.4072900712490082, | |
| "eval_objective": 0.40819329023361206, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.5144628286361694, | |
| "eval_regularize": 0.40819329023361206, | |
| "eval_runtime": 258.9254, | |
| "eval_samples_per_second": 22.362, | |
| "eval_steps_per_second": 0.935, | |
| "step": 100 | |
| }, | |
| { | |
| "dpo_loss": 0.6509627103805542, | |
| "epoch": 0.5951818611242324, | |
| "grad_norm": 15.031011715031442, | |
| "learning_rate": 4.994318112090048e-07, | |
| "logits": -1.4410721063613892, | |
| "logps": -85.85182189941406, | |
| "loss": 0.3711, | |
| "objective": 0.3764886260032654, | |
| "ranking_idealized": 0.7083333134651184, | |
| "ranking_idealized_expo": 0.5375000238418579, | |
| "ranking_simple": 0.574999988079071, | |
| "regularize": 0.3764886260032654, | |
| "step": 105 | |
| }, | |
| { | |
| "dpo_loss": 0.6576470136642456, | |
| "epoch": 0.6235238545111006, | |
| "grad_norm": 16.068373019347053, | |
| "learning_rate": 4.990486745229364e-07, | |
| "logits": -1.6439845561981201, | |
| "logps": -84.1036376953125, | |
| "loss": 0.3694, | |
| "objective": 0.39763620495796204, | |
| "ranking_idealized": 0.6541666388511658, | |
| "ranking_idealized_expo": 0.4583333432674408, | |
| "ranking_simple": 0.48750001192092896, | |
| "regularize": 0.39763620495796204, | |
| "step": 110 | |
| }, | |
| { | |
| "dpo_loss": 0.6543448567390442, | |
| "epoch": 0.6518658478979689, | |
| "grad_norm": 15.729444594038945, | |
| "learning_rate": 4.985675754429743e-07, | |
| "logits": -1.6000815629959106, | |
| "logps": -83.94436645507812, | |
| "loss": 0.3477, | |
| "objective": 0.3455929458141327, | |
| "ranking_idealized": 0.6583333611488342, | |
| "ranking_idealized_expo": 0.4833333194255829, | |
| "ranking_simple": 0.5, | |
| "regularize": 0.3455929458141327, | |
| "step": 115 | |
| }, | |
| { | |
| "dpo_loss": 0.6514815092086792, | |
| "epoch": 0.680207841284837, | |
| "grad_norm": 14.396427732147952, | |
| "learning_rate": 4.979887032076988e-07, | |
| "logits": -1.5459378957748413, | |
| "logps": -85.23513793945312, | |
| "loss": 0.3534, | |
| "objective": 0.3168259561061859, | |
| "ranking_idealized": 0.6583333611488342, | |
| "ranking_idealized_expo": 0.5333333611488342, | |
| "ranking_simple": 0.5458333492279053, | |
| "regularize": 0.3168259561061859, | |
| "step": 120 | |
| }, | |
| { | |
| "dpo_loss": 0.6492612957954407, | |
| "epoch": 0.7085498346717053, | |
| "grad_norm": 16.706445645247783, | |
| "learning_rate": 4.973122855144065e-07, | |
| "logits": -1.5174397230148315, | |
| "logps": -86.0051040649414, | |
| "loss": 0.3448, | |
| "objective": 0.3529473543167114, | |
| "ranking_idealized": 0.699999988079071, | |
| "ranking_idealized_expo": 0.550000011920929, | |
| "ranking_simple": 0.5708333253860474, | |
| "regularize": 0.3529473543167114, | |
| "step": 125 | |
| }, | |
| { | |
| "dpo_loss": 0.65309077501297, | |
| "epoch": 0.7368918280585735, | |
| "grad_norm": 15.417556754357976, | |
| "learning_rate": 4.965385884295466e-07, | |
| "logits": -1.664696455001831, | |
| "logps": -85.23889923095703, | |
| "loss": 0.3464, | |
| "objective": 0.33712950348854065, | |
| "ranking_idealized": 0.637499988079071, | |
| "ranking_idealized_expo": 0.4541666805744171, | |
| "ranking_simple": 0.4958333373069763, | |
| "regularize": 0.33712950348854065, | |
| "step": 130 | |
| }, | |
| { | |
| "dpo_loss": 0.6549941301345825, | |
| "epoch": 0.7652338214454416, | |
| "grad_norm": 13.59480500578719, | |
| "learning_rate": 4.956679162840645e-07, | |
| "logits": -1.626897931098938, | |
| "logps": -86.90068817138672, | |
| "loss": 0.3309, | |
| "objective": 0.34302666783332825, | |
| "ranking_idealized": 0.6416666507720947, | |
| "ranking_idealized_expo": 0.5333333611488342, | |
| "ranking_simple": 0.5458333492279053, | |
| "regularize": 0.34302666783332825, | |
| "step": 135 | |
| }, | |
| { | |
| "dpo_loss": 0.6500818729400635, | |
| "epoch": 0.7935758148323099, | |
| "grad_norm": 14.79485288903614, | |
| "learning_rate": 4.947006115536947e-07, | |
| "logits": -1.523794412612915, | |
| "logps": -86.5340576171875, | |
| "loss": 0.3244, | |
| "objective": 0.3356337249279022, | |
| "ranking_idealized": 0.6291666626930237, | |
| "ranking_idealized_expo": 0.5083333253860474, | |
| "ranking_simple": 0.5166666507720947, | |
| "regularize": 0.3356337249279022, | |
| "step": 140 | |
| }, | |
| { | |
| "dpo_loss": 0.6436840295791626, | |
| "epoch": 0.821917808219178, | |
| "grad_norm": 14.29078834943314, | |
| "learning_rate": 4.936370547242482e-07, | |
| "logits": -1.5991618633270264, | |
| "logps": -86.87805938720703, | |
| "loss": 0.3315, | |
| "objective": 0.35039833188056946, | |
| "ranking_idealized": 0.6458333134651184, | |
| "ranking_idealized_expo": 0.4833333194255829, | |
| "ranking_simple": 0.5, | |
| "regularize": 0.35039833188056946, | |
| "step": 145 | |
| }, | |
| { | |
| "dpo_loss": 0.6453251242637634, | |
| "epoch": 0.8502598016060463, | |
| "grad_norm": 14.662823673975787, | |
| "learning_rate": 4.924776641419512e-07, | |
| "logits": -1.5607432126998901, | |
| "logps": -84.04727935791016, | |
| "loss": 0.3231, | |
| "objective": 0.32859519124031067, | |
| "ranking_idealized": 0.6916666626930237, | |
| "ranking_idealized_expo": 0.5416666865348816, | |
| "ranking_simple": 0.5833333134651184, | |
| "regularize": 0.32859519124031067, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.8502598016060463, | |
| "eval_dpo_loss": 0.6880838871002197, | |
| "eval_logits": -1.6073634624481201, | |
| "eval_logps": -92.8072509765625, | |
| "eval_loss": 0.40292537212371826, | |
| "eval_objective": 0.4087039530277252, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.5185950398445129, | |
| "eval_regularize": 0.4087039530277252, | |
| "eval_runtime": 258.55, | |
| "eval_samples_per_second": 22.394, | |
| "eval_steps_per_second": 0.936, | |
| "step": 150 | |
| }, | |
| { | |
| "dpo_loss": 0.6386777758598328, | |
| "epoch": 0.8786017949929145, | |
| "grad_norm": 14.231064502358223, | |
| "learning_rate": 4.912228958488892e-07, | |
| "logits": -1.5854390859603882, | |
| "logps": -84.10832214355469, | |
| "loss": 0.3257, | |
| "objective": 0.3301841616630554, | |
| "ranking_idealized": 0.6791666746139526, | |
| "ranking_idealized_expo": 0.512499988079071, | |
| "ranking_simple": 0.5916666388511658, | |
| "regularize": 0.3301841616630554, | |
| "step": 155 | |
| }, | |
| { | |
| "dpo_loss": 0.635101854801178, | |
| "epoch": 0.9069437883797827, | |
| "grad_norm": 16.2268120086952, | |
| "learning_rate": 4.898732434036243e-07, | |
| "logits": -1.4904930591583252, | |
| "logps": -86.09799194335938, | |
| "loss": 0.3107, | |
| "objective": 0.32787373661994934, | |
| "ranking_idealized": 0.6625000238418579, | |
| "ranking_idealized_expo": 0.4958333373069763, | |
| "ranking_simple": 0.5458333492279053, | |
| "regularize": 0.32787373661994934, | |
| "step": 160 | |
| }, | |
| { | |
| "dpo_loss": 0.632634162902832, | |
| "epoch": 0.9352857817666509, | |
| "grad_norm": 16.041101199008867, | |
| "learning_rate": 4.884292376870567e-07, | |
| "logits": -1.5242409706115723, | |
| "logps": -86.48987579345703, | |
| "loss": 0.3212, | |
| "objective": 0.3137226700782776, | |
| "ranking_idealized": 0.6541666388511658, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.574999988079071, | |
| "regularize": 0.3137226700782776, | |
| "step": 165 | |
| }, | |
| { | |
| "dpo_loss": 0.6288425922393799, | |
| "epoch": 0.9636277751535192, | |
| "grad_norm": 16.89173365453321, | |
| "learning_rate": 4.868914466936037e-07, | |
| "logits": -1.5360677242279053, | |
| "logps": -86.72618103027344, | |
| "loss": 0.3151, | |
| "objective": 0.30297866463661194, | |
| "ranking_idealized": 0.6291666626930237, | |
| "ranking_idealized_expo": 0.512499988079071, | |
| "ranking_simple": 0.5708333253860474, | |
| "regularize": 0.30297866463661194, | |
| "step": 170 | |
| }, | |
| { | |
| "dpo_loss": 0.6335378289222717, | |
| "epoch": 0.9919697685403873, | |
| "grad_norm": 15.347989877166441, | |
| "learning_rate": 4.852604753077817e-07, | |
| "logits": -1.4790997505187988, | |
| "logps": -87.8569107055664, | |
| "loss": 0.3103, | |
| "objective": 0.29884618520736694, | |
| "ranking_idealized": 0.6833333373069763, | |
| "ranking_idealized_expo": 0.574999988079071, | |
| "ranking_simple": 0.6166666746139526, | |
| "regularize": 0.29884618520736694, | |
| "step": 175 | |
| }, | |
| { | |
| "dpo_loss": 0.6220327615737915, | |
| "epoch": 1.0203117619272555, | |
| "grad_norm": 15.022798279596007, | |
| "learning_rate": 4.835369650662767e-07, | |
| "logits": -1.6809762716293335, | |
| "logps": -87.00578308105469, | |
| "loss": 0.2902, | |
| "objective": 0.3023075461387634, | |
| "ranking_idealized": 0.6833333373069763, | |
| "ranking_idealized_expo": 0.5416666865348816, | |
| "ranking_simple": 0.5874999761581421, | |
| "regularize": 0.3023075461387634, | |
| "step": 180 | |
| }, | |
| { | |
| "dpo_loss": 0.6156979203224182, | |
| "epoch": 1.0486537553141237, | |
| "grad_norm": 17.673807880039096, | |
| "learning_rate": 4.817215939055985e-07, | |
| "logits": -1.54806387424469, | |
| "logps": -86.16964721679688, | |
| "loss": 0.2856, | |
| "objective": 0.30200377106666565, | |
| "ranking_idealized": 0.6541666388511658, | |
| "ranking_idealized_expo": 0.4833333194255829, | |
| "ranking_simple": 0.5416666865348816, | |
| "regularize": 0.30200377106666565, | |
| "step": 185 | |
| }, | |
| { | |
| "dpo_loss": 0.6105552315711975, | |
| "epoch": 1.076995748700992, | |
| "grad_norm": 16.47130075175902, | |
| "learning_rate": 4.798150758954164e-07, | |
| "logits": -1.6065795421600342, | |
| "logps": -88.57856750488281, | |
| "loss": 0.2661, | |
| "objective": 0.23887412250041962, | |
| "ranking_idealized": 0.6916666626930237, | |
| "ranking_idealized_expo": 0.5541666746139526, | |
| "ranking_simple": 0.6041666865348816, | |
| "regularize": 0.23887412250041962, | |
| "step": 190 | |
| }, | |
| { | |
| "dpo_loss": 0.6094806790351868, | |
| "epoch": 1.10533774208786, | |
| "grad_norm": 15.979183042956787, | |
| "learning_rate": 4.778181609576831e-07, | |
| "logits": -1.58108651638031, | |
| "logps": -86.33049011230469, | |
| "loss": 0.2734, | |
| "objective": 0.23615716397762299, | |
| "ranking_idealized": 0.7041666507720947, | |
| "ranking_idealized_expo": 0.5708333253860474, | |
| "ranking_simple": 0.6333333253860474, | |
| "regularize": 0.23615716397762299, | |
| "step": 195 | |
| }, | |
| { | |
| "dpo_loss": 0.6135362982749939, | |
| "epoch": 1.1336797354747283, | |
| "grad_norm": 15.241345178579065, | |
| "learning_rate": 4.757316345716553e-07, | |
| "logits": -1.6668376922607422, | |
| "logps": -85.64834594726562, | |
| "loss": 0.267, | |
| "objective": 0.2661064565181732, | |
| "ranking_idealized": 0.6499999761581421, | |
| "ranking_idealized_expo": 0.5083333253860474, | |
| "ranking_simple": 0.5541666746139526, | |
| "regularize": 0.2661064565181732, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 1.1336797354747283, | |
| "eval_dpo_loss": 0.6866354942321777, | |
| "eval_logits": -1.6423935890197754, | |
| "eval_logps": -94.7991943359375, | |
| "eval_loss": 0.4068562686443329, | |
| "eval_objective": 0.41099515557289124, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.5185950398445129, | |
| "eval_regularize": 0.41099515557289124, | |
| "eval_runtime": 258.6603, | |
| "eval_samples_per_second": 22.385, | |
| "eval_steps_per_second": 0.936, | |
| "step": 200 | |
| }, | |
| { | |
| "dpo_loss": 0.6104326844215393, | |
| "epoch": 1.1620217288615966, | |
| "grad_norm": 16.09148449696529, | |
| "learning_rate": 4.735563174649278e-07, | |
| "logits": -1.6373622417449951, | |
| "logps": -88.22838592529297, | |
| "loss": 0.2748, | |
| "objective": 0.2591724991798401, | |
| "ranking_idealized": 0.6458333134651184, | |
| "ranking_idealized_expo": 0.48750001192092896, | |
| "ranking_simple": 0.5541666746139526, | |
| "regularize": 0.2591724991798401, | |
| "step": 205 | |
| }, | |
| { | |
| "dpo_loss": 0.6027090549468994, | |
| "epoch": 1.1903637222484649, | |
| "grad_norm": 16.78316844909737, | |
| "learning_rate": 4.7129306529060407e-07, | |
| "logits": -1.604967474937439, | |
| "logps": -88.00846099853516, | |
| "loss": 0.2647, | |
| "objective": 0.28820380568504333, | |
| "ranking_idealized": 0.6291666626930237, | |
| "ranking_idealized_expo": 0.5, | |
| "ranking_simple": 0.5541666746139526, | |
| "regularize": 0.28820380568504333, | |
| "step": 210 | |
| }, | |
| { | |
| "dpo_loss": 0.6120165586471558, | |
| "epoch": 1.2187057156353331, | |
| "grad_norm": 17.38824297135803, | |
| "learning_rate": 4.6894276829072786e-07, | |
| "logits": -1.577252745628357, | |
| "logps": -88.0232925415039, | |
| "loss": 0.2457, | |
| "objective": 0.2474772185087204, | |
| "ranking_idealized": 0.6666666865348816, | |
| "ranking_idealized_expo": 0.48750001192092896, | |
| "ranking_simple": 0.5625, | |
| "regularize": 0.2474772185087204, | |
| "step": 215 | |
| }, | |
| { | |
| "dpo_loss": 0.6058060526847839, | |
| "epoch": 1.2470477090222012, | |
| "grad_norm": 16.33918535416867, | |
| "learning_rate": 4.6650635094610966e-07, | |
| "logits": -1.5445390939712524, | |
| "logps": -87.7970199584961, | |
| "loss": 0.25, | |
| "objective": 0.23831520974636078, | |
| "ranking_idealized": 0.7208333611488342, | |
| "ranking_idealized_expo": 0.6041666865348816, | |
| "ranking_simple": 0.637499988079071, | |
| "regularize": 0.23831520974636078, | |
| "step": 220 | |
| }, | |
| { | |
| "dpo_loss": 0.6021844744682312, | |
| "epoch": 1.2753897024090695, | |
| "grad_norm": 14.989005292751132, | |
| "learning_rate": 4.639847716126854e-07, | |
| "logits": -1.6192957162857056, | |
| "logps": -89.04407501220703, | |
| "loss": 0.2607, | |
| "objective": 0.26420968770980835, | |
| "ranking_idealized": 0.6791666746139526, | |
| "ranking_idealized_expo": 0.5291666388511658, | |
| "ranking_simple": 0.6041666865348816, | |
| "regularize": 0.26420965790748596, | |
| "step": 225 | |
| }, | |
| { | |
| "dpo_loss": 0.6108235120773315, | |
| "epoch": 1.3037316957959377, | |
| "grad_norm": 16.245321246774985, | |
| "learning_rate": 4.6137902214455106e-07, | |
| "logits": -1.5698676109313965, | |
| "logps": -89.06554412841797, | |
| "loss": 0.2454, | |
| "objective": 0.24457047879695892, | |
| "ranking_idealized": 0.6541666388511658, | |
| "ranking_idealized_expo": 0.5083333253860474, | |
| "ranking_simple": 0.5541666746139526, | |
| "regularize": 0.24457047879695892, | |
| "step": 230 | |
| }, | |
| { | |
| "dpo_loss": 0.59207683801651, | |
| "epoch": 1.3320736891828058, | |
| "grad_norm": 16.42864016636988, | |
| "learning_rate": 4.5869012750382004e-07, | |
| "logits": -1.6616859436035156, | |
| "logps": -87.82197570800781, | |
| "loss": 0.2583, | |
| "objective": 0.26300859451293945, | |
| "ranking_idealized": 0.6875, | |
| "ranking_idealized_expo": 0.48750001192092896, | |
| "ranking_simple": 0.6041666865348816, | |
| "regularize": 0.26300859451293945, | |
| "step": 235 | |
| }, | |
| { | |
| "dpo_loss": 0.5991641283035278, | |
| "epoch": 1.360415682569674, | |
| "grad_norm": 17.146089761318706, | |
| "learning_rate": 4.5591914535745817e-07, | |
| "logits": -1.5948702096939087, | |
| "logps": -89.31143188476562, | |
| "loss": 0.2442, | |
| "objective": 0.25130581855773926, | |
| "ranking_idealized": 0.637499988079071, | |
| "ranking_idealized_expo": 0.4791666567325592, | |
| "ranking_simple": 0.5666666626930237, | |
| "regularize": 0.25130581855773926, | |
| "step": 240 | |
| }, | |
| { | |
| "dpo_loss": 0.6016849279403687, | |
| "epoch": 1.3887576759565423, | |
| "grad_norm": 14.997203138603757, | |
| "learning_rate": 4.5306716566125433e-07, | |
| "logits": -1.6367671489715576, | |
| "logps": -88.17431640625, | |
| "loss": 0.2399, | |
| "objective": 0.23935823142528534, | |
| "ranking_idealized": 0.6833333373069763, | |
| "ranking_idealized_expo": 0.5208333134651184, | |
| "ranking_simple": 0.5833333134651184, | |
| "regularize": 0.23935823142528534, | |
| "step": 245 | |
| }, | |
| { | |
| "dpo_loss": 0.5918813943862915, | |
| "epoch": 1.4170996693434104, | |
| "grad_norm": 15.668454928081044, | |
| "learning_rate": 4.501353102310901e-07, | |
| "logits": -1.5877238512039185, | |
| "logps": -87.66322326660156, | |
| "loss": 0.2432, | |
| "objective": 0.2531537711620331, | |
| "ranking_idealized": 0.6625000238418579, | |
| "ranking_idealized_expo": 0.4749999940395355, | |
| "ranking_simple": 0.5625, | |
| "regularize": 0.2531537711620331, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 1.4170996693434104, | |
| "eval_dpo_loss": 0.6876620650291443, | |
| "eval_logits": -1.6720653772354126, | |
| "eval_logps": -96.13894653320312, | |
| "eval_loss": 0.4107522666454315, | |
| "eval_objective": 0.4137335419654846, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.51962810754776, | |
| "eval_regularize": 0.4137335419654846, | |
| "eval_runtime": 259.3309, | |
| "eval_samples_per_second": 22.327, | |
| "eval_steps_per_second": 0.933, | |
| "step": 250 | |
| }, | |
| { | |
| "dpo_loss": 0.5952737927436829, | |
| "epoch": 1.4454416627302786, | |
| "grad_norm": 16.40280338029817, | |
| "learning_rate": 4.471247323016777e-07, | |
| "logits": -1.5863794088363647, | |
| "logps": -89.24433898925781, | |
| "loss": 0.2442, | |
| "objective": 0.2290959656238556, | |
| "ranking_idealized": 0.675000011920929, | |
| "ranking_idealized_expo": 0.5458333492279053, | |
| "ranking_simple": 0.6291666626930237, | |
| "regularize": 0.2290959656238556, | |
| "step": 255 | |
| }, | |
| { | |
| "dpo_loss": 0.5954132080078125, | |
| "epoch": 1.473783656117147, | |
| "grad_norm": 16.11674277744465, | |
| "learning_rate": 4.440366160729392e-07, | |
| "logits": -1.6588572263717651, | |
| "logps": -89.44280242919922, | |
| "loss": 0.2443, | |
| "objective": 0.2354036122560501, | |
| "ranking_idealized": 0.6499999761581421, | |
| "ranking_idealized_expo": 0.48750001192092896, | |
| "ranking_simple": 0.5625, | |
| "regularize": 0.2354036122560501, | |
| "step": 260 | |
| }, | |
| { | |
| "dpo_loss": 0.5868396759033203, | |
| "epoch": 1.5021256495040152, | |
| "grad_norm": 16.753008834337265, | |
| "learning_rate": 4.4087217624420585e-07, | |
| "logits": -1.6106855869293213, | |
| "logps": -88.14371490478516, | |
| "loss": 0.239, | |
| "objective": 0.24956756830215454, | |
| "ranking_idealized": 0.6333333253860474, | |
| "ranking_idealized_expo": 0.5041666626930237, | |
| "ranking_simple": 0.5958333611488342, | |
| "regularize": 0.24956756830215454, | |
| "step": 265 | |
| }, | |
| { | |
| "dpo_loss": 0.5915893316268921, | |
| "epoch": 1.5304676428908834, | |
| "grad_norm": 17.579129679111187, | |
| "learning_rate": 4.3763265753642055e-07, | |
| "logits": -1.6173158884048462, | |
| "logps": -90.8720703125, | |
| "loss": 0.2421, | |
| "objective": 0.24301743507385254, | |
| "ranking_idealized": 0.6791666746139526, | |
| "ranking_idealized_expo": 0.512499988079071, | |
| "ranking_simple": 0.5958333611488342, | |
| "regularize": 0.24301742017269135, | |
| "step": 270 | |
| }, | |
| { | |
| "dpo_loss": 0.5958731770515442, | |
| "epoch": 1.5588096362777515, | |
| "grad_norm": 16.101798479127662, | |
| "learning_rate": 4.34319334202531e-07, | |
| "logits": -1.6187034845352173, | |
| "logps": -90.12999725341797, | |
| "loss": 0.2448, | |
| "objective": 0.22899790108203888, | |
| "ranking_idealized": 0.6916666626930237, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.6208333373069763, | |
| "regularize": 0.22899790108203888, | |
| "step": 275 | |
| }, | |
| { | |
| "dpo_loss": 0.5842003226280212, | |
| "epoch": 1.5871516296646198, | |
| "grad_norm": 15.709789047148108, | |
| "learning_rate": 4.309335095262675e-07, | |
| "logits": -1.5244942903518677, | |
| "logps": -88.604248046875, | |
| "loss": 0.2411, | |
| "objective": 0.23850402235984802, | |
| "ranking_idealized": 0.675000011920929, | |
| "ranking_idealized_expo": 0.5041666626930237, | |
| "ranking_simple": 0.6000000238418579, | |
| "regularize": 0.23850402235984802, | |
| "step": 280 | |
| }, | |
| { | |
| "dpo_loss": 0.5853084921836853, | |
| "epoch": 1.615493623051488, | |
| "grad_norm": 16.6854633771705, | |
| "learning_rate": 4.274765153095007e-07, | |
| "logits": -1.6502856016159058, | |
| "logps": -89.77727508544922, | |
| "loss": 0.2219, | |
| "objective": 0.21514521539211273, | |
| "ranking_idealized": 0.675000011920929, | |
| "ranking_idealized_expo": 0.5166666507720947, | |
| "ranking_simple": 0.6041666865348816, | |
| "regularize": 0.21514521539211273, | |
| "step": 285 | |
| }, | |
| { | |
| "dpo_loss": 0.5910848379135132, | |
| "epoch": 1.643835616438356, | |
| "grad_norm": 18.978761606300836, | |
| "learning_rate": 4.239497113483819e-07, | |
| "logits": -1.7089149951934814, | |
| "logps": -86.87386322021484, | |
| "loss": 0.2312, | |
| "objective": 0.23006680607795715, | |
| "ranking_idealized": 0.699999988079071, | |
| "ranking_idealized_expo": 0.5625, | |
| "ranking_simple": 0.6291666626930237, | |
| "regularize": 0.23006680607795715, | |
| "step": 290 | |
| }, | |
| { | |
| "dpo_loss": 0.5802692174911499, | |
| "epoch": 1.6721776098252243, | |
| "grad_norm": 16.652074965539576, | |
| "learning_rate": 4.203544848984728e-07, | |
| "logits": -1.5955086946487427, | |
| "logps": -86.49956512451172, | |
| "loss": 0.2276, | |
| "objective": 0.23742005228996277, | |
| "ranking_idealized": 0.6583333611488342, | |
| "ranking_idealized_expo": 0.4833333194255829, | |
| "ranking_simple": 0.5791666507720947, | |
| "regularize": 0.23742005228996277, | |
| "step": 295 | |
| }, | |
| { | |
| "dpo_loss": 0.5916833281517029, | |
| "epoch": 1.7005196032120926, | |
| "grad_norm": 16.884463449554712, | |
| "learning_rate": 4.166922501290729e-07, | |
| "logits": -1.6546835899353027, | |
| "logps": -88.2989730834961, | |
| "loss": 0.2252, | |
| "objective": 0.26394858956336975, | |
| "ranking_idealized": 0.7083333134651184, | |
| "ranking_idealized_expo": 0.574999988079071, | |
| "ranking_simple": 0.6208333373069763, | |
| "regularize": 0.26394858956336975, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 1.7005196032120926, | |
| "eval_dpo_loss": 0.6866207718849182, | |
| "eval_logits": -1.6648496389389038, | |
| "eval_logps": -95.62443542480469, | |
| "eval_loss": 0.410134494304657, | |
| "eval_objective": 0.4137687385082245, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.5216942429542542, | |
| "eval_regularize": 0.4137687385082245, | |
| "eval_runtime": 259.4458, | |
| "eval_samples_per_second": 22.317, | |
| "eval_steps_per_second": 0.933, | |
| "step": 300 | |
| }, | |
| { | |
| "dpo_loss": 0.5952399373054504, | |
| "epoch": 1.7288615965989607, | |
| "grad_norm": 16.24562342201146, | |
| "learning_rate": 4.129644475669616e-07, | |
| "logits": -1.6116312742233276, | |
| "logps": -88.82595825195312, | |
| "loss": 0.218, | |
| "objective": 0.2242499738931656, | |
| "ranking_idealized": 0.6291666626930237, | |
| "ranking_idealized_expo": 0.5083333253860474, | |
| "ranking_simple": 0.5833333134651184, | |
| "regularize": 0.2242499738931656, | |
| "step": 305 | |
| }, | |
| { | |
| "dpo_loss": 0.583368182182312, | |
| "epoch": 1.7572035899858292, | |
| "grad_norm": 18.099666352463437, | |
| "learning_rate": 4.0917254352977206e-07, | |
| "logits": -1.7004183530807495, | |
| "logps": -87.11441040039062, | |
| "loss": 0.2283, | |
| "objective": 0.2325660139322281, | |
| "ranking_idealized": 0.6583333611488342, | |
| "ranking_idealized_expo": 0.5333333611488342, | |
| "ranking_simple": 0.612500011920929, | |
| "regularize": 0.2325660139322281, | |
| "step": 310 | |
| }, | |
| { | |
| "dpo_loss": 0.5933206677436829, | |
| "epoch": 1.7855455833726972, | |
| "grad_norm": 16.545516113765466, | |
| "learning_rate": 4.053180295492202e-07, | |
| "logits": -1.602583408355713, | |
| "logps": -88.69900512695312, | |
| "loss": 0.2287, | |
| "objective": 0.21895338594913483, | |
| "ranking_idealized": 0.6541666388511658, | |
| "ranking_idealized_expo": 0.512499988079071, | |
| "ranking_simple": 0.5833333134651184, | |
| "regularize": 0.21895337104797363, | |
| "step": 315 | |
| }, | |
| { | |
| "dpo_loss": 0.5876157283782959, | |
| "epoch": 1.8138875767595655, | |
| "grad_norm": 18.487916312721516, | |
| "learning_rate": 4.0140242178441665e-07, | |
| "logits": -1.6777514219284058, | |
| "logps": -90.22407531738281, | |
| "loss": 0.2153, | |
| "objective": 0.20208925008773804, | |
| "ranking_idealized": 0.6625000238418579, | |
| "ranking_idealized_expo": 0.5041666626930237, | |
| "ranking_simple": 0.5874999761581421, | |
| "regularize": 0.20208925008773804, | |
| "step": 320 | |
| }, | |
| { | |
| "dpo_loss": 0.5746586918830872, | |
| "epoch": 1.8422295701464337, | |
| "grad_norm": 18.670087833334332, | |
| "learning_rate": 3.9742726042549053e-07, | |
| "logits": -1.7464016675949097, | |
| "logps": -91.97502899169922, | |
| "loss": 0.219, | |
| "objective": 0.2114688903093338, | |
| "ranking_idealized": 0.7166666388511658, | |
| "ranking_idealized_expo": 0.5208333134651184, | |
| "ranking_simple": 0.6416666507720947, | |
| "regularize": 0.2114688903093338, | |
| "step": 325 | |
| }, | |
| { | |
| "dpo_loss": 0.5767069458961487, | |
| "epoch": 1.8705715635333018, | |
| "grad_norm": 18.91235181922618, | |
| "learning_rate": 3.933941090877615e-07, | |
| "logits": -1.466091275215149, | |
| "logps": -90.11954498291016, | |
| "loss": 0.219, | |
| "objective": 0.2171897292137146, | |
| "ranking_idealized": 0.6499999761581421, | |
| "ranking_idealized_expo": 0.4749999940395355, | |
| "ranking_simple": 0.574999988079071, | |
| "regularize": 0.2171897292137146, | |
| "step": 330 | |
| }, | |
| { | |
| "dpo_loss": 0.5894278287887573, | |
| "epoch": 1.89891355692017, | |
| "grad_norm": 15.592599296406116, | |
| "learning_rate": 3.8930455419669744e-07, | |
| "logits": -1.6301844120025635, | |
| "logps": -89.44200134277344, | |
| "loss": 0.2112, | |
| "objective": 0.18907961249351501, | |
| "ranking_idealized": 0.6499999761581421, | |
| "ranking_idealized_expo": 0.4958333373069763, | |
| "ranking_simple": 0.6000000238418579, | |
| "regularize": 0.18907961249351501, | |
| "step": 335 | |
| }, | |
| { | |
| "dpo_loss": 0.5853725075721741, | |
| "epoch": 1.9272555503070383, | |
| "grad_norm": 15.83418724261755, | |
| "learning_rate": 3.851602043638994e-07, | |
| "logits": -1.660121202468872, | |
| "logps": -91.48560333251953, | |
| "loss": 0.2026, | |
| "objective": 0.18658672273159027, | |
| "ranking_idealized": 0.7291666865348816, | |
| "ranking_idealized_expo": 0.5458333492279053, | |
| "ranking_simple": 0.6416666507720947, | |
| "regularize": 0.18658672273159027, | |
| "step": 340 | |
| }, | |
| { | |
| "dpo_loss": 0.5825453400611877, | |
| "epoch": 1.9555975436939064, | |
| "grad_norm": 16.366640560133238, | |
| "learning_rate": 3.809626897543604e-07, | |
| "logits": -1.657557725906372, | |
| "logps": -90.72650909423828, | |
| "loss": 0.1961, | |
| "objective": 0.18415075540542603, | |
| "ranking_idealized": 0.6458333134651184, | |
| "ranking_idealized_expo": 0.5333333611488342, | |
| "ranking_simple": 0.6166666746139526, | |
| "regularize": 0.18415075540542603, | |
| "step": 345 | |
| }, | |
| { | |
| "dpo_loss": 0.5838915109634399, | |
| "epoch": 1.9839395370807746, | |
| "grad_norm": 17.651439137685784, | |
| "learning_rate": 3.7671366144524576e-07, | |
| "logits": -1.551125407218933, | |
| "logps": -91.74525451660156, | |
| "loss": 0.2082, | |
| "objective": 0.20508398115634918, | |
| "ranking_idealized": 0.6333333253860474, | |
| "ranking_idealized_expo": 0.49166667461395264, | |
| "ranking_simple": 0.5541666746139526, | |
| "regularize": 0.20508398115634918, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 1.9839395370807746, | |
| "eval_dpo_loss": 0.6863144040107727, | |
| "eval_logits": -1.6988588571548462, | |
| "eval_logps": -97.52546691894531, | |
| "eval_loss": 0.4102429747581482, | |
| "eval_objective": 0.4131539762020111, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.51962810754776, | |
| "eval_regularize": 0.4131539762020111, | |
| "eval_runtime": 258.5165, | |
| "eval_samples_per_second": 22.397, | |
| "eval_steps_per_second": 0.936, | |
| "step": 350 | |
| }, | |
| { | |
| "dpo_loss": 0.5745717287063599, | |
| "epoch": 2.012281530467643, | |
| "grad_norm": 15.784086525377202, | |
| "learning_rate": 3.724147907764478e-07, | |
| "logits": -1.5323989391326904, | |
| "logps": -90.18486785888672, | |
| "loss": 0.2055, | |
| "objective": 0.20713359117507935, | |
| "ranking_idealized": 0.699999988079071, | |
| "ranking_idealized_expo": 0.5166666507720947, | |
| "ranking_simple": 0.6458333134651184, | |
| "regularize": 0.20713359117507935, | |
| "step": 355 | |
| }, | |
| { | |
| "dpo_loss": 0.5675494074821472, | |
| "epoch": 2.040623523854511, | |
| "grad_norm": 20.55210866626824, | |
| "learning_rate": 3.6806776869317067e-07, | |
| "logits": -1.6239458322525024, | |
| "logps": -89.69377899169922, | |
| "loss": 0.1726, | |
| "objective": 0.17787505686283112, | |
| "ranking_idealized": 0.7250000238418579, | |
| "ranking_idealized_expo": 0.5333333611488342, | |
| "ranking_simple": 0.675000011920929, | |
| "regularize": 0.17787505686283112, | |
| "step": 360 | |
| }, | |
| { | |
| "dpo_loss": 0.5650666952133179, | |
| "epoch": 2.0689655172413794, | |
| "grad_norm": 16.544231581396616, | |
| "learning_rate": 3.636743050808028e-07, | |
| "logits": -1.6872822046279907, | |
| "logps": -91.26659393310547, | |
| "loss": 0.1866, | |
| "objective": 0.16895455121994019, | |
| "ranking_idealized": 0.6708333492279053, | |
| "ranking_idealized_expo": 0.5041666626930237, | |
| "ranking_simple": 0.6333333253860474, | |
| "regularize": 0.16895455121994019, | |
| "step": 365 | |
| }, | |
| { | |
| "dpo_loss": 0.575705349445343, | |
| "epoch": 2.0973075106282475, | |
| "grad_norm": 16.758890304778106, | |
| "learning_rate": 3.5923612809233984e-07, | |
| "logits": -1.662663221359253, | |
| "logps": -87.82825469970703, | |
| "loss": 0.1679, | |
| "objective": 0.1752353459596634, | |
| "ranking_idealized": 0.6541666388511658, | |
| "ranking_idealized_expo": 0.4833333194255829, | |
| "ranking_simple": 0.574999988079071, | |
| "regularize": 0.1752353310585022, | |
| "step": 370 | |
| }, | |
| { | |
| "dpo_loss": 0.5633853077888489, | |
| "epoch": 2.1256495040151155, | |
| "grad_norm": 16.99783941953761, | |
| "learning_rate": 3.5475498346862214e-07, | |
| "logits": -1.6271302700042725, | |
| "logps": -91.13916015625, | |
| "loss": 0.1726, | |
| "objective": 0.16911908984184265, | |
| "ranking_idealized": 0.7291666865348816, | |
| "ranking_idealized_expo": 0.5958333611488342, | |
| "ranking_simple": 0.6791666746139526, | |
| "regularize": 0.16911907494068146, | |
| "step": 375 | |
| }, | |
| { | |
| "dpo_loss": 0.5752108097076416, | |
| "epoch": 2.153991497401984, | |
| "grad_norm": 17.23340187781712, | |
| "learning_rate": 3.502326338516534e-07, | |
| "logits": -1.5394021272659302, | |
| "logps": -89.99533081054688, | |
| "loss": 0.179, | |
| "objective": 0.1650255024433136, | |
| "ranking_idealized": 0.6833333373069763, | |
| "ranking_idealized_expo": 0.5375000238418579, | |
| "ranking_simple": 0.6000000238418579, | |
| "regularize": 0.1650255024433136, | |
| "step": 380 | |
| }, | |
| { | |
| "dpo_loss": 0.571977972984314, | |
| "epoch": 2.182333490788852, | |
| "grad_norm": 15.78796183229778, | |
| "learning_rate": 3.4567085809127245e-07, | |
| "logits": -1.6716177463531494, | |
| "logps": -91.3305892944336, | |
| "loss": 0.1653, | |
| "objective": 0.13291777670383453, | |
| "ranking_idealized": 0.6458333134651184, | |
| "ranking_idealized_expo": 0.5083333253860474, | |
| "ranking_simple": 0.612500011920929, | |
| "regularize": 0.13291777670383453, | |
| "step": 385 | |
| }, | |
| { | |
| "dpo_loss": 0.5752423405647278, | |
| "epoch": 2.21067548417572, | |
| "grad_norm": 18.129151048308177, | |
| "learning_rate": 3.4107145054544855e-07, | |
| "logits": -1.5358682870864868, | |
| "logps": -91.15263366699219, | |
| "loss": 0.1744, | |
| "objective": 0.16379062831401825, | |
| "ranking_idealized": 0.6041666865348816, | |
| "ranking_idealized_expo": 0.4833333194255829, | |
| "ranking_simple": 0.5625, | |
| "regularize": 0.16379062831401825, | |
| "step": 390 | |
| }, | |
| { | |
| "dpo_loss": 0.5485681891441345, | |
| "epoch": 2.2390174775625886, | |
| "grad_norm": 16.313781937896024, | |
| "learning_rate": 3.3643622037447767e-07, | |
| "logits": -1.5593619346618652, | |
| "logps": -92.42921447753906, | |
| "loss": 0.1776, | |
| "objective": 0.1637614667415619, | |
| "ranking_idealized": 0.6916666626930237, | |
| "ranking_idealized_expo": 0.5166666507720947, | |
| "ranking_simple": 0.6416666507720947, | |
| "regularize": 0.1637614667415619, | |
| "step": 395 | |
| }, | |
| { | |
| "dpo_loss": 0.5597947239875793, | |
| "epoch": 2.2673594709494567, | |
| "grad_norm": 16.659127876259, | |
| "learning_rate": 3.317669908293554e-07, | |
| "logits": -1.631813645362854, | |
| "logps": -92.92410278320312, | |
| "loss": 0.1825, | |
| "objective": 0.196553573012352, | |
| "ranking_idealized": 0.7583333253860474, | |
| "ranking_idealized_expo": 0.5791666507720947, | |
| "ranking_simple": 0.6958333253860474, | |
| "regularize": 0.1965535581111908, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 2.2673594709494567, | |
| "eval_dpo_loss": 0.6862595677375793, | |
| "eval_logits": -1.6931663751602173, | |
| "eval_logps": -97.79962158203125, | |
| "eval_loss": 0.4124037027359009, | |
| "eval_objective": 0.4144473969936371, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.5206611752510071, | |
| "eval_regularize": 0.4144473969936371, | |
| "eval_runtime": 258.4529, | |
| "eval_samples_per_second": 22.403, | |
| "eval_steps_per_second": 0.936, | |
| "step": 400 | |
| }, | |
| { | |
| "dpo_loss": 0.5691500902175903, | |
| "epoch": 2.295701464336325, | |
| "grad_norm": 17.30117286858182, | |
| "learning_rate": 3.270655985346081e-07, | |
| "logits": -1.7139372825622559, | |
| "logps": -89.78938293457031, | |
| "loss": 0.1717, | |
| "objective": 0.18224166333675385, | |
| "ranking_idealized": 0.737500011920929, | |
| "ranking_idealized_expo": 0.5583333373069763, | |
| "ranking_simple": 0.6458333134651184, | |
| "regularize": 0.18224166333675385, | |
| "step": 405 | |
| }, | |
| { | |
| "dpo_loss": 0.5579439997673035, | |
| "epoch": 2.324043457723193, | |
| "grad_norm": 17.123218301010457, | |
| "learning_rate": 3.223338927658632e-07, | |
| "logits": -1.5741162300109863, | |
| "logps": -91.07009887695312, | |
| "loss": 0.1618, | |
| "objective": 0.15759395062923431, | |
| "ranking_idealized": 0.6499999761581421, | |
| "ranking_idealized_expo": 0.4833333194255829, | |
| "ranking_simple": 0.5958333611488342, | |
| "regularize": 0.15759395062923431, | |
| "step": 410 | |
| }, | |
| { | |
| "dpo_loss": 0.5704253315925598, | |
| "epoch": 2.3523854511100613, | |
| "grad_norm": 17.68931154440285, | |
| "learning_rate": 3.175737347224432e-07, | |
| "logits": -1.6476367712020874, | |
| "logps": -91.30075073242188, | |
| "loss": 0.1732, | |
| "objective": 0.19281157851219177, | |
| "ranking_idealized": 0.6458333134651184, | |
| "ranking_idealized_expo": 0.5333333611488342, | |
| "ranking_simple": 0.612500011920929, | |
| "regularize": 0.19281157851219177, | |
| "step": 415 | |
| }, | |
| { | |
| "dpo_loss": 0.5771389603614807, | |
| "epoch": 2.3807274444969297, | |
| "grad_norm": 17.274582557860825, | |
| "learning_rate": 3.1278699679526975e-07, | |
| "logits": -1.5415838956832886, | |
| "logps": -92.63572692871094, | |
| "loss": 0.1579, | |
| "objective": 0.15308959782123566, | |
| "ranking_idealized": 0.6458333134651184, | |
| "ranking_idealized_expo": 0.4749999940395355, | |
| "ranking_simple": 0.5958333611488342, | |
| "regularize": 0.15308959782123566, | |
| "step": 420 | |
| }, | |
| { | |
| "dpo_loss": 0.566936731338501, | |
| "epoch": 2.409069437883798, | |
| "grad_norm": 16.445557447346342, | |
| "learning_rate": 3.0797556183036575e-07, | |
| "logits": -1.5967096090316772, | |
| "logps": -91.4622802734375, | |
| "loss": 0.1607, | |
| "objective": 0.16068215668201447, | |
| "ranking_idealized": 0.6625000238418579, | |
| "ranking_idealized_expo": 0.5041666626930237, | |
| "ranking_simple": 0.6000000238418579, | |
| "regularize": 0.16068214178085327, | |
| "step": 425 | |
| }, | |
| { | |
| "dpo_loss": 0.5632474422454834, | |
| "epoch": 2.4374114312706663, | |
| "grad_norm": 15.962055488306607, | |
| "learning_rate": 3.0314132238824415e-07, | |
| "logits": -1.6247813701629639, | |
| "logps": -92.1604995727539, | |
| "loss": 0.1547, | |
| "objective": 0.1360505074262619, | |
| "ranking_idealized": 0.6708333492279053, | |
| "ranking_idealized_expo": 0.5333333611488342, | |
| "ranking_simple": 0.625, | |
| "regularize": 0.1360505074262619, | |
| "step": 430 | |
| }, | |
| { | |
| "dpo_loss": 0.566851019859314, | |
| "epoch": 2.4657534246575343, | |
| "grad_norm": 16.006081940650837, | |
| "learning_rate": 2.982861799994764e-07, | |
| "logits": -1.6544443368911743, | |
| "logps": -92.63692474365234, | |
| "loss": 0.1637, | |
| "objective": 0.17756709456443787, | |
| "ranking_idealized": 0.6333333253860474, | |
| "ranking_idealized_expo": 0.5083333253860474, | |
| "ranking_simple": 0.6208333373069763, | |
| "regularize": 0.17756709456443787, | |
| "step": 435 | |
| }, | |
| { | |
| "dpo_loss": 0.5565729141235352, | |
| "epoch": 2.4940954180444024, | |
| "grad_norm": 17.37344722468487, | |
| "learning_rate": 2.934120444167326e-07, | |
| "logits": -1.5883994102478027, | |
| "logps": -91.88066101074219, | |
| "loss": 0.159, | |
| "objective": 0.15150482952594757, | |
| "ranking_idealized": 0.6333333253860474, | |
| "ranking_idealized_expo": 0.5375000238418579, | |
| "ranking_simple": 0.6041666865348816, | |
| "regularize": 0.15150482952594757, | |
| "step": 440 | |
| }, | |
| { | |
| "dpo_loss": 0.565682590007782, | |
| "epoch": 2.5224374114312704, | |
| "grad_norm": 18.453788667979182, | |
| "learning_rate": 2.885208328635864e-07, | |
| "logits": -1.6123565435409546, | |
| "logps": -89.5006332397461, | |
| "loss": 0.1576, | |
| "objective": 0.1587233543395996, | |
| "ranking_idealized": 0.6958333253860474, | |
| "ranking_idealized_expo": 0.5166666507720947, | |
| "ranking_simple": 0.6458333134651184, | |
| "regularize": 0.1587233543395996, | |
| "step": 445 | |
| }, | |
| { | |
| "dpo_loss": 0.5824019312858582, | |
| "epoch": 2.550779404818139, | |
| "grad_norm": 19.424550718198045, | |
| "learning_rate": 2.83614469280383e-07, | |
| "logits": -1.6537593603134155, | |
| "logps": -91.4095230102539, | |
| "loss": 0.1504, | |
| "objective": 0.15120406448841095, | |
| "ranking_idealized": 0.6583333611488342, | |
| "ranking_idealized_expo": 0.5, | |
| "ranking_simple": 0.5791666507720947, | |
| "regularize": 0.15120406448841095, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 2.550779404818139, | |
| "eval_dpo_loss": 0.6864377856254578, | |
| "eval_logits": -1.711348056793213, | |
| "eval_logps": -99.202880859375, | |
| "eval_loss": 0.41492125391960144, | |
| "eval_objective": 0.4176488518714905, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.5216942429542542, | |
| "eval_regularize": 0.4176488518714905, | |
| "eval_runtime": 258.9375, | |
| "eval_samples_per_second": 22.361, | |
| "eval_steps_per_second": 0.935, | |
| "step": 450 | |
| }, | |
| { | |
| "dpo_loss": 0.5587320923805237, | |
| "epoch": 2.579121398205007, | |
| "grad_norm": 18.174711126742732, | |
| "learning_rate": 2.786948835674634e-07, | |
| "logits": -1.6923545598983765, | |
| "logps": -92.0631103515625, | |
| "loss": 0.1514, | |
| "objective": 0.15467478334903717, | |
| "ranking_idealized": 0.6666666865348816, | |
| "ranking_idealized_expo": 0.5041666626930237, | |
| "ranking_simple": 0.6416666507720947, | |
| "regularize": 0.15467478334903717, | |
| "step": 455 | |
| }, | |
| { | |
| "dpo_loss": 0.5529297590255737, | |
| "epoch": 2.6074633915918755, | |
| "grad_norm": 18.378396938924546, | |
| "learning_rate": 2.737640108260456e-07, | |
| "logits": -1.765284776687622, | |
| "logps": -92.5921401977539, | |
| "loss": 0.1544, | |
| "objective": 0.13981758058071136, | |
| "ranking_idealized": 0.6833333373069763, | |
| "ranking_idealized_expo": 0.512499988079071, | |
| "ranking_simple": 0.6291666626930237, | |
| "regularize": 0.13981756567955017, | |
| "step": 460 | |
| }, | |
| { | |
| "dpo_loss": 0.5604754090309143, | |
| "epoch": 2.6358053849787435, | |
| "grad_norm": 17.16312208138119, | |
| "learning_rate": 2.6882379059705953e-07, | |
| "logits": -1.6412590742111206, | |
| "logps": -91.83204650878906, | |
| "loss": 0.1571, | |
| "objective": 0.15992027521133423, | |
| "ranking_idealized": 0.612500011920929, | |
| "ranking_idealized_expo": 0.4958333373069763, | |
| "ranking_simple": 0.5791666507720947, | |
| "regularize": 0.15992026031017303, | |
| "step": 465 | |
| }, | |
| { | |
| "dpo_loss": 0.5741956830024719, | |
| "epoch": 2.6641473783656116, | |
| "grad_norm": 17.444271577746782, | |
| "learning_rate": 2.6387616609823504e-07, | |
| "logits": -1.6750518083572388, | |
| "logps": -91.33477020263672, | |
| "loss": 0.151, | |
| "objective": 0.17329135537147522, | |
| "ranking_idealized": 0.6625000238418579, | |
| "ranking_idealized_expo": 0.5583333373069763, | |
| "ranking_simple": 0.625, | |
| "regularize": 0.17329135537147522, | |
| "step": 470 | |
| }, | |
| { | |
| "dpo_loss": 0.5746079087257385, | |
| "epoch": 2.69248937175248, | |
| "grad_norm": 17.607595627923466, | |
| "learning_rate": 2.5892308345974514e-07, | |
| "logits": -1.6217347383499146, | |
| "logps": -90.19564819335938, | |
| "loss": 0.1521, | |
| "objective": 0.1534017026424408, | |
| "ranking_idealized": 0.6791666746139526, | |
| "ranking_idealized_expo": 0.4958333373069763, | |
| "ranking_simple": 0.6166666746139526, | |
| "regularize": 0.1534017026424408, | |
| "step": 475 | |
| }, | |
| { | |
| "dpo_loss": 0.56805020570755, | |
| "epoch": 2.720831365139348, | |
| "grad_norm": 18.441983400540806, | |
| "learning_rate": 2.53966490958702e-07, | |
| "logits": -1.7197903394699097, | |
| "logps": -90.20177459716797, | |
| "loss": 0.148, | |
| "objective": 0.14620445668697357, | |
| "ranking_idealized": 0.6833333373069763, | |
| "ranking_idealized_expo": 0.5, | |
| "ranking_simple": 0.6291666626930237, | |
| "regularize": 0.14620442688465118, | |
| "step": 480 | |
| }, | |
| { | |
| "dpo_loss": 0.5559974312782288, | |
| "epoch": 2.7491733585262166, | |
| "grad_norm": 17.00220355810742, | |
| "learning_rate": 2.4900833825280967e-07, | |
| "logits": -1.628369927406311, | |
| "logps": -93.048828125, | |
| "loss": 0.1488, | |
| "objective": 0.1451708972454071, | |
| "ranking_idealized": 0.6708333492279053, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.6583333611488342, | |
| "regularize": 0.1451708972454071, | |
| "step": 485 | |
| }, | |
| { | |
| "dpo_loss": 0.555105984210968, | |
| "epoch": 2.7775153519130846, | |
| "grad_norm": 17.798810379621077, | |
| "learning_rate": 2.4405057561347313e-07, | |
| "logits": -1.647185206413269, | |
| "logps": -90.4990463256836, | |
| "loss": 0.1613, | |
| "objective": 0.17200501263141632, | |
| "ranking_idealized": 0.6833333373069763, | |
| "ranking_idealized_expo": 0.4749999940395355, | |
| "ranking_simple": 0.6208333373069763, | |
| "regularize": 0.17200501263141632, | |
| "step": 490 | |
| }, | |
| { | |
| "dpo_loss": 0.5594576001167297, | |
| "epoch": 2.8058573452999527, | |
| "grad_norm": 18.076540126591944, | |
| "learning_rate": 2.39095153158666e-07, | |
| "logits": -1.6548616886138916, | |
| "logps": -90.19225311279297, | |
| "loss": 0.1504, | |
| "objective": 0.1365150660276413, | |
| "ranking_idealized": 0.6416666507720947, | |
| "ranking_idealized_expo": 0.4833333194255829, | |
| "ranking_simple": 0.6208333373069763, | |
| "regularize": 0.1365150511264801, | |
| "step": 495 | |
| }, | |
| { | |
| "dpo_loss": 0.5552747845649719, | |
| "epoch": 2.8341993386868207, | |
| "grad_norm": 17.278782223651127, | |
| "learning_rate": 2.3414402008585886e-07, | |
| "logits": -1.6857832670211792, | |
| "logps": -89.0853500366211, | |
| "loss": 0.1494, | |
| "objective": 0.15246258676052094, | |
| "ranking_idealized": 0.6916666626930237, | |
| "ranking_idealized_expo": 0.5208333134651184, | |
| "ranking_simple": 0.6416666507720947, | |
| "regularize": 0.15246258676052094, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 2.8341993386868207, | |
| "eval_dpo_loss": 0.6861580014228821, | |
| "eval_logits": -1.7174702882766724, | |
| "eval_logps": -99.17545318603516, | |
| "eval_loss": 0.41525644063949585, | |
| "eval_objective": 0.4182237386703491, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.5227272510528564, | |
| "eval_regularize": 0.4182237386703491, | |
| "eval_runtime": 259.2438, | |
| "eval_samples_per_second": 22.334, | |
| "eval_steps_per_second": 0.933, | |
| "step": 500 | |
| }, | |
| { | |
| "dpo_loss": 0.5660989284515381, | |
| "epoch": 2.862541332073689, | |
| "grad_norm": 18.182680782212074, | |
| "learning_rate": 2.2919912390530943e-07, | |
| "logits": -1.6143929958343506, | |
| "logps": -91.0888900756836, | |
| "loss": 0.1437, | |
| "objective": 0.16082407534122467, | |
| "ranking_idealized": 0.6583333611488342, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.6333333253860474, | |
| "regularize": 0.16082406044006348, | |
| "step": 505 | |
| }, | |
| { | |
| "dpo_loss": 0.5675150752067566, | |
| "epoch": 2.8908833254605573, | |
| "grad_norm": 16.373132303441977, | |
| "learning_rate": 2.2426240967401638e-07, | |
| "logits": -1.5807684659957886, | |
| "logps": -91.39689636230469, | |
| "loss": 0.1433, | |
| "objective": 0.1494457870721817, | |
| "ranking_idealized": 0.6958333253860474, | |
| "ranking_idealized_expo": 0.5291666388511658, | |
| "ranking_simple": 0.6291666626930237, | |
| "regularize": 0.1494457870721817, | |
| "step": 510 | |
| }, | |
| { | |
| "dpo_loss": 0.5627566576004028, | |
| "epoch": 2.9192253188474258, | |
| "grad_norm": 18.008132213394468, | |
| "learning_rate": 2.1933581923063837e-07, | |
| "logits": -1.7557440996170044, | |
| "logps": -91.32353210449219, | |
| "loss": 0.1448, | |
| "objective": 0.13260915875434875, | |
| "ranking_idealized": 0.6583333611488342, | |
| "ranking_idealized_expo": 0.4791666567325592, | |
| "ranking_simple": 0.6291666626930237, | |
| "regularize": 0.13260914385318756, | |
| "step": 515 | |
| }, | |
| { | |
| "dpo_loss": 0.5646940469741821, | |
| "epoch": 2.947567312234294, | |
| "grad_norm": 17.30767973762921, | |
| "learning_rate": 2.1442129043167873e-07, | |
| "logits": -1.610668420791626, | |
| "logps": -92.7865219116211, | |
| "loss": 0.1368, | |
| "objective": 0.11772733181715012, | |
| "ranking_idealized": 0.6583333611488342, | |
| "ranking_idealized_expo": 0.512499988079071, | |
| "ranking_simple": 0.6291666626930237, | |
| "regularize": 0.11772733181715012, | |
| "step": 520 | |
| }, | |
| { | |
| "dpo_loss": 0.5658089518547058, | |
| "epoch": 2.975909305621162, | |
| "grad_norm": 18.116492800551395, | |
| "learning_rate": 2.0952075638923652e-07, | |
| "logits": -1.6272333860397339, | |
| "logps": -92.43870544433594, | |
| "loss": 0.1424, | |
| "objective": 0.15236981213092804, | |
| "ranking_idealized": 0.6875, | |
| "ranking_idealized_expo": 0.5708333253860474, | |
| "ranking_simple": 0.6416666507720947, | |
| "regularize": 0.15236981213092804, | |
| "step": 525 | |
| }, | |
| { | |
| "dpo_loss": 0.5534684658050537, | |
| "epoch": 3.0042512990080303, | |
| "grad_norm": 18.337044286762765, | |
| "learning_rate": 2.0463614471062435e-07, | |
| "logits": -1.6210473775863647, | |
| "logps": -91.47294616699219, | |
| "loss": 0.1502, | |
| "objective": 0.17477649450302124, | |
| "ranking_idealized": 0.675000011920929, | |
| "ranking_idealized_expo": 0.512499988079071, | |
| "ranking_simple": 0.6291666626930237, | |
| "regularize": 0.17477649450302124, | |
| "step": 530 | |
| }, | |
| { | |
| "dpo_loss": 0.5659457445144653, | |
| "epoch": 3.0325932923948984, | |
| "grad_norm": 16.444884726429134, | |
| "learning_rate": 1.9976937674015026e-07, | |
| "logits": -1.6844907999038696, | |
| "logps": -93.2222671508789, | |
| "loss": 0.1284, | |
| "objective": 0.14268328249454498, | |
| "ranking_idealized": 0.6708333492279053, | |
| "ranking_idealized_expo": 0.5458333492279053, | |
| "ranking_simple": 0.6583333611488342, | |
| "regularize": 0.14268328249454498, | |
| "step": 535 | |
| }, | |
| { | |
| "dpo_loss": 0.5521051287651062, | |
| "epoch": 3.0609352857817664, | |
| "grad_norm": 19.963683437356444, | |
| "learning_rate": 1.9492236680336483e-07, | |
| "logits": -1.7760847806930542, | |
| "logps": -90.89082336425781, | |
| "loss": 0.1216, | |
| "objective": 0.10329335182905197, | |
| "ranking_idealized": 0.6625000238418579, | |
| "ranking_idealized_expo": 0.5375000238418579, | |
| "ranking_simple": 0.6541666388511658, | |
| "regularize": 0.10329335182905197, | |
| "step": 540 | |
| }, | |
| { | |
| "dpo_loss": 0.5619763731956482, | |
| "epoch": 3.089277279168635, | |
| "grad_norm": 17.450130382767895, | |
| "learning_rate": 1.9009702145406724e-07, | |
| "logits": -1.6995065212249756, | |
| "logps": -92.40625, | |
| "loss": 0.1232, | |
| "objective": 0.1230437308549881, | |
| "ranking_idealized": 0.637499988079071, | |
| "ranking_idealized_expo": 0.49166667461395264, | |
| "ranking_simple": 0.6000000238418579, | |
| "regularize": 0.1230437308549881, | |
| "step": 545 | |
| }, | |
| { | |
| "dpo_loss": 0.5528106689453125, | |
| "epoch": 3.117619272555503, | |
| "grad_norm": 18.245098236126562, | |
| "learning_rate": 1.8529523872436977e-07, | |
| "logits": -1.5086556673049927, | |
| "logps": -92.30103302001953, | |
| "loss": 0.1407, | |
| "objective": 0.12957319617271423, | |
| "ranking_idealized": 0.699999988079071, | |
| "ranking_idealized_expo": 0.5458333492279053, | |
| "ranking_simple": 0.6791666746139526, | |
| "regularize": 0.12957318127155304, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 3.117619272555503, | |
| "eval_dpo_loss": 0.6856257915496826, | |
| "eval_logits": -1.7183054685592651, | |
| "eval_logps": -99.2997055053711, | |
| "eval_loss": 0.4161340296268463, | |
| "eval_objective": 0.41743505001068115, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.5216942429542542, | |
| "eval_regularize": 0.41743505001068115, | |
| "eval_runtime": 258.7783, | |
| "eval_samples_per_second": 22.374, | |
| "eval_steps_per_second": 0.935, | |
| "step": 550 | |
| }, | |
| { | |
| "dpo_loss": 0.5473430752754211, | |
| "epoch": 3.1459612659423715, | |
| "grad_norm": 18.87722095427309, | |
| "learning_rate": 1.8051890737811393e-07, | |
| "logits": -1.6218358278274536, | |
| "logps": -93.05738067626953, | |
| "loss": 0.1336, | |
| "objective": 0.1305130124092102, | |
| "ranking_idealized": 0.6833333373069763, | |
| "ranking_idealized_expo": 0.550000011920929, | |
| "ranking_simple": 0.6625000238418579, | |
| "regularize": 0.1305130124092102, | |
| "step": 555 | |
| }, | |
| { | |
| "dpo_loss": 0.5478367209434509, | |
| "epoch": 3.1743032593292395, | |
| "grad_norm": 22.714698597290123, | |
| "learning_rate": 1.7576990616793137e-07, | |
| "logits": -1.601859211921692, | |
| "logps": -90.21554565429688, | |
| "loss": 0.1212, | |
| "objective": 0.10795855522155762, | |
| "ranking_idealized": 0.6666666865348816, | |
| "ranking_idealized_expo": 0.5041666626930237, | |
| "ranking_simple": 0.637499988079071, | |
| "regularize": 0.10795855522155762, | |
| "step": 560 | |
| }, | |
| { | |
| "dpo_loss": 0.5566601157188416, | |
| "epoch": 3.2026452527161076, | |
| "grad_norm": 24.322678833478967, | |
| "learning_rate": 1.710501030962438e-07, | |
| "logits": -1.663177728652954, | |
| "logps": -91.7726058959961, | |
| "loss": 0.1298, | |
| "objective": 0.13216590881347656, | |
| "ranking_idealized": 0.6916666626930237, | |
| "ranking_idealized_expo": 0.5458333492279053, | |
| "ranking_simple": 0.6458333134651184, | |
| "regularize": 0.13216587901115417, | |
| "step": 565 | |
| }, | |
| { | |
| "dpo_loss": 0.5519458055496216, | |
| "epoch": 3.230987246102976, | |
| "grad_norm": 19.102063233264193, | |
| "learning_rate": 1.663613546804912e-07, | |
| "logits": -1.5763607025146484, | |
| "logps": -91.98208618164062, | |
| "loss": 0.1293, | |
| "objective": 0.13738204538822174, | |
| "ranking_idealized": 0.6416666507720947, | |
| "ranking_idealized_expo": 0.4791666567325592, | |
| "ranking_simple": 0.5916666388511658, | |
| "regularize": 0.13738203048706055, | |
| "step": 570 | |
| }, | |
| { | |
| "dpo_loss": 0.5559364557266235, | |
| "epoch": 3.259329239489844, | |
| "grad_norm": 16.15481429380041, | |
| "learning_rate": 1.617055052228768e-07, | |
| "logits": -1.6705526113510132, | |
| "logps": -92.17435455322266, | |
| "loss": 0.1266, | |
| "objective": 0.12801046669483185, | |
| "ranking_idealized": 0.7083333134651184, | |
| "ranking_idealized_expo": 0.5041666626930237, | |
| "ranking_simple": 0.6499999761581421, | |
| "regularize": 0.12801046669483185, | |
| "step": 575 | |
| }, | |
| { | |
| "dpo_loss": 0.5649384260177612, | |
| "epoch": 3.287671232876712, | |
| "grad_norm": 17.44743081337015, | |
| "learning_rate": 1.5708438608491815e-07, | |
| "logits": -1.6591442823410034, | |
| "logps": -93.50952911376953, | |
| "loss": 0.1277, | |
| "objective": 0.11801984906196594, | |
| "ranking_idealized": 0.6666666865348816, | |
| "ranking_idealized_expo": 0.5583333373069763, | |
| "ranking_simple": 0.6291666626930237, | |
| "regularize": 0.11801984906196594, | |
| "step": 580 | |
| }, | |
| { | |
| "dpo_loss": 0.5595548152923584, | |
| "epoch": 3.3160132262635806, | |
| "grad_norm": 16.72082331684023, | |
| "learning_rate": 1.524998149670871e-07, | |
| "logits": -1.69523286819458, | |
| "logps": -93.74117279052734, | |
| "loss": 0.12, | |
| "objective": 0.10769928246736526, | |
| "ranking_idealized": 0.6958333253860474, | |
| "ranking_idealized_expo": 0.5375000238418579, | |
| "ranking_simple": 0.6583333611488342, | |
| "regularize": 0.10769927501678467, | |
| "step": 585 | |
| }, | |
| { | |
| "dpo_loss": 0.561581015586853, | |
| "epoch": 3.3443552196504487, | |
| "grad_norm": 19.465809423510365, | |
| "learning_rate": 1.479535951938243e-07, | |
| "logits": -1.7049933671951294, | |
| "logps": -93.83617401123047, | |
| "loss": 0.1205, | |
| "objective": 0.09809862077236176, | |
| "ranking_idealized": 0.6666666865348816, | |
| "ranking_idealized_expo": 0.4958333373069763, | |
| "ranking_simple": 0.637499988079071, | |
| "regularize": 0.09809862077236176, | |
| "step": 590 | |
| }, | |
| { | |
| "dpo_loss": 0.5538628101348877, | |
| "epoch": 3.372697213037317, | |
| "grad_norm": 17.81052400873953, | |
| "learning_rate": 1.43447515004208e-07, | |
| "logits": -1.613613247871399, | |
| "logps": -92.85578155517578, | |
| "loss": 0.1191, | |
| "objective": 0.12334737926721573, | |
| "ranking_idealized": 0.6458333134651184, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.637499988079071, | |
| "regularize": 0.12334737926721573, | |
| "step": 595 | |
| }, | |
| { | |
| "dpo_loss": 0.5357978940010071, | |
| "epoch": 3.4010392064241852, | |
| "grad_norm": 18.626853535104544, | |
| "learning_rate": 1.3898334684855645e-07, | |
| "logits": -1.624743938446045, | |
| "logps": -92.40316009521484, | |
| "loss": 0.1149, | |
| "objective": 0.13463754951953888, | |
| "ranking_idealized": 0.6916666626930237, | |
| "ranking_idealized_expo": 0.5041666626930237, | |
| "ranking_simple": 0.6458333134651184, | |
| "regularize": 0.13463754951953888, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 3.4010392064241852, | |
| "eval_dpo_loss": 0.6852067112922668, | |
| "eval_logits": -1.71807062625885, | |
| "eval_logps": -99.92455291748047, | |
| "eval_loss": 0.41705650091171265, | |
| "eval_objective": 0.41811424493789673, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.5247933864593506, | |
| "eval_regularize": 0.41811424493789673, | |
| "eval_runtime": 259.0859, | |
| "eval_samples_per_second": 22.348, | |
| "eval_steps_per_second": 0.934, | |
| "step": 600 | |
| }, | |
| { | |
| "dpo_loss": 0.5652448534965515, | |
| "epoch": 3.4293811998110533, | |
| "grad_norm": 17.958926430591173, | |
| "learning_rate": 1.3456284669124157e-07, | |
| "logits": -1.6740020513534546, | |
| "logps": -94.55862426757812, | |
| "loss": 0.1179, | |
| "objective": 0.11572790890932083, | |
| "ranking_idealized": 0.6583333611488342, | |
| "ranking_idealized_expo": 0.5333333611488342, | |
| "ranking_simple": 0.625, | |
| "regularize": 0.11572788655757904, | |
| "step": 605 | |
| }, | |
| { | |
| "dpo_loss": 0.5543821454048157, | |
| "epoch": 3.4577231931979218, | |
| "grad_norm": 17.326848783729876, | |
| "learning_rate": 1.301877533199859e-07, | |
| "logits": -1.6315828561782837, | |
| "logps": -92.49845886230469, | |
| "loss": 0.1149, | |
| "objective": 0.1067105308175087, | |
| "ranking_idealized": 0.6791666746139526, | |
| "ranking_idealized_expo": 0.5208333134651184, | |
| "ranking_simple": 0.6499999761581421, | |
| "regularize": 0.1067105159163475, | |
| "step": 610 | |
| }, | |
| { | |
| "dpo_loss": 0.5443283915519714, | |
| "epoch": 3.48606518658479, | |
| "grad_norm": 16.84586393500809, | |
| "learning_rate": 1.2585978766191724e-07, | |
| "logits": -1.664933681488037, | |
| "logps": -93.27455139160156, | |
| "loss": 0.1142, | |
| "objective": 0.10945113748311996, | |
| "ranking_idealized": 0.6458333134651184, | |
| "ranking_idealized_expo": 0.47083333134651184, | |
| "ranking_simple": 0.5958333611488342, | |
| "regularize": 0.10945113748311996, | |
| "step": 615 | |
| }, | |
| { | |
| "dpo_loss": 0.5419160723686218, | |
| "epoch": 3.514407179971658, | |
| "grad_norm": 17.32874521556865, | |
| "learning_rate": 1.2158065210664848e-07, | |
| "logits": -1.5332224369049072, | |
| "logps": -92.34308624267578, | |
| "loss": 0.1203, | |
| "objective": 0.12084861099720001, | |
| "ranking_idealized": 0.637499988079071, | |
| "ranking_idealized_expo": 0.5, | |
| "ranking_simple": 0.6166666746139526, | |
| "regularize": 0.12084860354661942, | |
| "step": 620 | |
| }, | |
| { | |
| "dpo_loss": 0.5541211366653442, | |
| "epoch": 3.5427491733585263, | |
| "grad_norm": 17.48530471086995, | |
| "learning_rate": 1.1735202983664802e-07, | |
| "logits": -1.6171096563339233, | |
| "logps": -91.3125991821289, | |
| "loss": 0.1178, | |
| "objective": 0.11125477403402328, | |
| "ranking_idealized": 0.6708333492279053, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.6416666507720947, | |
| "regularize": 0.11125477403402328, | |
| "step": 625 | |
| }, | |
| { | |
| "dpo_loss": 0.5698776245117188, | |
| "epoch": 3.5710911667453944, | |
| "grad_norm": 17.864701578880954, | |
| "learning_rate": 1.1317558416516696e-07, | |
| "logits": -1.697689175605774, | |
| "logps": -91.67240905761719, | |
| "loss": 0.1261, | |
| "objective": 0.13253255188465118, | |
| "ranking_idealized": 0.6333333253860474, | |
| "ranking_idealized_expo": 0.512499988079071, | |
| "ranking_simple": 0.6083333492279053, | |
| "regularize": 0.13253255188465118, | |
| "step": 630 | |
| }, | |
| { | |
| "dpo_loss": 0.5599467158317566, | |
| "epoch": 3.5994331601322624, | |
| "grad_norm": 17.33519253157568, | |
| "learning_rate": 1.090529578819799e-07, | |
| "logits": -1.6461411714553833, | |
| "logps": -91.57376098632812, | |
| "loss": 0.1157, | |
| "objective": 0.10732007026672363, | |
| "ranking_idealized": 0.637499988079071, | |
| "ranking_idealized_expo": 0.48750001192092896, | |
| "ranking_simple": 0.6208333373069763, | |
| "regularize": 0.10732006281614304, | |
| "step": 635 | |
| }, | |
| { | |
| "dpo_loss": 0.540539562702179, | |
| "epoch": 3.627775153519131, | |
| "grad_norm": 17.025667462047203, | |
| "learning_rate": 1.0498577260720048e-07, | |
| "logits": -1.5717778205871582, | |
| "logps": -93.14022827148438, | |
| "loss": 0.1146, | |
| "objective": 0.13500064611434937, | |
| "ranking_idealized": 0.6416666507720947, | |
| "ranking_idealized_expo": 0.4625000059604645, | |
| "ranking_simple": 0.574999988079071, | |
| "regularize": 0.13500064611434937, | |
| "step": 640 | |
| }, | |
| { | |
| "dpo_loss": 0.5469278693199158, | |
| "epoch": 3.656117146905999, | |
| "grad_norm": 17.536092815770388, | |
| "learning_rate": 1.0097562815342214e-07, | |
| "logits": -1.6058826446533203, | |
| "logps": -90.76680755615234, | |
| "loss": 0.1144, | |
| "objective": 0.1191474050283432, | |
| "ranking_idealized": 0.6666666865348816, | |
| "ranking_idealized_expo": 0.5249999761581421, | |
| "ranking_simple": 0.6291666626930237, | |
| "regularize": 0.1191474050283432, | |
| "step": 645 | |
| }, | |
| { | |
| "dpo_loss": 0.5611483454704285, | |
| "epoch": 3.6844591402928675, | |
| "grad_norm": 17.646206320924833, | |
| "learning_rate": 9.702410189643836e-08, | |
| "logits": -1.6121342182159424, | |
| "logps": -92.83375549316406, | |
| "loss": 0.1108, | |
| "objective": 0.09943919628858566, | |
| "ranking_idealized": 0.6458333134651184, | |
| "ranking_idealized_expo": 0.5083333253860474, | |
| "ranking_simple": 0.6083333492279053, | |
| "regularize": 0.09943918883800507, | |
| "step": 650 | |
| }, | |
| { | |
| "epoch": 3.6844591402928675, | |
| "eval_dpo_loss": 0.6852837800979614, | |
| "eval_logits": -1.7315040826797485, | |
| "eval_logps": -99.91177368164062, | |
| "eval_loss": 0.41784536838531494, | |
| "eval_objective": 0.41884875297546387, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.5247933864593506, | |
| "eval_regularize": 0.41884875297546387, | |
| "eval_runtime": 259.4431, | |
| "eval_samples_per_second": 22.317, | |
| "eval_steps_per_second": 0.933, | |
| "step": 650 | |
| }, | |
| { | |
| "dpo_loss": 0.5510907769203186, | |
| "epoch": 3.7128011336797355, | |
| "grad_norm": 17.84028807284025, | |
| "learning_rate": 9.313274815478698e-08, | |
| "logits": -1.6280105113983154, | |
| "logps": -92.27388763427734, | |
| "loss": 0.117, | |
| "objective": 0.10249165445566177, | |
| "ranking_idealized": 0.6583333611488342, | |
| "ranking_idealized_expo": 0.5208333134651184, | |
| "ranking_simple": 0.6291666626930237, | |
| "regularize": 0.10249165445566177, | |
| "step": 655 | |
| }, | |
| { | |
| "dpo_loss": 0.5551621913909912, | |
| "epoch": 3.7411431270666036, | |
| "grad_norm": 17.863904309670215, | |
| "learning_rate": 8.930309757836516e-08, | |
| "logits": -1.7605994939804077, | |
| "logps": -92.74076080322266, | |
| "loss": 0.1162, | |
| "objective": 0.11682406812906265, | |
| "ranking_idealized": 0.6916666626930237, | |
| "ranking_idealized_expo": 0.5541666746139526, | |
| "ranking_simple": 0.6625000238418579, | |
| "regularize": 0.11682406812906265, | |
| "step": 660 | |
| }, | |
| { | |
| "dpo_loss": 0.5553780198097229, | |
| "epoch": 3.769485120453472, | |
| "grad_norm": 17.287090327509993, | |
| "learning_rate": 8.553665654635342e-08, | |
| "logits": -1.6500779390335083, | |
| "logps": -92.00687408447266, | |
| "loss": 0.116, | |
| "objective": 0.11367592960596085, | |
| "ranking_idealized": 0.6958333253860474, | |
| "ranking_idealized_expo": 0.5166666507720947, | |
| "ranking_simple": 0.6541666388511658, | |
| "regularize": 0.11367591470479965, | |
| "step": 665 | |
| }, | |
| { | |
| "dpo_loss": 0.5561904311180115, | |
| "epoch": 3.79782711384034, | |
| "grad_norm": 18.08647866984471, | |
| "learning_rate": 8.183490657468686e-08, | |
| "logits": -1.7430044412612915, | |
| "logps": -92.36637878417969, | |
| "loss": 0.1153, | |
| "objective": 0.13086958229541779, | |
| "ranking_idealized": 0.7333333492279053, | |
| "ranking_idealized_expo": 0.5958333611488342, | |
| "ranking_simple": 0.7124999761581421, | |
| "regularize": 0.13086958229541779, | |
| "step": 670 | |
| }, | |
| { | |
| "dpo_loss": 0.5525475740432739, | |
| "epoch": 3.826169107227208, | |
| "grad_norm": 17.678219555630616, | |
| "learning_rate": 7.819930373330669e-08, | |
| "logits": -1.6892848014831543, | |
| "logps": -91.58055114746094, | |
| "loss": 0.1165, | |
| "objective": 0.10088498890399933, | |
| "ranking_idealized": 0.6708333492279053, | |
| "ranking_idealized_expo": 0.5458333492279053, | |
| "ranking_simple": 0.6416666507720947, | |
| "regularize": 0.10088498145341873, | |
| "step": 675 | |
| }, | |
| { | |
| "dpo_loss": 0.5708147883415222, | |
| "epoch": 3.8545111006140766, | |
| "grad_norm": 17.208804705028182, | |
| "learning_rate": 7.463127807341966e-08, | |
| "logits": -1.6462949514389038, | |
| "logps": -92.41487884521484, | |
| "loss": 0.1181, | |
| "objective": 0.11868777871131897, | |
| "ranking_idealized": 0.6541666388511658, | |
| "ranking_idealized_expo": 0.5083333253860474, | |
| "ranking_simple": 0.6208333373069763, | |
| "regularize": 0.11868777871131897, | |
| "step": 680 | |
| }, | |
| { | |
| "dpo_loss": 0.5442604422569275, | |
| "epoch": 3.8828530940009447, | |
| "grad_norm": 18.600166194890996, | |
| "learning_rate": 7.113223306499336e-08, | |
| "logits": -1.7259678840637207, | |
| "logps": -91.63528442382812, | |
| "loss": 0.1127, | |
| "objective": 0.10472600162029266, | |
| "ranking_idealized": 0.6875, | |
| "ranking_idealized_expo": 0.5166666507720947, | |
| "ranking_simple": 0.6499999761581421, | |
| "regularize": 0.10472600162029266, | |
| "step": 685 | |
| }, | |
| { | |
| "dpo_loss": 0.5496628284454346, | |
| "epoch": 3.9111950873878127, | |
| "grad_norm": 17.80496655704031, | |
| "learning_rate": 6.770354504470574e-08, | |
| "logits": -1.6540542840957642, | |
| "logps": -90.78262329101562, | |
| "loss": 0.1164, | |
| "objective": 0.10735266655683517, | |
| "ranking_idealized": 0.7041666507720947, | |
| "ranking_idealized_expo": 0.550000011920929, | |
| "ranking_simple": 0.6875, | |
| "regularize": 0.10735265165567398, | |
| "step": 690 | |
| }, | |
| { | |
| "dpo_loss": 0.5639461874961853, | |
| "epoch": 3.9395370807746812, | |
| "grad_norm": 16.492816826616206, | |
| "learning_rate": 6.434656267456842e-08, | |
| "logits": -1.6047898530960083, | |
| "logps": -92.38011932373047, | |
| "loss": 0.1193, | |
| "objective": 0.12910698354244232, | |
| "ranking_idealized": 0.6083333492279053, | |
| "ranking_idealized_expo": 0.46666666865348816, | |
| "ranking_simple": 0.5833333134651184, | |
| "regularize": 0.12910698354244232, | |
| "step": 695 | |
| }, | |
| { | |
| "dpo_loss": 0.5509793162345886, | |
| "epoch": 3.9678790741615493, | |
| "grad_norm": 19.17114576937693, | |
| "learning_rate": 6.106260641143546e-08, | |
| "logits": -1.6564711332321167, | |
| "logps": -92.65071868896484, | |
| "loss": 0.1146, | |
| "objective": 0.1030283123254776, | |
| "ranking_idealized": 0.612500011920929, | |
| "ranking_idealized_expo": 0.48750001192092896, | |
| "ranking_simple": 0.5958333611488342, | |
| "regularize": 0.103028304874897, | |
| "step": 700 | |
| }, | |
| { | |
| "epoch": 3.9678790741615493, | |
| "eval_dpo_loss": 0.6854080557823181, | |
| "eval_logits": -1.7319272756576538, | |
| "eval_logps": -99.89824676513672, | |
| "eval_loss": 0.4175797998905182, | |
| "eval_objective": 0.4186650514602661, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.5237603187561035, | |
| "eval_regularize": 0.4186650514602661, | |
| "eval_runtime": 258.7065, | |
| "eval_samples_per_second": 22.381, | |
| "eval_steps_per_second": 0.935, | |
| "step": 700 | |
| }, | |
| { | |
| "dpo_loss": 0.5571620464324951, | |
| "epoch": 3.9962210675484178, | |
| "grad_norm": 18.74884098276965, | |
| "learning_rate": 5.7852967987606e-08, | |
| "logits": -1.554320216178894, | |
| "logps": -90.9109878540039, | |
| "loss": 0.1135, | |
| "objective": 0.12702669203281403, | |
| "ranking_idealized": 0.6666666865348816, | |
| "ranking_idealized_expo": 0.5333333611488342, | |
| "ranking_simple": 0.6583333611488342, | |
| "regularize": 0.12702666223049164, | |
| "step": 705 | |
| }, | |
| { | |
| "dpo_loss": 0.5528541803359985, | |
| "epoch": 4.024563060935286, | |
| "grad_norm": 17.40621172754528, | |
| "learning_rate": 5.471890990272665e-08, | |
| "logits": -1.6468113660812378, | |
| "logps": -92.61128997802734, | |
| "loss": 0.1135, | |
| "objective": 0.1373264044523239, | |
| "ranking_idealized": 0.6916666626930237, | |
| "ranking_idealized_expo": 0.4791666567325592, | |
| "ranking_simple": 0.612500011920929, | |
| "regularize": 0.13732638955116272, | |
| "step": 710 | |
| }, | |
| { | |
| "dpo_loss": 0.5556226968765259, | |
| "epoch": 4.052905054322154, | |
| "grad_norm": 18.812501686123863, | |
| "learning_rate": 5.166166492719124e-08, | |
| "logits": -1.6049120426177979, | |
| "logps": -92.74799346923828, | |
| "loss": 0.1035, | |
| "objective": 0.10493499785661697, | |
| "ranking_idealized": 0.6833333373069763, | |
| "ranking_idealized_expo": 0.5416666865348816, | |
| "ranking_simple": 0.6458333134651184, | |
| "regularize": 0.10493497550487518, | |
| "step": 715 | |
| }, | |
| { | |
| "dpo_loss": 0.5424375534057617, | |
| "epoch": 4.081247047709022, | |
| "grad_norm": 17.435339866299028, | |
| "learning_rate": 4.868243561723534e-08, | |
| "logits": -1.511703372001648, | |
| "logps": -94.71248626708984, | |
| "loss": 0.0978, | |
| "objective": 0.09467672556638718, | |
| "ranking_idealized": 0.6583333611488342, | |
| "ranking_idealized_expo": 0.5208333134651184, | |
| "ranking_simple": 0.6166666746139526, | |
| "regularize": 0.09467671811580658, | |
| "step": 720 | |
| }, | |
| { | |
| "dpo_loss": 0.5652304291725159, | |
| "epoch": 4.109589041095891, | |
| "grad_norm": 17.507850551258127, | |
| "learning_rate": 4.578239384191529e-08, | |
| "logits": -1.6384118795394897, | |
| "logps": -92.1180191040039, | |
| "loss": 0.1076, | |
| "objective": 0.09941933304071426, | |
| "ranking_idealized": 0.6541666388511658, | |
| "ranking_idealized_expo": 0.5375000238418579, | |
| "ranking_simple": 0.637499988079071, | |
| "regularize": 0.09941932559013367, | |
| "step": 725 | |
| }, | |
| { | |
| "dpo_loss": 0.5497789978981018, | |
| "epoch": 4.137931034482759, | |
| "grad_norm": 18.803004868454263, | |
| "learning_rate": 4.296268032215733e-08, | |
| "logits": -1.7138111591339111, | |
| "logps": -91.83662414550781, | |
| "loss": 0.1086, | |
| "objective": 0.10822432488203049, | |
| "ranking_idealized": 0.675000011920929, | |
| "ranking_idealized_expo": 0.5375000238418579, | |
| "ranking_simple": 0.6208333373069763, | |
| "regularize": 0.10822432488203049, | |
| "step": 730 | |
| }, | |
| { | |
| "dpo_loss": 0.55525803565979, | |
| "epoch": 4.166273027869627, | |
| "grad_norm": 18.177348382836357, | |
| "learning_rate": 4.022440418205944e-08, | |
| "logits": -1.6232236623764038, | |
| "logps": -93.14463806152344, | |
| "loss": 0.1028, | |
| "objective": 0.10451411455869675, | |
| "ranking_idealized": 0.6791666746139526, | |
| "ranking_idealized_expo": 0.5291666388511658, | |
| "ranking_simple": 0.6541666388511658, | |
| "regularize": 0.10451411455869675, | |
| "step": 735 | |
| }, | |
| { | |
| "dpo_loss": 0.535234808921814, | |
| "epoch": 4.194615021256495, | |
| "grad_norm": 17.133543410858152, | |
| "learning_rate": 3.756864251262143e-08, | |
| "logits": -1.610323190689087, | |
| "logps": -93.36137390136719, | |
| "loss": 0.1062, | |
| "objective": 0.10550294071435928, | |
| "ranking_idealized": 0.6791666746139526, | |
| "ranking_idealized_expo": 0.5208333134651184, | |
| "ranking_simple": 0.6541666388511658, | |
| "regularize": 0.10550292581319809, | |
| "step": 740 | |
| }, | |
| { | |
| "dpo_loss": 0.546442449092865, | |
| "epoch": 4.222957014643363, | |
| "grad_norm": 18.78547392108143, | |
| "learning_rate": 3.4996439948074855e-08, | |
| "logits": -1.6879092454910278, | |
| "logps": -90.12301635742188, | |
| "loss": 0.1001, | |
| "objective": 0.10297367721796036, | |
| "ranking_idealized": 0.6291666626930237, | |
| "ranking_idealized_expo": 0.48750001192092896, | |
| "ranking_simple": 0.6208333373069763, | |
| "regularize": 0.10297366231679916, | |
| "step": 745 | |
| }, | |
| { | |
| "dpo_loss": 0.5416663289070129, | |
| "epoch": 4.251299008030231, | |
| "grad_norm": 19.462231175744662, | |
| "learning_rate": 3.250880825498026e-08, | |
| "logits": -1.8104737997055054, | |
| "logps": -92.32807922363281, | |
| "loss": 0.0986, | |
| "objective": 0.10099396854639053, | |
| "ranking_idealized": 0.6458333134651184, | |
| "ranking_idealized_expo": 0.5291666388511658, | |
| "ranking_simple": 0.6291666626930237, | |
| "regularize": 0.10099395364522934, | |
| "step": 750 | |
| }, | |
| { | |
| "epoch": 4.251299008030231, | |
| "eval_dpo_loss": 0.6853212714195251, | |
| "eval_logits": -1.7322306632995605, | |
| "eval_logps": -99.86943054199219, | |
| "eval_loss": 0.41747406125068665, | |
| "eval_objective": 0.41828420758247375, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.5237603187561035, | |
| "eval_regularize": 0.41828420758247375, | |
| "eval_runtime": 258.964, | |
| "eval_samples_per_second": 22.358, | |
| "eval_steps_per_second": 0.934, | |
| "step": 750 | |
| }, | |
| { | |
| "dpo_loss": 0.5468146800994873, | |
| "epoch": 4.2796410014171, | |
| "grad_norm": 18.270752967286892, | |
| "learning_rate": 3.010672593425209e-08, | |
| "logits": -1.7138711214065552, | |
| "logps": -92.11996459960938, | |
| "loss": 0.1147, | |
| "objective": 0.10049024224281311, | |
| "ranking_idealized": 0.6791666746139526, | |
| "ranking_idealized_expo": 0.5333333611488342, | |
| "ranking_simple": 0.6458333134651184, | |
| "regularize": 0.10049023479223251, | |
| "step": 755 | |
| }, | |
| { | |
| "dpo_loss": 0.5455428957939148, | |
| "epoch": 4.307982994803968, | |
| "grad_norm": 17.285027330879423, | |
| "learning_rate": 2.7791137836269158e-08, | |
| "logits": -1.6757961511611938, | |
| "logps": -93.05391693115234, | |
| "loss": 0.0961, | |
| "objective": 0.10086000710725784, | |
| "ranking_idealized": 0.7083333134651184, | |
| "ranking_idealized_expo": 0.5666666626930237, | |
| "ranking_simple": 0.6833333373069763, | |
| "regularize": 0.10086000710725784, | |
| "step": 760 | |
| }, | |
| { | |
| "dpo_loss": 0.5395826101303101, | |
| "epoch": 4.336324988190836, | |
| "grad_norm": 17.680760382133624, | |
| "learning_rate": 2.556295478922116e-08, | |
| "logits": -1.7200431823730469, | |
| "logps": -93.1734619140625, | |
| "loss": 0.1053, | |
| "objective": 0.12091321498155594, | |
| "ranking_idealized": 0.7291666865348816, | |
| "ranking_idealized_expo": 0.5291666388511658, | |
| "ranking_simple": 0.6833333373069763, | |
| "regularize": 0.12091320008039474, | |
| "step": 765 | |
| }, | |
| { | |
| "dpo_loss": 0.5380468964576721, | |
| "epoch": 4.364666981577704, | |
| "grad_norm": 17.361908211383366, | |
| "learning_rate": 2.3423053240837514e-08, | |
| "logits": -1.577264428138733, | |
| "logps": -91.18030548095703, | |
| "loss": 0.1064, | |
| "objective": 0.11028440296649933, | |
| "ranking_idealized": 0.6666666865348816, | |
| "ranking_idealized_expo": 0.5083333253860474, | |
| "ranking_simple": 0.637499988079071, | |
| "regularize": 0.11028438061475754, | |
| "step": 770 | |
| }, | |
| { | |
| "dpo_loss": 0.5494747161865234, | |
| "epoch": 4.393008974964572, | |
| "grad_norm": 16.640977636984772, | |
| "learning_rate": 2.137227491364016e-08, | |
| "logits": -1.627792239189148, | |
| "logps": -91.97000885009766, | |
| "loss": 0.1067, | |
| "objective": 0.10595239698886871, | |
| "ranking_idealized": 0.6666666865348816, | |
| "ranking_idealized_expo": 0.512499988079071, | |
| "ranking_simple": 0.6333333253860474, | |
| "regularize": 0.10595235228538513, | |
| "step": 775 | |
| }, | |
| { | |
| "dpo_loss": 0.5575358867645264, | |
| "epoch": 4.42135096835144, | |
| "grad_norm": 17.112345268128863, | |
| "learning_rate": 1.9411426473854687e-08, | |
| "logits": -1.693690538406372, | |
| "logps": -90.5418472290039, | |
| "loss": 0.1004, | |
| "objective": 0.10788667947053909, | |
| "ranking_idealized": 0.6541666388511658, | |
| "ranking_idealized_expo": 0.5041666626930237, | |
| "ranking_simple": 0.6208333373069763, | |
| "regularize": 0.10788667947053909, | |
| "step": 780 | |
| }, | |
| { | |
| "dpo_loss": 0.5676646828651428, | |
| "epoch": 4.449692961738309, | |
| "grad_norm": 16.657419373072543, | |
| "learning_rate": 1.7541279214111275e-08, | |
| "logits": -1.7215303182601929, | |
| "logps": -90.63499450683594, | |
| "loss": 0.1128, | |
| "objective": 0.1177934780716896, | |
| "ranking_idealized": 0.6583333611488342, | |
| "ranking_idealized_expo": 0.5291666388511658, | |
| "ranking_simple": 0.612500011920929, | |
| "regularize": 0.1177934780716896, | |
| "step": 785 | |
| }, | |
| { | |
| "dpo_loss": 0.539345920085907, | |
| "epoch": 4.478034955125177, | |
| "grad_norm": 17.089680176209615, | |
| "learning_rate": 1.57625687500596e-08, | |
| "logits": -1.6345340013504028, | |
| "logps": -93.4063720703125, | |
| "loss": 0.0998, | |
| "objective": 0.10817180573940277, | |
| "ranking_idealized": 0.7291666865348816, | |
| "ranking_idealized_expo": 0.5583333373069763, | |
| "ranking_simple": 0.6916666626930237, | |
| "regularize": 0.10817176848649979, | |
| "step": 790 | |
| }, | |
| { | |
| "dpo_loss": 0.547528088092804, | |
| "epoch": 4.506376948512045, | |
| "grad_norm": 17.320194779122446, | |
| "learning_rate": 1.4075994731016894e-08, | |
| "logits": -1.5627334117889404, | |
| "logps": -93.30286407470703, | |
| "loss": 0.1058, | |
| "objective": 0.102629154920578, | |
| "ranking_idealized": 0.6541666388511658, | |
| "ranking_idealized_expo": 0.550000011920929, | |
| "ranking_simple": 0.6291666626930237, | |
| "regularize": 0.10262913256883621, | |
| "step": 795 | |
| }, | |
| { | |
| "dpo_loss": 0.5486911535263062, | |
| "epoch": 4.534718941898913, | |
| "grad_norm": 17.9229448629881, | |
| "learning_rate": 1.2482220564763667e-08, | |
| "logits": -1.5870776176452637, | |
| "logps": -92.80538940429688, | |
| "loss": 0.1042, | |
| "objective": 0.11460768431425095, | |
| "ranking_idealized": 0.6875, | |
| "ranking_idealized_expo": 0.512499988079071, | |
| "ranking_simple": 0.6499999761581421, | |
| "regularize": 0.11460768431425095, | |
| "step": 800 | |
| }, | |
| { | |
| "epoch": 4.534718941898913, | |
| "eval_dpo_loss": 0.6852768659591675, | |
| "eval_logits": -1.7317209243774414, | |
| "eval_logps": -99.85995483398438, | |
| "eval_loss": 0.4175398349761963, | |
| "eval_objective": 0.4183157980442047, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.5237603187561035, | |
| "eval_regularize": 0.4183157980442047, | |
| "eval_runtime": 259.0856, | |
| "eval_samples_per_second": 22.348, | |
| "eval_steps_per_second": 0.934, | |
| "step": 800 | |
| }, | |
| { | |
| "dpo_loss": 0.5488670468330383, | |
| "epoch": 4.563060935285781, | |
| "grad_norm": 18.799592478883184, | |
| "learning_rate": 1.0981873156594379e-08, | |
| "logits": -1.627816081047058, | |
| "logps": -91.32179260253906, | |
| "loss": 0.1001, | |
| "objective": 0.10988225042819977, | |
| "ranking_idealized": 0.6458333134651184, | |
| "ranking_idealized_expo": 0.46666666865348816, | |
| "ranking_simple": 0.6041666865348816, | |
| "regularize": 0.10988224297761917, | |
| "step": 805 | |
| }, | |
| { | |
| "dpo_loss": 0.5488799810409546, | |
| "epoch": 4.59140292867265, | |
| "grad_norm": 16.566296936758206, | |
| "learning_rate": 9.575542662726754e-09, | |
| "logits": -1.7243562936782837, | |
| "logps": -91.10765075683594, | |
| "loss": 0.0996, | |
| "objective": 0.08869278430938721, | |
| "ranking_idealized": 0.5874999761581421, | |
| "ranking_idealized_expo": 0.4625000059604645, | |
| "ranking_simple": 0.5833333134651184, | |
| "regularize": 0.08869277685880661, | |
| "step": 810 | |
| }, | |
| { | |
| "dpo_loss": 0.5502530336380005, | |
| "epoch": 4.619744922059518, | |
| "grad_norm": 16.948787644578637, | |
| "learning_rate": 8.263782258165819e-09, | |
| "logits": -1.5700196027755737, | |
| "logps": -92.37843322753906, | |
| "loss": 0.0991, | |
| "objective": 0.07977009564638138, | |
| "ranking_idealized": 0.7250000238418579, | |
| "ranking_idealized_expo": 0.5916666388511658, | |
| "ranking_simple": 0.6958333253860474, | |
| "regularize": 0.07977008074522018, | |
| "step": 815 | |
| }, | |
| { | |
| "dpo_loss": 0.5550402402877808, | |
| "epoch": 4.648086915446386, | |
| "grad_norm": 21.501267763535818, | |
| "learning_rate": 7.047107919114586e-09, | |
| "logits": -1.6636712551116943, | |
| "logps": -92.61454010009766, | |
| "loss": 0.0982, | |
| "objective": 0.08367303013801575, | |
| "ranking_idealized": 0.6875, | |
| "ranking_idealized_expo": 0.5625, | |
| "ranking_simple": 0.6708333492279053, | |
| "regularize": 0.08367302268743515, | |
| "step": 820 | |
| }, | |
| { | |
| "dpo_loss": 0.5588962435722351, | |
| "epoch": 4.6764289088332545, | |
| "grad_norm": 16.315710057694485, | |
| "learning_rate": 5.925998220016659e-09, | |
| "logits": -1.5499807596206665, | |
| "logps": -90.22130584716797, | |
| "loss": 0.1018, | |
| "objective": 0.09301813691854477, | |
| "ranking_idealized": 0.6166666746139526, | |
| "ranking_idealized_expo": 0.5, | |
| "ranking_simple": 0.5833333134651184, | |
| "regularize": 0.09301812201738358, | |
| "step": 825 | |
| }, | |
| { | |
| "dpo_loss": 0.5498708486557007, | |
| "epoch": 4.7047709022201225, | |
| "grad_norm": 17.748669049129045, | |
| "learning_rate": 4.9008941453107525e-09, | |
| "logits": -1.7388263940811157, | |
| "logps": -92.17695617675781, | |
| "loss": 0.1092, | |
| "objective": 0.11468993872404099, | |
| "ranking_idealized": 0.675000011920929, | |
| "ranking_idealized_expo": 0.5166666507720947, | |
| "ranking_simple": 0.6416666507720947, | |
| "regularize": 0.1146899089217186, | |
| "step": 830 | |
| }, | |
| { | |
| "dpo_loss": 0.5327169299125671, | |
| "epoch": 4.733112895606991, | |
| "grad_norm": 17.389468155390862, | |
| "learning_rate": 3.9721989159709754e-09, | |
| "logits": -1.6580873727798462, | |
| "logps": -92.22929382324219, | |
| "loss": 0.1, | |
| "objective": 0.09779965132474899, | |
| "ranking_idealized": 0.6499999761581421, | |
| "ranking_idealized_expo": 0.48750001192092896, | |
| "ranking_simple": 0.6041666865348816, | |
| "regularize": 0.0977996364235878, | |
| "step": 835 | |
| }, | |
| { | |
| "dpo_loss": 0.5324665904045105, | |
| "epoch": 4.7614548889938595, | |
| "grad_norm": 18.51697686947663, | |
| "learning_rate": 3.140277830901428e-09, | |
| "logits": -1.6570351123809814, | |
| "logps": -92.75865173339844, | |
| "loss": 0.1079, | |
| "objective": 0.11856434494256973, | |
| "ranking_idealized": 0.6958333253860474, | |
| "ranking_idealized_expo": 0.5541666746139526, | |
| "ranking_simple": 0.6499999761581421, | |
| "regularize": 0.11856433004140854, | |
| "step": 840 | |
| }, | |
| { | |
| "dpo_loss": 0.560818612575531, | |
| "epoch": 4.7897968823807275, | |
| "grad_norm": 17.858581136510683, | |
| "learning_rate": 2.4054581232470785e-09, | |
| "logits": -1.6901015043258667, | |
| "logps": -92.13179016113281, | |
| "loss": 0.1067, | |
| "objective": 0.10368030518293381, | |
| "ranking_idealized": 0.6416666507720947, | |
| "ranking_idealized_expo": 0.4583333432674408, | |
| "ranking_simple": 0.6208333373069763, | |
| "regularize": 0.10368029028177261, | |
| "step": 845 | |
| }, | |
| { | |
| "dpo_loss": 0.5496495366096497, | |
| "epoch": 4.818138875767596, | |
| "grad_norm": 17.640915580271592, | |
| "learning_rate": 1.7680288316779256e-09, | |
| "logits": -1.6190950870513916, | |
| "logps": -90.9464340209961, | |
| "loss": 0.103, | |
| "objective": 0.10453298687934875, | |
| "ranking_idealized": 0.6833333373069763, | |
| "ranking_idealized_expo": 0.5291666388511658, | |
| "ranking_simple": 0.6333333253860474, | |
| "regularize": 0.10453297942876816, | |
| "step": 850 | |
| }, | |
| { | |
| "epoch": 4.818138875767596, | |
| "eval_dpo_loss": 0.6852567791938782, | |
| "eval_logits": -1.732380986213684, | |
| "eval_logps": -99.89720153808594, | |
| "eval_loss": 0.4175875782966614, | |
| "eval_objective": 0.4183763563632965, | |
| "eval_ranking_idealized": 0.6570248007774353, | |
| "eval_ranking_idealized_expo": 0.5113636255264282, | |
| "eval_ranking_simple": 0.5237603187561035, | |
| "eval_regularize": 0.4183763563632965, | |
| "eval_runtime": 259.1903, | |
| "eval_samples_per_second": 22.339, | |
| "eval_steps_per_second": 0.934, | |
| "step": 850 | |
| }, | |
| { | |
| "dpo_loss": 0.5584205389022827, | |
| "epoch": 4.846480869154464, | |
| "grad_norm": 17.56984774908714, | |
| "learning_rate": 1.2282406866966078e-09, | |
| "logits": -1.6185228824615479, | |
| "logps": -91.83565521240234, | |
| "loss": 0.0974, | |
| "objective": 0.09289266169071198, | |
| "ranking_idealized": 0.7083333134651184, | |
| "ranking_idealized_expo": 0.5375000238418579, | |
| "ranking_simple": 0.675000011920929, | |
| "regularize": 0.09289265424013138, | |
| "step": 855 | |
| }, | |
| { | |
| "dpo_loss": 0.5458131432533264, | |
| "epoch": 4.874822862541333, | |
| "grad_norm": 17.337457908328606, | |
| "learning_rate": 7.863060120144316e-10, | |
| "logits": -1.5824497938156128, | |
| "logps": -91.32083892822266, | |
| "loss": 0.0959, | |
| "objective": 0.11178465932607651, | |
| "ranking_idealized": 0.7166666388511658, | |
| "ranking_idealized_expo": 0.550000011920929, | |
| "ranking_simple": 0.6791666746139526, | |
| "regularize": 0.11178465187549591, | |
| "step": 860 | |
| }, | |
| { | |
| "dpo_loss": 0.555813729763031, | |
| "epoch": 4.903164855928201, | |
| "grad_norm": 16.568497702615847, | |
| "learning_rate": 4.4239864103465254e-10, | |
| "logits": -1.6553268432617188, | |
| "logps": -90.40623474121094, | |
| "loss": 0.1003, | |
| "objective": 0.1271737664937973, | |
| "ranking_idealized": 0.6625000238418579, | |
| "ranking_idealized_expo": 0.4958333373069763, | |
| "ranking_simple": 0.6041666865348816, | |
| "regularize": 0.1271737515926361, | |
| "step": 865 | |
| }, | |
| { | |
| "dpo_loss": 0.5469813942909241, | |
| "epoch": 4.931506849315069, | |
| "grad_norm": 17.67813635168232, | |
| "learning_rate": 1.966538484758362e-10, | |
| "logits": -1.7142003774642944, | |
| "logps": -92.42487335205078, | |
| "loss": 0.1053, | |
| "objective": 0.10629518330097198, | |
| "ranking_idealized": 0.6916666626930237, | |
| "ranking_idealized_expo": 0.512499988079071, | |
| "ranking_simple": 0.6625000238418579, | |
| "regularize": 0.1062951609492302, | |
| "step": 870 | |
| }, | |
| { | |
| "dpo_loss": 0.5456808805465698, | |
| "epoch": 4.959848842701937, | |
| "grad_norm": 17.372715214830695, | |
| "learning_rate": 4.9168297161839014e-11, | |
| "logits": -1.6318602561950684, | |
| "logps": -92.3662109375, | |
| "loss": 0.1017, | |
| "objective": 0.08666170388460159, | |
| "ranking_idealized": 0.6833333373069763, | |
| "ranking_idealized_expo": 0.5458333492279053, | |
| "ranking_simple": 0.6541666388511658, | |
| "regularize": 0.086661696434021, | |
| "step": 875 | |
| }, | |
| { | |
| "dpo_loss": 0.5451498627662659, | |
| "epoch": 4.988190836088805, | |
| "grad_norm": 17.608890670600516, | |
| "learning_rate": 0.0, | |
| "logits": -1.7033004760742188, | |
| "logps": -92.65689849853516, | |
| "loss": 0.0938, | |
| "objective": 0.09012699872255325, | |
| "ranking_idealized": 0.6708333492279053, | |
| "ranking_idealized_expo": 0.5291666388511658, | |
| "ranking_simple": 0.6541666388511658, | |
| "regularize": 0.09012699127197266, | |
| "step": 880 | |
| }, | |
| { | |
| "epoch": 4.988190836088805, | |
| "step": 880, | |
| "total_flos": 0.0, | |
| "train_loss": 0.19899855256080629, | |
| "train_runtime": 35117.4941, | |
| "train_samples_per_second": 7.233, | |
| "train_steps_per_second": 0.025 | |
| } | |
| ], | |
| "logging_steps": 5, | |
| "max_steps": 880, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 5, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": false, | |
| "should_training_stop": false | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 4, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |