{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 4.988190836088805,
  "eval_steps": 50,
  "global_step": 880,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "dpo_loss": 0.6931471824645996,
      "epoch": 0.005668398677373642,
      "grad_norm": 13.413600039235007,
      "learning_rate": 5.681818181818181e-09,
      "logits": -1.3147305250167847,
      "logps": -88.0877456665039,
      "loss": 0.4113,
      "objective": 0.41588976979255676,
      "ranking_idealized": 0.6875,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.5208333134651184,
      "regularize": 0.41588976979255676,
      "step": 1
    },
    {
      "dpo_loss": 0.6931512951850891,
      "epoch": 0.02834199338686821,
      "grad_norm": 13.318320815280419,
      "learning_rate": 2.8409090909090908e-08,
      "logits": -1.3678321838378906,
      "logps": -84.44427490234375,
      "loss": 0.4131,
      "objective": 0.3755509555339813,
      "ranking_idealized": 0.6510416865348816,
      "ranking_idealized_expo": 0.5572916865348816,
      "ranking_simple": 0.546875,
      "regularize": 0.3755509555339813,
      "step": 5
    },
    {
      "dpo_loss": 0.6927531361579895,
      "epoch": 0.05668398677373642,
      "grad_norm": 13.050623089340824,
      "learning_rate": 5.6818181818181815e-08,
      "logits": -1.4463988542556763,
      "logps": -83.39988708496094,
      "loss": 0.4176,
      "objective": 0.4423220753669739,
      "ranking_idealized": 0.675000011920929,
      "ranking_idealized_expo": 0.512499988079071,
      "ranking_simple": 0.512499988079071,
      "regularize": 0.4423220753669739,
      "step": 10
    },
    {
      "dpo_loss": 0.6927918195724487,
      "epoch": 0.08502598016060463,
      "grad_norm": 12.549385306441062,
      "learning_rate": 8.522727272727271e-08,
      "logits": -1.4107797145843506,
      "logps": -83.50421905517578,
      "loss": 0.4254,
      "objective": 0.41179904341697693,
      "ranking_idealized": 0.6541666388511658,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.5083333253860474,
      "regularize": 0.41179904341697693,
      "step": 15
    },
    {
      "dpo_loss": 0.6924694776535034,
      "epoch": 0.11336797354747284,
      "grad_norm": 13.269620119946596,
      "learning_rate": 1.1363636363636363e-07,
      "logits": -1.4003115892410278,
      "logps": -84.06736755371094,
      "loss": 0.4149,
      "objective": 0.40317121148109436,
      "ranking_idealized": 0.6791666746139526,
      "ranking_idealized_expo": 0.5375000238418579,
      "ranking_simple": 0.5375000238418579,
      "regularize": 0.40317121148109436,
      "step": 20
    },
    {
      "dpo_loss": 0.6906281113624573,
      "epoch": 0.14170996693434104,
      "grad_norm": 12.65234373247132,
      "learning_rate": 1.4204545454545455e-07,
      "logits": -1.4490704536437988,
      "logps": -83.72380065917969,
      "loss": 0.412,
      "objective": 0.4304184317588806,
      "ranking_idealized": 0.675000011920929,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.5,
      "regularize": 0.4304184317588806,
      "step": 25
    },
    {
      "dpo_loss": 0.6906370520591736,
      "epoch": 0.17005196032120926,
      "grad_norm": 13.419812147505471,
      "learning_rate": 1.7045454545454543e-07,
      "logits": -1.4248003959655762,
      "logps": -84.09757232666016,
      "loss": 0.4126,
      "objective": 0.41593801975250244,
      "ranking_idealized": 0.6875,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.41593801975250244,
      "step": 30
    },
    {
      "dpo_loss": 0.6881809234619141,
      "epoch": 0.19839395370807747,
      "grad_norm": 13.431894879328123,
      "learning_rate": 1.9886363636363636e-07,
      "logits": -1.398374319076538,
      "logps": -82.60546112060547,
      "loss": 0.4095,
      "objective": 0.3929609954357147,
      "ranking_idealized": 0.6583333611488342,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.5083333253860474,
      "regularize": 0.3929609954357147,
      "step": 35
    },
    {
      "dpo_loss": 0.6868489384651184,
      "epoch": 0.22673594709494568,
      "grad_norm": 15.0250838416837,
      "learning_rate": 2.2727272727272726e-07,
      "logits": -1.3904410600662231,
      "logps": -82.84651947021484,
      "loss": 0.42,
      "objective": 0.43919187784194946,
      "ranking_idealized": 0.6625000238418579,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.43919187784194946,
      "step": 40
    },
    {
      "dpo_loss": 0.6844364404678345,
      "epoch": 0.25507794048181387,
      "grad_norm": 13.128806663839857,
      "learning_rate": 2.5568181818181816e-07,
      "logits": -1.5230154991149902,
      "logps": -84.21646118164062,
      "loss": 0.4194,
      "objective": 0.4717731177806854,
      "ranking_idealized": 0.7208333611488342,
      "ranking_idealized_expo": 0.5375000238418579,
      "ranking_simple": 0.5458333492279053,
      "regularize": 0.4717731177806854,
      "step": 45
    },
    {
      "dpo_loss": 0.6831071376800537,
      "epoch": 0.2834199338686821,
      "grad_norm": 12.39410793472882,
      "learning_rate": 2.840909090909091e-07,
      "logits": -1.431780219078064,
      "logps": -82.2941665649414,
      "loss": 0.4122,
      "objective": 0.3948862850666046,
      "ranking_idealized": 0.637499988079071,
      "ranking_idealized_expo": 0.42500001192092896,
      "ranking_simple": 0.44999998807907104,
      "regularize": 0.3948862850666046,
      "step": 50
    },
    {
      "epoch": 0.2834199338686821,
      "eval_dpo_loss": 0.6914567947387695,
      "eval_logits": -1.4614633321762085,
      "eval_logps": -90.56139373779297,
      "eval_loss": 0.4102250635623932,
      "eval_objective": 0.40930914878845215,
      "eval_ranking_idealized": 0.6570248007774353,
      "eval_ranking_idealized_expo": 0.5113636255264282,
      "eval_ranking_simple": 0.5123966932296753,
      "eval_regularize": 0.40930914878845215,
      "eval_runtime": 260.1383,
      "eval_samples_per_second": 22.257,
      "eval_steps_per_second": 0.93,
      "step": 50
    },
    {
      "dpo_loss": 0.6807647943496704,
      "epoch": 0.3117619272555503,
      "grad_norm": 13.979478083508853,
      "learning_rate": 3.1249999999999997e-07,
      "logits": -1.4781759977340698,
      "logps": -84.0101089477539,
      "loss": 0.4052,
      "objective": 0.4063163101673126,
      "ranking_idealized": 0.7124999761581421,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.5458333492279053,
      "regularize": 0.4063163101673126,
      "step": 55
    },
    {
      "dpo_loss": 0.6824926137924194,
      "epoch": 0.3401039206424185,
      "grad_norm": 13.484676530515722,
      "learning_rate": 3.4090909090909085e-07,
      "logits": -1.4679373502731323,
      "logps": -83.09486389160156,
      "loss": 0.3992,
      "objective": 0.387731671333313,
      "ranking_idealized": 0.7041666507720947,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.387731671333313,
      "step": 60
    },
    {
      "dpo_loss": 0.6788213849067688,
      "epoch": 0.3684459140292867,
      "grad_norm": 13.535493104004898,
      "learning_rate": 3.693181818181818e-07,
      "logits": -1.4250341653823853,
      "logps": -83.52283477783203,
      "loss": 0.3842,
      "objective": 0.3719988465309143,
      "ranking_idealized": 0.6291666626930237,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.5083333253860474,
      "regularize": 0.3719988465309143,
      "step": 65
    },
    {
      "dpo_loss": 0.6763210296630859,
      "epoch": 0.39678790741615494,
      "grad_norm": 13.25897931133664,
      "learning_rate": 3.977272727272727e-07,
      "logits": -1.5077797174453735,
      "logps": -85.39080047607422,
      "loss": 0.3855,
      "objective": 0.42043933272361755,
      "ranking_idealized": 0.6916666626930237,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5291666388511658,
      "regularize": 0.42043933272361755,
      "step": 70
    },
    {
      "dpo_loss": 0.678033709526062,
      "epoch": 0.42512990080302315,
      "grad_norm": 14.035157652400327,
      "learning_rate": 4.2613636363636364e-07,
      "logits": -1.5349814891815186,
      "logps": -86.0143051147461,
      "loss": 0.3945,
      "objective": 0.41438591480255127,
      "ranking_idealized": 0.6791666746139526,
      "ranking_idealized_expo": 0.5458333492279053,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.41438591480255127,
      "step": 75
    },
    {
      "dpo_loss": 0.6748775243759155,
      "epoch": 0.45347189418989137,
      "grad_norm": 13.539091864104346,
      "learning_rate": 4.545454545454545e-07,
      "logits": -1.5832253694534302,
      "logps": -85.59701538085938,
      "loss": 0.3789,
      "objective": 0.37422579526901245,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.37422579526901245,
      "step": 80
    },
    {
      "dpo_loss": 0.6750870943069458,
      "epoch": 0.4818138875767596,
      "grad_norm": 12.78905385712093,
      "learning_rate": 4.829545454545455e-07,
      "logits": -1.5551499128341675,
      "logps": -84.24475860595703,
      "loss": 0.374,
      "objective": 0.39821094274520874,
      "ranking_idealized": 0.6541666388511658,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.5375000238418579,
      "regularize": 0.39821094274520874,
      "step": 85
    },
    {
      "dpo_loss": 0.6703960299491882,
      "epoch": 0.5101558809636277,
      "grad_norm": 14.26040681218726,
      "learning_rate": 4.999921328558332e-07,
      "logits": -1.37662935256958,
      "logps": -86.21568298339844,
      "loss": 0.3761,
      "objective": 0.3837045729160309,
      "ranking_idealized": 0.6291666626930237,
      "ranking_idealized_expo": 0.4791666567325592,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.3837045729160309,
      "step": 90
    },
    {
      "dpo_loss": 0.658724844455719,
      "epoch": 0.538497874350496,
      "grad_norm": 14.003866252787525,
      "learning_rate": 4.999036331701828e-07,
      "logits": -1.4695987701416016,
      "logps": -85.49458312988281,
      "loss": 0.3642,
      "objective": 0.39033612608909607,
      "ranking_idealized": 0.6833333373069763,
      "ranking_idealized_expo": 0.5458333492279053,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.39033612608909607,
      "step": 95
    },
    {
      "dpo_loss": 0.6546652317047119,
      "epoch": 0.5668398677373642,
      "grad_norm": 13.876424116810778,
      "learning_rate": 4.99716834795752e-07,
      "logits": -1.5616024732589722,
      "logps": -86.23612213134766,
      "loss": 0.374,
      "objective": 0.4149954915046692,
      "ranking_idealized": 0.6708333492279053,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.5458333492279053,
      "regularize": 0.4149954915046692,
      "step": 100
    },
    {
      "epoch": 0.5668398677373642,
      "eval_dpo_loss": 0.6882808208465576,
      "eval_logits": -1.5521211624145508,
      "eval_logps": -92.03614807128906,
      "eval_loss": 0.4072900712490082,
      "eval_objective": 0.40819329023361206,
      "eval_ranking_idealized": 0.6570248007774353,
      "eval_ranking_idealized_expo": 0.5113636255264282,
      "eval_ranking_simple": 0.5144628286361694,
      "eval_regularize": 0.40819329023361206,
      "eval_runtime": 258.9254,
      "eval_samples_per_second": 22.362,
      "eval_steps_per_second": 0.935,
      "step": 100
    },
    {
      "dpo_loss": 0.6509627103805542,
      "epoch": 0.5951818611242324,
      "grad_norm": 15.031011715031442,
      "learning_rate": 4.994318112090048e-07,
      "logits": -1.4410721063613892,
      "logps": -85.85182189941406,
      "loss": 0.3711,
      "objective": 0.3764886260032654,
      "ranking_idealized": 0.7083333134651184,
      "ranking_idealized_expo": 0.5375000238418579,
      "ranking_simple": 0.574999988079071,
      "regularize": 0.3764886260032654,
      "step": 105
    },
    {
      "dpo_loss": 0.6576470136642456,
      "epoch": 0.6235238545111006,
      "grad_norm": 16.068373019347053,
      "learning_rate": 4.990486745229364e-07,
      "logits": -1.6439845561981201,
      "logps": -84.1036376953125,
      "loss": 0.3694,
      "objective": 0.39763620495796204,
      "ranking_idealized": 0.6541666388511658,
      "ranking_idealized_expo": 0.4583333432674408,
      "ranking_simple": 0.48750001192092896,
      "regularize": 0.39763620495796204,
      "step": 110
    },
    {
      "dpo_loss": 0.6543448567390442,
      "epoch": 0.6518658478979689,
      "grad_norm": 15.729444594038945,
      "learning_rate": 4.985675754429743e-07,
      "logits": -1.6000815629959106,
      "logps": -83.94436645507812,
      "loss": 0.3477,
      "objective": 0.3455929458141327,
      "ranking_idealized": 0.6583333611488342,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.5,
      "regularize": 0.3455929458141327,
      "step": 115
    },
    {
      "dpo_loss": 0.6514815092086792,
      "epoch": 0.680207841284837,
      "grad_norm": 14.396427732147952,
      "learning_rate": 4.979887032076988e-07,
      "logits": -1.5459378957748413,
      "logps": -85.23513793945312,
      "loss": 0.3534,
      "objective": 0.3168259561061859,
      "ranking_idealized": 0.6583333611488342,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5458333492279053,
      "regularize": 0.3168259561061859,
      "step": 120
    },
    {
      "dpo_loss": 0.6492612957954407,
      "epoch": 0.7085498346717053,
      "grad_norm": 16.706445645247783,
      "learning_rate": 4.973122855144065e-07,
      "logits": -1.5174397230148315,
      "logps": -86.0051040649414,
      "loss": 0.3448,
      "objective": 0.3529473543167114,
      "ranking_idealized": 0.699999988079071,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.5708333253860474,
      "regularize": 0.3529473543167114,
      "step": 125
    },
    {
      "dpo_loss": 0.65309077501297,
      "epoch": 0.7368918280585735,
      "grad_norm": 15.417556754357976,
      "learning_rate": 4.965385884295466e-07,
      "logits": -1.664696455001831,
      "logps": -85.23889923095703,
      "loss": 0.3464,
      "objective": 0.33712950348854065,
      "ranking_idealized": 0.637499988079071,
      "ranking_idealized_expo": 0.4541666805744171,
      "ranking_simple": 0.4958333373069763,
      "regularize": 0.33712950348854065,
      "step": 130
    },
    {
      "dpo_loss": 0.6549941301345825,
      "epoch": 0.7652338214454416,
      "grad_norm": 13.59480500578719,
      "learning_rate": 4.956679162840645e-07,
      "logits": -1.626897931098938,
      "logps": -86.90068817138672,
      "loss": 0.3309,
      "objective": 0.34302666783332825,
      "ranking_idealized": 0.6416666507720947,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5458333492279053,
      "regularize": 0.34302666783332825,
      "step": 135
    },
    {
      "dpo_loss": 0.6500818729400635,
      "epoch": 0.7935758148323099,
      "grad_norm": 14.79485288903614,
      "learning_rate": 4.947006115536947e-07,
      "logits": -1.523794412612915,
      "logps": -86.5340576171875,
      "loss": 0.3244,
      "objective": 0.3356337249279022,
      "ranking_idealized": 0.6291666626930237,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.3356337249279022,
      "step": 140
    },
    {
      "dpo_loss": 0.6436840295791626,
      "epoch": 0.821917808219178,
      "grad_norm": 14.29078834943314,
      "learning_rate": 4.936370547242482e-07,
      "logits": -1.5991618633270264,
      "logps": -86.87805938720703,
      "loss": 0.3315,
      "objective": 0.35039833188056946,
      "ranking_idealized": 0.6458333134651184,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.5,
      "regularize": 0.35039833188056946,
      "step": 145
    },
    {
      "dpo_loss": 0.6453251242637634,
      "epoch": 0.8502598016060463,
      "grad_norm": 14.662823673975787,
      "learning_rate": 4.924776641419512e-07,
      "logits": -1.5607432126998901,
      "logps": -84.04727935791016,
      "loss": 0.3231,
      "objective": 0.32859519124031067,
      "ranking_idealized": 0.6916666626930237,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.32859519124031067,
      "step": 150
    },
    {
      "epoch": 0.8502598016060463,
      "eval_dpo_loss": 0.6880838871002197,
      "eval_logits": -1.6073634624481201,
      "eval_logps": -92.8072509765625,
      "eval_loss": 0.40292537212371826,
      "eval_objective": 0.4087039530277252,
      "eval_ranking_idealized": 0.6570248007774353,
      "eval_ranking_idealized_expo": 0.5113636255264282,
      "eval_ranking_simple": 0.5185950398445129,
      "eval_regularize": 0.4087039530277252,
      "eval_runtime": 258.55,
      "eval_samples_per_second": 22.394,
      "eval_steps_per_second": 0.936,
      "step": 150
    },
    {
      "dpo_loss": 0.6386777758598328,
      "epoch": 0.8786017949929145,
      "grad_norm": 14.231064502358223,
      "learning_rate": 4.912228958488892e-07,
      "logits": -1.5854390859603882,
      "logps": -84.10832214355469,
      "loss": 0.3257,
      "objective": 0.3301841616630554,
      "ranking_idealized": 0.6791666746139526,
      "ranking_idealized_expo": 0.512499988079071,
      "ranking_simple": 0.5916666388511658,
      "regularize": 0.3301841616630554,
      "step": 155
    },
    {
      "dpo_loss": 0.635101854801178,
      "epoch": 0.9069437883797827,
      "grad_norm": 16.2268120086952,
      "learning_rate": 4.898732434036243e-07,
      "logits": -1.4904930591583252,
      "logps": -86.09799194335938,
      "loss": 0.3107,
      "objective": 0.32787373661994934,
      "ranking_idealized": 0.6625000238418579,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.5458333492279053,
      "regularize": 0.32787373661994934,
      "step": 160
    },
    {
      "dpo_loss": 0.632634162902832,
      "epoch": 0.9352857817666509,
      "grad_norm": 16.041101199008867,
      "learning_rate": 4.884292376870567e-07,
      "logits": -1.5242409706115723,
      "logps": -86.48987579345703,
      "loss": 0.3212,
      "objective": 0.3137226700782776,
      "ranking_idealized": 0.6541666388511658,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.574999988079071,
      "regularize": 0.3137226700782776,
      "step": 165
    },
    {
      "dpo_loss": 0.6288425922393799,
      "epoch": 0.9636277751535192,
      "grad_norm": 16.89173365453321,
      "learning_rate": 4.868914466936037e-07,
      "logits": -1.5360677242279053,
      "logps": -86.72618103027344,
      "loss": 0.3151,
      "objective": 0.30297866463661194,
      "ranking_idealized": 0.6291666626930237,
      "ranking_idealized_expo": 0.512499988079071,
      "ranking_simple": 0.5708333253860474,
      "regularize": 0.30297866463661194,
      "step": 170
    },
    {
      "dpo_loss": 0.6335378289222717,
      "epoch": 0.9919697685403873,
      "grad_norm": 15.347989877166441,
      "learning_rate": 4.852604753077817e-07,
      "logits": -1.4790997505187988,
      "logps": -87.8569107055664,
      "loss": 0.3103,
      "objective": 0.29884618520736694,
      "ranking_idealized": 0.6833333373069763,
      "ranking_idealized_expo": 0.574999988079071,
      "ranking_simple": 0.6166666746139526,
      "regularize": 0.29884618520736694,
      "step": 175
    },
    {
      "dpo_loss": 0.6220327615737915,
      "epoch": 1.0203117619272555,
      "grad_norm": 15.022798279596007,
      "learning_rate": 4.835369650662767e-07,
      "logits": -1.6809762716293335,
      "logps": -87.00578308105469,
      "loss": 0.2902,
      "objective": 0.3023075461387634,
      "ranking_idealized": 0.6833333373069763,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.5874999761581421,
      "regularize": 0.3023075461387634,
      "step": 180
    },
    {
      "dpo_loss": 0.6156979203224182,
      "epoch": 1.0486537553141237,
      "grad_norm": 17.673807880039096,
      "learning_rate": 4.817215939055985e-07,
      "logits": -1.54806387424469,
      "logps": -86.16964721679688,
      "loss": 0.2856,
      "objective": 0.30200377106666565,
      "ranking_idealized": 0.6541666388511658,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.5416666865348816,
      "regularize": 0.30200377106666565,
      "step": 185
    },
    {
      "dpo_loss": 0.6105552315711975,
      "epoch": 1.076995748700992,
      "grad_norm": 16.47130075175902,
      "learning_rate": 4.798150758954164e-07,
      "logits": -1.6065795421600342,
      "logps": -88.57856750488281,
      "loss": 0.2661,
      "objective": 0.23887412250041962,
      "ranking_idealized": 0.6916666626930237,
      "ranking_idealized_expo": 0.5541666746139526,
      "ranking_simple": 0.6041666865348816,
      "regularize": 0.23887412250041962,
      "step": 190
    },
    {
      "dpo_loss": 0.6094806790351868,
      "epoch": 1.10533774208786,
      "grad_norm": 15.979183042956787,
      "learning_rate": 4.778181609576831e-07,
      "logits": -1.58108651638031,
      "logps": -86.33049011230469,
      "loss": 0.2734,
      "objective": 0.23615716397762299,
      "ranking_idealized": 0.7041666507720947,
      "ranking_idealized_expo": 0.5708333253860474,
      "ranking_simple": 0.6333333253860474,
      "regularize": 0.23615716397762299,
      "step": 195
    },
    {
      "dpo_loss": 0.6135362982749939,
      "epoch": 1.1336797354747283,
      "grad_norm": 15.241345178579065,
      "learning_rate": 4.757316345716553e-07,
      "logits": -1.6668376922607422,
      "logps": -85.64834594726562,
      "loss": 0.267,
      "objective": 0.2661064565181732,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.5541666746139526,
      "regularize": 0.2661064565181732,
      "step": 200
    },
    {
      "epoch": 1.1336797354747283,
      "eval_dpo_loss": 0.6866354942321777,
      "eval_logits": -1.6423935890197754,
      "eval_logps": -94.7991943359375,
      "eval_loss": 0.4068562686443329,
      "eval_objective": 0.41099515557289124,
      "eval_ranking_idealized": 0.6570248007774353,
      "eval_ranking_idealized_expo": 0.5113636255264282,
      "eval_ranking_simple": 0.5185950398445129,
      "eval_regularize": 0.41099515557289124,
      "eval_runtime": 258.6603,
      "eval_samples_per_second": 22.385,
      "eval_steps_per_second": 0.936,
      "step": 200
    },
    {
      "dpo_loss": 0.6104326844215393,
      "epoch": 1.1620217288615966,
      "grad_norm": 16.09148449696529,
      "learning_rate": 4.735563174649278e-07,
      "logits": -1.6373622417449951,
      "logps": -88.22838592529297,
      "loss": 0.2748,
      "objective": 0.2591724991798401,
      "ranking_idealized": 0.6458333134651184,
      "ranking_idealized_expo": 0.48750001192092896,
      "ranking_simple": 0.5541666746139526,
      "regularize": 0.2591724991798401,
      "step": 205
    },
    {
      "dpo_loss": 0.6027090549468994,
      "epoch": 1.1903637222484649,
      "grad_norm": 16.78316844909737,
      "learning_rate": 4.7129306529060407e-07,
      "logits": -1.604967474937439,
      "logps": -88.00846099853516,
      "loss": 0.2647,
      "objective": 0.28820380568504333,
      "ranking_idealized": 0.6291666626930237,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5541666746139526,
      "regularize": 0.28820380568504333,
      "step": 210
    },
    {
      "dpo_loss": 0.6120165586471558,
      "epoch": 1.2187057156353331,
      "grad_norm": 17.38824297135803,
      "learning_rate": 4.6894276829072786e-07,
      "logits": -1.577252745628357,
      "logps": -88.0232925415039,
      "loss": 0.2457,
      "objective": 0.2474772185087204,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.48750001192092896,
      "ranking_simple": 0.5625,
      "regularize": 0.2474772185087204,
      "step": 215
    },
    {
      "dpo_loss": 0.6058060526847839,
      "epoch": 1.2470477090222012,
      "grad_norm": 16.33918535416867,
      "learning_rate": 4.6650635094610966e-07,
      "logits": -1.5445390939712524,
      "logps": -87.7970199584961,
      "loss": 0.25,
      "objective": 0.23831520974636078,
      "ranking_idealized": 0.7208333611488342,
      "ranking_idealized_expo": 0.6041666865348816,
      "ranking_simple": 0.637499988079071,
      "regularize": 0.23831520974636078,
      "step": 220
    },
    {
      "dpo_loss": 0.6021844744682312,
      "epoch": 1.2753897024090695,
      "grad_norm": 14.989005292751132,
      "learning_rate": 4.639847716126854e-07,
      "logits": -1.6192957162857056,
      "logps": -89.04407501220703,
      "loss": 0.2607,
      "objective": 0.26420968770980835,
      "ranking_idealized": 0.6791666746139526,
      "ranking_idealized_expo": 0.5291666388511658,
      "ranking_simple": 0.6041666865348816,
      "regularize": 0.26420965790748596,
      "step": 225
    },
    {
      "dpo_loss": 0.6108235120773315,
      "epoch": 1.3037316957959377,
      "grad_norm": 16.245321246774985,
      "learning_rate": 4.6137902214455106e-07,
      "logits": -1.5698676109313965,
      "logps": -89.06554412841797,
      "loss": 0.2454,
      "objective": 0.24457047879695892,
      "ranking_idealized": 0.6541666388511658,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.5541666746139526,
      "regularize": 0.24457047879695892,
      "step": 230
    },
    {
      "dpo_loss": 0.59207683801651,
      "epoch": 1.3320736891828058,
      "grad_norm": 16.42864016636988,
      "learning_rate": 4.5869012750382004e-07,
      "logits": -1.6616859436035156,
      "logps": -87.82197570800781,
      "loss": 0.2583,
      "objective": 0.26300859451293945,
      "ranking_idealized": 0.6875,
      "ranking_idealized_expo": 0.48750001192092896,
      "ranking_simple": 0.6041666865348816,
      "regularize": 0.26300859451293945,
      "step": 235
    },
    {
      "dpo_loss": 0.5991641283035278,
      "epoch": 1.360415682569674,
      "grad_norm": 17.146089761318706,
      "learning_rate": 4.5591914535745817e-07,
      "logits": -1.5948702096939087,
      "logps": -89.31143188476562,
      "loss": 0.2442,
      "objective": 0.25130581855773926,
      "ranking_idealized": 0.637499988079071,
      "ranking_idealized_expo": 0.4791666567325592,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.25130581855773926,
      "step": 240
    },
    {
      "dpo_loss": 0.6016849279403687,
      "epoch": 1.3887576759565423,
      "grad_norm": 14.997203138603757,
      "learning_rate": 4.5306716566125433e-07,
      "logits": -1.6367671489715576,
      "logps": -88.17431640625,
      "loss": 0.2399,
      "objective": 0.23935823142528534,
      "ranking_idealized": 0.6833333373069763,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.23935823142528534,
      "step": 245
    },
    {
      "dpo_loss": 0.5918813943862915,
      "epoch": 1.4170996693434104,
      "grad_norm": 15.668454928081044,
      "learning_rate": 4.501353102310901e-07,
      "logits": -1.5877238512039185,
      "logps": -87.66322326660156,
      "loss": 0.2432,
      "objective": 0.2531537711620331,
      "ranking_idealized": 0.6625000238418579,
      "ranking_idealized_expo": 0.4749999940395355,
      "ranking_simple": 0.5625,
      "regularize": 0.2531537711620331,
      "step": 250
    },
    {
      "epoch": 1.4170996693434104,
      "eval_dpo_loss": 0.6876620650291443,
      "eval_logits": -1.6720653772354126,
      "eval_logps": -96.13894653320312,
      "eval_loss": 0.4107522666454315,
      "eval_objective": 0.4137335419654846,
      "eval_ranking_idealized": 0.6570248007774353,
      "eval_ranking_idealized_expo": 0.5113636255264282,
      "eval_ranking_simple": 0.51962810754776,
      "eval_regularize": 0.4137335419654846,
      "eval_runtime": 259.3309,
      "eval_samples_per_second": 22.327,
      "eval_steps_per_second": 0.933,
      "step": 250
    },
    {
      "dpo_loss": 0.5952737927436829,
      "epoch": 1.4454416627302786,
      "grad_norm": 16.40280338029817,
      "learning_rate": 4.471247323016777e-07,
      "logits": -1.5863794088363647,
      "logps": -89.24433898925781,
      "loss": 0.2442,
      "objective": 0.2290959656238556,
      "ranking_idealized": 0.675000011920929,
      "ranking_idealized_expo": 0.5458333492279053,
      "ranking_simple": 0.6291666626930237,
      "regularize": 0.2290959656238556,
      "step": 255
    },
    {
      "dpo_loss": 0.5954132080078125,
      "epoch": 1.473783656117147,
      "grad_norm": 16.11674277744465,
      "learning_rate": 4.440366160729392e-07,
      "logits": -1.6588572263717651,
      "logps": -89.44280242919922,
      "loss": 0.2443,
      "objective": 0.2354036122560501,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.48750001192092896,
      "ranking_simple": 0.5625,
      "regularize": 0.2354036122560501,
      "step": 260
    },
    {
      "dpo_loss": 0.5868396759033203,
      "epoch": 1.5021256495040152,
      "grad_norm": 16.753008834337265,
      "learning_rate": 4.4087217624420585e-07,
      "logits": -1.6106855869293213,
      "logps": -88.14371490478516,
      "loss": 0.239,
      "objective": 0.24956756830215454,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.5958333611488342,
      "regularize": 0.24956756830215454,
      "step": 265
    },
    {
      "dpo_loss": 0.5915893316268921,
      "epoch": 1.5304676428908834,
      "grad_norm": 17.579129679111187,
      "learning_rate": 4.3763265753642055e-07,
      "logits": -1.6173158884048462,
      "logps": -90.8720703125,
      "loss": 0.2421,
      "objective": 0.24301743507385254,
      "ranking_idealized": 0.6791666746139526,
      "ranking_idealized_expo": 0.512499988079071,
      "ranking_simple": 0.5958333611488342,
      "regularize": 0.24301742017269135,
      "step": 270
    },
    {
      "dpo_loss": 0.5958731770515442,
      "epoch": 1.5588096362777515,
      "grad_norm": 16.101798479127662,
      "learning_rate": 4.34319334202531e-07,
      "logits": -1.6187034845352173,
      "logps": -90.12999725341797,
      "loss": 0.2448,
      "objective": 0.22899790108203888,
      "ranking_idealized": 0.6916666626930237,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.6208333373069763,
      "regularize": 0.22899790108203888,
      "step": 275
    },
    {
      "dpo_loss": 0.5842003226280212,
      "epoch": 1.5871516296646198,
      "grad_norm": 15.709789047148108,
      "learning_rate": 4.309335095262675e-07,
      "logits": -1.5244942903518677,
      "logps": -88.604248046875,
      "loss": 0.2411,
      "objective": 0.23850402235984802,
      "ranking_idealized": 0.675000011920929,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.23850402235984802,
      "step": 280
    },
    {
      "dpo_loss": 0.5853084921836853,
      "epoch": 1.615493623051488,
      "grad_norm": 16.6854633771705,
      "learning_rate": 4.274765153095007e-07,
      "logits": -1.6502856016159058,
      "logps": -89.77727508544922,
      "loss": 0.2219,
      "objective": 0.21514521539211273,
      "ranking_idealized": 0.675000011920929,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.6041666865348816,
      "regularize": 0.21514521539211273,
      "step": 285
    },
    {
      "dpo_loss": 0.5910848379135132,
      "epoch": 1.643835616438356,
      "grad_norm": 18.978761606300836,
      "learning_rate": 4.239497113483819e-07,
      "logits": -1.7089149951934814,
      "logps": -86.87386322021484,
      "loss": 0.2312,
      "objective": 0.23006680607795715,
      "ranking_idealized": 0.699999988079071,
      "ranking_idealized_expo": 0.5625,
      "ranking_simple": 0.6291666626930237,
      "regularize": 0.23006680607795715,
      "step": 290
    },
    {
      "dpo_loss": 0.5802692174911499,
      "epoch": 1.6721776098252243,
      "grad_norm": 16.652074965539576,
      "learning_rate": 4.203544848984728e-07,
      "logits": -1.5955086946487427,
      "logps": -86.49956512451172,
      "loss": 0.2276,
      "objective": 0.23742005228996277,
      "ranking_idealized": 0.6583333611488342,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.5791666507720947,
      "regularize": 0.23742005228996277,
      "step": 295
    },
    {
      "dpo_loss": 0.5916833281517029,
      "epoch": 1.7005196032120926,
      "grad_norm": 16.884463449554712,
      "learning_rate": 4.166922501290729e-07,
      "logits": -1.6546835899353027,
      "logps": -88.2989730834961,
      "loss": 0.2252,
      "objective": 0.26394858956336975,
      "ranking_idealized": 0.7083333134651184,
      "ranking_idealized_expo": 0.574999988079071,
      "ranking_simple": 0.6208333373069763,
      "regularize": 0.26394858956336975,
      "step": 300
    },
    {
      "epoch": 1.7005196032120926,
      "eval_dpo_loss": 0.6866207718849182,
      "eval_logits": -1.6648496389389038,
      "eval_logps": -95.62443542480469,
      "eval_loss": 0.410134494304657,
      "eval_objective": 0.4137687385082245,
      "eval_ranking_idealized": 0.6570248007774353,
      "eval_ranking_idealized_expo": 0.5113636255264282,
      "eval_ranking_simple": 0.5216942429542542,
      "eval_regularize": 0.4137687385082245,
      "eval_runtime": 259.4458,
      "eval_samples_per_second": 22.317,
      "eval_steps_per_second": 0.933,
      "step": 300
    },
    {
      "dpo_loss": 0.5952399373054504,
      "epoch": 1.7288615965989607,
      "grad_norm": 16.24562342201146,
      "learning_rate": 4.129644475669616e-07,
      "logits": -1.6116312742233276,
      "logps": -88.82595825195312,
      "loss": 0.218,
      "objective": 0.2242499738931656,
      "ranking_idealized": 0.6291666626930237,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.2242499738931656,
      "step": 305
    },
    {
      "dpo_loss": 0.583368182182312,
      "epoch": 1.7572035899858292,
      "grad_norm": 18.099666352463437,
      "learning_rate": 4.0917254352977206e-07,
      "logits": -1.7004183530807495,
      "logps": -87.11441040039062,
      "loss": 0.2283,
      "objective": 0.2325660139322281,
      "ranking_idealized": 0.6583333611488342,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.612500011920929,
      "regularize": 0.2325660139322281,
      "step": 310
    },
    {
      "dpo_loss": 0.5933206677436829,
      "epoch": 1.7855455833726972,
      "grad_norm": 16.545516113765466,
      "learning_rate": 4.053180295492202e-07,
      "logits": -1.602583408355713,
      "logps": -88.69900512695312,
      "loss": 0.2287,
      "objective": 0.21895338594913483,
      "ranking_idealized": 0.6541666388511658,
      "ranking_idealized_expo": 0.512499988079071,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.21895337104797363,
      "step": 315
    },
    {
      "dpo_loss": 0.5876157283782959,
      "epoch": 1.8138875767595655,
      "grad_norm": 18.487916312721516,
      "learning_rate": 4.0140242178441665e-07,
      "logits": -1.6777514219284058,
      "logps": -90.22407531738281,
      "loss": 0.2153,
      "objective": 0.20208925008773804,
      "ranking_idealized": 0.6625000238418579,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.5874999761581421,
      "regularize": 0.20208925008773804,
      "step": 320
    },
    {
      "dpo_loss": 0.5746586918830872,
      "epoch": 1.8422295701464337,
      "grad_norm": 18.670087833334332,
      "learning_rate": 3.9742726042549053e-07,
      "logits": -1.7464016675949097,
      "logps": -91.97502899169922,
      "loss": 0.219,
      "objective": 0.2114688903093338,
      "ranking_idealized": 0.7166666388511658,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.6416666507720947,
      "regularize": 0.2114688903093338,
      "step": 325
    },
    {
      "dpo_loss": 0.5767069458961487,
      "epoch": 1.8705715635333018,
      "grad_norm": 18.91235181922618,
      "learning_rate": 3.933941090877615e-07,
      "logits": -1.466091275215149,
      "logps": -90.11954498291016,
      "loss": 0.219,
      "objective": 0.2171897292137146,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.4749999940395355,
      "ranking_simple": 0.574999988079071,
      "regularize": 0.2171897292137146,
      "step": 330
    },
    {
      "dpo_loss": 0.5894278287887573,
      "epoch": 1.89891355692017,
      "grad_norm": 15.592599296406116,
      "learning_rate": 3.8930455419669744e-07,
      "logits": -1.6301844120025635,
      "logps": -89.44200134277344,
      "loss": 0.2112,
      "objective": 0.18907961249351501,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.18907961249351501,
      "step": 335
    },
    {
      "dpo_loss": 0.5853725075721741,
      "epoch": 1.9272555503070383,
      "grad_norm": 15.83418724261755,
      "learning_rate": 3.851602043638994e-07,
      "logits": -1.660121202468872,
      "logps": -91.48560333251953,
      "loss": 0.2026,
      "objective": 0.18658672273159027,
      "ranking_idealized": 0.7291666865348816,
      "ranking_idealized_expo": 0.5458333492279053,
      "ranking_simple": 0.6416666507720947,
      "regularize": 0.18658672273159027,
      "step": 340
    },
    {
      "dpo_loss": 0.5825453400611877,
      "epoch": 1.9555975436939064,
      "grad_norm": 16.366640560133238,
      "learning_rate": 3.809626897543604e-07,
      "logits": -1.657557725906372,
      "logps": -90.72650909423828,
      "loss": 0.1961,
      "objective": 0.18415075540542603,
      "ranking_idealized": 0.6458333134651184,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.6166666746139526,
      "regularize": 0.18415075540542603,
      "step": 345
    },
    {
      "dpo_loss": 0.5838915109634399,
      "epoch": 1.9839395370807746,
      "grad_norm": 17.651439137685784,
      "learning_rate": 3.7671366144524576e-07,
      "logits": -1.551125407218933,
      "logps": -91.74525451660156,
      "loss": 0.2082,
      "objective": 0.20508398115634918,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.49166667461395264,
      "ranking_simple": 0.5541666746139526,
      "regularize": 0.20508398115634918,
      "step": 350
    },
    {
      "epoch": 1.9839395370807746,
      "eval_dpo_loss": 0.6863144040107727,
      "eval_logits": -1.6988588571548462,
      "eval_logps": -97.52546691894531,
      "eval_loss": 0.4102429747581482,
      "eval_objective": 0.4131539762020111,
      "eval_ranking_idealized": 0.6570248007774353,
      "eval_ranking_idealized_expo": 0.5113636255264282,
      "eval_ranking_simple": 0.51962810754776,
      "eval_regularize": 0.4131539762020111,
      "eval_runtime": 258.5165,
      "eval_samples_per_second": 22.397,
      "eval_steps_per_second": 0.936,
      "step": 350
    },
    {
      "dpo_loss": 0.5745717287063599,
      "epoch": 2.012281530467643,
      "grad_norm": 15.784086525377202,
      "learning_rate": 3.724147907764478e-07,
      "logits": -1.5323989391326904,
      "logps": -90.18486785888672,
      "loss": 0.2055,
      "objective": 0.20713359117507935,
      "ranking_idealized": 0.699999988079071,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.6458333134651184,
      "regularize": 0.20713359117507935,
      "step": 355
    },
    {
      "dpo_loss": 0.5675494074821472,
      "epoch": 2.040623523854511,
      "grad_norm": 20.55210866626824,
      "learning_rate": 3.6806776869317067e-07,
      "logits": -1.6239458322525024,
      "logps": -89.69377899169922,
      "loss": 0.1726,
      "objective": 0.17787505686283112,
      "ranking_idealized": 0.7250000238418579,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.675000011920929,
      "regularize": 0.17787505686283112,
      "step": 360
    },
    {
      "dpo_loss": 0.5650666952133179,
      "epoch": 2.0689655172413794,
      "grad_norm": 16.544231581396616,
      "learning_rate": 3.636743050808028e-07,
      "logits": -1.6872822046279907,
      "logps": -91.26659393310547,
      "loss": 0.1866,
      "objective": 0.16895455121994019,
      "ranking_idealized": 0.6708333492279053,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.6333333253860474,
      "regularize": 0.16895455121994019,
      "step": 365
    },
    {
      "dpo_loss": 0.575705349445343,
      "epoch": 2.0973075106282475,
      "grad_norm": 16.758890304778106,
      "learning_rate": 3.5923612809233984e-07,
      "logits": -1.662663221359253,
      "logps": -87.82825469970703,
      "loss": 0.1679,
      "objective": 0.1752353459596634,
      "ranking_idealized": 0.6541666388511658,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.574999988079071,
      "regularize": 0.1752353310585022,
      "step": 370
    },
    {
      "dpo_loss": 0.5633853077888489,
      "epoch": 2.1256495040151155,
      "grad_norm": 16.99783941953761,
      "learning_rate": 3.5475498346862214e-07,
      "logits": -1.6271302700042725,
      "logps": -91.13916015625,
      "loss": 0.1726,
      "objective": 0.16911908984184265,
      "ranking_idealized": 0.7291666865348816,
      "ranking_idealized_expo": 0.5958333611488342,
      "ranking_simple": 0.6791666746139526,
      "regularize": 0.16911907494068146,
      "step": 375
    },
    {
      "dpo_loss": 0.5752108097076416,
      "epoch": 2.153991497401984,
      "grad_norm": 17.23340187781712,
      "learning_rate": 3.502326338516534e-07,
      "logits": -1.5394021272659302,
      "logps": -89.99533081054688,
      "loss": 0.179,
      "objective": 0.1650255024433136,
      "ranking_idealized": 0.6833333373069763,
      "ranking_idealized_expo": 0.5375000238418579,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.1650255024433136,
      "step": 380
    },
    {
      "dpo_loss": 0.571977972984314,
      "epoch": 2.182333490788852,
      "grad_norm": 15.78796183229778,
      "learning_rate": 3.4567085809127245e-07,
      "logits": -1.6716177463531494,
      "logps": -91.3305892944336,
      "loss": 0.1653,
      "objective": 0.13291777670383453,
      "ranking_idealized": 0.6458333134651184,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.612500011920929,
      "regularize": 0.13291777670383453,
      "step": 385
    },
    {
      "dpo_loss": 0.5752423405647278,
      "epoch": 2.21067548417572,
      "grad_norm": 18.129151048308177,
      "learning_rate": 3.4107145054544855e-07,
      "logits": -1.5358682870864868,
      "logps": -91.15263366699219,
      "loss": 0.1744,
      "objective": 0.16379062831401825,
      "ranking_idealized": 0.6041666865348816,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.5625,
      "regularize": 0.16379062831401825,
      "step": 390
    },
    {
      "dpo_loss": 0.5485681891441345,
      "epoch": 2.2390174775625886,
      "grad_norm": 16.313781937896024,
      "learning_rate": 3.3643622037447767e-07,
      "logits": -1.5593619346618652,
      "logps": -92.42921447753906,
      "loss": 0.1776,
      "objective": 0.1637614667415619,
      "ranking_idealized": 0.6916666626930237,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.6416666507720947,
      "regularize": 0.1637614667415619,
      "step": 395
    },
    {
      "dpo_loss": 0.5597947239875793,
      "epoch": 2.2673594709494567,
      "grad_norm": 16.659127876259,
      "learning_rate": 3.317669908293554e-07,
      "logits": -1.631813645362854,
      "logps": -92.92410278320312,
      "loss": 0.1825,
      "objective": 0.196553573012352,
      "ranking_idealized": 0.7583333253860474,
      "ranking_idealized_expo": 0.5791666507720947,
      "ranking_simple": 0.6958333253860474,
      "regularize": 0.1965535581111908,
      "step": 400
    },
    {
      "epoch": 2.2673594709494567,
      "eval_dpo_loss": 0.6862595677375793,
      "eval_logits": -1.6931663751602173,
      "eval_logps": -97.79962158203125,
      "eval_loss": 0.4124037027359009,
      "eval_objective": 0.4144473969936371,
      "eval_ranking_idealized": 0.6570248007774353,
      "eval_ranking_idealized_expo": 0.5113636255264282,
      "eval_ranking_simple": 0.5206611752510071,
      "eval_regularize": 0.4144473969936371,
      "eval_runtime": 258.4529,
      "eval_samples_per_second": 22.403,
      "eval_steps_per_second": 0.936,
      "step": 400
    },
    {
      "dpo_loss": 0.5691500902175903,
      "epoch": 2.295701464336325,
      "grad_norm": 17.30117286858182,
      "learning_rate": 3.270655985346081e-07,
      "logits": -1.7139372825622559,
      "logps": -89.78938293457031,
      "loss": 0.1717,
      "objective": 0.18224166333675385,
      "ranking_idealized": 0.737500011920929,
      "ranking_idealized_expo": 0.5583333373069763,
      "ranking_simple": 0.6458333134651184,
      "regularize": 0.18224166333675385,
      "step": 405
    },
    {
      "dpo_loss": 0.5579439997673035,
      "epoch": 2.324043457723193,
      "grad_norm": 17.123218301010457,
      "learning_rate": 3.223338927658632e-07,
      "logits": -1.5741162300109863,
      "logps": -91.07009887695312,
      "loss": 0.1618,
      "objective": 0.15759395062923431,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.5958333611488342,
      "regularize": 0.15759395062923431,
      "step": 410
    },
    {
      "dpo_loss": 0.5704253315925598,
      "epoch": 2.3523854511100613,
      "grad_norm": 17.68931154440285,
      "learning_rate": 3.175737347224432e-07,
      "logits": -1.6476367712020874,
      "logps": -91.30075073242188,
      "loss": 0.1732,
      "objective": 0.19281157851219177,
      "ranking_idealized": 0.6458333134651184,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.612500011920929,
      "regularize": 0.19281157851219177,
      "step": 415
    },
    {
      "dpo_loss": 0.5771389603614807,
      "epoch": 2.3807274444969297,
      "grad_norm": 17.274582557860825,
      "learning_rate": 3.1278699679526975e-07,
      "logits": -1.5415838956832886,
      "logps": -92.63572692871094,
      "loss": 0.1579,
      "objective": 0.15308959782123566,
      "ranking_idealized": 0.6458333134651184,
      "ranking_idealized_expo": 0.4749999940395355,
      "ranking_simple": 0.5958333611488342,
      "regularize": 0.15308959782123566,
      "step": 420
    },
    {
      "dpo_loss": 0.566936731338501,
      "epoch": 2.409069437883798,
      "grad_norm": 16.445557447346342,
      "learning_rate": 3.0797556183036575e-07,
      "logits": -1.5967096090316772,
      "logps": -91.4622802734375,
      "loss": 0.1607,
      "objective": 0.16068215668201447,
      "ranking_idealized": 0.6625000238418579,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.16068214178085327,
      "step": 425
    },
    {
      "dpo_loss": 0.5632474422454834,
      "epoch": 2.4374114312706663,
      "grad_norm": 15.962055488306607,
      "learning_rate": 3.0314132238824415e-07,
      "logits": -1.6247813701629639,
      "logps": -92.1604995727539,
      "loss": 0.1547,
      "objective": 0.1360505074262619,
      "ranking_idealized": 0.6708333492279053,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.625,
      "regularize": 0.1360505074262619,
      "step": 430
    },
    {
      "dpo_loss": 0.566851019859314,
      "epoch": 2.4657534246575343,
      "grad_norm": 16.006081940650837,
      "learning_rate": 2.982861799994764e-07,
      "logits": -1.6544443368911743,
      "logps": -92.63692474365234,
      "loss": 0.1637,
      "objective": 0.17756709456443787,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.6208333373069763,
      "regularize": 0.17756709456443787,
      "step": 435
    },
    {
      "dpo_loss": 0.5565729141235352,
      "epoch": 2.4940954180444024,
      "grad_norm": 17.37344722468487,
      "learning_rate": 2.934120444167326e-07,
      "logits": -1.5883994102478027,
      "logps": -91.88066101074219,
      "loss": 0.159,
      "objective": 0.15150482952594757,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5375000238418579,
      "ranking_simple": 0.6041666865348816,
      "regularize": 0.15150482952594757,
      "step": 440
    },
    {
      "dpo_loss": 0.565682590007782,
      "epoch": 2.5224374114312704,
      "grad_norm": 18.453788667979182,
      "learning_rate": 2.885208328635864e-07,
      "logits": -1.6123565435409546,
      "logps": -89.5006332397461,
      "loss": 0.1576,
      "objective": 0.1587233543395996,
      "ranking_idealized": 0.6958333253860474,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.6458333134651184,
      "regularize": 0.1587233543395996,
      "step": 445
    },
    {
      "dpo_loss": 0.5824019312858582,
      "epoch": 2.550779404818139,
      "grad_norm": 19.424550718198045,
      "learning_rate": 2.83614469280383e-07,
      "logits": -1.6537593603134155,
      "logps": -91.4095230102539,
      "loss": 0.1504,
      "objective": 0.15120406448841095,
      "ranking_idealized": 0.6583333611488342,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5791666507720947,
      "regularize": 0.15120406448841095,
      "step": 450
    },
    {
      "epoch": 2.550779404818139,
      "eval_dpo_loss": 0.6864377856254578,
      "eval_logits": -1.711348056793213,
      "eval_logps": -99.202880859375,
      "eval_loss": 0.41492125391960144,
      "eval_objective": 0.4176488518714905,
      "eval_ranking_idealized": 0.6570248007774353,
      "eval_ranking_idealized_expo": 0.5113636255264282,
      "eval_ranking_simple": 0.5216942429542542,
      "eval_regularize": 0.4176488518714905,
      "eval_runtime": 258.9375,
      "eval_samples_per_second": 22.361,
      "eval_steps_per_second": 0.935,
      "step": 450
    },
    {
      "dpo_loss": 0.5587320923805237,
      "epoch": 2.579121398205007,
      "grad_norm": 18.174711126742732,
      "learning_rate": 2.786948835674634e-07,
      "logits": -1.6923545598983765,
      "logps": -92.0631103515625,
      "loss": 0.1514,
      "objective": 0.15467478334903717,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.6416666507720947,
      "regularize": 0.15467478334903717,
      "step": 455
    },
    {
      "dpo_loss": 0.5529297590255737,
      "epoch": 2.6074633915918755,
      "grad_norm": 18.378396938924546,
      "learning_rate": 2.737640108260456e-07,
      "logits": -1.765284776687622,
      "logps": -92.5921401977539,
      "loss": 0.1544,
      "objective": 0.13981758058071136,
      "ranking_idealized": 0.6833333373069763,
      "ranking_idealized_expo": 0.512499988079071,
      "ranking_simple": 0.6291666626930237,
      "regularize": 0.13981756567955017,
      "step": 460
    },
    {
      "dpo_loss": 0.5604754090309143,
      "epoch": 2.6358053849787435,
      "grad_norm": 17.16312208138119,
      "learning_rate": 2.6882379059705953e-07,
      "logits": -1.6412590742111206,
      "logps": -91.83204650878906,
      "loss": 0.1571,
      "objective": 0.15992027521133423,
      "ranking_idealized": 0.612500011920929,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.5791666507720947,
      "regularize": 0.15992026031017303,
      "step": 465
    },
    {
      "dpo_loss": 0.5741956830024719,
      "epoch": 2.6641473783656116,
      "grad_norm": 17.444271577746782,
      "learning_rate": 2.6387616609823504e-07,
      "logits": -1.6750518083572388,
      "logps": -91.33477020263672,
      "loss": 0.151,
      "objective": 0.17329135537147522,
      "ranking_idealized": 0.6625000238418579,
      "ranking_idealized_expo": 0.5583333373069763,
      "ranking_simple": 0.625,
      "regularize": 0.17329135537147522,
      "step": 470
    },
    {
      "dpo_loss": 0.5746079087257385,
      "epoch": 2.69248937175248,
      "grad_norm": 17.607595627923466,
      "learning_rate": 2.5892308345974514e-07,
      "logits": -1.6217347383499146,
      "logps": -90.19564819335938,
      "loss": 0.1521,
      "objective": 0.1534017026424408,
      "ranking_idealized": 0.6791666746139526,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.6166666746139526,
      "regularize": 0.1534017026424408,
      "step": 475
    },
    {
      "dpo_loss": 0.56805020570755,
      "epoch": 2.720831365139348,
      "grad_norm": 18.441983400540806,
      "learning_rate": 2.53966490958702e-07,
      "logits": -1.7197903394699097,
      "logps": -90.20177459716797,
      "loss": 0.148,
      "objective": 0.14620445668697357,
      "ranking_idealized": 0.6833333373069763,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.6291666626930237,
      "regularize": 0.14620442688465118,
      "step": 480
    },
    {
      "dpo_loss": 0.5559974312782288,
      "epoch": 2.7491733585262166,
      "grad_norm": 17.00220355810742,
      "learning_rate": 2.4900833825280967e-07,
      "logits": -1.628369927406311,
      "logps": -93.048828125,
      "loss": 0.1488,
      "objective": 0.1451708972454071,
      "ranking_idealized": 0.6708333492279053,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.6583333611488342,
      "regularize": 0.1451708972454071,
      "step": 485
    },
    {
      "dpo_loss": 0.555105984210968,
      "epoch": 2.7775153519130846,
      "grad_norm": 17.798810379621077,
      "learning_rate": 2.4405057561347313e-07,
      "logits": -1.647185206413269,
      "logps": -90.4990463256836,
      "loss": 0.1613,
      "objective": 0.17200501263141632,
      "ranking_idealized": 0.6833333373069763,
      "ranking_idealized_expo": 0.4749999940395355,
      "ranking_simple": 0.6208333373069763,
      "regularize": 0.17200501263141632,
      "step": 490
    },
    {
      "dpo_loss": 0.5594576001167297,
      "epoch": 2.8058573452999527,
      "grad_norm": 18.076540126591944,
      "learning_rate": 2.39095153158666e-07,
      "logits": -1.6548616886138916,
      "logps": -90.19225311279297,
      "loss": 0.1504,
      "objective": 0.1365150660276413,
      "ranking_idealized": 0.6416666507720947,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.6208333373069763,
      "regularize": 0.1365150511264801,
      "step": 495
    },
    {
      "dpo_loss": 0.5552747845649719,
      "epoch": 2.8341993386868207,
      "grad_norm": 17.278782223651127,
      "learning_rate": 2.3414402008585886e-07,
      "logits": -1.6857832670211792,
      "logps": -89.0853500366211,
      "loss": 0.1494,
      "objective": 0.15246258676052094,
      "ranking_idealized": 0.6916666626930237,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.6416666507720947,
      "regularize": 0.15246258676052094,
      "step": 500
    },
    {
      "epoch": 2.8341993386868207,
      "eval_dpo_loss": 0.6861580014228821,
      "eval_logits": -1.7174702882766724,
      "eval_logps": -99.17545318603516,
      "eval_loss": 0.41525644063949585,
      "eval_objective": 0.4182237386703491,
      "eval_ranking_idealized": 0.6570248007774353,
      "eval_ranking_idealized_expo": 0.5113636255264282,
      "eval_ranking_simple": 0.5227272510528564,
      "eval_regularize": 0.4182237386703491,
      "eval_runtime": 259.2438,
      "eval_samples_per_second": 22.334,
      "eval_steps_per_second": 0.933,
      "step": 500
    },
    {
      "dpo_loss": 0.5660989284515381,
      "epoch": 2.862541332073689,
      "grad_norm": 18.182680782212074,
      "learning_rate": 2.2919912390530943e-07,
      "logits": -1.6143929958343506,
      "logps": -91.0888900756836,
      "loss": 0.1437,
      "objective": 0.16082407534122467,
      "ranking_idealized": 0.6583333611488342,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.6333333253860474,
      "regularize": 0.16082406044006348,
      "step": 505
    },
    {
      "dpo_loss": 0.5675150752067566,
      "epoch": 2.8908833254605573,
      "grad_norm": 16.373132303441977,
      "learning_rate": 2.2426240967401638e-07,
      "logits": -1.5807684659957886,
      "logps": -91.39689636230469,
      "loss": 0.1433,
      "objective": 0.1494457870721817,
      "ranking_idealized": 0.6958333253860474,
      "ranking_idealized_expo": 0.5291666388511658,
      "ranking_simple": 0.6291666626930237,
      "regularize": 0.1494457870721817,
      "step": 510
    },
    {
      "dpo_loss": 0.5627566576004028,
      "epoch": 2.9192253188474258,
      "grad_norm": 18.008132213394468,
      "learning_rate": 2.1933581923063837e-07,
      "logits": -1.7557440996170044,
      "logps": -91.32353210449219,
      "loss": 0.1448,
      "objective": 0.13260915875434875,
      "ranking_idealized": 0.6583333611488342,
      "ranking_idealized_expo": 0.4791666567325592,
      "ranking_simple": 0.6291666626930237,
      "regularize": 0.13260914385318756,
      "step": 515
    },
    {
      "dpo_loss": 0.5646940469741821,
      "epoch": 2.947567312234294,
      "grad_norm": 17.30767973762921,
      "learning_rate": 2.1442129043167873e-07,
      "logits": -1.610668420791626,
      "logps": -92.7865219116211,
      "loss": 0.1368,
      "objective": 0.11772733181715012,
      "ranking_idealized": 0.6583333611488342,
      "ranking_idealized_expo": 0.512499988079071,
      "ranking_simple": 0.6291666626930237,
      "regularize": 0.11772733181715012,
      "step": 520
    },
    {
      "dpo_loss": 0.5658089518547058,
      "epoch": 2.975909305621162,
      "grad_norm": 18.116492800551395,
      "learning_rate": 2.0952075638923652e-07,
      "logits": -1.6272333860397339,
      "logps": -92.43870544433594,
      "loss": 0.1424,
      "objective": 0.15236981213092804,
      "ranking_idealized": 0.6875,
      "ranking_idealized_expo": 0.5708333253860474,
      "ranking_simple": 0.6416666507720947,
      "regularize": 0.15236981213092804,
      "step": 525
    },
    {
      "dpo_loss": 0.5534684658050537,
      "epoch": 3.0042512990080303,
      "grad_norm": 18.337044286762765,
      "learning_rate": 2.0463614471062435e-07,
      "logits": -1.6210473775863647,
      "logps": -91.47294616699219,
      "loss": 0.1502,
      "objective": 0.17477649450302124,
      "ranking_idealized": 0.675000011920929,
      "ranking_idealized_expo": 0.512499988079071,
      "ranking_simple": 0.6291666626930237,
      "regularize": 0.17477649450302124,
      "step": 530
    },
    {
      "dpo_loss": 0.5659457445144653,
      "epoch": 3.0325932923948984,
      "grad_norm": 16.444884726429134,
      "learning_rate": 1.9976937674015026e-07,
      "logits": -1.6844907999038696,
      "logps": -93.2222671508789,
      "loss": 0.1284,
      "objective": 0.14268328249454498,
      "ranking_idealized": 0.6708333492279053,
      "ranking_idealized_expo": 0.5458333492279053,
      "ranking_simple": 0.6583333611488342,
      "regularize": 0.14268328249454498,
      "step": 535
    },
    {
      "dpo_loss": 0.5521051287651062,
      "epoch": 3.0609352857817664,
      "grad_norm": 19.963683437356444,
      "learning_rate": 1.9492236680336483e-07,
      "logits": -1.7760847806930542,
      "logps": -90.89082336425781,
      "loss": 0.1216,
      "objective": 0.10329335182905197,
      "ranking_idealized": 0.6625000238418579,
      "ranking_idealized_expo": 0.5375000238418579,
      "ranking_simple": 0.6541666388511658,
      "regularize": 0.10329335182905197,
      "step": 540
    },
    {
      "dpo_loss": 0.5619763731956482,
      "epoch": 3.089277279168635,
      "grad_norm": 17.450130382767895,
      "learning_rate": 1.9009702145406724e-07,
      "logits": -1.6995065212249756,
      "logps": -92.40625,
      "loss": 0.1232,
      "objective": 0.1230437308549881,
      "ranking_idealized": 0.637499988079071,
      "ranking_idealized_expo": 0.49166667461395264,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.1230437308549881,
      "step": 545
    },
    {
      "dpo_loss": 0.5528106689453125,
      "epoch": 3.117619272555503,
      "grad_norm": 18.245098236126562,
      "learning_rate": 1.8529523872436977e-07,
      "logits": -1.5086556673049927,
      "logps": -92.30103302001953,
      "loss": 0.1407,
      "objective": 0.12957319617271423,
      "ranking_idealized": 0.699999988079071,
      "ranking_idealized_expo": 0.5458333492279053,
      "ranking_simple": 0.6791666746139526,
      "regularize": 0.12957318127155304,
      "step": 550
    },
    {
      "epoch": 3.117619272555503,
      "eval_dpo_loss": 0.6856257915496826,
      "eval_logits": -1.7183054685592651,
      "eval_logps": -99.2997055053711,
      "eval_loss": 0.4161340296268463,
      "eval_objective": 0.41743505001068115,
      "eval_ranking_idealized": 0.6570248007774353,
      "eval_ranking_idealized_expo": 0.5113636255264282,
      "eval_ranking_simple": 0.5216942429542542,
      "eval_regularize": 0.41743505001068115,
      "eval_runtime": 258.7783,
      "eval_samples_per_second": 22.374,
      "eval_steps_per_second": 0.935,
      "step": 550
    },
    {
      "dpo_loss": 0.5473430752754211,
      "epoch": 3.1459612659423715,
      "grad_norm": 18.87722095427309,
      "learning_rate": 1.8051890737811393e-07,
      "logits": -1.6218358278274536,
      "logps": -93.05738067626953,
      "loss": 0.1336,
      "objective": 0.1305130124092102,
      "ranking_idealized": 0.6833333373069763,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.6625000238418579,
      "regularize": 0.1305130124092102,
      "step": 555
    },
    {
      "dpo_loss": 0.5478367209434509,
      "epoch": 3.1743032593292395,
      "grad_norm": 22.714698597290123,
      "learning_rate": 1.7576990616793137e-07,
      "logits": -1.601859211921692,
      "logps": -90.21554565429688,
      "loss": 0.1212,
      "objective": 0.10795855522155762,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.637499988079071,
      "regularize": 0.10795855522155762,
      "step": 560
    },
    {
      "dpo_loss": 0.5566601157188416,
      "epoch": 3.2026452527161076,
      "grad_norm": 24.322678833478967,
      "learning_rate": 1.710501030962438e-07,
      "logits": -1.663177728652954,
      "logps": -91.7726058959961,
      "loss": 0.1298,
      "objective": 0.13216590881347656,
      "ranking_idealized": 0.6916666626930237,
      "ranking_idealized_expo": 0.5458333492279053,
      "ranking_simple": 0.6458333134651184,
      "regularize": 0.13216587901115417,
      "step": 565
    },
    {
      "dpo_loss": 0.5519458055496216,
      "epoch": 3.230987246102976,
      "grad_norm": 19.102063233264193,
      "learning_rate": 1.663613546804912e-07,
      "logits": -1.5763607025146484,
      "logps": -91.98208618164062,
      "loss": 0.1293,
      "objective": 0.13738204538822174,
      "ranking_idealized": 0.6416666507720947,
      "ranking_idealized_expo": 0.4791666567325592,
      "ranking_simple": 0.5916666388511658,
      "regularize": 0.13738203048706055,
      "step": 570
    },
    {
      "dpo_loss": 0.5559364557266235,
      "epoch": 3.259329239489844,
      "grad_norm": 16.15481429380041,
      "learning_rate": 1.617055052228768e-07,
      "logits": -1.6705526113510132,
      "logps": -92.17435455322266,
      "loss": 0.1266,
      "objective": 0.12801046669483185,
      "ranking_idealized": 0.7083333134651184,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.6499999761581421,
      "regularize": 0.12801046669483185,
      "step": 575
    },
    {
      "dpo_loss": 0.5649384260177612,
      "epoch": 3.287671232876712,
      "grad_norm": 17.44743081337015,
      "learning_rate": 1.5708438608491815e-07,
      "logits": -1.6591442823410034,
      "logps": -93.50952911376953,
      "loss": 0.1277,
      "objective": 0.11801984906196594,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.5583333373069763,
      "ranking_simple": 0.6291666626930237,
      "regularize": 0.11801984906196594,
      "step": 580
    },
    {
      "dpo_loss": 0.5595548152923584,
      "epoch": 3.3160132262635806,
      "grad_norm": 16.72082331684023,
      "learning_rate": 1.524998149670871e-07,
      "logits": -1.69523286819458,
      "logps": -93.74117279052734,
      "loss": 0.12,
      "objective": 0.10769928246736526,
      "ranking_idealized": 0.6958333253860474,
      "ranking_idealized_expo": 0.5375000238418579,
      "ranking_simple": 0.6583333611488342,
      "regularize": 0.10769927501678467,
      "step": 585
    },
    {
      "dpo_loss": 0.561581015586853,
      "epoch": 3.3443552196504487,
      "grad_norm": 19.465809423510365,
      "learning_rate": 1.479535951938243e-07,
      "logits": -1.7049933671951294,
      "logps": -93.83617401123047,
      "loss": 0.1205,
      "objective": 0.09809862077236176,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.637499988079071,
      "regularize": 0.09809862077236176,
      "step": 590
    },
    {
      "dpo_loss": 0.5538628101348877,
      "epoch": 3.372697213037317,
      "grad_norm": 17.81052400873953,
      "learning_rate": 1.43447515004208e-07,
      "logits": -1.613613247871399,
      "logps": -92.85578155517578,
      "loss": 0.1191,
      "objective": 0.12334737926721573,
      "ranking_idealized": 0.6458333134651184,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.637499988079071,
      "regularize": 0.12334737926721573,
      "step": 595
    },
    {
      "dpo_loss": 0.5357978940010071,
      "epoch": 3.4010392064241852,
      "grad_norm": 18.626853535104544,
      "learning_rate": 1.3898334684855645e-07,
      "logits": -1.624743938446045,
      "logps": -92.40316009521484,
      "loss": 0.1149,
      "objective": 0.13463754951953888,
      "ranking_idealized": 0.6916666626930237,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.6458333134651184,
      "regularize": 0.13463754951953888,
      "step": 600
    },
    {
      "epoch": 3.4010392064241852,
      "eval_dpo_loss": 0.6852067112922668,
      "eval_logits": -1.71807062625885,
      "eval_logps": -99.92455291748047,
      "eval_loss": 0.41705650091171265,
      "eval_objective": 0.41811424493789673,
      "eval_ranking_idealized": 0.6570248007774353,
      "eval_ranking_idealized_expo": 0.5113636255264282,
      "eval_ranking_simple": 0.5247933864593506,
      "eval_regularize": 0.41811424493789673,
      "eval_runtime": 259.0859,
      "eval_samples_per_second": 22.348,
      "eval_steps_per_second": 0.934,
      "step": 600
    },
    {
      "dpo_loss": 0.5652448534965515,
      "epoch": 3.4293811998110533,
      "grad_norm": 17.958926430591173,
      "learning_rate": 1.3456284669124157e-07,
      "logits": -1.6740020513534546,
      "logps": -94.55862426757812,
      "loss": 0.1179,
      "objective": 0.11572790890932083,
      "ranking_idealized": 0.6583333611488342,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.625,
      "regularize": 0.11572788655757904,
      "step": 605
    },
    {
      "dpo_loss": 0.5543821454048157,
      "epoch": 3.4577231931979218,
      "grad_norm": 17.326848783729876,
      "learning_rate": 1.301877533199859e-07,
      "logits": -1.6315828561782837,
      "logps": -92.49845886230469,
      "loss": 0.1149,
      "objective": 0.1067105308175087,
      "ranking_idealized": 0.6791666746139526,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.6499999761581421,
      "regularize": 0.1067105159163475,
      "step": 610
    },
    {
      "dpo_loss": 0.5443283915519714,
      "epoch": 3.48606518658479,
      "grad_norm": 16.84586393500809,
      "learning_rate": 1.2585978766191724e-07,
      "logits": -1.664933681488037,
      "logps": -93.27455139160156,
      "loss": 0.1142,
      "objective": 0.10945113748311996,
      "ranking_idealized": 0.6458333134651184,
      "ranking_idealized_expo": 0.47083333134651184,
      "ranking_simple": 0.5958333611488342,
      "regularize": 0.10945113748311996,
      "step": 615
    },
    {
      "dpo_loss": 0.5419160723686218,
      "epoch": 3.514407179971658,
      "grad_norm": 17.32874521556865,
      "learning_rate": 1.2158065210664848e-07,
      "logits": -1.5332224369049072,
      "logps": -92.34308624267578,
      "loss": 0.1203,
      "objective": 0.12084861099720001,
      "ranking_idealized": 0.637499988079071,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.6166666746139526,
      "regularize": 0.12084860354661942,
      "step": 620
    },
    {
      "dpo_loss": 0.5541211366653442,
      "epoch": 3.5427491733585263,
      "grad_norm": 17.48530471086995,
      "learning_rate": 1.1735202983664802e-07,
      "logits": -1.6171096563339233,
      "logps": -91.3125991821289,
      "loss": 0.1178,
      "objective": 0.11125477403402328,
      "ranking_idealized": 0.6708333492279053,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.6416666507720947,
      "regularize": 0.11125477403402328,
      "step": 625
    },
    {
      "dpo_loss": 0.5698776245117188,
      "epoch": 3.5710911667453944,
      "grad_norm": 17.864701578880954,
      "learning_rate": 1.1317558416516696e-07,
      "logits": -1.697689175605774,
      "logps": -91.67240905761719,
      "loss": 0.1261,
      "objective": 0.13253255188465118,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.512499988079071,
      "ranking_simple": 0.6083333492279053,
      "regularize": 0.13253255188465118,
      "step": 630
    },
    {
      "dpo_loss": 0.5599467158317566,
      "epoch": 3.5994331601322624,
      "grad_norm": 17.33519253157568,
      "learning_rate": 1.090529578819799e-07,
      "logits": -1.6461411714553833,
      "logps": -91.57376098632812,
      "loss": 0.1157,
      "objective": 0.10732007026672363,
      "ranking_idealized": 0.637499988079071,
      "ranking_idealized_expo": 0.48750001192092896,
      "ranking_simple": 0.6208333373069763,
      "regularize": 0.10732006281614304,
      "step": 635
    },
    {
      "dpo_loss": 0.540539562702179,
      "epoch": 3.627775153519131,
      "grad_norm": 17.025667462047203,
      "learning_rate": 1.0498577260720048e-07,
      "logits": -1.5717778205871582,
      "logps": -93.14022827148438,
      "loss": 0.1146,
      "objective": 0.13500064611434937,
      "ranking_idealized": 0.6416666507720947,
      "ranking_idealized_expo": 0.4625000059604645,
      "ranking_simple": 0.574999988079071,
      "regularize": 0.13500064611434937,
      "step": 640
    },
    {
      "dpo_loss": 0.5469278693199158,
      "epoch": 3.656117146905999,
      "grad_norm": 17.536092815770388,
      "learning_rate": 1.0097562815342214e-07,
      "logits": -1.6058826446533203,
      "logps": -90.76680755615234,
      "loss": 0.1144,
      "objective": 0.1191474050283432,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.6291666626930237,
      "regularize": 0.1191474050283432,
      "step": 645
    },
    {
      "dpo_loss": 0.5611483454704285,
      "epoch": 3.6844591402928675,
      "grad_norm": 17.646206320924833,
      "learning_rate": 9.702410189643836e-08,
      "logits": -1.6121342182159424,
      "logps": -92.83375549316406,
      "loss": 0.1108,
      "objective": 0.09943919628858566,
      "ranking_idealized": 0.6458333134651184,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.6083333492279053,
      "regularize": 0.09943918883800507,
      "step": 650
    },
    {
      "epoch": 3.6844591402928675,
      "eval_dpo_loss": 0.6852837800979614,
      "eval_logits": -1.7315040826797485,
      "eval_logps": -99.91177368164062,
      "eval_loss": 0.41784536838531494,
      "eval_objective": 0.41884875297546387,
      "eval_ranking_idealized": 0.6570248007774353,
      "eval_ranking_idealized_expo": 0.5113636255264282,
      "eval_ranking_simple": 0.5247933864593506,
      "eval_regularize": 0.41884875297546387,
      "eval_runtime": 259.4431,
      "eval_samples_per_second": 22.317,
      "eval_steps_per_second": 0.933,
      "step": 650
    },
    {
      "dpo_loss": 0.5510907769203186,
      "epoch": 3.7128011336797355,
      "grad_norm": 17.84028807284025,
      "learning_rate": 9.313274815478698e-08,
      "logits": -1.6280105113983154,
      "logps": -92.27388763427734,
      "loss": 0.117,
      "objective": 0.10249165445566177,
      "ranking_idealized": 0.6583333611488342,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.6291666626930237,
      "regularize": 0.10249165445566177,
      "step": 655
    },
    {
      "dpo_loss": 0.5551621913909912,
      "epoch": 3.7411431270666036,
      "grad_norm": 17.863904309670215,
      "learning_rate": 8.930309757836516e-08,
      "logits": -1.7605994939804077,
      "logps": -92.74076080322266,
      "loss": 0.1162,
      "objective": 0.11682406812906265,
      "ranking_idealized": 0.6916666626930237,
      "ranking_idealized_expo": 0.5541666746139526,
      "ranking_simple": 0.6625000238418579,
      "regularize": 0.11682406812906265,
      "step": 660
    },
    {
      "dpo_loss": 0.5553780198097229,
      "epoch": 3.769485120453472,
      "grad_norm": 17.287090327509993,
      "learning_rate": 8.553665654635342e-08,
      "logits": -1.6500779390335083,
      "logps": -92.00687408447266,
      "loss": 0.116,
      "objective": 0.11367592960596085,
      "ranking_idealized": 0.6958333253860474,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.6541666388511658,
      "regularize": 0.11367591470479965,
      "step": 665
    },
    {
      "dpo_loss": 0.5561904311180115,
      "epoch": 3.79782711384034,
      "grad_norm": 18.08647866984471,
      "learning_rate": 8.183490657468686e-08,
      "logits": -1.7430044412612915,
      "logps": -92.36637878417969,
      "loss": 0.1153,
      "objective": 0.13086958229541779,
      "ranking_idealized": 0.7333333492279053,
      "ranking_idealized_expo": 0.5958333611488342,
      "ranking_simple": 0.7124999761581421,
      "regularize": 0.13086958229541779,
      "step": 670
    },
    {
      "dpo_loss": 0.5525475740432739,
      "epoch": 3.826169107227208,
      "grad_norm": 17.678219555630616,
      "learning_rate": 7.819930373330669e-08,
      "logits": -1.6892848014831543,
      "logps": -91.58055114746094,
      "loss": 0.1165,
      "objective": 0.10088498890399933,
      "ranking_idealized": 0.6708333492279053,
      "ranking_idealized_expo": 0.5458333492279053,
      "ranking_simple": 0.6416666507720947,
      "regularize": 0.10088498145341873,
      "step": 675
    },
    {
      "dpo_loss": 0.5708147883415222,
      "epoch": 3.8545111006140766,
      "grad_norm": 17.208804705028182,
      "learning_rate": 7.463127807341966e-08,
      "logits": -1.6462949514389038,
      "logps": -92.41487884521484,
      "loss": 0.1181,
      "objective": 0.11868777871131897,
      "ranking_idealized": 0.6541666388511658,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.6208333373069763,
      "regularize": 0.11868777871131897,
      "step": 680
    },
    {
      "dpo_loss": 0.5442604422569275,
      "epoch": 3.8828530940009447,
      "grad_norm": 18.600166194890996,
      "learning_rate": 7.113223306499336e-08,
      "logits": -1.7259678840637207,
      "logps": -91.63528442382812,
      "loss": 0.1127,
      "objective": 0.10472600162029266,
      "ranking_idealized": 0.6875,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.6499999761581421,
      "regularize": 0.10472600162029266,
      "step": 685
    },
    {
      "dpo_loss": 0.5496628284454346,
      "epoch": 3.9111950873878127,
      "grad_norm": 17.80496655704031,
      "learning_rate": 6.770354504470574e-08,
      "logits": -1.6540542840957642,
      "logps": -90.78262329101562,
      "loss": 0.1164,
      "objective": 0.10735266655683517,
      "ranking_idealized": 0.7041666507720947,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.6875,
      "regularize": 0.10735265165567398,
      "step": 690
    },
    {
      "dpo_loss": 0.5639461874961853,
      "epoch": 3.9395370807746812,
      "grad_norm": 16.492816826616206,
      "learning_rate": 6.434656267456842e-08,
      "logits": -1.6047898530960083,
      "logps": -92.38011932373047,
      "loss": 0.1193,
      "objective": 0.12910698354244232,
      "ranking_idealized": 0.6083333492279053,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.12910698354244232,
      "step": 695
    },
    {
      "dpo_loss": 0.5509793162345886,
      "epoch": 3.9678790741615493,
      "grad_norm": 19.17114576937693,
      "learning_rate": 6.106260641143546e-08,
      "logits": -1.6564711332321167,
      "logps": -92.65071868896484,
      "loss": 0.1146,
      "objective": 0.1030283123254776,
      "ranking_idealized": 0.612500011920929,
      "ranking_idealized_expo": 0.48750001192092896,
      "ranking_simple": 0.5958333611488342,
      "regularize": 0.103028304874897,
      "step": 700
    },
    {
      "epoch": 3.9678790741615493,
      "eval_dpo_loss": 0.6854080557823181,
      "eval_logits": -1.7319272756576538,
      "eval_logps": -99.89824676513672,
      "eval_loss": 0.4175797998905182,
      "eval_objective": 0.4186650514602661,
      "eval_ranking_idealized": 0.6570248007774353,
      "eval_ranking_idealized_expo": 0.5113636255264282,
      "eval_ranking_simple": 0.5237603187561035,
      "eval_regularize": 0.4186650514602661,
      "eval_runtime": 258.7065,
      "eval_samples_per_second": 22.381,
      "eval_steps_per_second": 0.935,
      "step": 700
    },
    {
      "dpo_loss": 0.5571620464324951,
      "epoch": 3.9962210675484178,
      "grad_norm": 18.74884098276965,
      "learning_rate": 5.7852967987606e-08,
      "logits": -1.554320216178894,
      "logps": -90.9109878540039,
      "loss": 0.1135,
      "objective": 0.12702669203281403,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.6583333611488342,
      "regularize": 0.12702666223049164,
      "step": 705
    },
    {
      "dpo_loss": 0.5528541803359985,
      "epoch": 4.024563060935286,
      "grad_norm": 17.40621172754528,
      "learning_rate": 5.471890990272665e-08,
      "logits": -1.6468113660812378,
      "logps": -92.61128997802734,
      "loss": 0.1135,
      "objective": 0.1373264044523239,
      "ranking_idealized": 0.6916666626930237,
      "ranking_idealized_expo": 0.4791666567325592,
      "ranking_simple": 0.612500011920929,
      "regularize": 0.13732638955116272,
      "step": 710
    },
    {
      "dpo_loss": 0.5556226968765259,
      "epoch": 4.052905054322154,
      "grad_norm": 18.812501686123863,
      "learning_rate": 5.166166492719124e-08,
      "logits": -1.6049120426177979,
      "logps": -92.74799346923828,
      "loss": 0.1035,
      "objective": 0.10493499785661697,
      "ranking_idealized": 0.6833333373069763,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.6458333134651184,
      "regularize": 0.10493497550487518,
      "step": 715
    },
    {
      "dpo_loss": 0.5424375534057617,
      "epoch": 4.081247047709022,
      "grad_norm": 17.435339866299028,
      "learning_rate": 4.868243561723534e-08,
      "logits": -1.511703372001648,
      "logps": -94.71248626708984,
      "loss": 0.0978,
      "objective": 0.09467672556638718,
      "ranking_idealized": 0.6583333611488342,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.6166666746139526,
      "regularize": 0.09467671811580658,
      "step": 720
    },
    {
      "dpo_loss": 0.5652304291725159,
      "epoch": 4.109589041095891,
      "grad_norm": 17.507850551258127,
      "learning_rate": 4.578239384191529e-08,
      "logits": -1.6384118795394897,
      "logps": -92.1180191040039,
      "loss": 0.1076,
      "objective": 0.09941933304071426,
      "ranking_idealized": 0.6541666388511658,
      "ranking_idealized_expo": 0.5375000238418579,
      "ranking_simple": 0.637499988079071,
      "regularize": 0.09941932559013367,
      "step": 725
    },
    {
      "dpo_loss": 0.5497789978981018,
      "epoch": 4.137931034482759,
      "grad_norm": 18.803004868454263,
      "learning_rate": 4.296268032215733e-08,
      "logits": -1.7138111591339111,
      "logps": -91.83662414550781,
      "loss": 0.1086,
      "objective": 0.10822432488203049,
      "ranking_idealized": 0.675000011920929,
      "ranking_idealized_expo": 0.5375000238418579,
      "ranking_simple": 0.6208333373069763,
      "regularize": 0.10822432488203049,
      "step": 730
    },
    {
      "dpo_loss": 0.55525803565979,
      "epoch": 4.166273027869627,
      "grad_norm": 18.177348382836357,
      "learning_rate": 4.022440418205944e-08,
      "logits": -1.6232236623764038,
      "logps": -93.14463806152344,
      "loss": 0.1028,
      "objective": 0.10451411455869675,
      "ranking_idealized": 0.6791666746139526,
      "ranking_idealized_expo": 0.5291666388511658,
      "ranking_simple": 0.6541666388511658,
      "regularize": 0.10451411455869675,
      "step": 735
    },
    {
      "dpo_loss": 0.535234808921814,
      "epoch": 4.194615021256495,
      "grad_norm": 17.133543410858152,
      "learning_rate": 3.756864251262143e-08,
      "logits": -1.610323190689087,
      "logps": -93.36137390136719,
      "loss": 0.1062,
      "objective": 0.10550294071435928,
      "ranking_idealized": 0.6791666746139526,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.6541666388511658,
      "regularize": 0.10550292581319809,
      "step": 740
    },
    {
      "dpo_loss": 0.546442449092865,
      "epoch": 4.222957014643363,
      "grad_norm": 18.78547392108143,
      "learning_rate": 3.4996439948074855e-08,
      "logits": -1.6879092454910278,
      "logps": -90.12301635742188,
      "loss": 0.1001,
      "objective": 0.10297367721796036,
      "ranking_idealized": 0.6291666626930237,
      "ranking_idealized_expo": 0.48750001192092896,
      "ranking_simple": 0.6208333373069763,
      "regularize": 0.10297366231679916,
      "step": 745
    },
    {
      "dpo_loss": 0.5416663289070129,
      "epoch": 4.251299008030231,
      "grad_norm": 19.462231175744662,
      "learning_rate": 3.250880825498026e-08,
      "logits": -1.8104737997055054,
      "logps": -92.32807922363281,
      "loss": 0.0986,
      "objective": 0.10099396854639053,
      "ranking_idealized": 0.6458333134651184,
      "ranking_idealized_expo": 0.5291666388511658,
      "ranking_simple": 0.6291666626930237,
      "regularize": 0.10099395364522934,
      "step": 750
    },
    {
      "epoch": 4.251299008030231,
      "eval_dpo_loss": 0.6853212714195251,
      "eval_logits": -1.7322306632995605,
      "eval_logps": -99.86943054199219,
      "eval_loss": 0.41747406125068665,
      "eval_objective": 0.41828420758247375,
      "eval_ranking_idealized": 0.6570248007774353,
      "eval_ranking_idealized_expo": 0.5113636255264282,
      "eval_ranking_simple": 0.5237603187561035,
      "eval_regularize": 0.41828420758247375,
      "eval_runtime": 258.964,
      "eval_samples_per_second": 22.358,
      "eval_steps_per_second": 0.934,
      "step": 750
    },
    {
      "dpo_loss": 0.5468146800994873,
      "epoch": 4.2796410014171,
      "grad_norm": 18.270752967286892,
      "learning_rate": 3.010672593425209e-08,
      "logits": -1.7138711214065552,
      "logps": -92.11996459960938,
      "loss": 0.1147,
      "objective": 0.10049024224281311,
      "ranking_idealized": 0.6791666746139526,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.6458333134651184,
      "regularize": 0.10049023479223251,
      "step": 755
    },
    {
      "dpo_loss": 0.5455428957939148,
      "epoch": 4.307982994803968,
      "grad_norm": 17.285027330879423,
      "learning_rate": 2.7791137836269158e-08,
      "logits": -1.6757961511611938,
      "logps": -93.05391693115234,
      "loss": 0.0961,
      "objective": 0.10086000710725784,
      "ranking_idealized": 0.7083333134651184,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.6833333373069763,
      "regularize": 0.10086000710725784,
      "step": 760
    },
    {
      "dpo_loss": 0.5395826101303101,
      "epoch": 4.336324988190836,
      "grad_norm": 17.680760382133624,
      "learning_rate": 2.556295478922116e-08,
      "logits": -1.7200431823730469,
      "logps": -93.1734619140625,
      "loss": 0.1053,
      "objective": 0.12091321498155594,
      "ranking_idealized": 0.7291666865348816,
      "ranking_idealized_expo": 0.5291666388511658,
      "ranking_simple": 0.6833333373069763,
      "regularize": 0.12091320008039474,
      "step": 765
    },
    {
      "dpo_loss": 0.5380468964576721,
      "epoch": 4.364666981577704,
      "grad_norm": 17.361908211383366,
      "learning_rate": 2.3423053240837514e-08,
      "logits": -1.577264428138733,
      "logps": -91.18030548095703,
      "loss": 0.1064,
      "objective": 0.11028440296649933,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.637499988079071,
      "regularize": 0.11028438061475754,
      "step": 770
    },
    {
      "dpo_loss": 0.5494747161865234,
      "epoch": 4.393008974964572,
      "grad_norm": 16.640977636984772,
      "learning_rate": 2.137227491364016e-08,
      "logits": -1.627792239189148,
      "logps": -91.97000885009766,
      "loss": 0.1067,
      "objective": 0.10595239698886871,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.512499988079071,
      "ranking_simple": 0.6333333253860474,
      "regularize": 0.10595235228538513,
      "step": 775
    },
    {
      "dpo_loss": 0.5575358867645264,
      "epoch": 4.42135096835144,
      "grad_norm": 17.112345268128863,
      "learning_rate": 1.9411426473854687e-08,
      "logits": -1.693690538406372,
      "logps": -90.5418472290039,
      "loss": 0.1004,
      "objective": 0.10788667947053909,
      "ranking_idealized": 0.6541666388511658,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.6208333373069763,
      "regularize": 0.10788667947053909,
      "step": 780
    },
    {
      "dpo_loss": 0.5676646828651428,
      "epoch": 4.449692961738309,
      "grad_norm": 16.657419373072543,
      "learning_rate": 1.7541279214111275e-08,
      "logits": -1.7215303182601929,
      "logps": -90.63499450683594,
      "loss": 0.1128,
      "objective": 0.1177934780716896,
      "ranking_idealized": 0.6583333611488342,
      "ranking_idealized_expo": 0.5291666388511658,
      "ranking_simple": 0.612500011920929,
      "regularize": 0.1177934780716896,
      "step": 785
    },
    {
      "dpo_loss": 0.539345920085907,
      "epoch": 4.478034955125177,
      "grad_norm": 17.089680176209615,
      "learning_rate": 1.57625687500596e-08,
      "logits": -1.6345340013504028,
      "logps": -93.4063720703125,
      "loss": 0.0998,
      "objective": 0.10817180573940277,
      "ranking_idealized": 0.7291666865348816,
      "ranking_idealized_expo": 0.5583333373069763,
      "ranking_simple": 0.6916666626930237,
      "regularize": 0.10817176848649979,
      "step": 790
    },
    {
      "dpo_loss": 0.547528088092804,
      "epoch": 4.506376948512045,
      "grad_norm": 17.320194779122446,
      "learning_rate": 1.4075994731016894e-08,
      "logits": -1.5627334117889404,
      "logps": -93.30286407470703,
      "loss": 0.1058,
      "objective": 0.102629154920578,
      "ranking_idealized": 0.6541666388511658,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.6291666626930237,
      "regularize": 0.10262913256883621,
      "step": 795
    },
    {
      "dpo_loss": 0.5486911535263062,
      "epoch": 4.534718941898913,
      "grad_norm": 17.9229448629881,
      "learning_rate": 1.2482220564763667e-08,
      "logits": -1.5870776176452637,
      "logps": -92.80538940429688,
      "loss": 0.1042,
      "objective": 0.11460768431425095,
      "ranking_idealized": 0.6875,
      "ranking_idealized_expo": 0.512499988079071,
      "ranking_simple": 0.6499999761581421,
      "regularize": 0.11460768431425095,
      "step": 800
    },
    {
      "epoch": 4.534718941898913,
      "eval_dpo_loss": 0.6852768659591675,
      "eval_logits": -1.7317209243774414,
      "eval_logps": -99.85995483398438,
      "eval_loss": 0.4175398349761963,
      "eval_objective": 0.4183157980442047,
      "eval_ranking_idealized": 0.6570248007774353,
      "eval_ranking_idealized_expo": 0.5113636255264282,
      "eval_ranking_simple": 0.5237603187561035,
      "eval_regularize": 0.4183157980442047,
      "eval_runtime": 259.0856,
      "eval_samples_per_second": 22.348,
      "eval_steps_per_second": 0.934,
      "step": 800
    },
    {
      "dpo_loss": 0.5488670468330383,
      "epoch": 4.563060935285781,
      "grad_norm": 18.799592478883184,
      "learning_rate": 1.0981873156594379e-08,
      "logits": -1.627816081047058,
      "logps": -91.32179260253906,
      "loss": 0.1001,
      "objective": 0.10988225042819977,
      "ranking_idealized": 0.6458333134651184,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.6041666865348816,
      "regularize": 0.10988224297761917,
      "step": 805
    },
    {
      "dpo_loss": 0.5488799810409546,
      "epoch": 4.59140292867265,
      "grad_norm": 16.566296936758206,
      "learning_rate": 9.575542662726754e-09,
      "logits": -1.7243562936782837,
      "logps": -91.10765075683594,
      "loss": 0.0996,
      "objective": 0.08869278430938721,
      "ranking_idealized": 0.5874999761581421,
      "ranking_idealized_expo": 0.4625000059604645,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.08869277685880661,
      "step": 810
    },
    {
      "dpo_loss": 0.5502530336380005,
      "epoch": 4.619744922059518,
      "grad_norm": 16.948787644578637,
      "learning_rate": 8.263782258165819e-09,
      "logits": -1.5700196027755737,
      "logps": -92.37843322753906,
      "loss": 0.0991,
      "objective": 0.07977009564638138,
      "ranking_idealized": 0.7250000238418579,
      "ranking_idealized_expo": 0.5916666388511658,
      "ranking_simple": 0.6958333253860474,
      "regularize": 0.07977008074522018,
      "step": 815
    },
    {
      "dpo_loss": 0.5550402402877808,
      "epoch": 4.648086915446386,
      "grad_norm": 21.501267763535818,
      "learning_rate": 7.047107919114586e-09,
      "logits": -1.6636712551116943,
      "logps": -92.61454010009766,
      "loss": 0.0982,
      "objective": 0.08367303013801575,
      "ranking_idealized": 0.6875,
      "ranking_idealized_expo": 0.5625,
      "ranking_simple": 0.6708333492279053,
      "regularize": 0.08367302268743515,
      "step": 820
    },
    {
      "dpo_loss": 0.5588962435722351,
      "epoch": 4.6764289088332545,
      "grad_norm": 16.315710057694485,
      "learning_rate": 5.925998220016659e-09,
      "logits": -1.5499807596206665,
      "logps": -90.22130584716797,
      "loss": 0.1018,
      "objective": 0.09301813691854477,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.09301812201738358,
      "step": 825
    },
    {
      "dpo_loss": 0.5498708486557007,
      "epoch": 4.7047709022201225,
      "grad_norm": 17.748669049129045,
      "learning_rate": 4.9008941453107525e-09,
      "logits": -1.7388263940811157,
      "logps": -92.17695617675781,
      "loss": 0.1092,
      "objective": 0.11468993872404099,
      "ranking_idealized": 0.675000011920929,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.6416666507720947,
      "regularize": 0.1146899089217186,
      "step": 830
    },
    {
      "dpo_loss": 0.5327169299125671,
      "epoch": 4.733112895606991,
      "grad_norm": 17.389468155390862,
      "learning_rate": 3.9721989159709754e-09,
      "logits": -1.6580873727798462,
      "logps": -92.22929382324219,
      "loss": 0.1,
      "objective": 0.09779965132474899,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.48750001192092896,
      "ranking_simple": 0.6041666865348816,
      "regularize": 0.0977996364235878,
      "step": 835
    },
    {
      "dpo_loss": 0.5324665904045105,
      "epoch": 4.7614548889938595,
      "grad_norm": 18.51697686947663,
      "learning_rate": 3.140277830901428e-09,
      "logits": -1.6570351123809814,
      "logps": -92.75865173339844,
      "loss": 0.1079,
      "objective": 0.11856434494256973,
      "ranking_idealized": 0.6958333253860474,
      "ranking_idealized_expo": 0.5541666746139526,
      "ranking_simple": 0.6499999761581421,
      "regularize": 0.11856433004140854,
      "step": 840
    },
    {
      "dpo_loss": 0.560818612575531,
      "epoch": 4.7897968823807275,
      "grad_norm": 17.858581136510683,
      "learning_rate": 2.4054581232470785e-09,
      "logits": -1.6901015043258667,
      "logps": -92.13179016113281,
      "loss": 0.1067,
      "objective": 0.10368030518293381,
      "ranking_idealized": 0.6416666507720947,
      "ranking_idealized_expo": 0.4583333432674408,
      "ranking_simple": 0.6208333373069763,
      "regularize": 0.10368029028177261,
      "step": 845
    },
    {
      "dpo_loss": 0.5496495366096497,
      "epoch": 4.818138875767596,
      "grad_norm": 17.640915580271592,
      "learning_rate": 1.7680288316779256e-09,
      "logits": -1.6190950870513916,
      "logps": -90.9464340209961,
      "loss": 0.103,
      "objective": 0.10453298687934875,
      "ranking_idealized": 0.6833333373069763,
      "ranking_idealized_expo": 0.5291666388511658,
      "ranking_simple": 0.6333333253860474,
      "regularize": 0.10453297942876816,
      "step": 850
    },
    {
      "epoch": 4.818138875767596,
      "eval_dpo_loss": 0.6852567791938782,
      "eval_logits": -1.732380986213684,
      "eval_logps": -99.89720153808594,
      "eval_loss": 0.4175875782966614,
      "eval_objective": 0.4183763563632965,
      "eval_ranking_idealized": 0.6570248007774353,
      "eval_ranking_idealized_expo": 0.5113636255264282,
      "eval_ranking_simple": 0.5237603187561035,
      "eval_regularize": 0.4183763563632965,
      "eval_runtime": 259.1903,
      "eval_samples_per_second": 22.339,
      "eval_steps_per_second": 0.934,
      "step": 850
    },
    {
      "dpo_loss": 0.5584205389022827,
      "epoch": 4.846480869154464,
      "grad_norm": 17.56984774908714,
      "learning_rate": 1.2282406866966078e-09,
      "logits": -1.6185228824615479,
      "logps": -91.83565521240234,
      "loss": 0.0974,
      "objective": 0.09289266169071198,
      "ranking_idealized": 0.7083333134651184,
      "ranking_idealized_expo": 0.5375000238418579,
      "ranking_simple": 0.675000011920929,
      "regularize": 0.09289265424013138,
      "step": 855
    },
    {
      "dpo_loss": 0.5458131432533264,
      "epoch": 4.874822862541333,
      "grad_norm": 17.337457908328606,
      "learning_rate": 7.863060120144316e-10,
      "logits": -1.5824497938156128,
      "logps": -91.32083892822266,
      "loss": 0.0959,
      "objective": 0.11178465932607651,
      "ranking_idealized": 0.7166666388511658,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.6791666746139526,
      "regularize": 0.11178465187549591,
      "step": 860
    },
    {
      "dpo_loss": 0.555813729763031,
      "epoch": 4.903164855928201,
      "grad_norm": 16.568497702615847,
      "learning_rate": 4.4239864103465254e-10,
      "logits": -1.6553268432617188,
      "logps": -90.40623474121094,
      "loss": 0.1003,
      "objective": 0.1271737664937973,
      "ranking_idealized": 0.6625000238418579,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.6041666865348816,
      "regularize": 0.1271737515926361,
      "step": 865
    },
    {
      "dpo_loss": 0.5469813942909241,
      "epoch": 4.931506849315069,
      "grad_norm": 17.67813635168232,
      "learning_rate": 1.966538484758362e-10,
      "logits": -1.7142003774642944,
      "logps": -92.42487335205078,
      "loss": 0.1053,
      "objective": 0.10629518330097198,
      "ranking_idealized": 0.6916666626930237,
      "ranking_idealized_expo": 0.512499988079071,
      "ranking_simple": 0.6625000238418579,
      "regularize": 0.1062951609492302,
      "step": 870
    },
    {
      "dpo_loss": 0.5456808805465698,
      "epoch": 4.959848842701937,
      "grad_norm": 17.372715214830695,
      "learning_rate": 4.9168297161839014e-11,
      "logits": -1.6318602561950684,
      "logps": -92.3662109375,
      "loss": 0.1017,
      "objective": 0.08666170388460159,
      "ranking_idealized": 0.6833333373069763,
      "ranking_idealized_expo": 0.5458333492279053,
      "ranking_simple": 0.6541666388511658,
      "regularize": 0.086661696434021,
      "step": 875
    },
    {
      "dpo_loss": 0.5451498627662659,
      "epoch": 4.988190836088805,
      "grad_norm": 17.608890670600516,
      "learning_rate": 0.0,
      "logits": -1.7033004760742188,
      "logps": -92.65689849853516,
      "loss": 0.0938,
      "objective": 0.09012699872255325,
      "ranking_idealized": 0.6708333492279053,
      "ranking_idealized_expo": 0.5291666388511658,
      "ranking_simple": 0.6541666388511658,
      "regularize": 0.09012699127197266,
      "step": 880
    },
    {
      "epoch": 4.988190836088805,
      "step": 880,
      "total_flos": 0.0,
      "train_loss": 0.19899855256080629,
      "train_runtime": 35117.4941,
      "train_samples_per_second": 7.233,
      "train_steps_per_second": 0.025
    }
  ],
  "logging_steps": 5,
  "max_steps": 880,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 5,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}