{
  "best_metric": 0.5423553586006165,
  "best_model_checkpoint": "./qwen2.5-0.5b/qwen2.5-0.5b-expo-DPO-ES-TRY/checkpoint-371",
  "epoch": 2.9976381672177608,
  "eval_steps": 53,
  "global_step": 528,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "dpo_loss": 0.6931471824645996,
      "dpo_wo_beta": -0.6931471824645996,
      "epoch": 0.005668398677373642,
      "grad_norm": 13.433600669124935,
      "learning_rate": 9.433962264150944e-08,
      "logits": -1.3874311447143555,
      "logps": -88.43561553955078,
      "loss": 0.6931,
      "objective": 0.6931471824645996,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.5208333134651184,
      "regularize": 0.6931471824645996,
      "step": 1
    },
    {
      "dpo_loss": 0.693236768245697,
      "dpo_wo_beta": -0.6993356347084045,
      "epoch": 0.02834199338686821,
      "grad_norm": 13.640653628388394,
      "learning_rate": 4.716981132075472e-07,
      "logits": -1.4090652465820312,
      "logps": -84.34337615966797,
      "loss": 0.693,
      "objective": 0.693236768245697,
      "ranking_idealized": 0.6197916865348816,
      "ranking_idealized_expo": 0.546875,
      "ranking_simple": 0.546875,
      "regularize": 0.693236768245697,
      "step": 5
    },
    {
      "dpo_loss": 0.6845630407333374,
      "dpo_wo_beta": -0.7111619710922241,
      "epoch": 0.05668398677373642,
      "grad_norm": 12.626074407134174,
      "learning_rate": 9.433962264150944e-07,
      "logits": -1.4784893989562988,
      "logps": -81.94055938720703,
      "loss": 0.6892,
      "objective": 0.6845630407333374,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.512499988079071,
      "regularize": 0.6845630407333374,
      "step": 10
    },
    {
      "dpo_loss": 0.6825469136238098,
      "dpo_wo_beta": -0.8259204626083374,
      "epoch": 0.08502598016060463,
      "grad_norm": 12.374180595083178,
      "learning_rate": 1.4150943396226415e-06,
      "logits": -1.4932299852371216,
      "logps": -81.52880096435547,
      "loss": 0.6814,
      "objective": 0.6825469136238098,
      "ranking_idealized": 0.5916666388511658,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.512499988079071,
      "regularize": 0.6825469136238098,
      "step": 15
    },
    {
      "dpo_loss": 0.6950914263725281,
      "dpo_wo_beta": -1.2390469312667847,
      "epoch": 0.11336797354747284,
      "grad_norm": 14.839934392200913,
      "learning_rate": 1.8867924528301889e-06,
      "logits": -1.5371100902557373,
      "logps": -82.72624969482422,
      "loss": 0.6711,
      "objective": 0.6950914263725281,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.5375000238418579,
      "ranking_simple": 0.5249999761581421,
      "regularize": 0.6950914263725281,
      "step": 20
    },
    {
      "dpo_loss": 0.6556071043014526,
      "dpo_wo_beta": -1.110619068145752,
      "epoch": 0.14170996693434104,
      "grad_norm": 12.89805052529156,
      "learning_rate": 2.358490566037736e-06,
      "logits": -1.6399922370910645,
      "logps": -81.59695434570312,
      "loss": 0.6589,
      "objective": 0.6556071043014526,
      "ranking_idealized": 0.5958333611488342,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.5249999761581421,
      "regularize": 0.6556071043014526,
      "step": 25
    },
    {
      "dpo_loss": 0.6518108248710632,
      "dpo_wo_beta": -1.2506839036941528,
      "epoch": 0.17005196032120926,
      "grad_norm": 12.64998937636519,
      "learning_rate": 2.830188679245283e-06,
      "logits": -1.6404598951339722,
      "logps": -83.20111846923828,
      "loss": 0.6451,
      "objective": 0.6518108248710632,
      "ranking_idealized": 0.6041666865348816,
      "ranking_idealized_expo": 0.5291666388511658,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.6518108248710632,
      "step": 30
    },
    {
      "dpo_loss": 0.6226770877838135,
      "dpo_wo_beta": -1.394917368888855,
      "epoch": 0.19839395370807747,
      "grad_norm": 13.760162421635227,
      "learning_rate": 3.30188679245283e-06,
      "logits": -1.6237396001815796,
      "logps": -87.80964660644531,
      "loss": 0.6189,
      "objective": 0.6226770877838135,
      "ranking_idealized": 0.612500011920929,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.5583333373069763,
      "regularize": 0.6226770877838135,
      "step": 35
    },
    {
      "dpo_loss": 0.5924390554428101,
      "dpo_wo_beta": -1.422450304031372,
      "epoch": 0.22673594709494568,
      "grad_norm": 16.810886476613117,
      "learning_rate": 3.7735849056603777e-06,
      "logits": -1.620682954788208,
      "logps": -91.93690490722656,
      "loss": 0.6076,
      "objective": 0.5924390554428101,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.5708333253860474,
      "regularize": 0.5924390554428101,
      "step": 40
    },
    {
      "dpo_loss": 0.573756217956543,
      "dpo_wo_beta": -1.3691534996032715,
      "epoch": 0.25507794048181387,
      "grad_norm": 13.798774501924722,
      "learning_rate": 4.245283018867925e-06,
      "logits": -1.7814558744430542,
      "logps": -92.24474334716797,
      "loss": 0.5989,
      "objective": 0.573756217956543,
      "ranking_idealized": 0.6208333373069763,
      "ranking_idealized_expo": 0.5375000238418579,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.573756217956543,
      "step": 45
    },
    {
      "dpo_loss": 0.5726417899131775,
      "dpo_wo_beta": -1.3605374097824097,
      "epoch": 0.2834199338686821,
      "grad_norm": 12.568473894025988,
      "learning_rate": 4.716981132075472e-06,
      "logits": -1.808895468711853,
      "logps": -90.65751647949219,
      "loss": 0.5954,
      "objective": 0.5726417899131775,
      "ranking_idealized": 0.5416666865348816,
      "ranking_idealized_expo": 0.42500001192092896,
      "ranking_simple": 0.574999988079071,
      "regularize": 0.5726417899131775,
      "step": 50
    },
    {
      "epoch": 0.300425129900803,
      "eval_dpo_loss": 0.7112604975700378,
      "eval_dpo_wo_beta": -2.2659413814544678,
      "eval_logits": -1.892814040184021,
      "eval_logps": -101.36742401123047,
      "eval_loss": 0.6816489100456238,
      "eval_objective": 0.7112604975700378,
      "eval_ranking_idealized": 0.5888429880142212,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5237603187561035,
      "eval_regularize": 0.7112604975700378,
      "eval_runtime": 211.6587,
      "eval_samples_per_second": 27.355,
      "eval_steps_per_second": 1.143,
      "step": 53
    },
    {
      "dpo_loss": 0.5827316045761108,
      "dpo_wo_beta": -1.6213361024856567,
      "epoch": 0.3117619272555503,
      "grad_norm": 14.442715913160086,
      "learning_rate": 4.999781286194085e-06,
      "logits": -1.8762638568878174,
      "logps": -93.41423797607422,
      "loss": 0.5721,
      "objective": 0.5827316045761108,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.6041666865348816,
      "regularize": 0.5827316045761108,
      "step": 55
    },
    {
      "dpo_loss": 0.5553872585296631,
      "dpo_wo_beta": -1.6468366384506226,
      "epoch": 0.3401039206424185,
      "grad_norm": 13.845514282811145,
      "learning_rate": 4.997321195347154e-06,
      "logits": -1.8914529085159302,
      "logps": -90.59642028808594,
      "loss": 0.5756,
      "objective": 0.5553872585296631,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.5791666507720947,
      "regularize": 0.5553872585296631,
      "step": 60
    },
    {
      "dpo_loss": 0.5302771329879761,
      "dpo_wo_beta": -1.3166770935058594,
      "epoch": 0.3684459140292867,
      "grad_norm": 10.846857687148022,
      "learning_rate": 4.992130320438411e-06,
      "logits": -1.8399535417556763,
      "logps": -86.60197448730469,
      "loss": 0.5586,
      "objective": 0.5302771329879761,
      "ranking_idealized": 0.5708333253860474,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.6208333373069763,
      "regularize": 0.5302771329879761,
      "step": 65
    },
    {
      "dpo_loss": 0.5711485743522644,
      "dpo_wo_beta": -1.7437169551849365,
      "epoch": 0.39678790741615494,
      "grad_norm": 13.787840238803502,
      "learning_rate": 4.984214337613357e-06,
      "logits": -1.8178967237472534,
      "logps": -91.10688781738281,
      "loss": 0.5701,
      "objective": 0.5711485743522644,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5874999761581421,
      "regularize": 0.5711485743522644,
      "step": 70
    },
    {
      "dpo_loss": 0.523643434047699,
      "dpo_wo_beta": -1.669514536857605,
      "epoch": 0.42512990080302315,
      "grad_norm": 13.192298437287352,
      "learning_rate": 4.97358190288299e-06,
      "logits": -1.8182169198989868,
      "logps": -94.8000717163086,
      "loss": 0.5205,
      "objective": 0.523643434047699,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.5458333492279053,
      "ranking_simple": 0.6583333611488342,
      "regularize": 0.523643434047699,
      "step": 75
    },
    {
      "dpo_loss": 0.51079261302948,
      "dpo_wo_beta": -1.7271808385849,
      "epoch": 0.45347189418989137,
      "grad_norm": 15.151373786996814,
      "learning_rate": 4.9602446426585845e-06,
      "logits": -1.8920824527740479,
      "logps": -93.58238220214844,
      "loss": 0.5285,
      "objective": 0.51079261302948,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.6458333134651184,
      "regularize": 0.51079261302948,
      "step": 80
    },
    {
      "dpo_loss": 0.5066012144088745,
      "dpo_wo_beta": -1.5956443548202515,
      "epoch": 0.4818138875767596,
      "grad_norm": 12.328960275584794,
      "learning_rate": 4.944217141038379e-06,
      "logits": -1.8741406202316284,
      "logps": -87.06742858886719,
      "loss": 0.5202,
      "objective": 0.5066012144088745,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.6208333373069763,
      "regularize": 0.5066012144088745,
      "step": 85
    },
    {
      "dpo_loss": 0.5358369946479797,
      "dpo_wo_beta": -1.9357556104660034,
      "epoch": 0.5101558809636277,
      "grad_norm": 12.694483590051824,
      "learning_rate": 4.925516923860083e-06,
      "logits": -1.7968534231185913,
      "logps": -86.77802276611328,
      "loss": 0.4858,
      "objective": 0.5358369946479797,
      "ranking_idealized": 0.5458333492279053,
      "ranking_idealized_expo": 0.4749999940395355,
      "ranking_simple": 0.5874999761581421,
      "regularize": 0.5358369946479797,
      "step": 90
    },
    {
      "dpo_loss": 0.4783257842063904,
      "dpo_wo_beta": -1.9098786115646362,
      "epoch": 0.538497874350496,
      "grad_norm": 14.474706973531484,
      "learning_rate": 4.904164439536626e-06,
      "logits": -1.8568389415740967,
      "logps": -88.12813568115234,
      "loss": 0.4865,
      "objective": 0.4783257842063904,
      "ranking_idealized": 0.6291666626930237,
      "ranking_idealized_expo": 0.5458333492279053,
      "ranking_simple": 0.6916666626930237,
      "regularize": 0.4783257842063904,
      "step": 95
    },
    {
      "dpo_loss": 0.4654810130596161,
      "dpo_wo_beta": -1.9254087209701538,
      "epoch": 0.5668398677373642,
      "grad_norm": 13.577084707122001,
      "learning_rate": 4.880183036696123e-06,
      "logits": -1.938937783241272,
      "logps": -92.29436492919922,
      "loss": 0.5016,
      "objective": 0.4654810130596161,
      "ranking_idealized": 0.612500011920929,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.6875,
      "regularize": 0.4654810130596161,
      "step": 100
    },
    {
      "dpo_loss": 0.4374677240848541,
      "dpo_wo_beta": -1.4267934560775757,
      "epoch": 0.5951818611242324,
      "grad_norm": 11.14545328639218,
      "learning_rate": 4.853598938650487e-06,
      "logits": -1.8158982992172241,
      "logps": -90.21449279785156,
      "loss": 0.4618,
      "objective": 0.4374677240848541,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.6666666865348816,
      "regularize": 0.4374677240848541,
      "step": 105
    },
    {
      "epoch": 0.600850259801606,
      "eval_dpo_loss": 0.6936022639274597,
      "eval_dpo_wo_beta": -2.462427854537964,
      "eval_logits": -1.9007418155670166,
      "eval_logps": -94.35714721679688,
      "eval_loss": 0.6912521123886108,
      "eval_objective": 0.6936022639274597,
      "eval_ranking_idealized": 0.5888429880142212,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5351239442825317,
      "eval_regularize": 0.6936022639274597,
      "eval_runtime": 210.2297,
      "eval_samples_per_second": 27.541,
      "eval_steps_per_second": 1.151,
      "step": 106
    },
    {
      "dpo_loss": 0.47933149337768555,
      "dpo_wo_beta": -1.9683055877685547,
      "epoch": 0.6235238545111006,
      "grad_norm": 12.39392340166307,
      "learning_rate": 4.824441214720629e-06,
      "logits": -1.9334439039230347,
      "logps": -87.35523223876953,
      "loss": 0.4633,
      "objective": 0.47933149337768555,
      "ranking_idealized": 0.5708333253860474,
      "ranking_idealized_expo": 0.4541666805744171,
      "ranking_simple": 0.6625000238418579,
      "regularize": 0.47933149337768555,
      "step": 110
    },
    {
      "dpo_loss": 0.4749464690685272,
      "dpo_wo_beta": -1.7375919818878174,
      "epoch": 0.6518658478979689,
      "grad_norm": 12.612865651893962,
      "learning_rate": 4.7927417484495756e-06,
      "logits": -1.9057692289352417,
      "logps": -87.68991088867188,
      "loss": 0.4712,
      "objective": 0.4749464690685272,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.6333333253860474,
      "regularize": 0.4749464690685272,
      "step": 115
    },
    {
      "dpo_loss": 0.4848935306072235,
      "dpo_wo_beta": -1.9273093938827515,
      "epoch": 0.680207841284837,
      "grad_norm": 13.836239066838136,
      "learning_rate": 4.758535202738287e-06,
      "logits": -1.8775906562805176,
      "logps": -87.8878173828125,
      "loss": 0.4641,
      "objective": 0.4848935306072235,
      "ranking_idealized": 0.5874999761581421,
      "ranking_idealized_expo": 0.5291666388511658,
      "ranking_simple": 0.6625000238418579,
      "regularize": 0.4848935306072235,
      "step": 120
    },
    {
      "dpo_loss": 0.4785127639770508,
      "dpo_wo_beta": -1.814666748046875,
      "epoch": 0.7085498346717053,
      "grad_norm": 12.105170057238437,
      "learning_rate": 4.721858981942284e-06,
      "logits": -1.8346068859100342,
      "logps": -86.40522766113281,
      "loss": 0.4801,
      "objective": 0.4785127639770508,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.6875,
      "regularize": 0.4785127639770508,
      "step": 125
    },
    {
      "dpo_loss": 0.4548089802265167,
      "dpo_wo_beta": -1.4164987802505493,
      "epoch": 0.7368918280585735,
      "grad_norm": 11.895980627109102,
      "learning_rate": 4.682753190970533e-06,
      "logits": -1.9488608837127686,
      "logps": -79.42195129394531,
      "loss": 0.4538,
      "objective": 0.4548089802265167,
      "ranking_idealized": 0.5416666865348816,
      "ranking_idealized_expo": 0.44999998807907104,
      "ranking_simple": 0.6291666626930237,
      "regularize": 0.4548089802265167,
      "step": 130
    },
    {
      "dpo_loss": 0.49760884046554565,
      "dpo_wo_beta": -1.994195818901062,
      "epoch": 0.7652338214454416,
      "grad_norm": 12.298776298341995,
      "learning_rate": 4.641260591431315e-06,
      "logits": -1.9813282489776611,
      "logps": -82.40634155273438,
      "loss": 0.4433,
      "objective": 0.49760884046554565,
      "ranking_idealized": 0.5874999761581421,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.6583333611488342,
      "regularize": 0.49760884046554565,
      "step": 135
    },
    {
      "dpo_loss": 0.41459351778030396,
      "dpo_wo_beta": -1.187635064125061,
      "epoch": 0.7935758148323099,
      "grad_norm": 12.618720178096575,
      "learning_rate": 4.597426554873037e-06,
      "logits": -1.97609281539917,
      "logps": -83.44467163085938,
      "loss": 0.4236,
      "objective": 0.41459351778030396,
      "ranking_idealized": 0.5791666507720947,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.6791666746139526,
      "regularize": 0.41459351778030396,
      "step": 140
    },
    {
      "dpo_loss": 0.4073801636695862,
      "dpo_wo_beta": -1.311059832572937,
      "epoch": 0.821917808219178,
      "grad_norm": 14.417917904409194,
      "learning_rate": 4.551299013171111e-06,
      "logits": -2.0718839168548584,
      "logps": -84.2674560546875,
      "loss": 0.4215,
      "objective": 0.4073801636695862,
      "ranking_idealized": 0.5791666507720947,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.6916666626930237,
      "regularize": 0.4073801636695862,
      "step": 145
    },
    {
      "dpo_loss": 0.4207518398761749,
      "dpo_wo_beta": -1.50857675075531,
      "epoch": 0.8502598016060463,
      "grad_norm": 11.543599868064442,
      "learning_rate": 4.502928406115152e-06,
      "logits": -2.0730583667755127,
      "logps": -82.68958282470703,
      "loss": 0.4276,
      "objective": 0.4207518398761749,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.7208333611488342,
      "regularize": 0.4207518398761749,
      "step": 150
    },
    {
      "dpo_loss": 0.3847941756248474,
      "dpo_wo_beta": -1.4449684619903564,
      "epoch": 0.8786017949929145,
      "grad_norm": 12.08771803065001,
      "learning_rate": 4.452367626253805e-06,
      "logits": -2.0991933345794678,
      "logps": -85.211181640625,
      "loss": 0.3986,
      "objective": 0.3847941756248474,
      "ranking_idealized": 0.612500011920929,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.7250000238418579,
      "regularize": 0.3847941756248474,
      "step": 155
    },
    {
      "epoch": 0.9012753897024091,
      "eval_dpo_loss": 0.7214789390563965,
      "eval_dpo_wo_beta": -3.1229145526885986,
      "eval_logits": -2.1450352668762207,
      "eval_logps": -95.60012817382812,
      "eval_loss": 0.7013870477676392,
      "eval_objective": 0.7214789390563965,
      "eval_ranking_idealized": 0.5888429880142212,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5351239442825317,
      "eval_regularize": 0.7214789390563965,
      "eval_runtime": 210.3593,
      "eval_samples_per_second": 27.524,
      "eval_steps_per_second": 1.15,
      "step": 159
    },
    {
      "dpo_loss": 0.4162478744983673,
      "dpo_wo_beta": -1.6461573839187622,
      "epoch": 0.9069437883797827,
      "grad_norm": 12.82345397067452,
      "learning_rate": 4.399671961057523e-06,
      "logits": -2.0759384632110596,
      "logps": -89.25846862792969,
      "loss": 0.4236,
      "objective": 0.4162478744983673,
      "ranking_idealized": 0.5708333253860474,
      "ranking_idealized_expo": 0.49166667461395264,
      "ranking_simple": 0.699999988079071,
      "regularize": 0.4162478744983673,
      "step": 160
    },
    {
      "dpo_loss": 0.41358453035354614,
      "dpo_wo_beta": -1.648630976676941,
      "epoch": 0.9352857817666509,
      "grad_norm": 12.860537676624453,
      "learning_rate": 4.3448990324625244e-06,
      "logits": -2.024477481842041,
      "logps": -88.03329467773438,
      "loss": 0.4026,
      "objective": 0.41358453035354614,
      "ranking_idealized": 0.6041666865348816,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.7333333492279053,
      "regularize": 0.41358453035354614,
      "step": 165
    },
    {
      "dpo_loss": 0.378000408411026,
      "dpo_wo_beta": -1.2966532707214355,
      "epoch": 0.9636277751535192,
      "grad_norm": 11.533711130228069,
      "learning_rate": 4.288108733862064e-06,
      "logits": -2.042527437210083,
      "logps": -90.26854705810547,
      "loss": 0.3925,
      "objective": 0.378000408411026,
      "ranking_idealized": 0.5791666507720947,
      "ranking_idealized_expo": 0.512499988079071,
      "ranking_simple": 0.7166666388511658,
      "regularize": 0.378000408411026,
      "step": 170
    },
    {
      "dpo_loss": 0.3764660954475403,
      "dpo_wo_beta": -1.3978971242904663,
      "epoch": 0.9919697685403873,
      "grad_norm": 12.165192869157089,
      "learning_rate": 4.229363164613874e-06,
      "logits": -2.0610477924346924,
      "logps": -89.8354721069336,
      "loss": 0.3793,
      "objective": 0.3764660954475403,
      "ranking_idealized": 0.6541666388511658,
      "ranking_idealized_expo": 0.574999988079071,
      "ranking_simple": 0.7916666865348816,
      "regularize": 0.3764660954475403,
      "step": 175
    },
    {
      "dpo_loss": 0.27626773715019226,
      "dpo_wo_beta": -0.8504549860954285,
      "epoch": 1.0203117619272555,
      "grad_norm": 10.141692447282386,
      "learning_rate": 4.168726562135432e-06,
      "logits": -2.2514243125915527,
      "logps": -90.8476333618164,
      "loss": 0.2852,
      "objective": 0.27626773715019226,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.8083333373069763,
      "regularize": 0.27626773715019226,
      "step": 180
    },
    {
      "dpo_loss": 0.23696589469909668,
      "dpo_wo_beta": -0.6947117447853088,
      "epoch": 1.0486537553141237,
      "grad_norm": 13.78702272812957,
      "learning_rate": 4.106265231661292e-06,
      "logits": -2.158977746963501,
      "logps": -95.00120544433594,
      "loss": 0.2429,
      "objective": 0.23696589469909668,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.49166667461395264,
      "ranking_simple": 0.8083333373069763,
      "regularize": 0.23696589469909668,
      "step": 185
    },
    {
      "dpo_loss": 0.26388806104660034,
      "dpo_wo_beta": -0.9112051725387573,
      "epoch": 1.076995748700992,
      "grad_norm": 14.740228375586371,
      "learning_rate": 4.042047473739278e-06,
      "logits": -2.1533920764923096,
      "logps": -101.71949768066406,
      "loss": 0.2517,
      "objective": 0.26388806104660034,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.5541666746139526,
      "ranking_simple": 0.8416666388511658,
      "regularize": 0.26388806104660034,
      "step": 190
    },
    {
      "dpo_loss": 0.2244579941034317,
      "dpo_wo_beta": -0.6430780291557312,
      "epoch": 1.10533774208786,
      "grad_norm": 10.169064121599527,
      "learning_rate": 3.976143509544843e-06,
      "logits": -2.1589295864105225,
      "logps": -96.5248031616211,
      "loss": 0.2467,
      "objective": 0.2244579941034317,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.8083333373069763,
      "regularize": 0.2244579941034317,
      "step": 195
    },
    {
      "dpo_loss": 0.24179764091968536,
      "dpo_wo_beta": -0.6332272291183472,
      "epoch": 1.1336797354747283,
      "grad_norm": 9.444774343787891,
      "learning_rate": 3.908625404095242e-06,
      "logits": -2.2753493785858154,
      "logps": -91.93312072753906,
      "loss": 0.2563,
      "objective": 0.24179764091968536,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.8458333611488342,
      "regularize": 0.24179764091968536,
      "step": 200
    },
    {
      "dpo_loss": 0.25683078169822693,
      "dpo_wo_beta": -0.8531176447868347,
      "epoch": 1.1620217288615966,
      "grad_norm": 9.240319326762517,
      "learning_rate": 3.839566987447492e-06,
      "logits": -2.2432618141174316,
      "logps": -91.3159408569336,
      "loss": 0.2584,
      "objective": 0.25683078169822693,
      "ranking_idealized": 0.5708333253860474,
      "ranking_idealized_expo": 0.49166667461395264,
      "ranking_simple": 0.8166666626930237,
      "regularize": 0.25683078169822693,
      "step": 205
    },
    {
      "dpo_loss": 0.24292893707752228,
      "dpo_wo_beta": -0.8205318450927734,
      "epoch": 1.1903637222484649,
      "grad_norm": 9.283856100785183,
      "learning_rate": 3.7690437739662928e-06,
      "logits": -2.2361652851104736,
      "logps": -90.6613998413086,
      "loss": 0.2551,
      "objective": 0.24292893707752228,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.800000011920929,
      "regularize": 0.24292893707752228,
      "step": 210
    },
    {
      "epoch": 1.201700519603212,
      "eval_dpo_loss": 0.7525234222412109,
      "eval_dpo_wo_beta": -3.7749528884887695,
      "eval_logits": -2.267778158187866,
      "eval_logps": -98.14269256591797,
      "eval_loss": 0.7350714206695557,
      "eval_objective": 0.7525234222412109,
      "eval_ranking_idealized": 0.5888429880142212,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5371900796890259,
      "eval_regularize": 0.7525234222412109,
      "eval_runtime": 210.8898,
      "eval_samples_per_second": 27.455,
      "eval_steps_per_second": 1.148,
      "step": 212
    },
    {
      "dpo_loss": 0.289533793926239,
      "dpo_wo_beta": -0.8810125589370728,
      "epoch": 1.2187057156353331,
      "grad_norm": 10.72372972136692,
      "learning_rate": 3.697132879750174e-06,
      "logits": -2.1757090091705322,
      "logps": -93.64250183105469,
      "loss": 0.2578,
      "objective": 0.289533793926239,
      "ranking_idealized": 0.5583333373069763,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.7875000238418579,
      "regularize": 0.289533793926239,
      "step": 215
    },
    {
      "dpo_loss": 0.25134381651878357,
      "dpo_wo_beta": -0.8703542947769165,
      "epoch": 1.2470477090222012,
      "grad_norm": 12.940604838816247,
      "learning_rate": 3.6239129383061764e-06,
      "logits": -2.121750593185425,
      "logps": -94.44015502929688,
      "loss": 0.2676,
      "objective": 0.25134381651878357,
      "ranking_idealized": 0.6541666388511658,
      "ranking_idealized_expo": 0.6041666865348816,
      "ranking_simple": 0.8208333253860474,
      "regularize": 0.25134381651878357,
      "step": 220
    },
    {
      "dpo_loss": 0.23937886953353882,
      "dpo_wo_beta": -0.7396827936172485,
      "epoch": 1.2753897024090695,
      "grad_norm": 9.645711793319885,
      "learning_rate": 3.5494640145652647e-06,
      "logits": -2.0901684761047363,
      "logps": -94.10260772705078,
      "loss": 0.2637,
      "objective": 0.23937886953353882,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.8541666865348816,
      "regularize": 0.23937886953353882,
      "step": 225
    },
    {
      "dpo_loss": 0.2818019688129425,
      "dpo_wo_beta": -1.1170729398727417,
      "epoch": 1.3037316957959377,
      "grad_norm": 8.80210598601974,
      "learning_rate": 3.4738675173325008e-06,
      "logits": -1.9860222339630127,
      "logps": -92.9978256225586,
      "loss": 0.2776,
      "objective": 0.2818019688129425,
      "ranking_idealized": 0.5791666507720947,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.7749999761581421,
      "regularize": 0.2818019688129425,
      "step": 230
    },
    {
      "dpo_loss": 0.22621506452560425,
      "dpo_wo_beta": -0.35843732953071594,
      "epoch": 1.3320736891828058,
      "grad_norm": 9.267612473930496,
      "learning_rate": 3.397206110267713e-06,
      "logits": -2.1131467819213867,
      "logps": -87.49403381347656,
      "loss": 0.2618,
      "objective": 0.22621506452560425,
      "ranking_idealized": 0.6041666865348816,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.8333333134651184,
      "regularize": 0.22621506452560425,
      "step": 235
    },
    {
      "dpo_loss": 0.23632274568080902,
      "dpo_wo_beta": -0.6697984933853149,
      "epoch": 1.360415682569674,
      "grad_norm": 10.68594080832048,
      "learning_rate": 3.3195636214939943e-06,
      "logits": -2.130047559738159,
      "logps": -91.7619857788086,
      "loss": 0.2584,
      "objective": 0.23632274568080902,
      "ranking_idealized": 0.5541666746139526,
      "ranking_idealized_expo": 0.4791666567325592,
      "ranking_simple": 0.8125,
      "regularize": 0.23632274568080902,
      "step": 240
    },
    {
      "dpo_loss": 0.2982023358345032,
      "dpo_wo_beta": -1.1124054193496704,
      "epoch": 1.3887576759565423,
      "grad_norm": 10.330360151122868,
      "learning_rate": 3.2410249519328848e-06,
      "logits": -2.1718757152557373,
      "logps": -93.45353698730469,
      "loss": 0.2692,
      "objective": 0.2982023358345032,
      "ranking_idealized": 0.5916666388511658,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.8041666746139526,
      "regularize": 0.2982023358345032,
      "step": 245
    },
    {
      "dpo_loss": 0.2403133064508438,
      "dpo_wo_beta": -0.7000442147254944,
      "epoch": 1.4170996693434104,
      "grad_norm": 13.026578288520353,
      "learning_rate": 3.1616759824664543e-06,
      "logits": -2.145325183868408,
      "logps": -94.18195343017578,
      "loss": 0.269,
      "objective": 0.2403133064508438,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.47083333134651184,
      "ranking_simple": 0.824999988079071,
      "regularize": 0.2403133064508438,
      "step": 250
    },
    {
      "dpo_loss": 0.23977436125278473,
      "dpo_wo_beta": -0.5784927010536194,
      "epoch": 1.4454416627302786,
      "grad_norm": 10.959901566104394,
      "learning_rate": 3.081603480027826e-06,
      "logits": -2.108074426651001,
      "logps": -94.5383529663086,
      "loss": 0.2625,
      "objective": 0.23977436125278473,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5458333492279053,
      "ranking_simple": 0.8374999761581421,
      "regularize": 0.23977436125278473,
      "step": 255
    },
    {
      "dpo_loss": 0.25297579169273376,
      "dpo_wo_beta": -0.7996426820755005,
      "epoch": 1.473783656117147,
      "grad_norm": 9.578050078679867,
      "learning_rate": 3.0008950027228035e-06,
      "logits": -2.1828908920288086,
      "logps": -92.77781677246094,
      "loss": 0.232,
      "objective": 0.25297579169273376,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.48750001192092896,
      "ranking_simple": 0.8041666746139526,
      "regularize": 0.25297579169273376,
      "step": 260
    },
    {
      "dpo_loss": 0.2830916941165924,
      "dpo_wo_beta": -1.124144434928894,
      "epoch": 1.5021256495040152,
      "grad_norm": 11.27765707111355,
      "learning_rate": 2.9196388040863695e-06,
      "logits": -2.1150081157684326,
      "logps": -95.04662322998047,
      "loss": 0.2623,
      "objective": 0.2830916941165924,
      "ranking_idealized": 0.5791666507720947,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.8041666746139526,
      "regularize": 0.2830916941165924,
      "step": 265
    },
    {
      "epoch": 1.5021256495040152,
      "eval_dpo_loss": 0.7739136815071106,
      "eval_dpo_wo_beta": -4.163427829742432,
      "eval_logits": -2.1478331089019775,
      "eval_logps": -100.8313217163086,
      "eval_loss": 0.7400166392326355,
      "eval_objective": 0.7739136815071106,
      "eval_ranking_idealized": 0.5888429880142212,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.53925621509552,
      "eval_regularize": 0.7739136815071106,
      "eval_runtime": 210.8657,
      "eval_samples_per_second": 27.458,
      "eval_steps_per_second": 1.148,
      "step": 265
    },
    {
      "dpo_loss": 0.2930367887020111,
      "dpo_wo_beta": -1.3651045560836792,
      "epoch": 1.5304676428908834,
      "grad_norm": 11.715215816813723,
      "learning_rate": 2.8379237365787426e-06,
      "logits": -2.035703182220459,
      "logps": -97.7331771850586,
      "loss": 0.253,
      "objective": 0.2930367887020111,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.7916666865348816,
      "regularize": 0.2930367887020111,
      "step": 270
    },
    {
      "dpo_loss": 0.24886849522590637,
      "dpo_wo_beta": -0.8069366216659546,
      "epoch": 1.5588096362777515,
      "grad_norm": 8.958944325794365,
      "learning_rate": 2.7558391544265127e-06,
      "logits": -1.9700883626937866,
      "logps": -97.53855895996094,
      "loss": 0.2491,
      "objective": 0.24886849522590637,
      "ranking_idealized": 0.6208333373069763,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.8125,
      "regularize": 0.24886849522590637,
      "step": 275
    },
    {
      "dpo_loss": 0.22936613857746124,
      "dpo_wo_beta": -0.6120084524154663,
      "epoch": 1.5871516296646198,
      "grad_norm": 10.814739938498821,
      "learning_rate": 2.6734748159151104e-06,
      "logits": -1.9118597507476807,
      "logps": -98.06639099121094,
      "loss": 0.2491,
      "objective": 0.22936613857746124,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.8374999761581421,
      "regularize": 0.22936613857746124,
      "step": 280
    },
    {
      "dpo_loss": 0.22401383519172668,
      "dpo_wo_beta": -0.5180224776268005,
      "epoch": 1.615493623051488,
      "grad_norm": 11.270657822712987,
      "learning_rate": 2.5909207852394363e-06,
      "logits": -1.9585484266281128,
      "logps": -100.70836639404297,
      "loss": 0.2348,
      "objective": 0.22401383519172668,
      "ranking_idealized": 0.5874999761581421,
      "ranking_idealized_expo": 0.512499988079071,
      "ranking_simple": 0.8291666507720947,
      "regularize": 0.22401383519172668,
      "step": 285
    },
    {
      "dpo_loss": 0.2646006941795349,
      "dpo_wo_beta": -0.7763135433197021,
      "epoch": 1.643835616438356,
      "grad_norm": 10.585292794409252,
      "learning_rate": 2.508267334019988e-06,
      "logits": -1.9566444158554077,
      "logps": -97.0122299194336,
      "loss": 0.2532,
      "objective": 0.2646006941795349,
      "ranking_idealized": 0.612500011920929,
      "ranking_idealized_expo": 0.5625,
      "ranking_simple": 0.8041666746139526,
      "regularize": 0.2646006941795349,
      "step": 290
    },
    {
      "dpo_loss": 0.23113909363746643,
      "dpo_wo_beta": -0.6497251987457275,
      "epoch": 1.6721776098252243,
      "grad_norm": 11.90240881956814,
      "learning_rate": 2.4256048425921693e-06,
      "logits": -1.8574607372283936,
      "logps": -94.91531372070312,
      "loss": 0.2476,
      "objective": 0.23113909363746643,
      "ranking_idealized": 0.5791666507720947,
      "ranking_idealized_expo": 0.4749999940395355,
      "ranking_simple": 0.8333333134651184,
      "regularize": 0.23113909363746643,
      "step": 295
    },
    {
      "dpo_loss": 0.22116926312446594,
      "dpo_wo_beta": -0.6268281936645508,
      "epoch": 1.7005196032120926,
      "grad_norm": 11.745161783871675,
      "learning_rate": 2.3430237011767166e-06,
      "logits": -1.895004153251648,
      "logps": -97.79885864257812,
      "loss": 0.2266,
      "objective": 0.22116926312446594,
      "ranking_idealized": 0.6583333611488342,
      "ranking_idealized_expo": 0.574999988079071,
      "ranking_simple": 0.8333333134651184,
      "regularize": 0.22116926312446594,
      "step": 300
    },
    {
      "dpo_loss": 0.24756571650505066,
      "dpo_wo_beta": -0.9131773114204407,
      "epoch": 1.7288615965989607,
      "grad_norm": 12.299641904512029,
      "learning_rate": 2.2606142110393248e-06,
      "logits": -1.8061485290527344,
      "logps": -96.69060516357422,
      "loss": 0.2379,
      "objective": 0.24756571650505066,
      "ranking_idealized": 0.5625,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.8291666507720947,
      "regularize": 0.24756571650505066,
      "step": 305
    },
    {
      "dpo_loss": 0.2321903556585312,
      "dpo_wo_beta": -0.6867564916610718,
      "epoch": 1.7572035899858292,
      "grad_norm": 13.489735935272718,
      "learning_rate": 2.1784664857475356e-06,
      "logits": -1.8388514518737793,
      "logps": -95.04447937011719,
      "loss": 0.2456,
      "objective": 0.2321903556585312,
      "ranking_idealized": 0.6083333492279053,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.824999988079071,
      "regularize": 0.2321903556585312,
      "step": 310
    },
    {
      "dpo_loss": 0.2901044189929962,
      "dpo_wo_beta": -1.1286156177520752,
      "epoch": 1.7855455833726972,
      "grad_norm": 10.887596324980125,
      "learning_rate": 2.096670352632873e-06,
      "logits": -1.75984525680542,
      "logps": -94.63612365722656,
      "loss": 0.2571,
      "objective": 0.2901044189929962,
      "ranking_idealized": 0.6041666865348816,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.8083333373069763,
      "regularize": 0.2901044189929962,
      "step": 315
    },
    {
      "epoch": 1.8025507794048181,
      "eval_dpo_loss": 0.7664583325386047,
      "eval_dpo_wo_beta": -4.09501838684082,
      "eval_logits": -1.9888346195220947,
      "eval_logps": -102.3712158203125,
      "eval_loss": 0.7400712966918945,
      "eval_objective": 0.7664583325386047,
      "eval_ranking_idealized": 0.5888429880142212,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.53925621509552,
      "eval_regularize": 0.7664583325386047,
      "eval_runtime": 210.274,
      "eval_samples_per_second": 27.535,
      "eval_steps_per_second": 1.151,
      "step": 318
    },
    {
      "dpo_loss": 0.2219768464565277,
      "dpo_wo_beta": -0.47742757201194763,
      "epoch": 1.8138875767595655,
      "grad_norm": 11.029480506309918,
      "learning_rate": 2.01531525456598e-06,
      "logits": -1.9175788164138794,
      "logps": -99.74655151367188,
      "loss": 0.2404,
      "objective": 0.2219768464565277,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.824999988079071,
      "regularize": 0.2219768464565277,
      "step": 320
    },
    {
      "dpo_loss": 0.24908211827278137,
      "dpo_wo_beta": -0.8014059066772461,
      "epoch": 1.8422295701464337,
      "grad_norm": 12.92850322071669,
      "learning_rate": 1.93449015215215e-06,
      "logits": -2.0084919929504395,
      "logps": -101.09780883789062,
      "loss": 0.2586,
      "objective": 0.24908211827278137,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.8333333134651184,
      "regularize": 0.24908211827278137,
      "step": 325
    },
    {
      "dpo_loss": 0.1984507441520691,
      "dpo_wo_beta": -0.3766098618507385,
      "epoch": 1.8705715635333018,
      "grad_norm": 10.415606016359964,
      "learning_rate": 1.8542834264542091e-06,
      "logits": -1.851909875869751,
      "logps": -94.5366439819336,
      "loss": 0.2496,
      "objective": 0.1984507441520691,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.4791666567325592,
      "ranking_simple": 0.8291666507720947,
      "regularize": 0.1984507441520691,
      "step": 330
    },
    {
      "dpo_loss": 0.26707762479782104,
      "dpo_wo_beta": -0.9339324831962585,
      "epoch": 1.89891355692017,
      "grad_norm": 10.078352873471246,
      "learning_rate": 1.7747827823491253e-06,
      "logits": -1.9827288389205933,
      "logps": -94.26249694824219,
      "loss": 0.2463,
      "objective": 0.26707762479782104,
      "ranking_idealized": 0.5958333611488342,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.8125,
      "regularize": 0.26707762479782104,
      "step": 335
    },
    {
      "dpo_loss": 0.2447831928730011,
      "dpo_wo_beta": -0.7387041449546814,
      "epoch": 1.9272555503070383,
      "grad_norm": 10.88136655004607,
      "learning_rate": 1.6960751526240122e-06,
      "logits": -1.9671465158462524,
      "logps": -98.63937377929688,
      "loss": 0.2399,
      "objective": 0.2447831928730011,
      "ranking_idealized": 0.6416666507720947,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.8583333492279053,
      "regularize": 0.2447831928730011,
      "step": 340
    },
    {
      "dpo_loss": 0.2123527079820633,
      "dpo_wo_beta": -0.5544185638427734,
      "epoch": 1.9555975436939064,
      "grad_norm": 11.18260747105762,
      "learning_rate": 1.6182466029163974e-06,
      "logits": -1.9572845697402954,
      "logps": -100.18721008300781,
      "loss": 0.2211,
      "objective": 0.2123527079820633,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.8208333253860474,
      "regularize": 0.2123527079820633,
      "step": 345
    },
    {
      "dpo_loss": 0.2570362389087677,
      "dpo_wo_beta": -0.7474013566970825,
      "epoch": 1.9839395370807746,
      "grad_norm": 11.061918116138507,
      "learning_rate": 1.541382237602721e-06,
      "logits": -1.8960832357406616,
      "logps": -101.65901947021484,
      "loss": 0.2316,
      "objective": 0.2570362389087677,
      "ranking_idealized": 0.5791666507720947,
      "ranking_idealized_expo": 0.48750001192092896,
      "ranking_simple": 0.7916666865348816,
      "regularize": 0.2570362389087677,
      "step": 350
    },
    {
      "dpo_loss": 0.19961656630039215,
      "dpo_wo_beta": -0.5642960667610168,
      "epoch": 2.012281530467643,
      "grad_norm": 7.569515164252156,
      "learning_rate": 1.465566106737942e-06,
      "logits": -1.8380100727081299,
      "logps": -102.71571350097656,
      "loss": 0.2103,
      "objective": 0.19961656630039215,
      "ranking_idealized": 0.5958333611488342,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.8374999761581421,
      "regularize": 0.19961656630039215,
      "step": 355
    },
    {
      "dpo_loss": 0.11018560826778412,
      "dpo_wo_beta": -0.12253165245056152,
      "epoch": 2.040623523854511,
      "grad_norm": 6.632276986432463,
      "learning_rate": 1.3908811141480408e-06,
      "logits": -1.867693543434143,
      "logps": -103.06665802001953,
      "loss": 0.118,
      "objective": 0.11018560826778412,
      "ranking_idealized": 0.6625000238418579,
      "ranking_idealized_expo": 0.5375000238418579,
      "ranking_simple": 0.9041666388511658,
      "regularize": 0.11018560826778412,
      "step": 360
    },
    {
      "dpo_loss": 0.12077057361602783,
      "dpo_wo_beta": -0.197490856051445,
      "epoch": 2.0689655172413794,
      "grad_norm": 10.213186193965676,
      "learning_rate": 1.3174089267758983e-06,
      "logits": -1.8255099058151245,
      "logps": -110.3724136352539,
      "loss": 0.118,
      "objective": 0.12077057361602783,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.8666666746139526,
      "regularize": 0.12077057361602783,
      "step": 365
    },
    {
      "dpo_loss": 0.1337815225124359,
      "dpo_wo_beta": -0.27523547410964966,
      "epoch": 2.0973075106282475,
      "grad_norm": 9.926730675582434,
      "learning_rate": 1.245229885379699e-06,
      "logits": -1.7588540315628052,
      "logps": -111.99506378173828,
      "loss": 0.1227,
      "objective": 0.1337815225124359,
      "ranking_idealized": 0.5708333253860474,
      "ranking_idealized_expo": 0.4791666567325592,
      "ranking_simple": 0.8916666507720947,
      "regularize": 0.1337815225124359,
      "step": 370
    },
    {
      "epoch": 2.1029759093056213,
      "eval_dpo_loss": 0.9223728179931641,
      "eval_dpo_wo_beta": -6.4510064125061035,
      "eval_logits": -1.8644566535949707,
      "eval_logps": -122.00161743164062,
      "eval_loss": 0.8844180107116699,
      "eval_objective": 0.9223728179931641,
      "eval_ranking_idealized": 0.5888429880142212,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5423553586006165,
      "eval_regularize": 0.9223728179931641,
      "eval_runtime": 210.7356,
      "eval_samples_per_second": 27.475,
      "eval_steps_per_second": 1.148,
      "step": 371
    },
    {
      "dpo_loss": 0.10664375871419907,
      "dpo_wo_beta": -0.2532973289489746,
      "epoch": 2.1256495040151155,
      "grad_norm": 9.740007111179482,
      "learning_rate": 1.1744229166814889e-06,
      "logits": -1.696647047996521,
      "logps": -118.39366149902344,
      "loss": 0.1103,
      "objective": 0.10664375871419907,
      "ranking_idealized": 0.675000011920929,
      "ranking_idealized_expo": 0.5916666388511658,
      "ranking_simple": 0.925000011920929,
      "regularize": 0.10664375871419907,
      "step": 375
    },
    {
      "dpo_loss": 0.12854978442192078,
      "dpo_wo_beta": -0.27664583921432495,
      "epoch": 2.153991497401984,
      "grad_norm": 9.699256456859702,
      "learning_rate": 1.1050654470619602e-06,
      "logits": -1.700494647026062,
      "logps": -114.1063232421875,
      "loss": 0.1208,
      "objective": 0.12854978442192078,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.8999999761581421,
      "regularize": 0.12854978442192078,
      "step": 380
    },
    {
      "dpo_loss": 0.10418140888214111,
      "dpo_wo_beta": -0.09889766573905945,
      "epoch": 2.182333490788852,
      "grad_norm": 9.620361843085416,
      "learning_rate": 1.0372333178958462e-06,
      "logits": -1.8633235692977905,
      "logps": -110.55794525146484,
      "loss": 0.1244,
      "objective": 0.10418140888214111,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.9125000238418579,
      "regularize": 0.10418140888214111,
      "step": 385
    },
    {
      "dpo_loss": 0.12462247163057327,
      "dpo_wo_beta": -0.2658768594264984,
      "epoch": 2.21067548417572,
      "grad_norm": 11.000881222201947,
      "learning_rate": 9.710007026204896e-07,
      "logits": -1.7877620458602905,
      "logps": -112.08268737792969,
      "loss": 0.1204,
      "objective": 0.12462247163057327,
      "ranking_idealized": 0.5249999761581421,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.8833333253860474,
      "regularize": 0.12462247163057327,
      "step": 390
    },
    {
      "dpo_loss": 0.11183874309062958,
      "dpo_wo_beta": -0.3540593981742859,
      "epoch": 2.2390174775625886,
      "grad_norm": 8.717110295390793,
      "learning_rate": 9.064400256282757e-07,
      "logits": -1.8010636568069458,
      "logps": -110.48490142822266,
      "loss": 0.1248,
      "objective": 0.11183874309062958,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.9041666388511658,
      "regularize": 0.11183874309062958,
      "step": 395
    },
    {
      "dpo_loss": 0.12893003225326538,
      "dpo_wo_beta": -0.3680768311023712,
      "epoch": 2.2673594709494567,
      "grad_norm": 9.562073048936949,
      "learning_rate": 8.436218830716259e-07,
      "logits": -1.8909595012664795,
      "logps": -111.70219421386719,
      "loss": 0.1193,
      "objective": 0.12893003225326538,
      "ranking_idealized": 0.6625000238418579,
      "ranking_idealized_expo": 0.5708333253860474,
      "ranking_simple": 0.9166666865348816,
      "regularize": 0.12893003225326538,
      "step": 400
    },
    {
      "dpo_loss": 0.13196416199207306,
      "dpo_wo_beta": -0.17852090299129486,
      "epoch": 2.295701464336325,
      "grad_norm": 9.166021194752298,
      "learning_rate": 7.826149656671386e-07,
      "logits": -1.9320632219314575,
      "logps": -108.1246566772461,
      "loss": 0.1267,
      "objective": 0.13196416199207306,
      "ranking_idealized": 0.6416666507720947,
      "ranking_idealized_expo": 0.5583333373069763,
      "ranking_simple": 0.8958333134651184,
      "regularize": 0.13196416199207306,
      "step": 405
    },
    {
      "dpo_loss": 0.11071384698152542,
      "dpo_wo_beta": -0.1424117088317871,
      "epoch": 2.324043457723193,
      "grad_norm": 8.918983804471582,
      "learning_rate": 7.234859835833022e-07,
      "logits": -1.8304682970046997,
      "logps": -111.2301025390625,
      "loss": 0.112,
      "objective": 0.11071384698152542,
      "ranking_idealized": 0.5708333253860474,
      "ranking_idealized_expo": 0.4791666567325592,
      "ranking_simple": 0.9041666388511658,
      "regularize": 0.11071384698152542,
      "step": 410
    },
    {
      "dpo_loss": 0.1223960742354393,
      "dpo_wo_beta": -0.1956464648246765,
      "epoch": 2.3523854511100613,
      "grad_norm": 9.386393866562546,
      "learning_rate": 6.662995934939007e-07,
      "logits": -1.8708041906356812,
      "logps": -111.06449890136719,
      "loss": 0.1155,
      "objective": 0.1223960742354393,
      "ranking_idealized": 0.5958333611488342,
      "ranking_idealized_expo": 0.5291666388511658,
      "ranking_simple": 0.9083333611488342,
      "regularize": 0.1223960742354393,
      "step": 415
    },
    {
      "dpo_loss": 0.12930770218372345,
      "dpo_wo_beta": -0.21560731530189514,
      "epoch": 2.3807274444969297,
      "grad_norm": 11.0131183307354,
      "learning_rate": 6.111183278768956e-07,
      "logits": -1.860797643661499,
      "logps": -113.08780670166016,
      "loss": 0.133,
      "objective": 0.12930770218372345,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.4749999940395355,
      "ranking_simple": 0.9208333492279053,
      "regularize": 0.12930770218372345,
      "step": 420
    },
    {
      "epoch": 2.403401039206424,
      "eval_dpo_loss": 0.8785684108734131,
      "eval_dpo_wo_beta": -5.887755870819092,
      "eval_logits": -2.0276894569396973,
      "eval_logps": -117.1216812133789,
      "eval_loss": 0.8447906374931335,
      "eval_objective": 0.8785684108734131,
      "eval_ranking_idealized": 0.5888429880142212,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5413222908973694,
      "eval_regularize": 0.8785684108734131,
      "eval_runtime": 209.8564,
      "eval_samples_per_second": 27.59,
      "eval_steps_per_second": 1.153,
      "step": 424
    },
    {
      "dpo_loss": 0.117975153028965,
      "dpo_wo_beta": -0.1884605884552002,
      "epoch": 2.413793103448276,
      "grad_norm": 11.036168833651558,
      "learning_rate": 5.580025266360764e-07,
      "logits": -1.7822004556655884,
      "logps": -114.43038177490234,
      "loss": 0.1465,
      "objective": 0.117975153028965,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.5625,
      "ranking_simple": 0.9375,
      "regularize": 0.117975153028965,
      "step": 425
    },
    {
      "dpo_loss": 0.1465020477771759,
      "dpo_wo_beta": -0.2595965266227722,
      "epoch": 2.442135096835144,
      "grad_norm": 10.595070818850646,
      "learning_rate": 5.070102711202606e-07,
      "logits": -1.8692681789398193,
      "logps": -110.2762680053711,
      "loss": 0.1276,
      "objective": 0.1465020477771759,
      "ranking_idealized": 0.5916666388511658,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.8958333134651184,
      "regularize": 0.1465020477771759,
      "step": 430
    },
    {
      "dpo_loss": 0.09775053709745407,
      "dpo_wo_beta": -0.12755917012691498,
      "epoch": 2.4704770902220123,
      "grad_norm": 9.393206692367766,
      "learning_rate": 4.581973206121948e-07,
      "logits": -1.8968538045883179,
      "logps": -112.28767395019531,
      "loss": 0.1175,
      "objective": 0.09775053709745407,
      "ranking_idealized": 0.5874999761581421,
      "ranking_idealized_expo": 0.512499988079071,
      "ranking_simple": 0.9083333611488342,
      "regularize": 0.09775053709745407,
      "step": 435
    },
    {
      "dpo_loss": 0.14228057861328125,
      "dpo_wo_beta": -0.3639788329601288,
      "epoch": 2.4988190836088804,
      "grad_norm": 8.020134663378592,
      "learning_rate": 4.116170513565942e-07,
      "logits": -1.8666160106658936,
      "logps": -109.18843078613281,
      "loss": 0.1167,
      "objective": 0.14228057861328125,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.875,
      "regularize": 0.14228057861328125,
      "step": 440
    },
    {
      "dpo_loss": 0.13583588600158691,
      "dpo_wo_beta": -0.2074100226163864,
      "epoch": 2.527161076995749,
      "grad_norm": 9.224367796824264,
      "learning_rate": 3.6732039819400686e-07,
      "logits": -1.8071045875549316,
      "logps": -107.2675552368164,
      "loss": 0.1319,
      "objective": 0.13583588600158691,
      "ranking_idealized": 0.5625,
      "ranking_idealized_expo": 0.49166667461395264,
      "ranking_simple": 0.8791666626930237,
      "regularize": 0.13583588600158691,
      "step": 445
    },
    {
      "dpo_loss": 0.17114870250225067,
      "dpo_wo_beta": -0.43270742893218994,
      "epoch": 2.555503070382617,
      "grad_norm": 11.265861710797749,
      "learning_rate": 3.253557988643072e-07,
      "logits": -1.9256045818328857,
      "logps": -111.20384216308594,
      "loss": 0.1288,
      "objective": 0.17114870250225067,
      "ranking_idealized": 0.5874999761581421,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.862500011920929,
      "regularize": 0.17114870250225067,
      "step": 450
    },
    {
      "dpo_loss": 0.10827689617872238,
      "dpo_wo_beta": -0.1751028150320053,
      "epoch": 2.583845063769485,
      "grad_norm": 9.605136286662574,
      "learning_rate": 2.8576914104074425e-07,
      "logits": -1.9289051294326782,
      "logps": -109.37706756591797,
      "loss": 0.1168,
      "objective": 0.10827689617872238,
      "ranking_idealized": 0.5958333611488342,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.9125000238418579,
      "regularize": 0.10827689617872238,
      "step": 455
    },
    {
      "dpo_loss": 0.11124877631664276,
      "dpo_wo_beta": -0.28054580092430115,
      "epoch": 2.6121870571563535,
      "grad_norm": 9.957466667064367,
      "learning_rate": 2.486037121524448e-07,
      "logits": -1.93342924118042,
      "logps": -113.2356948852539,
      "loss": 0.1169,
      "objective": 0.11124877631664276,
      "ranking_idealized": 0.6291666626930237,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.9375,
      "regularize": 0.11124877631664276,
      "step": 460
    },
    {
      "dpo_loss": 0.12714476883411407,
      "dpo_wo_beta": -0.22146105766296387,
      "epoch": 2.6405290505432215,
      "grad_norm": 10.04326854921629,
      "learning_rate": 2.13900152050239e-07,
      "logits": -1.8874350786209106,
      "logps": -108.94982147216797,
      "loss": 0.1189,
      "objective": 0.12714476883411407,
      "ranking_idealized": 0.5458333492279053,
      "ranking_idealized_expo": 0.4791666567325592,
      "ranking_simple": 0.862500011920929,
      "regularize": 0.12714476883411407,
      "step": 465
    },
    {
      "dpo_loss": 0.12151040881872177,
      "dpo_wo_beta": -0.26416900753974915,
      "epoch": 2.66887104393009,
      "grad_norm": 8.777820527737605,
      "learning_rate": 1.8169640856758652e-07,
      "logits": -1.9314534664154053,
      "logps": -112.75170135498047,
      "loss": 0.1254,
      "objective": 0.12151040881872177,
      "ranking_idealized": 0.6291666626930237,
      "ranking_idealized_expo": 0.5541666746139526,
      "ranking_simple": 0.9083333611488342,
      "regularize": 0.12151040881872177,
      "step": 470
    },
    {
      "dpo_loss": 0.12749101221561432,
      "dpo_wo_beta": -0.2816121280193329,
      "epoch": 2.697213037316958,
      "grad_norm": 9.221778751171357,
      "learning_rate": 1.5202769602517514e-07,
      "logits": -1.8307260274887085,
      "logps": -109.39693450927734,
      "loss": 0.1211,
      "objective": 0.12749101221561432,
      "ranking_idealized": 0.6041666865348816,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.8999999761581421,
      "regularize": 0.12749101221561432,
      "step": 475
    },
    {
      "epoch": 2.708549834671705,
      "eval_dpo_loss": 0.8738968372344971,
      "eval_dpo_wo_beta": -5.815241813659668,
      "eval_logits": -2.0271613597869873,
      "eval_logps": -116.42301177978516,
      "eval_loss": 0.8371492624282837,
      "eval_objective": 0.8738968372344971,
      "eval_ranking_idealized": 0.5888429880142212,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5402892827987671,
      "eval_regularize": 0.8738968372344971,
      "eval_runtime": 211.9437,
      "eval_samples_per_second": 27.319,
      "eval_steps_per_second": 1.142,
      "step": 477
    },
    {
      "dpo_loss": 0.13781045377254486,
      "dpo_wo_beta": -0.2485995590686798,
      "epoch": 2.725555030703826,
      "grad_norm": 10.971551462649595,
      "learning_rate": 1.2492645672457838e-07,
      "logits": -1.9437103271484375,
      "logps": -108.93817901611328,
      "loss": 0.1267,
      "objective": 0.13781045377254486,
      "ranking_idealized": 0.5916666388511658,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.8999999761581421,
      "regularize": 0.13781045377254486,
      "step": 480
    },
    {
      "dpo_loss": 0.11082082241773605,
      "dpo_wo_beta": -0.10876031965017319,
      "epoch": 2.753897024090694,
      "grad_norm": 10.884940640535042,
      "learning_rate": 1.004223254730749e-07,
      "logits": -1.7556992769241333,
      "logps": -114.1142807006836,
      "loss": 0.1222,
      "objective": 0.11082082241773605,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.9041666388511658,
      "regularize": 0.11082082241773605,
      "step": 485
    },
    {
      "dpo_loss": 0.09154360741376877,
      "dpo_wo_beta": -0.05899694189429283,
      "epoch": 2.7822390174775626,
      "grad_norm": 11.015982469457516,
      "learning_rate": 7.854209717842231e-08,
      "logits": -1.8848822116851807,
      "logps": -110.15470886230469,
      "loss": 0.1058,
      "objective": 0.09154360741376877,
      "ranking_idealized": 0.5874999761581421,
      "ranking_idealized_expo": 0.47083333134651184,
      "ranking_simple": 0.9333333373069763,
      "regularize": 0.09154360741376877,
      "step": 490
    },
    {
      "dpo_loss": 0.10964310169219971,
      "dpo_wo_beta": -0.07648710906505585,
      "epoch": 2.8105810108644307,
      "grad_norm": 10.079416267782939,
      "learning_rate": 5.930969754901844e-08,
      "logits": -1.8575230836868286,
      "logps": -108.52234649658203,
      "loss": 0.1192,
      "objective": 0.10964310169219971,
      "ranking_idealized": 0.5583333373069763,
      "ranking_idealized_expo": 0.4749999940395355,
      "ranking_simple": 0.8666666746139526,
      "regularize": 0.10964310169219971,
      "step": 495
    },
    {
      "dpo_loss": 0.09479068219661713,
      "dpo_wo_beta": -0.03411731496453285,
      "epoch": 2.838923004251299,
      "grad_norm": 9.84080114767598,
      "learning_rate": 4.2746156931490756e-08,
      "logits": -1.8439643383026123,
      "logps": -109.77281188964844,
      "loss": 0.1213,
      "objective": 0.09479068219661713,
      "ranking_idealized": 0.6208333373069763,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.925000011920929,
      "regularize": 0.09479068219661713,
      "step": 500
    },
    {
      "dpo_loss": 0.12725140154361725,
      "dpo_wo_beta": -0.18973813951015472,
      "epoch": 2.8672649976381672,
      "grad_norm": 9.973754192936779,
      "learning_rate": 2.8869587314321324e-08,
      "logits": -1.8574442863464355,
      "logps": -110.32710266113281,
      "loss": 0.132,
      "objective": 0.12725140154361725,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.8999999761581421,
      "regularize": 0.12725140154361725,
      "step": 505
    },
    {
      "dpo_loss": 0.10469380766153336,
      "dpo_wo_beta": -0.1985001415014267,
      "epoch": 2.8956069910250353,
      "grad_norm": 8.936464383287202,
      "learning_rate": 1.7695162522652352e-08,
      "logits": -1.8629390001296997,
      "logps": -113.56767272949219,
      "loss": 0.1218,
      "objective": 0.10469380766153336,
      "ranking_idealized": 0.6041666865348816,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.8916666507720947,
      "regularize": 0.10469380766153336,
      "step": 510
    },
    {
      "dpo_loss": 0.11260154843330383,
      "dpo_wo_beta": -0.15691885352134705,
      "epoch": 2.9239489844119038,
      "grad_norm": 9.442300088571939,
      "learning_rate": 9.235101625932885e-09,
      "logits": -1.946829915046692,
      "logps": -108.54016876220703,
      "loss": 0.1258,
      "objective": 0.11260154843330383,
      "ranking_idealized": 0.5791666507720947,
      "ranking_idealized_expo": 0.48750001192092896,
      "ranking_simple": 0.925000011920929,
      "regularize": 0.11260154843330383,
      "step": 515
    },
    {
      "dpo_loss": 0.12230218201875687,
      "dpo_wo_beta": -0.10489177703857422,
      "epoch": 2.952290977798772,
      "grad_norm": 9.279898048101137,
      "learning_rate": 3.4986555765434415e-09,
      "logits": -1.8482831716537476,
      "logps": -114.20655059814453,
      "loss": 0.1228,
      "objective": 0.12230218201875687,
      "ranking_idealized": 0.6291666626930237,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.8708333373069763,
      "regularize": 0.12230218201875687,
      "step": 520
    },
    {
      "dpo_loss": 0.13335375487804413,
      "dpo_wo_beta": -0.35261282324790955,
      "epoch": 2.9806329711856403,
      "grad_norm": 11.094809681697281,
      "learning_rate": 4.920970940180958e-10,
      "logits": -1.876869797706604,
      "logps": -111.03084564208984,
      "loss": 0.1235,
      "objective": 0.13335375487804413,
      "ranking_idealized": 0.6291666626930237,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.8958333134651184,
      "regularize": 0.13335375487804413,
      "step": 525
    },
    {
      "epoch": 2.9976381672177608,
      "step": 528,
      "total_flos": 0.0,
      "train_loss": 0.023984534440167023,
      "train_runtime": 3310.3799,
      "train_samples_per_second": 46.039,
      "train_steps_per_second": 0.159
    }
  ],
  "logging_steps": 5,
  "max_steps": 528,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 53,
  "stateful_callbacks": {
    "EarlyStoppingCallback": {
      "args": {
        "early_stopping_patience": 5,
        "early_stopping_threshold": 0.0
      },
      "attributes": {
        "early_stopping_patience_counter": 0
      }
    },
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}