{
  "best_metric": 0.5869565010070801,
  "best_model_checkpoint": "./qwen2.5-0.5b/qwen2.5-0.5b-expo-DPO-ES-TRY/checkpoint-583",
  "epoch": 2.9976381672177608,
  "eval_steps": 53,
  "global_step": 1056,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "dpo_loss": 0.6931471824645996,
      "dpo_wo_beta": -0.6931471824645996,
      "epoch": 0.005668398677373642,
      "grad_norm": 13.433600669124935,
      "learning_rate": 9.433962264150944e-08,
      "logits": -1.3874311447143555,
      "logps": -88.43561553955078,
      "loss": 0.6931,
      "objective": 0.6931471824645996,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.5208333134651184,
      "regularize": 0.6931471824645996,
      "step": 1
    },
    {
      "dpo_loss": 0.693236768245697,
      "dpo_wo_beta": -0.6993356347084045,
      "epoch": 0.02834199338686821,
      "grad_norm": 13.640653628388394,
      "learning_rate": 4.716981132075472e-07,
      "logits": -1.4090652465820312,
      "logps": -84.34337615966797,
      "loss": 0.693,
      "objective": 0.693236768245697,
      "ranking_idealized": 0.6197916865348816,
      "ranking_idealized_expo": 0.546875,
      "ranking_simple": 0.546875,
      "regularize": 0.693236768245697,
      "step": 5
    },
    {
      "dpo_loss": 0.6845630407333374,
      "dpo_wo_beta": -0.7111619710922241,
      "epoch": 0.05668398677373642,
      "grad_norm": 12.626074407134174,
      "learning_rate": 9.433962264150944e-07,
      "logits": -1.4784893989562988,
      "logps": -81.94055938720703,
      "loss": 0.6892,
      "objective": 0.6845630407333374,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.512499988079071,
      "regularize": 0.6845630407333374,
      "step": 10
    },
    {
      "dpo_loss": 0.6825469136238098,
      "dpo_wo_beta": -0.8259204626083374,
      "epoch": 0.08502598016060463,
      "grad_norm": 12.374180595083178,
      "learning_rate": 1.4150943396226415e-06,
      "logits": -1.4932299852371216,
      "logps": -81.52880096435547,
      "loss": 0.6814,
      "objective": 0.6825469136238098,
      "ranking_idealized": 0.5916666388511658,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.512499988079071,
      "regularize": 0.6825469136238098,
      "step": 15
    },
    {
      "dpo_loss": 0.6950914263725281,
      "dpo_wo_beta": -1.2390469312667847,
      "epoch": 0.11336797354747284,
      "grad_norm": 14.839934392200913,
      "learning_rate": 1.8867924528301889e-06,
      "logits": -1.5371100902557373,
      "logps": -82.72624969482422,
      "loss": 0.6711,
      "objective": 0.6950914263725281,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.5375000238418579,
      "ranking_simple": 0.5249999761581421,
      "regularize": 0.6950914263725281,
      "step": 20
    },
    {
      "dpo_loss": 0.6556071043014526,
      "dpo_wo_beta": -1.110619068145752,
      "epoch": 0.14170996693434104,
      "grad_norm": 12.89805052529156,
      "learning_rate": 2.358490566037736e-06,
      "logits": -1.6399922370910645,
      "logps": -81.59695434570312,
      "loss": 0.6589,
      "objective": 0.6556071043014526,
      "ranking_idealized": 0.5958333611488342,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.5249999761581421,
      "regularize": 0.6556071043014526,
      "step": 25
    },
    {
      "dpo_loss": 0.6518108248710632,
      "dpo_wo_beta": -1.2506839036941528,
      "epoch": 0.17005196032120926,
      "grad_norm": 12.64998937636519,
      "learning_rate": 2.830188679245283e-06,
      "logits": -1.6404598951339722,
      "logps": -83.20111846923828,
      "loss": 0.6451,
      "objective": 0.6518108248710632,
      "ranking_idealized": 0.6041666865348816,
      "ranking_idealized_expo": 0.5291666388511658,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.6518108248710632,
      "step": 30
    },
    {
      "dpo_loss": 0.6226770877838135,
      "dpo_wo_beta": -1.394917368888855,
      "epoch": 0.19839395370807747,
      "grad_norm": 13.760162421635227,
      "learning_rate": 3.30188679245283e-06,
      "logits": -1.6237396001815796,
      "logps": -87.80964660644531,
      "loss": 0.6189,
      "objective": 0.6226770877838135,
      "ranking_idealized": 0.612500011920929,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.5583333373069763,
      "regularize": 0.6226770877838135,
      "step": 35
    },
    {
      "dpo_loss": 0.5924390554428101,
      "dpo_wo_beta": -1.422450304031372,
      "epoch": 0.22673594709494568,
      "grad_norm": 16.810886476613117,
      "learning_rate": 3.7735849056603777e-06,
      "logits": -1.620682954788208,
      "logps": -91.93690490722656,
      "loss": 0.6076,
      "objective": 0.5924390554428101,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.5708333253860474,
      "regularize": 0.5924390554428101,
      "step": 40
    },
    {
      "dpo_loss": 0.573756217956543,
      "dpo_wo_beta": -1.3691534996032715,
      "epoch": 0.25507794048181387,
      "grad_norm": 13.798774501924722,
      "learning_rate": 4.245283018867925e-06,
      "logits": -1.7814558744430542,
      "logps": -92.24474334716797,
      "loss": 0.5989,
      "objective": 0.573756217956543,
      "ranking_idealized": 0.6208333373069763,
      "ranking_idealized_expo": 0.5375000238418579,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.573756217956543,
      "step": 45
    },
    {
      "dpo_loss": 0.5726417899131775,
      "dpo_wo_beta": -1.3605374097824097,
      "epoch": 0.2834199338686821,
      "grad_norm": 12.568473894025988,
      "learning_rate": 4.716981132075472e-06,
      "logits": -1.808895468711853,
      "logps": -90.65751647949219,
      "loss": 0.5954,
      "objective": 0.5726417899131775,
      "ranking_idealized": 0.5416666865348816,
      "ranking_idealized_expo": 0.42500001192092896,
      "ranking_simple": 0.574999988079071,
      "regularize": 0.5726417899131775,
      "step": 50
    },
    {
      "epoch": 0.300425129900803,
      "eval_dpo_loss": 0.7112604975700378,
      "eval_dpo_wo_beta": -2.2659413814544678,
      "eval_logits": -1.892814040184021,
      "eval_logps": -101.36742401123047,
      "eval_loss": 0.6816489100456238,
      "eval_objective": 0.7112604975700378,
      "eval_ranking_idealized": 0.5888429880142212,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5237603187561035,
      "eval_regularize": 0.7112604975700378,
      "eval_runtime": 211.6587,
      "eval_samples_per_second": 27.355,
      "eval_steps_per_second": 1.143,
      "step": 53
    },
    {
      "dpo_loss": 0.5827316045761108,
      "dpo_wo_beta": -1.6213361024856567,
      "epoch": 0.3117619272555503,
      "grad_norm": 14.442715913160086,
      "learning_rate": 4.999781286194085e-06,
      "logits": -1.8762638568878174,
      "logps": -93.41423797607422,
      "loss": 0.5721,
      "objective": 0.5827316045761108,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.6041666865348816,
      "regularize": 0.5827316045761108,
      "step": 55
    },
    {
      "dpo_loss": 0.5553872585296631,
      "dpo_wo_beta": -1.6468366384506226,
      "epoch": 0.3401039206424185,
      "grad_norm": 13.845514282811145,
      "learning_rate": 4.997321195347154e-06,
      "logits": -1.8914529085159302,
      "logps": -90.59642028808594,
      "loss": 0.5756,
      "objective": 0.5553872585296631,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.5791666507720947,
      "regularize": 0.5553872585296631,
      "step": 60
    },
    {
      "dpo_loss": 0.5302771329879761,
      "dpo_wo_beta": -1.3166770935058594,
      "epoch": 0.3684459140292867,
      "grad_norm": 10.846857687148022,
      "learning_rate": 4.992130320438411e-06,
      "logits": -1.8399535417556763,
      "logps": -86.60197448730469,
      "loss": 0.5586,
      "objective": 0.5302771329879761,
      "ranking_idealized": 0.5708333253860474,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.6208333373069763,
      "regularize": 0.5302771329879761,
      "step": 65
    },
    {
      "dpo_loss": 0.5711485743522644,
      "dpo_wo_beta": -1.7437169551849365,
      "epoch": 0.39678790741615494,
      "grad_norm": 13.787840238803502,
      "learning_rate": 4.984214337613357e-06,
      "logits": -1.8178967237472534,
      "logps": -91.10688781738281,
      "loss": 0.5701,
      "objective": 0.5711485743522644,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5874999761581421,
      "regularize": 0.5711485743522644,
      "step": 70
    },
    {
      "dpo_loss": 0.523643434047699,
      "dpo_wo_beta": -1.669514536857605,
      "epoch": 0.42512990080302315,
      "grad_norm": 13.192298437287352,
      "learning_rate": 4.97358190288299e-06,
      "logits": -1.8182169198989868,
      "logps": -94.8000717163086,
      "loss": 0.5205,
      "objective": 0.523643434047699,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.5458333492279053,
      "ranking_simple": 0.6583333611488342,
      "regularize": 0.523643434047699,
      "step": 75
    },
    {
      "dpo_loss": 0.51079261302948,
      "dpo_wo_beta": -1.7271808385849,
      "epoch": 0.45347189418989137,
      "grad_norm": 15.151373786996814,
      "learning_rate": 4.9602446426585845e-06,
      "logits": -1.8920824527740479,
      "logps": -93.58238220214844,
      "loss": 0.5285,
      "objective": 0.51079261302948,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.6458333134651184,
      "regularize": 0.51079261302948,
      "step": 80
    },
    {
      "dpo_loss": 0.5066012144088745,
      "dpo_wo_beta": -1.5956443548202515,
      "epoch": 0.4818138875767596,
      "grad_norm": 12.328960275584794,
      "learning_rate": 4.944217141038379e-06,
      "logits": -1.8741406202316284,
      "logps": -87.06742858886719,
      "loss": 0.5202,
      "objective": 0.5066012144088745,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.6208333373069763,
      "regularize": 0.5066012144088745,
      "step": 85
    },
    {
      "dpo_loss": 0.5358369946479797,
      "dpo_wo_beta": -1.9357556104660034,
      "epoch": 0.5101558809636277,
      "grad_norm": 12.694483590051824,
      "learning_rate": 4.925516923860083e-06,
      "logits": -1.7968534231185913,
      "logps": -86.77802276611328,
      "loss": 0.4858,
      "objective": 0.5358369946479797,
      "ranking_idealized": 0.5458333492279053,
      "ranking_idealized_expo": 0.4749999940395355,
      "ranking_simple": 0.5874999761581421,
      "regularize": 0.5358369946479797,
      "step": 90
    },
    {
      "dpo_loss": 0.4783257842063904,
      "dpo_wo_beta": -1.9098786115646362,
      "epoch": 0.538497874350496,
      "grad_norm": 14.474706973531484,
      "learning_rate": 4.904164439536626e-06,
      "logits": -1.8568389415740967,
      "logps": -88.12813568115234,
      "loss": 0.4865,
      "objective": 0.4783257842063904,
      "ranking_idealized": 0.6291666626930237,
      "ranking_idealized_expo": 0.5458333492279053,
      "ranking_simple": 0.6916666626930237,
      "regularize": 0.4783257842063904,
      "step": 95
    },
    {
      "dpo_loss": 0.4654810130596161,
      "dpo_wo_beta": -1.9254087209701538,
      "epoch": 0.5668398677373642,
      "grad_norm": 13.577084707122001,
      "learning_rate": 4.880183036696123e-06,
      "logits": -1.938937783241272,
      "logps": -92.29436492919922,
      "loss": 0.5016,
      "objective": 0.4654810130596161,
      "ranking_idealized": 0.612500011920929,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.6875,
      "regularize": 0.4654810130596161,
      "step": 100
    },
    {
      "dpo_loss": 0.4374677240848541,
      "dpo_wo_beta": -1.4267934560775757,
      "epoch": 0.5951818611242324,
      "grad_norm": 11.14545328639218,
      "learning_rate": 4.853598938650487e-06,
      "logits": -1.8158982992172241,
      "logps": -90.21449279785156,
      "loss": 0.4618,
      "objective": 0.4374677240848541,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.6666666865348816,
      "regularize": 0.4374677240848541,
      "step": 105
    },
    {
      "epoch": 0.600850259801606,
      "eval_dpo_loss": 0.6936022639274597,
      "eval_dpo_wo_beta": -2.462427854537964,
      "eval_logits": -1.9007418155670166,
      "eval_logps": -94.35714721679688,
      "eval_loss": 0.6912521123886108,
      "eval_objective": 0.6936022639274597,
      "eval_ranking_idealized": 0.5888429880142212,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5351239442825317,
      "eval_regularize": 0.6936022639274597,
      "eval_runtime": 210.2297,
      "eval_samples_per_second": 27.541,
      "eval_steps_per_second": 1.151,
      "step": 106
    },
    {
      "dpo_loss": 0.47933149337768555,
      "dpo_wo_beta": -1.9683055877685547,
      "epoch": 0.6235238545111006,
      "grad_norm": 12.39392340166307,
      "learning_rate": 4.824441214720629e-06,
      "logits": -1.9334439039230347,
      "logps": -87.35523223876953,
      "loss": 0.4633,
      "objective": 0.47933149337768555,
      "ranking_idealized": 0.5708333253860474,
      "ranking_idealized_expo": 0.4541666805744171,
      "ranking_simple": 0.6625000238418579,
      "regularize": 0.47933149337768555,
      "step": 110
    },
    {
      "dpo_loss": 0.4749464690685272,
      "dpo_wo_beta": -1.7375919818878174,
      "epoch": 0.6518658478979689,
      "grad_norm": 12.612865651893962,
      "learning_rate": 4.7927417484495756e-06,
      "logits": -1.9057692289352417,
      "logps": -87.68991088867188,
      "loss": 0.4712,
      "objective": 0.4749464690685272,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.6333333253860474,
      "regularize": 0.4749464690685272,
      "step": 115
    },
    {
      "dpo_loss": 0.4848935306072235,
      "dpo_wo_beta": -1.9273093938827515,
      "epoch": 0.680207841284837,
      "grad_norm": 13.836239066838136,
      "learning_rate": 4.758535202738287e-06,
      "logits": -1.8775906562805176,
      "logps": -87.8878173828125,
      "loss": 0.4641,
      "objective": 0.4848935306072235,
      "ranking_idealized": 0.5874999761581421,
      "ranking_idealized_expo": 0.5291666388511658,
      "ranking_simple": 0.6625000238418579,
      "regularize": 0.4848935306072235,
      "step": 120
    },
    {
      "dpo_loss": 0.4785127639770508,
      "dpo_wo_beta": -1.814666748046875,
      "epoch": 0.7085498346717053,
      "grad_norm": 12.105170057238437,
      "learning_rate": 4.721858981942284e-06,
      "logits": -1.8346068859100342,
      "logps": -86.40522766113281,
      "loss": 0.4801,
      "objective": 0.4785127639770508,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.6875,
      "regularize": 0.4785127639770508,
      "step": 125
    },
    {
      "dpo_loss": 0.4548089802265167,
      "dpo_wo_beta": -1.4164987802505493,
      "epoch": 0.7368918280585735,
      "grad_norm": 11.895980627109102,
      "learning_rate": 4.682753190970533e-06,
      "logits": -1.9488608837127686,
      "logps": -79.42195129394531,
      "loss": 0.4538,
      "objective": 0.4548089802265167,
      "ranking_idealized": 0.5416666865348816,
      "ranking_idealized_expo": 0.44999998807907104,
      "ranking_simple": 0.6291666626930237,
      "regularize": 0.4548089802265167,
      "step": 130
    },
    {
      "dpo_loss": 0.49760884046554565,
      "dpo_wo_beta": -1.994195818901062,
      "epoch": 0.7652338214454416,
      "grad_norm": 12.298776298341995,
      "learning_rate": 4.641260591431315e-06,
      "logits": -1.9813282489776611,
      "logps": -82.40634155273438,
      "loss": 0.4433,
      "objective": 0.49760884046554565,
      "ranking_idealized": 0.5874999761581421,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.6583333611488342,
      "regularize": 0.49760884046554565,
      "step": 135
    },
    {
      "dpo_loss": 0.41459351778030396,
      "dpo_wo_beta": -1.187635064125061,
      "epoch": 0.7935758148323099,
      "grad_norm": 12.618720178096575,
      "learning_rate": 4.597426554873037e-06,
      "logits": -1.97609281539917,
      "logps": -83.44467163085938,
      "loss": 0.4236,
      "objective": 0.41459351778030396,
      "ranking_idealized": 0.5791666507720947,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.6791666746139526,
      "regularize": 0.41459351778030396,
      "step": 140
    },
    {
      "dpo_loss": 0.4073801636695862,
      "dpo_wo_beta": -1.311059832572937,
      "epoch": 0.821917808219178,
      "grad_norm": 14.417917904409194,
      "learning_rate": 4.551299013171111e-06,
      "logits": -2.0718839168548584,
      "logps": -84.2674560546875,
      "loss": 0.4215,
      "objective": 0.4073801636695862,
      "ranking_idealized": 0.5791666507720947,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.6916666626930237,
      "regularize": 0.4073801636695862,
      "step": 145
    },
    {
      "dpo_loss": 0.4207518398761749,
      "dpo_wo_beta": -1.50857675075531,
      "epoch": 0.8502598016060463,
      "grad_norm": 11.543599868064442,
      "learning_rate": 4.502928406115152e-06,
      "logits": -2.0730583667755127,
      "logps": -82.68958282470703,
      "loss": 0.4276,
      "objective": 0.4207518398761749,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.7208333611488342,
      "regularize": 0.4207518398761749,
      "step": 150
    },
    {
      "dpo_loss": 0.3847941756248474,
      "dpo_wo_beta": -1.4449684619903564,
      "epoch": 0.8786017949929145,
      "grad_norm": 12.08771803065001,
      "learning_rate": 4.452367626253805e-06,
      "logits": -2.0991933345794678,
      "logps": -85.211181640625,
      "loss": 0.3986,
      "objective": 0.3847941756248474,
      "ranking_idealized": 0.612500011920929,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.7250000238418579,
      "regularize": 0.3847941756248474,
      "step": 155
    },
    {
      "epoch": 0.9012753897024091,
      "eval_dpo_loss": 0.7214789390563965,
      "eval_dpo_wo_beta": -3.1229145526885986,
      "eval_logits": -2.1450352668762207,
      "eval_logps": -95.60012817382812,
      "eval_loss": 0.7013870477676392,
      "eval_objective": 0.7214789390563965,
      "eval_ranking_idealized": 0.5888429880142212,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5351239442825317,
      "eval_regularize": 0.7214789390563965,
      "eval_runtime": 210.3593,
      "eval_samples_per_second": 27.524,
      "eval_steps_per_second": 1.15,
      "step": 159
    },
    {
      "dpo_loss": 0.4162478744983673,
      "dpo_wo_beta": -1.6461573839187622,
      "epoch": 0.9069437883797827,
      "grad_norm": 12.82345397067452,
      "learning_rate": 4.399671961057523e-06,
      "logits": -2.0759384632110596,
      "logps": -89.25846862792969,
      "loss": 0.4236,
      "objective": 0.4162478744983673,
      "ranking_idealized": 0.5708333253860474,
      "ranking_idealized_expo": 0.49166667461395264,
      "ranking_simple": 0.699999988079071,
      "regularize": 0.4162478744983673,
      "step": 160
    },
    {
      "dpo_loss": 0.41358453035354614,
      "dpo_wo_beta": -1.648630976676941,
      "epoch": 0.9352857817666509,
      "grad_norm": 12.860537676624453,
      "learning_rate": 4.3448990324625244e-06,
      "logits": -2.024477481842041,
      "logps": -88.03329467773438,
      "loss": 0.4026,
      "objective": 0.41358453035354614,
      "ranking_idealized": 0.6041666865348816,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.7333333492279053,
      "regularize": 0.41358453035354614,
      "step": 165
    },
    {
      "dpo_loss": 0.378000408411026,
      "dpo_wo_beta": -1.2966532707214355,
      "epoch": 0.9636277751535192,
      "grad_norm": 11.533711130228069,
      "learning_rate": 4.288108733862064e-06,
      "logits": -2.042527437210083,
      "logps": -90.26854705810547,
      "loss": 0.3925,
      "objective": 0.378000408411026,
      "ranking_idealized": 0.5791666507720947,
      "ranking_idealized_expo": 0.512499988079071,
      "ranking_simple": 0.7166666388511658,
      "regularize": 0.378000408411026,
      "step": 170
    },
    {
      "dpo_loss": 0.3764660954475403,
      "dpo_wo_beta": -1.3978971242904663,
      "epoch": 0.9919697685403873,
      "grad_norm": 12.165192869157089,
      "learning_rate": 4.229363164613874e-06,
      "logits": -2.0610477924346924,
      "logps": -89.8354721069336,
      "loss": 0.3793,
      "objective": 0.3764660954475403,
      "ranking_idealized": 0.6541666388511658,
      "ranking_idealized_expo": 0.574999988079071,
      "ranking_simple": 0.7916666865348816,
      "regularize": 0.3764660954475403,
      "step": 175
    },
    {
      "dpo_loss": 0.27626773715019226,
      "dpo_wo_beta": -0.8504549860954285,
      "epoch": 1.0203117619272555,
      "grad_norm": 10.141692447282386,
      "learning_rate": 4.168726562135432e-06,
      "logits": -2.2514243125915527,
      "logps": -90.8476333618164,
      "loss": 0.2852,
      "objective": 0.27626773715019226,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.8083333373069763,
      "regularize": 0.27626773715019226,
      "step": 180
    },
    {
      "dpo_loss": 0.23696589469909668,
      "dpo_wo_beta": -0.6947117447853088,
      "epoch": 1.0486537553141237,
      "grad_norm": 13.78702272812957,
      "learning_rate": 4.106265231661292e-06,
      "logits": -2.158977746963501,
      "logps": -95.00120544433594,
      "loss": 0.2429,
      "objective": 0.23696589469909668,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.49166667461395264,
      "ranking_simple": 0.8083333373069763,
      "regularize": 0.23696589469909668,
      "step": 185
    },
    {
      "dpo_loss": 0.26388806104660034,
      "dpo_wo_beta": -0.9112051725387573,
      "epoch": 1.076995748700992,
      "grad_norm": 14.740228375586371,
      "learning_rate": 4.042047473739278e-06,
      "logits": -2.1533920764923096,
      "logps": -101.71949768066406,
      "loss": 0.2517,
      "objective": 0.26388806104660034,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.5541666746139526,
      "ranking_simple": 0.8416666388511658,
      "regularize": 0.26388806104660034,
      "step": 190
    },
    {
      "dpo_loss": 0.2244579941034317,
      "dpo_wo_beta": -0.6430780291557312,
      "epoch": 1.10533774208786,
      "grad_norm": 10.169064121599527,
      "learning_rate": 3.976143509544843e-06,
      "logits": -2.1589295864105225,
      "logps": -96.5248031616211,
      "loss": 0.2467,
      "objective": 0.2244579941034317,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.8083333373069763,
      "regularize": 0.2244579941034317,
      "step": 195
    },
    {
      "dpo_loss": 0.24179764091968536,
      "dpo_wo_beta": -0.6332272291183472,
      "epoch": 1.1336797354747283,
      "grad_norm": 9.444774343787891,
      "learning_rate": 3.908625404095242e-06,
      "logits": -2.2753493785858154,
      "logps": -91.93312072753906,
      "loss": 0.2563,
      "objective": 0.24179764091968536,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.8458333611488342,
      "regularize": 0.24179764091968536,
      "step": 200
    },
    {
      "dpo_loss": 0.25683078169822693,
      "dpo_wo_beta": -0.8531176447868347,
      "epoch": 1.1620217288615966,
      "grad_norm": 9.240319326762517,
      "learning_rate": 3.839566987447492e-06,
      "logits": -2.2432618141174316,
      "logps": -91.3159408569336,
      "loss": 0.2584,
      "objective": 0.25683078169822693,
      "ranking_idealized": 0.5708333253860474,
      "ranking_idealized_expo": 0.49166667461395264,
      "ranking_simple": 0.8166666626930237,
      "regularize": 0.25683078169822693,
      "step": 205
    },
    {
      "dpo_loss": 0.24292893707752228,
      "dpo_wo_beta": -0.8205318450927734,
      "epoch": 1.1903637222484649,
      "grad_norm": 9.283856100785183,
      "learning_rate": 3.7690437739662928e-06,
      "logits": -2.2361652851104736,
      "logps": -90.6613998413086,
      "loss": 0.2551,
      "objective": 0.24292893707752228,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.800000011920929,
      "regularize": 0.24292893707752228,
      "step": 210
    },
    {
      "epoch": 1.201700519603212,
      "eval_dpo_loss": 0.7525234222412109,
      "eval_dpo_wo_beta": -3.7749528884887695,
      "eval_logits": -2.267778158187866,
      "eval_logps": -98.14269256591797,
      "eval_loss": 0.7350714206695557,
      "eval_objective": 0.7525234222412109,
      "eval_ranking_idealized": 0.5888429880142212,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5371900796890259,
      "eval_regularize": 0.7525234222412109,
      "eval_runtime": 210.8898,
      "eval_samples_per_second": 27.455,
      "eval_steps_per_second": 1.148,
      "step": 212
    },
    {
      "dpo_loss": 0.289533793926239,
      "dpo_wo_beta": -0.8810125589370728,
      "epoch": 1.2187057156353331,
      "grad_norm": 10.72372972136692,
      "learning_rate": 3.697132879750174e-06,
      "logits": -2.1757090091705322,
      "logps": -93.64250183105469,
      "loss": 0.2578,
      "objective": 0.289533793926239,
      "ranking_idealized": 0.5583333373069763,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.7875000238418579,
      "regularize": 0.289533793926239,
      "step": 215
    },
    {
      "dpo_loss": 0.25134381651878357,
      "dpo_wo_beta": -0.8703542947769165,
      "epoch": 1.2470477090222012,
      "grad_norm": 12.940604838816247,
      "learning_rate": 3.6239129383061764e-06,
      "logits": -2.121750593185425,
      "logps": -94.44015502929688,
      "loss": 0.2676,
      "objective": 0.25134381651878357,
      "ranking_idealized": 0.6541666388511658,
      "ranking_idealized_expo": 0.6041666865348816,
      "ranking_simple": 0.8208333253860474,
      "regularize": 0.25134381651878357,
      "step": 220
    },
    {
      "dpo_loss": 0.23937886953353882,
      "dpo_wo_beta": -0.7396827936172485,
      "epoch": 1.2753897024090695,
      "grad_norm": 9.645711793319885,
      "learning_rate": 3.5494640145652647e-06,
      "logits": -2.0901684761047363,
      "logps": -94.10260772705078,
      "loss": 0.2637,
      "objective": 0.23937886953353882,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.8541666865348816,
      "regularize": 0.23937886953353882,
      "step": 225
    },
    {
      "dpo_loss": 0.2818019688129425,
      "dpo_wo_beta": -1.1170729398727417,
      "epoch": 1.3037316957959377,
      "grad_norm": 8.80210598601974,
      "learning_rate": 3.4738675173325008e-06,
      "logits": -1.9860222339630127,
      "logps": -92.9978256225586,
      "loss": 0.2776,
      "objective": 0.2818019688129425,
      "ranking_idealized": 0.5791666507720947,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.7749999761581421,
      "regularize": 0.2818019688129425,
      "step": 230
    },
    {
      "dpo_loss": 0.22621506452560425,
      "dpo_wo_beta": -0.35843732953071594,
      "epoch": 1.3320736891828058,
      "grad_norm": 9.267612473930496,
      "learning_rate": 3.397206110267713e-06,
      "logits": -2.1131467819213867,
      "logps": -87.49403381347656,
      "loss": 0.2618,
      "objective": 0.22621506452560425,
      "ranking_idealized": 0.6041666865348816,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.8333333134651184,
      "regularize": 0.22621506452560425,
      "step": 235
    },
    {
      "dpo_loss": 0.23632274568080902,
      "dpo_wo_beta": -0.6697984933853149,
      "epoch": 1.360415682569674,
      "grad_norm": 10.68594080832048,
      "learning_rate": 3.3195636214939943e-06,
      "logits": -2.130047559738159,
      "logps": -91.7619857788086,
      "loss": 0.2584,
      "objective": 0.23632274568080902,
      "ranking_idealized": 0.5541666746139526,
      "ranking_idealized_expo": 0.4791666567325592,
      "ranking_simple": 0.8125,
      "regularize": 0.23632274568080902,
      "step": 240
    },
    {
      "dpo_loss": 0.2982023358345032,
      "dpo_wo_beta": -1.1124054193496704,
      "epoch": 1.3887576759565423,
      "grad_norm": 10.330360151122868,
      "learning_rate": 3.2410249519328848e-06,
      "logits": -2.1718757152557373,
      "logps": -93.45353698730469,
      "loss": 0.2692,
      "objective": 0.2982023358345032,
      "ranking_idealized": 0.5916666388511658,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.8041666746139526,
      "regularize": 0.2982023358345032,
      "step": 245
    },
    {
      "dpo_loss": 0.2403133064508438,
      "dpo_wo_beta": -0.7000442147254944,
      "epoch": 1.4170996693434104,
      "grad_norm": 13.026578288520353,
      "learning_rate": 3.1616759824664543e-06,
      "logits": -2.145325183868408,
      "logps": -94.18195343017578,
      "loss": 0.269,
      "objective": 0.2403133064508438,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.47083333134651184,
      "ranking_simple": 0.824999988079071,
      "regularize": 0.2403133064508438,
      "step": 250
    },
    {
      "dpo_loss": 0.23977436125278473,
      "dpo_wo_beta": -0.5784927010536194,
      "epoch": 1.4454416627302786,
      "grad_norm": 10.959901566104394,
      "learning_rate": 3.081603480027826e-06,
      "logits": -2.108074426651001,
      "logps": -94.5383529663086,
      "loss": 0.2625,
      "objective": 0.23977436125278473,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5458333492279053,
      "ranking_simple": 0.8374999761581421,
      "regularize": 0.23977436125278473,
      "step": 255
    },
    {
      "dpo_loss": 0.25297579169273376,
      "dpo_wo_beta": -0.7996426820755005,
      "epoch": 1.473783656117147,
      "grad_norm": 9.578050078679867,
      "learning_rate": 3.0008950027228035e-06,
      "logits": -2.1828908920288086,
      "logps": -92.77781677246094,
      "loss": 0.232,
      "objective": 0.25297579169273376,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.48750001192092896,
      "ranking_simple": 0.8041666746139526,
      "regularize": 0.25297579169273376,
      "step": 260
    },
    {
      "dpo_loss": 0.2830916941165924,
      "dpo_wo_beta": -1.124144434928894,
      "epoch": 1.5021256495040152,
      "grad_norm": 11.27765707111355,
      "learning_rate": 2.9196388040863695e-06,
      "logits": -2.1150081157684326,
      "logps": -95.04662322998047,
      "loss": 0.2623,
      "objective": 0.2830916941165924,
      "ranking_idealized": 0.5791666507720947,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.8041666746139526,
      "regularize": 0.2830916941165924,
      "step": 265
    },
    {
      "epoch": 1.5021256495040152,
      "eval_dpo_loss": 0.7739136815071106,
      "eval_dpo_wo_beta": -4.163427829742432,
      "eval_logits": -2.1478331089019775,
      "eval_logps": -100.8313217163086,
      "eval_loss": 0.7400166392326355,
      "eval_objective": 0.7739136815071106,
      "eval_ranking_idealized": 0.5888429880142212,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.53925621509552,
      "eval_regularize": 0.7739136815071106,
      "eval_runtime": 210.8657,
      "eval_samples_per_second": 27.458,
      "eval_steps_per_second": 1.148,
      "step": 265
    },
    {
      "dpo_loss": 0.2930367887020111,
      "dpo_wo_beta": -1.3651045560836792,
      "epoch": 1.5304676428908834,
      "grad_norm": 11.715215816813723,
      "learning_rate": 2.8379237365787426e-06,
      "logits": -2.035703182220459,
      "logps": -97.7331771850586,
      "loss": 0.253,
      "objective": 0.2930367887020111,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.7916666865348816,
      "regularize": 0.2930367887020111,
      "step": 270
    },
    {
      "dpo_loss": 0.24886849522590637,
      "dpo_wo_beta": -0.8069366216659546,
      "epoch": 1.5588096362777515,
      "grad_norm": 8.958944325794365,
      "learning_rate": 2.7558391544265127e-06,
      "logits": -1.9700883626937866,
      "logps": -97.53855895996094,
      "loss": 0.2491,
      "objective": 0.24886849522590637,
      "ranking_idealized": 0.6208333373069763,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.8125,
      "regularize": 0.24886849522590637,
      "step": 275
    },
    {
      "dpo_loss": 0.22936613857746124,
      "dpo_wo_beta": -0.6120084524154663,
      "epoch": 1.5871516296646198,
      "grad_norm": 10.814739938498821,
      "learning_rate": 2.6734748159151104e-06,
      "logits": -1.9118597507476807,
      "logps": -98.06639099121094,
      "loss": 0.2491,
      "objective": 0.22936613857746124,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.8374999761581421,
      "regularize": 0.22936613857746124,
      "step": 280
    },
    {
      "dpo_loss": 0.22401383519172668,
      "dpo_wo_beta": -0.5180224776268005,
      "epoch": 1.615493623051488,
      "grad_norm": 11.270657822712987,
      "learning_rate": 2.5909207852394363e-06,
      "logits": -1.9585484266281128,
      "logps": -100.70836639404297,
      "loss": 0.2348,
      "objective": 0.22401383519172668,
      "ranking_idealized": 0.5874999761581421,
      "ranking_idealized_expo": 0.512499988079071,
      "ranking_simple": 0.8291666507720947,
      "regularize": 0.22401383519172668,
      "step": 285
    },
    {
      "dpo_loss": 0.2646006941795349,
      "dpo_wo_beta": -0.7763135433197021,
      "epoch": 1.643835616438356,
      "grad_norm": 10.585292794409252,
      "learning_rate": 2.508267334019988e-06,
      "logits": -1.9566444158554077,
      "logps": -97.0122299194336,
      "loss": 0.2532,
      "objective": 0.2646006941795349,
      "ranking_idealized": 0.612500011920929,
      "ranking_idealized_expo": 0.5625,
      "ranking_simple": 0.8041666746139526,
      "regularize": 0.2646006941795349,
      "step": 290
    },
    {
      "dpo_loss": 0.23113909363746643,
      "dpo_wo_beta": -0.6497251987457275,
      "epoch": 1.6721776098252243,
      "grad_norm": 11.90240881956814,
      "learning_rate": 2.4256048425921693e-06,
      "logits": -1.8574607372283936,
      "logps": -94.91531372070312,
      "loss": 0.2476,
      "objective": 0.23113909363746643,
      "ranking_idealized": 0.5791666507720947,
      "ranking_idealized_expo": 0.4749999940395355,
      "ranking_simple": 0.8333333134651184,
      "regularize": 0.23113909363746643,
      "step": 295
    },
    {
      "dpo_loss": 0.22116926312446594,
      "dpo_wo_beta": -0.6268281936645508,
      "epoch": 1.7005196032120926,
      "grad_norm": 11.745161783871675,
      "learning_rate": 2.3430237011767166e-06,
      "logits": -1.895004153251648,
      "logps": -97.79885864257812,
      "loss": 0.2266,
      "objective": 0.22116926312446594,
      "ranking_idealized": 0.6583333611488342,
      "ranking_idealized_expo": 0.574999988079071,
      "ranking_simple": 0.8333333134651184,
      "regularize": 0.22116926312446594,
      "step": 300
    },
    {
      "dpo_loss": 0.24756571650505066,
      "dpo_wo_beta": -0.9131773114204407,
      "epoch": 1.7288615965989607,
      "grad_norm": 12.299641904512029,
      "learning_rate": 2.2606142110393248e-06,
      "logits": -1.8061485290527344,
      "logps": -96.69060516357422,
      "loss": 0.2379,
      "objective": 0.24756571650505066,
      "ranking_idealized": 0.5625,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.8291666507720947,
      "regularize": 0.24756571650505066,
      "step": 305
    },
    {
      "dpo_loss": 0.2321903556585312,
      "dpo_wo_beta": -0.6867564916610718,
      "epoch": 1.7572035899858292,
      "grad_norm": 13.489735935272718,
      "learning_rate": 2.1784664857475356e-06,
      "logits": -1.8388514518737793,
      "logps": -95.04447937011719,
      "loss": 0.2456,
      "objective": 0.2321903556585312,
      "ranking_idealized": 0.6083333492279053,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.824999988079071,
      "regularize": 0.2321903556585312,
      "step": 310
    },
    {
      "dpo_loss": 0.2901044189929962,
      "dpo_wo_beta": -1.1286156177520752,
      "epoch": 1.7855455833726972,
      "grad_norm": 10.887596324980125,
      "learning_rate": 2.096670352632873e-06,
      "logits": -1.75984525680542,
      "logps": -94.63612365722656,
      "loss": 0.2571,
      "objective": 0.2901044189929962,
      "ranking_idealized": 0.6041666865348816,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.8083333373069763,
      "regularize": 0.2901044189929962,
      "step": 315
    },
    {
      "epoch": 1.8025507794048181,
      "eval_dpo_loss": 0.7664583325386047,
      "eval_dpo_wo_beta": -4.09501838684082,
      "eval_logits": -1.9888346195220947,
      "eval_logps": -102.3712158203125,
      "eval_loss": 0.7400712966918945,
      "eval_objective": 0.7664583325386047,
      "eval_ranking_idealized": 0.5888429880142212,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.53925621509552,
      "eval_regularize": 0.7664583325386047,
      "eval_runtime": 210.274,
      "eval_samples_per_second": 27.535,
      "eval_steps_per_second": 1.151,
      "step": 318
    },
    {
      "dpo_loss": 0.2219768464565277,
      "dpo_wo_beta": -0.47742757201194763,
      "epoch": 1.8138875767595655,
      "grad_norm": 11.029480506309918,
      "learning_rate": 2.01531525456598e-06,
      "logits": -1.9175788164138794,
      "logps": -99.74655151367188,
      "loss": 0.2404,
      "objective": 0.2219768464565277,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.824999988079071,
      "regularize": 0.2219768464565277,
      "step": 320
    },
    {
      "dpo_loss": 0.24908211827278137,
      "dpo_wo_beta": -0.8014059066772461,
      "epoch": 1.8422295701464337,
      "grad_norm": 12.92850322071669,
      "learning_rate": 1.93449015215215e-06,
      "logits": -2.0084919929504395,
      "logps": -101.09780883789062,
      "loss": 0.2586,
      "objective": 0.24908211827278137,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.8333333134651184,
      "regularize": 0.24908211827278137,
      "step": 325
    },
    {
      "dpo_loss": 0.1984507441520691,
      "dpo_wo_beta": -0.3766098618507385,
      "epoch": 1.8705715635333018,
      "grad_norm": 10.415606016359964,
      "learning_rate": 1.8542834264542091e-06,
      "logits": -1.851909875869751,
      "logps": -94.5366439819336,
      "loss": 0.2496,
      "objective": 0.1984507441520691,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.4791666567325592,
      "ranking_simple": 0.8291666507720947,
      "regularize": 0.1984507441520691,
      "step": 330
    },
    {
      "dpo_loss": 0.26707762479782104,
      "dpo_wo_beta": -0.9339324831962585,
      "epoch": 1.89891355692017,
      "grad_norm": 10.078352873471246,
      "learning_rate": 1.7747827823491253e-06,
      "logits": -1.9827288389205933,
      "logps": -94.26249694824219,
      "loss": 0.2463,
      "objective": 0.26707762479782104,
      "ranking_idealized": 0.5958333611488342,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.8125,
      "regularize": 0.26707762479782104,
      "step": 335
    },
    {
      "dpo_loss": 0.2447831928730011,
      "dpo_wo_beta": -0.7387041449546814,
      "epoch": 1.9272555503070383,
      "grad_norm": 10.88136655004607,
      "learning_rate": 1.6960751526240122e-06,
      "logits": -1.9671465158462524,
      "logps": -98.63937377929688,
      "loss": 0.2399,
      "objective": 0.2447831928730011,
      "ranking_idealized": 0.6416666507720947,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.8583333492279053,
      "regularize": 0.2447831928730011,
      "step": 340
    },
    {
      "dpo_loss": 0.2123527079820633,
      "dpo_wo_beta": -0.5544185638427734,
      "epoch": 1.9555975436939064,
      "grad_norm": 11.18260747105762,
      "learning_rate": 1.6182466029163974e-06,
      "logits": -1.9572845697402954,
      "logps": -100.18721008300781,
      "loss": 0.2211,
      "objective": 0.2123527079820633,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.8208333253860474,
      "regularize": 0.2123527079820633,
      "step": 345
    },
    {
      "dpo_loss": 0.2570362389087677,
      "dpo_wo_beta": -0.7474013566970825,
      "epoch": 1.9839395370807746,
      "grad_norm": 11.061918116138507,
      "learning_rate": 1.541382237602721e-06,
      "logits": -1.8960832357406616,
      "logps": -101.65901947021484,
      "loss": 0.2316,
      "objective": 0.2570362389087677,
      "ranking_idealized": 0.5791666507720947,
      "ranking_idealized_expo": 0.48750001192092896,
      "ranking_simple": 0.7916666865348816,
      "regularize": 0.2570362389087677,
      "step": 350
    },
    {
      "dpo_loss": 0.19961656630039215,
      "dpo_wo_beta": -0.5642960667610168,
      "epoch": 2.012281530467643,
      "grad_norm": 7.569515164252156,
      "learning_rate": 1.465566106737942e-06,
      "logits": -1.8380100727081299,
      "logps": -102.71571350097656,
      "loss": 0.2103,
      "objective": 0.19961656630039215,
      "ranking_idealized": 0.5958333611488342,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.8374999761581421,
      "regularize": 0.19961656630039215,
      "step": 355
    },
    {
      "dpo_loss": 0.11018560826778412,
      "dpo_wo_beta": -0.12253165245056152,
      "epoch": 2.040623523854511,
      "grad_norm": 6.632276986432463,
      "learning_rate": 1.3908811141480408e-06,
      "logits": -1.867693543434143,
      "logps": -103.06665802001953,
      "loss": 0.118,
      "objective": 0.11018560826778412,
      "ranking_idealized": 0.6625000238418579,
      "ranking_idealized_expo": 0.5375000238418579,
      "ranking_simple": 0.9041666388511658,
      "regularize": 0.11018560826778412,
      "step": 360
    },
    {
      "dpo_loss": 0.12077057361602783,
      "dpo_wo_beta": -0.197490856051445,
      "epoch": 2.0689655172413794,
      "grad_norm": 10.213186193965676,
      "learning_rate": 1.3174089267758983e-06,
      "logits": -1.8255099058151245,
      "logps": -110.3724136352539,
      "loss": 0.118,
      "objective": 0.12077057361602783,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.8666666746139526,
      "regularize": 0.12077057361602783,
      "step": 365
    },
    {
      "dpo_loss": 0.1337815225124359,
      "dpo_wo_beta": -0.27523547410964966,
      "epoch": 2.0973075106282475,
      "grad_norm": 9.926730675582434,
      "learning_rate": 1.245229885379699e-06,
      "logits": -1.7588540315628052,
      "logps": -111.99506378173828,
      "loss": 0.1227,
      "objective": 0.1337815225124359,
      "ranking_idealized": 0.5708333253860474,
      "ranking_idealized_expo": 0.4791666567325592,
      "ranking_simple": 0.8916666507720947,
      "regularize": 0.1337815225124359,
      "step": 370
    },
    {
      "epoch": 2.1029759093056213,
      "eval_dpo_loss": 0.9223728179931641,
      "eval_dpo_wo_beta": -6.4510064125061035,
      "eval_logits": -1.8644566535949707,
      "eval_logps": -122.00161743164062,
      "eval_loss": 0.8844180107116699,
      "eval_objective": 0.9223728179931641,
      "eval_ranking_idealized": 0.5888429880142212,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5423553586006165,
      "eval_regularize": 0.9223728179931641,
      "eval_runtime": 210.7356,
      "eval_samples_per_second": 27.475,
      "eval_steps_per_second": 1.148,
      "step": 371
    },
    {
      "dpo_loss": 0.10664375871419907,
      "dpo_wo_beta": -0.2532973289489746,
      "epoch": 2.1256495040151155,
      "grad_norm": 9.740007111179482,
      "learning_rate": 1.1744229166814889e-06,
      "logits": -1.696647047996521,
      "logps": -118.39366149902344,
      "loss": 0.1103,
      "objective": 0.10664375871419907,
      "ranking_idealized": 0.675000011920929,
      "ranking_idealized_expo": 0.5916666388511658,
      "ranking_simple": 0.925000011920929,
      "regularize": 0.10664375871419907,
      "step": 375
    },
    {
      "dpo_loss": 0.12854978442192078,
      "dpo_wo_beta": -0.27664583921432495,
      "epoch": 2.153991497401984,
      "grad_norm": 9.699256456859702,
      "learning_rate": 1.1050654470619602e-06,
      "logits": -1.700494647026062,
      "logps": -114.1063232421875,
      "loss": 0.1208,
      "objective": 0.12854978442192078,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.8999999761581421,
      "regularize": 0.12854978442192078,
      "step": 380
    },
    {
      "dpo_loss": 0.10418140888214111,
      "dpo_wo_beta": -0.09889766573905945,
      "epoch": 2.182333490788852,
      "grad_norm": 9.620361843085416,
      "learning_rate": 1.0372333178958462e-06,
      "logits": -1.8633235692977905,
      "logps": -110.55794525146484,
      "loss": 0.1244,
      "objective": 0.10418140888214111,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.9125000238418579,
      "regularize": 0.10418140888214111,
      "step": 385
    },
    {
      "dpo_loss": 0.12462247163057327,
      "dpo_wo_beta": -0.2658768594264984,
      "epoch": 2.21067548417572,
      "grad_norm": 11.000881222201947,
      "learning_rate": 9.710007026204896e-07,
      "logits": -1.7877620458602905,
      "logps": -112.08268737792969,
      "loss": 0.1204,
      "objective": 0.12462247163057327,
      "ranking_idealized": 0.5249999761581421,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.8833333253860474,
      "regularize": 0.12462247163057327,
      "step": 390
    },
    {
      "dpo_loss": 0.11183874309062958,
      "dpo_wo_beta": -0.3540593981742859,
      "epoch": 2.2390174775625886,
      "grad_norm": 8.717110295390793,
      "learning_rate": 9.064400256282757e-07,
      "logits": -1.8010636568069458,
      "logps": -110.48490142822266,
      "loss": 0.1248,
      "objective": 0.11183874309062958,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.9041666388511658,
      "regularize": 0.11183874309062958,
      "step": 395
    },
    {
      "dpo_loss": 0.12893003225326538,
      "dpo_wo_beta": -0.3680768311023712,
      "epoch": 2.2673594709494567,
      "grad_norm": 9.562073048936949,
      "learning_rate": 8.436218830716259e-07,
      "logits": -1.8909595012664795,
      "logps": -111.70219421386719,
      "loss": 0.1193,
      "objective": 0.12893003225326538,
      "ranking_idealized": 0.6625000238418579,
      "ranking_idealized_expo": 0.5708333253860474,
      "ranking_simple": 0.9166666865348816,
      "regularize": 0.12893003225326538,
      "step": 400
    },
    {
      "dpo_loss": 0.13196416199207306,
      "dpo_wo_beta": -0.17852090299129486,
      "epoch": 2.295701464336325,
      "grad_norm": 9.166021194752298,
      "learning_rate": 7.826149656671386e-07,
      "logits": -1.9320632219314575,
      "logps": -108.1246566772461,
      "loss": 0.1267,
      "objective": 0.13196416199207306,
      "ranking_idealized": 0.6416666507720947,
      "ranking_idealized_expo": 0.5583333373069763,
      "ranking_simple": 0.8958333134651184,
      "regularize": 0.13196416199207306,
      "step": 405
    },
    {
      "dpo_loss": 0.11071384698152542,
      "dpo_wo_beta": -0.1424117088317871,
      "epoch": 2.324043457723193,
      "grad_norm": 8.918983804471582,
      "learning_rate": 7.234859835833022e-07,
      "logits": -1.8304682970046997,
      "logps": -111.2301025390625,
      "loss": 0.112,
      "objective": 0.11071384698152542,
      "ranking_idealized": 0.5708333253860474,
      "ranking_idealized_expo": 0.4791666567325592,
      "ranking_simple": 0.9041666388511658,
      "regularize": 0.11071384698152542,
      "step": 410
    },
    {
      "dpo_loss": 0.1223960742354393,
      "dpo_wo_beta": -0.1956464648246765,
      "epoch": 2.3523854511100613,
      "grad_norm": 9.386393866562546,
      "learning_rate": 6.662995934939007e-07,
      "logits": -1.8708041906356812,
      "logps": -111.06449890136719,
      "loss": 0.1155,
      "objective": 0.1223960742354393,
      "ranking_idealized": 0.5958333611488342,
      "ranking_idealized_expo": 0.5291666388511658,
      "ranking_simple": 0.9083333611488342,
      "regularize": 0.1223960742354393,
      "step": 415
    },
    {
      "dpo_loss": 0.12930770218372345,
      "dpo_wo_beta": -0.21560731530189514,
      "epoch": 2.3807274444969297,
      "grad_norm": 11.0131183307354,
      "learning_rate": 6.111183278768956e-07,
      "logits": -1.860797643661499,
      "logps": -113.08780670166016,
      "loss": 0.133,
      "objective": 0.12930770218372345,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.4749999940395355,
      "ranking_simple": 0.9208333492279053,
      "regularize": 0.12930770218372345,
      "step": 420
    },
    {
      "epoch": 2.403401039206424,
      "eval_dpo_loss": 0.8785684108734131,
      "eval_dpo_wo_beta": -5.887755870819092,
      "eval_logits": -2.0276894569396973,
      "eval_logps": -117.1216812133789,
      "eval_loss": 0.8447906374931335,
      "eval_objective": 0.8785684108734131,
      "eval_ranking_idealized": 0.5888429880142212,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5413222908973694,
      "eval_regularize": 0.8785684108734131,
      "eval_runtime": 209.8564,
      "eval_samples_per_second": 27.59,
      "eval_steps_per_second": 1.153,
      "step": 424
    },
    {
      "dpo_loss": 0.117975153028965,
      "dpo_wo_beta": -0.1884605884552002,
      "epoch": 2.413793103448276,
      "grad_norm": 11.036168833651558,
      "learning_rate": 5.580025266360764e-07,
      "logits": -1.7822004556655884,
      "logps": -114.43038177490234,
      "loss": 0.1465,
      "objective": 0.117975153028965,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.5625,
      "ranking_simple": 0.9375,
      "regularize": 0.117975153028965,
      "step": 425
    },
    {
      "dpo_loss": 0.1465020477771759,
      "dpo_wo_beta": -0.2595965266227722,
      "epoch": 2.442135096835144,
      "grad_norm": 10.595070818850646,
      "learning_rate": 5.070102711202606e-07,
      "logits": -1.8692681789398193,
      "logps": -110.2762680053711,
      "loss": 0.1276,
      "objective": 0.1465020477771759,
      "ranking_idealized": 0.5916666388511658,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.8958333134651184,
      "regularize": 0.1465020477771759,
      "step": 430
    },
    {
      "dpo_loss": 0.09775053709745407,
      "dpo_wo_beta": -0.12755917012691498,
      "epoch": 2.4704770902220123,
      "grad_norm": 9.393206692367766,
      "learning_rate": 4.581973206121948e-07,
      "logits": -1.8968538045883179,
      "logps": -112.28767395019531,
      "loss": 0.1175,
      "objective": 0.09775053709745407,
      "ranking_idealized": 0.5874999761581421,
      "ranking_idealized_expo": 0.512499988079071,
      "ranking_simple": 0.9083333611488342,
      "regularize": 0.09775053709745407,
      "step": 435
    },
    {
      "dpo_loss": 0.14228057861328125,
      "dpo_wo_beta": -0.3639788329601288,
      "epoch": 2.4988190836088804,
      "grad_norm": 8.020134663378592,
      "learning_rate": 4.116170513565942e-07,
      "logits": -1.8666160106658936,
      "logps": -109.18843078613281,
      "loss": 0.1167,
      "objective": 0.14228057861328125,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.875,
      "regularize": 0.14228057861328125,
      "step": 440
    },
    {
      "dpo_loss": 0.13583588600158691,
      "dpo_wo_beta": -0.2074100226163864,
      "epoch": 2.527161076995749,
      "grad_norm": 9.224367796824264,
      "learning_rate": 3.6732039819400686e-07,
      "logits": -1.8071045875549316,
      "logps": -107.2675552368164,
      "loss": 0.1319,
      "objective": 0.13583588600158691,
      "ranking_idealized": 0.5625,
      "ranking_idealized_expo": 0.49166667461395264,
      "ranking_simple": 0.8791666626930237,
      "regularize": 0.13583588600158691,
      "step": 445
    },
    {
      "dpo_loss": 0.17114870250225067,
      "dpo_wo_beta": -0.43270742893218994,
      "epoch": 2.555503070382617,
      "grad_norm": 11.265861710797749,
      "learning_rate": 3.253557988643072e-07,
      "logits": -1.9256045818328857,
      "logps": -111.20384216308594,
      "loss": 0.1288,
      "objective": 0.17114870250225067,
      "ranking_idealized": 0.5874999761581421,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.862500011920929,
      "regularize": 0.17114870250225067,
      "step": 450
    },
    {
      "dpo_loss": 0.10827689617872238,
      "dpo_wo_beta": -0.1751028150320053,
      "epoch": 2.583845063769485,
      "grad_norm": 9.605136286662574,
      "learning_rate": 2.8576914104074425e-07,
      "logits": -1.9289051294326782,
      "logps": -109.37706756591797,
      "loss": 0.1168,
      "objective": 0.10827689617872238,
      "ranking_idealized": 0.5958333611488342,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.9125000238418579,
      "regularize": 0.10827689617872238,
      "step": 455
    },
    {
      "dpo_loss": 0.11124877631664276,
      "dpo_wo_beta": -0.28054580092430115,
      "epoch": 2.6121870571563535,
      "grad_norm": 9.957466667064367,
      "learning_rate": 2.486037121524448e-07,
      "logits": -1.93342924118042,
      "logps": -113.2356948852539,
      "loss": 0.1169,
      "objective": 0.11124877631664276,
      "ranking_idealized": 0.6291666626930237,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.9375,
      "regularize": 0.11124877631664276,
      "step": 460
    },
    {
      "dpo_loss": 0.12714476883411407,
      "dpo_wo_beta": -0.22146105766296387,
      "epoch": 2.6405290505432215,
      "grad_norm": 10.04326854921629,
      "learning_rate": 2.13900152050239e-07,
      "logits": -1.8874350786209106,
      "logps": -108.94982147216797,
      "loss": 0.1189,
      "objective": 0.12714476883411407,
      "ranking_idealized": 0.5458333492279053,
      "ranking_idealized_expo": 0.4791666567325592,
      "ranking_simple": 0.862500011920929,
      "regularize": 0.12714476883411407,
      "step": 465
    },
    {
      "dpo_loss": 0.12151040881872177,
      "dpo_wo_beta": -0.26416900753974915,
      "epoch": 2.66887104393009,
      "grad_norm": 8.777820527737605,
      "learning_rate": 1.8169640856758652e-07,
      "logits": -1.9314534664154053,
      "logps": -112.75170135498047,
      "loss": 0.1254,
      "objective": 0.12151040881872177,
      "ranking_idealized": 0.6291666626930237,
      "ranking_idealized_expo": 0.5541666746139526,
      "ranking_simple": 0.9083333611488342,
      "regularize": 0.12151040881872177,
      "step": 470
    },
    {
      "dpo_loss": 0.12749101221561432,
      "dpo_wo_beta": -0.2816121280193329,
      "epoch": 2.697213037316958,
      "grad_norm": 9.221778751171357,
      "learning_rate": 1.5202769602517514e-07,
      "logits": -1.8307260274887085,
      "logps": -109.39693450927734,
      "loss": 0.1211,
      "objective": 0.12749101221561432,
      "ranking_idealized": 0.6041666865348816,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.8999999761581421,
      "regularize": 0.12749101221561432,
      "step": 475
    },
    {
      "epoch": 2.708549834671705,
      "eval_dpo_loss": 0.8738968372344971,
      "eval_dpo_wo_beta": -5.815241813659668,
      "eval_logits": -2.0271613597869873,
      "eval_logps": -116.42301177978516,
      "eval_loss": 0.8371492624282837,
      "eval_objective": 0.8738968372344971,
      "eval_ranking_idealized": 0.5888429880142212,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5402892827987671,
      "eval_regularize": 0.8738968372344971,
      "eval_runtime": 211.9437,
      "eval_samples_per_second": 27.319,
      "eval_steps_per_second": 1.142,
      "step": 477
    },
    {
      "dpo_loss": 0.13781045377254486,
      "dpo_wo_beta": -0.2485995590686798,
      "epoch": 2.725555030703826,
      "grad_norm": 10.971551462649595,
      "learning_rate": 1.2492645672457838e-07,
      "logits": -1.9437103271484375,
      "logps": -108.93817901611328,
      "loss": 0.1267,
      "objective": 0.13781045377254486,
      "ranking_idealized": 0.5916666388511658,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.8999999761581421,
      "regularize": 0.13781045377254486,
      "step": 480
    },
    {
      "dpo_loss": 0.11082082241773605,
      "dpo_wo_beta": -0.10876031965017319,
      "epoch": 2.753897024090694,
      "grad_norm": 10.884940640535042,
      "learning_rate": 1.004223254730749e-07,
      "logits": -1.7556992769241333,
      "logps": -114.1142807006836,
      "loss": 0.1222,
      "objective": 0.11082082241773605,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.9041666388511658,
      "regularize": 0.11082082241773605,
      "step": 485
    },
    {
      "dpo_loss": 0.09154360741376877,
      "dpo_wo_beta": -0.05899694189429283,
      "epoch": 2.7822390174775626,
      "grad_norm": 11.015982469457516,
      "learning_rate": 7.854209717842231e-08,
      "logits": -1.8848822116851807,
      "logps": -110.15470886230469,
      "loss": 0.1058,
      "objective": 0.09154360741376877,
      "ranking_idealized": 0.5874999761581421,
      "ranking_idealized_expo": 0.47083333134651184,
      "ranking_simple": 0.9333333373069763,
      "regularize": 0.09154360741376877,
      "step": 490
    },
    {
      "dpo_loss": 0.10964310169219971,
      "dpo_wo_beta": -0.07648710906505585,
      "epoch": 2.8105810108644307,
      "grad_norm": 10.079416267782939,
      "learning_rate": 5.930969754901844e-08,
      "logits": -1.8575230836868286,
      "logps": -108.52234649658203,
      "loss": 0.1192,
      "objective": 0.10964310169219971,
      "ranking_idealized": 0.5583333373069763,
      "ranking_idealized_expo": 0.4749999940395355,
      "ranking_simple": 0.8666666746139526,
      "regularize": 0.10964310169219971,
      "step": 495
    },
    {
      "dpo_loss": 0.09479068219661713,
      "dpo_wo_beta": -0.03411731496453285,
      "epoch": 2.838923004251299,
      "grad_norm": 9.84080114767598,
      "learning_rate": 4.2746156931490756e-08,
      "logits": -1.8439643383026123,
      "logps": -109.77281188964844,
      "loss": 0.1213,
      "objective": 0.09479068219661713,
      "ranking_idealized": 0.6208333373069763,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.925000011920929,
      "regularize": 0.09479068219661713,
      "step": 500
    },
    {
      "dpo_loss": 0.12725140154361725,
      "dpo_wo_beta": -0.18973813951015472,
      "epoch": 2.8672649976381672,
      "grad_norm": 9.973754192936779,
      "learning_rate": 2.8869587314321324e-08,
      "logits": -1.8574442863464355,
      "logps": -110.32710266113281,
      "loss": 0.132,
      "objective": 0.12725140154361725,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.8999999761581421,
      "regularize": 0.12725140154361725,
      "step": 505
    },
    {
      "dpo_loss": 0.10469380766153336,
      "dpo_wo_beta": -0.1985001415014267,
      "epoch": 2.8956069910250353,
      "grad_norm": 8.936464383287202,
      "learning_rate": 1.7695162522652352e-08,
      "logits": -1.8629390001296997,
      "logps": -113.56767272949219,
      "loss": 0.1218,
      "objective": 0.10469380766153336,
      "ranking_idealized": 0.6041666865348816,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.8916666507720947,
      "regularize": 0.10469380766153336,
      "step": 510
    },
    {
      "dpo_loss": 0.11260154843330383,
      "dpo_wo_beta": -0.15691885352134705,
      "epoch": 2.9239489844119038,
      "grad_norm": 9.442300088571939,
      "learning_rate": 9.235101625932885e-09,
      "logits": -1.946829915046692,
      "logps": -108.54016876220703,
      "loss": 0.1258,
      "objective": 0.11260154843330383,
      "ranking_idealized": 0.5791666507720947,
      "ranking_idealized_expo": 0.48750001192092896,
      "ranking_simple": 0.925000011920929,
      "regularize": 0.11260154843330383,
      "step": 515
    },
    {
      "dpo_loss": 0.12230218201875687,
      "dpo_wo_beta": -0.10489177703857422,
      "epoch": 2.952290977798772,
      "grad_norm": 9.279898048101137,
      "learning_rate": 3.4986555765434415e-09,
      "logits": -1.8482831716537476,
      "logps": -114.20655059814453,
      "loss": 0.1228,
      "objective": 0.12230218201875687,
      "ranking_idealized": 0.6291666626930237,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.8708333373069763,
      "regularize": 0.12230218201875687,
      "step": 520
    },
    {
      "dpo_loss": 0.13335375487804413,
      "dpo_wo_beta": -0.35261282324790955,
      "epoch": 2.9806329711856403,
      "grad_norm": 11.094809681697281,
      "learning_rate": 4.920970940180958e-10,
      "logits": -1.876869797706604,
      "logps": -111.03084564208984,
      "loss": 0.1235,
      "objective": 0.13335375487804413,
      "ranking_idealized": 0.6291666626930237,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.8958333134651184,
      "regularize": 0.13335375487804413,
      "step": 525
    },
    {
      "dpo_loss": 0.08052093535661697,
      "dpo_wo_beta": -0.004379949066787958,
      "epoch": 1.5044874822862542,
      "grad_norm": 9.015683090612498,
      "learning_rate": 2.9196388040863695e-06,
      "logits": -1.833287239074707,
      "logps": -114.10733795166016,
      "loss": 0.0858,
      "objective": 0.08052093535661697,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.9375,
      "regularize": 0.08052093535661697,
      "step": 530
    },
    {
      "epoch": 1.5044874822862542,
      "eval_dpo_loss": 0.8753401041030884,
      "eval_dpo_wo_beta": -5.922874450683594,
      "eval_logits": -2.4529590606689453,
      "eval_logps": -118.25288391113281,
      "eval_loss": 0.8505071997642517,
      "eval_objective": 0.8753401041030884,
      "eval_ranking_idealized": 0.6045548915863037,
      "eval_ranking_idealized_expo": 0.5279502868652344,
      "eval_ranking_simple": 0.5683229565620422,
      "eval_regularize": 0.8753401041030884,
      "eval_runtime": 344.4459,
      "eval_samples_per_second": 16.81,
      "eval_steps_per_second": 1.402,
      "step": 530
    },
    {
      "dpo_loss": 0.06306228041648865,
      "dpo_wo_beta": -0.08629266172647476,
      "epoch": 1.5186584789796882,
      "grad_norm": 12.634106279395441,
      "learning_rate": 2.8788330549198512e-06,
      "logits": -1.8371531963348389,
      "logps": -115.510009765625,
      "loss": 0.0766,
      "objective": 0.06306228041648865,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.9166666865348816,
      "regularize": 0.06306228041648865,
      "step": 535
    },
    {
      "dpo_loss": 0.13102607429027557,
      "dpo_wo_beta": -0.41934680938720703,
      "epoch": 1.5328294756731222,
      "grad_norm": 14.583212522491863,
      "learning_rate": 2.8379237365787426e-06,
      "logits": -1.7649121284484863,
      "logps": -127.44564819335938,
      "loss": 0.0928,
      "objective": 0.13102607429027557,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.8999999761581421,
      "regularize": 0.13102607429027557,
      "step": 540
    },
    {
      "dpo_loss": 0.12010473757982254,
      "dpo_wo_beta": -0.2892196476459503,
      "epoch": 1.5470004723665565,
      "grad_norm": 18.74636858022955,
      "learning_rate": 2.7969220332622004e-06,
      "logits": -1.7403244972229004,
      "logps": -126.89453125,
      "loss": 0.1114,
      "objective": 0.12010473757982254,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.925000011920929,
      "regularize": 0.12010473757982254,
      "step": 545
    },
    {
      "dpo_loss": 0.05550822243094444,
      "dpo_wo_beta": -0.0006199590279720724,
      "epoch": 1.5611714690599907,
      "grad_norm": 16.65748534309405,
      "learning_rate": 2.7558391544265127e-06,
      "logits": -1.7434070110321045,
      "logps": -120.79520416259766,
      "loss": 0.0872,
      "objective": 0.05550822243094444,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.9750000238418579,
      "regularize": 0.05550822243094444,
      "step": 550
    },
    {
      "dpo_loss": 0.12240471690893173,
      "dpo_wo_beta": -0.22643856704235077,
      "epoch": 1.5753424657534247,
      "grad_norm": 12.27653222683965,
      "learning_rate": 2.714686331720543e-06,
      "logits": -1.8163702487945557,
      "logps": -123.55280303955078,
      "loss": 0.103,
      "objective": 0.12240471690893173,
      "ranking_idealized": 0.6083333492279053,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.875,
      "regularize": 0.12240471690893173,
      "step": 555
    },
    {
      "dpo_loss": 0.1260891556739807,
      "dpo_wo_beta": -0.384922057390213,
      "epoch": 1.5895134624468588,
      "grad_norm": 14.054554594902122,
      "learning_rate": 2.6734748159151104e-06,
      "logits": -1.8347235918045044,
      "logps": -115.50324249267578,
      "loss": 0.1171,
      "objective": 0.1260891556739807,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.4749999940395355,
      "ranking_simple": 0.8916666507720947,
      "regularize": 0.1260891556739807,
      "step": 560
    },
    {
      "dpo_loss": 0.11428937315940857,
      "dpo_wo_beta": -0.142462819814682,
      "epoch": 1.6036844591402928,
      "grad_norm": 8.888993907970832,
      "learning_rate": 2.632215873827142e-06,
      "logits": -1.9027162790298462,
      "logps": -117.14366912841797,
      "loss": 0.1142,
      "objective": 0.11428937315940857,
      "ranking_idealized": 0.5416666865348816,
      "ranking_idealized_expo": 0.4583333432674408,
      "ranking_simple": 0.8916666507720947,
      "regularize": 0.11428937315940857,
      "step": 565
    },
    {
      "dpo_loss": 0.14770367741584778,
      "dpo_wo_beta": -0.5861695408821106,
      "epoch": 1.6178554558337268,
      "grad_norm": 9.396256315509223,
      "learning_rate": 2.5909207852394363e-06,
      "logits": -2.088587760925293,
      "logps": -113.1082992553711,
      "loss": 0.1098,
      "objective": 0.14770367741584778,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.9083333611488342,
      "regularize": 0.14770367741584778,
      "step": 570
    },
    {
      "dpo_loss": 0.1413314789533615,
      "dpo_wo_beta": -0.3796103298664093,
      "epoch": 1.632026452527161,
      "grad_norm": 16.325704474226345,
      "learning_rate": 2.5496008398168844e-06,
      "logits": -1.9472541809082031,
      "logps": -117.12679290771484,
      "loss": 0.1404,
      "objective": 0.1413314789533615,
      "ranking_idealized": 0.6583333611488342,
      "ranking_idealized_expo": 0.6166666746139526,
      "ranking_simple": 0.9166666865348816,
      "regularize": 0.1413314789533615,
      "step": 575
    },
    {
      "dpo_loss": 0.1715042144060135,
      "dpo_wo_beta": -0.48858124017715454,
      "epoch": 1.6461974492205953,
      "grad_norm": 11.97898567349363,
      "learning_rate": 2.508267334019988e-06,
      "logits": -1.935112476348877,
      "logps": -107.23208618164062,
      "loss": 0.1274,
      "objective": 0.1715042144060135,
      "ranking_idealized": 0.5583333373069763,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.8999999761581421,
      "regularize": 0.1715042144060135,
      "step": 580
    },
    {
      "epoch": 1.6547000472366555,
      "eval_dpo_loss": 0.8264312148094177,
      "eval_dpo_wo_beta": -5.2847466468811035,
      "eval_logits": -2.4379913806915283,
      "eval_logps": -119.59071350097656,
      "eval_loss": 0.808626651763916,
      "eval_objective": 0.8264312148094177,
      "eval_ranking_idealized": 0.6045548915863037,
      "eval_ranking_idealized_expo": 0.5279502868652344,
      "eval_ranking_simple": 0.5869565010070801,
      "eval_regularize": 0.8264312148094177,
      "eval_runtime": 355.6593,
      "eval_samples_per_second": 16.28,
      "eval_steps_per_second": 1.358,
      "step": 583
    },
    {
      "dpo_loss": 0.1341490000486374,
      "dpo_wo_beta": -0.5180007815361023,
      "epoch": 1.6603684459140293,
      "grad_norm": 15.27561747473412,
      "learning_rate": 2.46693156801652e-06,
      "logits": -1.8945667743682861,
      "logps": -110.8204116821289,
      "loss": 0.1227,
      "objective": 0.1341490000486374,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.9083333611488342,
      "regularize": 0.1341490000486374,
      "step": 585
    },
    {
      "dpo_loss": 0.13649246096611023,
      "dpo_wo_beta": -0.5407892465591431,
      "epoch": 1.6745394426074633,
      "grad_norm": 8.066407586221112,
      "learning_rate": 2.4256048425921693e-06,
      "logits": -1.8402847051620483,
      "logps": -117.5975112915039,
      "loss": 0.1263,
      "objective": 0.13649246096611023,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.9083333611488342,
      "regularize": 0.13649246096611023,
      "step": 590
    },
    {
      "dpo_loss": 0.14796380698680878,
      "dpo_wo_beta": -0.5909832119941711,
      "epoch": 1.6887104393008974,
      "grad_norm": 17.4246056389601,
      "learning_rate": 2.384298456061023e-06,
      "logits": -1.8368481397628784,
      "logps": -117.76559448242188,
      "loss": 0.1544,
      "objective": 0.14796380698680878,
      "ranking_idealized": 0.675000011920929,
      "ranking_idealized_expo": 0.5916666388511658,
      "ranking_simple": 0.8916666507720947,
      "regularize": 0.14796380698680878,
      "step": 595
    },
    {
      "dpo_loss": 0.15043622255325317,
      "dpo_wo_beta": -0.6607655882835388,
      "epoch": 1.7028814359943316,
      "grad_norm": 18.447927668502214,
      "learning_rate": 2.3430237011767166e-06,
      "logits": -1.839497447013855,
      "logps": -112.93590545654297,
      "loss": 0.15,
      "objective": 0.15043622255325317,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.875,
      "regularize": 0.15043622255325317,
      "step": 600
    },
    {
      "dpo_loss": 0.11465544998645782,
      "dpo_wo_beta": -0.34068503975868225,
      "epoch": 1.7170524326877659,
      "grad_norm": 12.561303443006217,
      "learning_rate": 2.30179186204511e-06,
      "logits": -1.7405670881271362,
      "logps": -112.9049072265625,
      "loss": 0.1278,
      "objective": 0.11465544998645782,
      "ranking_idealized": 0.5583333373069763,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.8999999761581421,
      "regularize": 0.11465544998645782,
      "step": 605
    },
    {
      "dpo_loss": 0.1477740854024887,
      "dpo_wo_beta": -0.4079127609729767,
      "epoch": 1.7312234293811999,
      "grad_norm": 17.631548474702864,
      "learning_rate": 2.2606142110393248e-06,
      "logits": -1.7865701913833618,
      "logps": -109.32853698730469,
      "loss": 0.1331,
      "objective": 0.1477740854024887,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.875,
      "regularize": 0.1477740854024887,
      "step": 610
    },
    {
      "dpo_loss": 0.18279190361499786,
      "dpo_wo_beta": -0.8598226308822632,
      "epoch": 1.745394426074634,
      "grad_norm": 10.88386046876842,
      "learning_rate": 2.2195020057179897e-06,
      "logits": -1.7821184396743774,
      "logps": -111.59290313720703,
      "loss": 0.136,
      "objective": 0.18279190361499786,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.875,
      "regularize": 0.18279190361499786,
      "step": 615
    },
    {
      "dpo_loss": 0.1030309647321701,
      "dpo_wo_beta": -0.05606275424361229,
      "epoch": 1.759565422768068,
      "grad_norm": 12.949368950505932,
      "learning_rate": 2.1784664857475356e-06,
      "logits": -1.780458688735962,
      "logps": -108.14535522460938,
      "loss": 0.1303,
      "objective": 0.1030309647321701,
      "ranking_idealized": 0.6416666507720947,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.9666666388511658,
      "regularize": 0.1030309647321701,
      "step": 620
    },
    {
      "dpo_loss": 0.158640518784523,
      "dpo_wo_beta": -0.4262932240962982,
      "epoch": 1.7737364194615022,
      "grad_norm": 15.36682532028932,
      "learning_rate": 2.1375188698293855e-06,
      "logits": -1.657003402709961,
      "logps": -110.36738586425781,
      "loss": 0.161,
      "objective": 0.158640518784523,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.9083333611488342,
      "regularize": 0.158640518784523,
      "step": 625
    },
    {
      "dpo_loss": 0.21892648935317993,
      "dpo_wo_beta": -0.9036411046981812,
      "epoch": 1.7879074161549362,
      "grad_norm": 14.959909753098716,
      "learning_rate": 2.096670352632873e-06,
      "logits": -1.7339377403259277,
      "logps": -107.9156723022461,
      "loss": 0.1673,
      "objective": 0.21892648935317993,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.49166667461395264,
      "ranking_simple": 0.7833333611488342,
      "regularize": 0.21892648935317993,
      "step": 630
    },
    {
      "dpo_loss": 0.14205454289913177,
      "dpo_wo_beta": -0.47540512681007385,
      "epoch": 1.8020784128483704,
      "grad_norm": 22.25702396524442,
      "learning_rate": 2.0559321017347286e-06,
      "logits": -2.000821352005005,
      "logps": -119.52375793457031,
      "loss": 0.1614,
      "objective": 0.14205454289913177,
      "ranking_idealized": 0.675000011920929,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.8999999761581421,
      "regularize": 0.14205454289913177,
      "step": 635
    },
    {
      "epoch": 1.8049126121870571,
      "eval_dpo_loss": 0.8243346214294434,
      "eval_dpo_wo_beta": -5.281310081481934,
      "eval_logits": -2.4850430488586426,
      "eval_logps": -117.85846710205078,
      "eval_loss": 0.8209081292152405,
      "eval_objective": 0.8243346214294434,
      "eval_ranking_idealized": 0.6045548915863037,
      "eval_ranking_idealized_expo": 0.5279502868652344,
      "eval_ranking_simple": 0.5817805528640747,
      "eval_regularize": 0.8243346214294434,
      "eval_runtime": 351.818,
      "eval_samples_per_second": 16.457,
      "eval_steps_per_second": 1.373,
      "step": 636
    },
    {
      "dpo_loss": 0.13119691610336304,
      "dpo_wo_beta": -0.36706313490867615,
      "epoch": 1.8162494095418045,
      "grad_norm": 20.631621209893076,
      "learning_rate": 2.01531525456598e-06,
      "logits": -1.8479942083358765,
      "logps": -111.6270523071289,
      "loss": 0.1417,
      "objective": 0.13119691610336304,
      "ranking_idealized": 0.5249999761581421,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.8583333492279053,
      "regularize": 0.13119691610336304,
      "step": 640
    },
    {
      "dpo_loss": 0.1571781039237976,
      "dpo_wo_beta": -0.5080724954605103,
      "epoch": 1.8304204062352385,
      "grad_norm": 20.508347622723388,
      "learning_rate": 1.974830915367086e-06,
      "logits": -1.9629262685775757,
      "logps": -122.33467102050781,
      "loss": 0.1732,
      "objective": 0.1571781039237976,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.875,
      "regularize": 0.1571781039237976,
      "step": 645
    },
    {
      "dpo_loss": 0.18711452186107635,
      "dpo_wo_beta": -0.5334885120391846,
      "epoch": 1.8445914029286725,
      "grad_norm": 17.78377983289135,
      "learning_rate": 1.93449015215215e-06,
      "logits": -1.960700511932373,
      "logps": -116.45585632324219,
      "loss": 0.1873,
      "objective": 0.18711452186107635,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.875,
      "regularize": 0.18711452186107635,
      "step": 650
    },
    {
      "dpo_loss": 0.14089132845401764,
      "dpo_wo_beta": -0.257717490196228,
      "epoch": 1.8587623996221068,
      "grad_norm": 17.423684978791957,
      "learning_rate": 1.8943039936830347e-06,
      "logits": -1.7539128065109253,
      "logps": -105.96385192871094,
      "loss": 0.1703,
      "objective": 0.14089132845401764,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.8416666388511658,
      "regularize": 0.14089132845401764,
      "step": 655
    },
    {
      "dpo_loss": 0.12644439935684204,
      "dpo_wo_beta": -0.18994450569152832,
      "epoch": 1.872933396315541,
      "grad_norm": 14.114136095930363,
      "learning_rate": 1.8542834264542091e-06,
      "logits": -1.8740805387496948,
      "logps": -112.58109283447266,
      "loss": 0.1679,
      "objective": 0.12644439935684204,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.8833333253860474,
      "regularize": 0.12644439935684204,
      "step": 660
    },
    {
      "dpo_loss": 0.1596754938364029,
      "dpo_wo_beta": -0.5118387341499329,
      "epoch": 1.887104393008975,
      "grad_norm": 17.02657500840854,
      "learning_rate": 1.814439391689151e-06,
      "logits": -2.012057065963745,
      "logps": -106.6546401977539,
      "loss": 0.1748,
      "objective": 0.1596754938364029,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.8583333492279053,
      "regularize": 0.1596754938364029,
      "step": 665
    },
    {
      "dpo_loss": 0.1905670315027237,
      "dpo_wo_beta": -0.7486369609832764,
      "epoch": 1.901275389702409,
      "grad_norm": 14.652072743525036,
      "learning_rate": 1.7747827823491253e-06,
      "logits": -1.7807596921920776,
      "logps": -107.37975311279297,
      "loss": 0.1695,
      "objective": 0.1905670315027237,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.8666666746139526,
      "regularize": 0.1905670315027237,
      "step": 670
    },
    {
      "dpo_loss": 0.14669080078601837,
      "dpo_wo_beta": -0.3554477095603943,
      "epoch": 1.915446386395843,
      "grad_norm": 15.076428386388748,
      "learning_rate": 1.7353244401551566e-06,
      "logits": -1.8374218940734863,
      "logps": -107.03987884521484,
      "loss": 0.1555,
      "objective": 0.14669080078601837,
      "ranking_idealized": 0.6416666507720947,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.8999999761581421,
      "regularize": 0.14669080078601837,
      "step": 675
    },
    {
      "dpo_loss": 0.15277433395385742,
      "dpo_wo_beta": -0.5579003095626831,
      "epoch": 1.9296173830892773,
      "grad_norm": 14.063016020977683,
      "learning_rate": 1.6960751526240122e-06,
      "logits": -1.9377697706222534,
      "logps": -118.40213775634766,
      "loss": 0.1487,
      "objective": 0.15277433395385742,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.8833333253860474,
      "regularize": 0.15277433395385742,
      "step": 680
    },
    {
      "dpo_loss": 0.1468452513217926,
      "dpo_wo_beta": -0.34248843789100647,
      "epoch": 1.9437883797827114,
      "grad_norm": 20.037932580562952,
      "learning_rate": 1.6570456501189996e-06,
      "logits": -1.822561502456665,
      "logps": -116.04502868652344,
      "loss": 0.1616,
      "objective": 0.1468452513217926,
      "ranking_idealized": 0.5083333253860474,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.8999999761581421,
      "regularize": 0.1468452513217926,
      "step": 685
    },
    {
      "epoch": 1.9551251771374587,
      "eval_dpo_loss": 0.857575535774231,
      "eval_dpo_wo_beta": -5.723405361175537,
      "eval_logits": -2.465576410293579,
      "eval_logps": -119.32209777832031,
      "eval_loss": 0.8382942080497742,
      "eval_objective": 0.857575535774231,
      "eval_ranking_idealized": 0.6045548915863037,
      "eval_ranking_idealized_expo": 0.5279502868652344,
      "eval_ranking_simple": 0.5797101259231567,
      "eval_regularize": 0.857575535774231,
      "eval_runtime": 359.3448,
      "eval_samples_per_second": 16.113,
      "eval_steps_per_second": 1.344,
      "step": 689
    },
    {
      "dpo_loss": 0.11434569954872131,
      "dpo_wo_beta": -0.045675624161958694,
      "epoch": 1.9579593764761456,
      "grad_norm": 13.437704226981893,
      "learning_rate": 1.6182466029163974e-06,
      "logits": -1.8100759983062744,
      "logps": -115.22444915771484,
      "loss": 0.1309,
      "objective": 0.11434569954872131,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.574999988079071,
      "ranking_simple": 0.8999999761581421,
      "regularize": 0.11434569954872131,
      "step": 690
    },
    {
      "dpo_loss": 0.19163179397583008,
      "dpo_wo_beta": -0.46952199935913086,
      "epoch": 1.9721303731695796,
      "grad_norm": 20.21550526964092,
      "learning_rate": 1.5796886182883053e-06,
      "logits": -1.829925775527954,
      "logps": -114.886962890625,
      "loss": 0.1753,
      "objective": 0.19163179397583008,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.4416666626930237,
      "ranking_simple": 0.8166666626930237,
      "regularize": 0.19163179397583008,
      "step": 695
    },
    {
      "dpo_loss": 0.18629121780395508,
      "dpo_wo_beta": -0.7477880120277405,
      "epoch": 1.9863013698630136,
      "grad_norm": 14.017066914132515,
      "learning_rate": 1.541382237602721e-06,
      "logits": -1.7919304370880127,
      "logps": -115.88526916503906,
      "loss": 0.176,
      "objective": 0.18629121780395508,
      "ranking_idealized": 0.5916666388511658,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.8833333253860474,
      "regularize": 0.18629121780395508,
      "step": 700
    },
    {
      "dpo_loss": 0.20123900473117828,
      "dpo_wo_beta": -0.6124710440635681,
      "epoch": 2.0004723665564477,
      "grad_norm": 21.887245138739928,
      "learning_rate": 1.5033379334416376e-06,
      "logits": -1.786551594734192,
      "logps": -114.37857055664062,
      "loss": 0.2015,
      "objective": 0.20123900473117828,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.8333333134651184,
      "regularize": 0.20123900473117828,
      "step": 705
    },
    {
      "dpo_loss": 0.08176574856042862,
      "dpo_wo_beta": -0.0886942520737648,
      "epoch": 2.0146433632498817,
      "grad_norm": 12.07978325252889,
      "learning_rate": 1.465566106737942e-06,
      "logits": -1.8880345821380615,
      "logps": -114.7125244140625,
      "loss": 0.1005,
      "objective": 0.08176574856042862,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.925000011920929,
      "regularize": 0.08176574856042862,
      "step": 710
    },
    {
      "dpo_loss": 0.07800193130970001,
      "dpo_wo_beta": -0.004813884384930134,
      "epoch": 2.028814359943316,
      "grad_norm": 10.892350166600437,
      "learning_rate": 1.4280770839319073e-06,
      "logits": -1.8223975896835327,
      "logps": -109.677001953125,
      "loss": 0.1064,
      "objective": 0.07800193130970001,
      "ranking_idealized": 0.6916666626930237,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.949999988079071,
      "regularize": 0.07800193130970001,
      "step": 715
    },
    {
      "dpo_loss": 0.07249681651592255,
      "dpo_wo_beta": -0.03800208121538162,
      "epoch": 2.04298535663675,
      "grad_norm": 10.585775059236884,
      "learning_rate": 1.3908811141480408e-06,
      "logits": -1.7804607152938843,
      "logps": -119.00810241699219,
      "loss": 0.1011,
      "objective": 0.07249681651592255,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.949999988079071,
      "regularize": 0.07249681651592255,
      "step": 720
    },
    {
      "dpo_loss": 0.10758433490991592,
      "dpo_wo_beta": -0.4225389361381531,
      "epoch": 2.057156353330184,
      "grad_norm": 16.634608237356296,
      "learning_rate": 1.353988366393083e-06,
      "logits": -1.8837405443191528,
      "logps": -122.06383514404297,
      "loss": 0.1068,
      "objective": 0.10758433490991592,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.5583333373069763,
      "ranking_simple": 0.925000011920929,
      "regularize": 0.10758433490991592,
      "step": 725
    },
    {
      "dpo_loss": 0.16440601646900177,
      "dpo_wo_beta": -0.5969924926757812,
      "epoch": 2.0713273500236182,
      "grad_norm": 12.618281047842604,
      "learning_rate": 1.3174089267758983e-06,
      "logits": -1.7312003374099731,
      "logps": -118.56900024414062,
      "loss": 0.107,
      "objective": 0.16440601646900177,
      "ranking_idealized": 0.5166666507720947,
      "ranking_idealized_expo": 0.4333333373069763,
      "ranking_simple": 0.8583333492279053,
      "regularize": 0.16440601646900177,
      "step": 730
    },
    {
      "dpo_loss": 0.15623989701271057,
      "dpo_wo_beta": -0.9556598663330078,
      "epoch": 2.0854983467170523,
      "grad_norm": 16.05614648021729,
      "learning_rate": 1.2811527957500344e-06,
      "logits": -1.6055046319961548,
      "logps": -127.75144958496094,
      "loss": 0.1095,
      "objective": 0.15623989701271057,
      "ranking_idealized": 0.5583333373069763,
      "ranking_idealized_expo": 0.46666666865348816,
      "ranking_simple": 0.949999988079071,
      "regularize": 0.15623989701271057,
      "step": 735
    },
    {
      "dpo_loss": 0.11614324897527695,
      "dpo_wo_beta": -0.22688980400562286,
      "epoch": 2.0996693434104867,
      "grad_norm": 14.923960342956232,
      "learning_rate": 1.245229885379699e-06,
      "logits": -1.688416838645935,
      "logps": -123.08829498291016,
      "loss": 0.1063,
      "objective": 0.11614324897527695,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.949999988079071,
      "regularize": 0.11614324897527695,
      "step": 740
    },
    {
      "epoch": 2.10533774208786,
      "eval_dpo_loss": 0.9823706746101379,
      "eval_dpo_wo_beta": -7.331023216247559,
      "eval_logits": -2.2711641788482666,
      "eval_logps": -133.36373901367188,
      "eval_loss": 0.9485942721366882,
      "eval_objective": 0.9823706746101379,
      "eval_ranking_idealized": 0.6045548915863037,
      "eval_ranking_idealized_expo": 0.5279502868652344,
      "eval_ranking_simple": 0.5517598390579224,
      "eval_regularize": 0.9823706746101379,
      "eval_runtime": 364.9494,
      "eval_samples_per_second": 15.865,
      "eval_steps_per_second": 1.323,
      "step": 742
    },
    {
      "dpo_loss": 0.11820446699857712,
      "dpo_wo_beta": -0.5452965497970581,
      "epoch": 2.1138403401039207,
      "grad_norm": 14.405167306826879,
      "learning_rate": 1.2096500166298992e-06,
      "logits": -1.4971224069595337,
      "logps": -132.55892944335938,
      "loss": 0.1031,
      "objective": 0.11820446699857712,
      "ranking_idealized": 0.6916666626930237,
      "ranking_idealized_expo": 0.6166666746139526,
      "ranking_simple": 0.925000011920929,
      "regularize": 0.11820446699857712,
      "step": 745
    },
    {
      "dpo_loss": 0.08092837035655975,
      "dpo_wo_beta": -0.21483030915260315,
      "epoch": 2.1280113367973548,
      "grad_norm": 12.035987152428243,
      "learning_rate": 1.1744229166814889e-06,
      "logits": -1.698511004447937,
      "logps": -129.5937042236328,
      "loss": 0.0957,
      "objective": 0.08092837035655975,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.574999988079071,
      "ranking_simple": 0.9583333134651184,
      "regularize": 0.08092837035655975,
      "step": 750
    },
    {
      "dpo_loss": 0.09838299453258514,
      "dpo_wo_beta": -0.330010324716568,
      "epoch": 2.142182333490789,
      "grad_norm": 15.638085348810199,
      "learning_rate": 1.1395582162718524e-06,
      "logits": -1.6223360300064087,
      "logps": -128.86538696289062,
      "loss": 0.1147,
      "objective": 0.09838299453258514,
      "ranking_idealized": 0.5916666388511658,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.925000011920929,
      "regularize": 0.09838299453258514,
      "step": 755
    },
    {
      "dpo_loss": 0.10984232276678085,
      "dpo_wo_beta": -0.2521561086177826,
      "epoch": 2.156353330184223,
      "grad_norm": 18.280317761955644,
      "learning_rate": 1.1050654470619602e-06,
      "logits": -1.6547772884368896,
      "logps": -118.33650970458984,
      "loss": 0.1127,
      "objective": 0.10984232276678085,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.9166666865348816,
      "regularize": 0.10984232276678085,
      "step": 760
    },
    {
      "dpo_loss": 0.11440528929233551,
      "dpo_wo_beta": -0.2931906580924988,
      "epoch": 2.170524326877657,
      "grad_norm": 12.536707104746414,
      "learning_rate": 1.0709540390305061e-06,
      "logits": -1.692717432975769,
      "logps": -118.69541931152344,
      "loss": 0.1215,
      "objective": 0.11440528929233551,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.9333333373069763,
      "regularize": 0.11440528929233551,
      "step": 765
    },
    {
      "dpo_loss": 0.06226298585534096,
      "dpo_wo_beta": -0.023008961230516434,
      "epoch": 2.1846953235710913,
      "grad_norm": 12.278836680753214,
      "learning_rate": 1.0372333178958462e-06,
      "logits": -1.8234201669692993,
      "logps": -122.00631713867188,
      "loss": 0.1046,
      "objective": 0.06226298585534096,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.49166667461395264,
      "ranking_simple": 0.9583333134651184,
      "regularize": 0.06226298585534096,
      "step": 770
    },
    {
      "dpo_loss": 0.13250760734081268,
      "dpo_wo_beta": -0.465701699256897,
      "epoch": 2.1988663202645253,
      "grad_norm": 18.99094104963921,
      "learning_rate": 1.0039125025664392e-06,
      "logits": -1.7803070545196533,
      "logps": -124.71762084960938,
      "loss": 0.1111,
      "objective": 0.13250760734081268,
      "ranking_idealized": 0.5416666865348816,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.875,
      "regularize": 0.13250760734081268,
      "step": 775
    },
    {
      "dpo_loss": 0.13606421649456024,
      "dpo_wo_beta": -0.6162300705909729,
      "epoch": 2.2130373169579594,
      "grad_norm": 16.36095800479351,
      "learning_rate": 9.710007026204896e-07,
      "logits": -1.7376734018325806,
      "logps": -122.50430297851562,
      "loss": 0.1045,
      "objective": 0.13606421649456024,
      "ranking_idealized": 0.5,
      "ranking_idealized_expo": 0.4333333373069763,
      "ranking_simple": 0.8999999761581421,
      "regularize": 0.13606421649456024,
      "step": 780
    },
    {
      "dpo_loss": 0.11954029649496078,
      "dpo_wo_beta": -0.5017859935760498,
      "epoch": 2.2272083136513934,
      "grad_norm": 14.897356026126795,
      "learning_rate": 9.385069158154805e-07,
      "logits": -1.7108873128890991,
      "logps": -119.73731994628906,
      "loss": 0.1223,
      "objective": 0.11954029649496078,
      "ranking_idealized": 0.5583333373069763,
      "ranking_idealized_expo": 0.4583333432674408,
      "ranking_simple": 0.8916666507720947,
      "regularize": 0.11954029649496078,
      "step": 785
    },
    {
      "dpo_loss": 0.0877259224653244,
      "dpo_wo_beta": -0.3332770764827728,
      "epoch": 2.2413793103448274,
      "grad_norm": 14.08354106572375,
      "learning_rate": 9.064400256282757e-07,
      "logits": -1.7486475706100464,
      "logps": -122.89460754394531,
      "loss": 0.1063,
      "objective": 0.0877259224653244,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.949999988079071,
      "regularize": 0.0877259224653244,
      "step": 790
    },
    {
      "dpo_loss": 0.09928978979587555,
      "dpo_wo_beta": -0.10460276901721954,
      "epoch": 2.255550307038262,
      "grad_norm": 15.557463813809257,
      "learning_rate": 8.74808798826467e-07,
      "logits": -1.8421998023986816,
      "logps": -120.40747833251953,
      "loss": 0.1017,
      "objective": 0.09928978979587555,
      "ranking_idealized": 0.699999988079071,
      "ranking_idealized_expo": 0.6000000238418579,
      "ranking_simple": 0.9416666626930237,
      "regularize": 0.09928978979587555,
      "step": 795
    },
    {
      "epoch": 2.255550307038262,
      "eval_dpo_loss": 0.8903655409812927,
      "eval_dpo_wo_beta": -6.205545902252197,
      "eval_logits": -2.4489927291870117,
      "eval_logps": -123.57449340820312,
      "eval_loss": 0.8711386919021606,
      "eval_objective": 0.8903655409812927,
      "eval_ranking_idealized": 0.6045548915863037,
      "eval_ranking_idealized_expo": 0.5279502868652344,
      "eval_ranking_simple": 0.5683229565620422,
      "eval_regularize": 0.8903655409812927,
      "eval_runtime": 347.0103,
      "eval_samples_per_second": 16.685,
      "eval_steps_per_second": 1.392,
      "step": 795
    },
    {
      "dpo_loss": 0.1340600550174713,
      "dpo_wo_beta": -0.5811701416969299,
      "epoch": 2.269721303731696,
      "grad_norm": 13.668309173603514,
      "learning_rate": 8.436218830716259e-07,
      "logits": -1.8454309701919556,
      "logps": -120.35511779785156,
      "loss": 0.1264,
      "objective": 0.1340600550174713,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.925000011920929,
      "regularize": 0.1340600550174713,
      "step": 800
    },
    {
      "dpo_loss": 0.10816308110952377,
      "dpo_wo_beta": -0.1633211225271225,
      "epoch": 2.28389230042513,
      "grad_norm": 13.957508503091352,
      "learning_rate": 8.1288780455512e-07,
      "logits": -1.8811193704605103,
      "logps": -119.9056625366211,
      "loss": 0.1133,
      "objective": 0.10816308110952377,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.9166666865348816,
      "regularize": 0.10816308110952377,
      "step": 805
    },
    {
      "dpo_loss": 0.1715787798166275,
      "dpo_wo_beta": -0.7547404766082764,
      "epoch": 2.298063297118564,
      "grad_norm": 12.765274097098457,
      "learning_rate": 7.826149656671386e-07,
      "logits": -1.9726245403289795,
      "logps": -114.58699798583984,
      "loss": 0.1146,
      "objective": 0.1715787798166275,
      "ranking_idealized": 0.6416666507720947,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.9166666865348816,
      "regularize": 0.1715787798166275,
      "step": 810
    },
    {
      "dpo_loss": 0.09882104396820068,
      "dpo_wo_beta": -0.18077202141284943,
      "epoch": 2.312234293811998,
      "grad_norm": 11.818365173405272,
      "learning_rate": 7.528116426995605e-07,
      "logits": -1.7834192514419556,
      "logps": -118.5341796875,
      "loss": 0.0843,
      "objective": 0.09882104396820068,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.4333333373069763,
      "ranking_simple": 0.8916666507720947,
      "regularize": 0.09882104396820068,
      "step": 815
    },
    {
      "dpo_loss": 0.1070082038640976,
      "dpo_wo_beta": -0.44529280066490173,
      "epoch": 2.3264052905054324,
      "grad_norm": 13.918285230217345,
      "learning_rate": 7.234859835833022e-07,
      "logits": -1.8069401979446411,
      "logps": -123.11463928222656,
      "loss": 0.1153,
      "objective": 0.1070082038640976,
      "ranking_idealized": 0.6083333492279053,
      "ranking_idealized_expo": 0.5583333373069763,
      "ranking_simple": 0.949999988079071,
      "regularize": 0.1070082038640976,
      "step": 820
    },
    {
      "dpo_loss": 0.14794430136680603,
      "dpo_wo_beta": -0.5907248258590698,
      "epoch": 2.3405762871988665,
      "grad_norm": 14.849580151366643,
      "learning_rate": 6.94646005660749e-07,
      "logits": -1.808493971824646,
      "logps": -116.64714050292969,
      "loss": 0.1107,
      "objective": 0.14794430136680603,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5583333373069763,
      "ranking_simple": 0.9083333611488342,
      "regularize": 0.14794430136680603,
      "step": 825
    },
    {
      "dpo_loss": 0.08306514471769333,
      "dpo_wo_beta": -0.10000230371952057,
      "epoch": 2.3547472838923005,
      "grad_norm": 12.857101782602014,
      "learning_rate": 6.662995934939007e-07,
      "logits": -1.7857582569122314,
      "logps": -123.92733764648438,
      "loss": 0.1063,
      "objective": 0.08306514471769333,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.9583333134651184,
      "regularize": 0.08306514471769333,
      "step": 830
    },
    {
      "dpo_loss": 0.11827471107244492,
      "dpo_wo_beta": -0.40130358934402466,
      "epoch": 2.3689182805857345,
      "grad_norm": 13.825829745811577,
      "learning_rate": 6.384544967088063e-07,
      "logits": -1.8356945514678955,
      "logps": -122.48320770263672,
      "loss": 0.124,
      "objective": 0.11827471107244492,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.4583333432674408,
      "ranking_simple": 0.9333333373069763,
      "regularize": 0.11827471107244492,
      "step": 835
    },
    {
      "dpo_loss": 0.14169135689735413,
      "dpo_wo_beta": -0.3359481692314148,
      "epoch": 2.3830892772791685,
      "grad_norm": 18.243146391325514,
      "learning_rate": 6.111183278768956e-07,
      "logits": -1.8658840656280518,
      "logps": -123.42705535888672,
      "loss": 0.1321,
      "objective": 0.14169135689735413,
      "ranking_idealized": 0.5,
      "ranking_idealized_expo": 0.44999998807907104,
      "ranking_simple": 0.9083333611488342,
      "regularize": 0.14169135689735413,
      "step": 840
    },
    {
      "dpo_loss": 0.09381429105997086,
      "dpo_wo_beta": -0.14171645045280457,
      "epoch": 2.3972602739726026,
      "grad_norm": 24.500088347031785,
      "learning_rate": 5.842985604337769e-07,
      "logits": -1.7731019258499146,
      "logps": -125.81861877441406,
      "loss": 0.1225,
      "objective": 0.09381429105997086,
      "ranking_idealized": 0.6083333492279053,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.8999999761581421,
      "regularize": 0.09381429105997086,
      "step": 845
    },
    {
      "epoch": 2.4057628719886632,
      "eval_dpo_loss": 0.9035148620605469,
      "eval_dpo_wo_beta": -6.352902889251709,
      "eval_logits": -2.4742591381073,
      "eval_logps": -124.53355407714844,
      "eval_loss": 0.882164478302002,
      "eval_objective": 0.9035148620605469,
      "eval_ranking_idealized": 0.6045548915863037,
      "eval_ranking_idealized_expo": 0.5279502868652344,
      "eval_ranking_simple": 0.5569358468055725,
      "eval_regularize": 0.9035148620605469,
      "eval_runtime": 366.478,
      "eval_samples_per_second": 15.799,
      "eval_steps_per_second": 1.318,
      "step": 848
    },
    {
      "dpo_loss": 0.10245585441589355,
      "dpo_wo_beta": -0.2030431628227234,
      "epoch": 2.413793103448276,
      "grad_norm": 22.363955547911008,
      "learning_rate": 5.580025266360764e-07,
      "logits": -1.7464776039123535,
      "logps": -122.80999755859375,
      "loss": 0.1449,
      "objective": 0.10245585441589355,
      "ranking_idealized": 0.6666666865348816,
      "ranking_idealized_expo": 0.5625,
      "ranking_simple": 0.9583333134651184,
      "regularize": 0.10245585441589355,
      "step": 850
    },
    {
      "dpo_loss": 0.14077231287956238,
      "dpo_wo_beta": -0.15207929909229279,
      "epoch": 2.42796410014171,
      "grad_norm": 14.848409925173506,
      "learning_rate": 5.322374155568688e-07,
      "logits": -1.8929237127304077,
      "logps": -115.12696838378906,
      "loss": 0.1124,
      "objective": 0.14077231287956238,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.9166666865348816,
      "regularize": 0.14077231287956238,
      "step": 855
    },
    {
      "dpo_loss": 0.1414971649646759,
      "dpo_wo_beta": -0.5974557995796204,
      "epoch": 2.442135096835144,
      "grad_norm": 11.710235822935351,
      "learning_rate": 5.070102711202606e-07,
      "logits": -1.7974507808685303,
      "logps": -121.47347259521484,
      "loss": 0.1083,
      "objective": 0.1414971649646759,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5583333373069763,
      "ranking_simple": 0.9333333373069763,
      "regularize": 0.1414971649646759,
      "step": 860
    },
    {
      "dpo_loss": 0.09845638275146484,
      "dpo_wo_beta": -0.1998511403799057,
      "epoch": 2.4563060935285783,
      "grad_norm": 15.827446562598988,
      "learning_rate": 4.823279901756498e-07,
      "logits": -1.816353440284729,
      "logps": -122.7919692993164,
      "loss": 0.1063,
      "objective": 0.09845638275146484,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.9083333611488342,
      "regularize": 0.09845638275146484,
      "step": 865
    },
    {
      "dpo_loss": 0.11614971607923508,
      "dpo_wo_beta": -0.34934201836586,
      "epoch": 2.4704770902220123,
      "grad_norm": 15.618833316654502,
      "learning_rate": 4.581973206121948e-07,
      "logits": -1.9400283098220825,
      "logps": -119.73554992675781,
      "loss": 0.1153,
      "objective": 0.11614971607923508,
      "ranking_idealized": 0.5916666388511658,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.925000011920929,
      "regularize": 0.11614971607923508,
      "step": 870
    },
    {
      "dpo_loss": 0.2065262645483017,
      "dpo_wo_beta": -1.1751881837844849,
      "epoch": 2.4846480869154464,
      "grad_norm": 17.223610041291963,
      "learning_rate": 4.3462485951401126e-07,
      "logits": -1.7437902688980103,
      "logps": -120.61251831054688,
      "loss": 0.1238,
      "objective": 0.2065262645483017,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.574999988079071,
      "ranking_simple": 0.8666666746139526,
      "regularize": 0.2065262645483017,
      "step": 875
    },
    {
      "dpo_loss": 0.13995474576950073,
      "dpo_wo_beta": -0.27328214049339294,
      "epoch": 2.4988190836088804,
      "grad_norm": 11.937037662011573,
      "learning_rate": 4.116170513565942e-07,
      "logits": -1.9172199964523315,
      "logps": -115.04023742675781,
      "loss": 0.1093,
      "objective": 0.13995474576950073,
      "ranking_idealized": 0.5249999761581421,
      "ranking_idealized_expo": 0.49166667461395264,
      "ranking_simple": 0.8999999761581421,
      "regularize": 0.13995474576950073,
      "step": 880
    },
    {
      "dpo_loss": 0.13740381598472595,
      "dpo_wo_beta": -0.38727322220802307,
      "epoch": 2.5129900803023144,
      "grad_norm": 16.35266837824131,
      "learning_rate": 3.891801862449629e-07,
      "logits": -1.8533929586410522,
      "logps": -115.91497039794922,
      "loss": 0.1385,
      "objective": 0.13740381598472595,
      "ranking_idealized": 0.6083333492279053,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.8916666507720947,
      "regularize": 0.13740381598472595,
      "step": 885
    },
    {
      "dpo_loss": 0.11444827914237976,
      "dpo_wo_beta": -0.31205496191978455,
      "epoch": 2.527161076995749,
      "grad_norm": 12.141683375579714,
      "learning_rate": 3.6732039819400686e-07,
      "logits": -1.6747931241989136,
      "logps": -116.25071716308594,
      "loss": 0.1173,
      "objective": 0.11444827914237976,
      "ranking_idealized": 0.5166666507720947,
      "ranking_idealized_expo": 0.4416666626930237,
      "ranking_simple": 0.9083333611488342,
      "regularize": 0.11444827914237976,
      "step": 890
    },
    {
      "dpo_loss": 0.21558310091495514,
      "dpo_wo_beta": -0.7720097303390503,
      "epoch": 2.541332073689183,
      "grad_norm": 17.5258932616616,
      "learning_rate": 3.46043663451511e-07,
      "logits": -1.938331127166748,
      "logps": -121.76246643066406,
      "loss": 0.1324,
      "objective": 0.21558310091495514,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.8333333134651184,
      "regularize": 0.21558310091495514,
      "step": 895
    },
    {
      "dpo_loss": 0.1236240565776825,
      "dpo_wo_beta": -0.21698738634586334,
      "epoch": 2.555503070382617,
      "grad_norm": 16.69855766001795,
      "learning_rate": 3.253557988643072e-07,
      "logits": -1.8755207061767578,
      "logps": -119.14775848388672,
      "loss": 0.1157,
      "objective": 0.1236240565776825,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.9083333611488342,
      "regularize": 0.1236240565776825,
      "step": 900
    },
    {
      "epoch": 2.558337269721304,
      "eval_dpo_loss": 0.894111156463623,
      "eval_dpo_wo_beta": -6.213596343994141,
      "eval_logits": -2.4885809421539307,
      "eval_logps": -124.45829010009766,
      "eval_loss": 0.8717960715293884,
      "eval_objective": 0.894111156463623,
      "eval_ranking_idealized": 0.6045548915863037,
      "eval_ranking_idealized_expo": 0.5279502868652344,
      "eval_ranking_simple": 0.5621117949485779,
      "eval_regularize": 0.894111156463623,
      "eval_runtime": 342.6574,
      "eval_samples_per_second": 16.897,
      "eval_steps_per_second": 1.41,
      "step": 901
    },
    {
      "dpo_loss": 0.09204068034887314,
      "dpo_wo_beta": -0.17730669677257538,
      "epoch": 2.569674067076051,
      "grad_norm": 14.161016786369684,
      "learning_rate": 3.052624602880064e-07,
      "logits": -1.8424724340438843,
      "logps": -117.90782928466797,
      "loss": 0.0986,
      "objective": 0.09204068034887314,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.925000011920929,
      "regularize": 0.09204068034887314,
      "step": 905
    },
    {
      "dpo_loss": 0.11361932754516602,
      "dpo_wo_beta": -0.38244467973709106,
      "epoch": 2.583845063769485,
      "grad_norm": 12.421803606538058,
      "learning_rate": 2.8576914104074425e-07,
      "logits": -2.0089211463928223,
      "logps": -116.39904022216797,
      "loss": 0.1122,
      "objective": 0.11361932754516602,
      "ranking_idealized": 0.5916666388511658,
      "ranking_idealized_expo": 0.4583333432674408,
      "ranking_simple": 0.9166666865348816,
      "regularize": 0.11361932754516602,
      "step": 910
    },
    {
      "dpo_loss": 0.13471105694770813,
      "dpo_wo_beta": -0.5014829039573669,
      "epoch": 2.5980160604629194,
      "grad_norm": 15.12060209950672,
      "learning_rate": 2.6688117040136463e-07,
      "logits": -1.981037974357605,
      "logps": -121.86930084228516,
      "loss": 0.1153,
      "objective": 0.13471105694770813,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.949999988079071,
      "regularize": 0.13471105694770813,
      "step": 915
    },
    {
      "dpo_loss": 0.08700807392597198,
      "dpo_wo_beta": -0.1876840889453888,
      "epoch": 2.6121870571563535,
      "grad_norm": 17.811975214160903,
      "learning_rate": 2.486037121524448e-07,
      "logits": -1.898934245109558,
      "logps": -120.13009643554688,
      "loss": 0.114,
      "objective": 0.08700807392597198,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.9416666626930237,
      "regularize": 0.08700807392597198,
      "step": 920
    },
    {
      "dpo_loss": 0.0918925479054451,
      "dpo_wo_beta": -0.14420188963413239,
      "epoch": 2.6263580538497875,
      "grad_norm": 16.6284598820075,
      "learning_rate": 2.3094176316856982e-07,
      "logits": -1.8268101215362549,
      "logps": -121.2860107421875,
      "loss": 0.1064,
      "objective": 0.0918925479054451,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.4749999940395355,
      "ranking_simple": 0.9083333611488342,
      "regularize": 0.0918925479054451,
      "step": 925
    },
    {
      "dpo_loss": 0.15886452794075012,
      "dpo_wo_beta": -0.7213179469108582,
      "epoch": 2.6405290505432215,
      "grad_norm": 15.32952011074324,
      "learning_rate": 2.13900152050239e-07,
      "logits": -1.9606980085372925,
      "logps": -110.05363464355469,
      "loss": 0.1179,
      "objective": 0.15886452794075012,
      "ranking_idealized": 0.5416666865348816,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.8999999761581421,
      "regularize": 0.15886452794075012,
      "step": 930
    },
    {
      "dpo_loss": 0.13784056901931763,
      "dpo_wo_beta": -0.5006576776504517,
      "epoch": 2.6547000472366555,
      "grad_norm": 14.658353796887713,
      "learning_rate": 1.9748353780377234e-07,
      "logits": -1.9395031929016113,
      "logps": -119.9269790649414,
      "loss": 0.1254,
      "objective": 0.13784056901931763,
      "ranking_idealized": 0.6583333611488342,
      "ranking_idealized_expo": 0.6000000238418579,
      "ranking_simple": 0.8916666507720947,
      "regularize": 0.13784056901931763,
      "step": 935
    },
    {
      "dpo_loss": 0.12213913351297379,
      "dpo_wo_beta": -0.43640393018722534,
      "epoch": 2.66887104393009,
      "grad_norm": 14.844912954939305,
      "learning_rate": 1.8169640856758652e-07,
      "logits": -1.952646017074585,
      "logps": -121.75474548339844,
      "loss": 0.1117,
      "objective": 0.12213913351297379,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.9166666865348816,
      "regularize": 0.12213913351297379,
      "step": 940
    },
    {
      "dpo_loss": 0.1136295348405838,
      "dpo_wo_beta": -0.17752434313297272,
      "epoch": 2.6830420406235236,
      "grad_norm": 13.220834082246482,
      "learning_rate": 1.6654308038518057e-07,
      "logits": -1.7970060110092163,
      "logps": -117.90103149414062,
      "loss": 0.1054,
      "objective": 0.1136295348405838,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.49166667461395264,
      "ranking_simple": 0.9083333611488342,
      "regularize": 0.1136295348405838,
      "step": 945
    },
    {
      "dpo_loss": 0.16729401051998138,
      "dpo_wo_beta": -0.7473469972610474,
      "epoch": 2.697213037316958,
      "grad_norm": 14.369145851061829,
      "learning_rate": 1.5202769602517514e-07,
      "logits": -1.8816020488739014,
      "logps": -115.14582061767578,
      "loss": 0.1387,
      "objective": 0.16729401051998138,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.8999999761581421,
      "regularize": 0.16729401051998138,
      "step": 950
    },
    {
      "epoch": 2.708549834671705,
      "eval_dpo_loss": 0.8891981840133667,
      "eval_dpo_wo_beta": -6.16098690032959,
      "eval_logits": -2.508572816848755,
      "eval_logps": -123.41434478759766,
      "eval_loss": 0.8687644004821777,
      "eval_objective": 0.8891981840133667,
      "eval_ranking_idealized": 0.6045548915863037,
      "eval_ranking_idealized_expo": 0.5279502868652344,
      "eval_ranking_simple": 0.5579710006713867,
      "eval_regularize": 0.8891981840133667,
      "eval_runtime": 379.9179,
      "eval_samples_per_second": 15.24,
      "eval_steps_per_second": 1.271,
      "step": 954
    },
    {
      "dpo_loss": 0.16232462227344513,
      "dpo_wo_beta": -0.5988053679466248,
      "epoch": 2.711384034010392,
      "grad_norm": 18.96223814657741,
      "learning_rate": 1.381542238487188e-07,
      "logits": -1.8838648796081543,
      "logps": -119.06771087646484,
      "loss": 0.1298,
      "objective": 0.16232462227344513,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.8999999761581421,
      "regularize": 0.16232462227344513,
      "step": 955
    },
    {
      "dpo_loss": 0.14441066980361938,
      "dpo_wo_beta": -0.4333815276622772,
      "epoch": 2.725555030703826,
      "grad_norm": 13.716940458471806,
      "learning_rate": 1.2492645672457838e-07,
      "logits": -2.032045364379883,
      "logps": -113.05756378173828,
      "loss": 0.1189,
      "objective": 0.14441066980361938,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.8999999761581421,
      "regularize": 0.14441066980361938,
      "step": 960
    },
    {
      "dpo_loss": 0.0981813296675682,
      "dpo_wo_beta": -0.19802381098270416,
      "epoch": 2.73972602739726,
      "grad_norm": 15.207158850260772,
      "learning_rate": 1.1234801099220787e-07,
      "logits": -1.7988998889923096,
      "logps": -122.7618408203125,
      "loss": 0.1111,
      "objective": 0.0981813296675682,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.9333333373069763,
      "regularize": 0.0981813296675682,
      "step": 965
    },
    {
      "dpo_loss": 0.1318301260471344,
      "dpo_wo_beta": -0.2933843731880188,
      "epoch": 2.753897024090694,
      "grad_norm": 19.758130540339153,
      "learning_rate": 1.004223254730749e-07,
      "logits": -1.7169368267059326,
      "logps": -120.43925476074219,
      "loss": 0.1278,
      "objective": 0.1318301260471344,
      "ranking_idealized": 0.6083333492279053,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.8916666507720947,
      "regularize": 0.1318301260471344,
      "step": 970
    },
    {
      "dpo_loss": 0.058168552815914154,
      "dpo_wo_beta": -8.394511678488925e-05,
      "epoch": 2.7680680207841286,
      "grad_norm": 12.662905511630496,
      "learning_rate": 8.915266053052374e-08,
      "logits": -1.902711033821106,
      "logps": -116.11229705810547,
      "loss": 0.0999,
      "objective": 0.058168552815914154,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.925000011920929,
      "regularize": 0.058168552815914154,
      "step": 975
    },
    {
      "dpo_loss": 0.10343047231435776,
      "dpo_wo_beta": -0.17137210071086884,
      "epoch": 2.7822390174775626,
      "grad_norm": 18.09899373584344,
      "learning_rate": 7.854209717842231e-08,
      "logits": -1.8915067911148071,
      "logps": -118.23885345458984,
      "loss": 0.1108,
      "objective": 0.10343047231435776,
      "ranking_idealized": 0.5416666865348816,
      "ranking_idealized_expo": 0.4416666626930237,
      "ranking_simple": 0.949999988079071,
      "regularize": 0.10343047231435776,
      "step": 980
    },
    {
      "dpo_loss": 0.11380515992641449,
      "dpo_wo_beta": -0.2935677468776703,
      "epoch": 2.7964100141709967,
      "grad_norm": 15.185634936609162,
      "learning_rate": 6.859353623884569e-08,
      "logits": -1.851272463798523,
      "logps": -114.0352783203125,
      "loss": 0.1005,
      "objective": 0.11380515992641449,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.8833333253860474,
      "regularize": 0.11380515992641449,
      "step": 985
    },
    {
      "dpo_loss": 0.10726428776979446,
      "dpo_wo_beta": -0.24069656431674957,
      "epoch": 2.8105810108644307,
      "grad_norm": 16.660819641239655,
      "learning_rate": 5.930969754901844e-08,
      "logits": -1.8860033750534058,
      "logps": -116.69475555419922,
      "loss": 0.1285,
      "objective": 0.10726428776979446,
      "ranking_idealized": 0.49166667461395264,
      "ranking_idealized_expo": 0.4333333373069763,
      "ranking_simple": 0.8833333253860474,
      "regularize": 0.10726428776979446,
      "step": 990
    },
    {
      "dpo_loss": 0.12048947066068649,
      "dpo_wo_beta": -0.4086553454399109,
      "epoch": 2.8247520075578647,
      "grad_norm": 18.459518556479477,
      "learning_rate": 5.069311921774039e-08,
      "logits": -1.8980218172073364,
      "logps": -120.89018249511719,
      "loss": 0.1282,
      "objective": 0.12048947066068649,
      "ranking_idealized": 0.6416666507720947,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.925000011920929,
      "regularize": 0.12048947066068649,
      "step": 995
    },
    {
      "dpo_loss": 0.08448319137096405,
      "dpo_wo_beta": -0.17254652082920074,
      "epoch": 2.838923004251299,
      "grad_norm": 13.073827319036173,
      "learning_rate": 4.2746156931490756e-08,
      "logits": -1.8106515407562256,
      "logps": -113.39408874511719,
      "loss": 0.1237,
      "objective": 0.08448319137096405,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.49166667461395264,
      "ranking_simple": 0.9333333373069763,
      "regularize": 0.08448319137096405,
      "step": 1000
    },
    {
      "dpo_loss": 0.12625885009765625,
      "dpo_wo_beta": -0.28751522302627563,
      "epoch": 2.853094000944733,
      "grad_norm": 11.352338165734487,
      "learning_rate": 3.547098331040916e-08,
      "logits": -1.8715885877609253,
      "logps": -115.85346984863281,
      "loss": 0.1219,
      "objective": 0.12625885009765625,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.4749999940395355,
      "ranking_simple": 0.8833333253860474,
      "regularize": 0.12625885009765625,
      "step": 1005
    },
    {
      "epoch": 2.8587623996221065,
      "eval_dpo_loss": 0.8881875872612,
      "eval_dpo_wo_beta": -6.153732776641846,
      "eval_logits": -2.5127134323120117,
      "eval_logps": -123.14542388916016,
      "eval_loss": 0.868183434009552,
      "eval_objective": 0.8881875872612,
      "eval_ranking_idealized": 0.6045548915863037,
      "eval_ranking_idealized_expo": 0.5279502868652344,
      "eval_ranking_simple": 0.5600414276123047,
      "eval_regularize": 0.8881875872612,
      "eval_runtime": 375.0227,
      "eval_samples_per_second": 15.439,
      "eval_steps_per_second": 1.288,
      "step": 1007
    },
    {
      "dpo_loss": 0.12578138709068298,
      "dpo_wo_beta": -0.4055772125720978,
      "epoch": 2.8672649976381672,
      "grad_norm": 13.654143333766829,
      "learning_rate": 2.8869587314321324e-08,
      "logits": -1.871021032333374,
      "logps": -118.54988861083984,
      "loss": 0.1301,
      "objective": 0.12578138709068298,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.9166666865348816,
      "regularize": 0.12578138709068298,
      "step": 1010
    },
    {
      "dpo_loss": 0.11141829192638397,
      "dpo_wo_beta": -0.452913373708725,
      "epoch": 2.8814359943316012,
      "grad_norm": 14.00967768494451,
      "learning_rate": 2.2943773698977935e-08,
      "logits": -1.8538991212844849,
      "logps": -119.40221405029297,
      "loss": 0.1157,
      "objective": 0.11141829192638397,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.9416666626930237,
      "regularize": 0.11141829192638397,
      "step": 1015
    },
    {
      "dpo_loss": 0.10787668824195862,
      "dpo_wo_beta": -0.35029396414756775,
      "epoch": 2.8956069910250353,
      "grad_norm": 12.590503217808422,
      "learning_rate": 1.7695162522652352e-08,
      "logits": -1.9000986814498901,
      "logps": -122.90519714355469,
      "loss": 0.1076,
      "objective": 0.10787668824195862,
      "ranking_idealized": 0.6083333492279053,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.8999999761581421,
      "regularize": 0.10787668824195862,
      "step": 1020
    },
    {
      "dpo_loss": 0.11394120752811432,
      "dpo_wo_beta": -0.36261746287345886,
      "epoch": 2.9097779877184697,
      "grad_norm": 10.606906345600125,
      "learning_rate": 1.3125188703233815e-08,
      "logits": -1.8986002206802368,
      "logps": -116.403564453125,
      "loss": 0.1089,
      "objective": 0.11394120752811432,
      "ranking_idealized": 0.5583333373069763,
      "ranking_idealized_expo": 0.44999998807907104,
      "ranking_simple": 0.9333333373069763,
      "regularize": 0.11394120752811432,
      "step": 1025
    },
    {
      "dpo_loss": 0.0906638652086258,
      "dpo_wo_beta": -0.1376449316740036,
      "epoch": 2.9239489844119038,
      "grad_norm": 12.750602018189479,
      "learning_rate": 9.235101625932885e-09,
      "logits": -2.033400058746338,
      "logps": -113.65220642089844,
      "loss": 0.1197,
      "objective": 0.0906638652086258,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.925000011920929,
      "regularize": 0.0906638652086258,
      "step": 1030
    },
    {
      "dpo_loss": 0.10265343636274338,
      "dpo_wo_beta": -0.1061137467622757,
      "epoch": 2.938119981105338,
      "grad_norm": 15.296605965797069,
      "learning_rate": 6.025964801714412e-09,
      "logits": -1.8468897342681885,
      "logps": -119.85134887695312,
      "loss": 0.1049,
      "objective": 0.10265343636274338,
      "ranking_idealized": 0.6083333492279053,
      "ranking_idealized_expo": 0.49166667461395264,
      "ranking_simple": 0.8916666507720947,
      "regularize": 0.10265343636274338,
      "step": 1035
    },
    {
      "dpo_loss": 0.09396873414516449,
      "dpo_wo_beta": -0.1912200003862381,
      "epoch": 2.952290977798772,
      "grad_norm": 15.90972962002085,
      "learning_rate": 3.4986555765434415e-09,
      "logits": -1.8800926208496094,
      "logps": -122.51961517333984,
      "loss": 0.0994,
      "objective": 0.09396873414516449,
      "ranking_idealized": 0.6499999761581421,
      "ranking_idealized_expo": 0.5583333373069763,
      "ranking_simple": 0.9083333611488342,
      "regularize": 0.09396873414516449,
      "step": 1040
    },
    {
      "dpo_loss": 0.14692950248718262,
      "dpo_wo_beta": -0.6586350798606873,
      "epoch": 2.966461974492206,
      "grad_norm": 8.0573279067109,
      "learning_rate": 1.6538648915270794e-09,
      "logits": -1.8756026029586792,
      "logps": -119.65303039550781,
      "loss": 0.1082,
      "objective": 0.14692950248718262,
      "ranking_idealized": 0.6416666507720947,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.9166666865348816,
      "regularize": 0.14692950248718262,
      "step": 1045
    },
    {
      "dpo_loss": 0.07982174307107925,
      "dpo_wo_beta": -0.0584401935338974,
      "epoch": 2.9806329711856403,
      "grad_norm": 9.88886611903832,
      "learning_rate": 4.920970940180958e-10,
      "logits": -1.912126898765564,
      "logps": -116.61032104492188,
      "loss": 0.0891,
      "objective": 0.07982174307107925,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.925000011920929,
      "regularize": 0.07982174307107925,
      "step": 1050
    },
    {
      "dpo_loss": 0.07663024961948395,
      "dpo_wo_beta": -0.015355088748037815,
      "epoch": 2.9948039678790743,
      "grad_norm": 13.025122561532541,
      "learning_rate": 1.3669799732163314e-11,
      "logits": -1.775391697883606,
      "logps": -116.977294921875,
      "loss": 0.0869,
      "objective": 0.07663024961948395,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.8999999761581421,
      "regularize": 0.07663024961948395,
      "step": 1055
    },
    {
      "epoch": 2.9976381672177608,
      "step": 1056,
      "total_flos": 0.0,
      "train_loss": 0.022545777056648425,
      "train_runtime": 4386.5835,
      "train_samples_per_second": 34.744,
      "train_steps_per_second": 0.241
    }
  ],
  "logging_steps": 5,
  "max_steps": 1056,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 53,
  "stateful_callbacks": {
    "EarlyStoppingCallback": {
      "args": {
        "early_stopping_patience": 5,
        "early_stopping_threshold": 0.0
      },
      "attributes": {
        "early_stopping_patience_counter": 0
      }
    },
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}