{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.992914501653283,
  "eval_steps": 50,
  "global_step": 1056,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "dpo_loss": 0.6931471824645996,
      "epoch": 0.002834199338686821,
      "grad_norm": 18.39703870091124,
      "learning_rate": 9.433962264150943e-09,
      "logits": -1.2867579460144043,
      "logps": -84.34933471679688,
      "loss": 0.6931,
      "objective": 0.6931471824645996,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.6931471824645996,
      "step": 1,
      "wo_beta": 5.2708282470703125
    },
    {
      "dpo_loss": 0.6930365562438965,
      "epoch": 0.014170996693434105,
      "grad_norm": 18.759209504732585,
      "learning_rate": 4.7169811320754715e-08,
      "logits": -1.429247498512268,
      "logps": -83.84073638916016,
      "loss": 0.6931,
      "objective": 0.6930365562438965,
      "ranking_simple": 0.4895833432674408,
      "regularize": 0.6930365562438965,
      "step": 5,
      "wo_beta": 7.670312404632568
    },
    {
      "dpo_loss": 0.6933786869049072,
      "epoch": 0.02834199338686821,
      "grad_norm": 20.439115048712264,
      "learning_rate": 9.433962264150943e-08,
      "logits": -1.40127694606781,
      "logps": -84.69605255126953,
      "loss": 0.6932,
      "objective": 0.6933786869049072,
      "ranking_simple": 0.5791666507720947,
      "regularize": 0.6933786869049072,
      "step": 10,
      "wo_beta": 5.937962055206299
    },
    {
      "dpo_loss": 0.6909440755844116,
      "epoch": 0.042512990080302314,
      "grad_norm": 18.777743501771415,
      "learning_rate": 1.4150943396226414e-07,
      "logits": -1.5383602380752563,
      "logps": -84.21109771728516,
      "loss": 0.6921,
      "objective": 0.6909440755844116,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.6909440755844116,
      "step": 15,
      "wo_beta": 6.64866828918457
    },
    {
      "dpo_loss": 0.6903365254402161,
      "epoch": 0.05668398677373642,
      "grad_norm": 18.03474083688221,
      "learning_rate": 1.8867924528301886e-07,
      "logits": -1.3631871938705444,
      "logps": -83.19156646728516,
      "loss": 0.6904,
      "objective": 0.6903365254402161,
      "ranking_simple": 0.48750001192092896,
      "regularize": 0.6903365254402161,
      "step": 20,
      "wo_beta": 8.294000625610352
    },
    {
      "dpo_loss": 0.6920856237411499,
      "epoch": 0.07085498346717052,
      "grad_norm": 17.418659582335064,
      "learning_rate": 2.3584905660377358e-07,
      "logits": -1.3736237287521362,
      "logps": -83.09373474121094,
      "loss": 0.6904,
      "objective": 0.6920856237411499,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.6920856237411499,
      "step": 25,
      "wo_beta": 6.352960109710693
    },
    {
      "dpo_loss": 0.686029851436615,
      "epoch": 0.08502598016060463,
      "grad_norm": 16.102325248569215,
      "learning_rate": 2.830188679245283e-07,
      "logits": -1.4392812252044678,
      "logps": -82.93743133544922,
      "loss": 0.6866,
      "objective": 0.686029851436615,
      "ranking_simple": 0.48750001192092896,
      "regularize": 0.686029851436615,
      "step": 30,
      "wo_beta": 8.934405326843262
    },
    {
      "dpo_loss": 0.692076563835144,
      "epoch": 0.09919697685403873,
      "grad_norm": 17.863783183076404,
      "learning_rate": 3.30188679245283e-07,
      "logits": -1.4204154014587402,
      "logps": -82.21839904785156,
      "loss": 0.6871,
      "objective": 0.692076563835144,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.692076563835144,
      "step": 35,
      "wo_beta": 6.40147590637207
    },
    {
      "dpo_loss": 0.68491530418396,
      "epoch": 0.11336797354747284,
      "grad_norm": 19.094434967317397,
      "learning_rate": 3.773584905660377e-07,
      "logits": -1.4201196432113647,
      "logps": -82.76591491699219,
      "loss": 0.6809,
      "objective": 0.68491530418396,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.68491530418396,
      "step": 40,
      "wo_beta": 6.802278518676758
    },
    {
      "dpo_loss": 0.681048572063446,
      "epoch": 0.12753897024090693,
      "grad_norm": 19.32762735250478,
      "learning_rate": 4.2452830188679244e-07,
      "logits": -1.448870301246643,
      "logps": -82.9134292602539,
      "loss": 0.6834,
      "objective": 0.681048572063446,
      "ranking_simple": 0.5249999761581421,
      "regularize": 0.681048572063446,
      "step": 45,
      "wo_beta": 6.972365856170654
    },
    {
      "dpo_loss": 0.676356852054596,
      "epoch": 0.14170996693434104,
      "grad_norm": 18.06878353401125,
      "learning_rate": 4.7169811320754717e-07,
      "logits": -1.4718233346939087,
      "logps": -83.58888244628906,
      "loss": 0.6719,
      "objective": 0.676356852054596,
      "ranking_simple": 0.4958333373069763,
      "regularize": 0.676356852054596,
      "step": 50,
      "wo_beta": 7.712605953216553
    },
    {
      "epoch": 0.14170996693434104,
      "eval_dpo_loss": 0.6879124045372009,
      "eval_logits": -1.4697412252426147,
      "eval_logps": -89.67761993408203,
      "eval_loss": 0.685627281665802,
      "eval_objective": 0.6879124045372009,
      "eval_ranking_simple": 0.5269151329994202,
      "eval_regularize": 0.6879124045372009,
      "eval_runtime": 368.3427,
      "eval_samples_per_second": 15.719,
      "eval_steps_per_second": 1.311,
      "eval_wo_beta": 7.922134876251221,
      "step": 50
    },
    {
      "dpo_loss": 0.6702864766120911,
      "epoch": 0.15588096362777515,
      "grad_norm": 19.335503669780774,
      "learning_rate": 5.188679245283019e-07,
      "logits": -1.4462206363677979,
      "logps": -83.2486801147461,
      "loss": 0.6744,
      "objective": 0.6702864766120911,
      "ranking_simple": 0.4958333373069763,
      "regularize": 0.6702864766120911,
      "step": 55,
      "wo_beta": 7.6053338050842285
    },
    {
      "dpo_loss": 0.6743167638778687,
      "epoch": 0.17005196032120926,
      "grad_norm": 19.32608740622528,
      "learning_rate": 5.660377358490566e-07,
      "logits": -1.3781672716140747,
      "logps": -84.30441284179688,
      "loss": 0.6703,
      "objective": 0.6743167638778687,
      "ranking_simple": 0.5375000238418579,
      "regularize": 0.6743167638778687,
      "step": 60,
      "wo_beta": 6.470597267150879
    },
    {
      "dpo_loss": 0.6568139791488647,
      "epoch": 0.18422295701464336,
      "grad_norm": 21.002392018495687,
      "learning_rate": 6.132075471698112e-07,
      "logits": -1.4774647951126099,
      "logps": -81.55781555175781,
      "loss": 0.6619,
      "objective": 0.6568139791488647,
      "ranking_simple": 0.5375000238418579,
      "regularize": 0.6568139791488647,
      "step": 65,
      "wo_beta": 6.502650260925293
    },
    {
      "dpo_loss": 0.6504150629043579,
      "epoch": 0.19839395370807747,
      "grad_norm": 18.63585310886347,
      "learning_rate": 6.60377358490566e-07,
      "logits": -1.5209298133850098,
      "logps": -84.6080551147461,
      "loss": 0.6501,
      "objective": 0.6504150629043579,
      "ranking_simple": 0.5625,
      "regularize": 0.6504150629043579,
      "step": 70,
      "wo_beta": 6.957874774932861
    },
    {
      "dpo_loss": 0.653372049331665,
      "epoch": 0.21256495040151158,
      "grad_norm": 20.115959212821757,
      "learning_rate": 7.075471698113207e-07,
      "logits": -1.4559004306793213,
      "logps": -85.126953125,
      "loss": 0.6557,
      "objective": 0.653372049331665,
      "ranking_simple": 0.5458333492279053,
      "regularize": 0.653372049331665,
      "step": 75,
      "wo_beta": 6.480815410614014
    },
    {
      "dpo_loss": 0.6524909734725952,
      "epoch": 0.22673594709494568,
      "grad_norm": 23.881541586489003,
      "learning_rate": 7.547169811320754e-07,
      "logits": -1.567487120628357,
      "logps": -84.74055480957031,
      "loss": 0.6601,
      "objective": 0.6524909734725952,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.6524909734725952,
      "step": 80,
      "wo_beta": 6.057122707366943
    },
    {
      "dpo_loss": 0.6443823575973511,
      "epoch": 0.2409069437883798,
      "grad_norm": 18.676003369955335,
      "learning_rate": 8.018867924528302e-07,
      "logits": -1.5786373615264893,
      "logps": -84.9271011352539,
      "loss": 0.6489,
      "objective": 0.6443823575973511,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.6443823575973511,
      "step": 85,
      "wo_beta": 5.872949600219727
    },
    {
      "dpo_loss": 0.6308037638664246,
      "epoch": 0.25507794048181387,
      "grad_norm": 18.36712876918143,
      "learning_rate": 8.490566037735849e-07,
      "logits": -1.693374752998352,
      "logps": -85.01197052001953,
      "loss": 0.6429,
      "objective": 0.6308037638664246,
      "ranking_simple": 0.5458333492279053,
      "regularize": 0.6308037638664246,
      "step": 90,
      "wo_beta": 6.442193031311035
    },
    {
      "dpo_loss": 0.6293771266937256,
      "epoch": 0.269248937175248,
      "grad_norm": 19.143294691787162,
      "learning_rate": 8.962264150943396e-07,
      "logits": -1.6244534254074097,
      "logps": -84.94630432128906,
      "loss": 0.6392,
      "objective": 0.6293771266937256,
      "ranking_simple": 0.5208333134651184,
      "regularize": 0.6293771266937256,
      "step": 95,
      "wo_beta": 7.510855197906494
    },
    {
      "dpo_loss": 0.6378400921821594,
      "epoch": 0.2834199338686821,
      "grad_norm": 18.923136932397636,
      "learning_rate": 9.433962264150943e-07,
      "logits": -1.531855821609497,
      "logps": -84.18727111816406,
      "loss": 0.6459,
      "objective": 0.6378400921821594,
      "ranking_simple": 0.5291666388511658,
      "regularize": 0.6378400921821594,
      "step": 100,
      "wo_beta": 6.881345748901367
    },
    {
      "epoch": 0.2834199338686821,
      "eval_dpo_loss": 0.6793138384819031,
      "eval_logits": -1.6510542631149292,
      "eval_logps": -92.99544525146484,
      "eval_loss": 0.6764773726463318,
      "eval_objective": 0.6793138384819031,
      "eval_ranking_simple": 0.534679114818573,
      "eval_regularize": 0.6793138384819031,
      "eval_runtime": 367.8383,
      "eval_samples_per_second": 15.741,
      "eval_steps_per_second": 1.313,
      "eval_wo_beta": 7.872693061828613,
      "step": 100
    },
    {
      "dpo_loss": 0.6514045000076294,
      "epoch": 0.2975909305621162,
      "grad_norm": 24.32830193794003,
      "learning_rate": 9.90566037735849e-07,
      "logits": -1.7460098266601562,
      "logps": -88.28630828857422,
      "loss": 0.6307,
      "objective": 0.6514045000076294,
      "ranking_simple": 0.574999988079071,
      "regularize": 0.6514045000076294,
      "step": 105,
      "wo_beta": 6.015078067779541
    },
    {
      "dpo_loss": 0.6186487674713135,
      "epoch": 0.3117619272555503,
      "grad_norm": 21.425599904477778,
      "learning_rate": 9.99956257238817e-07,
      "logits": -1.6647683382034302,
      "logps": -88.7564468383789,
      "loss": 0.619,
      "objective": 0.6186487674713135,
      "ranking_simple": 0.5249999761581421,
      "regularize": 0.6186487674713135,
      "step": 110,
      "wo_beta": 6.501535892486572
    },
    {
      "dpo_loss": 0.6382968425750732,
      "epoch": 0.32593292394898443,
      "grad_norm": 23.57030350866866,
      "learning_rate": 9.997785653888834e-07,
      "logits": -1.6553956270217896,
      "logps": -88.90452575683594,
      "loss": 0.6244,
      "objective": 0.6382968425750732,
      "ranking_simple": 0.5666666626930237,
      "regularize": 0.6382968425750732,
      "step": 115,
      "wo_beta": 6.804749488830566
    },
    {
      "dpo_loss": 0.613271176815033,
      "epoch": 0.3401039206424185,
      "grad_norm": 17.498249311505813,
      "learning_rate": 9.994642390694308e-07,
      "logits": -1.6392256021499634,
      "logps": -87.69567108154297,
      "loss": 0.6212,
      "objective": 0.613271176815033,
      "ranking_simple": 0.574999988079071,
      "regularize": 0.613271176815033,
      "step": 120,
      "wo_beta": 7.195651054382324
    },
    {
      "dpo_loss": 0.6102784872055054,
      "epoch": 0.35427491733585265,
      "grad_norm": 17.38350046383242,
      "learning_rate": 9.990133642141357e-07,
      "logits": -1.6804019212722778,
      "logps": -89.06767272949219,
      "loss": 0.6218,
      "objective": 0.6102784872055054,
      "ranking_simple": 0.5208333134651184,
      "regularize": 0.6102784872055054,
      "step": 125,
      "wo_beta": 7.266691207885742
    },
    {
      "dpo_loss": 0.6158983111381531,
      "epoch": 0.3684459140292867,
      "grad_norm": 18.419918693556507,
      "learning_rate": 9.98426064087682e-07,
      "logits": -1.6602026224136353,
      "logps": -86.5518569946289,
      "loss": 0.6238,
      "objective": 0.6158983111381531,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.6158983111381531,
      "step": 130,
      "wo_beta": 5.715666770935059
    },
    {
      "dpo_loss": 0.6510148644447327,
      "epoch": 0.3826169107227208,
      "grad_norm": 21.943398704973582,
      "learning_rate": 9.977024992520601e-07,
      "logits": -1.7066783905029297,
      "logps": -88.5932846069336,
      "loss": 0.6289,
      "objective": 0.6510148644447327,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.6510148644447327,
      "step": 135,
      "wo_beta": 7.166342258453369
    },
    {
      "dpo_loss": 0.6169579029083252,
      "epoch": 0.39678790741615494,
      "grad_norm": 19.9956879065625,
      "learning_rate": 9.968428675226713e-07,
      "logits": -1.6791799068450928,
      "logps": -89.6558837890625,
      "loss": 0.6144,
      "objective": 0.6169579029083252,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.6169579029083252,
      "step": 140,
      "wo_beta": 6.211281776428223
    },
    {
      "dpo_loss": 0.5698094964027405,
      "epoch": 0.410958904109589,
      "grad_norm": 20.71491947708098,
      "learning_rate": 9.958474039142469e-07,
      "logits": -1.6847442388534546,
      "logps": -89.56360626220703,
      "loss": 0.5875,
      "objective": 0.5698094964027405,
      "ranking_simple": 0.6499999761581421,
      "regularize": 0.5698094964027405,
      "step": 145,
      "wo_beta": 5.130297660827637
    },
    {
      "dpo_loss": 0.6153029203414917,
      "epoch": 0.42512990080302315,
      "grad_norm": 19.57556543746145,
      "learning_rate": 9.947163805765979e-07,
      "logits": -1.70354425907135,
      "logps": -89.73717498779297,
      "loss": 0.5993,
      "objective": 0.6153029203414917,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.6153029203414917,
      "step": 150,
      "wo_beta": 8.4924955368042
    },
    {
      "epoch": 0.42512990080302315,
      "eval_dpo_loss": 0.6804503202438354,
      "eval_logits": -1.6963125467300415,
      "eval_logps": -95.27294921875,
      "eval_loss": 0.6770597696304321,
      "eval_objective": 0.6804503202438354,
      "eval_ranking_simple": 0.534679114818573,
      "eval_regularize": 0.6804503202438354,
      "eval_runtime": 370.2983,
      "eval_samples_per_second": 15.636,
      "eval_steps_per_second": 1.304,
      "eval_wo_beta": 8.2155179977417,
      "step": 150
    },
    {
      "dpo_loss": 0.59377521276474,
      "epoch": 0.43930089749645723,
      "grad_norm": 23.61683273964934,
      "learning_rate": 9.934501067202117e-07,
      "logits": -1.7478511333465576,
      "logps": -87.8529052734375,
      "loss": 0.5935,
      "objective": 0.59377521276474,
      "ranking_simple": 0.5375000238418579,
      "regularize": 0.59377521276474,
      "step": 155,
      "wo_beta": 7.3897318840026855
    },
    {
      "dpo_loss": 0.589878261089325,
      "epoch": 0.45347189418989137,
      "grad_norm": 19.912032474590337,
      "learning_rate": 9.92048928531717e-07,
      "logits": -1.6176892518997192,
      "logps": -88.71855926513672,
      "loss": 0.5812,
      "objective": 0.589878261089325,
      "ranking_simple": 0.5874999761581421,
      "regularize": 0.589878261089325,
      "step": 160,
      "wo_beta": 6.206328868865967
    },
    {
      "dpo_loss": 0.5920292735099792,
      "epoch": 0.46764289088332545,
      "grad_norm": 21.374874744599914,
      "learning_rate": 9.905132290792392e-07,
      "logits": -1.745171308517456,
      "logps": -91.21366119384766,
      "loss": 0.5859,
      "objective": 0.5920292735099792,
      "ranking_simple": 0.5416666865348816,
      "regularize": 0.5920292735099792,
      "step": 165,
      "wo_beta": 6.208406448364258
    },
    {
      "dpo_loss": 0.6067599654197693,
      "epoch": 0.4818138875767596,
      "grad_norm": 19.998053262503415,
      "learning_rate": 9.888434282076757e-07,
      "logits": -1.8083672523498535,
      "logps": -91.927734375,
      "loss": 0.5862,
      "objective": 0.6067599654197693,
      "ranking_simple": 0.5375000238418579,
      "regularize": 0.6067599654197693,
      "step": 170,
      "wo_beta": 7.094420433044434
    },
    {
      "dpo_loss": 0.5829775929450989,
      "epoch": 0.49598488427019366,
      "grad_norm": 22.779470361992754,
      "learning_rate": 9.870399824239114e-07,
      "logits": -1.6424000263214111,
      "logps": -91.81340789794922,
      "loss": 0.5844,
      "objective": 0.5829775929450989,
      "ranking_simple": 0.5708333253860474,
      "regularize": 0.5829775929450989,
      "step": 175,
      "wo_beta": 6.69417667388916
    },
    {
      "dpo_loss": 0.5724970698356628,
      "epoch": 0.5101558809636277,
      "grad_norm": 19.56386680488567,
      "learning_rate": 9.851033847720164e-07,
      "logits": -1.6553891897201538,
      "logps": -91.84220123291016,
      "loss": 0.5461,
      "objective": 0.5724970698356628,
      "ranking_simple": 0.5375000238418579,
      "regularize": 0.5724970698356628,
      "step": 180,
      "wo_beta": 6.825948238372803
    },
    {
      "dpo_loss": 0.5537912845611572,
      "epoch": 0.5243268776570619,
      "grad_norm": 23.895057754013074,
      "learning_rate": 9.83034164698452e-07,
      "logits": -1.6311272382736206,
      "logps": -90.77505493164062,
      "loss": 0.5628,
      "objective": 0.5537912845611572,
      "ranking_simple": 0.5791666507720947,
      "regularize": 0.5537912845611572,
      "step": 185,
      "wo_beta": 5.920953750610352
    },
    {
      "dpo_loss": 0.5373654961585999,
      "epoch": 0.538497874350496,
      "grad_norm": 23.921385246249166,
      "learning_rate": 9.808328879073251e-07,
      "logits": -1.727196216583252,
      "logps": -92.44843292236328,
      "loss": 0.5475,
      "objective": 0.5373654961585999,
      "ranking_simple": 0.6291666626930237,
      "regularize": 0.5373654961585999,
      "step": 190,
      "wo_beta": 5.952234745025635
    },
    {
      "dpo_loss": 0.5564671754837036,
      "epoch": 0.5526688710439301,
      "grad_norm": 24.71069656183637,
      "learning_rate": 9.78500156205731e-07,
      "logits": -1.7691571712493896,
      "logps": -102.49636840820312,
      "loss": 0.5838,
      "objective": 0.5564671754837036,
      "ranking_simple": 0.6041666865348816,
      "regularize": 0.5564671754837036,
      "step": 195,
      "wo_beta": 6.274513244628906
    },
    {
      "dpo_loss": 0.5659002661705017,
      "epoch": 0.5668398677373642,
      "grad_norm": 18.721765242615913,
      "learning_rate": 9.760366073392244e-07,
      "logits": -1.7705143690109253,
      "logps": -108.12271118164062,
      "loss": 0.5557,
      "objective": 0.5659002661705017,
      "ranking_simple": 0.5583333373069763,
      "regularize": 0.5659002661705017,
      "step": 200,
      "wo_beta": 6.690428733825684
    },
    {
      "epoch": 0.5668398677373642,
      "eval_dpo_loss": 0.6866137981414795,
      "eval_logits": -1.8150068521499634,
      "eval_logps": -115.46802520751953,
      "eval_loss": 0.6857941150665283,
      "eval_objective": 0.6866137981414795,
      "eval_ranking_simple": 0.5295031070709229,
      "eval_regularize": 0.6866137981414795,
      "eval_runtime": 369.0235,
      "eval_samples_per_second": 15.69,
      "eval_steps_per_second": 1.309,
      "eval_wo_beta": 7.96071720123291,
      "step": 200
    },
    {
      "dpo_loss": 0.5478584170341492,
      "epoch": 0.5810108644307983,
      "grad_norm": 21.07102836538459,
      "learning_rate": 9.734429148174674e-07,
      "logits": -1.6568210124969482,
      "logps": -108.47331237792969,
      "loss": 0.545,
      "objective": 0.5478584170341492,
      "ranking_simple": 0.5874999761581421,
      "regularize": 0.5478584170341492,
      "step": 205,
      "wo_beta": 7.015623092651367
    },
    {
      "dpo_loss": 0.5228937268257141,
      "epoch": 0.5951818611242324,
      "grad_norm": 20.651776585733355,
      "learning_rate": 9.707197877300973e-07,
      "logits": -1.7239099740982056,
      "logps": -108.58413696289062,
      "loss": 0.5299,
      "objective": 0.5228937268257141,
      "ranking_simple": 0.625,
      "regularize": 0.5228937268257141,
      "step": 210,
      "wo_beta": 5.675539970397949
    },
    {
      "dpo_loss": 0.5450037121772766,
      "epoch": 0.6093528578176665,
      "grad_norm": 23.006204591176342,
      "learning_rate": 9.678679705528698e-07,
      "logits": -1.818426251411438,
      "logps": -108.85588073730469,
      "loss": 0.5504,
      "objective": 0.5450037121772766,
      "ranking_simple": 0.5625,
      "regularize": 0.5450037121772766,
      "step": 215,
      "wo_beta": 6.871977806091309
    },
    {
      "dpo_loss": 0.5016953945159912,
      "epoch": 0.6235238545111006,
      "grad_norm": 21.55242582954206,
      "learning_rate": 9.648882429441256e-07,
      "logits": -1.796720266342163,
      "logps": -106.0498046875,
      "loss": 0.5337,
      "objective": 0.5016953945159912,
      "ranking_simple": 0.5958333611488342,
      "regularize": 0.5016953945159912,
      "step": 220,
      "wo_beta": 6.975856304168701
    },
    {
      "dpo_loss": 0.5535920262336731,
      "epoch": 0.6376948512045347,
      "grad_norm": 18.336769425586553,
      "learning_rate": 9.61781419531641e-07,
      "logits": -1.8808425664901733,
      "logps": -103.36882781982422,
      "loss": 0.5444,
      "objective": 0.5535920262336731,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.5535920262336731,
      "step": 225,
      "wo_beta": 6.945814609527588
    },
    {
      "dpo_loss": 0.5067029595375061,
      "epoch": 0.6518658478979689,
      "grad_norm": 20.395537847593562,
      "learning_rate": 9.585483496899149e-07,
      "logits": -1.781941533088684,
      "logps": -102.78409576416016,
      "loss": 0.5275,
      "objective": 0.5067029595375061,
      "ranking_simple": 0.6541666388511658,
      "regularize": 0.5067029595375061,
      "step": 230,
      "wo_beta": 4.95630407333374
    },
    {
      "dpo_loss": 0.5263319611549377,
      "epoch": 0.6660368445914029,
      "grad_norm": 24.579908080459226,
      "learning_rate": 9.551899173079606e-07,
      "logits": -1.7018815279006958,
      "logps": -107.19641876220703,
      "loss": 0.5235,
      "objective": 0.5263319611549377,
      "ranking_simple": 0.5958333611488342,
      "regularize": 0.5263319611549377,
      "step": 235,
      "wo_beta": 6.136026859283447
    },
    {
      "dpo_loss": 0.5419639945030212,
      "epoch": 0.680207841284837,
      "grad_norm": 25.457069150013837,
      "learning_rate": 9.517070405476574e-07,
      "logits": -1.8716365098953247,
      "logps": -109.35581970214844,
      "loss": 0.5391,
      "objective": 0.5419639945030212,
      "ranking_simple": 0.637499988079071,
      "regularize": 0.5419639945030212,
      "step": 240,
      "wo_beta": 7.386639595031738
    },
    {
      "dpo_loss": 0.599401593208313,
      "epoch": 0.6943788379782712,
      "grad_norm": 20.952521291073488,
      "learning_rate": 9.481006715927351e-07,
      "logits": -1.8737353086471558,
      "logps": -105.67475128173828,
      "loss": 0.5586,
      "objective": 0.599401593208313,
      "ranking_simple": 0.625,
      "regularize": 0.599401593208313,
      "step": 245,
      "wo_beta": 6.23760986328125
    },
    {
      "dpo_loss": 0.5379212498664856,
      "epoch": 0.7085498346717053,
      "grad_norm": 17.076599406497994,
      "learning_rate": 9.443717963884568e-07,
      "logits": -1.6231579780578613,
      "logps": -97.79161834716797,
      "loss": 0.5428,
      "objective": 0.5379212498664856,
      "ranking_simple": 0.6083333492279053,
      "regularize": 0.5379212498664856,
      "step": 250,
      "wo_beta": 5.470564842224121
    },
    {
      "epoch": 0.7085498346717053,
      "eval_dpo_loss": 0.674086332321167,
      "eval_logits": -1.8494712114334106,
      "eval_logps": -102.56678771972656,
      "eval_loss": 0.6744823455810547,
      "eval_objective": 0.674086332321167,
      "eval_ranking_simple": 0.5367494821548462,
      "eval_regularize": 0.674086332321167,
      "eval_runtime": 369.2355,
      "eval_samples_per_second": 15.681,
      "eval_steps_per_second": 1.308,
      "eval_wo_beta": 7.989133358001709,
      "step": 250
    },
    {
      "dpo_loss": 0.4980238378047943,
      "epoch": 0.7227208313651393,
      "grad_norm": 18.645503031645394,
      "learning_rate": 9.405214343720706e-07,
      "logits": -1.8677008152008057,
      "logps": -94.05529022216797,
      "loss": 0.5176,
      "objective": 0.4980238378047943,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.4980238378047943,
      "step": 255,
      "wo_beta": 5.837521553039551
    },
    {
      "dpo_loss": 0.5208728909492493,
      "epoch": 0.7368918280585735,
      "grad_norm": 22.040310265005534,
      "learning_rate": 9.365506381941065e-07,
      "logits": -1.8145065307617188,
      "logps": -96.94599914550781,
      "loss": 0.5381,
      "objective": 0.5208728909492493,
      "ranking_simple": 0.5708333253860474,
      "regularize": 0.5208728909492493,
      "step": 260,
      "wo_beta": 6.736252784729004
    },
    {
      "dpo_loss": 0.4980500638484955,
      "epoch": 0.7510628247520076,
      "grad_norm": 23.39044199948042,
      "learning_rate": 9.32460493430591e-07,
      "logits": -1.7500866651535034,
      "logps": -96.90016174316406,
      "loss": 0.5023,
      "objective": 0.4980500638484955,
      "ranking_simple": 0.6583333611488342,
      "regularize": 0.4980500638484955,
      "step": 265,
      "wo_beta": 6.221852779388428
    },
    {
      "dpo_loss": 0.5561876893043518,
      "epoch": 0.7652338214454416,
      "grad_norm": 20.73702481860498,
      "learning_rate": 9.282521182862629e-07,
      "logits": -1.8606762886047363,
      "logps": -96.93506622314453,
      "loss": 0.5259,
      "objective": 0.5561876893043518,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.5561876893043518,
      "step": 270,
      "wo_beta": 6.527937412261963
    },
    {
      "dpo_loss": 0.5460684299468994,
      "epoch": 0.7794048181388757,
      "grad_norm": 19.98766091687606,
      "learning_rate": 9.239266632888658e-07,
      "logits": -1.697192668914795,
      "logps": -96.09431457519531,
      "loss": 0.5114,
      "objective": 0.5460684299468994,
      "ranking_simple": 0.574999988079071,
      "regularize": 0.5460684299468994,
      "step": 275,
      "wo_beta": 6.768658638000488
    },
    {
      "dpo_loss": 0.47327375411987305,
      "epoch": 0.7935758148323099,
      "grad_norm": 21.02838944330306,
      "learning_rate": 9.194853109746072e-07,
      "logits": -1.8069101572036743,
      "logps": -97.56507873535156,
      "loss": 0.491,
      "objective": 0.47327375411987305,
      "ranking_simple": 0.625,
      "regularize": 0.47327375411987305,
      "step": 280,
      "wo_beta": 5.885926723480225
    },
    {
      "dpo_loss": 0.5277642607688904,
      "epoch": 0.807746811525744,
      "grad_norm": 22.871584835681148,
      "learning_rate": 9.14929275564863e-07,
      "logits": -1.8146883249282837,
      "logps": -99.09355926513672,
      "loss": 0.5087,
      "objective": 0.5277642607688904,
      "ranking_simple": 0.5791666507720947,
      "regularize": 0.5277642607688904,
      "step": 285,
      "wo_beta": 7.316993236541748
    },
    {
      "dpo_loss": 0.43708011507987976,
      "epoch": 0.821917808219178,
      "grad_norm": 25.529916734053177,
      "learning_rate": 9.102598026342222e-07,
      "logits": -1.8029001951217651,
      "logps": -96.13628387451172,
      "loss": 0.4688,
      "objective": 0.43708011507987976,
      "ranking_simple": 0.625,
      "regularize": 0.43708011507987976,
      "step": 290,
      "wo_beta": 4.752199649810791
    },
    {
      "dpo_loss": 0.49055105447769165,
      "epoch": 0.8360888049126122,
      "grad_norm": 26.014797277152265,
      "learning_rate": 9.0547816876996e-07,
      "logits": -1.8255597352981567,
      "logps": -96.79292297363281,
      "loss": 0.4938,
      "objective": 0.49055105447769165,
      "ranking_simple": 0.5708333253860474,
      "regularize": 0.49055105447769165,
      "step": 295,
      "wo_beta": 6.931264400482178
    },
    {
      "dpo_loss": 0.4934200048446655,
      "epoch": 0.8502598016060463,
      "grad_norm": 22.235912832826426,
      "learning_rate": 9.005856812230304e-07,
      "logits": -1.8234219551086426,
      "logps": -98.131103515625,
      "loss": 0.4987,
      "objective": 0.4934200048446655,
      "ranking_simple": 0.6541666388511658,
      "regularize": 0.4934200048446655,
      "step": 300,
      "wo_beta": 4.583770275115967
    },
    {
      "epoch": 0.8502598016060463,
      "eval_dpo_loss": 0.7202683091163635,
      "eval_logits": -1.9276690483093262,
      "eval_logps": -110.09490966796875,
      "eval_loss": 0.711872398853302,
      "eval_objective": 0.7202683091163635,
      "eval_ranking_simple": 0.5372670888900757,
      "eval_regularize": 0.7202683091163635,
      "eval_runtime": 369.1999,
      "eval_samples_per_second": 15.683,
      "eval_steps_per_second": 1.308,
      "eval_wo_beta": 8.926669120788574,
      "step": 300
    },
    {
      "dpo_loss": 0.5209147334098816,
      "epoch": 0.8644307982994804,
      "grad_norm": 21.742768863559323,
      "learning_rate": 8.955836775506775e-07,
      "logits": -1.8012293577194214,
      "logps": -102.54885864257812,
      "loss": 0.4739,
      "objective": 0.5209147334098816,
      "ranking_simple": 0.6333333253860474,
      "regularize": 0.5209147334098816,
      "step": 305,
      "wo_beta": 5.649385929107666
    },
    {
      "dpo_loss": 0.4898562729358673,
      "epoch": 0.8786017949929145,
      "grad_norm": 22.266286975377888,
      "learning_rate": 8.904735252507609e-07,
      "logits": -1.8041430711746216,
      "logps": -101.15433502197266,
      "loss": 0.488,
      "objective": 0.4898562729358673,
      "ranking_simple": 0.6499999761581421,
      "regularize": 0.4898562729358673,
      "step": 310,
      "wo_beta": 6.073896408081055
    },
    {
      "dpo_loss": 0.4698447585105896,
      "epoch": 0.8927727916863486,
      "grad_norm": 22.62826678390999,
      "learning_rate": 8.852566213878946e-07,
      "logits": -1.798747181892395,
      "logps": -101.55271911621094,
      "loss": 0.4839,
      "objective": 0.4698447585105896,
      "ranking_simple": 0.6083333492279053,
      "regularize": 0.4698447585105896,
      "step": 315,
      "wo_beta": 5.962237358093262
    },
    {
      "dpo_loss": 0.4936090409755707,
      "epoch": 0.9069437883797827,
      "grad_norm": 25.39576707145419,
      "learning_rate": 8.799343922115043e-07,
      "logits": -1.7406024932861328,
      "logps": -110.01341247558594,
      "loss": 0.5003,
      "objective": 0.4936090409755707,
      "ranking_simple": 0.6666666865348816,
      "regularize": 0.4936090409755707,
      "step": 320,
      "wo_beta": 4.6922502517700195
    },
    {
      "dpo_loss": 0.464478075504303,
      "epoch": 0.9211147850732169,
      "grad_norm": 22.63000678324253,
      "learning_rate": 8.745082927659046e-07,
      "logits": -1.8573758602142334,
      "logps": -106.48271179199219,
      "loss": 0.4601,
      "objective": 0.464478075504303,
      "ranking_simple": 0.637499988079071,
      "regularize": 0.464478075504303,
      "step": 325,
      "wo_beta": 5.424474239349365
    },
    {
      "dpo_loss": 0.4378047287464142,
      "epoch": 0.9352857817666509,
      "grad_norm": 26.572088004567764,
      "learning_rate": 8.689798064925048e-07,
      "logits": -1.6883081197738647,
      "logps": -104.9384765625,
      "loss": 0.4724,
      "objective": 0.4378047287464142,
      "ranking_simple": 0.6416666507720947,
      "regularize": 0.4378047287464142,
      "step": 330,
      "wo_beta": 5.635779857635498
    },
    {
      "dpo_loss": 0.4327344000339508,
      "epoch": 0.949456778460085,
      "grad_norm": 28.535014872524457,
      "learning_rate": 8.633504448242504e-07,
      "logits": -1.6694140434265137,
      "logps": -107.3168716430664,
      "loss": 0.4595,
      "objective": 0.4327344000339508,
      "ranking_simple": 0.6708333492279053,
      "regularize": 0.4327344000339508,
      "step": 335,
      "wo_beta": 5.270318508148193
    },
    {
      "dpo_loss": 0.4696439504623413,
      "epoch": 0.9636277751535192,
      "grad_norm": 29.414333709201422,
      "learning_rate": 8.576217467724127e-07,
      "logits": -1.7229362726211548,
      "logps": -106.86974334716797,
      "loss": 0.5044,
      "objective": 0.4696439504623413,
      "ranking_simple": 0.637499988079071,
      "regularize": 0.4696439504623413,
      "step": 340,
      "wo_beta": 5.814812660217285
    },
    {
      "dpo_loss": 0.43202081322669983,
      "epoch": 0.9777987718469532,
      "grad_norm": 21.299611385252437,
      "learning_rate": 8.517952785058384e-07,
      "logits": -1.7784336805343628,
      "logps": -101.36027526855469,
      "loss": 0.4585,
      "objective": 0.43202081322669983,
      "ranking_simple": 0.6958333253860474,
      "regularize": 0.43202081322669983,
      "step": 345,
      "wo_beta": 4.479858875274658
    },
    {
      "dpo_loss": 0.4188024699687958,
      "epoch": 0.9919697685403873,
      "grad_norm": 20.8027959046757,
      "learning_rate": 8.458726329227747e-07,
      "logits": -1.6966959238052368,
      "logps": -99.99575805664062,
      "loss": 0.4599,
      "objective": 0.4188024699687958,
      "ranking_simple": 0.762499988079071,
      "regularize": 0.4188024699687958,
      "step": 350,
      "wo_beta": 4.140859603881836
    },
    {
      "epoch": 0.9919697685403873,
      "eval_dpo_loss": 0.6912401914596558,
      "eval_logits": -1.8473907709121704,
      "eval_logps": -104.9832763671875,
      "eval_loss": 0.6885838508605957,
      "eval_objective": 0.6912401914596558,
      "eval_ranking_simple": 0.5351966619491577,
      "eval_regularize": 0.6912401914596558,
      "eval_runtime": 375.4224,
      "eval_samples_per_second": 15.423,
      "eval_steps_per_second": 1.287,
      "eval_wo_beta": 8.374881744384766,
      "step": 350
    },
    {
      "dpo_loss": 0.4273616671562195,
      "epoch": 1.0061407652338215,
      "grad_norm": 19.801974743070023,
      "learning_rate": 8.398554292153865e-07,
      "logits": -1.9038132429122925,
      "logps": -97.71568298339844,
      "loss": 0.4198,
      "objective": 0.4273616671562195,
      "ranking_simple": 0.6708333492279053,
      "regularize": 0.4273616671562195,
      "step": 355,
      "wo_beta": 4.851585388183594
    },
    {
      "dpo_loss": 0.3563633859157562,
      "epoch": 1.0203117619272555,
      "grad_norm": 27.347143857547685,
      "learning_rate": 8.337453124270862e-07,
      "logits": -1.720730185508728,
      "logps": -104.91397094726562,
      "loss": 0.3668,
      "objective": 0.3563633859157562,
      "ranking_simple": 0.6958333253860474,
      "regularize": 0.3563633859157562,
      "step": 360,
      "wo_beta": 4.480281352996826
    },
    {
      "dpo_loss": 0.34869903326034546,
      "epoch": 1.0344827586206897,
      "grad_norm": 21.348824435063978,
      "learning_rate": 8.275439530027947e-07,
      "logits": -1.730131983757019,
      "logps": -111.44975280761719,
      "loss": 0.3663,
      "objective": 0.34869903326034546,
      "ranking_simple": 0.7041666507720947,
      "regularize": 0.34869903326034546,
      "step": 365,
      "wo_beta": 3.829502820968628
    },
    {
      "dpo_loss": 0.3821018934249878,
      "epoch": 1.0486537553141237,
      "grad_norm": 23.44162059864065,
      "learning_rate": 8.212530463322582e-07,
      "logits": -1.7056537866592407,
      "logps": -104.91122436523438,
      "loss": 0.3708,
      "objective": 0.3821018934249878,
      "ranking_simple": 0.6875,
      "regularize": 0.3821018934249878,
      "step": 370,
      "wo_beta": 5.17144250869751
    },
    {
      "dpo_loss": 0.3470642864704132,
      "epoch": 1.0628247520075578,
      "grad_norm": 21.682188897692722,
      "learning_rate": 8.148743122865463e-07,
      "logits": -1.7548179626464844,
      "logps": -104.17964172363281,
      "loss": 0.3665,
      "objective": 0.3470642864704132,
      "ranking_simple": 0.6875,
      "regularize": 0.3470642864704132,
      "step": 375,
      "wo_beta": 3.6793243885040283
    },
    {
      "dpo_loss": 0.35725000500679016,
      "epoch": 1.076995748700992,
      "grad_norm": 22.228388769880265,
      "learning_rate": 8.084094947478554e-07,
      "logits": -1.7486780881881714,
      "logps": -107.26901245117188,
      "loss": 0.3593,
      "objective": 0.35725000500679016,
      "ranking_simple": 0.7083333134651184,
      "regularize": 0.35725000500679016,
      "step": 380,
      "wo_beta": 3.6982295513153076
    },
    {
      "dpo_loss": 0.36463692784309387,
      "epoch": 1.091166745394426,
      "grad_norm": 21.586757686789323,
      "learning_rate": 8.018603611327504e-07,
      "logits": -1.6958861351013184,
      "logps": -106.55477905273438,
      "loss": 0.3642,
      "objective": 0.36463692784309387,
      "ranking_simple": 0.7041666507720947,
      "regularize": 0.36463692784309387,
      "step": 385,
      "wo_beta": 4.191780090332031
    },
    {
      "dpo_loss": 0.32307326793670654,
      "epoch": 1.10533774208786,
      "grad_norm": 21.44019149145441,
      "learning_rate": 7.952287019089685e-07,
      "logits": -1.6498711109161377,
      "logps": -109.88943481445312,
      "loss": 0.3408,
      "objective": 0.32307326793670654,
      "ranking_simple": 0.737500011920929,
      "regularize": 0.32307326793670654,
      "step": 390,
      "wo_beta": 4.2303996086120605
    },
    {
      "dpo_loss": 0.3521862328052521,
      "epoch": 1.1195087387812943,
      "grad_norm": 25.820266993638842,
      "learning_rate": 7.88516330105925e-07,
      "logits": -1.726596713066101,
      "logps": -108.69520568847656,
      "loss": 0.347,
      "objective": 0.3521862328052521,
      "ranking_simple": 0.7291666865348816,
      "regularize": 0.3521862328052521,
      "step": 395,
      "wo_beta": 3.9551658630371094
    },
    {
      "dpo_loss": 0.343461811542511,
      "epoch": 1.1336797354747283,
      "grad_norm": 22.721697115040456,
      "learning_rate": 7.817250808190483e-07,
      "logits": -1.8199702501296997,
      "logps": -106.22103118896484,
      "loss": 0.3498,
      "objective": 0.343461811542511,
      "ranking_simple": 0.7208333611488342,
      "regularize": 0.343461811542511,
      "step": 400,
      "wo_beta": 3.51658034324646
    },
    {
      "epoch": 1.1336797354747283,
      "eval_dpo_loss": 0.751783549785614,
      "eval_logits": -1.8806556463241577,
      "eval_logps": -115.0888671875,
      "eval_loss": 0.7462870478630066,
      "eval_objective": 0.751783549785614,
      "eval_ranking_simple": 0.5517598390579224,
      "eval_regularize": 0.751783549785614,
      "eval_runtime": 370.3425,
      "eval_samples_per_second": 15.634,
      "eval_steps_per_second": 1.304,
      "eval_wo_beta": 9.550463676452637,
      "step": 400
    },
    {
      "dpo_loss": 0.3792721927165985,
      "epoch": 1.1478507321681626,
      "grad_norm": 32.9676442992408,
      "learning_rate": 7.74856810708083e-07,
      "logits": -1.7736860513687134,
      "logps": -107.8192367553711,
      "loss": 0.3694,
      "objective": 0.3792721927165985,
      "ranking_simple": 0.7208333611488342,
      "regularize": 0.3792721927165985,
      "step": 405,
      "wo_beta": 5.114300727844238
    },
    {
      "dpo_loss": 0.3683342933654785,
      "epoch": 1.1620217288615966,
      "grad_norm": 21.343924046898124,
      "learning_rate": 7.679133974894982e-07,
      "logits": -1.7912460565567017,
      "logps": -105.35082244873047,
      "loss": 0.3582,
      "objective": 0.3683342933654785,
      "ranking_simple": 0.7166666388511658,
      "regularize": 0.3683342933654785,
      "step": 410,
      "wo_beta": 5.410634517669678
    },
    {
      "dpo_loss": 0.3712156116962433,
      "epoch": 1.1761927255550306,
      "grad_norm": 20.631633414155974,
      "learning_rate": 7.608967394231386e-07,
      "logits": -1.7320811748504639,
      "logps": -101.5705337524414,
      "loss": 0.3623,
      "objective": 0.3712156116962433,
      "ranking_simple": 0.6958333253860474,
      "regularize": 0.3712156116962433,
      "step": 415,
      "wo_beta": 5.24213171005249
    },
    {
      "dpo_loss": 0.34116995334625244,
      "epoch": 1.1903637222484649,
      "grad_norm": 21.634750653170887,
      "learning_rate": 7.538087547932584e-07,
      "logits": -1.67872953414917,
      "logps": -101.14533233642578,
      "loss": 0.359,
      "objective": 0.34116995334625244,
      "ranking_simple": 0.6791666746139526,
      "regularize": 0.34116995334625244,
      "step": 420,
      "wo_beta": 4.399470329284668
    },
    {
      "dpo_loss": 0.3757858872413635,
      "epoch": 1.204534718941899,
      "grad_norm": 21.74672077991223,
      "learning_rate": 7.466513813840824e-07,
      "logits": -1.733936071395874,
      "logps": -99.8553466796875,
      "loss": 0.3472,
      "objective": 0.3757858872413635,
      "ranking_simple": 0.7083333134651184,
      "regularize": 0.3757858872413635,
      "step": 425,
      "wo_beta": 4.1172003746032715
    },
    {
      "dpo_loss": 0.3544313311576843,
      "epoch": 1.2187057156353331,
      "grad_norm": 25.498669816549643,
      "learning_rate": 7.394265759500347e-07,
      "logits": -1.7176556587219238,
      "logps": -105.41299438476562,
      "loss": 0.3569,
      "objective": 0.3544313311576843,
      "ranking_simple": 0.6958333253860474,
      "regularize": 0.3544313311576843,
      "step": 430,
      "wo_beta": 4.10382604598999
    },
    {
      "dpo_loss": 0.35303547978401184,
      "epoch": 1.2328767123287672,
      "grad_norm": 24.236679665592966,
      "learning_rate": 7.321363136807818e-07,
      "logits": -1.6603659391403198,
      "logps": -107.16277313232422,
      "loss": 0.3778,
      "objective": 0.35303547978401184,
      "ranking_simple": 0.75,
      "regularize": 0.35303547978401184,
      "step": 435,
      "wo_beta": 3.8118536472320557
    },
    {
      "dpo_loss": 0.32170751690864563,
      "epoch": 1.2470477090222012,
      "grad_norm": 25.528958844465567,
      "learning_rate": 7.247825876612352e-07,
      "logits": -1.6398621797561646,
      "logps": -105.9729995727539,
      "loss": 0.3429,
      "objective": 0.32170751690864563,
      "ranking_simple": 0.762499988079071,
      "regularize": 0.32170751690864563,
      "step": 440,
      "wo_beta": 3.466229200363159
    },
    {
      "dpo_loss": 0.35508811473846436,
      "epoch": 1.2612187057156352,
      "grad_norm": 20.894838905133128,
      "learning_rate": 7.173674083266623e-07,
      "logits": -1.6645927429199219,
      "logps": -106.34298706054688,
      "loss": 0.3613,
      "objective": 0.35508811473846436,
      "ranking_simple": 0.737500011920929,
      "regularize": 0.35508811473846436,
      "step": 445,
      "wo_beta": 4.097968101501465
    },
    {
      "dpo_loss": 0.32759609818458557,
      "epoch": 1.2753897024090695,
      "grad_norm": 21.051410520730908,
      "learning_rate": 7.098928029130528e-07,
      "logits": -1.8029848337173462,
      "logps": -107.71712493896484,
      "loss": 0.3361,
      "objective": 0.32759609818458557,
      "ranking_simple": 0.7583333253860474,
      "regularize": 0.32759609818458557,
      "step": 450,
      "wo_beta": 3.633894443511963
    },
    {
      "epoch": 1.2753897024090695,
      "eval_dpo_loss": 0.7672637701034546,
      "eval_logits": -1.8356177806854248,
      "eval_logps": -116.80036926269531,
      "eval_loss": 0.7562825679779053,
      "eval_objective": 0.7672637701034546,
      "eval_ranking_simple": 0.5419254899024963,
      "eval_regularize": 0.7672637701034546,
      "eval_runtime": 367.9761,
      "eval_samples_per_second": 15.735,
      "eval_steps_per_second": 1.313,
      "eval_wo_beta": 9.725224494934082,
      "step": 450
    },
    {
      "dpo_loss": 0.3631579279899597,
      "epoch": 1.2895606991025035,
      "grad_norm": 27.512782619434457,
      "learning_rate": 7.023608149028936e-07,
      "logits": -1.6034198999404907,
      "logps": -110.81619262695312,
      "loss": 0.3689,
      "objective": 0.3631579279899597,
      "ranking_simple": 0.6708333492279053,
      "regularize": 0.3631579279899597,
      "step": 455,
      "wo_beta": 4.727202892303467
    },
    {
      "dpo_loss": 0.3907562792301178,
      "epoch": 1.3037316957959377,
      "grad_norm": 24.168580409550643,
      "learning_rate": 6.947735034665001e-07,
      "logits": -1.7300904989242554,
      "logps": -108.0628890991211,
      "loss": 0.3712,
      "objective": 0.3907562792301178,
      "ranking_simple": 0.6583333611488342,
      "regularize": 0.3907562792301178,
      "step": 460,
      "wo_beta": 5.317975997924805
    },
    {
      "dpo_loss": 0.3442947268486023,
      "epoch": 1.3179026924893718,
      "grad_norm": 22.349252886711223,
      "learning_rate": 6.871329428990601e-07,
      "logits": -1.7600762844085693,
      "logps": -102.89717102050781,
      "loss": 0.366,
      "objective": 0.3442947268486023,
      "ranking_simple": 0.7166666388511658,
      "regularize": 0.3442947268486023,
      "step": 465,
      "wo_beta": 4.05583381652832
    },
    {
      "dpo_loss": 0.36818769574165344,
      "epoch": 1.3320736891828058,
      "grad_norm": 21.171077905427786,
      "learning_rate": 6.794412220535425e-07,
      "logits": -1.8482578992843628,
      "logps": -100.47290802001953,
      "loss": 0.3612,
      "objective": 0.36818769574165344,
      "ranking_simple": 0.7250000238418579,
      "regularize": 0.36818769574165344,
      "step": 470,
      "wo_beta": 3.844135046005249
    },
    {
      "dpo_loss": 0.3290488123893738,
      "epoch": 1.34624468587624,
      "grad_norm": 23.17750388094903,
      "learning_rate": 6.717004437696249e-07,
      "logits": -1.6789878606796265,
      "logps": -102.9361572265625,
      "loss": 0.3486,
      "objective": 0.3290488123893738,
      "ranking_simple": 0.7541666626930237,
      "regularize": 0.3290488123893738,
      "step": 475,
      "wo_beta": 3.849888324737549
    },
    {
      "dpo_loss": 0.3331288993358612,
      "epoch": 1.360415682569674,
      "grad_norm": 31.57784191234236,
      "learning_rate": 6.639127242987987e-07,
      "logits": -1.7997510433197021,
      "logps": -105.81382751464844,
      "loss": 0.3418,
      "objective": 0.3331288993358612,
      "ranking_simple": 0.7124999761581421,
      "regularize": 0.3331288993358612,
      "step": 480,
      "wo_beta": 5.502153396606445
    },
    {
      "dpo_loss": 0.3518763482570648,
      "epoch": 1.3745866792631083,
      "grad_norm": 25.72021920875409,
      "learning_rate": 6.560801927258079e-07,
      "logits": -1.763397455215454,
      "logps": -105.35774993896484,
      "loss": 0.3606,
      "objective": 0.3518763482570648,
      "ranking_simple": 0.7416666746139526,
      "regularize": 0.3518763482570648,
      "step": 485,
      "wo_beta": 4.792428016662598
    },
    {
      "dpo_loss": 0.3617197871208191,
      "epoch": 1.3887576759565423,
      "grad_norm": 22.21535306569678,
      "learning_rate": 6.482049903865768e-07,
      "logits": -1.7518786191940308,
      "logps": -109.02607727050781,
      "loss": 0.3414,
      "objective": 0.3617197871208191,
      "ranking_simple": 0.7124999761581421,
      "regularize": 0.3617197871208191,
      "step": 490,
      "wo_beta": 3.807429790496826
    },
    {
      "dpo_loss": 0.33107537031173706,
      "epoch": 1.4029286726499763,
      "grad_norm": 28.27182569867929,
      "learning_rate": 6.402892702827916e-07,
      "logits": -1.7524651288986206,
      "logps": -110.29097747802734,
      "loss": 0.3453,
      "objective": 0.33107537031173706,
      "ranking_simple": 0.7083333134651184,
      "regularize": 0.33107537031173706,
      "step": 495,
      "wo_beta": 4.099748611450195
    },
    {
      "dpo_loss": 0.3612217307090759,
      "epoch": 1.4170996693434104,
      "grad_norm": 26.907643867295747,
      "learning_rate": 6.323351964932908e-07,
      "logits": -1.6837791204452515,
      "logps": -109.02066802978516,
      "loss": 0.3584,
      "objective": 0.3612217307090759,
      "ranking_simple": 0.7041666507720947,
      "regularize": 0.3612217307090759,
      "step": 500,
      "wo_beta": 3.91571307182312
    },
    {
      "epoch": 1.4170996693434104,
      "eval_dpo_loss": 0.7694771885871887,
      "eval_logits": -1.8626275062561035,
      "eval_logps": -117.51673126220703,
      "eval_loss": 0.7635300159454346,
      "eval_objective": 0.7694771885871887,
      "eval_ranking_simple": 0.5419254899024963,
      "eval_regularize": 0.7694771885871887,
      "eval_runtime": 370.4437,
      "eval_samples_per_second": 15.63,
      "eval_steps_per_second": 1.304,
      "eval_wo_beta": 9.631916999816895,
      "step": 500
    },
    {
      "dpo_loss": 0.2975477874279022,
      "epoch": 1.4312706660368446,
      "grad_norm": 22.51215190081199,
      "learning_rate": 6.243449435824276e-07,
      "logits": -1.7515524625778198,
      "logps": -109.30506134033203,
      "loss": 0.3308,
      "objective": 0.2975477874279022,
      "ranking_simple": 0.75,
      "regularize": 0.2975477874279022,
      "step": 505,
      "wo_beta": 3.399846076965332
    },
    {
      "dpo_loss": 0.3239024877548218,
      "epoch": 1.4454416627302786,
      "grad_norm": 25.756520666755268,
      "learning_rate": 6.163206960055652e-07,
      "logits": -1.7505611181259155,
      "logps": -108.20401000976562,
      "loss": 0.3502,
      "objective": 0.3239024877548218,
      "ranking_simple": 0.75,
      "regularize": 0.3239024877548218,
      "step": 510,
      "wo_beta": 3.145045042037964
    },
    {
      "dpo_loss": 0.3271316587924957,
      "epoch": 1.4596126594237129,
      "grad_norm": 22.00239796246049,
      "learning_rate": 6.082646475118699e-07,
      "logits": -1.8232632875442505,
      "logps": -107.82081604003906,
      "loss": 0.3356,
      "objective": 0.3271316587924957,
      "ranking_simple": 0.6708333492279053,
      "regularize": 0.3271316587924957,
      "step": 515,
      "wo_beta": 4.939964771270752
    },
    {
      "dpo_loss": 0.30039647221565247,
      "epoch": 1.473783656117147,
      "grad_norm": 22.8094809920846,
      "learning_rate": 6.001790005445606e-07,
      "logits": -1.6817113161087036,
      "logps": -106.67537689208984,
      "loss": 0.3092,
      "objective": 0.30039647221565247,
      "ranking_simple": 0.7541666626930237,
      "regularize": 0.30039647221565247,
      "step": 520,
      "wo_beta": 3.6650784015655518
    },
    {
      "dpo_loss": 0.34032100439071655,
      "epoch": 1.487954652810581,
      "grad_norm": 25.538184332619608,
      "learning_rate": 5.920659656387836e-07,
      "logits": -1.5860577821731567,
      "logps": -107.61659240722656,
      "loss": 0.3466,
      "objective": 0.34032100439071655,
      "ranking_simple": 0.75,
      "regularize": 0.34032100439071655,
      "step": 525,
      "wo_beta": 4.031210422515869
    },
    {
      "dpo_loss": 0.3295021951198578,
      "epoch": 1.5021256495040152,
      "grad_norm": 22.986849455924027,
      "learning_rate": 5.839277608172738e-07,
      "logits": -1.743402123451233,
      "logps": -111.25679016113281,
      "loss": 0.3262,
      "objective": 0.3295021951198578,
      "ranking_simple": 0.737500011920929,
      "regularize": 0.3295021951198578,
      "step": 530,
      "wo_beta": 3.9925944805145264
    },
    {
      "dpo_loss": 0.29601436853408813,
      "epoch": 1.5162966461974492,
      "grad_norm": 25.044023904201577,
      "learning_rate": 5.757666109839702e-07,
      "logits": -1.7071605920791626,
      "logps": -112.61083984375,
      "loss": 0.3088,
      "objective": 0.29601436853408813,
      "ranking_simple": 0.7541666626930237,
      "regularize": 0.29601436853408813,
      "step": 535,
      "wo_beta": 3.535855531692505
    },
    {
      "dpo_loss": 0.350533127784729,
      "epoch": 1.5304676428908834,
      "grad_norm": 22.77917543570306,
      "learning_rate": 5.675847473157485e-07,
      "logits": -1.6420516967773438,
      "logps": -114.60887145996094,
      "loss": 0.338,
      "objective": 0.350533127784729,
      "ranking_simple": 0.7708333134651184,
      "regularize": 0.350533127784729,
      "step": 540,
      "wo_beta": 3.302687644958496
    },
    {
      "dpo_loss": 0.31907784938812256,
      "epoch": 1.5446386395843175,
      "grad_norm": 27.52281539562519,
      "learning_rate": 5.5938440665244e-07,
      "logits": -1.7368324995040894,
      "logps": -117.18190002441406,
      "loss": 0.3311,
      "objective": 0.31907784938812256,
      "ranking_simple": 0.7875000238418579,
      "regularize": 0.31907784938812256,
      "step": 545,
      "wo_beta": 2.7296268939971924
    },
    {
      "dpo_loss": 0.3072589933872223,
      "epoch": 1.5588096362777515,
      "grad_norm": 24.90909396316388,
      "learning_rate": 5.511678308853025e-07,
      "logits": -1.7334787845611572,
      "logps": -118.51893615722656,
      "loss": 0.3343,
      "objective": 0.3072589933872223,
      "ranking_simple": 0.7208333611488342,
      "regularize": 0.3072589933872223,
      "step": 550,
      "wo_beta": 3.902778148651123
    },
    {
      "epoch": 1.5588096362777515,
      "eval_dpo_loss": 0.7814171314239502,
      "eval_logits": -1.8209288120269775,
      "eval_logps": -123.38627624511719,
      "eval_loss": 0.7698224186897278,
      "eval_objective": 0.7814171314239502,
      "eval_ranking_simple": 0.5351966619491577,
      "eval_regularize": 0.7814171314239502,
      "eval_runtime": 367.7587,
      "eval_samples_per_second": 15.744,
      "eval_steps_per_second": 1.313,
      "eval_wo_beta": 9.825753211975098,
      "step": 550
    },
    {
      "dpo_loss": 0.34100914001464844,
      "epoch": 1.5729806329711855,
      "grad_norm": 24.299530028848974,
      "learning_rate": 5.429372663441085e-07,
      "logits": -1.5191646814346313,
      "logps": -116.5060806274414,
      "loss": 0.339,
      "objective": 0.34100914001464844,
      "ranking_simple": 0.737500011920929,
      "regularize": 0.34100914001464844,
      "step": 555,
      "wo_beta": 3.8065884113311768
    },
    {
      "dpo_loss": 0.3122609257698059,
      "epoch": 1.5871516296646198,
      "grad_norm": 26.61780255104318,
      "learning_rate": 5.34694963183022e-07,
      "logits": -1.5900938510894775,
      "logps": -112.42914581298828,
      "loss": 0.334,
      "objective": 0.3122609257698059,
      "ranking_simple": 0.7666666507720947,
      "regularize": 0.3122609257698059,
      "step": 560,
      "wo_beta": 4.191233158111572
    },
    {
      "dpo_loss": 0.3086826205253601,
      "epoch": 1.601322626358054,
      "grad_norm": 23.97171206395212,
      "learning_rate": 5.264431747654283e-07,
      "logits": -1.5865463018417358,
      "logps": -112.55115509033203,
      "loss": 0.334,
      "objective": 0.3086826205253601,
      "ranking_simple": 0.7333333492279053,
      "regularize": 0.3086826205253601,
      "step": 565,
      "wo_beta": 3.760244846343994
    },
    {
      "dpo_loss": 0.3280898928642273,
      "epoch": 1.615493623051488,
      "grad_norm": 26.790227869518926,
      "learning_rate": 5.181841570478872e-07,
      "logits": -1.7293529510498047,
      "logps": -114.50617980957031,
      "loss": 0.3065,
      "objective": 0.3280898928642273,
      "ranking_simple": 0.7166666388511658,
      "regularize": 0.3280898928642273,
      "step": 570,
      "wo_beta": 4.093240737915039
    },
    {
      "dpo_loss": 0.356030136346817,
      "epoch": 1.629664619744922,
      "grad_norm": 29.38280925874381,
      "learning_rate": 5.099201679633768e-07,
      "logits": -1.7021836042404175,
      "logps": -112.18016052246094,
      "loss": 0.3442,
      "objective": 0.356030136346817,
      "ranking_simple": 0.7458333373069763,
      "regularize": 0.356030136346817,
      "step": 575,
      "wo_beta": 3.8104941844940186
    },
    {
      "dpo_loss": 0.2992390990257263,
      "epoch": 1.643835616438356,
      "grad_norm": 24.710654774900764,
      "learning_rate": 5.016534668039976e-07,
      "logits": -1.73283052444458,
      "logps": -111.73848724365234,
      "loss": 0.3113,
      "objective": 0.2992390990257263,
      "ranking_simple": 0.762499988079071,
      "regularize": 0.2992390990257263,
      "step": 580,
      "wo_beta": 2.8948888778686523
    },
    {
      "dpo_loss": 0.30101045966148376,
      "epoch": 1.6580066131317903,
      "grad_norm": 25.736134751129057,
      "learning_rate": 4.933863136033039e-07,
      "logits": -1.5684159994125366,
      "logps": -111.7412109375,
      "loss": 0.3245,
      "objective": 0.30101045966148376,
      "ranking_simple": 0.7124999761581421,
      "regularize": 0.30101045966148376,
      "step": 585,
      "wo_beta": 3.5469541549682617
    },
    {
      "dpo_loss": 0.29352518916130066,
      "epoch": 1.6721776098252243,
      "grad_norm": 28.292224202430326,
      "learning_rate": 4.851209685184338e-07,
      "logits": -1.6094284057617188,
      "logps": -114.35250854492188,
      "loss": 0.3264,
      "objective": 0.29352518916130066,
      "ranking_simple": 0.7458333373069763,
      "regularize": 0.29352518916130066,
      "step": 590,
      "wo_beta": 3.287980794906616
    },
    {
      "dpo_loss": 0.32477742433547974,
      "epoch": 1.6863486065186586,
      "grad_norm": 26.952229320357798,
      "learning_rate": 4.768596912122045e-07,
      "logits": -1.5896912813186646,
      "logps": -114.52155303955078,
      "loss": 0.3252,
      "objective": 0.32477742433547974,
      "ranking_simple": 0.7416666746139526,
      "regularize": 0.32477742433547974,
      "step": 595,
      "wo_beta": 4.611125469207764
    },
    {
      "dpo_loss": 0.2708142399787903,
      "epoch": 1.7005196032120926,
      "grad_norm": 26.06064163090054,
      "learning_rate": 4.686047402353433e-07,
      "logits": -1.6145151853561401,
      "logps": -114.16989135742188,
      "loss": 0.3105,
      "objective": 0.2708142399787903,
      "ranking_simple": 0.7708333134651184,
      "regularize": 0.2708142399787903,
      "step": 600,
      "wo_beta": 3.207538604736328
    },
    {
      "epoch": 1.7005196032120926,
      "eval_dpo_loss": 0.7761210799217224,
      "eval_logits": -1.7865931987762451,
      "eval_logps": -119.82308959960938,
      "eval_loss": 0.7679409980773926,
      "eval_objective": 0.7761210799217224,
      "eval_ranking_simple": 0.5383023023605347,
      "eval_regularize": 0.7761210799217224,
      "eval_runtime": 372.6791,
      "eval_samples_per_second": 15.536,
      "eval_steps_per_second": 1.296,
      "eval_wo_beta": 9.803099632263184,
      "step": 600
    },
    {
      "dpo_loss": 0.32439085841178894,
      "epoch": 1.7146905999055266,
      "grad_norm": 25.8504927528126,
      "learning_rate": 4.60358372409022e-07,
      "logits": -1.602834701538086,
      "logps": -112.0418472290039,
      "loss": 0.3292,
      "objective": 0.32439085841178894,
      "ranking_simple": 0.7250000238418579,
      "regularize": 0.32439085841178894,
      "step": 605,
      "wo_beta": 4.832021713256836
    },
    {
      "dpo_loss": 0.36444205045700073,
      "epoch": 1.7288615965989607,
      "grad_norm": 26.17653629180954,
      "learning_rate": 4.521228422078649e-07,
      "logits": -1.6547633409500122,
      "logps": -110.03471374511719,
      "loss": 0.3239,
      "objective": 0.36444205045700073,
      "ranking_simple": 0.75,
      "regularize": 0.36444205045700073,
      "step": 610,
      "wo_beta": 4.15641975402832
    },
    {
      "dpo_loss": 0.3479357659816742,
      "epoch": 1.743032593292395,
      "grad_norm": 25.45060113638119,
      "learning_rate": 4.439004011435979e-07,
      "logits": -1.6919087171554565,
      "logps": -109.51602172851562,
      "loss": 0.3279,
      "objective": 0.3479357659816742,
      "ranking_simple": 0.7250000238418579,
      "regularize": 0.3479357659816742,
      "step": 615,
      "wo_beta": 4.716867446899414
    },
    {
      "dpo_loss": 0.33088982105255127,
      "epoch": 1.7572035899858292,
      "grad_norm": 26.887147654271335,
      "learning_rate": 4.3569329714950703e-07,
      "logits": -1.7291035652160645,
      "logps": -109.82331085205078,
      "loss": 0.3425,
      "objective": 0.33088982105255127,
      "ranking_simple": 0.7583333253860474,
      "regularize": 0.33088982105255127,
      "step": 620,
      "wo_beta": 3.5512850284576416
    },
    {
      "dpo_loss": 0.3090989291667938,
      "epoch": 1.7713745866792632,
      "grad_norm": 26.389383350627952,
      "learning_rate": 4.275037739658771e-07,
      "logits": -1.590990424156189,
      "logps": -110.23678588867188,
      "loss": 0.3234,
      "objective": 0.3090989291667938,
      "ranking_simple": 0.7250000238418579,
      "regularize": 0.3090989291667938,
      "step": 625,
      "wo_beta": 3.666703462600708
    },
    {
      "dpo_loss": 0.33155128359794617,
      "epoch": 1.7855455833726972,
      "grad_norm": 29.336966725897334,
      "learning_rate": 4.193340705265745e-07,
      "logits": -1.635860562324524,
      "logps": -114.24417114257812,
      "loss": 0.3204,
      "objective": 0.33155128359794617,
      "ranking_simple": 0.7083333134651184,
      "regularize": 0.33155128359794617,
      "step": 630,
      "wo_beta": 4.568333625793457
    },
    {
      "dpo_loss": 0.3347407877445221,
      "epoch": 1.7997165800661312,
      "grad_norm": 31.313014707447742,
      "learning_rate": 4.1118642034694565e-07,
      "logits": -1.7257325649261475,
      "logps": -116.03430938720703,
      "loss": 0.3376,
      "objective": 0.3347407877445221,
      "ranking_simple": 0.7250000238418579,
      "regularize": 0.3347407877445221,
      "step": 635,
      "wo_beta": 4.352345943450928
    },
    {
      "dpo_loss": 0.30713585019111633,
      "epoch": 1.8138875767595655,
      "grad_norm": 25.70905071821337,
      "learning_rate": 4.030630509131959e-07,
      "logits": -1.6731877326965332,
      "logps": -116.4244155883789,
      "loss": 0.3098,
      "objective": 0.30713585019111633,
      "ranking_simple": 0.7583333253860474,
      "regularize": 0.30713585019111633,
      "step": 640,
      "wo_beta": 4.2585673332214355
    },
    {
      "dpo_loss": 0.29875513911247253,
      "epoch": 1.8280585734529995,
      "grad_norm": 23.791400324977506,
      "learning_rate": 3.9496618307341713e-07,
      "logits": -1.7462607622146606,
      "logps": -116.83419036865234,
      "loss": 0.3005,
      "objective": 0.29875513911247253,
      "ranking_simple": 0.7791666388511658,
      "regularize": 0.29875513911247253,
      "step": 645,
      "wo_beta": 3.3428430557250977
    },
    {
      "dpo_loss": 0.32320088148117065,
      "epoch": 1.8422295701464337,
      "grad_norm": 32.03839388955998,
      "learning_rate": 3.8689803043042996e-07,
      "logits": -1.8062669038772583,
      "logps": -117.27250671386719,
      "loss": 0.3412,
      "objective": 0.32320088148117065,
      "ranking_simple": 0.7583333253860474,
      "regularize": 0.32320088148117065,
      "step": 650,
      "wo_beta": 3.019827127456665
    },
    {
      "epoch": 1.8422295701464337,
      "eval_dpo_loss": 0.7847943305969238,
      "eval_logits": -1.832274079322815,
      "eval_logps": -122.29435729980469,
      "eval_loss": 0.7749964594841003,
      "eval_objective": 0.7847943305969238,
      "eval_ranking_simple": 0.5383023023605347,
      "eval_regularize": 0.7847943305969238,
      "eval_runtime": 369.3337,
      "eval_samples_per_second": 15.677,
      "eval_steps_per_second": 1.308,
      "eval_wo_beta": 9.94935417175293,
      "step": 650
    },
    {
      "dpo_loss": 0.3198917508125305,
      "epoch": 1.8564005668398678,
      "grad_norm": 24.453465336538663,
      "learning_rate": 3.788607987366069e-07,
      "logits": -1.6524808406829834,
      "logps": -113.18647766113281,
      "loss": 0.333,
      "objective": 0.3198917508125305,
      "ranking_simple": 0.737500011920929,
      "regularize": 0.3198917508125305,
      "step": 655,
      "wo_beta": 3.779681444168091
    },
    {
      "dpo_loss": 0.30827051401138306,
      "epoch": 1.8705715635333018,
      "grad_norm": 25.30249944675523,
      "learning_rate": 3.708566852908418e-07,
      "logits": -1.722990870475769,
      "logps": -117.13253021240234,
      "loss": 0.3093,
      "objective": 0.30827051401138306,
      "ranking_simple": 0.7250000238418579,
      "regularize": 0.30827051401138306,
      "step": 660,
      "wo_beta": 3.8630990982055664
    },
    {
      "dpo_loss": 0.30362004041671753,
      "epoch": 1.8847425602267358,
      "grad_norm": 26.577906210584395,
      "learning_rate": 3.6288787833783016e-07,
      "logits": -1.6925681829452515,
      "logps": -114.92183685302734,
      "loss": 0.3287,
      "objective": 0.30362004041671753,
      "ranking_simple": 0.7749999761581421,
      "regularize": 0.30362004041671753,
      "step": 665,
      "wo_beta": 3.2407257556915283
    },
    {
      "dpo_loss": 0.33854812383651733,
      "epoch": 1.89891355692017,
      "grad_norm": 26.608491612605135,
      "learning_rate": 3.5495655646982503e-07,
      "logits": -1.632662057876587,
      "logps": -113.74718475341797,
      "loss": 0.3206,
      "objective": 0.33854812383651733,
      "ranking_simple": 0.7124999761581421,
      "regularize": 0.33854812383651733,
      "step": 670,
      "wo_beta": 4.751885890960693
    },
    {
      "dpo_loss": 0.30450791120529175,
      "epoch": 1.9130845536136043,
      "grad_norm": 27.707963800825,
      "learning_rate": 3.470648880310313e-07,
      "logits": -1.7001540660858154,
      "logps": -112.93498992919922,
      "loss": 0.3286,
      "objective": 0.30450791120529175,
      "ranking_simple": 0.800000011920929,
      "regularize": 0.30450791120529175,
      "step": 675,
      "wo_beta": 2.8013789653778076
    },
    {
      "dpo_loss": 0.29233846068382263,
      "epoch": 1.9272555503070383,
      "grad_norm": 26.318074274927174,
      "learning_rate": 3.3921503052480236e-07,
      "logits": -1.7435904741287231,
      "logps": -115.07746887207031,
      "loss": 0.3108,
      "objective": 0.29233846068382263,
      "ranking_simple": 0.7833333611488342,
      "regularize": 0.29233846068382263,
      "step": 680,
      "wo_beta": 3.6125741004943848
    },
    {
      "dpo_loss": 0.2992376685142517,
      "epoch": 1.9414265470004723,
      "grad_norm": 26.980724381093548,
      "learning_rate": 3.314091300237999e-07,
      "logits": -1.6790062189102173,
      "logps": -115.0765380859375,
      "loss": 0.2818,
      "objective": 0.2992376685142517,
      "ranking_simple": 0.7333333492279053,
      "regularize": 0.2992376685142517,
      "step": 685,
      "wo_beta": 4.020833969116211
    },
    {
      "dpo_loss": 0.2989169955253601,
      "epoch": 1.9555975436939064,
      "grad_norm": 28.956498459281256,
      "learning_rate": 3.236493205832794e-07,
      "logits": -1.7138378620147705,
      "logps": -116.5325927734375,
      "loss": 0.3112,
      "objective": 0.2989169955253601,
      "ranking_simple": 0.762499988079071,
      "regularize": 0.2989169955253601,
      "step": 690,
      "wo_beta": 3.235595941543579
    },
    {
      "dpo_loss": 0.3340277373790741,
      "epoch": 1.9697685403873406,
      "grad_norm": 29.008978725819244,
      "learning_rate": 3.15937723657661e-07,
      "logits": -1.5779744386672974,
      "logps": -117.33056640625,
      "loss": 0.3178,
      "objective": 0.3340277373790741,
      "ranking_simple": 0.7416666746139526,
      "regularize": 0.3340277373790741,
      "step": 695,
      "wo_beta": 4.477265357971191
    },
    {
      "dpo_loss": 0.3072899878025055,
      "epoch": 1.9839395370807746,
      "grad_norm": 26.91852361479357,
      "learning_rate": 3.082764475205442e-07,
      "logits": -1.5716139078140259,
      "logps": -120.74311065673828,
      "loss": 0.3156,
      "objective": 0.3072899878025055,
      "ranking_simple": 0.7583333253860474,
      "regularize": 0.3072899878025055,
      "step": 700,
      "wo_beta": 3.845552682876587
    },
    {
      "epoch": 1.9839395370807746,
      "eval_dpo_loss": 0.813927948474884,
      "eval_logits": -1.8338414430618286,
      "eval_logps": -126.39391326904297,
      "eval_loss": 0.8013313412666321,
      "eval_objective": 0.813927948474884,
      "eval_ranking_simple": 0.5377846956253052,
      "eval_regularize": 0.813927948474884,
      "eval_runtime": 375.3584,
      "eval_samples_per_second": 15.425,
      "eval_steps_per_second": 1.287,
      "eval_wo_beta": 10.32465934753418,
      "step": 700
    },
    {
      "dpo_loss": 0.3367713391780853,
      "epoch": 1.9981105337742089,
      "grad_norm": 30.634677985646626,
      "learning_rate": 3.006675866883275e-07,
      "logits": -1.486984133720398,
      "logps": -120.58149719238281,
      "loss": 0.3202,
      "objective": 0.3367713391780853,
      "ranking_simple": 0.699999988079071,
      "regularize": 0.3367713391780853,
      "step": 705,
      "wo_beta": 5.604025840759277
    },
    {
      "dpo_loss": 0.26506486535072327,
      "epoch": 2.012281530467643,
      "grad_norm": 21.811542358240928,
      "learning_rate": 2.931132213475884e-07,
      "logits": -1.652250051498413,
      "logps": -117.85489654541016,
      "loss": 0.247,
      "objective": 0.26506486535072327,
      "ranking_simple": 0.7958333492279053,
      "regularize": 0.26506486535072327,
      "step": 710,
      "wo_beta": 2.941725015640259
    },
    {
      "dpo_loss": 0.21083328127861023,
      "epoch": 2.026452527161077,
      "grad_norm": 22.009413556660768,
      "learning_rate": 2.856154167863814e-07,
      "logits": -1.7095146179199219,
      "logps": -118.10084533691406,
      "loss": 0.2213,
      "objective": 0.21083328127861023,
      "ranking_simple": 0.8166666626930237,
      "regularize": 0.21083328127861023,
      "step": 715,
      "wo_beta": 1.996678113937378
    },
    {
      "dpo_loss": 0.1858675181865692,
      "epoch": 2.040623523854511,
      "grad_norm": 19.31295851177097,
      "learning_rate": 2.7817622282960813e-07,
      "logits": -1.6148954629898071,
      "logps": -123.07051086425781,
      "loss": 0.2017,
      "objective": 0.1858675181865692,
      "ranking_simple": 0.824999988079071,
      "regularize": 0.1858675181865692,
      "step": 720,
      "wo_beta": 2.2542519569396973
    },
    {
      "dpo_loss": 0.2097151279449463,
      "epoch": 2.0547945205479454,
      "grad_norm": 24.65512776088815,
      "learning_rate": 2.707976732786166e-07,
      "logits": -1.686496376991272,
      "logps": -126.76118469238281,
      "loss": 0.2047,
      "objective": 0.2097151279449463,
      "ranking_simple": 0.8083333373069763,
      "regularize": 0.2097151279449463,
      "step": 725,
      "wo_beta": 2.5733823776245117
    },
    {
      "dpo_loss": 0.23622439801692963,
      "epoch": 2.0689655172413794,
      "grad_norm": 27.027861051451126,
      "learning_rate": 2.6348178535517965e-07,
      "logits": -1.7227706909179688,
      "logps": -126.69216918945312,
      "loss": 0.2175,
      "objective": 0.23622439801692963,
      "ranking_simple": 0.7749999761581421,
      "regularize": 0.23622439801692963,
      "step": 730,
      "wo_beta": 3.341784715652466
    },
    {
      "dpo_loss": 0.23194490373134613,
      "epoch": 2.0831365139348135,
      "grad_norm": 29.247659239837198,
      "learning_rate": 2.5623055915000686e-07,
      "logits": -1.6128872632980347,
      "logps": -126.96920776367188,
      "loss": 0.2248,
      "objective": 0.23194490373134613,
      "ranking_simple": 0.7875000238418579,
      "regularize": 0.23194490373134613,
      "step": 735,
      "wo_beta": 3.5192384719848633
    },
    {
      "dpo_loss": 0.2231958657503128,
      "epoch": 2.0973075106282475,
      "grad_norm": 25.19144911793584,
      "learning_rate": 2.490459770759398e-07,
      "logits": -1.6792032718658447,
      "logps": -123.05906677246094,
      "loss": 0.2328,
      "objective": 0.2231958657503128,
      "ranking_simple": 0.8125,
      "regularize": 0.2231958657503128,
      "step": 740,
      "wo_beta": 2.758615016937256
    },
    {
      "dpo_loss": 0.20981092751026154,
      "epoch": 2.1114785073216815,
      "grad_norm": 23.2038862143533,
      "learning_rate": 2.419300033259798e-07,
      "logits": -1.6360180377960205,
      "logps": -124.81270599365234,
      "loss": 0.2182,
      "objective": 0.20981092751026154,
      "ranking_simple": 0.7791666388511658,
      "regularize": 0.20981092751026154,
      "step": 745,
      "wo_beta": 3.1798312664031982
    },
    {
      "dpo_loss": 0.22788210213184357,
      "epoch": 2.1256495040151155,
      "grad_norm": 22.72550221909295,
      "learning_rate": 2.3488458333629773e-07,
      "logits": -1.6554747819900513,
      "logps": -122.24414825439453,
      "loss": 0.2183,
      "objective": 0.22788210213184357,
      "ranking_simple": 0.8041666746139526,
      "regularize": 0.22788210213184357,
      "step": 750,
      "wo_beta": 2.6969196796417236
    },
    {
      "epoch": 2.1256495040151155,
      "eval_dpo_loss": 0.8603518009185791,
      "eval_logits": -1.7998822927474976,
      "eval_logps": -131.12570190429688,
      "eval_loss": 0.8466524481773376,
      "eval_objective": 0.8603518009185791,
      "eval_ranking_simple": 0.5351966619491577,
      "eval_regularize": 0.8603518009185791,
      "eval_runtime": 370.8383,
      "eval_samples_per_second": 15.613,
      "eval_steps_per_second": 1.302,
      "eval_wo_beta": 10.893115997314453,
      "step": 750
    },
    {
      "dpo_loss": 0.2121828943490982,
      "epoch": 2.13982050070855,
      "grad_norm": 20.710190502520984,
      "learning_rate": 2.2791164325437046e-07,
      "logits": -1.6399970054626465,
      "logps": -126.25770568847656,
      "loss": 0.2128,
      "objective": 0.2121828943490982,
      "ranking_simple": 0.8125,
      "regularize": 0.2121828943490982,
      "step": 755,
      "wo_beta": 3.0046989917755127
    },
    {
      "dpo_loss": 0.24169200658798218,
      "epoch": 2.153991497401984,
      "grad_norm": 22.67213482006544,
      "learning_rate": 2.21013089412392e-07,
      "logits": -1.6009403467178345,
      "logps": -119.49600219726562,
      "loss": 0.2344,
      "objective": 0.24169200658798218,
      "ranking_simple": 0.7916666865348816,
      "regularize": 0.24169200658798218,
      "step": 760,
      "wo_beta": 2.758802890777588
    },
    {
      "dpo_loss": 0.2243640273809433,
      "epoch": 2.168162494095418,
      "grad_norm": 20.64816775047569,
      "learning_rate": 2.1419080780610122e-07,
      "logits": -1.6553146839141846,
      "logps": -120.87618255615234,
      "loss": 0.2361,
      "objective": 0.2243640273809433,
      "ranking_simple": 0.8208333253860474,
      "regularize": 0.2243640273809433,
      "step": 765,
      "wo_beta": 2.57806396484375
    },
    {
      "dpo_loss": 0.2182272970676422,
      "epoch": 2.182333490788852,
      "grad_norm": 24.316819536544447,
      "learning_rate": 2.0744666357916925e-07,
      "logits": -1.6933544874191284,
      "logps": -120.17996978759766,
      "loss": 0.2274,
      "objective": 0.2182272970676422,
      "ranking_simple": 0.7916666865348816,
      "regularize": 0.2182272970676422,
      "step": 770,
      "wo_beta": 2.544562816619873
    },
    {
      "dpo_loss": 0.20803479850292206,
      "epoch": 2.196504487482286,
      "grad_norm": 23.36541458579555,
      "learning_rate": 2.0078250051328782e-07,
      "logits": -1.6226826906204224,
      "logps": -122.02224731445312,
      "loss": 0.2183,
      "objective": 0.20803479850292206,
      "ranking_simple": 0.8208333253860474,
      "regularize": 0.20803479850292206,
      "step": 775,
      "wo_beta": 2.7029988765716553
    },
    {
      "dpo_loss": 0.22727744281291962,
      "epoch": 2.21067548417572,
      "grad_norm": 24.538113581222706,
      "learning_rate": 1.942001405240979e-07,
      "logits": -1.5558067560195923,
      "logps": -124.58464813232422,
      "loss": 0.2171,
      "objective": 0.22727744281291962,
      "ranking_simple": 0.7541666626930237,
      "regularize": 0.22727744281291962,
      "step": 780,
      "wo_beta": 3.0669195652008057
    },
    {
      "dpo_loss": 0.21583274006843567,
      "epoch": 2.2248464808691546,
      "grad_norm": 28.544460286246334,
      "learning_rate": 1.877013831630961e-07,
      "logits": -1.5640733242034912,
      "logps": -123.09439849853516,
      "loss": 0.2152,
      "objective": 0.21583274006843567,
      "ranking_simple": 0.7958333492279053,
      "regularize": 0.21583274006843567,
      "step": 785,
      "wo_beta": 3.0127415657043457
    },
    {
      "dpo_loss": 0.1936068832874298,
      "epoch": 2.2390174775625886,
      "grad_norm": 23.5808144420256,
      "learning_rate": 1.812880051256551e-07,
      "logits": -1.5408331155776978,
      "logps": -125.1629638671875,
      "loss": 0.2122,
      "objective": 0.1936068832874298,
      "ranking_simple": 0.8583333492279053,
      "regularize": 0.1936068832874298,
      "step": 790,
      "wo_beta": 2.2015647888183594
    },
    {
      "dpo_loss": 0.21609367430210114,
      "epoch": 2.2531884742560226,
      "grad_norm": 23.539955858336363,
      "learning_rate": 1.7496175976529337e-07,
      "logits": -1.6351306438446045,
      "logps": -125.9599609375,
      "loss": 0.2139,
      "objective": 0.21609367430210114,
      "ranking_simple": 0.8374999761581421,
      "regularize": 0.21609367430210114,
      "step": 795,
      "wo_beta": 2.052849292755127
    },
    {
      "dpo_loss": 0.2304428219795227,
      "epoch": 2.2673594709494567,
      "grad_norm": 30.254441621642492,
      "learning_rate": 1.6872437661432516e-07,
      "logits": -1.6573865413665771,
      "logps": -127.17088317871094,
      "loss": 0.2338,
      "objective": 0.2304428219795227,
      "ranking_simple": 0.8291666507720947,
      "regularize": 0.2304428219795227,
      "step": 800,
      "wo_beta": 2.813727617263794
    },
    {
      "epoch": 2.2673594709494567,
      "eval_dpo_loss": 0.864130973815918,
      "eval_logits": -1.8069664239883423,
      "eval_logps": -132.1160125732422,
      "eval_loss": 0.8479817509651184,
      "eval_objective": 0.864130973815918,
      "eval_ranking_simple": 0.5351966619491577,
      "eval_regularize": 0.864130973815918,
      "eval_runtime": 369.7595,
      "eval_samples_per_second": 15.659,
      "eval_steps_per_second": 1.306,
      "eval_wo_beta": 10.98104476928711,
      "step": 800
    },
    {
      "dpo_loss": 0.24076558649539948,
      "epoch": 2.2815304676428907,
      "grad_norm": 22.143907441363098,
      "learning_rate": 1.62577560911024e-07,
      "logits": -1.645892858505249,
      "logps": -124.22594451904297,
      "loss": 0.2325,
      "objective": 0.24076558649539948,
      "ranking_simple": 0.762499988079071,
      "regularize": 0.24076558649539948,
      "step": 805,
      "wo_beta": 3.4595048427581787
    },
    {
      "dpo_loss": 0.2532159686088562,
      "epoch": 2.295701464336325,
      "grad_norm": 22.775287720493225,
      "learning_rate": 1.565229931334277e-07,
      "logits": -1.7806832790374756,
      "logps": -121.71713256835938,
      "loss": 0.229,
      "objective": 0.2532159686088562,
      "ranking_simple": 0.8041666746139526,
      "regularize": 0.2532159686088562,
      "step": 810,
      "wo_beta": 2.6662535667419434
    },
    {
      "dpo_loss": 0.21587124466896057,
      "epoch": 2.309872461029759,
      "grad_norm": 20.40928105754396,
      "learning_rate": 1.5056232853991208e-07,
      "logits": -1.6817148923873901,
      "logps": -123.68048858642578,
      "loss": 0.1989,
      "objective": 0.21587124466896057,
      "ranking_simple": 0.800000011920929,
      "regularize": 0.21587124466896057,
      "step": 815,
      "wo_beta": 2.549884557723999
    },
    {
      "dpo_loss": 0.21313165128231049,
      "epoch": 2.324043457723193,
      "grad_norm": 24.789515132235113,
      "learning_rate": 1.4469719671666043e-07,
      "logits": -1.6495434045791626,
      "logps": -124.32107543945312,
      "loss": 0.2149,
      "objective": 0.21313165128231049,
      "ranking_simple": 0.8083333373069763,
      "regularize": 0.21313165128231049,
      "step": 820,
      "wo_beta": 2.404456377029419
    },
    {
      "dpo_loss": 0.21146027743816376,
      "epoch": 2.3382144544166272,
      "grad_norm": 21.660632425555523,
      "learning_rate": 1.389292011321498e-07,
      "logits": -1.692821979522705,
      "logps": -122.83853149414062,
      "loss": 0.2168,
      "objective": 0.21146027743816376,
      "ranking_simple": 0.8166666626930237,
      "regularize": 0.21146027743816376,
      "step": 825,
      "wo_beta": 2.4975247383117676
    },
    {
      "dpo_loss": 0.19700536131858826,
      "epoch": 2.3523854511100613,
      "grad_norm": 27.701358075237533,
      "learning_rate": 1.3325991869878012e-07,
      "logits": -1.6795495748519897,
      "logps": -126.65140533447266,
      "loss": 0.1967,
      "objective": 0.19700536131858826,
      "ranking_simple": 0.8374999761581421,
      "regularize": 0.19700536131858826,
      "step": 830,
      "wo_beta": 2.146075487136841
    },
    {
      "dpo_loss": 0.21794618666172028,
      "epoch": 2.3665564478034957,
      "grad_norm": 29.238202346662906,
      "learning_rate": 1.2769089934176126e-07,
      "logits": -1.647661566734314,
      "logps": -125.17839813232422,
      "loss": 0.2288,
      "objective": 0.21794618666172028,
      "ranking_simple": 0.8416666388511658,
      "regularize": 0.21794618666172028,
      "step": 835,
      "wo_beta": 2.8974695205688477
    },
    {
      "dpo_loss": 0.21900226175785065,
      "epoch": 2.3807274444969297,
      "grad_norm": 24.009821339334245,
      "learning_rate": 1.222236655753791e-07,
      "logits": -1.6093765497207642,
      "logps": -125.18759155273438,
      "loss": 0.2184,
      "objective": 0.21900226175785065,
      "ranking_simple": 0.7833333611488342,
      "regularize": 0.21900226175785065,
      "step": 840,
      "wo_beta": 2.749152898788452
    },
    {
      "dpo_loss": 0.24857226014137268,
      "epoch": 2.3948984411903638,
      "grad_norm": 27.5079137044431,
      "learning_rate": 1.1685971208675538e-07,
      "logits": -1.6249310970306396,
      "logps": -127.47116088867188,
      "loss": 0.2262,
      "objective": 0.24857226014137268,
      "ranking_simple": 0.7916666865348816,
      "regularize": 0.24857226014137268,
      "step": 845,
      "wo_beta": 2.7568535804748535
    },
    {
      "dpo_loss": 0.2085387259721756,
      "epoch": 2.409069437883798,
      "grad_norm": 25.878018469347772,
      "learning_rate": 1.1160050532721527e-07,
      "logits": -1.67723548412323,
      "logps": -125.76563262939453,
      "loss": 0.2015,
      "objective": 0.2085387259721756,
      "ranking_simple": 0.7958333492279053,
      "regularize": 0.2085387259721756,
      "step": 850,
      "wo_beta": 2.600330352783203
    },
    {
      "epoch": 2.409069437883798,
      "eval_dpo_loss": 0.8720477223396301,
      "eval_logits": -1.80176842212677,
      "eval_logps": -133.381103515625,
      "eval_loss": 0.8571510910987854,
      "eval_objective": 0.8720477223396301,
      "eval_ranking_simple": 0.5377846956253052,
      "eval_regularize": 0.8720477223396301,
      "eval_runtime": 369.6782,
      "eval_samples_per_second": 15.662,
      "eval_steps_per_second": 1.307,
      "eval_wo_beta": 11.025221824645996,
      "step": 850
    },
    {
      "dpo_loss": 0.2096593677997589,
      "epoch": 2.423240434577232,
      "grad_norm": 25.499456648461173,
      "learning_rate": 1.0644748311137375e-07,
      "logits": -1.6905667781829834,
      "logps": -123.53242492675781,
      "loss": 0.2195,
      "objective": 0.2096593677997589,
      "ranking_simple": 0.8458333611488342,
      "regularize": 0.2096593677997589,
      "step": 855,
      "wo_beta": 1.9907649755477905
    },
    {
      "dpo_loss": 0.21979431807994843,
      "epoch": 2.4374114312706663,
      "grad_norm": 24.848581285581997,
      "learning_rate": 1.0140205422405212e-07,
      "logits": -1.6141736507415771,
      "logps": -127.53914642333984,
      "loss": 0.2157,
      "objective": 0.21979431807994843,
      "ranking_simple": 0.7916666865348816,
      "regularize": 0.21979431807994843,
      "step": 860,
      "wo_beta": 3.0604774951934814
    },
    {
      "dpo_loss": 0.2267284095287323,
      "epoch": 2.4515824279641003,
      "grad_norm": 26.5384738263413,
      "learning_rate": 9.646559803512993e-08,
      "logits": -1.6312936544418335,
      "logps": -125.63304138183594,
      "loss": 0.2262,
      "objective": 0.2267284095287323,
      "ranking_simple": 0.7708333134651184,
      "regularize": 0.2267284095287323,
      "step": 865,
      "wo_beta": 3.2560784816741943
    },
    {
      "dpo_loss": 0.196714848279953,
      "epoch": 2.4657534246575343,
      "grad_norm": 22.187392253095393,
      "learning_rate": 9.163946412243895e-08,
      "logits": -1.6977574825286865,
      "logps": -125.96756744384766,
      "loss": 0.2037,
      "objective": 0.196714848279953,
      "ranking_simple": 0.824999988079071,
      "regularize": 0.196714848279953,
      "step": 870,
      "wo_beta": 2.583970308303833
    },
    {
      "dpo_loss": 0.1924598515033722,
      "epoch": 2.4799244213509684,
      "grad_norm": 25.12579870814084,
      "learning_rate": 8.692497190280224e-08,
      "logits": -1.5863568782806396,
      "logps": -126.04496765136719,
      "loss": 0.2038,
      "objective": 0.1924598515033722,
      "ranking_simple": 0.8583333492279053,
      "regularize": 0.1924598515033722,
      "step": 875,
      "wo_beta": 2.296090602874756
    },
    {
      "dpo_loss": 0.2359813004732132,
      "epoch": 2.4940954180444024,
      "grad_norm": 23.176269104907163,
      "learning_rate": 8.232341027131883e-08,
      "logits": -1.5722264051437378,
      "logps": -125.54722595214844,
      "loss": 0.2224,
      "objective": 0.2359813004732132,
      "ranking_simple": 0.7666666507720947,
      "regularize": 0.2359813004732132,
      "step": 880,
      "wo_beta": 3.9758193492889404
    },
    {
      "dpo_loss": 0.2542850375175476,
      "epoch": 2.5082664147378364,
      "grad_norm": 28.503952001031568,
      "learning_rate": 7.783603724899257e-08,
      "logits": -1.724973201751709,
      "logps": -123.99037170410156,
      "loss": 0.2326,
      "objective": 0.2542850375175476,
      "ranking_simple": 0.7958333492279053,
      "regularize": 0.2542850375175476,
      "step": 885,
      "wo_beta": 3.509568929672241
    },
    {
      "dpo_loss": 0.20098893344402313,
      "epoch": 2.5224374114312704,
      "grad_norm": 24.488143250342084,
      "learning_rate": 7.346407963880136e-08,
      "logits": -1.595066785812378,
      "logps": -123.90083312988281,
      "loss": 0.2056,
      "objective": 0.20098893344402313,
      "ranking_simple": 0.8291666507720947,
      "regularize": 0.20098893344402313,
      "step": 890,
      "wo_beta": 2.0867483615875244
    },
    {
      "dpo_loss": 0.22675950825214386,
      "epoch": 2.536608408124705,
      "grad_norm": 26.1383864365769,
      "learning_rate": 6.92087326903022e-08,
      "logits": -1.6463298797607422,
      "logps": -126.22193908691406,
      "loss": 0.2235,
      "objective": 0.22675950825214386,
      "ranking_simple": 0.7833333611488342,
      "regularize": 0.22675950825214386,
      "step": 895,
      "wo_beta": 3.0455052852630615
    },
    {
      "dpo_loss": 0.26797404885292053,
      "epoch": 2.550779404818139,
      "grad_norm": 29.177118299023412,
      "learning_rate": 6.507115977286143e-08,
      "logits": -1.5577763319015503,
      "logps": -123.61392974853516,
      "loss": 0.2348,
      "objective": 0.26797404885292053,
      "ranking_simple": 0.7749999761581421,
      "regularize": 0.26797404885292053,
      "step": 900,
      "wo_beta": 3.1443541049957275
    },
    {
      "epoch": 2.550779404818139,
      "eval_dpo_loss": 0.8675441145896912,
      "eval_logits": -1.8113691806793213,
      "eval_logps": -133.67962646484375,
      "eval_loss": 0.8529655933380127,
      "eval_objective": 0.8675441145896912,
      "eval_ranking_simple": 0.5377846956253052,
      "eval_regularize": 0.8675441145896912,
      "eval_runtime": 369.0633,
      "eval_samples_per_second": 15.688,
      "eval_steps_per_second": 1.309,
      "eval_wo_beta": 10.942305564880371,
      "step": 900
    },
    {
      "dpo_loss": 0.20284312963485718,
      "epoch": 2.564950401511573,
      "grad_norm": 19.950697729864736,
      "learning_rate": 6.105249205760127e-08,
      "logits": -1.6392539739608765,
      "logps": -123.07585906982422,
      "loss": 0.2053,
      "objective": 0.20284312963485718,
      "ranking_simple": 0.8333333134651184,
      "regularize": 0.20284312963485718,
      "step": 905,
      "wo_beta": 2.37481951713562
    },
    {
      "dpo_loss": 0.23266027867794037,
      "epoch": 2.579121398205007,
      "grad_norm": 30.055735152469072,
      "learning_rate": 5.7153828208148846e-08,
      "logits": -1.6491973400115967,
      "logps": -127.52837371826172,
      "loss": 0.217,
      "objective": 0.23266027867794037,
      "ranking_simple": 0.800000011920929,
      "regularize": 0.23266027867794037,
      "step": 910,
      "wo_beta": 3.36558198928833
    },
    {
      "dpo_loss": 0.23302534222602844,
      "epoch": 2.593292394898441,
      "grad_norm": 22.890822851586893,
      "learning_rate": 5.337623408027292e-08,
      "logits": -1.7552512884140015,
      "logps": -127.87796783447266,
      "loss": 0.2121,
      "objective": 0.23302534222602844,
      "ranking_simple": 0.8208333253860474,
      "regularize": 0.23302534222602844,
      "step": 915,
      "wo_beta": 2.502667188644409
    },
    {
      "dpo_loss": 0.20162495970726013,
      "epoch": 2.6074633915918755,
      "grad_norm": 24.20273693189473,
      "learning_rate": 4.972074243048896e-08,
      "logits": -1.5814868211746216,
      "logps": -127.01163482666016,
      "loss": 0.2068,
      "objective": 0.20162495970726013,
      "ranking_simple": 0.8333333134651184,
      "regularize": 0.20162495970726013,
      "step": 920,
      "wo_beta": 1.8146071434020996
    },
    {
      "dpo_loss": 0.21291884779930115,
      "epoch": 2.6216343882853095,
      "grad_norm": 22.388349767286606,
      "learning_rate": 4.6188352633713956e-08,
      "logits": -1.5830769538879395,
      "logps": -126.23300170898438,
      "loss": 0.2195,
      "objective": 0.21291884779930115,
      "ranking_simple": 0.800000011920929,
      "regularize": 0.21291884779930115,
      "step": 925,
      "wo_beta": 3.1612021923065186
    },
    {
      "dpo_loss": 0.24175626039505005,
      "epoch": 2.6358053849787435,
      "grad_norm": 21.949168917598172,
      "learning_rate": 4.2780030410047796e-08,
      "logits": -1.6357704401016235,
      "logps": -125.97001647949219,
      "loss": 0.2179,
      "objective": 0.24175626039505005,
      "ranking_simple": 0.7708333134651184,
      "regularize": 0.24175626039505005,
      "step": 930,
      "wo_beta": 3.436295747756958
    },
    {
      "dpo_loss": 0.22544310986995697,
      "epoch": 2.6499763816721775,
      "grad_norm": 28.54777226492267,
      "learning_rate": 3.949670756075446e-08,
      "logits": -1.6567264795303345,
      "logps": -123.90028381347656,
      "loss": 0.2428,
      "objective": 0.22544310986995697,
      "ranking_simple": 0.8125,
      "regularize": 0.22544310986995697,
      "step": 935,
      "wo_beta": 2.002594470977783
    },
    {
      "dpo_loss": 0.19537684321403503,
      "epoch": 2.6641473783656116,
      "grad_norm": 27.292491316004774,
      "learning_rate": 3.63392817135173e-08,
      "logits": -1.6852660179138184,
      "logps": -128.0804901123047,
      "loss": 0.2163,
      "objective": 0.19537684321403503,
      "ranking_simple": 0.8083333373069763,
      "regularize": 0.19537684321403503,
      "step": 940,
      "wo_beta": 3.2341361045837402
    },
    {
      "dpo_loss": 0.21889939904212952,
      "epoch": 2.678318375059046,
      "grad_norm": 23.60505865361295,
      "learning_rate": 3.330861607703611e-08,
      "logits": -1.7046397924423218,
      "logps": -125.53500366210938,
      "loss": 0.206,
      "objective": 0.21889939904212952,
      "ranking_simple": 0.7875000238418579,
      "regularize": 0.21889939904212952,
      "step": 945,
      "wo_beta": 3.0100159645080566
    },
    {
      "dpo_loss": 0.20888622105121613,
      "epoch": 2.69248937175248,
      "grad_norm": 24.640589732429273,
      "learning_rate": 3.040553920503502e-08,
      "logits": -1.5867191553115845,
      "logps": -123.99597930908203,
      "loss": 0.2268,
      "objective": 0.20888622105121613,
      "ranking_simple": 0.8208333253860474,
      "regularize": 0.20888622105121613,
      "step": 950,
      "wo_beta": 2.38991379737854
    },
    {
      "epoch": 2.69248937175248,
      "eval_dpo_loss": 0.8684250116348267,
      "eval_logits": -1.8135783672332764,
      "eval_logps": -133.2828826904297,
      "eval_loss": 0.8525474667549133,
      "eval_objective": 0.8684250116348267,
      "eval_ranking_simple": 0.533643901348114,
      "eval_regularize": 0.8684250116348267,
      "eval_runtime": 373.8008,
      "eval_samples_per_second": 15.49,
      "eval_steps_per_second": 1.292,
      "eval_wo_beta": 10.9784517288208,
      "step": 950
    },
    {
      "dpo_loss": 0.21907640993595123,
      "epoch": 2.706660368445914,
      "grad_norm": 25.73877923734523,
      "learning_rate": 2.7630844769743756e-08,
      "logits": -1.696962833404541,
      "logps": -125.86216735839844,
      "loss": 0.217,
      "objective": 0.21907640993595123,
      "ranking_simple": 0.8333333134651184,
      "regularize": 0.21907640993595123,
      "step": 955,
      "wo_beta": 2.471001148223877
    },
    {
      "dpo_loss": 0.24159465730190277,
      "epoch": 2.720831365139348,
      "grad_norm": 27.398682783569342,
      "learning_rate": 2.4985291344915673e-08,
      "logits": -1.6559653282165527,
      "logps": -125.80078125,
      "loss": 0.2243,
      "objective": 0.24159465730190277,
      "ranking_simple": 0.8041666746139526,
      "regularize": 0.24159465730190277,
      "step": 960,
      "wo_beta": 3.4911081790924072
    },
    {
      "dpo_loss": 0.17974473536014557,
      "epoch": 2.735002361832782,
      "grad_norm": 26.129978647933363,
      "learning_rate": 2.2469602198441573e-08,
      "logits": -1.6600605249404907,
      "logps": -125.82157897949219,
      "loss": 0.2216,
      "objective": 0.17974473536014557,
      "ranking_simple": 0.8458333611488342,
      "regularize": 0.17974473536014557,
      "step": 965,
      "wo_beta": 1.8301441669464111
    },
    {
      "dpo_loss": 0.2389921396970749,
      "epoch": 2.7491733585262166,
      "grad_norm": 24.93627247502558,
      "learning_rate": 2.008446509461498e-08,
      "logits": -1.6814639568328857,
      "logps": -127.4894027709961,
      "loss": 0.2119,
      "objective": 0.2389921396970749,
      "ranking_simple": 0.8333333134651184,
      "regularize": 0.2389921396970749,
      "step": 970,
      "wo_beta": 2.071624517440796
    },
    {
      "dpo_loss": 0.21195697784423828,
      "epoch": 2.7633443552196506,
      "grad_norm": 24.386661793540505,
      "learning_rate": 1.7830532106104746e-08,
      "logits": -1.6035431623458862,
      "logps": -124.46774291992188,
      "loss": 0.2154,
      "objective": 0.21195697784423828,
      "ranking_simple": 0.8333333134651184,
      "regularize": 0.21195697784423828,
      "step": 975,
      "wo_beta": 3.027677059173584
    },
    {
      "dpo_loss": 0.2177601158618927,
      "epoch": 2.7775153519130846,
      "grad_norm": 30.722672013551914,
      "learning_rate": 1.570841943568446e-08,
      "logits": -1.7569483518600464,
      "logps": -126.10541534423828,
      "loss": 0.2135,
      "objective": 0.2177601158618927,
      "ranking_simple": 0.824999988079071,
      "regularize": 0.2177601158618927,
      "step": 980,
      "wo_beta": 1.981053113937378
    },
    {
      "dpo_loss": 0.21660226583480835,
      "epoch": 2.7916863486065187,
      "grad_norm": 24.268020278235447,
      "learning_rate": 1.3718707247769134e-08,
      "logits": -1.5750867128372192,
      "logps": -122.41696166992188,
      "loss": 0.2142,
      "objective": 0.21660226583480835,
      "ranking_simple": 0.8208333253860474,
      "regularize": 0.21660226583480835,
      "step": 985,
      "wo_beta": 2.762291431427002
    },
    {
      "dpo_loss": 0.23230423033237457,
      "epoch": 2.8058573452999527,
      "grad_norm": 25.3957119119687,
      "learning_rate": 1.1861939509803686e-08,
      "logits": -1.626227617263794,
      "logps": -125.03184509277344,
      "loss": 0.2309,
      "objective": 0.23230423033237457,
      "ranking_simple": 0.7916666865348816,
      "regularize": 0.23230423033237457,
      "step": 990,
      "wo_beta": 3.499976396560669
    },
    {
      "dpo_loss": 0.22918492555618286,
      "epoch": 2.820028341993387,
      "grad_norm": 26.589296858875272,
      "learning_rate": 1.0138623843548078e-08,
      "logits": -1.691062092781067,
      "logps": -126.53192901611328,
      "loss": 0.2269,
      "objective": 0.22918492555618286,
      "ranking_simple": 0.7916666865348816,
      "regularize": 0.22918492555618286,
      "step": 995,
      "wo_beta": 3.2248499393463135
    },
    {
      "dpo_loss": 0.22611786425113678,
      "epoch": 2.8341993386868207,
      "grad_norm": 23.891964045353923,
      "learning_rate": 8.54923138629815e-09,
      "logits": -1.6503469944000244,
      "logps": -122.57675170898438,
      "loss": 0.2198,
      "objective": 0.22611786425113678,
      "ranking_simple": 0.8333333134651184,
      "regularize": 0.22611786425113678,
      "step": 1000,
      "wo_beta": 2.427152395248413
    },
    {
      "epoch": 2.8341993386868207,
      "eval_dpo_loss": 0.8652148246765137,
      "eval_logits": -1.8167296648025513,
      "eval_logps": -132.88087463378906,
      "eval_loss": 0.8492961525917053,
      "eval_objective": 0.8652148246765137,
      "eval_ranking_simple": 0.5341615080833435,
      "eval_regularize": 0.8652148246765137,
      "eval_runtime": 368.2335,
      "eval_samples_per_second": 15.724,
      "eval_steps_per_second": 1.312,
      "eval_wo_beta": 10.938254356384277,
      "step": 1000
    },
    {
      "dpo_loss": 0.24608786404132843,
      "epoch": 2.848370335380255,
      "grad_norm": 27.602968631540815,
      "learning_rate": 7.09419666208183e-09,
      "logits": -1.6333565711975098,
      "logps": -123.5007553100586,
      "loss": 0.2273,
      "objective": 0.24608786404132843,
      "ranking_simple": 0.7916666865348816,
      "regularize": 0.24608786404132843,
      "step": 1005,
      "wo_beta": 3.1694223880767822
    },
    {
      "dpo_loss": 0.20009997487068176,
      "epoch": 2.862541332073689,
      "grad_norm": 25.090299269603783,
      "learning_rate": 5.773917462864264e-09,
      "logits": -1.6704093217849731,
      "logps": -125.71635437011719,
      "loss": 0.2037,
      "objective": 0.20009997487068176,
      "ranking_simple": 0.800000011920929,
      "regularize": 0.20009997487068176,
      "step": 1010,
      "wo_beta": 2.577218532562256
    },
    {
      "dpo_loss": 0.21036744117736816,
      "epoch": 2.8767123287671232,
      "grad_norm": 28.329606007932064,
      "learning_rate": 4.588754739795586e-09,
      "logits": -1.6348304748535156,
      "logps": -122.55280303955078,
      "loss": 0.2113,
      "objective": 0.21036744117736816,
      "ranking_simple": 0.8083333373069763,
      "regularize": 0.21036744117736816,
      "step": 1015,
      "wo_beta": 2.199587345123291
    },
    {
      "dpo_loss": 0.21461248397827148,
      "epoch": 2.8908833254605573,
      "grad_norm": 24.62375043047653,
      "learning_rate": 3.53903250453047e-09,
      "logits": -1.5652154684066772,
      "logps": -125.25798797607422,
      "loss": 0.2252,
      "objective": 0.21461248397827148,
      "ranking_simple": 0.7708333134651184,
      "regularize": 0.21461248397827148,
      "step": 1020,
      "wo_beta": 3.251589059829712
    },
    {
      "dpo_loss": 0.23392058908939362,
      "epoch": 2.9050543221539913,
      "grad_norm": 25.1548357787363,
      "learning_rate": 2.6250377406467627e-09,
      "logits": -1.6649322509765625,
      "logps": -124.19398498535156,
      "loss": 0.2291,
      "objective": 0.23392058908939362,
      "ranking_simple": 0.8083333373069763,
      "regularize": 0.23392058908939362,
      "step": 1025,
      "wo_beta": 2.934882879257202
    },
    {
      "dpo_loss": 0.19529716670513153,
      "epoch": 2.9192253188474258,
      "grad_norm": 23.978894569817292,
      "learning_rate": 1.8470203251865768e-09,
      "logits": -1.7404934167861938,
      "logps": -123.60317993164062,
      "loss": 0.2177,
      "objective": 0.19529716670513153,
      "ranking_simple": 0.7666666507720947,
      "regularize": 0.19529716670513153,
      "step": 1030,
      "wo_beta": 2.584693670272827
    },
    {
      "dpo_loss": 0.19891007244586945,
      "epoch": 2.9333963155408598,
      "grad_norm": 22.489637587674896,
      "learning_rate": 1.2051929603428823e-09,
      "logits": -1.6503539085388184,
      "logps": -126.50735473632812,
      "loss": 0.2124,
      "objective": 0.19891007244586945,
      "ranking_simple": 0.8125,
      "regularize": 0.19891007244586945,
      "step": 1035,
      "wo_beta": 2.176048755645752
    },
    {
      "dpo_loss": 0.21993538737297058,
      "epoch": 2.947567312234294,
      "grad_norm": 23.591827890954175,
      "learning_rate": 6.997311153086882e-10,
      "logits": -1.6743519306182861,
      "logps": -125.27057647705078,
      "loss": 0.2151,
      "objective": 0.21993538737297058,
      "ranking_simple": 0.800000011920929,
      "regularize": 0.21993538737297058,
      "step": 1040,
      "wo_beta": 3.0266594886779785
    },
    {
      "dpo_loss": 0.22178266942501068,
      "epoch": 2.961738308927728,
      "grad_norm": 26.41640596804321,
      "learning_rate": 3.3077297830541585e-10,
      "logits": -1.6230467557907104,
      "logps": -128.82781982421875,
      "loss": 0.2256,
      "objective": 0.22178266942501068,
      "ranking_simple": 0.8041666746139526,
      "regularize": 0.22178266942501068,
      "step": 1045,
      "wo_beta": 3.258082866668701
    },
    {
      "dpo_loss": 0.21878859400749207,
      "epoch": 2.975909305621162,
      "grad_norm": 25.001515767941786,
      "learning_rate": 9.841941880361914e-11,
      "logits": -1.6594524383544922,
      "logps": -121.61707305908203,
      "loss": 0.2221,
      "objective": 0.21878859400749207,
      "ranking_simple": 0.7958333492279053,
      "regularize": 0.21878859400749207,
      "step": 1050,
      "wo_beta": 2.6373982429504395
    },
    {
      "epoch": 2.975909305621162,
      "eval_dpo_loss": 0.8653033375740051,
      "eval_logits": -1.816504955291748,
      "eval_logps": -132.85667419433594,
      "eval_loss": 0.8493290543556213,
      "eval_objective": 0.8653033375740051,
      "eval_ranking_simple": 0.534679114818573,
      "eval_regularize": 0.8653033375740051,
      "eval_runtime": 369.7292,
      "eval_samples_per_second": 15.66,
      "eval_steps_per_second": 1.306,
      "eval_wo_beta": 10.94184398651123,
      "step": 1050
    },
    {
      "dpo_loss": 0.22742925584316254,
      "epoch": 2.9900803023145963,
      "grad_norm": 25.264967316936268,
      "learning_rate": 2.7339599464326622e-12,
      "logits": -1.6433926820755005,
      "logps": -123.5830078125,
      "loss": 0.2221,
      "objective": 0.22742925584316254,
      "ranking_simple": 0.800000011920929,
      "regularize": 0.22742925584316254,
      "step": 1055,
      "wo_beta": 2.7176921367645264
    },
    {
      "epoch": 2.992914501653283,
      "step": 1056,
      "total_flos": 0.0,
      "train_loss": 0.37872021553586377,
      "train_runtime": 34657.2213,
      "train_samples_per_second": 4.398,
      "train_steps_per_second": 0.03
    }
  ],
  "logging_steps": 5,
  "max_steps": 1056,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}