{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 100,
  "global_step": 1065,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "dpo_losses": 0.6931471824645996,
      "epoch": 0.0,
      "grad_norm": 2.1182166269339366,
      "learning_rate": 4.672897196261682e-08,
      "logits/chosen": -2.8477635383605957,
      "logits/rejected": -2.8469698429107666,
      "logps/chosen": -522.6112670898438,
      "logps/rejected": -359.48583984375,
      "loss": 0.6931,
      "positive_losses": 0.0,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/margins_max": 0.0,
      "rewards/margins_min": 0.0,
      "rewards/margins_std": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "dpo_losses": 0.6931488513946533,
      "epoch": 0.03,
      "grad_norm": 18.90606759984328,
      "learning_rate": 4.6728971962616824e-07,
      "logits/chosen": -2.921452522277832,
      "logits/rejected": -2.7972779273986816,
      "logps/chosen": -313.4413146972656,
      "logps/rejected": -170.33502197265625,
      "loss": 0.698,
      "positive_losses": 0.056756019592285156,
      "rewards/accuracies": 0.5555555820465088,
      "rewards/chosen": 0.00029136601369827986,
      "rewards/margins": -2.1007015220675385e-06,
      "rewards/margins_max": 0.0011362881632521749,
      "rewards/margins_min": -0.001140489592216909,
      "rewards/margins_std": 0.0016099249478429556,
      "rewards/rejected": 0.0002934667863883078,
      "step": 10
    },
    {
      "dpo_losses": 0.6920409202575684,
      "epoch": 0.06,
      "grad_norm": 2.048573611790903,
      "learning_rate": 9.345794392523365e-07,
      "logits/chosen": -2.7631053924560547,
      "logits/rejected": -2.7107467651367188,
      "logps/chosen": -380.45953369140625,
      "logps/rejected": -243.90365600585938,
      "loss": 0.6928,
      "positive_losses": 0.02124938927590847,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.006503595970571041,
      "rewards/margins": 0.0022183754481375217,
      "rewards/margins_max": 0.004751545377075672,
      "rewards/margins_min": -0.00031479448080062866,
      "rewards/margins_std": 0.0035824428778141737,
      "rewards/rejected": 0.004285220988094807,
      "step": 20
    },
    {
      "dpo_losses": 0.6899846792221069,
      "epoch": 0.08,
      "grad_norm": 2.2814863944700297,
      "learning_rate": 1.4018691588785047e-06,
      "logits/chosen": -2.87530779838562,
      "logits/rejected": -2.823071002960205,
      "logps/chosen": -374.14593505859375,
      "logps/rejected": -251.3402099609375,
      "loss": 0.6884,
      "positive_losses": 0.0,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.01957734487950802,
      "rewards/margins": 0.0063508180901408195,
      "rewards/margins_max": 0.011502384208142757,
      "rewards/margins_min": 0.0011992522049695253,
      "rewards/margins_std": 0.007285414729267359,
      "rewards/rejected": 0.013226528652012348,
      "step": 30
    },
    {
      "dpo_losses": 0.6816079616546631,
      "epoch": 0.11,
      "grad_norm": 1.9338584641637362,
      "learning_rate": 1.869158878504673e-06,
      "logits/chosen": -2.732996940612793,
      "logits/rejected": -2.7668612003326416,
      "logps/chosen": -302.9008483886719,
      "logps/rejected": -315.843505859375,
      "loss": 0.6807,
      "positive_losses": 0.0,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.04024823382496834,
      "rewards/margins": 0.023371612653136253,
      "rewards/margins_max": 0.03446139022707939,
      "rewards/margins_min": 0.01228183414787054,
      "rewards/margins_std": 0.015683313831686974,
      "rewards/rejected": 0.016876617446541786,
      "step": 40
    },
    {
      "dpo_losses": 0.6704033613204956,
      "epoch": 0.14,
      "grad_norm": 2.241837849751868,
      "learning_rate": 2.3364485981308413e-06,
      "logits/chosen": -2.7844934463500977,
      "logits/rejected": -2.696681022644043,
      "logps/chosen": -238.6322784423828,
      "logps/rejected": -172.29949951171875,
      "loss": 0.6657,
      "positive_losses": 0.0,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.06590737402439117,
      "rewards/margins": 0.046577345579862595,
      "rewards/margins_max": 0.06979880481958389,
      "rewards/margins_min": 0.023355895653367043,
      "rewards/margins_std": 0.03284009173512459,
      "rewards/rejected": 0.019330020993947983,
      "step": 50
    },
    {
      "dpo_losses": 0.6440416574478149,
      "epoch": 0.17,
      "grad_norm": 2.210137428049826,
      "learning_rate": 2.8037383177570094e-06,
      "logits/chosen": -2.7359554767608643,
      "logits/rejected": -2.6721925735473633,
      "logps/chosen": -255.8380584716797,
      "logps/rejected": -234.24276733398438,
      "loss": 0.6442,
      "positive_losses": 0.0,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.11597372591495514,
      "rewards/margins": 0.10343287885189056,
      "rewards/margins_max": 0.15465359389781952,
      "rewards/margins_min": 0.0522121861577034,
      "rewards/margins_std": 0.07243702560663223,
      "rewards/rejected": 0.012540824711322784,
      "step": 60
    },
    {
      "dpo_losses": 0.6189829111099243,
      "epoch": 0.2,
      "grad_norm": 1.721316554071705,
      "learning_rate": 3.2710280373831774e-06,
      "logits/chosen": -2.6506264209747314,
      "logits/rejected": -2.65425443649292,
      "logps/chosen": -316.86090087890625,
      "logps/rejected": -213.46066284179688,
      "loss": 0.6213,
      "positive_losses": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17079836130142212,
      "rewards/margins": 0.15738530457019806,
      "rewards/margins_max": 0.23612920939922333,
      "rewards/margins_min": 0.078641377389431,
      "rewards/margins_std": 0.11136071383953094,
      "rewards/rejected": 0.013413062319159508,
      "step": 70
    },
    {
      "dpo_losses": 0.59584641456604,
      "epoch": 0.23,
      "grad_norm": 5.833494928937326,
      "learning_rate": 3.738317757009346e-06,
      "logits/chosen": -2.8657782077789307,
      "logits/rejected": -2.7893776893615723,
      "logps/chosen": -318.1582946777344,
      "logps/rejected": -289.5111999511719,
      "loss": 0.6052,
      "positive_losses": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18891094624996185,
      "rewards/margins": 0.2127668410539627,
      "rewards/margins_max": 0.30381911993026733,
      "rewards/margins_min": 0.12171456962823868,
      "rewards/margins_std": 0.12876734137535095,
      "rewards/rejected": -0.023855898529291153,
      "step": 80
    },
    {
      "dpo_losses": 0.5957463383674622,
      "epoch": 0.25,
      "grad_norm": 2.2476761791524478,
      "learning_rate": 4.205607476635514e-06,
      "logits/chosen": -2.668989658355713,
      "logits/rejected": -2.6688218116760254,
      "logps/chosen": -256.0438537597656,
      "logps/rejected": -203.91128540039062,
      "loss": 0.5735,
      "positive_losses": 0.0,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.20931395888328552,
      "rewards/margins": 0.21628114581108093,
      "rewards/margins_max": 0.35886088013648987,
      "rewards/margins_min": 0.07370143383741379,
      "rewards/margins_std": 0.20163817703723907,
      "rewards/rejected": -0.00696719903498888,
      "step": 90
    },
    {
      "dpo_losses": 0.5217684507369995,
      "epoch": 0.28,
      "grad_norm": 1.9962825797807737,
      "learning_rate": 4.6728971962616825e-06,
      "logits/chosen": -2.8356316089630127,
      "logits/rejected": -2.7704596519470215,
      "logps/chosen": -424.61712646484375,
      "logps/rejected": -347.96246337890625,
      "loss": 0.5432,
      "positive_losses": 0.040345001965761185,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.336413711309433,
      "rewards/margins": 0.3945561945438385,
      "rewards/margins_max": 0.5599964261054993,
      "rewards/margins_min": 0.22911591827869415,
      "rewards/margins_std": 0.2339678257703781,
      "rewards/rejected": -0.058142442256212234,
      "step": 100
    },
    {
      "epoch": 0.28,
      "eval_dpo_losses": 0.6723248958587646,
      "eval_logits/chosen": -2.7099392414093018,
      "eval_logits/rejected": -2.6677489280700684,
      "eval_logps/chosen": -290.291748046875,
      "eval_logps/rejected": -269.3353576660156,
      "eval_loss": 1.5490143299102783,
      "eval_positive_losses": 8.368339538574219,
      "eval_rewards/accuracies": 0.5992063283920288,
      "eval_rewards/chosen": -0.05070570856332779,
      "eval_rewards/margins": 0.05082136392593384,
      "eval_rewards/margins_max": 0.25667881965637207,
      "eval_rewards/margins_min": -0.14138472080230713,
      "eval_rewards/margins_std": 0.17566871643066406,
      "eval_rewards/rejected": -0.10152707248926163,
      "eval_runtime": 282.2019,
      "eval_samples_per_second": 7.087,
      "eval_steps_per_second": 0.223,
      "step": 100
    },
    {
      "dpo_losses": 0.5300568342208862,
      "epoch": 0.31,
      "grad_norm": 2.4142164656542064,
      "learning_rate": 4.999879018839288e-06,
      "logits/chosen": -2.694153070449829,
      "logits/rejected": -2.5792508125305176,
      "logps/chosen": -301.54388427734375,
      "logps/rejected": -258.5564270019531,
      "loss": 0.5372,
      "positive_losses": 0.238017275929451,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.26155275106430054,
      "rewards/margins": 0.3829374313354492,
      "rewards/margins_max": 0.5613982677459717,
      "rewards/margins_min": 0.20447655022144318,
      "rewards/margins_std": 0.2523817718029022,
      "rewards/rejected": -0.12138471752405167,
      "step": 110
    },
    {
      "dpo_losses": 0.488577663898468,
      "epoch": 0.34,
      "grad_norm": 9.3268800860026,
      "learning_rate": 4.99772856836941e-06,
      "logits/chosen": -2.8129353523254395,
      "logits/rejected": -2.79107403755188,
      "logps/chosen": -337.3607177734375,
      "logps/rejected": -347.9256286621094,
      "loss": 0.5066,
      "positive_losses": 0.0004665374872274697,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.345800518989563,
      "rewards/margins": 0.5089500546455383,
      "rewards/margins_max": 0.725378692150116,
      "rewards/margins_min": 0.29252126812934875,
      "rewards/margins_std": 0.3060764968395233,
      "rewards/rejected": -0.16314946115016937,
      "step": 120
    },
    {
      "dpo_losses": 0.42630523443222046,
      "epoch": 0.37,
      "grad_norm": 6.838233431406268,
      "learning_rate": 4.992892309373227e-06,
      "logits/chosen": -2.64375376701355,
      "logits/rejected": -2.5765233039855957,
      "logps/chosen": -352.48126220703125,
      "logps/rejected": -287.73388671875,
      "loss": 0.4592,
      "positive_losses": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3849102854728699,
      "rewards/margins": 0.679017186164856,
      "rewards/margins_max": 0.8494507670402527,
      "rewards/margins_min": 0.5085835456848145,
      "rewards/margins_std": 0.24102959036827087,
      "rewards/rejected": -0.2941069006919861,
      "step": 130
    },
    {
      "dpo_losses": 0.4316105842590332,
      "epoch": 0.39,
      "grad_norm": 24.291983530934388,
      "learning_rate": 4.985375442281969e-06,
      "logits/chosen": -2.6181836128234863,
      "logits/rejected": -2.5858638286590576,
      "logps/chosen": -320.45770263671875,
      "logps/rejected": -268.80950927734375,
      "loss": 0.4791,
      "positive_losses": 0.11783752590417862,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.3075354993343353,
      "rewards/margins": 0.6785004138946533,
      "rewards/margins_max": 0.9275323748588562,
      "rewards/margins_min": 0.42946839332580566,
      "rewards/margins_std": 0.35218438506126404,
      "rewards/rejected": -0.37096482515335083,
      "step": 140
    },
    {
      "dpo_losses": 0.43038830161094666,
      "epoch": 0.42,
      "grad_norm": 16.31539388542976,
      "learning_rate": 4.9751860499858175e-06,
      "logits/chosen": -2.6524252891540527,
      "logits/rejected": -2.582724094390869,
      "logps/chosen": -269.40289306640625,
      "logps/rejected": -286.25921630859375,
      "loss": 0.4405,
      "positive_losses": 0.45142823457717896,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.27419334650039673,
      "rewards/margins": 0.6590893268585205,
      "rewards/margins_max": 0.8764106631278992,
      "rewards/margins_min": 0.44176802039146423,
      "rewards/margins_std": 0.307338684797287,
      "rewards/rejected": -0.3848959803581238,
      "step": 150
    },
    {
      "dpo_losses": 0.40865588188171387,
      "epoch": 0.45,
      "grad_norm": 9.196914105554178,
      "learning_rate": 4.962335089142376e-06,
      "logits/chosen": -2.671811580657959,
      "logits/rejected": -2.557887554168701,
      "logps/chosen": -298.1056823730469,
      "logps/rejected": -286.0025329589844,
      "loss": 0.4273,
      "positive_losses": 1.3531090021133423,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3361397683620453,
      "rewards/margins": 0.7648395299911499,
      "rewards/margins_max": 1.1010096073150635,
      "rewards/margins_min": 0.42866945266723633,
      "rewards/margins_std": 0.47541624307632446,
      "rewards/rejected": -0.4286997318267822,
      "step": 160
    },
    {
      "dpo_losses": 0.36792081594467163,
      "epoch": 0.48,
      "grad_norm": 3.4842547066626572,
      "learning_rate": 4.946836378394967e-06,
      "logits/chosen": -2.719280242919922,
      "logits/rejected": -2.560034990310669,
      "logps/chosen": -358.5220642089844,
      "logps/rejected": -287.3617858886719,
      "loss": 0.4612,
      "positive_losses": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4195915162563324,
      "rewards/margins": 0.9577536582946777,
      "rewards/margins_max": 1.3000526428222656,
      "rewards/margins_min": 0.6154545545578003,
      "rewards/margins_std": 0.48408395051956177,
      "rewards/rejected": -0.5381620526313782,
      "step": 170
    },
    {
      "dpo_losses": 0.3783532977104187,
      "epoch": 0.51,
      "grad_norm": 2.2304162655616295,
      "learning_rate": 4.928706583513441e-06,
      "logits/chosen": -2.599997043609619,
      "logits/rejected": -2.5286357402801514,
      "logps/chosen": -297.5575866699219,
      "logps/rejected": -439.0782165527344,
      "loss": 0.4178,
      "positive_losses": 0.7186470031738281,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.26578420400619507,
      "rewards/margins": 0.8594935536384583,
      "rewards/margins_max": 1.1510531902313232,
      "rewards/margins_min": 0.5679339170455933,
      "rewards/margins_std": 0.4123275876045227,
      "rewards/rejected": -0.5937093496322632,
      "step": 180
    },
    {
      "dpo_losses": 0.3120216727256775,
      "epoch": 0.54,
      "grad_norm": 2.502842717440193,
      "learning_rate": 4.907965199473471e-06,
      "logits/chosen": -2.5457229614257812,
      "logits/rejected": -2.3784427642822266,
      "logps/chosen": -403.69097900390625,
      "logps/rejected": -277.13079833984375,
      "loss": 0.4291,
      "positive_losses": 0.1399887055158615,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.524212658405304,
      "rewards/margins": 1.0702520608901978,
      "rewards/margins_max": 1.2478736639022827,
      "rewards/margins_min": 0.892630398273468,
      "rewards/margins_std": 0.25119495391845703,
      "rewards/rejected": -0.546039342880249,
      "step": 190
    },
    {
      "dpo_losses": 0.340221643447876,
      "epoch": 0.56,
      "grad_norm": 39.20326200653621,
      "learning_rate": 4.884634529493591e-06,
      "logits/chosen": -2.686732769012451,
      "logits/rejected": -2.5898447036743164,
      "logps/chosen": -290.22259521484375,
      "logps/rejected": -267.71923828125,
      "loss": 0.4843,
      "positive_losses": 1.2471590042114258,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.405730664730072,
      "rewards/margins": 1.0374891757965088,
      "rewards/margins_max": 1.3196719884872437,
      "rewards/margins_min": 0.7553063035011292,
      "rewards/margins_std": 0.3990669250488281,
      "rewards/rejected": -0.6317585110664368,
      "step": 200
    },
    {
      "epoch": 0.56,
      "eval_dpo_losses": 0.6414885520935059,
      "eval_logits/chosen": -2.6000208854675293,
      "eval_logits/rejected": -2.5589327812194824,
      "eval_logps/chosen": -310.5942687988281,
      "eval_logps/rejected": -302.1485595703125,
      "eval_loss": 3.6353535652160645,
      "eval_positive_losses": 28.93216323852539,
      "eval_rewards/accuracies": 0.6349206566810608,
      "eval_rewards/chosen": -0.2537309527397156,
      "eval_rewards/margins": 0.17592783272266388,
      "eval_rewards/margins_max": 0.7364258766174316,
      "eval_rewards/margins_min": -0.35331711173057556,
      "eval_rewards/margins_std": 0.48577553033828735,
      "eval_rewards/rejected": -0.42965877056121826,
      "eval_runtime": 280.7482,
      "eval_samples_per_second": 7.124,
      "eval_steps_per_second": 0.224,
      "step": 200
    },
    {
      "dpo_losses": 0.4035143256187439,
      "epoch": 0.59,
      "grad_norm": 2.0960462134704727,
      "learning_rate": 4.858739661052539e-06,
      "logits/chosen": -2.449962854385376,
      "logits/rejected": -2.364846706390381,
      "logps/chosen": -316.26300048828125,
      "logps/rejected": -311.43115234375,
      "loss": 0.3993,
      "positive_losses": 0.28444308042526245,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.4057347774505615,
      "rewards/margins": 0.8319534063339233,
      "rewards/margins_max": 1.267531156539917,
      "rewards/margins_min": 0.3963755965232849,
      "rewards/margins_std": 0.6160000562667847,
      "rewards/rejected": -0.4262186586856842,
      "step": 210
    },
    {
      "dpo_losses": 0.2952147424221039,
      "epoch": 0.62,
      "grad_norm": 2.161286043094922,
      "learning_rate": 4.830308438912687e-06,
      "logits/chosen": -2.7107901573181152,
      "logits/rejected": -2.560708522796631,
      "logps/chosen": -364.57659912109375,
      "logps/rejected": -361.52545166015625,
      "loss": 0.3728,
      "positive_losses": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4469282031059265,
      "rewards/margins": 1.2792309522628784,
      "rewards/margins_max": 1.8157857656478882,
      "rewards/margins_min": 0.7426761388778687,
      "rewards/margins_std": 0.7588031888008118,
      "rewards/rejected": -0.8323026895523071,
      "step": 220
    },
    {
      "dpo_losses": 0.26847127079963684,
      "epoch": 0.65,
      "grad_norm": 14.213158348490241,
      "learning_rate": 4.799371435178544e-06,
      "logits/chosen": -2.694364070892334,
      "logits/rejected": -2.611956834793091,
      "logps/chosen": -353.65203857421875,
      "logps/rejected": -447.9458923339844,
      "loss": 0.4538,
      "positive_losses": 0.512774646282196,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2710634768009186,
      "rewards/margins": 1.3846604824066162,
      "rewards/margins_max": 1.9593979120254517,
      "rewards/margins_min": 0.809922993183136,
      "rewards/margins_std": 0.8128012418746948,
      "rewards/rejected": -1.113596796989441,
      "step": 230
    },
    {
      "dpo_losses": 0.3169272541999817,
      "epoch": 0.68,
      "grad_norm": 19.218706332521528,
      "learning_rate": 4.765961916422575e-06,
      "logits/chosen": -2.664609670639038,
      "logits/rejected": -2.5199811458587646,
      "logps/chosen": -309.8558349609375,
      "logps/rejected": -378.1949157714844,
      "loss": 0.4789,
      "positive_losses": 2.6823112964630127,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.3129195272922516,
      "rewards/margins": 1.156752586364746,
      "rewards/margins_max": 1.4397625923156738,
      "rewards/margins_min": 0.8737425804138184,
      "rewards/margins_std": 0.4002366065979004,
      "rewards/rejected": -0.8438330888748169,
      "step": 240
    },
    {
      "dpo_losses": 0.2593916356563568,
      "epoch": 0.7,
      "grad_norm": 10.943604253629541,
      "learning_rate": 4.730115807913627e-06,
      "logits/chosen": -2.6513824462890625,
      "logits/rejected": -2.482832431793213,
      "logps/chosen": -369.9505920410156,
      "logps/rejected": -348.71759033203125,
      "loss": 0.3659,
      "positive_losses": 0.7899643182754517,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.45276421308517456,
      "rewards/margins": 1.3846436738967896,
      "rewards/margins_max": 1.6470266580581665,
      "rewards/margins_min": 1.1222608089447021,
      "rewards/margins_std": 0.3710656762123108,
      "rewards/rejected": -0.9318795204162598,
      "step": 250
    },
    {
      "dpo_losses": 0.3340635895729065,
      "epoch": 0.73,
      "grad_norm": 12.721859863392186,
      "learning_rate": 4.691871654986485e-06,
      "logits/chosen": -2.592149019241333,
      "logits/rejected": -2.553668975830078,
      "logps/chosen": -290.64971923828125,
      "logps/rejected": -312.02752685546875,
      "loss": 0.5868,
      "positive_losses": 1.0232231616973877,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3117257356643677,
      "rewards/margins": 1.1895887851715088,
      "rewards/margins_max": 1.7240755558013916,
      "rewards/margins_min": 0.6551022529602051,
      "rewards/margins_std": 0.755878210067749,
      "rewards/rejected": -0.8778631091117859,
      "step": 260
    },
    {
      "dpo_losses": 0.2871754467487335,
      "epoch": 0.76,
      "grad_norm": 20.74321957766599,
      "learning_rate": 4.651270581594054e-06,
      "logits/chosen": -2.6253020763397217,
      "logits/rejected": -2.486907482147217,
      "logps/chosen": -381.5191955566406,
      "logps/rejected": -312.0382995605469,
      "loss": 0.4914,
      "positive_losses": 0.877484142780304,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4747743606567383,
      "rewards/margins": 1.2697714567184448,
      "rewards/margins_max": 1.6828867197036743,
      "rewards/margins_min": 0.8566561937332153,
      "rewards/margins_std": 0.5842332243919373,
      "rewards/rejected": -0.7949970960617065,
      "step": 270
    },
    {
      "dpo_losses": 0.325231671333313,
      "epoch": 0.79,
      "grad_norm": 24.63939637076414,
      "learning_rate": 4.6083562460867545e-06,
      "logits/chosen": -2.4967987537384033,
      "logits/rejected": -2.4599080085754395,
      "logps/chosen": -295.0630798339844,
      "logps/rejected": -352.3880615234375,
      "loss": 0.4194,
      "positive_losses": 1.5066649913787842,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.34013646841049194,
      "rewards/margins": 1.2071036100387573,
      "rewards/margins_max": 1.8113868236541748,
      "rewards/margins_min": 0.6028203964233398,
      "rewards/margins_std": 0.8545855283737183,
      "rewards/rejected": -0.8669670820236206,
      "step": 280
    },
    {
      "dpo_losses": 0.29185962677001953,
      "epoch": 0.82,
      "grad_norm": 49.4876607292477,
      "learning_rate": 4.563174794266684e-06,
      "logits/chosen": -2.725825071334839,
      "logits/rejected": -2.5793709754943848,
      "logps/chosen": -316.76153564453125,
      "logps/rejected": -357.36334228515625,
      "loss": 0.4944,
      "positive_losses": 1.7035411596298218,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.45084109902381897,
      "rewards/margins": 1.3104431629180908,
      "rewards/margins_max": 1.7685630321502686,
      "rewards/margins_min": 0.8523231744766235,
      "rewards/margins_std": 0.6478795409202576,
      "rewards/rejected": -0.859602153301239,
      "step": 290
    },
    {
      "dpo_losses": 0.24671301245689392,
      "epoch": 0.85,
      "grad_norm": 38.65216195794793,
      "learning_rate": 4.5157748097670125e-06,
      "logits/chosen": -2.6136484146118164,
      "logits/rejected": -2.5544955730438232,
      "logps/chosen": -294.6103820800781,
      "logps/rejected": -422.00531005859375,
      "loss": 0.2828,
      "positive_losses": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.43132978677749634,
      "rewards/margins": 1.5242297649383545,
      "rewards/margins_max": 1.9953060150146484,
      "rewards/margins_min": 1.0531535148620605,
      "rewards/margins_std": 0.6662023067474365,
      "rewards/rejected": -1.092900037765503,
      "step": 300
    },
    {
      "epoch": 0.85,
      "eval_dpo_losses": 0.6346198320388794,
      "eval_logits/chosen": -2.5758955478668213,
      "eval_logits/rejected": -2.534869909286499,
      "eval_logps/chosen": -345.25262451171875,
      "eval_logps/rejected": -344.2117004394531,
      "eval_loss": 6.804558753967285,
      "eval_positive_losses": 61.76890182495117,
      "eval_rewards/accuracies": 0.6507936716079712,
      "eval_rewards/chosen": -0.6003143787384033,
      "eval_rewards/margins": 0.24997644126415253,
      "eval_rewards/margins_max": 1.0084712505340576,
      "eval_rewards/margins_min": -0.48679542541503906,
      "eval_rewards/margins_std": 0.6679101586341858,
      "eval_rewards/rejected": -0.8502907156944275,
      "eval_runtime": 281.0888,
      "eval_samples_per_second": 7.115,
      "eval_steps_per_second": 0.224,
      "step": 300
    },
    {
      "dpo_losses": 0.2701479196548462,
      "epoch": 0.87,
      "grad_norm": 2.3545752201762977,
      "learning_rate": 4.466207261809989e-06,
      "logits/chosen": -2.7792999744415283,
      "logits/rejected": -2.558169364929199,
      "logps/chosen": -376.7308044433594,
      "logps/rejected": -358.2815246582031,
      "loss": 0.4216,
      "positive_losses": 0.41081467270851135,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.47516530752182007,
      "rewards/margins": 1.4481419324874878,
      "rewards/margins_max": 1.9792436361312866,
      "rewards/margins_min": 0.9170401692390442,
      "rewards/margins_std": 0.7510912418365479,
      "rewards/rejected": -0.972976565361023,
      "step": 310
    },
    {
      "dpo_losses": 0.2924317717552185,
      "epoch": 0.9,
      "grad_norm": 12.007334692835927,
      "learning_rate": 4.414525450399713e-06,
      "logits/chosen": -2.6015143394470215,
      "logits/rejected": -2.5146775245666504,
      "logps/chosen": -315.1227111816406,
      "logps/rejected": -350.9872131347656,
      "loss": 0.3361,
      "positive_losses": 2.5894155502319336,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.3113010823726654,
      "rewards/margins": 1.4402223825454712,
      "rewards/margins_max": 2.083007335662842,
      "rewards/margins_min": 0.7974374890327454,
      "rewards/margins_std": 0.9090349078178406,
      "rewards/rejected": -1.128921389579773,
      "step": 320
    },
    {
      "dpo_losses": 0.2676263153553009,
      "epoch": 0.93,
      "grad_norm": 29.603619163509805,
      "learning_rate": 4.360784949008615e-06,
      "logits/chosen": -2.705920934677124,
      "logits/rejected": -2.5720715522766113,
      "logps/chosen": -338.70623779296875,
      "logps/rejected": -376.7459716796875,
      "loss": 0.4384,
      "positive_losses": 2.4138710498809814,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.40980419516563416,
      "rewards/margins": 1.7029374837875366,
      "rewards/margins_max": 2.3179752826690674,
      "rewards/margins_min": 1.087899923324585,
      "rewards/margins_std": 0.8697945475578308,
      "rewards/rejected": -1.2931334972381592,
      "step": 330
    },
    {
      "dpo_losses": 0.3142815828323364,
      "epoch": 0.96,
      "grad_norm": 18.502276306325797,
      "learning_rate": 4.30504354481929e-06,
      "logits/chosen": -2.5686209201812744,
      "logits/rejected": -2.424360990524292,
      "logps/chosen": -333.12530517578125,
      "logps/rejected": -292.06707763671875,
      "loss": 0.3621,
      "positive_losses": 0.5738517642021179,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3537023365497589,
      "rewards/margins": 1.2313454151153564,
      "rewards/margins_max": 1.7322998046875,
      "rewards/margins_min": 0.7303910255432129,
      "rewards/margins_std": 0.7084565758705139,
      "rewards/rejected": -0.8776431083679199,
      "step": 340
    },
    {
      "dpo_losses": 0.18595710396766663,
      "epoch": 0.99,
      "grad_norm": 137.9970915114699,
      "learning_rate": 4.247361176585904e-06,
      "logits/chosen": -2.5403497219085693,
      "logits/rejected": -2.4443917274475098,
      "logps/chosen": -386.3079833984375,
      "logps/rejected": -441.9620056152344,
      "loss": 0.4456,
      "positive_losses": 0.9461520910263062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4209546446800232,
      "rewards/margins": 1.8566910028457642,
      "rewards/margins_max": 2.370370388031006,
      "rewards/margins_min": 1.3430118560791016,
      "rewards/margins_std": 0.7264522910118103,
      "rewards/rejected": -1.4357364177703857,
      "step": 350
    },
    {
      "dpo_losses": 0.24505066871643066,
      "epoch": 1.01,
      "grad_norm": 3.181861527878417,
      "learning_rate": 4.187799870182038e-06,
      "logits/chosen": -2.5239622592926025,
      "logits/rejected": -2.4095091819763184,
      "logps/chosen": -305.76092529296875,
      "logps/rejected": -311.29388427734375,
      "loss": 0.3141,
      "positive_losses": 0.24010220170021057,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4044499397277832,
      "rewards/margins": 1.4896382093429565,
      "rewards/margins_max": 2.010220527648926,
      "rewards/margins_min": 0.9690560102462769,
      "rewards/margins_std": 0.7362144589424133,
      "rewards/rejected": -1.0851882696151733,
      "step": 360
    },
    {
      "dpo_losses": 0.24201233685016632,
      "epoch": 1.04,
      "grad_norm": 54.166595052035994,
      "learning_rate": 4.1264236719042365e-06,
      "logits/chosen": -2.362025737762451,
      "logits/rejected": -2.3736584186553955,
      "logps/chosen": -307.57098388671875,
      "logps/rejected": -404.64447021484375,
      "loss": 0.3927,
      "positive_losses": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.44086480140686035,
      "rewards/margins": 1.8494741916656494,
      "rewards/margins_max": 2.651545763015747,
      "rewards/margins_min": 1.0474025011062622,
      "rewards/margins_std": 1.1343004703521729,
      "rewards/rejected": -1.40860915184021,
      "step": 370
    },
    {
      "dpo_losses": 0.22759906947612762,
      "epoch": 1.07,
      "grad_norm": 38.27158679057807,
      "learning_rate": 4.063298579603001e-06,
      "logits/chosen": -2.472991466522217,
      "logits/rejected": -2.2599282264709473,
      "logps/chosen": -351.54620361328125,
      "logps/rejected": -319.0135803222656,
      "loss": 0.3817,
      "positive_losses": 3.08628511428833,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3943432867527008,
      "rewards/margins": 1.7396957874298096,
      "rewards/margins_max": 2.3970978260040283,
      "rewards/margins_min": 1.0822933912277222,
      "rewards/margins_std": 0.9297070503234863,
      "rewards/rejected": -1.3453524112701416,
      "step": 380
    },
    {
      "dpo_losses": 0.134817972779274,
      "epoch": 1.1,
      "grad_norm": 2.791305623027733,
      "learning_rate": 3.998492471715272e-06,
      "logits/chosen": -2.496638059616089,
      "logits/rejected": -2.455773115158081,
      "logps/chosen": -349.44073486328125,
      "logps/rejected": -536.924560546875,
      "loss": 0.3072,
      "positive_losses": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4807213246822357,
      "rewards/margins": 2.533548355102539,
      "rewards/margins_max": 3.3716654777526855,
      "rewards/margins_min": 1.695431113243103,
      "rewards/margins_std": 1.185276746749878,
      "rewards/rejected": -2.0528271198272705,
      "step": 390
    },
    {
      "dpo_losses": 0.23253802955150604,
      "epoch": 1.13,
      "grad_norm": 1.567698262162245,
      "learning_rate": 3.932075034274723e-06,
      "logits/chosen": -2.385937213897705,
      "logits/rejected": -2.3955626487731934,
      "logps/chosen": -266.9366455078125,
      "logps/rejected": -367.39764404296875,
      "loss": 0.3355,
      "positive_losses": 0.33393630385398865,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3180273175239563,
      "rewards/margins": 1.719641923904419,
      "rewards/margins_max": 2.0190768241882324,
      "rewards/margins_min": 1.4202073812484741,
      "rewards/margins_std": 0.4234645366668701,
      "rewards/rejected": -1.401614785194397,
      "step": 400
    },
    {
      "epoch": 1.13,
      "eval_dpo_losses": 0.6571967601776123,
      "eval_logits/chosen": -2.415469169616699,
      "eval_logits/rejected": -2.3773038387298584,
      "eval_logps/chosen": -392.8340759277344,
      "eval_logps/rejected": -401.27020263671875,
      "eval_loss": 11.415841102600098,
      "eval_positive_losses": 108.73988342285156,
      "eval_rewards/accuracies": 0.6547619104385376,
      "eval_rewards/chosen": -1.0761287212371826,
      "eval_rewards/margins": 0.34474655985832214,
      "eval_rewards/margins_max": 1.462611198425293,
      "eval_rewards/margins_min": -0.7661140561103821,
      "eval_rewards/margins_std": 0.9968340992927551,
      "eval_rewards/rejected": -1.4208753108978271,
      "eval_runtime": 282.7743,
      "eval_samples_per_second": 7.073,
      "eval_steps_per_second": 0.223,
      "step": 400
    },
    {
      "dpo_losses": 0.23369018733501434,
      "epoch": 1.15,
      "grad_norm": 1.6752309896133104,
      "learning_rate": 3.864117685978339e-06,
      "logits/chosen": -2.4605586528778076,
      "logits/rejected": -2.4261014461517334,
      "logps/chosen": -264.14556884765625,
      "logps/rejected": -375.3333740234375,
      "loss": 0.3689,
      "positive_losses": 3.3056740760803223,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.3054488003253937,
      "rewards/margins": 1.9768092632293701,
      "rewards/margins_max": 2.963430166244507,
      "rewards/margins_min": 0.990188479423523,
      "rewards/margins_std": 1.3952926397323608,
      "rewards/rejected": -1.6713603734970093,
      "step": 410
    },
    {
      "dpo_losses": 0.1611286997795105,
      "epoch": 1.18,
      "grad_norm": 19.99801702205629,
      "learning_rate": 3.794693501389861e-06,
      "logits/chosen": -2.549091100692749,
      "logits/rejected": -2.4567952156066895,
      "logps/chosen": -349.955810546875,
      "logps/rejected": -440.55987548828125,
      "loss": 0.3233,
      "positive_losses": 0.5550443530082703,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4979848861694336,
      "rewards/margins": 2.247861385345459,
      "rewards/margins_max": 3.1490349769592285,
      "rewards/margins_min": 1.3466877937316895,
      "rewards/margins_std": 1.2744518518447876,
      "rewards/rejected": -1.7498763799667358,
      "step": 420
    },
    {
      "dpo_losses": 0.18394428491592407,
      "epoch": 1.21,
      "grad_norm": 89.7451007324007,
      "learning_rate": 3.7238771323626822e-06,
      "logits/chosen": -2.5034220218658447,
      "logits/rejected": -2.4015450477600098,
      "logps/chosen": -339.3304748535156,
      "logps/rejected": -436.32769775390625,
      "loss": 0.3026,
      "positive_losses": 0.09120301902294159,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4775872230529785,
      "rewards/margins": 2.2062175273895264,
      "rewards/margins_max": 2.9012348651885986,
      "rewards/margins_min": 1.5112000703811646,
      "rewards/margins_std": 0.9829031229019165,
      "rewards/rejected": -1.7286304235458374,
      "step": 430
    },
    {
      "dpo_losses": 0.18616007268428802,
      "epoch": 1.24,
      "grad_norm": 3.3128247371778987,
      "learning_rate": 3.651744727766676e-06,
      "logits/chosen": -2.5054969787597656,
      "logits/rejected": -2.38596773147583,
      "logps/chosen": -275.51483154296875,
      "logps/rejected": -352.8285827636719,
      "loss": 0.3883,
      "positive_losses": 0.630480945110321,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.39738765358924866,
      "rewards/margins": 1.9772555828094482,
      "rewards/margins_max": 2.764988422393799,
      "rewards/margins_min": 1.1895228624343872,
      "rewards/margins_std": 1.1140224933624268,
      "rewards/rejected": -1.579867959022522,
      "step": 440
    },
    {
      "dpo_losses": 0.16775104403495789,
      "epoch": 1.27,
      "grad_norm": 2.373995116616861,
      "learning_rate": 3.57837385160529e-06,
      "logits/chosen": -2.431580066680908,
      "logits/rejected": -2.3474247455596924,
      "logps/chosen": -304.8125915527344,
      "logps/rejected": -460.9014587402344,
      "loss": 0.3246,
      "positive_losses": 2.069793224334717,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.33361926674842834,
      "rewards/margins": 2.2662878036499023,
      "rewards/margins_max": 2.9109106063842773,
      "rewards/margins_min": 1.6216650009155273,
      "rewards/margins_std": 0.9116341471672058,
      "rewards/rejected": -1.9326684474945068,
      "step": 450
    },
    {
      "dpo_losses": 0.13223373889923096,
      "epoch": 1.3,
      "grad_norm": 43.7205383502172,
      "learning_rate": 3.503843399610941e-06,
      "logits/chosen": -2.416710615158081,
      "logits/rejected": -2.4022932052612305,
      "logps/chosen": -355.0602111816406,
      "logps/rejected": -625.78369140625,
      "loss": 0.3215,
      "positive_losses": 3.091512680053711,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.40217727422714233,
      "rewards/margins": 2.6340858936309814,
      "rewards/margins_max": 3.5210165977478027,
      "rewards/margins_min": 1.7471544742584229,
      "rewards/margins_std": 1.2543103694915771,
      "rewards/rejected": -2.2319083213806152,
      "step": 460
    },
    {
      "dpo_losses": 0.13370443880558014,
      "epoch": 1.32,
      "grad_norm": 93.10254553112391,
      "learning_rate": 3.4282335144083985e-06,
      "logits/chosen": -2.4071993827819824,
      "logits/rejected": -2.301910638809204,
      "logps/chosen": -323.6948547363281,
      "logps/rejected": -422.40106201171875,
      "loss": 0.3844,
      "positive_losses": 0.8020246624946594,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.33555978536605835,
      "rewards/margins": 2.352768659591675,
      "rewards/margins_max": 2.6321401596069336,
      "rewards/margins_min": 2.073397159576416,
      "rewards/margins_std": 0.395090788602829,
      "rewards/rejected": -2.017209053039551,
      "step": 470
    },
    {
      "dpo_losses": 0.16256804764270782,
      "epoch": 1.35,
      "grad_norm": 25.781294981021368,
      "learning_rate": 3.351625499337395e-06,
      "logits/chosen": -2.5639257431030273,
      "logits/rejected": -2.3825831413269043,
      "logps/chosen": -368.14764404296875,
      "logps/rejected": -455.31170654296875,
      "loss": 0.3346,
      "positive_losses": 5.322105407714844,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.47703951597213745,
      "rewards/margins": 2.266021251678467,
      "rewards/margins_max": 2.826010227203369,
      "rewards/margins_min": 1.706032156944275,
      "rewards/margins_std": 0.791944146156311,
      "rewards/rejected": -1.7889817953109741,
      "step": 480
    },
    {
      "dpo_losses": 0.27087026834487915,
      "epoch": 1.38,
      "grad_norm": 1.9694210301585493,
      "learning_rate": 3.2741017310271056e-06,
      "logits/chosen": -2.434044361114502,
      "logits/rejected": -2.2838988304138184,
      "logps/chosen": -185.84713745117188,
      "logps/rejected": -353.2933044433594,
      "loss": 0.4342,
      "positive_losses": 2.383211851119995,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.31434357166290283,
      "rewards/margins": 1.6815185546875,
      "rewards/margins_max": 2.3663582801818848,
      "rewards/margins_min": 0.9966787099838257,
      "rewards/margins_std": 0.9685096740722656,
      "rewards/rejected": -1.3671748638153076,
      "step": 490
    },
    {
      "dpo_losses": 0.1878516972064972,
      "epoch": 1.41,
      "grad_norm": 37.46970781719882,
      "learning_rate": 3.195745570816532e-06,
      "logits/chosen": -2.3778724670410156,
      "logits/rejected": -2.298856258392334,
      "logps/chosen": -339.0552978515625,
      "logps/rejected": -400.7374572753906,
      "loss": 0.3438,
      "positive_losses": 1.1473572254180908,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4514932632446289,
      "rewards/margins": 2.0797715187072754,
      "rewards/margins_max": 2.5511765480041504,
      "rewards/margins_min": 1.6083663702011108,
      "rewards/margins_std": 0.6666676998138428,
      "rewards/rejected": -1.628278136253357,
      "step": 500
    },
    {
      "epoch": 1.41,
      "eval_dpo_losses": 0.6380993127822876,
      "eval_logits/chosen": -2.490675449371338,
      "eval_logits/rejected": -2.4471161365509033,
      "eval_logps/chosen": -385.29376220703125,
      "eval_logps/rejected": -393.24566650390625,
      "eval_loss": 10.641342163085938,
      "eval_positive_losses": 101.35254669189453,
      "eval_rewards/accuracies": 0.6865079402923584,
      "eval_rewards/chosen": -1.0007256269454956,
      "eval_rewards/margins": 0.33990418910980225,
      "eval_rewards/margins_max": 1.3353358507156372,
      "eval_rewards/margins_min": -0.6337663531303406,
      "eval_rewards/margins_std": 0.8805232644081116,
      "eval_rewards/rejected": -1.3406296968460083,
      "eval_runtime": 281.3613,
      "eval_samples_per_second": 7.108,
      "eval_steps_per_second": 0.224,
      "step": 500
    },
    {
      "dpo_losses": 0.15203723311424255,
      "epoch": 1.44,
      "grad_norm": 7.547244740150183,
      "learning_rate": 3.116641275116018e-06,
      "logits/chosen": -2.2004570960998535,
      "logits/rejected": -2.1605918407440186,
      "logps/chosen": -256.17730712890625,
      "logps/rejected": -509.5887756347656,
      "loss": 0.3429,
      "positive_losses": 2.6612415313720703,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3650303781032562,
      "rewards/margins": 2.308377504348755,
      "rewards/margins_max": 3.1350021362304688,
      "rewards/margins_min": 1.481752634048462,
      "rewards/margins_std": 1.1690237522125244,
      "rewards/rejected": -1.9433467388153076,
      "step": 510
    },
    {
      "dpo_losses": 0.12489266693592072,
      "epoch": 1.46,
      "grad_norm": 145.8887364997723,
      "learning_rate": 3.0368739048062956e-06,
      "logits/chosen": -2.448385238647461,
      "logits/rejected": -2.354788064956665,
      "logps/chosen": -307.94976806640625,
      "logps/rejected": -450.015380859375,
      "loss": 0.5303,
      "positive_losses": 3.1661620140075684,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3366628587245941,
      "rewards/margins": 2.504631519317627,
      "rewards/margins_max": 3.252946138381958,
      "rewards/margins_min": 1.7563165426254272,
      "rewards/margins_std": 1.058276891708374,
      "rewards/rejected": -2.16796875,
      "step": 520
    },
    {
      "dpo_losses": 0.14785408973693848,
      "epoch": 1.49,
      "grad_norm": 19.752652919627817,
      "learning_rate": 2.956529233772492e-06,
      "logits/chosen": -2.3587186336517334,
      "logits/rejected": -2.3671178817749023,
      "logps/chosen": -339.9403076171875,
      "logps/rejected": -447.1279296875,
      "loss": 0.2974,
      "positive_losses": 0.8550773859024048,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.39065319299697876,
      "rewards/margins": 2.165048122406006,
      "rewards/margins_max": 2.7594637870788574,
      "rewards/margins_min": 1.5706324577331543,
      "rewards/margins_std": 0.8406306505203247,
      "rewards/rejected": -1.7743949890136719,
      "step": 530
    },
    {
      "dpo_losses": 0.1259264200925827,
      "epoch": 1.52,
      "grad_norm": 75.31648718698649,
      "learning_rate": 2.8756936566714317e-06,
      "logits/chosen": -2.5206494331359863,
      "logits/rejected": -2.379965305328369,
      "logps/chosen": -351.1200256347656,
      "logps/rejected": -426.8377990722656,
      "loss": 0.4729,
      "positive_losses": 1.8278591632843018,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.48433223366737366,
      "rewards/margins": 2.3780269622802734,
      "rewards/margins_max": 2.926578998565674,
      "rewards/margins_min": 1.8294748067855835,
      "rewards/margins_std": 0.7757696509361267,
      "rewards/rejected": -1.893694281578064,
      "step": 540
    },
    {
      "dpo_losses": 0.20844757556915283,
      "epoch": 1.55,
      "grad_norm": 3.349676851087858,
      "learning_rate": 2.794454096031429e-06,
      "logits/chosen": -2.4806084632873535,
      "logits/rejected": -2.4055755138397217,
      "logps/chosen": -274.53900146484375,
      "logps/rejected": -464.46307373046875,
      "loss": 0.2124,
      "positive_losses": 1.989834189414978,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.36409634351730347,
      "rewards/margins": 2.338207960128784,
      "rewards/margins_max": 3.5889182090759277,
      "rewards/margins_min": 1.0874969959259033,
      "rewards/margins_std": 1.7687723636627197,
      "rewards/rejected": -1.974111557006836,
      "step": 550
    },
    {
      "dpo_losses": 0.18082240223884583,
      "epoch": 1.58,
      "grad_norm": 2.0680076581049485,
      "learning_rate": 2.71289790878446e-06,
      "logits/chosen": -2.375046730041504,
      "logits/rejected": -2.316608190536499,
      "logps/chosen": -295.595703125,
      "logps/rejected": -556.7249755859375,
      "loss": 0.3351,
      "positive_losses": 4.852077484130859,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.21087315678596497,
      "rewards/margins": 2.390120267868042,
      "rewards/margins_max": 3.270312786102295,
      "rewards/margins_min": 1.5099279880523682,
      "rewards/margins_std": 1.24478018283844,
      "rewards/rejected": -2.1792471408843994,
      "step": 560
    },
    {
      "dpo_losses": 0.11622228473424911,
      "epoch": 1.61,
      "grad_norm": 53.09814348729035,
      "learning_rate": 2.6311127923312156e-06,
      "logits/chosen": -2.38314151763916,
      "logits/rejected": -2.2578694820404053,
      "logps/chosen": -386.07366943359375,
      "logps/rejected": -540.0533447265625,
      "loss": 0.2485,
      "positive_losses": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4612743854522705,
      "rewards/margins": 2.703238010406494,
      "rewards/margins_max": 3.3717334270477295,
      "rewards/margins_min": 2.0347423553466797,
      "rewards/margins_std": 0.9453955888748169,
      "rewards/rejected": -2.2419633865356445,
      "step": 570
    },
    {
      "dpo_losses": 0.11324157565832138,
      "epoch": 1.63,
      "grad_norm": 28.84783476798183,
      "learning_rate": 2.549186690240057e-06,
      "logits/chosen": -2.3538641929626465,
      "logits/rejected": -2.2661383152008057,
      "logps/chosen": -265.77618408203125,
      "logps/rejected": -461.02362060546875,
      "loss": 0.3852,
      "positive_losses": 0.5109559893608093,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.326893150806427,
      "rewards/margins": 2.722642660140991,
      "rewards/margins_max": 3.37109375,
      "rewards/margins_min": 2.0741915702819824,
      "rewards/margins_std": 0.9170483350753784,
      "rewards/rejected": -2.395749568939209,
      "step": 580
    },
    {
      "dpo_losses": 0.19121481478214264,
      "epoch": 1.66,
      "grad_norm": 3.235430116905917,
      "learning_rate": 2.4672076976812548e-06,
      "logits/chosen": -2.256422281265259,
      "logits/rejected": -2.129117965698242,
      "logps/chosen": -335.0826721191406,
      "logps/rejected": -496.73138427734375,
      "loss": 0.3539,
      "positive_losses": 0.32223206758499146,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3417799472808838,
      "rewards/margins": 2.386916399002075,
      "rewards/margins_max": 3.4707369804382324,
      "rewards/margins_min": 1.3030953407287598,
      "rewards/margins_std": 1.5327543020248413,
      "rewards/rejected": -2.0451362133026123,
      "step": 590
    },
    {
      "dpo_losses": 0.2605344355106354,
      "epoch": 1.69,
      "grad_norm": 13.287666189135933,
      "learning_rate": 2.3852639666982218e-06,
      "logits/chosen": -2.4083638191223145,
      "logits/rejected": -2.3480491638183594,
      "logps/chosen": -243.49948120117188,
      "logps/rejected": -401.8130798339844,
      "loss": 0.2144,
      "positive_losses": 0.026648616418242455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.35153132677078247,
      "rewards/margins": 1.764525055885315,
      "rewards/margins_max": 2.218550205230713,
      "rewards/margins_min": 1.310499668121338,
      "rewards/margins_std": 0.642088770866394,
      "rewards/rejected": -1.4129936695098877,
      "step": 600
    },
    {
      "epoch": 1.69,
      "eval_dpo_losses": 0.6267468929290771,
      "eval_logits/chosen": -2.3438363075256348,
      "eval_logits/rejected": -2.302872896194458,
      "eval_logps/chosen": -363.3914489746094,
      "eval_logps/rejected": -380.5317687988281,
      "eval_loss": 8.589625358581543,
      "eval_positive_losses": 79.79975891113281,
      "eval_rewards/accuracies": 0.6865079402923584,
      "eval_rewards/chosen": -0.7817028760910034,
      "eval_rewards/margins": 0.4317886233329773,
      "eval_rewards/margins_max": 1.5951305627822876,
      "eval_rewards/margins_min": -0.6660595536231995,
      "eval_rewards/margins_std": 1.0046793222427368,
      "eval_rewards/rejected": -1.2134915590286255,
      "eval_runtime": 280.6736,
      "eval_samples_per_second": 7.126,
      "eval_steps_per_second": 0.224,
      "step": 600
    },
    {
      "dpo_losses": 0.21777740120887756,
      "epoch": 1.72,
      "grad_norm": 3.6630257967936934,
      "learning_rate": 2.303443611417584e-06,
      "logits/chosen": -2.2210001945495605,
      "logits/rejected": -2.135100841522217,
      "logps/chosen": -324.7200622558594,
      "logps/rejected": -427.8872985839844,
      "loss": 0.4399,
      "positive_losses": 1.8486969470977783,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4076939523220062,
      "rewards/margins": 2.0033507347106934,
      "rewards/margins_max": 2.8805928230285645,
      "rewards/margins_min": 1.12610924243927,
      "rewards/margins_std": 1.2406072616577148,
      "rewards/rejected": -1.5956569910049438,
      "step": 610
    },
    {
      "dpo_losses": 0.16951796412467957,
      "epoch": 1.75,
      "grad_norm": 171.16064616866518,
      "learning_rate": 2.2218346133000264e-06,
      "logits/chosen": -2.2120890617370605,
      "logits/rejected": -2.1098124980926514,
      "logps/chosen": -283.6782531738281,
      "logps/rejected": -384.42193603515625,
      "loss": 0.445,
      "positive_losses": 6.724704742431641,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.2760510742664337,
      "rewards/margins": 2.14823317527771,
      "rewards/margins_max": 2.824097156524658,
      "rewards/margins_min": 1.4723690748214722,
      "rewards/margins_std": 0.9558159708976746,
      "rewards/rejected": -1.8721821308135986,
      "step": 620
    },
    {
      "dpo_losses": 0.15758368372917175,
      "epoch": 1.77,
      "grad_norm": 24.7162522936105,
      "learning_rate": 2.140524726533792e-06,
      "logits/chosen": -2.237668514251709,
      "logits/rejected": -2.1284663677215576,
      "logps/chosen": -338.96942138671875,
      "logps/rejected": -402.83697509765625,
      "loss": 0.3769,
      "positive_losses": 0.14499235153198242,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5579421520233154,
      "rewards/margins": 2.344921350479126,
      "rewards/margins_max": 2.9339561462402344,
      "rewards/margins_min": 1.7558867931365967,
      "rewards/margins_std": 0.833020806312561,
      "rewards/rejected": -1.7869793176651,
      "step": 630
    },
    {
      "dpo_losses": 0.18143755197525024,
      "epoch": 1.8,
      "grad_norm": 28.54900764303978,
      "learning_rate": 2.059601383672566e-06,
      "logits/chosen": -2.421058177947998,
      "logits/rejected": -2.359043598175049,
      "logps/chosen": -261.0716857910156,
      "logps/rejected": -371.5032653808594,
      "loss": 0.2764,
      "positive_losses": 0.5634332895278931,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4140376150608063,
      "rewards/margins": 2.1301121711730957,
      "rewards/margins_max": 2.458400011062622,
      "rewards/margins_min": 1.8018243312835693,
      "rewards/margins_std": 0.46426907181739807,
      "rewards/rejected": -1.7160745859146118,
      "step": 640
    },
    {
      "dpo_losses": 0.21564999222755432,
      "epoch": 1.83,
      "grad_norm": 43.347219579951144,
      "learning_rate": 1.9791516016192214e-06,
      "logits/chosen": -2.4876837730407715,
      "logits/rejected": -2.327549457550049,
      "logps/chosen": -288.0809020996094,
      "logps/rejected": -400.9809875488281,
      "loss": 0.2601,
      "positive_losses": 2.1945955753326416,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.4631883203983307,
      "rewards/margins": 2.244870662689209,
      "rewards/margins_max": 3.217092990875244,
      "rewards/margins_min": 1.2726480960845947,
      "rewards/margins_std": 1.3749301433563232,
      "rewards/rejected": -1.7816823720932007,
      "step": 650
    },
    {
      "dpo_losses": 0.21185970306396484,
      "epoch": 1.86,
      "grad_norm": 4.895621870086889,
      "learning_rate": 1.8992618880565039e-06,
      "logits/chosen": -2.147622585296631,
      "logits/rejected": -2.0906789302825928,
      "logps/chosen": -282.22711181640625,
      "logps/rejected": -362.41119384765625,
      "loss": 0.4916,
      "positive_losses": 3.9535961151123047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4179634153842926,
      "rewards/margins": 2.141587495803833,
      "rewards/margins_max": 3.0650925636291504,
      "rewards/margins_min": 1.218082308769226,
      "rewards/margins_std": 1.3060333728790283,
      "rewards/rejected": -1.7236239910125732,
      "step": 660
    },
    {
      "dpo_losses": 0.11567674577236176,
      "epoch": 1.89,
      "grad_norm": 259.9155211313861,
      "learning_rate": 1.8200181484252888e-06,
      "logits/chosen": -2.390129566192627,
      "logits/rejected": -2.4018349647521973,
      "logps/chosen": -326.493896484375,
      "logps/rejected": -549.7818603515625,
      "loss": 0.2561,
      "positive_losses": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.562630295753479,
      "rewards/margins": 2.886990785598755,
      "rewards/margins_max": 3.560823440551758,
      "rewards/margins_min": 2.2131576538085938,
      "rewards/margins_std": 0.9529436826705933,
      "rewards/rejected": -2.3243603706359863,
      "step": 670
    },
    {
      "dpo_losses": 0.1450214684009552,
      "epoch": 1.92,
      "grad_norm": 3.1779421953498246,
      "learning_rate": 1.7415055935504234e-06,
      "logits/chosen": -2.4584078788757324,
      "logits/rejected": -2.2816786766052246,
      "logps/chosen": -351.3116760253906,
      "logps/rejected": -523.8319091796875,
      "loss": 0.4032,
      "positive_losses": 3.919013500213623,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4204772114753723,
      "rewards/margins": 2.6883883476257324,
      "rewards/margins_max": 3.6081409454345703,
      "rewards/margins_min": 1.7686359882354736,
      "rewards/margins_std": 1.300726294517517,
      "rewards/rejected": -2.267911434173584,
      "step": 680
    },
    {
      "dpo_losses": 0.22765210270881653,
      "epoch": 1.94,
      "grad_norm": 290.4438144891366,
      "learning_rate": 1.6638086480134954e-06,
      "logits/chosen": -2.2493913173675537,
      "logits/rejected": -2.2027366161346436,
      "logps/chosen": -208.2427520751953,
      "logps/rejected": -294.7981872558594,
      "loss": 0.3929,
      "positive_losses": 4.185807228088379,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.2872256338596344,
      "rewards/margins": 1.9401639699935913,
      "rewards/margins_max": 2.823594331741333,
      "rewards/margins_min": 1.0567338466644287,
      "rewards/margins_std": 1.249358892440796,
      "rewards/rejected": -1.6529382467269897,
      "step": 690
    },
    {
      "dpo_losses": 0.11677996069192886,
      "epoch": 1.97,
      "grad_norm": 2.93724058913164,
      "learning_rate": 1.5870108593710473e-06,
      "logits/chosen": -2.169877290725708,
      "logits/rejected": -2.062708854675293,
      "logps/chosen": -379.1634826660156,
      "logps/rejected": -440.5103454589844,
      "loss": 0.3314,
      "positive_losses": 0.9755552411079407,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.478015273809433,
      "rewards/margins": 2.7329649925231934,
      "rewards/margins_max": 3.358954906463623,
      "rewards/margins_min": 2.1069746017456055,
      "rewards/margins_std": 0.8852837681770325,
      "rewards/rejected": -2.2549493312835693,
      "step": 700
    },
    {
      "epoch": 1.97,
      "eval_dpo_losses": 0.6525446176528931,
      "eval_logits/chosen": -2.343222141265869,
      "eval_logits/rejected": -2.30248761177063,
      "eval_logps/chosen": -391.17047119140625,
      "eval_logps/rejected": -410.6868591308594,
      "eval_loss": 11.165140151977539,
      "eval_positive_losses": 107.29693603515625,
      "eval_rewards/accuracies": 0.6626983880996704,
      "eval_rewards/chosen": -1.0594924688339233,
      "eval_rewards/margins": 0.4555494785308838,
      "eval_rewards/margins_max": 1.7776461839675903,
      "eval_rewards/margins_min": -0.8450111150741577,
      "eval_rewards/margins_std": 1.16599440574646,
      "eval_rewards/rejected": -1.5150419473648071,
      "eval_runtime": 281.451,
      "eval_samples_per_second": 7.106,
      "eval_steps_per_second": 0.224,
      "step": 700
    },
    {
      "dpo_losses": 0.2008899748325348,
      "epoch": 2.0,
      "grad_norm": 1.5771255349778672,
      "learning_rate": 1.511194808315853e-06,
      "logits/chosen": -2.2691843509674072,
      "logits/rejected": -2.16029691696167,
      "logps/chosen": -264.2540588378906,
      "logps/rejected": -368.97308349609375,
      "loss": 0.2577,
      "positive_losses": 1.1838890314102173,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4201991558074951,
      "rewards/margins": 2.272444248199463,
      "rewards/margins_max": 3.1316885948181152,
      "rewards/margins_min": 1.413199782371521,
      "rewards/margins_std": 1.2151552438735962,
      "rewards/rejected": -1.8522450923919678,
      "step": 710
    },
    {
      "dpo_losses": 0.11133086681365967,
      "epoch": 2.03,
      "grad_norm": 39.920300238021134,
      "learning_rate": 1.4364420198778662e-06,
      "logits/chosen": -2.3695783615112305,
      "logits/rejected": -2.292491912841797,
      "logps/chosen": -303.01556396484375,
      "logps/rejected": -584.0543212890625,
      "loss": 0.2033,
      "positive_losses": 1.3720115423202515,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.44599366188049316,
      "rewards/margins": 2.9826126098632812,
      "rewards/margins_max": 3.897780179977417,
      "rewards/margins_min": 2.0674448013305664,
      "rewards/margins_std": 1.2942426204681396,
      "rewards/rejected": -2.536618709564209,
      "step": 720
    },
    {
      "dpo_losses": 0.06813247501850128,
      "epoch": 2.06,
      "grad_norm": 61.35667528119903,
      "learning_rate": 1.3628328757603243e-06,
      "logits/chosen": -2.3125011920928955,
      "logits/rejected": -2.234978437423706,
      "logps/chosen": -380.5849609375,
      "logps/rejected": -546.2708129882812,
      "loss": 0.2931,
      "positive_losses": 3.713000535964966,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.42165422439575195,
      "rewards/margins": 3.356724500656128,
      "rewards/margins_max": 3.9914348125457764,
      "rewards/margins_min": 2.7220141887664795,
      "rewards/margins_std": 0.8976157903671265,
      "rewards/rejected": -2.935070276260376,
      "step": 730
    },
    {
      "dpo_losses": 0.1471334546804428,
      "epoch": 2.08,
      "grad_norm": 81.02956274757072,
      "learning_rate": 1.2904465279052725e-06,
      "logits/chosen": -2.3687386512756348,
      "logits/rejected": -2.246112585067749,
      "logps/chosen": -307.13690185546875,
      "logps/rejected": -456.7256774902344,
      "loss": 0.2511,
      "positive_losses": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.44816774129867554,
      "rewards/margins": 2.7628419399261475,
      "rewards/margins_max": 3.8002758026123047,
      "rewards/margins_min": 1.7254081964492798,
      "rewards/margins_std": 1.4671531915664673,
      "rewards/rejected": -2.314674139022827,
      "step": 740
    },
    {
      "dpo_losses": 0.18885108828544617,
      "epoch": 2.11,
      "grad_norm": 37.34403765579587,
      "learning_rate": 1.219360813381446e-06,
      "logits/chosen": -2.1814396381378174,
      "logits/rejected": -2.133742570877075,
      "logps/chosen": -167.7379913330078,
      "logps/rejected": -342.445068359375,
      "loss": 0.2246,
      "positive_losses": 3.9296765327453613,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.2083771526813507,
      "rewards/margins": 2.266416072845459,
      "rewards/margins_max": 3.0310301780700684,
      "rewards/margins_min": 1.501802682876587,
      "rewards/margins_std": 1.0813268423080444,
      "rewards/rejected": -2.0580391883850098,
      "step": 750
    },
    {
      "dpo_losses": 0.07930545508861542,
      "epoch": 2.14,
      "grad_norm": 53.31665337719304,
      "learning_rate": 1.1496521706860392e-06,
      "logits/chosen": -2.3485515117645264,
      "logits/rejected": -2.2168877124786377,
      "logps/chosen": -298.58966064453125,
      "logps/rejected": -549.2097778320312,
      "loss": 0.2258,
      "positive_losses": 3.5301315784454346,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.344715416431427,
      "rewards/margins": 3.2088115215301514,
      "rewards/margins_max": 4.005995750427246,
      "rewards/margins_min": 2.411628246307373,
      "rewards/margins_std": 1.1273881196975708,
      "rewards/rejected": -2.8640968799591064,
      "step": 760
    },
    {
      "dpo_losses": 0.13964664936065674,
      "epoch": 2.17,
      "grad_norm": 1.2653188452887754,
      "learning_rate": 1.0813955575503588e-06,
      "logits/chosen": -2.278029680252075,
      "logits/rejected": -2.2471253871917725,
      "logps/chosen": -296.67205810546875,
      "logps/rejected": -532.7224731445312,
      "loss": 0.2746,
      "positive_losses": 1.8862769603729248,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.36964336037635803,
      "rewards/margins": 3.1202445030212402,
      "rewards/margins_max": 4.3073296546936035,
      "rewards/margins_min": 1.9331591129302979,
      "rewards/margins_std": 1.6787922382354736,
      "rewards/rejected": -2.750600814819336,
      "step": 770
    },
    {
      "dpo_losses": 0.18134805560112,
      "epoch": 2.2,
      "grad_norm": 49.5060636871573,
      "learning_rate": 1.0146643703377488e-06,
      "logits/chosen": -2.3735833168029785,
      "logits/rejected": -2.250998020172119,
      "logps/chosen": -292.6219177246094,
      "logps/rejected": -449.14068603515625,
      "loss": 0.2041,
      "positive_losses": 1.1141910552978516,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3442000448703766,
      "rewards/margins": 2.5554795265197754,
      "rewards/margins_max": 3.6242382526397705,
      "rewards/margins_min": 1.4867204427719116,
      "rewards/margins_std": 1.51145339012146,
      "rewards/rejected": -2.2112793922424316,
      "step": 780
    },
    {
      "dpo_losses": 0.09527120739221573,
      "epoch": 2.23,
      "grad_norm": 22.61766042791389,
      "learning_rate": 9.495303651204496e-07,
      "logits/chosen": -2.317281484603882,
      "logits/rejected": -2.2382774353027344,
      "logps/chosen": -320.7598571777344,
      "logps/rejected": -518.2537841796875,
      "loss": 0.1938,
      "positive_losses": 0.0011909485328942537,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4521896243095398,
      "rewards/margins": 2.8671340942382812,
      "rewards/margins_max": 3.515406847000122,
      "rewards/margins_min": 2.2188615798950195,
      "rewards/margins_std": 0.9167959094047546,
      "rewards/rejected": -2.414944648742676,
      "step": 790
    },
    {
      "dpo_losses": 0.09652134031057358,
      "epoch": 2.25,
      "grad_norm": 55.89571807239017,
      "learning_rate": 8.860635805202616e-07,
      "logits/chosen": -2.334362506866455,
      "logits/rejected": -2.2386062145233154,
      "logps/chosen": -323.18060302734375,
      "logps/rejected": -481.80712890625,
      "loss": 0.1352,
      "positive_losses": 0.3718675673007965,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4021781086921692,
      "rewards/margins": 2.7669034004211426,
      "rewards/margins_max": 3.376081943511963,
      "rewards/margins_min": 2.1577250957489014,
      "rewards/margins_std": 0.8615081906318665,
      "rewards/rejected": -2.364725112915039,
      "step": 800
    },
    {
      "epoch": 2.25,
      "eval_dpo_losses": 0.6700040102005005,
      "eval_logits/chosen": -2.3320279121398926,
      "eval_logits/rejected": -2.290130376815796,
      "eval_logps/chosen": -415.07861328125,
      "eval_logps/rejected": -441.023681640625,
      "eval_loss": 13.35706615447998,
      "eval_positive_losses": 130.9070281982422,
      "eval_rewards/accuracies": 0.6626983880996704,
      "eval_rewards/chosen": -1.2985737323760986,
      "eval_rewards/margins": 0.5198364853858948,
      "eval_rewards/margins_max": 2.022522211074829,
      "eval_rewards/margins_min": -0.9602744579315186,
      "eval_rewards/margins_std": 1.3295913934707642,
      "eval_rewards/rejected": -1.8184101581573486,
      "eval_runtime": 281.6798,
      "eval_samples_per_second": 7.1,
      "eval_steps_per_second": 0.224,
      "step": 800
    },
    {
      "dpo_losses": 0.11107480525970459,
      "epoch": 2.28,
      "grad_norm": 1.211879634445009,
      "learning_rate": 8.24332262395994e-07,
      "logits/chosen": -2.4299230575561523,
      "logits/rejected": -2.3509979248046875,
      "logps/chosen": -259.64581298828125,
      "logps/rejected": -510.4208984375,
      "loss": 0.4972,
      "positive_losses": 1.13177490234375,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.34921571612358093,
      "rewards/margins": 3.014543056488037,
      "rewards/margins_max": 4.297389984130859,
      "rewards/margins_min": 1.7316957712173462,
      "rewards/margins_std": 1.8142198324203491,
      "rewards/rejected": -2.665327310562134,
      "step": 810
    },
    {
      "dpo_losses": 0.15867747366428375,
      "epoch": 2.31,
      "grad_norm": 1.9974082008055192,
      "learning_rate": 7.644027904586587e-07,
      "logits/chosen": -2.368762731552124,
      "logits/rejected": -2.2775216102600098,
      "logps/chosen": -270.0508728027344,
      "logps/rejected": -446.8939514160156,
      "loss": 0.3191,
      "positive_losses": 0.852569580078125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3547489047050476,
      "rewards/margins": 2.747641086578369,
      "rewards/margins_max": 3.639145612716675,
      "rewards/margins_min": 1.8561369180679321,
      "rewards/margins_std": 1.2607777118682861,
      "rewards/rejected": -2.392892360687256,
      "step": 820
    },
    {
      "dpo_losses": 0.06032683700323105,
      "epoch": 2.34,
      "grad_norm": 79.25640130890059,
      "learning_rate": 7.06339606893347e-07,
      "logits/chosen": -2.3674511909484863,
      "logits/rejected": -2.254099130630493,
      "logps/chosen": -410.2024841308594,
      "logps/rejected": -582.8692016601562,
      "loss": 0.212,
      "positive_losses": 3.602843761444092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4369390904903412,
      "rewards/margins": 3.7233974933624268,
      "rewards/margins_max": 4.673032283782959,
      "rewards/margins_min": 2.7737619876861572,
      "rewards/margins_std": 1.3429871797561646,
      "rewards/rejected": -3.2864582538604736,
      "step": 830
    },
    {
      "dpo_losses": 0.12710335850715637,
      "epoch": 2.37,
      "grad_norm": 5.277843753781562,
      "learning_rate": 6.502051470645149e-07,
      "logits/chosen": -2.486243486404419,
      "logits/rejected": -2.29672908782959,
      "logps/chosen": -348.5108947753906,
      "logps/rejected": -525.3292846679688,
      "loss": 0.346,
      "positive_losses": 0.0202178955078125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4295215606689453,
      "rewards/margins": 2.8306221961975098,
      "rewards/margins_max": 3.4442856311798096,
      "rewards/margins_min": 2.216958522796631,
      "rewards/margins_std": 0.8678513765335083,
      "rewards/rejected": -2.4011006355285645,
      "step": 840
    },
    {
      "dpo_losses": 0.10362281650304794,
      "epoch": 2.39,
      "grad_norm": 172.11522416865105,
      "learning_rate": 5.960597723792194e-07,
      "logits/chosen": -2.351360559463501,
      "logits/rejected": -2.166510581970215,
      "logps/chosen": -337.3572692871094,
      "logps/rejected": -561.3741455078125,
      "loss": 0.2227,
      "positive_losses": 0.215586856007576,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.47708138823509216,
      "rewards/margins": 3.5102126598358154,
      "rewards/margins_max": 4.734680652618408,
      "rewards/margins_min": 2.28574538230896,
      "rewards/margins_std": 1.731658697128296,
      "rewards/rejected": -3.0331313610076904,
      "step": 850
    },
    {
      "dpo_losses": 0.10238895565271378,
      "epoch": 2.42,
      "grad_norm": 82.67868906193421,
      "learning_rate": 5.43961705380465e-07,
      "logits/chosen": -2.389751434326172,
      "logits/rejected": -2.2885704040527344,
      "logps/chosen": -315.6968078613281,
      "logps/rejected": -601.2724609375,
      "loss": 0.2321,
      "positive_losses": 0.0,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.3972366154193878,
      "rewards/margins": 3.726912260055542,
      "rewards/margins_max": 4.830050945281982,
      "rewards/margins_min": 2.6237740516662598,
      "rewards/margins_std": 1.5600733757019043,
      "rewards/rejected": -3.3296761512756348,
      "step": 860
    },
    {
      "dpo_losses": 0.10260417312383652,
      "epoch": 2.45,
      "grad_norm": 24.22347862103844,
      "learning_rate": 4.939669671404871e-07,
      "logits/chosen": -2.3038783073425293,
      "logits/rejected": -2.185800552368164,
      "logps/chosen": -305.2945556640625,
      "logps/rejected": -611.5335693359375,
      "loss": 0.1672,
      "positive_losses": 0.3565734922885895,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.35259321331977844,
      "rewards/margins": 3.309342861175537,
      "rewards/margins_max": 4.305205345153809,
      "rewards/margins_min": 2.313480854034424,
      "rewards/margins_std": 1.4083621501922607,
      "rewards/rejected": -2.956749439239502,
      "step": 870
    },
    {
      "dpo_losses": 0.0972672775387764,
      "epoch": 2.48,
      "grad_norm": 2.5325654932498978,
      "learning_rate": 4.461293170212644e-07,
      "logits/chosen": -2.366685390472412,
      "logits/rejected": -2.244584798812866,
      "logps/chosen": -284.2767028808594,
      "logps/rejected": -525.98681640625,
      "loss": 0.1957,
      "positive_losses": 4.633613586425781,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3293871283531189,
      "rewards/margins": 3.1354575157165527,
      "rewards/margins_max": 4.131924629211426,
      "rewards/margins_min": 2.138990640640259,
      "rewards/margins_std": 1.409217119216919,
      "rewards/rejected": -2.806070327758789,
      "step": 880
    },
    {
      "dpo_losses": 0.07743240892887115,
      "epoch": 2.51,
      "grad_norm": 246.3549744525164,
      "learning_rate": 4.005001948670606e-07,
      "logits/chosen": -2.3962912559509277,
      "logits/rejected": -2.299287796020508,
      "logps/chosen": -425.3636169433594,
      "logps/rejected": -637.3418579101562,
      "loss": 0.2586,
      "positive_losses": 1.6243082284927368,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5158067941665649,
      "rewards/margins": 3.6285767555236816,
      "rewards/margins_max": 4.66379451751709,
      "rewards/margins_min": 2.593358278274536,
      "rewards/margins_std": 1.464019536972046,
      "rewards/rejected": -3.112769603729248,
      "step": 890
    },
    {
      "dpo_losses": 0.0913797914981842,
      "epoch": 2.54,
      "grad_norm": 31.034651555446352,
      "learning_rate": 3.571286656911377e-07,
      "logits/chosen": -2.356581211090088,
      "logits/rejected": -2.1674458980560303,
      "logps/chosen": -353.3656005859375,
      "logps/rejected": -581.6492309570312,
      "loss": 0.2348,
      "positive_losses": 2.3227431774139404,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.46973830461502075,
      "rewards/margins": 3.632810115814209,
      "rewards/margins_max": 5.023132801055908,
      "rewards/margins_min": 2.242488145828247,
      "rewards/margins_std": 1.966212511062622,
      "rewards/rejected": -3.163072109222412,
      "step": 900
    },
    {
      "epoch": 2.54,
      "eval_dpo_losses": 0.6903655529022217,
      "eval_logits/chosen": -2.3085381984710693,
      "eval_logits/rejected": -2.2660651206970215,
      "eval_logps/chosen": -430.1044006347656,
      "eval_logps/rejected": -459.71075439453125,
      "eval_loss": 14.72413444519043,
      "eval_positive_losses": 145.90809631347656,
      "eval_rewards/accuracies": 0.670634925365448,
      "eval_rewards/chosen": -1.4488320350646973,
      "eval_rewards/margins": 0.5564488768577576,
      "eval_rewards/margins_max": 2.180058240890503,
      "eval_rewards/margins_min": -1.0958278179168701,
      "eval_rewards/margins_std": 1.4586230516433716,
      "eval_rewards/rejected": -2.0052807331085205,
      "eval_runtime": 280.6688,
      "eval_samples_per_second": 7.126,
      "eval_steps_per_second": 0.224,
      "step": 900
    },
    {
      "dpo_losses": 0.10229980945587158,
      "epoch": 2.56,
      "grad_norm": 1.3207112449434741,
      "learning_rate": 3.1606136691612555e-07,
      "logits/chosen": -2.450596809387207,
      "logits/rejected": -2.320343255996704,
      "logps/chosen": -373.2110900878906,
      "logps/rejected": -512.347412109375,
      "loss": 0.2148,
      "positive_losses": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.45970940589904785,
      "rewards/margins": 3.199089527130127,
      "rewards/margins_max": 4.215831756591797,
      "rewards/margins_min": 2.182347297668457,
      "rewards/margins_std": 1.4378905296325684,
      "rewards/rejected": -2.739380121231079,
      "step": 910
    },
    {
      "dpo_losses": 0.10751441866159439,
      "epoch": 2.59,
      "grad_norm": 50.58623371052272,
      "learning_rate": 2.773424582247844e-07,
      "logits/chosen": -2.3134539127349854,
      "logits/rejected": -2.1395092010498047,
      "logps/chosen": -311.353271484375,
      "logps/rejected": -480.4183654785156,
      "loss": 0.1833,
      "positive_losses": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4951690137386322,
      "rewards/margins": 3.3741965293884277,
      "rewards/margins_max": 4.4515910148620605,
      "rewards/margins_min": 2.296802043914795,
      "rewards/margins_std": 1.5236659049987793,
      "rewards/rejected": -2.8790273666381836,
      "step": 920
    },
    {
      "dpo_losses": 0.08198712766170502,
      "epoch": 2.62,
      "grad_norm": 34.59409580902137,
      "learning_rate": 2.410135740750821e-07,
      "logits/chosen": -2.3756449222564697,
      "logits/rejected": -2.2801589965820312,
      "logps/chosen": -309.6001892089844,
      "logps/rejected": -594.0984497070312,
      "loss": 0.2129,
      "positive_losses": 0.7936180233955383,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.37005850672721863,
      "rewards/margins": 3.6145331859588623,
      "rewards/margins_max": 4.759096145629883,
      "rewards/margins_min": 2.4699695110321045,
      "rewards/margins_std": 1.618657112121582,
      "rewards/rejected": -3.244474411010742,
      "step": 930
    },
    {
      "dpo_losses": 0.18878915905952454,
      "epoch": 2.65,
      "grad_norm": 127.23596281859544,
      "learning_rate": 2.0711377893064182e-07,
      "logits/chosen": -2.362793207168579,
      "logits/rejected": -2.2321505546569824,
      "logps/chosen": -313.73736572265625,
      "logps/rejected": -428.6400451660156,
      "loss": 0.2036,
      "positive_losses": 2.974576711654663,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.2547670602798462,
      "rewards/margins": 2.439239978790283,
      "rewards/margins_max": 3.555748701095581,
      "rewards/margins_min": 1.3227306604385376,
      "rewards/margins_std": 1.5789823532104492,
      "rewards/rejected": -2.1844725608825684,
      "step": 940
    },
    {
      "dpo_losses": 0.1424863040447235,
      "epoch": 2.68,
      "grad_norm": 3.7675671139759395,
      "learning_rate": 1.756795252547111e-07,
      "logits/chosen": -2.324215888977051,
      "logits/rejected": -2.20219087600708,
      "logps/chosen": -260.1328430175781,
      "logps/rejected": -470.79412841796875,
      "loss": 0.3619,
      "positive_losses": 0.20402908325195312,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.42012229561805725,
      "rewards/margins": 3.099902629852295,
      "rewards/margins_max": 4.072454452514648,
      "rewards/margins_min": 2.1273510456085205,
      "rewards/margins_std": 1.3753960132598877,
      "rewards/rejected": -2.6797804832458496,
      "step": 950
    },
    {
      "dpo_losses": 0.10174594074487686,
      "epoch": 2.7,
      "grad_norm": 1.8085505107547237,
      "learning_rate": 1.4674461431281013e-07,
      "logits/chosen": -2.525442361831665,
      "logits/rejected": -2.4085853099823,
      "logps/chosen": -290.04345703125,
      "logps/rejected": -531.1943969726562,
      "loss": 0.3038,
      "positive_losses": 0.1788475066423416,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3685051202774048,
      "rewards/margins": 3.2411983013153076,
      "rewards/margins_max": 4.235571384429932,
      "rewards/margins_min": 2.246825695037842,
      "rewards/margins_std": 1.4062554836273193,
      "rewards/rejected": -2.8726933002471924,
      "step": 960
    },
    {
      "dpo_losses": 0.09436773508787155,
      "epoch": 2.73,
      "grad_norm": 38.21579384775439,
      "learning_rate": 1.2034015982622243e-07,
      "logits/chosen": -2.403714418411255,
      "logits/rejected": -2.282444477081299,
      "logps/chosen": -336.4191589355469,
      "logps/rejected": -650.2848510742188,
      "loss": 0.2727,
      "positive_losses": 4.7762908935546875,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.38717547059059143,
      "rewards/margins": 3.8552498817443848,
      "rewards/margins_max": 5.095088481903076,
      "rewards/margins_min": 2.615410327911377,
      "rewards/margins_std": 1.7533977031707764,
      "rewards/rejected": -3.468074083328247,
      "step": 970
    },
    {
      "dpo_losses": 0.0995684489607811,
      "epoch": 2.76,
      "grad_norm": 34.214640164153415,
      "learning_rate": 9.649455451539419e-08,
      "logits/chosen": -2.195786714553833,
      "logits/rejected": -2.1557459831237793,
      "logps/chosen": -205.28829956054688,
      "logps/rejected": -449.69903564453125,
      "loss": 0.362,
      "positive_losses": 1.5731815099716187,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.3235073983669281,
      "rewards/margins": 3.076923131942749,
      "rewards/margins_max": 3.9705424308776855,
      "rewards/margins_min": 2.1833040714263916,
      "rewards/margins_std": 1.2637684345245361,
      "rewards/rejected": -2.753415584564209,
      "step": 980
    },
    {
      "dpo_losses": 0.09841219335794449,
      "epoch": 2.79,
      "grad_norm": 3.551041227418815,
      "learning_rate": 7.523343956923196e-08,
      "logits/chosen": -2.4305670261383057,
      "logits/rejected": -2.3597071170806885,
      "logps/chosen": -302.9432678222656,
      "logps/rejected": -600.987060546875,
      "loss": 0.2684,
      "positive_losses": 1.013157606124878,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.49134930968284607,
      "rewards/margins": 3.6477818489074707,
      "rewards/margins_max": 4.790149688720703,
      "rewards/margins_min": 2.5054140090942383,
      "rewards/margins_std": 1.6155517101287842,
      "rewards/rejected": -3.156432628631592,
      "step": 990
    },
    {
      "dpo_losses": 0.11016283929347992,
      "epoch": 2.82,
      "grad_norm": 17.885804652199482,
      "learning_rate": 5.657967707312195e-08,
      "logits/chosen": -2.2564501762390137,
      "logits/rejected": -2.223057985305786,
      "logps/chosen": -243.24911499023438,
      "logps/rejected": -581.9591064453125,
      "loss": 0.1369,
      "positive_losses": 0.5390418171882629,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3867679238319397,
      "rewards/margins": 3.3442111015319824,
      "rewards/margins_max": 4.438827991485596,
      "rewards/margins_min": 2.2495944499969482,
      "rewards/margins_std": 1.5480217933654785,
      "rewards/rejected": -2.9574437141418457,
      "step": 1000
    },
    {
      "epoch": 2.82,
      "eval_dpo_losses": 0.6868842244148254,
      "eval_logits/chosen": -2.3165292739868164,
      "eval_logits/rejected": -2.2738356590270996,
      "eval_logps/chosen": -428.1341857910156,
      "eval_logps/rejected": -461.68865966796875,
      "eval_loss": 14.59554386138916,
      "eval_positive_losses": 143.93893432617188,
      "eval_rewards/accuracies": 0.6626983880996704,
      "eval_rewards/chosen": -1.4291293621063232,
      "eval_rewards/margins": 0.5959304571151733,
      "eval_rewards/margins_max": 2.2952685356140137,
      "eval_rewards/margins_min": -1.107314109802246,
      "eval_rewards/margins_std": 1.5052008628845215,
      "eval_rewards/rejected": -2.025059700012207,
      "eval_runtime": 280.8142,
      "eval_samples_per_second": 7.122,
      "eval_steps_per_second": 0.224,
      "step": 1000
    },
    {
      "dpo_losses": 0.13428188860416412,
      "epoch": 2.85,
      "grad_norm": 223.7670035438648,
      "learning_rate": 4.055332542531959e-08,
      "logits/chosen": -2.418339490890503,
      "logits/rejected": -2.3237392902374268,
      "logps/chosen": -274.25958251953125,
      "logps/rejected": -552.5701904296875,
      "loss": 0.4402,
      "positive_losses": 0.35679930448532104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.41407138109207153,
      "rewards/margins": 3.3448989391326904,
      "rewards/margins_max": 4.5948896408081055,
      "rewards/margins_min": 2.094907522201538,
      "rewards/margins_std": 1.7677549123764038,
      "rewards/rejected": -2.9308273792266846,
      "step": 1010
    },
    {
      "dpo_losses": 0.09611718356609344,
      "epoch": 2.87,
      "grad_norm": 4.2164881541545896,
      "learning_rate": 2.7171617768147472e-08,
      "logits/chosen": -2.3242077827453613,
      "logits/rejected": -2.20931339263916,
      "logps/chosen": -273.04669189453125,
      "logps/rejected": -532.8912963867188,
      "loss": 0.2835,
      "positive_losses": 1.0188411474227905,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.38434693217277527,
      "rewards/margins": 3.382856845855713,
      "rewards/margins_max": 4.1625471115112305,
      "rewards/margins_min": 2.6031665802001953,
      "rewards/margins_std": 1.1026487350463867,
      "rewards/rejected": -2.9985098838806152,
      "step": 1020
    },
    {
      "dpo_losses": 0.09702328592538834,
      "epoch": 2.9,
      "grad_norm": 37.444607526485036,
      "learning_rate": 1.6448943457189616e-08,
      "logits/chosen": -2.3770041465759277,
      "logits/rejected": -2.312561511993408,
      "logps/chosen": -323.7328186035156,
      "logps/rejected": -590.1715698242188,
      "loss": 0.1692,
      "positive_losses": 0.22979411482810974,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4143516421318054,
      "rewards/margins": 3.550870418548584,
      "rewards/margins_max": 4.831820487976074,
      "rewards/margins_min": 2.269920825958252,
      "rewards/margins_std": 1.8115367889404297,
      "rewards/rejected": -3.136518955230713,
      "step": 1030
    },
    {
      "dpo_losses": 0.05908944085240364,
      "epoch": 2.93,
      "grad_norm": 4.414870466706013,
      "learning_rate": 8.39683258841123e-09,
      "logits/chosen": -2.2308764457702637,
      "logits/rejected": -2.1003758907318115,
      "logps/chosen": -295.9768371582031,
      "logps/rejected": -529.7901611328125,
      "loss": 0.2377,
      "positive_losses": 3.635768175125122,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5024327039718628,
      "rewards/margins": 3.5135796070098877,
      "rewards/margins_max": 4.2515668869018555,
      "rewards/margins_min": 2.7755913734436035,
      "rewards/margins_std": 1.0436723232269287,
      "rewards/rejected": -3.0111465454101562,
      "step": 1040
    },
    {
      "dpo_losses": 0.13825824856758118,
      "epoch": 2.96,
      "grad_norm": 2.4504883804901993,
      "learning_rate": 3.0239435998430376e-09,
      "logits/chosen": -2.348167896270752,
      "logits/rejected": -2.233564615249634,
      "logps/chosen": -286.636962890625,
      "logps/rejected": -520.432861328125,
      "loss": 0.2494,
      "positive_losses": 3.874138593673706,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4494766294956207,
      "rewards/margins": 3.075880527496338,
      "rewards/margins_max": 4.244940280914307,
      "rewards/margins_min": 1.9068210124969482,
      "rewards/margins_std": 1.6533000469207764,
      "rewards/rejected": -2.62640380859375,
      "step": 1050
    },
    {
      "dpo_losses": 0.10627589374780655,
      "epoch": 2.99,
      "grad_norm": 2.8525382266880834,
      "learning_rate": 3.3605396115826695e-10,
      "logits/chosen": -2.2836787700653076,
      "logits/rejected": -2.183605432510376,
      "logps/chosen": -289.5533447265625,
      "logps/rejected": -472.84375,
      "loss": 0.3088,
      "positive_losses": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3630369305610657,
      "rewards/margins": 2.99712872505188,
      "rewards/margins_max": 3.9373364448547363,
      "rewards/margins_min": 2.0569205284118652,
      "rewards/margins_std": 1.3296549320220947,
      "rewards/rejected": -2.63409161567688,
      "step": 1060
    },
    {
      "epoch": 3.0,
      "step": 1065,
      "total_flos": 0.0,
      "train_loss": 0.36563416943303856,
      "train_runtime": 9271.2095,
      "train_samples_per_second": 1.837,
      "train_steps_per_second": 0.115
    }
  ],
  "logging_steps": 10,
  "max_steps": 1065,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}