{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9974099364257123,
  "eval_steps": 50,
  "global_step": 353,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "chosen_logps": -92.0396499633789,
      "chosen_rewards": 0.0,
      "epoch": 0.0028255238992229807,
      "grad_norm": 12.203563280219937,
      "learning_rate": 2.7777777777777774e-08,
      "log_diff_policy": 2.422942638397217,
      "logits": -1.2872235774993896,
      "logp_accuracy": 0.625,
      "loss": 0.6931,
      "objective": 0.6931471824645996,
      "ranking_simple": 0.625,
      "rejected_logps": -94.46258544921875,
      "rejected_rewards": 0.0,
      "reward_accuracy": 0.0,
      "step": 1
    },
    {
      "chosen_logps": -94.69133758544922,
      "chosen_rewards": -0.00179989542812109,
      "epoch": 0.014127619496114905,
      "grad_norm": 13.769130010396626,
      "learning_rate": 1.3888888888888888e-07,
      "log_diff_policy": -0.9685585498809814,
      "logits": -1.223134994506836,
      "logp_accuracy": 0.4739583432674408,
      "loss": 0.6927,
      "objective": 0.6933824419975281,
      "ranking_simple": 0.4739583432674408,
      "rejected_logps": -93.7227783203125,
      "rejected_rewards": -0.0013557692291215062,
      "reward_accuracy": 0.3697916567325592,
      "step": 5
    },
    {
      "chosen_logps": -95.4830551147461,
      "chosen_rewards": -0.02674178034067154,
      "epoch": 0.02825523899222981,
      "grad_norm": 13.285605890484408,
      "learning_rate": 2.7777777777777776e-07,
      "log_diff_policy": -1.0804779529571533,
      "logits": -1.239374041557312,
      "logp_accuracy": 0.4833333194255829,
      "loss": 0.691,
      "objective": 0.6885988712310791,
      "ranking_simple": 0.4833333194255829,
      "rejected_logps": -94.40258026123047,
      "rejected_rewards": -0.03924858197569847,
      "reward_accuracy": 0.5416666865348816,
      "step": 10
    },
    {
      "chosen_logps": -95.61571502685547,
      "chosen_rewards": -0.051721397787332535,
      "epoch": 0.042382858488344714,
      "grad_norm": 12.319792066922028,
      "learning_rate": 4.1666666666666667e-07,
      "log_diff_policy": -0.5638642907142639,
      "logits": -1.2099182605743408,
      "logp_accuracy": 0.5,
      "loss": 0.6872,
      "objective": 0.6906775236129761,
      "ranking_simple": 0.5,
      "rejected_logps": -95.0518569946289,
      "rejected_rewards": -0.06499442458152771,
      "reward_accuracy": 0.5375000238418579,
      "step": 15
    },
    {
      "chosen_logps": -96.26466369628906,
      "chosen_rewards": 0.06014590337872505,
      "epoch": 0.05651047798445962,
      "grad_norm": 12.196449773003847,
      "learning_rate": 5.555555555555555e-07,
      "log_diff_policy": 0.15712101757526398,
      "logits": -1.3111374378204346,
      "logp_accuracy": 0.5166666507720947,
      "loss": 0.6904,
      "objective": 0.6839234828948975,
      "ranking_simple": 0.5166666507720947,
      "rejected_logps": -96.42178344726562,
      "rejected_rewards": 0.029283961281180382,
      "reward_accuracy": 0.5791666507720947,
      "step": 20
    },
    {
      "chosen_logps": -89.93444061279297,
      "chosen_rewards": 0.19082187116146088,
      "epoch": 0.07063809748057452,
      "grad_norm": 11.658480489992849,
      "learning_rate": 6.944444444444444e-07,
      "log_diff_policy": 1.8696421384811401,
      "logits": -1.275890588760376,
      "logp_accuracy": 0.5333333611488342,
      "loss": 0.6882,
      "objective": 0.6797720789909363,
      "ranking_simple": 0.5333333611488342,
      "rejected_logps": -91.80408477783203,
      "rejected_rewards": 0.14768332242965698,
      "reward_accuracy": 0.5916666388511658,
      "step": 25
    },
    {
      "chosen_logps": -91.81427764892578,
      "chosen_rewards": 0.14018863439559937,
      "epoch": 0.08476571697668943,
      "grad_norm": 15.037591193931753,
      "learning_rate": 8.333333333333333e-07,
      "log_diff_policy": 2.39349102973938,
      "logits": -1.212536334991455,
      "logp_accuracy": 0.5458333492279053,
      "loss": 0.6845,
      "objective": 0.6916062831878662,
      "ranking_simple": 0.5458333492279053,
      "rejected_logps": -94.20777130126953,
      "rejected_rewards": 0.11354698240756989,
      "reward_accuracy": 0.5458333492279053,
      "step": 30
    },
    {
      "chosen_logps": -90.74636840820312,
      "chosen_rewards": 0.15737393498420715,
      "epoch": 0.09889333647280434,
      "grad_norm": 10.604933845962982,
      "learning_rate": 9.722222222222222e-07,
      "log_diff_policy": 1.6841083765029907,
      "logits": -1.174207329750061,
      "logp_accuracy": 0.512499988079071,
      "loss": 0.671,
      "objective": 0.6680408716201782,
      "ranking_simple": 0.512499988079071,
      "rejected_logps": -92.43048858642578,
      "rejected_rewards": 0.08380100876092911,
      "reward_accuracy": 0.612500011920929,
      "step": 35
    },
    {
      "chosen_logps": -95.68059539794922,
      "chosen_rewards": 0.0697811022400856,
      "epoch": 0.11302095596891924,
      "grad_norm": 11.456756804174194,
      "learning_rate": 9.996071883688332e-07,
      "log_diff_policy": -0.9100133180618286,
      "logits": -1.1049649715423584,
      "logp_accuracy": 0.48750001192092896,
      "loss": 0.6785,
      "objective": 0.6945610642433167,
      "ranking_simple": 0.48750001192092896,
      "rejected_logps": -94.77057647705078,
      "rejected_rewards": 0.040437690913677216,
      "reward_accuracy": 0.5166666507720947,
      "step": 40
    },
    {
      "chosen_logps": -93.884765625,
      "chosen_rewards": -0.09519442170858383,
      "epoch": 0.12714857546503414,
      "grad_norm": 12.792596919876136,
      "learning_rate": 9.980124488638773e-07,
      "log_diff_policy": 1.2741607427597046,
      "logits": -1.2097392082214355,
      "logp_accuracy": 0.512499988079071,
      "loss": 0.6877,
      "objective": 0.6832014918327332,
      "ranking_simple": 0.512499988079071,
      "rejected_logps": -95.158935546875,
      "rejected_rewards": -0.15457913279533386,
      "reward_accuracy": 0.5083333253860474,
      "step": 45
    },
    {
      "chosen_logps": -96.72931671142578,
      "chosen_rewards": -0.19579552114009857,
      "epoch": 0.14127619496114904,
      "grad_norm": 11.519646507473169,
      "learning_rate": 9.95195142656885e-07,
      "log_diff_policy": 0.6534870862960815,
      "logits": -1.207014799118042,
      "logp_accuracy": 0.5208333134651184,
      "loss": 0.6657,
      "objective": 0.6546425223350525,
      "ranking_simple": 0.5208333134651184,
      "rejected_logps": -97.38280487060547,
      "rejected_rewards": -0.30787384510040283,
      "reward_accuracy": 0.5791666507720947,
      "step": 50
    },
    {
      "epoch": 0.14127619496114904,
      "eval_chosen_logps": -96.49317932128906,
      "eval_chosen_rewards": -0.23376186192035675,
      "eval_log_diff_policy": 1.5223942995071411,
      "eval_logits": -1.105320930480957,
      "eval_logp_accuracy": 0.5193236470222473,
      "eval_loss": 0.6741412878036499,
      "eval_objective": 0.6712923049926758,
      "eval_ranking_simple": 0.5193236470222473,
      "eval_rejected_logps": -98.01555633544922,
      "eval_rejected_rewards": -0.3036104738712311,
      "eval_reward_accuracy": 0.5851449370384216,
      "eval_runtime": 591.2486,
      "eval_samples_per_second": 16.8,
      "eval_steps_per_second": 0.7,
      "step": 50
    },
    {
      "chosen_logps": -95.2642593383789,
      "chosen_rewards": -0.2244856208562851,
      "epoch": 0.15540381445726395,
      "grad_norm": 11.816802501387212,
      "learning_rate": 9.91162185929904e-07,
      "log_diff_policy": 2.4890987873077393,
      "logits": -1.2382166385650635,
      "logp_accuracy": 0.5666666626930237,
      "loss": 0.6625,
      "objective": 0.6524732708930969,
      "ranking_simple": 0.5666666626930237,
      "rejected_logps": -97.75334930419922,
      "rejected_rewards": -0.3347455561161041,
      "reward_accuracy": 0.6541666388511658,
      "step": 55
    },
    {
      "chosen_logps": -96.38868713378906,
      "chosen_rewards": -0.31856828927993774,
      "epoch": 0.16953143395337886,
      "grad_norm": 15.028016832871401,
      "learning_rate": 9.859234791555355e-07,
      "log_diff_policy": 2.37857985496521,
      "logits": -1.270559549331665,
      "logp_accuracy": 0.5041666626930237,
      "loss": 0.658,
      "objective": 0.6463251709938049,
      "ranking_simple": 0.5041666626930237,
      "rejected_logps": -98.76726531982422,
      "rejected_rewards": -0.4456353485584259,
      "reward_accuracy": 0.625,
      "step": 60
    },
    {
      "chosen_logps": -96.91040802001953,
      "chosen_rewards": -0.38393592834472656,
      "epoch": 0.18365905344949376,
      "grad_norm": 12.842079906907882,
      "learning_rate": 9.794918827923456e-07,
      "log_diff_policy": 2.754225730895996,
      "logits": -1.2090051174163818,
      "logp_accuracy": 0.5333333611488342,
      "loss": 0.6552,
      "objective": 0.659140944480896,
      "ranking_simple": 0.5333333611488342,
      "rejected_logps": -99.66463470458984,
      "rejected_rewards": -0.5064182877540588,
      "reward_accuracy": 0.5874999761581421,
      "step": 65
    },
    {
      "chosen_logps": -99.7697525024414,
      "chosen_rewards": -0.36345040798187256,
      "epoch": 0.19778667294560867,
      "grad_norm": 11.642459528097591,
      "learning_rate": 9.718831857138307e-07,
      "log_diff_policy": -0.3823479115962982,
      "logits": -1.2754501104354858,
      "logp_accuracy": 0.5166666507720947,
      "loss": 0.652,
      "objective": 0.6486606597900391,
      "ranking_simple": 0.5166666507720947,
      "rejected_logps": -99.38741302490234,
      "rejected_rewards": -0.5056981444358826,
      "reward_accuracy": 0.6541666388511658,
      "step": 70
    },
    {
      "chosen_logps": -98.25231170654297,
      "chosen_rewards": -0.44365358352661133,
      "epoch": 0.21191429244172358,
      "grad_norm": 13.778053208316393,
      "learning_rate": 9.631160664484398e-07,
      "log_diff_policy": 2.420900821685791,
      "logits": -1.3257941007614136,
      "logp_accuracy": 0.5416666865348816,
      "loss": 0.645,
      "objective": 0.6380378603935242,
      "ranking_simple": 0.5416666865348816,
      "rejected_logps": -100.67321014404297,
      "rejected_rewards": -0.6107330918312073,
      "reward_accuracy": 0.6416666507720947,
      "step": 75
    },
    {
      "chosen_logps": -102.11405181884766,
      "chosen_rewards": -0.5838820338249207,
      "epoch": 0.22604191193783849,
      "grad_norm": 11.54119422499371,
      "learning_rate": 9.532120473258074e-07,
      "log_diff_policy": 0.3980850577354431,
      "logits": -1.410662055015564,
      "logp_accuracy": 0.5291666388511658,
      "loss": 0.6505,
      "objective": 0.6602904796600342,
      "ranking_simple": 0.5291666388511658,
      "rejected_logps": -102.51213836669922,
      "rejected_rewards": -0.716762125492096,
      "reward_accuracy": 0.612500011920929,
      "step": 80
    },
    {
      "chosen_logps": -99.7666015625,
      "chosen_rewards": -0.6009302735328674,
      "epoch": 0.2401695314339534,
      "grad_norm": 12.196843414681535,
      "learning_rate": 9.421954416417624e-07,
      "log_diff_policy": 3.4806392192840576,
      "logits": -1.403199315071106,
      "logp_accuracy": 0.550000011920929,
      "loss": 0.6356,
      "objective": 0.6589958071708679,
      "ranking_simple": 0.550000011920929,
      "rejected_logps": -103.24723815917969,
      "rejected_rewards": -0.7273377776145935,
      "reward_accuracy": 0.6041666865348816,
      "step": 85
    },
    {
      "chosen_logps": -99.78125,
      "chosen_rewards": -0.6102154850959778,
      "epoch": 0.25429715093006827,
      "grad_norm": 11.884683628001579,
      "learning_rate": 9.300932939718157e-07,
      "log_diff_policy": 4.280076026916504,
      "logits": -1.3959789276123047,
      "logp_accuracy": 0.5708333253860474,
      "loss": 0.6393,
      "objective": 0.6565433144569397,
      "ranking_simple": 0.5708333253860474,
      "rejected_logps": -104.06131744384766,
      "rejected_rewards": -0.732826292514801,
      "reward_accuracy": 0.637499988079071,
      "step": 90
    },
    {
      "chosen_logps": -100.35063171386719,
      "chosen_rewards": -0.5101503133773804,
      "epoch": 0.2684247704261832,
      "grad_norm": 11.580908746020523,
      "learning_rate": 9.169353137796533e-07,
      "log_diff_policy": 2.76478910446167,
      "logits": -1.453112006187439,
      "logp_accuracy": 0.5541666746139526,
      "loss": 0.6437,
      "objective": 0.6427834033966064,
      "ranking_simple": 0.5541666746139526,
      "rejected_logps": -103.11542510986328,
      "rejected_rewards": -0.6783679127693176,
      "reward_accuracy": 0.6499999761581421,
      "step": 95
    },
    {
      "chosen_logps": -98.85354614257812,
      "chosen_rewards": -0.5268033146858215,
      "epoch": 0.2825523899222981,
      "grad_norm": 13.403248828603669,
      "learning_rate": 9.027538024836141e-07,
      "log_diff_policy": 3.670335531234741,
      "logits": -1.3224259614944458,
      "logp_accuracy": 0.5791666507720947,
      "loss": 0.6364,
      "objective": 0.6203222274780273,
      "ranking_simple": 0.5791666507720947,
      "rejected_logps": -102.5239028930664,
      "rejected_rewards": -0.7540122270584106,
      "reward_accuracy": 0.6333333253860474,
      "step": 100
    },
    {
      "epoch": 0.2825523899222981,
      "eval_chosen_logps": -100.81198120117188,
      "eval_chosen_rewards": -0.6656423211097717,
      "eval_log_diff_policy": 2.193570375442505,
      "eval_logits": -1.2277085781097412,
      "eval_logp_accuracy": 0.5404589176177979,
      "eval_loss": 0.6705255508422852,
      "eval_objective": 0.6646167635917664,
      "eval_ranking_simple": 0.5404589176177979,
      "eval_rejected_logps": -103.00554656982422,
      "eval_rejected_rewards": -0.8026086091995239,
      "eval_reward_accuracy": 0.5972222089767456,
      "eval_runtime": 581.9012,
      "eval_samples_per_second": 17.07,
      "eval_steps_per_second": 0.711,
      "step": 100
    },
    {
      "chosen_logps": -102.4442367553711,
      "chosen_rewards": -0.6342382431030273,
      "epoch": 0.296680009418413,
      "grad_norm": 13.882342204535156,
      "learning_rate": 8.875835741602029e-07,
      "log_diff_policy": 1.7450906038284302,
      "logits": -1.2776763439178467,
      "logp_accuracy": 0.5375000238418579,
      "loss": 0.6375,
      "objective": 0.638464093208313,
      "ranking_simple": 0.5375000238418579,
      "rejected_logps": -104.18933868408203,
      "rejected_rewards": -0.8383311629295349,
      "reward_accuracy": 0.6291666626930237,
      "step": 105
    },
    {
      "chosen_logps": -98.33605194091797,
      "chosen_rewards": -0.4787895083427429,
      "epoch": 0.3108076289145279,
      "grad_norm": 11.129672527849843,
      "learning_rate": 8.714618700792975e-07,
      "log_diff_policy": 2.9387338161468506,
      "logits": -1.260119915008545,
      "logp_accuracy": 0.5333333611488342,
      "loss": 0.639,
      "objective": 0.6362030506134033,
      "ranking_simple": 0.5333333611488342,
      "rejected_logps": -101.2747802734375,
      "rejected_rewards": -0.6714141964912415,
      "reward_accuracy": 0.6458333134651184,
      "step": 110
    },
    {
      "chosen_logps": -98.392822265625,
      "chosen_rewards": -0.3023277521133423,
      "epoch": 0.3249352484106428,
      "grad_norm": 11.136874519373743,
      "learning_rate": 8.544282672808578e-07,
      "log_diff_policy": 1.8658004999160767,
      "logits": -1.2985230684280396,
      "logp_accuracy": 0.5416666865348816,
      "loss": 0.6397,
      "objective": 0.6552284359931946,
      "ranking_simple": 0.5416666865348816,
      "rejected_logps": -100.25862884521484,
      "rejected_rewards": -0.44404318928718567,
      "reward_accuracy": 0.637499988079071,
      "step": 115
    },
    {
      "chosen_logps": -97.15535736083984,
      "chosen_rewards": -0.30868878960609436,
      "epoch": 0.3390628679067577,
      "grad_norm": 10.158587216555272,
      "learning_rate": 8.365245814175743e-07,
      "log_diff_policy": 1.9471272230148315,
      "logits": -1.3143467903137207,
      "logp_accuracy": 0.5166666507720947,
      "loss": 0.6249,
      "objective": 0.6263554096221924,
      "ranking_simple": 0.5166666507720947,
      "rejected_logps": -99.10248565673828,
      "rejected_rewards": -0.5047985911369324,
      "reward_accuracy": 0.675000011920929,
      "step": 120
    },
    {
      "chosen_logps": -97.54443359375,
      "chosen_rewards": -0.41942229866981506,
      "epoch": 0.3531904874028726,
      "grad_norm": 11.091561717382723,
      "learning_rate": 8.17794764101962e-07,
      "log_diff_policy": 2.919205665588379,
      "logits": -1.3511691093444824,
      "logp_accuracy": 0.5625,
      "loss": 0.6224,
      "objective": 0.6153813004493713,
      "ranking_simple": 0.5625,
      "rejected_logps": -100.46363830566406,
      "rejected_rewards": -0.6644426584243774,
      "reward_accuracy": 0.6291666626930237,
      "step": 125
    },
    {
      "chosen_logps": -98.49415588378906,
      "chosen_rewards": -0.6381217241287231,
      "epoch": 0.36731810689898753,
      "grad_norm": 13.19781497022718,
      "learning_rate": 7.982847950099055e-07,
      "log_diff_policy": 2.8969905376434326,
      "logits": -1.2903234958648682,
      "logp_accuracy": 0.5625,
      "loss": 0.6365,
      "objective": 0.6469713449478149,
      "ranking_simple": 0.5625,
      "rejected_logps": -101.39115142822266,
      "rejected_rewards": -0.8170153498649597,
      "reward_accuracy": 0.6291666626930237,
      "step": 130
    },
    {
      "chosen_logps": -100.84342193603516,
      "chosen_rewards": -0.6817469000816345,
      "epoch": 0.38144572639510244,
      "grad_norm": 12.642228781148605,
      "learning_rate": 7.780425690055274e-07,
      "log_diff_policy": 2.6813039779663086,
      "logits": -1.3165868520736694,
      "logp_accuracy": 0.5166666507720947,
      "loss": 0.6271,
      "objective": 0.6345203518867493,
      "ranking_simple": 0.5166666507720947,
      "rejected_logps": -103.52472686767578,
      "rejected_rewards": -0.9179552793502808,
      "reward_accuracy": 0.5958333611488342,
      "step": 135
    },
    {
      "chosen_logps": -100.93543243408203,
      "chosen_rewards": -0.5448787808418274,
      "epoch": 0.39557334589121734,
      "grad_norm": 11.726885643648867,
      "learning_rate": 7.571177785644766e-07,
      "log_diff_policy": -0.2563031017780304,
      "logits": -1.3719837665557861,
      "logp_accuracy": 0.5208333134651184,
      "loss": 0.6264,
      "objective": 0.6360562443733215,
      "ranking_simple": 0.5208333134651184,
      "rejected_logps": -100.67912292480469,
      "rejected_rewards": -0.7725273370742798,
      "reward_accuracy": 0.6666666865348816,
      "step": 140
    },
    {
      "chosen_logps": -99.08710479736328,
      "chosen_rewards": -0.442624032497406,
      "epoch": 0.40970096538733225,
      "grad_norm": 12.322663872224792,
      "learning_rate": 7.35561791784275e-07,
      "log_diff_policy": 1.6613519191741943,
      "logits": -1.4259085655212402,
      "logp_accuracy": 0.5041666626930237,
      "loss": 0.6134,
      "objective": 0.6079509854316711,
      "ranking_simple": 0.5041666626930237,
      "rejected_logps": -100.74845123291016,
      "rejected_rewards": -0.7379883527755737,
      "reward_accuracy": 0.6708333492279053,
      "step": 145
    },
    {
      "chosen_logps": -97.87836456298828,
      "chosen_rewards": -0.4350048005580902,
      "epoch": 0.42382858488344716,
      "grad_norm": 11.246546091497954,
      "learning_rate": 7.134275262811934e-07,
      "log_diff_policy": 0.15529422461986542,
      "logits": -1.3733505010604858,
      "logp_accuracy": 0.5083333253860474,
      "loss": 0.6244,
      "objective": 0.6462458968162537,
      "ranking_simple": 0.5083333253860474,
      "rejected_logps": -98.03366088867188,
      "rejected_rewards": -0.6196256279945374,
      "reward_accuracy": 0.6333333253860474,
      "step": 150
    },
    {
      "epoch": 0.42382858488344716,
      "eval_chosen_logps": -98.61821746826172,
      "eval_chosen_rewards": -0.4462670087814331,
      "eval_log_diff_policy": 2.290201187133789,
      "eval_logits": -1.3178818225860596,
      "eval_logp_accuracy": 0.5368357300758362,
      "eval_loss": 0.6577388048171997,
      "eval_objective": 0.6551039814949036,
      "eval_ranking_simple": 0.5368357300758362,
      "eval_rejected_logps": -100.90841674804688,
      "eval_rejected_rewards": -0.5928963422775269,
      "eval_reward_accuracy": 0.6086956262588501,
      "eval_runtime": 578.9907,
      "eval_samples_per_second": 17.156,
      "eval_steps_per_second": 0.715,
      "step": 150
    },
    {
      "chosen_logps": -96.16273498535156,
      "chosen_rewards": -0.3375628888607025,
      "epoch": 0.43795620437956206,
      "grad_norm": 10.921897162913947,
      "learning_rate": 6.907693192832262e-07,
      "log_diff_policy": 3.625213146209717,
      "logits": -1.4034603834152222,
      "logp_accuracy": 0.5458333492279053,
      "loss": 0.6199,
      "objective": 0.6075600981712341,
      "ranking_simple": 0.5458333492279053,
      "rejected_logps": -99.7879409790039,
      "rejected_rewards": -0.5962837934494019,
      "reward_accuracy": 0.6916666626930237,
      "step": 155
    },
    {
      "chosen_logps": -95.9808349609375,
      "chosen_rewards": -0.23865890502929688,
      "epoch": 0.45208382387567697,
      "grad_norm": 10.607229520458072,
      "learning_rate": 6.676427942380741e-07,
      "log_diff_policy": 3.0014472007751465,
      "logits": -1.33468759059906,
      "logp_accuracy": 0.512499988079071,
      "loss": 0.6085,
      "objective": 0.6010857820510864,
      "ranking_simple": 0.512499988079071,
      "rejected_logps": -98.9822769165039,
      "rejected_rewards": -0.5223438739776611,
      "reward_accuracy": 0.6458333134651184,
      "step": 160
    },
    {
      "chosen_logps": -99.7357406616211,
      "chosen_rewards": -0.28111231327056885,
      "epoch": 0.4662114433717919,
      "grad_norm": 13.093121089444578,
      "learning_rate": 6.441047242635946e-07,
      "log_diff_policy": 2.3945486545562744,
      "logits": -1.4235426187515259,
      "logp_accuracy": 0.5208333134651184,
      "loss": 0.608,
      "objective": 0.6093403100967407,
      "ranking_simple": 0.5208333134651184,
      "rejected_logps": -102.13028717041016,
      "rejected_rewards": -0.570986807346344,
      "reward_accuracy": 0.6666666865348816,
      "step": 165
    },
    {
      "chosen_logps": -97.75370788574219,
      "chosen_rewards": -0.318727046251297,
      "epoch": 0.4803390628679068,
      "grad_norm": 11.412190377149692,
      "learning_rate": 6.20212892775939e-07,
      "log_diff_policy": 1.4041647911071777,
      "logits": -1.3291094303131104,
      "logp_accuracy": 0.5166666507720947,
      "loss": 0.6015,
      "objective": 0.6086380481719971,
      "ranking_simple": 0.5166666507720947,
      "rejected_logps": -99.15787506103516,
      "rejected_rewards": -0.5835815072059631,
      "reward_accuracy": 0.6791666746139526,
      "step": 170
    },
    {
      "chosen_logps": -99.83616638183594,
      "chosen_rewards": -0.4011620283126831,
      "epoch": 0.49446668236402164,
      "grad_norm": 12.49488940662025,
      "learning_rate": 5.960259516375133e-07,
      "log_diff_policy": 4.319859027862549,
      "logits": -1.42414128780365,
      "logp_accuracy": 0.5791666507720947,
      "loss": 0.6082,
      "objective": 0.5974529385566711,
      "ranking_simple": 0.5791666507720947,
      "rejected_logps": -104.1560287475586,
      "rejected_rewards": -0.6947523951530457,
      "reward_accuracy": 0.6916666626930237,
      "step": 175
    },
    {
      "chosen_logps": -99.31816864013672,
      "chosen_rewards": -0.44157731533050537,
      "epoch": 0.5085943018601365,
      "grad_norm": 14.016856666647364,
      "learning_rate": 5.716032771730007e-07,
      "log_diff_policy": 2.4290826320648193,
      "logits": -1.3892205953598022,
      "logp_accuracy": 0.5833333134651184,
      "loss": 0.6092,
      "objective": 0.59227454662323,
      "ranking_simple": 0.5833333134651184,
      "rejected_logps": -101.74726104736328,
      "rejected_rewards": -0.7656379342079163,
      "reward_accuracy": 0.7124999761581421,
      "step": 180
    },
    {
      "chosen_logps": -99.5169906616211,
      "chosen_rewards": -0.39382320642471313,
      "epoch": 0.5227219213562515,
      "grad_norm": 12.18994561625345,
      "learning_rate": 5.470048244069055e-07,
      "log_diff_policy": 3.986778974533081,
      "logits": -1.364132285118103,
      "logp_accuracy": 0.5625,
      "loss": 0.6046,
      "objective": 0.5779610276222229,
      "ranking_simple": 0.5625,
      "rejected_logps": -103.50377655029297,
      "rejected_rewards": -0.7597634792327881,
      "reward_accuracy": 0.7250000238418579,
      "step": 185
    },
    {
      "chosen_logps": -100.92544555664062,
      "chosen_rewards": -0.47165414690971375,
      "epoch": 0.5368495408523664,
      "grad_norm": 13.57415809299262,
      "learning_rate": 5.222909798804514e-07,
      "log_diff_policy": 1.5828216075897217,
      "logits": -1.4004924297332764,
      "logp_accuracy": 0.5583333373069763,
      "loss": 0.6109,
      "objective": 0.6085138916969299,
      "ranking_simple": 0.5583333373069763,
      "rejected_logps": -102.50827026367188,
      "rejected_rewards": -0.7680691480636597,
      "reward_accuracy": 0.6333333253860474,
      "step": 190
    },
    {
      "chosen_logps": -98.55032348632812,
      "chosen_rewards": -0.42258918285369873,
      "epoch": 0.5509771603484813,
      "grad_norm": 12.118293622606005,
      "learning_rate": 4.97522413409155e-07,
      "log_diff_policy": 2.5970427989959717,
      "logits": -1.3773627281188965,
      "logp_accuracy": 0.5458333492279053,
      "loss": 0.5975,
      "objective": 0.578415036201477,
      "ranking_simple": 0.5458333492279053,
      "rejected_logps": -101.1473617553711,
      "rejected_rewards": -0.7796388864517212,
      "reward_accuracy": 0.7291666865348816,
      "step": 195
    },
    {
      "chosen_logps": -99.49516296386719,
      "chosen_rewards": -0.38766908645629883,
      "epoch": 0.5651047798445962,
      "grad_norm": 11.832876951855871,
      "learning_rate": 4.7275992914498865e-07,
      "log_diff_policy": 0.977001428604126,
      "logits": -1.3331760168075562,
      "logp_accuracy": 0.5083333253860474,
      "loss": 0.5938,
      "objective": 0.6072806119918823,
      "ranking_simple": 0.5083333253860474,
      "rejected_logps": -100.47218322753906,
      "rejected_rewards": -0.6541637182235718,
      "reward_accuracy": 0.637499988079071,
      "step": 200
    },
    {
      "epoch": 0.5651047798445962,
      "eval_chosen_logps": -99.37232208251953,
      "eval_chosen_rewards": -0.5216771960258484,
      "eval_log_diff_policy": 2.475198745727539,
      "eval_logits": -1.2857621908187866,
      "eval_logp_accuracy": 0.5362318754196167,
      "eval_loss": 0.6590211391448975,
      "eval_objective": 0.6558353304862976,
      "eval_ranking_simple": 0.5362318754196167,
      "eval_rejected_logps": -101.84752655029297,
      "eval_rejected_rewards": -0.6868062615394592,
      "eval_reward_accuracy": 0.6159420013427734,
      "eval_runtime": 585.2119,
      "eval_samples_per_second": 16.973,
      "eval_steps_per_second": 0.707,
      "step": 200
    },
    {
      "chosen_logps": -96.24632263183594,
      "chosen_rewards": -0.3795066773891449,
      "epoch": 0.5792323993407111,
      "grad_norm": 11.462490179104472,
      "learning_rate": 4.4806431630876436e-07,
      "log_diff_policy": 4.63961124420166,
      "logits": -1.266701102256775,
      "logp_accuracy": 0.550000011920929,
      "loss": 0.6066,
      "objective": 0.6177704334259033,
      "ranking_simple": 0.550000011920929,
      "rejected_logps": -100.88591766357422,
      "rejected_rewards": -0.6309120655059814,
      "reward_accuracy": 0.637499988079071,
      "step": 205
    },
    {
      "chosen_logps": -100.29016876220703,
      "chosen_rewards": -0.4374944269657135,
      "epoch": 0.593360018836826,
      "grad_norm": 11.468388905148808,
      "learning_rate": 4.234961999591705e-07,
      "log_diff_policy": 5.617704391479492,
      "logits": -1.4379286766052246,
      "logp_accuracy": 0.6083333492279053,
      "loss": 0.6018,
      "objective": 0.60161954164505,
      "ranking_simple": 0.6083333492279053,
      "rejected_logps": -105.90787506103516,
      "rejected_rewards": -0.7697920203208923,
      "reward_accuracy": 0.6583333611488342,
      "step": 210
    },
    {
      "chosen_logps": -98.36739349365234,
      "chosen_rewards": -0.47102805972099304,
      "epoch": 0.607487638332941,
      "grad_norm": 11.544777522117554,
      "learning_rate": 3.9911589216480955e-07,
      "log_diff_policy": 1.489052176475525,
      "logits": -1.4814375638961792,
      "logp_accuracy": 0.5041666626930237,
      "loss": 0.6048,
      "objective": 0.6275675296783447,
      "ranking_simple": 0.5041666626930237,
      "rejected_logps": -99.8564453125,
      "rejected_rewards": -0.7114861011505127,
      "reward_accuracy": 0.675000011920929,
      "step": 215
    },
    {
      "chosen_logps": -98.07539367675781,
      "chosen_rewards": -0.40747135877609253,
      "epoch": 0.6216152578290558,
      "grad_norm": 11.7772812850058,
      "learning_rate": 3.7498324394459245e-07,
      "log_diff_policy": 3.3520638942718506,
      "logits": -1.4648287296295166,
      "logp_accuracy": 0.550000011920929,
      "loss": 0.5986,
      "objective": 0.5826771855354309,
      "ranking_simple": 0.550000011920929,
      "rejected_logps": -101.42745971679688,
      "rejected_rewards": -0.748247504234314,
      "reward_accuracy": 0.6875,
      "step": 220
    },
    {
      "chosen_logps": -99.81163787841797,
      "chosen_rewards": -0.42688456177711487,
      "epoch": 0.6357428773251708,
      "grad_norm": 11.823543045006824,
      "learning_rate": 3.511574983399599e-07,
      "log_diff_policy": 3.882382392883301,
      "logits": -1.356088399887085,
      "logp_accuracy": 0.5916666388511658,
      "loss": 0.5795,
      "objective": 0.5695621967315674,
      "ranking_simple": 0.5916666388511658,
      "rejected_logps": -103.69401550292969,
      "rejected_rewards": -0.8101120591163635,
      "reward_accuracy": 0.7041666507720947,
      "step": 225
    },
    {
      "chosen_logps": -98.75402069091797,
      "chosen_rewards": -0.5289642214775085,
      "epoch": 0.6498704968212856,
      "grad_norm": 12.384550673986663,
      "learning_rate": 3.276971449796223e-07,
      "log_diff_policy": 3.189291477203369,
      "logits": -1.4445135593414307,
      "logp_accuracy": 0.550000011920929,
      "loss": 0.6056,
      "objective": 0.6145560145378113,
      "ranking_simple": 0.550000011920929,
      "rejected_logps": -101.94329071044922,
      "rejected_rewards": -0.8118120431900024,
      "reward_accuracy": 0.6625000238418579,
      "step": 230
    },
    {
      "chosen_logps": -100.55665588378906,
      "chosen_rewards": -0.4955964684486389,
      "epoch": 0.6639981163174005,
      "grad_norm": 11.935195296010395,
      "learning_rate": 3.046597764938481e-07,
      "log_diff_policy": 5.328332424163818,
      "logits": -1.4324222803115845,
      "logp_accuracy": 0.6083333492279053,
      "loss": 0.5993,
      "objective": 0.5901253819465637,
      "ranking_simple": 0.6083333492279053,
      "rejected_logps": -105.88497924804688,
      "rejected_rewards": -0.821084201335907,
      "reward_accuracy": 0.6958333253860474,
      "step": 235
    },
    {
      "chosen_logps": -98.99813079833984,
      "chosen_rewards": -0.4493686258792877,
      "epoch": 0.6781257358135154,
      "grad_norm": 14.08286875676345,
      "learning_rate": 2.8210194713078404e-07,
      "log_diff_policy": 2.7355611324310303,
      "logits": -1.379309892654419,
      "logp_accuracy": 0.5708333253860474,
      "loss": 0.6072,
      "objective": 0.6106851100921631,
      "ranking_simple": 0.5708333253860474,
      "rejected_logps": -101.73368072509766,
      "rejected_rewards": -0.7426342368125916,
      "reward_accuracy": 0.6708333492279053,
      "step": 240
    },
    {
      "chosen_logps": -98.74683380126953,
      "chosen_rewards": -0.36279481649398804,
      "epoch": 0.6922533553096303,
      "grad_norm": 11.505971149373092,
      "learning_rate": 2.600790339218926e-07,
      "log_diff_policy": 4.0673909187316895,
      "logits": -1.4577367305755615,
      "logp_accuracy": 0.5625,
      "loss": 0.5975,
      "objective": 0.5751992464065552,
      "ranking_simple": 0.5625,
      "rejected_logps": -102.8142318725586,
      "rejected_rewards": -0.728181779384613,
      "reward_accuracy": 0.7250000238418579,
      "step": 245
    },
    {
      "chosen_logps": -97.71234893798828,
      "chosen_rewards": -0.34638115763664246,
      "epoch": 0.7063809748057452,
      "grad_norm": 11.594437077167564,
      "learning_rate": 2.3864510073732915e-07,
      "log_diff_policy": 2.69929575920105,
      "logits": -1.3644834756851196,
      "logp_accuracy": 0.5458333492279053,
      "loss": 0.5876,
      "objective": 0.5611613988876343,
      "ranking_simple": 0.5458333492279053,
      "rejected_logps": -100.41165161132812,
      "rejected_rewards": -0.7614852786064148,
      "reward_accuracy": 0.7083333134651184,
      "step": 250
    },
    {
      "epoch": 0.7063809748057452,
      "eval_chosen_logps": -99.82035827636719,
      "eval_chosen_rewards": -0.566480278968811,
      "eval_log_diff_policy": 2.69968581199646,
      "eval_logits": -1.3214609622955322,
      "eval_logp_accuracy": 0.5446860194206238,
      "eval_loss": 0.6542993187904358,
      "eval_objective": 0.6504490971565247,
      "eval_ranking_simple": 0.5446860194206238,
      "eval_rejected_logps": -102.5200424194336,
      "eval_rejected_rewards": -0.7540581226348877,
      "eval_reward_accuracy": 0.6171497702598572,
      "eval_runtime": 580.2016,
      "eval_samples_per_second": 17.12,
      "eval_steps_per_second": 0.714,
      "step": 250
    },
    {
      "chosen_logps": -98.32154846191406,
      "chosen_rewards": -0.4018653333187103,
      "epoch": 0.7205085943018601,
      "grad_norm": 12.212780646567717,
      "learning_rate": 2.1785276556498678e-07,
      "log_diff_policy": 4.165632724761963,
      "logits": -1.4318310022354126,
      "logp_accuracy": 0.574999988079071,
      "loss": 0.5952,
      "objective": 0.6162487268447876,
      "ranking_simple": 0.574999988079071,
      "rejected_logps": -102.4871826171875,
      "rejected_rewards": -0.6852520108222961,
      "reward_accuracy": 0.6499999761581421,
      "step": 255
    },
    {
      "chosen_logps": -98.31494140625,
      "chosen_rewards": -0.39665502309799194,
      "epoch": 0.7346362137979751,
      "grad_norm": 12.485927390030215,
      "learning_rate": 1.9775307133902806e-07,
      "log_diff_policy": 2.1429994106292725,
      "logits": -1.3612717390060425,
      "logp_accuracy": 0.5708333253860474,
      "loss": 0.605,
      "objective": 0.6075990796089172,
      "ranking_simple": 0.5708333253860474,
      "rejected_logps": -100.45793151855469,
      "rejected_rewards": -0.6959177851676941,
      "reward_accuracy": 0.6416666507720947,
      "step": 260
    },
    {
      "chosen_logps": -96.93352508544922,
      "chosen_rewards": -0.4520092010498047,
      "epoch": 0.7487638332940899,
      "grad_norm": 12.390709218829844,
      "learning_rate": 1.783953606350005e-07,
      "log_diff_policy": 2.5331625938415527,
      "logits": -1.3640118837356567,
      "logp_accuracy": 0.5416666865348816,
      "loss": 0.5989,
      "objective": 0.6267301440238953,
      "ranking_simple": 0.5416666865348816,
      "rejected_logps": -99.46669006347656,
      "rejected_rewards": -0.7122800350189209,
      "reward_accuracy": 0.6416666507720947,
      "step": 265
    },
    {
      "chosen_logps": -99.09416961669922,
      "chosen_rewards": -0.469031423330307,
      "epoch": 0.7628914527902049,
      "grad_norm": 13.216199579795669,
      "learning_rate": 1.5982715453915079e-07,
      "log_diff_policy": 3.8577356338500977,
      "logits": -1.4884754419326782,
      "logp_accuracy": 0.5874999761581421,
      "loss": 0.5841,
      "objective": 0.6081915497779846,
      "ranking_simple": 0.5874999761581421,
      "rejected_logps": -102.95191192626953,
      "rejected_rewards": -0.7843472957611084,
      "reward_accuracy": 0.6791666746139526,
      "step": 270
    },
    {
      "chosen_logps": -100.04994201660156,
      "chosen_rewards": -0.4179188311100006,
      "epoch": 0.7770190722863197,
      "grad_norm": 11.947579077894389,
      "learning_rate": 1.4209403598929708e-07,
      "log_diff_policy": 3.1525373458862305,
      "logits": -1.424578309059143,
      "logp_accuracy": 0.5291666388511658,
      "loss": 0.5992,
      "objective": 0.6024218797683716,
      "ranking_simple": 0.5291666388511658,
      "rejected_logps": -103.2024917602539,
      "rejected_rewards": -0.7272025942802429,
      "reward_accuracy": 0.6708333492279053,
      "step": 275
    },
    {
      "chosen_logps": -100.48685455322266,
      "chosen_rewards": -0.4961775243282318,
      "epoch": 0.7911466917824347,
      "grad_norm": 12.469165432741715,
      "learning_rate": 1.2523953787364722e-07,
      "log_diff_policy": 1.6572238206863403,
      "logits": -1.3310333490371704,
      "logp_accuracy": 0.5458333492279053,
      "loss": 0.588,
      "objective": 0.6101647615432739,
      "ranking_simple": 0.5458333492279053,
      "rejected_logps": -102.14408111572266,
      "rejected_rewards": -0.7782385349273682,
      "reward_accuracy": 0.6875,
      "step": 280
    },
    {
      "chosen_logps": -98.08577728271484,
      "chosen_rewards": -0.48601558804512024,
      "epoch": 0.8052743112785495,
      "grad_norm": 12.31765364922954,
      "learning_rate": 1.0930503616226495e-07,
      "log_diff_policy": 3.3555397987365723,
      "logits": -1.4751113653182983,
      "logp_accuracy": 0.5916666388511658,
      "loss": 0.6025,
      "objective": 0.6169189214706421,
      "ranking_simple": 0.5916666388511658,
      "rejected_logps": -101.44131469726562,
      "rejected_rewards": -0.7556989789009094,
      "reward_accuracy": 0.6708333492279053,
      "step": 285
    },
    {
      "chosen_logps": -100.12785339355469,
      "chosen_rewards": -0.5353098511695862,
      "epoch": 0.8194019307746645,
      "grad_norm": 12.570256325871375,
      "learning_rate": 9.432964833353946e-08,
      "log_diff_policy": 3.7209763526916504,
      "logits": -1.4481749534606934,
      "logp_accuracy": 0.574999988079071,
      "loss": 0.5859,
      "objective": 0.5787585973739624,
      "ranking_simple": 0.574999988079071,
      "rejected_logps": -103.84882354736328,
      "rejected_rewards": -0.8981534242630005,
      "reward_accuracy": 0.6916666626930237,
      "step": 290
    },
    {
      "chosen_logps": -97.55231475830078,
      "chosen_rewards": -0.4495824873447418,
      "epoch": 0.8335295502707794,
      "grad_norm": 12.980085764300096,
      "learning_rate": 8.035013734500557e-08,
      "log_diff_policy": 6.113182544708252,
      "logits": -1.4768887758255005,
      "logp_accuracy": 0.6000000238418579,
      "loss": 0.5855,
      "objective": 0.5812606811523438,
      "ranking_simple": 0.6000000238418579,
      "rejected_logps": -103.66551208496094,
      "rejected_rewards": -0.8177840113639832,
      "reward_accuracy": 0.7041666507720947,
      "step": 295
    },
    {
      "chosen_logps": -100.5435562133789,
      "chosen_rewards": -0.47136637568473816,
      "epoch": 0.8476571697668943,
      "grad_norm": 11.431502808997152,
      "learning_rate": 6.740082138425962e-08,
      "log_diff_policy": 4.680129051208496,
      "logits": -1.441601037979126,
      "logp_accuracy": 0.5791666507720947,
      "loss": 0.5705,
      "objective": 0.542003333568573,
      "ranking_simple": 0.5791666507720947,
      "rejected_logps": -105.22367095947266,
      "rejected_rewards": -0.9355214238166809,
      "reward_accuracy": 0.7416666746139526,
      "step": 300
    },
    {
      "epoch": 0.8476571697668943,
      "eval_chosen_logps": -100.62620544433594,
      "eval_chosen_rewards": -0.6470655202865601,
      "eval_log_diff_policy": 2.7566163539886475,
      "eval_logits": -1.3272062540054321,
      "eval_logp_accuracy": 0.5446860194206238,
      "eval_loss": 0.6553571224212646,
      "eval_objective": 0.6504380106925964,
      "eval_ranking_simple": 0.5446860194206238,
      "eval_rejected_logps": -103.38282012939453,
      "eval_rejected_rewards": -0.8403363823890686,
      "eval_reward_accuracy": 0.6189613342285156,
      "eval_runtime": 577.7358,
      "eval_samples_per_second": 17.193,
      "eval_steps_per_second": 0.717,
      "step": 300
    },
    {
      "chosen_logps": -99.48948669433594,
      "chosen_rewards": -0.5248011350631714,
      "epoch": 0.8617847892630092,
      "grad_norm": 13.252372937407342,
      "learning_rate": 5.551348962151964e-08,
      "log_diff_policy": 2.7193262577056885,
      "logits": -1.4033145904541016,
      "logp_accuracy": 0.5625,
      "loss": 0.6042,
      "objective": 0.617713987827301,
      "ranking_simple": 0.5625,
      "rejected_logps": -102.20880889892578,
      "rejected_rewards": -0.8076351284980774,
      "reward_accuracy": 0.6541666388511658,
      "step": 305
    },
    {
      "chosen_logps": -98.88239288330078,
      "chosen_rewards": -0.5424375534057617,
      "epoch": 0.8759124087591241,
      "grad_norm": 11.513198778387665,
      "learning_rate": 4.471732417065144e-08,
      "log_diff_policy": 5.485601902008057,
      "logits": -1.4526277780532837,
      "logp_accuracy": 0.612500011920929,
      "loss": 0.5898,
      "objective": 0.5874204039573669,
      "ranking_simple": 0.612500011920929,
      "rejected_logps": -104.36799621582031,
      "rejected_rewards": -0.8886787295341492,
      "reward_accuracy": 0.6916666626930237,
      "step": 310
    },
    {
      "chosen_logps": -95.83468627929688,
      "chosen_rewards": -0.3822983205318451,
      "epoch": 0.890040028255239,
      "grad_norm": 12.210262508024565,
      "learning_rate": 3.503882845023387e-08,
      "log_diff_policy": 4.727508068084717,
      "logits": -1.4003602266311646,
      "logp_accuracy": 0.6000000238418579,
      "loss": 0.584,
      "objective": 0.5880329608917236,
      "ranking_simple": 0.6000000238418579,
      "rejected_logps": -100.56217956542969,
      "rejected_rewards": -0.7324024438858032,
      "reward_accuracy": 0.6916666626930237,
      "step": 315
    },
    {
      "chosen_logps": -100.13945770263672,
      "chosen_rewards": -0.40757814049720764,
      "epoch": 0.9041676477513539,
      "grad_norm": 11.629225422410343,
      "learning_rate": 2.65017621205339e-08,
      "log_diff_policy": 1.5231914520263672,
      "logits": -1.4287190437316895,
      "logp_accuracy": 0.5375000238418579,
      "loss": 0.5795,
      "objective": 0.5747238993644714,
      "ranking_simple": 0.5375000238418579,
      "rejected_logps": -101.66265106201172,
      "rejected_rewards": -0.7921539545059204,
      "reward_accuracy": 0.7124999761581421,
      "step": 320
    },
    {
      "chosen_logps": -98.47615814208984,
      "chosen_rewards": -0.4354442059993744,
      "epoch": 0.9182952672474688,
      "grad_norm": 11.50196158969457,
      "learning_rate": 1.9127082756109138e-08,
      "log_diff_policy": 4.035438060760498,
      "logits": -1.4835073947906494,
      "logp_accuracy": 0.5916666388511658,
      "loss": 0.5836,
      "objective": 0.5947951078414917,
      "ranking_simple": 0.5916666388511658,
      "rejected_logps": -102.51158905029297,
      "rejected_rewards": -0.7917211055755615,
      "reward_accuracy": 0.6791666746139526,
      "step": 325
    },
    {
      "chosen_logps": -98.91030883789062,
      "chosen_rewards": -0.5017069578170776,
      "epoch": 0.9324228867435838,
      "grad_norm": 11.552117625756214,
      "learning_rate": 1.293289439722961e-08,
      "log_diff_policy": 6.066530704498291,
      "logits": -1.376121163368225,
      "logp_accuracy": 0.6041666865348816,
      "loss": 0.5902,
      "objective": 0.6051034331321716,
      "ranking_simple": 0.6041666865348816,
      "rejected_logps": -104.97685241699219,
      "rejected_rewards": -0.7912607789039612,
      "reward_accuracy": 0.7041666507720947,
      "step": 330
    },
    {
      "chosen_logps": -100.66015625,
      "chosen_rewards": -0.5485278367996216,
      "epoch": 0.9465505062396986,
      "grad_norm": 11.501179850402595,
      "learning_rate": 7.934403106416243e-09,
      "log_diff_policy": 3.832066774368286,
      "logits": -1.4517931938171387,
      "logp_accuracy": 0.5583333373069763,
      "loss": 0.5689,
      "objective": 0.5892359614372253,
      "ranking_simple": 0.5583333373069763,
      "rejected_logps": -104.49221801757812,
      "rejected_rewards": -0.8999612331390381,
      "reward_accuracy": 0.7041666507720947,
      "step": 335
    },
    {
      "chosen_logps": -99.37032318115234,
      "chosen_rewards": -0.5224529504776001,
      "epoch": 0.9606781257358136,
      "grad_norm": 12.734967304317747,
      "learning_rate": 4.143879639202541e-09,
      "log_diff_policy": 1.4650262594223022,
      "logits": -1.4669277667999268,
      "logp_accuracy": 0.5375000238418579,
      "loss": 0.5857,
      "objective": 0.6081200838088989,
      "ranking_simple": 0.5375000238418579,
      "rejected_logps": -100.83534240722656,
      "rejected_rewards": -0.8256459832191467,
      "reward_accuracy": 0.637499988079071,
      "step": 340
    },
    {
      "chosen_logps": -98.70734405517578,
      "chosen_rewards": -0.5224943161010742,
      "epoch": 0.9748057452319284,
      "grad_norm": 12.0257535848254,
      "learning_rate": 1.5706293207561893e-09,
      "log_diff_policy": 4.583549976348877,
      "logits": -1.3987572193145752,
      "logp_accuracy": 0.5708333253860474,
      "loss": 0.585,
      "objective": 0.5911051034927368,
      "ranking_simple": 0.5708333253860474,
      "rejected_logps": -103.2908935546875,
      "rejected_rewards": -0.8825021982192993,
      "reward_accuracy": 0.6875,
      "step": 345
    },
    {
      "chosen_logps": -99.29621124267578,
      "chosen_rewards": -0.4423524737358093,
      "epoch": 0.9889333647280433,
      "grad_norm": 12.354645024534335,
      "learning_rate": 2.209692023126819e-10,
      "log_diff_policy": 2.835632801055908,
      "logits": -1.471450686454773,
      "logp_accuracy": 0.5,
      "loss": 0.5864,
      "objective": 0.5840578079223633,
      "ranking_simple": 0.5,
      "rejected_logps": -102.13184356689453,
      "rejected_rewards": -0.7969415187835693,
      "reward_accuracy": 0.75,
      "step": 350
    },
    {
      "epoch": 0.9889333647280433,
      "eval_chosen_logps": -100.49518585205078,
      "eval_chosen_rewards": -0.6339634656906128,
      "eval_log_diff_policy": 2.7547223567962646,
      "eval_logits": -1.327553391456604,
      "eval_logp_accuracy": 0.54347825050354,
      "eval_loss": 0.6549956798553467,
      "eval_objective": 0.6503260731697083,
      "eval_ranking_simple": 0.54347825050354,
      "eval_rejected_logps": -103.24991607666016,
      "eval_rejected_rewards": -0.8270449638366699,
      "eval_reward_accuracy": 0.6183574795722961,
      "eval_runtime": 582.9627,
      "eval_samples_per_second": 17.039,
      "eval_steps_per_second": 0.71,
      "step": 350
    },
    {
      "epoch": 0.9974099364257123,
      "step": 353,
      "total_flos": 0.0,
      "train_loss": 0.6203088365263034,
      "train_runtime": 17039.3075,
      "train_samples_per_second": 5.981,
      "train_steps_per_second": 0.021
    }
  ],
  "logging_steps": 5,
  "max_steps": 353,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}