{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.995276334435522,
  "eval_steps": 50,
  "global_step": 704,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "dpo_loss": 0.6931471824645996,
      "epoch": 0.002834199338686821,
      "grad_norm": 183.97025063743072,
      "learning_rate": 7.042253521126761e-08,
      "logits": -1.2867579460144043,
      "logps": -84.34933471679688,
      "loss": 0.6931,
      "objective": 0.6931471824645996,
      "ranking_simple": 0.5833333134651184,
      "step": 1
    },
    {
      "dpo_loss": 0.6891745924949646,
      "epoch": 0.014170996693434105,
      "grad_norm": 175.97134717956817,
      "learning_rate": 3.521126760563381e-07,
      "logits": -1.430897831916809,
      "logps": -83.63750457763672,
      "loss": 0.6935,
      "objective": 0.6891745924949646,
      "ranking_simple": 0.4895833432674408,
      "step": 5
    },
    {
      "dpo_loss": 0.6994127631187439,
      "epoch": 0.02834199338686821,
      "grad_norm": 210.24216843481642,
      "learning_rate": 7.042253521126762e-07,
      "logits": -1.4024795293807983,
      "logps": -84.05953216552734,
      "loss": 0.7084,
      "objective": 0.6994127631187439,
      "ranking_simple": 0.5791666507720947,
      "step": 10
    },
    {
      "dpo_loss": 0.7201023101806641,
      "epoch": 0.042512990080302314,
      "grad_norm": 217.21520494236623,
      "learning_rate": 1.0563380281690142e-06,
      "logits": -1.5396863222122192,
      "logps": -83.98353576660156,
      "loss": 0.6992,
      "objective": 0.7201023101806641,
      "ranking_simple": 0.5708333253860474,
      "step": 15
    },
    {
      "dpo_loss": 0.6394654512405396,
      "epoch": 0.05668398677373642,
      "grad_norm": 178.00062840147226,
      "learning_rate": 1.4084507042253523e-06,
      "logits": -1.3805947303771973,
      "logps": -82.83494567871094,
      "loss": 0.6982,
      "objective": 0.6394654512405396,
      "ranking_simple": 0.4833333194255829,
      "step": 20
    },
    {
      "dpo_loss": 0.7543553113937378,
      "epoch": 0.07085498346717052,
      "grad_norm": 232.13126995990035,
      "learning_rate": 1.7605633802816902e-06,
      "logits": -1.406501054763794,
      "logps": -83.180908203125,
      "loss": 0.7413,
      "objective": 0.7543553113937378,
      "ranking_simple": 0.5833333134651184,
      "step": 25
    },
    {
      "dpo_loss": 0.8550678491592407,
      "epoch": 0.08502598016060463,
      "grad_norm": 178.51107981018083,
      "learning_rate": 2.1126760563380285e-06,
      "logits": -1.450936198234558,
      "logps": -83.1689682006836,
      "loss": 0.8038,
      "objective": 0.8550678491592407,
      "ranking_simple": 0.4958333373069763,
      "step": 30
    },
    {
      "dpo_loss": 0.8976810574531555,
      "epoch": 0.09919697685403873,
      "grad_norm": 159.63407046559615,
      "learning_rate": 2.4647887323943666e-06,
      "logits": -1.4125274419784546,
      "logps": -83.51966857910156,
      "loss": 0.8446,
      "objective": 0.8976810574531555,
      "ranking_simple": 0.550000011920929,
      "step": 35
    },
    {
      "dpo_loss": 0.9427976608276367,
      "epoch": 0.11336797354747284,
      "grad_norm": 204.8220203673265,
      "learning_rate": 2.8169014084507046e-06,
      "logits": -1.370052456855774,
      "logps": -85.57767486572266,
      "loss": 0.8988,
      "objective": 0.9427976608276367,
      "ranking_simple": 0.5,
      "step": 40
    },
    {
      "dpo_loss": 0.9266307950019836,
      "epoch": 0.12753897024090693,
      "grad_norm": 166.47204979724842,
      "learning_rate": 3.1690140845070427e-06,
      "logits": -1.3562687635421753,
      "logps": -82.7292251586914,
      "loss": 0.9677,
      "objective": 0.9266307950019836,
      "ranking_simple": 0.5041666626930237,
      "step": 45
    },
    {
      "dpo_loss": 1.1937092542648315,
      "epoch": 0.14170996693434104,
      "grad_norm": 162.3910297296546,
      "learning_rate": 3.5211267605633804e-06,
      "logits": -1.3756095170974731,
      "logps": -83.8311538696289,
      "loss": 1.1219,
      "objective": 1.1937092542648315,
      "ranking_simple": 0.5249999761581421,
      "step": 50
    },
    {
      "epoch": 0.14170996693434104,
      "eval_dpo_loss": 1.127982258796692,
      "eval_logits": -1.3893742561340332,
      "eval_logps": -91.15036010742188,
      "eval_loss": 1.1406893730163574,
      "eval_objective": 1.127982258796692,
      "eval_ranking_simple": 0.5274327397346497,
      "eval_runtime": 605.2256,
      "eval_samples_per_second": 9.567,
      "eval_steps_per_second": 0.798,
      "step": 50
    },
    {
      "dpo_loss": 1.2814911603927612,
      "epoch": 0.15588096362777515,
      "grad_norm": 163.57940740751664,
      "learning_rate": 3.873239436619718e-06,
      "logits": -1.3777908086776733,
      "logps": -84.10151672363281,
      "loss": 1.1332,
      "objective": 1.2814911603927612,
      "ranking_simple": 0.5041666626930237,
      "step": 55
    },
    {
      "dpo_loss": 1.0759367942810059,
      "epoch": 0.17005196032120926,
      "grad_norm": 161.38779700951656,
      "learning_rate": 4.225352112676057e-06,
      "logits": -1.3657281398773193,
      "logps": -80.9452133178711,
      "loss": 1.1561,
      "objective": 1.0759367942810059,
      "ranking_simple": 0.5541666746139526,
      "step": 60
    },
    {
      "dpo_loss": 1.2120552062988281,
      "epoch": 0.18422295701464336,
      "grad_norm": 160.82431949475915,
      "learning_rate": 4.577464788732395e-06,
      "logits": -1.3958569765090942,
      "logps": -77.98729705810547,
      "loss": 1.1297,
      "objective": 1.2120552062988281,
      "ranking_simple": 0.574999988079071,
      "step": 65
    },
    {
      "dpo_loss": 1.182289958000183,
      "epoch": 0.19839395370807747,
      "grad_norm": 153.19784131936885,
      "learning_rate": 4.929577464788733e-06,
      "logits": -1.3969266414642334,
      "logps": -81.52046203613281,
      "loss": 1.1094,
      "objective": 1.182289958000183,
      "ranking_simple": 0.5791666507720947,
      "step": 70
    },
    {
      "dpo_loss": 1.1960618495941162,
      "epoch": 0.21256495040151158,
      "grad_norm": 146.00340475417016,
      "learning_rate": 4.999507384516835e-06,
      "logits": -1.3441658020019531,
      "logps": -80.9335708618164,
      "loss": 1.2518,
      "objective": 1.1960618495941162,
      "ranking_simple": 0.5833333134651184,
      "step": 75
    },
    {
      "dpo_loss": 1.1959224939346313,
      "epoch": 0.22673594709494568,
      "grad_norm": 164.14111542380238,
      "learning_rate": 4.997506466835171e-06,
      "logits": -1.4696060419082642,
      "logps": -78.75235748291016,
      "loss": 1.2283,
      "objective": 1.1959224939346313,
      "ranking_simple": 0.612500011920929,
      "step": 80
    },
    {
      "dpo_loss": 1.3274264335632324,
      "epoch": 0.2409069437883798,
      "grad_norm": 116.34306089867205,
      "learning_rate": 4.9939676896203576e-06,
      "logits": -1.3812180757522583,
      "logps": -77.95438385009766,
      "loss": 1.2276,
      "objective": 1.3274264335632324,
      "ranking_simple": 0.5833333134651184,
      "step": 85
    },
    {
      "dpo_loss": 1.0321040153503418,
      "epoch": 0.25507794048181387,
      "grad_norm": 130.12375600026587,
      "learning_rate": 4.9888932319026994e-06,
      "logits": -1.406469702720642,
      "logps": -78.24427795410156,
      "loss": 1.2313,
      "objective": 1.0321040153503418,
      "ranking_simple": 0.5916666388511658,
      "step": 90
    },
    {
      "dpo_loss": 1.0937633514404297,
      "epoch": 0.269248937175248,
      "grad_norm": 138.54409786316114,
      "learning_rate": 4.982286218320023e-06,
      "logits": -1.3431053161621094,
      "logps": -78.20173645019531,
      "loss": 1.0961,
      "objective": 1.0937633514404297,
      "ranking_simple": 0.5333333611488342,
      "step": 95
    },
    {
      "dpo_loss": 1.2458388805389404,
      "epoch": 0.2834199338686821,
      "grad_norm": 135.72867701135627,
      "learning_rate": 4.974150717193654e-06,
      "logits": -1.2822611331939697,
      "logps": -75.05729675292969,
      "loss": 1.3821,
      "objective": 1.2458388805389404,
      "ranking_simple": 0.5666666626930237,
      "step": 100
    },
    {
      "epoch": 0.2834199338686821,
      "eval_dpo_loss": 1.4947212934494019,
      "eval_logits": -1.3773702383041382,
      "eval_logps": -81.2234115600586,
      "eval_loss": 1.530389428138733,
      "eval_objective": 1.4947212934494019,
      "eval_ranking_simple": 0.5289855003356934,
      "eval_runtime": 599.6473,
      "eval_samples_per_second": 9.656,
      "eval_steps_per_second": 0.805,
      "step": 100
    },
    {
      "dpo_loss": 1.673114538192749,
      "epoch": 0.2975909305621162,
      "grad_norm": 165.43372686768916,
      "learning_rate": 4.964491738023321e-06,
      "logits": -1.4061511754989624,
      "logps": -76.94784545898438,
      "loss": 1.4977,
      "objective": 1.673114538192749,
      "ranking_simple": 0.5916666388511658,
      "step": 105
    },
    {
      "dpo_loss": 1.381745457649231,
      "epoch": 0.3117619272555503,
      "grad_norm": 134.94435406621707,
      "learning_rate": 4.953315228402512e-06,
      "logits": -1.2966018915176392,
      "logps": -75.49461364746094,
      "loss": 1.4307,
      "objective": 1.381745457649231,
      "ranking_simple": 0.5541666746139526,
      "step": 110
    },
    {
      "dpo_loss": 1.525147795677185,
      "epoch": 0.32593292394898443,
      "grad_norm": 146.42714865280993,
      "learning_rate": 4.9406280703561944e-06,
      "logits": -1.203226923942566,
      "logps": -73.4133529663086,
      "loss": 1.4613,
      "objective": 1.525147795677185,
      "ranking_simple": 0.5708333253860474,
      "step": 115
    },
    {
      "dpo_loss": 1.2190510034561157,
      "epoch": 0.3401039206424185,
      "grad_norm": 145.8596102648603,
      "learning_rate": 4.926438076103162e-06,
      "logits": -1.192163109779358,
      "logps": -72.815673828125,
      "loss": 1.3059,
      "objective": 1.2190510034561157,
      "ranking_simple": 0.5708333253860474,
      "step": 120
    },
    {
      "dpo_loss": 1.5624465942382812,
      "epoch": 0.35427491733585265,
      "grad_norm": 131.5829174213295,
      "learning_rate": 4.910753983245589e-06,
      "logits": -1.190179467201233,
      "logps": -73.50247192382812,
      "loss": 1.4015,
      "objective": 1.5624465942382812,
      "ranking_simple": 0.5541666746139526,
      "step": 125
    },
    {
      "dpo_loss": 1.3345781564712524,
      "epoch": 0.3684459140292867,
      "grad_norm": 121.07722918376957,
      "learning_rate": 4.893585449388786e-06,
      "logits": -1.1411052942276,
      "logps": -70.5329818725586,
      "loss": 1.4099,
      "objective": 1.3345781564712524,
      "ranking_simple": 0.6416666507720947,
      "step": 130
    },
    {
      "dpo_loss": 1.0789151191711426,
      "epoch": 0.3826169107227208,
      "grad_norm": 118.85794111154456,
      "learning_rate": 4.8749430461944536e-06,
      "logits": -1.088476300239563,
      "logps": -71.05619812011719,
      "loss": 1.2611,
      "objective": 1.0789151191711426,
      "ranking_simple": 0.5874999761581421,
      "step": 135
    },
    {
      "dpo_loss": 1.8989228010177612,
      "epoch": 0.39678790741615494,
      "grad_norm": 131.90266764687357,
      "learning_rate": 4.854838252871097e-06,
      "logits": -1.0452989339828491,
      "logps": -71.00385284423828,
      "loss": 1.5343,
      "objective": 1.8989228010177612,
      "ranking_simple": 0.574999988079071,
      "step": 140
    },
    {
      "dpo_loss": 1.576548457145691,
      "epoch": 0.410958904109589,
      "grad_norm": 140.01764826482653,
      "learning_rate": 4.833283449105609e-06,
      "logits": -1.1004178524017334,
      "logps": -73.08991241455078,
      "loss": 1.6065,
      "objective": 1.576548457145691,
      "ranking_simple": 0.625,
      "step": 145
    },
    {
      "dpo_loss": 1.3776977062225342,
      "epoch": 0.42512990080302315,
      "grad_norm": 126.62570019000385,
      "learning_rate": 4.810291907440382e-06,
      "logits": -1.0991231203079224,
      "logps": -74.44639587402344,
      "loss": 1.4062,
      "objective": 1.3776977062225342,
      "ranking_simple": 0.574999988079071,
      "step": 150
    },
    {
      "epoch": 0.42512990080302315,
      "eval_dpo_loss": 1.8192129135131836,
      "eval_logits": -1.1641029119491577,
      "eval_logps": -79.7786636352539,
      "eval_loss": 1.8817615509033203,
      "eval_objective": 1.8192129135131836,
      "eval_ranking_simple": 0.5429606437683105,
      "eval_runtime": 600.2453,
      "eval_samples_per_second": 9.646,
      "eval_steps_per_second": 0.805,
      "step": 150
    },
    {
      "dpo_loss": 1.2696728706359863,
      "epoch": 0.43930089749645723,
      "grad_norm": 122.22748192353309,
      "learning_rate": 4.785877785100633e-06,
      "logits": -1.187027096748352,
      "logps": -75.60051727294922,
      "loss": 1.3364,
      "objective": 1.2696728706359863,
      "ranking_simple": 0.5708333253860474,
      "step": 155
    },
    {
      "dpo_loss": 1.7951911687850952,
      "epoch": 0.45347189418989137,
      "grad_norm": 125.17292960366125,
      "learning_rate": 4.7600561152769795e-06,
      "logits": -1.1456806659698486,
      "logps": -74.9968032836914,
      "loss": 1.5773,
      "objective": 1.7951911687850952,
      "ranking_simple": 0.550000011920929,
      "step": 160
    },
    {
      "dpo_loss": 1.2346725463867188,
      "epoch": 0.46764289088332545,
      "grad_norm": 108.82881778945708,
      "learning_rate": 4.732842797868631e-06,
      "logits": -1.2308270931243896,
      "logps": -75.75040435791016,
      "loss": 1.3947,
      "objective": 1.2346725463867188,
      "ranking_simple": 0.6083333492279053,
      "step": 165
    },
    {
      "dpo_loss": 1.3481464385986328,
      "epoch": 0.4818138875767596,
      "grad_norm": 108.10724884444645,
      "learning_rate": 4.704254589692903e-06,
      "logits": -1.2144527435302734,
      "logps": -75.97994232177734,
      "loss": 1.305,
      "objective": 1.3481464385986328,
      "ranking_simple": 0.5583333373069763,
      "step": 170
    },
    {
      "dpo_loss": 1.4393560886383057,
      "epoch": 0.49598488427019366,
      "grad_norm": 104.8074907192406,
      "learning_rate": 4.6743090941670675e-06,
      "logits": -1.147178292274475,
      "logps": -76.960205078125,
      "loss": 1.3896,
      "objective": 1.4393560886383057,
      "ranking_simple": 0.5583333373069763,
      "step": 175
    },
    {
      "dpo_loss": 1.6493242979049683,
      "epoch": 0.5101558809636277,
      "grad_norm": 107.1323631395759,
      "learning_rate": 4.643024750468913e-06,
      "logits": -1.1330630779266357,
      "logps": -75.9026870727539,
      "loss": 1.3334,
      "objective": 1.6493242979049683,
      "ranking_simple": 0.5291666388511658,
      "step": 180
    },
    {
      "dpo_loss": 1.1641370058059692,
      "epoch": 0.5243268776570619,
      "grad_norm": 113.77830856113329,
      "learning_rate": 4.610420822182671e-06,
      "logits": -1.1537328958511353,
      "logps": -74.29253387451172,
      "loss": 1.1361,
      "objective": 1.1641370058059692,
      "ranking_simple": 0.5916666388511658,
      "step": 185
    },
    {
      "dpo_loss": 1.2755895853042603,
      "epoch": 0.538497874350496,
      "grad_norm": 109.44630427090762,
      "learning_rate": 4.576517385437315e-06,
      "logits": -1.1797126531600952,
      "logps": -74.75035858154297,
      "loss": 1.2583,
      "objective": 1.2755895853042603,
      "ranking_simple": 0.637499988079071,
      "step": 190
    },
    {
      "dpo_loss": 1.073716402053833,
      "epoch": 0.5526688710439301,
      "grad_norm": 98.88901766370496,
      "learning_rate": 4.541335316544514e-06,
      "logits": -1.1745376586914062,
      "logps": -74.90966796875,
      "loss": 1.2475,
      "objective": 1.073716402053833,
      "ranking_simple": 0.637499988079071,
      "step": 195
    },
    {
      "dpo_loss": 1.185225248336792,
      "epoch": 0.5668398677373642,
      "grad_norm": 94.03051977106067,
      "learning_rate": 4.5048962791438885e-06,
      "logits": -1.1321061849594116,
      "logps": -74.69174194335938,
      "loss": 1.2275,
      "objective": 1.185225248336792,
      "ranking_simple": 0.5791666507720947,
      "step": 200
    },
    {
      "epoch": 0.5668398677373642,
      "eval_dpo_loss": 1.9716603755950928,
      "eval_logits": -1.1289294958114624,
      "eval_logps": -77.9853515625,
      "eval_loss": 2.0358171463012695,
      "eval_objective": 1.9716603755950928,
      "eval_ranking_simple": 0.534679114818573,
      "eval_runtime": 601.7719,
      "eval_samples_per_second": 9.622,
      "eval_steps_per_second": 0.803,
      "step": 200
    },
    {
      "dpo_loss": 1.2547378540039062,
      "epoch": 0.5810108644307983,
      "grad_norm": 109.12199093644038,
      "learning_rate": 4.467222710863444e-06,
      "logits": -0.9467021226882935,
      "logps": -74.9859390258789,
      "loss": 1.2613,
      "objective": 1.2547378540039062,
      "ranking_simple": 0.5874999761581421,
      "step": 205
    },
    {
      "dpo_loss": 0.9887279868125916,
      "epoch": 0.5951818611242324,
      "grad_norm": 110.59494347158504,
      "learning_rate": 4.428337809503425e-06,
      "logits": -0.9453111290931702,
      "logps": -75.48457336425781,
      "loss": 1.2147,
      "objective": 0.9887279868125916,
      "ranking_simple": 0.5958333611488342,
      "step": 210
    },
    {
      "dpo_loss": 1.297851800918579,
      "epoch": 0.6093528578176665,
      "grad_norm": 95.93031839243162,
      "learning_rate": 4.388265518752085e-06,
      "logits": -0.9826800227165222,
      "logps": -75.59996032714844,
      "loss": 1.1658,
      "objective": 1.297851800918579,
      "ranking_simple": 0.574999988079071,
      "step": 215
    },
    {
      "dpo_loss": 1.0760146379470825,
      "epoch": 0.6235238545111006,
      "grad_norm": 93.17621109620359,
      "learning_rate": 4.347030513442168e-06,
      "logits": -1.0486137866973877,
      "logps": -75.58748626708984,
      "loss": 1.0822,
      "objective": 1.0760146379470825,
      "ranking_simple": 0.5333333611488342,
      "step": 220
    },
    {
      "dpo_loss": 1.1693772077560425,
      "epoch": 0.6376948512045347,
      "grad_norm": 97.88558873864356,
      "learning_rate": 4.304658184357186e-06,
      "logits": -1.0480105876922607,
      "logps": -75.87112426757812,
      "loss": 1.2411,
      "objective": 1.1693772077560425,
      "ranking_simple": 0.5791666507720947,
      "step": 225
    },
    {
      "dpo_loss": 1.0544065237045288,
      "epoch": 0.6518658478979689,
      "grad_norm": 96.24074077951633,
      "learning_rate": 4.261174622596835e-06,
      "logits": -0.9322254657745361,
      "logps": -74.73289489746094,
      "loss": 1.2067,
      "objective": 1.0544065237045288,
      "ranking_simple": 0.6499999761581421,
      "step": 230
    },
    {
      "dpo_loss": 1.3109701871871948,
      "epoch": 0.6660368445914029,
      "grad_norm": 99.32952244494642,
      "learning_rate": 4.216606603511202e-06,
      "logits": -0.8552966713905334,
      "logps": -76.02928161621094,
      "loss": 1.1741,
      "objective": 1.3109701871871948,
      "ranking_simple": 0.5916666388511658,
      "step": 235
    },
    {
      "dpo_loss": 1.1290283203125,
      "epoch": 0.680207841284837,
      "grad_norm": 90.5291171770579,
      "learning_rate": 4.170981570213621e-06,
      "logits": -1.0387074947357178,
      "logps": -74.89482116699219,
      "loss": 1.1403,
      "objective": 1.1290283203125,
      "ranking_simple": 0.5916666388511658,
      "step": 240
    },
    {
      "dpo_loss": 1.3167603015899658,
      "epoch": 0.6943788379782712,
      "grad_norm": 111.7529832893929,
      "learning_rate": 4.124327616682362e-06,
      "logits": -1.05294930934906,
      "logps": -73.70958709716797,
      "loss": 1.1308,
      "objective": 1.3167603015899658,
      "ranking_simple": 0.612500011920929,
      "step": 245
    },
    {
      "dpo_loss": 1.0333999395370483,
      "epoch": 0.7085498346717053,
      "grad_norm": 103.1933415885334,
      "learning_rate": 4.076673470461538e-06,
      "logits": -0.8845551609992981,
      "logps": -73.25338745117188,
      "loss": 1.1914,
      "objective": 1.0333999395370483,
      "ranking_simple": 0.6208333373069763,
      "step": 250
    },
    {
      "epoch": 0.7085498346717053,
      "eval_dpo_loss": 1.9460629224777222,
      "eval_logits": -1.0883058309555054,
      "eval_logps": -78.33851623535156,
      "eval_loss": 2.0083987712860107,
      "eval_objective": 1.9460629224777222,
      "eval_ranking_simple": 0.534679114818573,
      "eval_runtime": 607.1642,
      "eval_samples_per_second": 9.536,
      "eval_steps_per_second": 0.796,
      "step": 250
    },
    {
      "dpo_loss": 1.0360175371170044,
      "epoch": 0.7227208313651393,
      "grad_norm": 86.03208381847595,
      "learning_rate": 4.028048474971889e-06,
      "logits": -1.0679165124893188,
      "logps": -73.73800659179688,
      "loss": 1.0854,
      "objective": 1.0360175371170044,
      "ranking_simple": 0.5458333492279053,
      "step": 255
    },
    {
      "dpo_loss": 1.0874184370040894,
      "epoch": 0.7368918280585735,
      "grad_norm": 103.07844444269477,
      "learning_rate": 3.978482571442339e-06,
      "logits": -1.0024687051773071,
      "logps": -77.03482818603516,
      "loss": 1.12,
      "objective": 1.0874184370040894,
      "ranking_simple": 0.612500011920929,
      "step": 260
    },
    {
      "dpo_loss": 0.8361913561820984,
      "epoch": 0.7510628247520076,
      "grad_norm": 99.23229150586383,
      "learning_rate": 3.928006280473445e-06,
      "logits": -0.9436743259429932,
      "logps": -76.39049530029297,
      "loss": 1.0349,
      "objective": 0.8361913561820984,
      "ranking_simple": 0.6708333492279053,
      "step": 265
    },
    {
      "dpo_loss": 1.0280269384384155,
      "epoch": 0.7652338214454416,
      "grad_norm": 95.81187840710191,
      "learning_rate": 3.876650683244093e-06,
      "logits": -0.9881510138511658,
      "logps": -77.2256851196289,
      "loss": 1.0768,
      "objective": 1.0280269384384155,
      "ranking_simple": 0.5458333492279053,
      "step": 270
    },
    {
      "dpo_loss": 0.9129766821861267,
      "epoch": 0.7794048181388757,
      "grad_norm": 89.52663856813324,
      "learning_rate": 3.8244474023730155e-06,
      "logits": -0.8726529479026794,
      "logps": -76.8287124633789,
      "loss": 1.0978,
      "objective": 0.9129766821861267,
      "ranking_simple": 0.5874999761581421,
      "step": 275
    },
    {
      "dpo_loss": 0.8938360810279846,
      "epoch": 0.7935758148323099,
      "grad_norm": 88.2587417355247,
      "learning_rate": 3.771428582446908e-06,
      "logits": -0.9136671423912048,
      "logps": -76.3465347290039,
      "loss": 0.9242,
      "objective": 0.8938360810279846,
      "ranking_simple": 0.6083333492279053,
      "step": 280
    },
    {
      "dpo_loss": 1.147481083869934,
      "epoch": 0.807746811525744,
      "grad_norm": 106.04454752282797,
      "learning_rate": 3.7176268702271468e-06,
      "logits": -0.8765575885772705,
      "logps": -77.12137603759766,
      "loss": 0.9999,
      "objective": 1.147481083869934,
      "ranking_simple": 0.5583333373069763,
      "step": 285
    },
    {
      "dpo_loss": 0.9309877157211304,
      "epoch": 0.821917808219178,
      "grad_norm": 97.07071302996712,
      "learning_rate": 3.6630753945472854e-06,
      "logits": -0.8515520095825195,
      "logps": -76.22865295410156,
      "loss": 0.8822,
      "objective": 0.9309877157211304,
      "ranking_simple": 0.5916666388511658,
      "step": 290
    },
    {
      "dpo_loss": 0.8610218167304993,
      "epoch": 0.8360888049126122,
      "grad_norm": 95.1741444029312,
      "learning_rate": 3.6078077459137097e-06,
      "logits": -0.9089216589927673,
      "logps": -76.60446166992188,
      "loss": 1.093,
      "objective": 0.8610218167304993,
      "ranking_simple": 0.550000011920929,
      "step": 295
    },
    {
      "dpo_loss": 0.7812207341194153,
      "epoch": 0.8502598016060463,
      "grad_norm": 86.74577338149517,
      "learning_rate": 3.5518579558220144e-06,
      "logits": -0.885455846786499,
      "logps": -76.61499786376953,
      "loss": 1.0378,
      "objective": 0.7812207341194153,
      "ranking_simple": 0.6499999761581421,
      "step": 300
    },
    {
      "epoch": 0.8502598016060463,
      "eval_dpo_loss": 2.0357348918914795,
      "eval_logits": -0.9324438571929932,
      "eval_logps": -83.47068786621094,
      "eval_loss": 2.091830015182495,
      "eval_objective": 2.0357348918914795,
      "eval_ranking_simple": 0.5351966619491577,
      "eval_runtime": 611.0424,
      "eval_samples_per_second": 9.476,
      "eval_steps_per_second": 0.79,
      "step": 300
    },
    {
      "dpo_loss": 1.2579916715621948,
      "epoch": 0.8644307982994804,
      "grad_norm": 85.57616018620313,
      "learning_rate": 3.495260475801841e-06,
      "logits": -0.8382306694984436,
      "logps": -79.23892974853516,
      "loss": 0.9276,
      "objective": 1.2579916715621948,
      "ranking_simple": 0.637499988079071,
      "step": 305
    },
    {
      "dpo_loss": 0.6779700517654419,
      "epoch": 0.8786017949929145,
      "grad_norm": 99.75714946886596,
      "learning_rate": 3.4380501562030704e-06,
      "logits": -0.7922627329826355,
      "logps": -78.47576141357422,
      "loss": 0.8132,
      "objective": 0.6779700517654419,
      "ranking_simple": 0.6166666746139526,
      "step": 310
    },
    {
      "dpo_loss": 0.8982964754104614,
      "epoch": 0.8927727916863486,
      "grad_norm": 77.85557552766669,
      "learning_rate": 3.3802622247364446e-06,
      "logits": -0.7617653608322144,
      "logps": -78.67091369628906,
      "loss": 0.9992,
      "objective": 0.8982964754104614,
      "ranking_simple": 0.5874999761581421,
      "step": 315
    },
    {
      "dpo_loss": 0.8807361125946045,
      "epoch": 0.9069437883797827,
      "grad_norm": 97.29848811322167,
      "learning_rate": 3.321932264781822e-06,
      "logits": -0.7270026803016663,
      "logps": -80.3343734741211,
      "loss": 0.9675,
      "objective": 0.8807361125946045,
      "ranking_simple": 0.6208333373069763,
      "step": 320
    },
    {
      "dpo_loss": 0.7385057806968689,
      "epoch": 0.9211147850732169,
      "grad_norm": 80.05333271753152,
      "learning_rate": 3.2630961934774265e-06,
      "logits": -0.8284817934036255,
      "logps": -80.14983367919922,
      "loss": 0.8071,
      "objective": 0.7385057806968689,
      "ranking_simple": 0.6041666865348816,
      "step": 325
    },
    {
      "dpo_loss": 0.8387654423713684,
      "epoch": 0.9352857817666509,
      "grad_norm": 85.76862598705932,
      "learning_rate": 3.203790239603583e-06,
      "logits": -0.7765657305717468,
      "logps": -79.01445007324219,
      "loss": 0.9652,
      "objective": 0.8387654423713684,
      "ranking_simple": 0.5958333611488342,
      "step": 330
    },
    {
      "dpo_loss": 0.838505208492279,
      "epoch": 0.949456778460085,
      "grad_norm": 88.47139227896243,
      "learning_rate": 3.1440509212745584e-06,
      "logits": -0.7588701844215393,
      "logps": -78.80551147460938,
      "loss": 0.8953,
      "objective": 0.838505208492279,
      "ranking_simple": 0.6166666746139526,
      "step": 335
    },
    {
      "dpo_loss": 1.108252763748169,
      "epoch": 0.9636277751535192,
      "grad_norm": 69.45012344359534,
      "learning_rate": 3.0839150234522404e-06,
      "logits": -0.8102169036865234,
      "logps": -77.67143249511719,
      "loss": 0.9601,
      "objective": 1.108252763748169,
      "ranking_simple": 0.5874999761581421,
      "step": 340
    },
    {
      "dpo_loss": 0.9010140299797058,
      "epoch": 0.9777987718469532,
      "grad_norm": 82.40549141111858,
      "learning_rate": 3.0234195752955032e-06,
      "logits": -0.8673346638679504,
      "logps": -75.89323425292969,
      "loss": 0.969,
      "objective": 0.9010140299797058,
      "ranking_simple": 0.6541666388511658,
      "step": 345
    },
    {
      "dpo_loss": 0.7752221822738647,
      "epoch": 0.9919697685403873,
      "grad_norm": 80.80020112901609,
      "learning_rate": 2.962601827359208e-06,
      "logits": -0.7917401790618896,
      "logps": -77.88353729248047,
      "loss": 0.8334,
      "objective": 0.7752221822738647,
      "ranking_simple": 0.6875,
      "step": 350
    },
    {
      "epoch": 0.9919697685403873,
      "eval_dpo_loss": 1.9975330829620361,
      "eval_logits": -0.8755112886428833,
      "eval_logps": -81.17398071289062,
      "eval_loss": 2.1142992973327637,
      "eval_objective": 1.9975330829620361,
      "eval_ranking_simple": 0.5388198494911194,
      "eval_runtime": 610.9883,
      "eval_samples_per_second": 9.476,
      "eval_steps_per_second": 0.791,
      "step": 350
    },
    {
      "dpo_loss": 0.602699875831604,
      "epoch": 1.0061407652338215,
      "grad_norm": 47.23395645047123,
      "learning_rate": 2.9014992286568773e-06,
      "logits": -0.8253915309906006,
      "logps": -76.2964096069336,
      "loss": 0.6347,
      "objective": 0.602699875831604,
      "ranking_simple": 0.6708333492279053,
      "step": 355
    },
    {
      "dpo_loss": 0.6031454205513,
      "epoch": 1.0203117619272555,
      "grad_norm": 58.81365077909613,
      "learning_rate": 2.840149403601166e-06,
      "logits": -0.7789632081985474,
      "logps": -77.5028305053711,
      "loss": 0.4974,
      "objective": 0.6031454205513,
      "ranking_simple": 0.6499999761581421,
      "step": 360
    },
    {
      "dpo_loss": 0.45107781887054443,
      "epoch": 1.0344827586206897,
      "grad_norm": 53.96463042562765,
      "learning_rate": 2.7785901288363253e-06,
      "logits": -0.7713039517402649,
      "logps": -77.14903259277344,
      "loss": 0.4946,
      "objective": 0.45107781887054443,
      "ranking_simple": 0.625,
      "step": 365
    },
    {
      "dpo_loss": 0.4809951186180115,
      "epoch": 1.0486537553141237,
      "grad_norm": 64.07795252956707,
      "learning_rate": 2.7168593099769414e-06,
      "logits": -0.7404767870903015,
      "logps": -74.05150604248047,
      "loss": 0.4291,
      "objective": 0.4809951186180115,
      "ranking_simple": 0.6208333373069763,
      "step": 370
    },
    {
      "dpo_loss": 0.4239761233329773,
      "epoch": 1.0628247520075578,
      "grad_norm": 55.70979139140137,
      "learning_rate": 2.654994958267241e-06,
      "logits": -0.8401414752006531,
      "logps": -75.16132354736328,
      "loss": 0.4015,
      "objective": 0.4239761233329773,
      "ranking_simple": 0.6416666507720947,
      "step": 375
    },
    {
      "dpo_loss": 0.4185657799243927,
      "epoch": 1.076995748700992,
      "grad_norm": 57.189539552967226,
      "learning_rate": 2.5930351671753707e-06,
      "logits": -0.8457431793212891,
      "logps": -76.84170532226562,
      "loss": 0.4414,
      "objective": 0.4185657799243927,
      "ranking_simple": 0.6875,
      "step": 380
    },
    {
      "dpo_loss": 0.49603796005249023,
      "epoch": 1.091166745394426,
      "grad_norm": 47.834305883433586,
      "learning_rate": 2.5310180889370374e-06,
      "logits": -0.778562605381012,
      "logps": -75.62349700927734,
      "loss": 0.4482,
      "objective": 0.49603796005249023,
      "ranking_simple": 0.6333333253860474,
      "step": 385
    },
    {
      "dpo_loss": 0.42004406452178955,
      "epoch": 1.10533774208786,
      "grad_norm": 48.511623077068286,
      "learning_rate": 2.468981911062964e-06,
      "logits": -0.7501899600028992,
      "logps": -77.29354095458984,
      "loss": 0.3694,
      "objective": 0.42004406452178955,
      "ranking_simple": 0.6916666626930237,
      "step": 390
    },
    {
      "dpo_loss": 0.4247871935367584,
      "epoch": 1.1195087387812943,
      "grad_norm": 57.912950107999485,
      "learning_rate": 2.4069648328246305e-06,
      "logits": -0.7564467787742615,
      "logps": -76.87413024902344,
      "loss": 0.5005,
      "objective": 0.4247871935367584,
      "ranking_simple": 0.637499988079071,
      "step": 395
    },
    {
      "dpo_loss": 0.3325573801994324,
      "epoch": 1.1336797354747283,
      "grad_norm": 58.38348871817592,
      "learning_rate": 2.3450050417327593e-06,
      "logits": -0.7802504301071167,
      "logps": -75.82648468017578,
      "loss": 0.4251,
      "objective": 0.3325573801994324,
      "ranking_simple": 0.6583333611488342,
      "step": 400
    },
    {
      "epoch": 1.1336797354747283,
      "eval_dpo_loss": 2.0240566730499268,
      "eval_logits": -0.8002918362617493,
      "eval_logps": -81.16890716552734,
      "eval_loss": 2.06406831741333,
      "eval_objective": 2.0240566730499268,
      "eval_ranking_simple": 0.54347825050354,
      "eval_runtime": 604.5313,
      "eval_samples_per_second": 9.578,
      "eval_steps_per_second": 0.799,
      "step": 400
    },
    {
      "dpo_loss": 0.6402778029441833,
      "epoch": 1.1478507321681626,
      "grad_norm": 57.999816664931764,
      "learning_rate": 2.2831406900230586e-06,
      "logits": -0.7289463877677917,
      "logps": -77.95418548583984,
      "loss": 0.4798,
      "objective": 0.6402778029441833,
      "ranking_simple": 0.6166666746139526,
      "step": 405
    },
    {
      "dpo_loss": 0.5912865400314331,
      "epoch": 1.1620217288615966,
      "grad_norm": 50.45480604272446,
      "learning_rate": 2.221409871163675e-06,
      "logits": -0.747395932674408,
      "logps": -77.24584197998047,
      "loss": 0.4554,
      "objective": 0.5912865400314331,
      "ranking_simple": 0.6541666388511658,
      "step": 410
    },
    {
      "dpo_loss": 0.3779014050960541,
      "epoch": 1.1761927255550306,
      "grad_norm": 64.88698660251025,
      "learning_rate": 2.1598505963988354e-06,
      "logits": -0.6771641969680786,
      "logps": -75.90592956542969,
      "loss": 0.4899,
      "objective": 0.3779014050960541,
      "ranking_simple": 0.6291666626930237,
      "step": 415
    },
    {
      "dpo_loss": 0.35035714507102966,
      "epoch": 1.1903637222484649,
      "grad_norm": 57.50741792623655,
      "learning_rate": 2.098500771343124e-06,
      "logits": -0.6913995742797852,
      "logps": -76.21831512451172,
      "loss": 0.4117,
      "objective": 0.35035714507102966,
      "ranking_simple": 0.6291666626930237,
      "step": 420
    },
    {
      "dpo_loss": 0.48492512106895447,
      "epoch": 1.204534718941899,
      "grad_norm": 59.717637187759834,
      "learning_rate": 2.037398172640793e-06,
      "logits": -0.7741472125053406,
      "logps": -74.0752944946289,
      "loss": 0.4692,
      "objective": 0.48492512106895447,
      "ranking_simple": 0.6541666388511658,
      "step": 425
    },
    {
      "dpo_loss": 0.613711953163147,
      "epoch": 1.2187057156353331,
      "grad_norm": 58.710504416860545,
      "learning_rate": 1.976580424704498e-06,
      "logits": -0.7667457461357117,
      "logps": -76.66746520996094,
      "loss": 0.3957,
      "objective": 0.613711953163147,
      "ranking_simple": 0.6291666626930237,
      "step": 430
    },
    {
      "dpo_loss": 0.28608372807502747,
      "epoch": 1.2328767123287672,
      "grad_norm": 88.90691891983907,
      "learning_rate": 1.9160849765477604e-06,
      "logits": -0.6787567734718323,
      "logps": -76.04117584228516,
      "loss": 0.4339,
      "objective": 0.28608372807502747,
      "ranking_simple": 0.699999988079071,
      "step": 435
    },
    {
      "dpo_loss": 0.4388173222541809,
      "epoch": 1.2470477090222012,
      "grad_norm": 63.01131335623436,
      "learning_rate": 1.8559490787254423e-06,
      "logits": -0.7236205339431763,
      "logps": -76.31954956054688,
      "loss": 0.4457,
      "objective": 0.4388173222541809,
      "ranking_simple": 0.7083333134651184,
      "step": 440
    },
    {
      "dpo_loss": 0.5651527643203735,
      "epoch": 1.2612187057156352,
      "grad_norm": 65.93704260731457,
      "learning_rate": 1.7962097603964177e-06,
      "logits": -0.7464514374732971,
      "logps": -76.93856048583984,
      "loss": 0.4889,
      "objective": 0.5651527643203735,
      "ranking_simple": 0.699999988079071,
      "step": 445
    },
    {
      "dpo_loss": 0.4833088517189026,
      "epoch": 1.2753897024090695,
      "grad_norm": 56.25706142129644,
      "learning_rate": 1.7369038065225743e-06,
      "logits": -0.8744809627532959,
      "logps": -76.16983795166016,
      "loss": 0.3886,
      "objective": 0.4833088517189026,
      "ranking_simple": 0.6416666507720947,
      "step": 450
    },
    {
      "epoch": 1.2753897024090695,
      "eval_dpo_loss": 1.9598380327224731,
      "eval_logits": -0.8998873233795166,
      "eval_logps": -79.8813247680664,
      "eval_loss": 2.0085067749023438,
      "eval_objective": 1.9598380327224731,
      "eval_ranking_simple": 0.5388198494911194,
      "eval_runtime": 617.7574,
      "eval_samples_per_second": 9.373,
      "eval_steps_per_second": 0.782,
      "step": 450
    },
    {
      "dpo_loss": 0.5061320662498474,
      "epoch": 1.2895606991025035,
      "grad_norm": 54.696067906167414,
      "learning_rate": 1.6780677352181781e-06,
      "logits": -0.7727634310722351,
      "logps": -76.43891906738281,
      "loss": 0.4525,
      "objective": 0.5061320662498474,
      "ranking_simple": 0.5874999761581421,
      "step": 455
    },
    {
      "dpo_loss": 0.45641031861305237,
      "epoch": 1.3037316957959377,
      "grad_norm": 57.70332958198004,
      "learning_rate": 1.6197377752635563e-06,
      "logits": -0.8627158999443054,
      "logps": -75.46236419677734,
      "loss": 0.3956,
      "objective": 0.45641031861305237,
      "ranking_simple": 0.6333333253860474,
      "step": 460
    },
    {
      "dpo_loss": 0.35006803274154663,
      "epoch": 1.3179026924893718,
      "grad_norm": 60.819739611222175,
      "learning_rate": 1.5619498437969302e-06,
      "logits": -0.8722900748252869,
      "logps": -73.59480285644531,
      "loss": 0.4224,
      "objective": 0.35006803274154663,
      "ranking_simple": 0.637499988079071,
      "step": 465
    },
    {
      "dpo_loss": 0.4906507432460785,
      "epoch": 1.3320736891828058,
      "grad_norm": 49.64412942112177,
      "learning_rate": 1.5047395241981606e-06,
      "logits": -0.9396683573722839,
      "logps": -73.02540588378906,
      "loss": 0.5003,
      "objective": 0.4906507432460785,
      "ranking_simple": 0.6499999761581421,
      "step": 470
    },
    {
      "dpo_loss": 0.30290400981903076,
      "epoch": 1.34624468587624,
      "grad_norm": 79.3006324136739,
      "learning_rate": 1.4481420441779862e-06,
      "logits": -0.8443289995193481,
      "logps": -75.5423583984375,
      "loss": 0.3808,
      "objective": 0.30290400981903076,
      "ranking_simple": 0.6958333253860474,
      "step": 475
    },
    {
      "dpo_loss": 0.5529125928878784,
      "epoch": 1.360415682569674,
      "grad_norm": 70.13856035677702,
      "learning_rate": 1.3921922540862907e-06,
      "logits": -0.8475839495658875,
      "logps": -76.10958862304688,
      "loss": 0.476,
      "objective": 0.5529125928878784,
      "ranking_simple": 0.637499988079071,
      "step": 480
    },
    {
      "dpo_loss": 0.4004639685153961,
      "epoch": 1.3745866792631083,
      "grad_norm": 78.28600119390228,
      "learning_rate": 1.3369246054527152e-06,
      "logits": -0.8136020302772522,
      "logps": -74.83834838867188,
      "loss": 0.4712,
      "objective": 0.4004639685153961,
      "ranking_simple": 0.625,
      "step": 485
    },
    {
      "dpo_loss": 0.54802006483078,
      "epoch": 1.3887576759565423,
      "grad_norm": 61.45793050447296,
      "learning_rate": 1.2823731297728536e-06,
      "logits": -0.7907751798629761,
      "logps": -77.5755844116211,
      "loss": 0.3945,
      "objective": 0.54802006483078,
      "ranking_simple": 0.625,
      "step": 490
    },
    {
      "dpo_loss": 0.2221187800168991,
      "epoch": 1.4029286726499763,
      "grad_norm": 66.41905989355433,
      "learning_rate": 1.2285714175530936e-06,
      "logits": -0.805623471736908,
      "logps": -77.69929504394531,
      "loss": 0.3891,
      "objective": 0.2221187800168991,
      "ranking_simple": 0.625,
      "step": 495
    },
    {
      "dpo_loss": 0.42266911268234253,
      "epoch": 1.4170996693434104,
      "grad_norm": 66.25308663743711,
      "learning_rate": 1.1755525976269851e-06,
      "logits": -0.750647783279419,
      "logps": -76.04014587402344,
      "loss": 0.4352,
      "objective": 0.42266911268234253,
      "ranking_simple": 0.6583333611488342,
      "step": 500
    },
    {
      "epoch": 1.4170996693434104,
      "eval_dpo_loss": 1.9818733930587769,
      "eval_logits": -0.8633737564086914,
      "eval_logps": -80.73566436767578,
      "eval_loss": 2.0448944568634033,
      "eval_objective": 1.9818733930587769,
      "eval_ranking_simple": 0.5367494821548462,
      "eval_runtime": 608.89,
      "eval_samples_per_second": 9.509,
      "eval_steps_per_second": 0.793,
      "step": 500
    },
    {
      "dpo_loss": 0.31959182024002075,
      "epoch": 1.4312706660368446,
      "grad_norm": 52.73552059138801,
      "learning_rate": 1.1233493167559065e-06,
      "logits": -0.7827561497688293,
      "logps": -76.80120086669922,
      "loss": 0.3302,
      "objective": 0.31959182024002075,
      "ranking_simple": 0.6458333134651184,
      "step": 505
    },
    {
      "dpo_loss": 0.3922279179096222,
      "epoch": 1.4454416627302786,
      "grad_norm": 61.50180104681149,
      "learning_rate": 1.0719937195265555e-06,
      "logits": -0.8109835386276245,
      "logps": -76.9507827758789,
      "loss": 0.3945,
      "objective": 0.3922279179096222,
      "ranking_simple": 0.6625000238418579,
      "step": 510
    },
    {
      "dpo_loss": 0.3581198453903198,
      "epoch": 1.4596126594237129,
      "grad_norm": 44.945879303706135,
      "learning_rate": 1.0215174285576615e-06,
      "logits": -0.8377180695533752,
      "logps": -78.14236450195312,
      "loss": 0.4044,
      "objective": 0.3581198453903198,
      "ranking_simple": 0.6083333492279053,
      "step": 515
    },
    {
      "dpo_loss": 0.3747193217277527,
      "epoch": 1.473783656117147,
      "grad_norm": 52.10326588262369,
      "learning_rate": 9.719515250281122e-07,
      "logits": -0.79498291015625,
      "logps": -75.8031234741211,
      "loss": 0.3132,
      "objective": 0.3747193217277527,
      "ranking_simple": 0.675000011920929,
      "step": 520
    },
    {
      "dpo_loss": 0.3443489670753479,
      "epoch": 1.487954652810581,
      "grad_norm": 51.25471401996847,
      "learning_rate": 9.233265295384624e-07,
      "logits": -0.6887346506118774,
      "logps": -75.2328872680664,
      "loss": 0.3972,
      "objective": 0.3443489670753479,
      "ranking_simple": 0.6541666388511658,
      "step": 525
    },
    {
      "dpo_loss": 0.3036381006240845,
      "epoch": 1.5021256495040152,
      "grad_norm": 58.55859781711994,
      "learning_rate": 8.756723833176376e-07,
      "logits": -0.8320663571357727,
      "logps": -77.42900085449219,
      "loss": 0.3414,
      "objective": 0.3036381006240845,
      "ranking_simple": 0.637499988079071,
      "step": 530
    },
    {
      "dpo_loss": 0.2670062184333801,
      "epoch": 1.5162966461974492,
      "grad_norm": 47.62625405878976,
      "learning_rate": 8.290184297863793e-07,
      "logits": -0.8374890685081482,
      "logps": -76.14564514160156,
      "loss": 0.3198,
      "objective": 0.2670062184333801,
      "ranking_simple": 0.6499999761581421,
      "step": 535
    },
    {
      "dpo_loss": 0.49031582474708557,
      "epoch": 1.5304676428908834,
      "grad_norm": 62.51314897054348,
      "learning_rate": 7.833933964887985e-07,
      "logits": -0.7449190020561218,
      "logps": -76.09394073486328,
      "loss": 0.3338,
      "objective": 0.49031582474708557,
      "ranking_simple": 0.699999988079071,
      "step": 540
    },
    {
      "dpo_loss": 0.4148392975330353,
      "epoch": 1.5446386395843175,
      "grad_norm": 52.636990202390535,
      "learning_rate": 7.388253774031659e-07,
      "logits": -0.8080311417579651,
      "logps": -76.78816986083984,
      "loss": 0.3883,
      "objective": 0.4148392975330353,
      "ranking_simple": 0.7250000238418579,
      "step": 545
    },
    {
      "dpo_loss": 0.22362883388996124,
      "epoch": 1.5588096362777515,
      "grad_norm": 46.69497634920009,
      "learning_rate": 6.953418156428152e-07,
      "logits": -0.8403748869895935,
      "logps": -76.63642883300781,
      "loss": 0.3103,
      "objective": 0.22362883388996124,
      "ranking_simple": 0.6499999761581421,
      "step": 550
    },
    {
      "epoch": 1.5588096362777515,
      "eval_dpo_loss": 1.9073150157928467,
      "eval_logits": -0.8671907186508179,
      "eval_logps": -80.88265228271484,
      "eval_loss": 1.9783891439437866,
      "eval_objective": 1.9073150157928467,
      "eval_ranking_simple": 0.5372670888900757,
      "eval_runtime": 612.8866,
      "eval_samples_per_second": 9.447,
      "eval_steps_per_second": 0.788,
      "step": 550
    },
    {
      "dpo_loss": 0.41253456473350525,
      "epoch": 1.5729806329711855,
      "grad_norm": 50.727318491856984,
      "learning_rate": 6.529694865578318e-07,
      "logits": -0.6723602414131165,
      "logps": -77.01154327392578,
      "loss": 0.3626,
      "objective": 0.41253456473350525,
      "ranking_simple": 0.6291666626930237,
      "step": 555
    },
    {
      "dpo_loss": 0.3846481144428253,
      "epoch": 1.5871516296646198,
      "grad_norm": 69.96953957426896,
      "learning_rate": 6.117344812479154e-07,
      "logits": -0.7329820990562439,
      "logps": -75.9783935546875,
      "loss": 0.3651,
      "objective": 0.3846481144428253,
      "ranking_simple": 0.6791666746139526,
      "step": 560
    },
    {
      "dpo_loss": 0.3101910650730133,
      "epoch": 1.601322626358054,
      "grad_norm": 66.15790086947261,
      "learning_rate": 5.71662190496575e-07,
      "logits": -0.7286636233329773,
      "logps": -76.75426483154297,
      "loss": 0.349,
      "objective": 0.3101910650730133,
      "ranking_simple": 0.612500011920929,
      "step": 565
    },
    {
      "dpo_loss": 0.2726050615310669,
      "epoch": 1.615493623051488,
      "grad_norm": 56.0158605948958,
      "learning_rate": 5.327772891365565e-07,
      "logits": -0.8405836224555969,
      "logps": -77.85334014892578,
      "loss": 0.2979,
      "objective": 0.2726050615310669,
      "ranking_simple": 0.6541666388511658,
      "step": 570
    },
    {
      "dpo_loss": 0.4396015405654907,
      "epoch": 1.629664619744922,
      "grad_norm": 67.51231381478947,
      "learning_rate": 4.951037208561116e-07,
      "logits": -0.8009175658226013,
      "logps": -75.7006607055664,
      "loss": 0.3867,
      "objective": 0.4396015405654907,
      "ranking_simple": 0.6833333373069763,
      "step": 575
    },
    {
      "dpo_loss": 0.3268950283527374,
      "epoch": 1.643835616438356,
      "grad_norm": 55.29442626424949,
      "learning_rate": 4.586646834554864e-07,
      "logits": -0.8186151385307312,
      "logps": -75.92921447753906,
      "loss": 0.3658,
      "objective": 0.3268950283527374,
      "ranking_simple": 0.6625000238418579,
      "step": 580
    },
    {
      "dpo_loss": 0.2519194185733795,
      "epoch": 1.6580066131317903,
      "grad_norm": 53.89781546053762,
      "learning_rate": 4.234826145626855e-07,
      "logits": -0.7085616588592529,
      "logps": -75.17672729492188,
      "loss": 0.2996,
      "objective": 0.2519194185733795,
      "ranking_simple": 0.5874999761581421,
      "step": 585
    },
    {
      "dpo_loss": 0.31272074580192566,
      "epoch": 1.6721776098252243,
      "grad_norm": 59.39748892759291,
      "learning_rate": 3.8957917781732883e-07,
      "logits": -0.7312564253807068,
      "logps": -76.37480163574219,
      "loss": 0.3241,
      "objective": 0.31272074580192566,
      "ranking_simple": 0.675000011920929,
      "step": 590
    },
    {
      "dpo_loss": 0.27419552206993103,
      "epoch": 1.6863486065186586,
      "grad_norm": 53.45471180782799,
      "learning_rate": 3.569752495310877e-07,
      "logits": -0.7288424372673035,
      "logps": -76.92625427246094,
      "loss": 0.2888,
      "objective": 0.27419552206993103,
      "ranking_simple": 0.6458333134651184,
      "step": 595
    },
    {
      "dpo_loss": 0.1944553703069687,
      "epoch": 1.7005196032120926,
      "grad_norm": 50.57431920897434,
      "learning_rate": 3.2569090583293356e-07,
      "logits": -0.7772687673568726,
      "logps": -77.40592193603516,
      "loss": 0.2489,
      "objective": 0.1944553703069687,
      "ranking_simple": 0.699999988079071,
      "step": 600
    },
    {
      "epoch": 1.7005196032120926,
      "eval_dpo_loss": 1.8850661516189575,
      "eval_logits": -0.8421351313591003,
      "eval_logps": -81.0832748413086,
      "eval_loss": 1.9488178491592407,
      "eval_objective": 1.8850661516189575,
      "eval_ranking_simple": 0.5367494821548462,
      "eval_runtime": 603.5072,
      "eval_samples_per_second": 9.594,
      "eval_steps_per_second": 0.8,
      "step": 600
    },
    {
      "dpo_loss": 0.35472404956817627,
      "epoch": 1.7146905999055266,
      "grad_norm": 65.83086255827398,
      "learning_rate": 2.957454103070978e-07,
      "logits": -0.7188993096351624,
      "logps": -78.14205169677734,
      "loss": 0.3777,
      "objective": 0.35472404956817627,
      "ranking_simple": 0.6499999761581421,
      "step": 605
    },
    {
      "dpo_loss": 0.286739319562912,
      "epoch": 1.7288615965989607,
      "grad_norm": 69.02122554997335,
      "learning_rate": 2.6715720213136955e-07,
      "logits": -0.7877374887466431,
      "logps": -76.47672271728516,
      "loss": 0.3096,
      "objective": 0.286739319562912,
      "ranking_simple": 0.6791666746139526,
      "step": 610
    },
    {
      "dpo_loss": 0.35429847240448,
      "epoch": 1.743032593292395,
      "grad_norm": 58.94519405559986,
      "learning_rate": 2.399438847230212e-07,
      "logits": -0.8013178110122681,
      "logps": -76.39897155761719,
      "loss": 0.3637,
      "objective": 0.35429847240448,
      "ranking_simple": 0.6166666746139526,
      "step": 615
    },
    {
      "dpo_loss": 0.2548845112323761,
      "epoch": 1.7572035899858292,
      "grad_norm": 53.576700184407876,
      "learning_rate": 2.1412221489936796e-07,
      "logits": -0.8240512609481812,
      "logps": -76.49982452392578,
      "loss": 0.3083,
      "objective": 0.2548845112323761,
      "ranking_simple": 0.6875,
      "step": 620
    },
    {
      "dpo_loss": 0.2307281196117401,
      "epoch": 1.7713745866792632,
      "grad_norm": 64.55540353999235,
      "learning_rate": 1.897080925596187e-07,
      "logits": -0.7509959936141968,
      "logps": -75.8747329711914,
      "loss": 0.321,
      "objective": 0.2307281196117401,
      "ranking_simple": 0.6416666507720947,
      "step": 625
    },
    {
      "dpo_loss": 0.5376826524734497,
      "epoch": 1.7855455833726972,
      "grad_norm": 61.55828638335153,
      "learning_rate": 1.6671655089439186e-07,
      "logits": -0.7904254794120789,
      "logps": -77.4958267211914,
      "loss": 0.4286,
      "objective": 0.5376826524734497,
      "ranking_simple": 0.6166666746139526,
      "step": 630
    },
    {
      "dpo_loss": 0.32618287205696106,
      "epoch": 1.7997165800661312,
      "grad_norm": 67.7723818410267,
      "learning_rate": 1.4516174712890406e-07,
      "logits": -0.8496752381324768,
      "logps": -77.06031036376953,
      "loss": 0.3605,
      "objective": 0.32618287205696106,
      "ranking_simple": 0.7041666507720947,
      "step": 635
    },
    {
      "dpo_loss": 0.264717161655426,
      "epoch": 1.8138875767595655,
      "grad_norm": 49.987295020521714,
      "learning_rate": 1.2505695380554712e-07,
      "logits": -0.8106148838996887,
      "logps": -77.50927734375,
      "loss": 0.2895,
      "objective": 0.264717161655426,
      "ranking_simple": 0.6499999761581421,
      "step": 640
    },
    {
      "dpo_loss": 0.3132435977458954,
      "epoch": 1.8280585734529995,
      "grad_norm": 57.443846637348,
      "learning_rate": 1.0641455061121519e-07,
      "logits": -0.8152769804000854,
      "logps": -78.20152282714844,
      "loss": 0.4079,
      "objective": 0.3132435977458954,
      "ranking_simple": 0.6958333253860474,
      "step": 645
    },
    {
      "dpo_loss": 0.3075469732284546,
      "epoch": 1.8422295701464337,
      "grad_norm": 63.674814735474534,
      "learning_rate": 8.924601675441207e-08,
      "logits": -0.8521133065223694,
      "logps": -78.01354217529297,
      "loss": 0.3631,
      "objective": 0.3075469732284546,
      "ranking_simple": 0.6708333492279053,
      "step": 650
    },
    {
      "epoch": 1.8422295701464337,
      "eval_dpo_loss": 1.8805330991744995,
      "eval_logits": -0.8528652787208557,
      "eval_logps": -81.17214965820312,
      "eval_loss": 1.9417303800582886,
      "eval_objective": 1.8805330991744995,
      "eval_ranking_simple": 0.534679114818573,
      "eval_runtime": 608.284,
      "eval_samples_per_second": 9.519,
      "eval_steps_per_second": 0.794,
      "step": 650
    },
    {
      "dpo_loss": 0.2915767729282379,
      "epoch": 1.8564005668398678,
      "grad_norm": 51.326241873653245,
      "learning_rate": 7.356192389683825e-08,
      "logits": -0.7731481790542603,
      "logps": -75.27713012695312,
      "loss": 0.3407,
      "objective": 0.2915767729282379,
      "ranking_simple": 0.6166666746139526,
      "step": 655
    },
    {
      "dpo_loss": 0.40089327096939087,
      "epoch": 1.8705715635333018,
      "grad_norm": 51.09671637748102,
      "learning_rate": 5.937192964380556e-08,
      "logits": -0.7996050715446472,
      "logps": -78.20768737792969,
      "loss": 0.3594,
      "objective": 0.40089327096939087,
      "ranking_simple": 0.6583333611488342,
      "step": 660
    },
    {
      "dpo_loss": 0.32784855365753174,
      "epoch": 1.8847425602267358,
      "grad_norm": 59.65383844014325,
      "learning_rate": 4.668477159748858e-08,
      "logits": -0.824116587638855,
      "logps": -76.31925201416016,
      "loss": 0.3228,
      "objective": 0.32784855365753174,
      "ranking_simple": 0.7041666507720947,
      "step": 665
    },
    {
      "dpo_loss": 0.4064708948135376,
      "epoch": 1.89891355692017,
      "grad_norm": 52.69018895294125,
      "learning_rate": 3.5508261976678894e-08,
      "logits": -0.7589130997657776,
      "logps": -75.46687316894531,
      "loss": 0.366,
      "objective": 0.4064708948135376,
      "ranking_simple": 0.625,
      "step": 670
    },
    {
      "dpo_loss": 0.3538086712360382,
      "epoch": 1.9130845536136043,
      "grad_norm": 57.27565478185342,
      "learning_rate": 2.5849282806345855e-08,
      "logits": -0.7437042593955994,
      "logps": -76.25951385498047,
      "loss": 0.3801,
      "objective": 0.3538086712360382,
      "ranking_simple": 0.675000011920929,
      "step": 675
    },
    {
      "dpo_loss": 0.2823149859905243,
      "epoch": 1.9272555503070383,
      "grad_norm": 55.3247140739336,
      "learning_rate": 1.771378167997745e-08,
      "logits": -0.788720428943634,
      "logps": -78.25997161865234,
      "loss": 0.389,
      "objective": 0.2823149859905243,
      "ranking_simple": 0.6875,
      "step": 680
    },
    {
      "dpo_loss": 0.30807915329933167,
      "epoch": 1.9414265470004723,
      "grad_norm": 42.581364917436595,
      "learning_rate": 1.1106768097300657e-08,
      "logits": -0.769125759601593,
      "logps": -75.96378326416016,
      "loss": 0.3183,
      "objective": 0.30807915329933167,
      "ranking_simple": 0.6083333492279053,
      "step": 685
    },
    {
      "dpo_loss": 0.21873274445533752,
      "epoch": 1.9555975436939064,
      "grad_norm": 37.39828456518223,
      "learning_rate": 6.032310379642803e-09,
      "logits": -0.7863041758537292,
      "logps": -76.51734924316406,
      "loss": 0.3202,
      "objective": 0.21873274445533752,
      "ranking_simple": 0.6708333492279053,
      "step": 690
    },
    {
      "dpo_loss": 0.24407407641410828,
      "epoch": 1.9697685403873406,
      "grad_norm": 75.42620047856866,
      "learning_rate": 2.4935331648298644e-09,
      "logits": -0.7426345944404602,
      "logps": -77.34226989746094,
      "loss": 0.3361,
      "objective": 0.24407407641410828,
      "ranking_simple": 0.625,
      "step": 695
    },
    {
      "dpo_loss": 0.2809065580368042,
      "epoch": 1.9839395370807746,
      "grad_norm": 63.61763495221277,
      "learning_rate": 4.926154831655372e-10,
      "logits": -0.6787734627723694,
      "logps": -78.1474609375,
      "loss": 0.3009,
      "objective": 0.2809065580368042,
      "ranking_simple": 0.6541666388511658,
      "step": 700
    },
    {
      "epoch": 1.9839395370807746,
      "eval_dpo_loss": 1.8786609172821045,
      "eval_logits": -0.850897490978241,
      "eval_logps": -81.16839599609375,
      "eval_loss": 1.9399291276931763,
      "eval_objective": 1.8786609172821045,
      "eval_ranking_simple": 0.534679114818573,
      "eval_runtime": 605.8989,
      "eval_samples_per_second": 9.556,
      "eval_steps_per_second": 0.797,
      "step": 700
    },
    {
      "epoch": 1.995276334435522,
      "step": 704,
      "total_flos": 0.0,
      "train_loss": 0.7561936149881645,
      "train_runtime": 36938.8609,
      "train_samples_per_second": 2.751,
      "train_steps_per_second": 0.019
    }
  ],
  "logging_steps": 5,
  "max_steps": 704,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}