{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.995276334435522,
  "eval_steps": 50,
  "global_step": 352,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "dpo_loss": 0.6931471824645996,
      "epoch": 0.005668398677373642,
      "grad_norm": 134136.2646513337,
      "learning_rate": 1.3888888888888888e-07,
      "logits": -1.3147305250167847,
      "logps": -88.0877456665039,
      "loss": 0.6931,
      "objective": 0.6931471824645996,
      "ranking_idealized": 0.5208333134651184,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.5208333134651184,
      "regularize": 0.6931471824645996,
      "step": 1
    },
    {
      "dpo_loss": 64.41354370117188,
      "epoch": 0.02834199338686821,
      "grad_norm": 255114.66209497701,
      "learning_rate": 6.944444444444446e-07,
      "logits": -1.3676921129226685,
      "logps": -84.29489135742188,
      "loss": 55.937,
      "objective": 64.41354370117188,
      "ranking_idealized": 0.546875,
      "ranking_idealized_expo": 0.546875,
      "ranking_simple": 0.546875,
      "regularize": 64.41354370117188,
      "step": 5
    },
    {
      "dpo_loss": 108.1563491821289,
      "epoch": 0.05668398677373642,
      "grad_norm": 223799.3920856216,
      "learning_rate": 1.3888888888888892e-06,
      "logits": -1.4503705501556396,
      "logps": -83.40940856933594,
      "loss": 124.4302,
      "objective": 108.1563491821289,
      "ranking_idealized": 0.5083333253860474,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.512499988079071,
      "regularize": 108.1563491821289,
      "step": 10
    },
    {
      "dpo_loss": 283.9921875,
      "epoch": 0.08502598016060463,
      "grad_norm": 161472.8179788942,
      "learning_rate": 2.0833333333333334e-06,
      "logits": -1.419437289237976,
      "logps": -83.344970703125,
      "loss": 220.6537,
      "objective": 283.9921875,
      "ranking_idealized": 0.5041666626930237,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.5083333253860474,
      "regularize": 283.9921875,
      "step": 15
    },
    {
      "dpo_loss": 417.5802917480469,
      "epoch": 0.11336797354747284,
      "grad_norm": 161556.43107400293,
      "learning_rate": 2.7777777777777783e-06,
      "logits": -1.392155408859253,
      "logps": -83.57788848876953,
      "loss": 392.664,
      "objective": 417.5802917480469,
      "ranking_idealized": 0.5375000238418579,
      "ranking_idealized_expo": 0.5375000238418579,
      "ranking_simple": 0.550000011920929,
      "regularize": 417.5802917480469,
      "step": 20
    },
    {
      "dpo_loss": 486.9968566894531,
      "epoch": 0.14170996693434104,
      "grad_norm": 170622.7766192396,
      "learning_rate": 3.4722222222222224e-06,
      "logits": -1.4195655584335327,
      "logps": -83.60600280761719,
      "loss": 527.6271,
      "objective": 486.9968566894531,
      "ranking_idealized": 0.4958333373069763,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.5,
      "regularize": 486.9968566894531,
      "step": 25
    },
    {
      "dpo_loss": 729.6851806640625,
      "epoch": 0.17005196032120926,
      "grad_norm": 142228.9251647543,
      "learning_rate": 4.166666666666667e-06,
      "logits": -1.4255249500274658,
      "logps": -83.0840835571289,
      "loss": 636.9524,
      "objective": 729.6851806640625,
      "ranking_idealized": 0.5416666865348816,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.5458333492279053,
      "regularize": 729.6851806640625,
      "step": 30
    },
    {
      "dpo_loss": 861.8379516601562,
      "epoch": 0.19839395370807747,
      "grad_norm": 126868.58867593313,
      "learning_rate": 4.861111111111111e-06,
      "logits": -1.4066874980926514,
      "logps": -82.34830474853516,
      "loss": 742.5448,
      "objective": 861.8379516601562,
      "ranking_idealized": 0.5041666626930237,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.5249999761581421,
      "regularize": 861.8379516601562,
      "step": 35
    },
    {
      "dpo_loss": 705.9973754882812,
      "epoch": 0.22673594709494568,
      "grad_norm": 118822.56408811941,
      "learning_rate": 4.998023493068255e-06,
      "logits": -1.4295450448989868,
      "logps": -81.06497192382812,
      "loss": 792.4565,
      "objective": 705.9973754882812,
      "ranking_idealized": 0.5208333134651184,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.5625,
      "regularize": 705.9973754882812,
      "step": 40
    },
    {
      "dpo_loss": 844.502197265625,
      "epoch": 0.25507794048181387,
      "grad_norm": 108533.82718376529,
      "learning_rate": 4.989999289644993e-06,
      "logits": -1.5902892351150513,
      "logps": -85.04265594482422,
      "loss": 883.6045,
      "objective": 844.502197265625,
      "ranking_idealized": 0.5375000238418579,
      "ranking_idealized_expo": 0.5375000238418579,
      "ranking_simple": 0.574999988079071,
      "regularize": 844.502197265625,
      "step": 45
    },
    {
      "dpo_loss": 878.5912475585938,
      "epoch": 0.2834199338686821,
      "grad_norm": 148276.51176076065,
      "learning_rate": 4.975823666181256e-06,
      "logits": -1.507038950920105,
      "logps": -84.19004821777344,
      "loss": 932.0532,
      "objective": 878.5912475585938,
      "ranking_idealized": 0.42500001192092896,
      "ranking_idealized_expo": 0.42500001192092896,
      "ranking_simple": 0.5208333134651184,
      "regularize": 878.5912475585938,
      "step": 50
    },
    {
      "epoch": 0.2834199338686821,
      "eval_dpo_loss": 972.7210693359375,
      "eval_logits": -1.5321499109268188,
      "eval_logps": -90.58901977539062,
      "eval_loss": 928.5909423828125,
      "eval_objective": 972.7210693359375,
      "eval_ranking_idealized": 0.5092975497245789,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5103305578231812,
      "eval_regularize": 972.7210693359375,
      "eval_runtime": 259.4233,
      "eval_samples_per_second": 22.319,
      "eval_steps_per_second": 0.933,
      "step": 50
    },
    {
      "dpo_loss": 1206.24169921875,
      "epoch": 0.3117619272555503,
      "grad_norm": 111585.26682268211,
      "learning_rate": 4.955531642853404e-06,
      "logits": -1.5654302835464478,
      "logps": -83.04993438720703,
      "loss": 1017.4186,
      "objective": 1206.24169921875,
      "ranking_idealized": 0.5416666865348816,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.5625,
      "regularize": 1206.24169921875,
      "step": 55
    },
    {
      "dpo_loss": 898.1983642578125,
      "epoch": 0.3401039206424185,
      "grad_norm": 128343.25042066336,
      "learning_rate": 4.929173350101025e-06,
      "logits": -1.4497432708740234,
      "logps": -79.0652847290039,
      "loss": 1083.7694,
      "objective": 898.1983642578125,
      "ranking_idealized": 0.512499988079071,
      "ranking_idealized_expo": 0.512499988079071,
      "ranking_simple": 0.5541666746139526,
      "regularize": 898.1983642578125,
      "step": 60
    },
    {
      "dpo_loss": 1027.238525390625,
      "epoch": 0.3684459140292867,
      "grad_norm": 109384.11689799791,
      "learning_rate": 4.896813904782162e-06,
      "logits": -1.31638503074646,
      "logps": -78.72539520263672,
      "loss": 1093.5172,
      "objective": 1027.238525390625,
      "ranking_idealized": 0.5083333253860474,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.5625,
      "regularize": 1027.238525390625,
      "step": 65
    },
    {
      "dpo_loss": 1033.64208984375,
      "epoch": 0.39678790741615494,
      "grad_norm": 110045.37824804982,
      "learning_rate": 4.858533249305337e-06,
      "logits": -1.3451343774795532,
      "logps": -78.35610961914062,
      "loss": 1202.9476,
      "objective": 1033.64208984375,
      "ranking_idealized": 0.5166666507720947,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5291666388511658,
      "regularize": 1033.64208984375,
      "step": 70
    },
    {
      "dpo_loss": 1102.832763671875,
      "epoch": 0.42512990080302315,
      "grad_norm": 104408.08358553851,
      "learning_rate": 4.814425954135786e-06,
      "logits": -1.3275989294052124,
      "logps": -79.08589172363281,
      "loss": 1223.3896,
      "objective": 1102.832763671875,
      "ranking_idealized": 0.5458333492279053,
      "ranking_idealized_expo": 0.5458333492279053,
      "ranking_simple": 0.5833333134651184,
      "regularize": 1102.832763671875,
      "step": 75
    },
    {
      "dpo_loss": 1157.8853759765625,
      "epoch": 0.45347189418989137,
      "grad_norm": 107564.4516387397,
      "learning_rate": 4.764600984163809e-06,
      "logits": -1.3979977369308472,
      "logps": -77.84270477294922,
      "loss": 1194.6248,
      "objective": 1157.8853759765625,
      "ranking_idealized": 0.5166666507720947,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5874999761581421,
      "regularize": 1157.8853759765625,
      "step": 80
    },
    {
      "dpo_loss": 886.0385131835938,
      "epoch": 0.4818138875767596,
      "grad_norm": 91109.60154489939,
      "learning_rate": 4.709181429512391e-06,
      "logits": -1.4160219430923462,
      "logps": -74.69747161865234,
      "loss": 1088.332,
      "objective": 886.0385131835938,
      "ranking_idealized": 0.5249999761581421,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.5833333134651184,
      "regularize": 886.0385131835938,
      "step": 85
    },
    {
      "dpo_loss": 1261.300537109375,
      "epoch": 0.5101558809636277,
      "grad_norm": 102202.86060183049,
      "learning_rate": 4.648304201449153e-06,
      "logits": -1.353415846824646,
      "logps": -76.67017364501953,
      "loss": 1141.1982,
      "objective": 1261.300537109375,
      "ranking_idealized": 0.4791666567325592,
      "ranking_idealized_expo": 0.4791666567325592,
      "ranking_simple": 0.4958333373069763,
      "regularize": 1261.300537109375,
      "step": 90
    },
    {
      "dpo_loss": 1023.5733032226562,
      "epoch": 0.538497874350496,
      "grad_norm": 94567.63130429093,
      "learning_rate": 4.582119694153834e-06,
      "logits": -1.3932024240493774,
      "logps": -74.20214080810547,
      "loss": 1027.5987,
      "objective": 1023.5733032226562,
      "ranking_idealized": 0.5416666865348816,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.612500011920929,
      "regularize": 1023.5733032226562,
      "step": 95
    },
    {
      "dpo_loss": 864.1242065429688,
      "epoch": 0.5668398677373642,
      "grad_norm": 95992.36362377643,
      "learning_rate": 4.510791413176912e-06,
      "logits": -1.3935141563415527,
      "logps": -75.58827209472656,
      "loss": 1035.9887,
      "objective": 864.1242065429688,
      "ranking_idealized": 0.5416666865348816,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.6166666746139526,
      "regularize": 864.1242065429688,
      "step": 100
    },
    {
      "epoch": 0.5668398677373642,
      "eval_dpo_loss": 1629.795166015625,
      "eval_logits": -1.3577184677124023,
      "eval_logps": -80.15082550048828,
      "eval_loss": 1589.5357666015625,
      "eval_objective": 1629.795166015625,
      "eval_ranking_idealized": 0.5092975497245789,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5144628286361694,
      "eval_regularize": 1629.795166015625,
      "eval_runtime": 259.4698,
      "eval_samples_per_second": 22.315,
      "eval_steps_per_second": 0.933,
      "step": 100
    },
    {
      "dpo_loss": 696.0361938476562,
      "epoch": 0.5951818611242324,
      "grad_norm": 88621.72973085915,
      "learning_rate": 4.4344955715072344e-06,
      "logits": -1.2187793254852295,
      "logps": -75.67195892333984,
      "loss": 1000.3678,
      "objective": 696.0361938476562,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.612500011920929,
      "regularize": 696.0361938476562,
      "step": 105
    },
    {
      "dpo_loss": 1065.71435546875,
      "epoch": 0.6235238545111006,
      "grad_norm": 85877.4825538592,
      "learning_rate": 4.353420654246546e-06,
      "logits": -1.3558900356292725,
      "logps": -72.48388671875,
      "loss": 985.9748,
      "objective": 1065.71435546875,
      "ranking_idealized": 0.44999998807907104,
      "ranking_idealized_expo": 0.44999998807907104,
      "ranking_simple": 0.5458333492279053,
      "regularize": 1065.71435546875,
      "step": 110
    },
    {
      "dpo_loss": 973.6234741210938,
      "epoch": 0.6518658478979689,
      "grad_norm": 82967.12972491249,
      "learning_rate": 4.267766952966369e-06,
      "logits": -1.352988839149475,
      "logps": -72.43819427490234,
      "loss": 1006.1971,
      "objective": 973.6234741210938,
      "ranking_idealized": 0.4833333194255829,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.5625,
      "regularize": 973.6234741210938,
      "step": 115
    },
    {
      "dpo_loss": 1370.5382080078125,
      "epoch": 0.680207841284837,
      "grad_norm": 83463.66668111246,
      "learning_rate": 4.177746070897593e-06,
      "logits": -1.2847944498062134,
      "logps": -72.26421356201172,
      "loss": 987.543,
      "objective": 1370.5382080078125,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5958333611488342,
      "regularize": 1370.5382080078125,
      "step": 120
    },
    {
      "dpo_loss": 1089.014892578125,
      "epoch": 0.7085498346717053,
      "grad_norm": 94174.35846112414,
      "learning_rate": 4.083580400175153e-06,
      "logits": -1.2737981081008911,
      "logps": -69.4671859741211,
      "loss": 1030.364,
      "objective": 1089.014892578125,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.5958333611488342,
      "regularize": 1089.014892578125,
      "step": 125
    },
    {
      "dpo_loss": 954.4219970703125,
      "epoch": 0.7368918280585735,
      "grad_norm": 89736.61293259053,
      "learning_rate": 3.985502572429277e-06,
      "logits": -1.330850601196289,
      "logps": -68.98014068603516,
      "loss": 1012.2323,
      "objective": 954.4219970703125,
      "ranking_idealized": 0.44999998807907104,
      "ranking_idealized_expo": 0.44999998807907104,
      "ranking_simple": 0.5083333253860474,
      "regularize": 954.4219970703125,
      "step": 130
    },
    {
      "dpo_loss": 809.591552734375,
      "epoch": 0.7652338214454416,
      "grad_norm": 87825.68289803407,
      "learning_rate": 3.8837548840805395e-06,
      "logits": -1.2969688177108765,
      "logps": -73.1766586303711,
      "loss": 936.7706,
      "objective": 809.591552734375,
      "ranking_idealized": 0.5291666388511658,
      "ranking_idealized_expo": 0.5291666388511658,
      "ranking_simple": 0.550000011920929,
      "regularize": 809.591552734375,
      "step": 135
    },
    {
      "dpo_loss": 874.1357421875,
      "epoch": 0.7935758148323099,
      "grad_norm": 76163.02166937232,
      "learning_rate": 3.7785886977585562e-06,
      "logits": -1.2195788621902466,
      "logps": -73.08373260498047,
      "loss": 892.2579,
      "objective": 874.1357421875,
      "ranking_idealized": 0.5041666626930237,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.5666666626930237,
      "regularize": 874.1357421875,
      "step": 140
    },
    {
      "dpo_loss": 570.5612182617188,
      "epoch": 0.821917808219178,
      "grad_norm": 84012.91126243533,
      "learning_rate": 3.6702638213230344e-06,
      "logits": -1.204200267791748,
      "logps": -72.65382385253906,
      "loss": 783.0793,
      "objective": 570.5612182617188,
      "ranking_idealized": 0.4833333194255829,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.5625,
      "regularize": 570.5612182617188,
      "step": 145
    },
    {
      "dpo_loss": 908.3119506835938,
      "epoch": 0.8502598016060463,
      "grad_norm": 81573.56077641094,
      "learning_rate": 3.5590478660213214e-06,
      "logits": -1.1704277992248535,
      "logps": -71.41789245605469,
      "loss": 835.8459,
      "objective": 908.3119506835938,
      "ranking_idealized": 0.5375000238418579,
      "ranking_idealized_expo": 0.5375000238418579,
      "ranking_simple": 0.5874999761581421,
      "regularize": 908.3119506835938,
      "step": 150
    },
    {
      "epoch": 0.8502598016060463,
      "eval_dpo_loss": 1554.7244873046875,
      "eval_logits": -1.1902146339416504,
      "eval_logps": -79.13040924072266,
      "eval_loss": 1554.2149658203125,
      "eval_objective": 1554.7244873046875,
      "eval_ranking_idealized": 0.5092975497245789,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5237603187561035,
      "eval_regularize": 1554.7244873046875,
      "eval_runtime": 263.62,
      "eval_samples_per_second": 21.963,
      "eval_steps_per_second": 0.918,
      "step": 150
    },
    {
      "dpo_loss": 788.9295654296875,
      "epoch": 0.8786017949929145,
      "grad_norm": 70779.228204101,
      "learning_rate": 3.4452155853680457e-06,
      "logits": -1.1143957376480103,
      "logps": -72.65587615966797,
      "loss": 781.2987,
      "objective": 788.9295654296875,
      "ranking_idealized": 0.512499988079071,
      "ranking_idealized_expo": 0.512499988079071,
      "ranking_simple": 0.6083333492279053,
      "regularize": 788.9295654296875,
      "step": 155
    },
    {
      "dpo_loss": 894.7807006835938,
      "epoch": 0.9069437883797827,
      "grad_norm": 69553.50356561082,
      "learning_rate": 3.32904819638017e-06,
      "logits": -0.9945003390312195,
      "logps": -75.07295989990234,
      "loss": 799.9777,
      "objective": 894.7807006835938,
      "ranking_idealized": 0.4958333373069763,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.574999988079071,
      "regularize": 894.7807006835938,
      "step": 160
    },
    {
      "dpo_loss": 727.5079956054688,
      "epoch": 0.9352857817666509,
      "grad_norm": 64190.392142686775,
      "learning_rate": 3.2108326848442507e-06,
      "logits": -1.0413709878921509,
      "logps": -75.90446472167969,
      "loss": 716.1521,
      "objective": 727.5079956054688,
      "ranking_idealized": 0.5208333134651184,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.6041666865348816,
      "regularize": 727.5079956054688,
      "step": 165
    },
    {
      "dpo_loss": 878.9271850585938,
      "epoch": 0.9636277751535192,
      "grad_norm": 71997.48430459863,
      "learning_rate": 3.090861096332263e-06,
      "logits": -1.084736704826355,
      "logps": -76.09172058105469,
      "loss": 747.5494,
      "objective": 878.9271850585938,
      "ranking_idealized": 0.5083333253860474,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.5666666626930237,
      "regularize": 878.9271850585938,
      "step": 170
    },
    {
      "dpo_loss": 649.3298950195312,
      "epoch": 0.9919697685403873,
      "grad_norm": 75396.04374564675,
      "learning_rate": 2.9694298147174566e-06,
      "logits": -1.0796034336090088,
      "logps": -75.43477630615234,
      "loss": 706.0721,
      "objective": 649.3298950195312,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.574999988079071,
      "ranking_simple": 0.6499999761581421,
      "regularize": 649.3298950195312,
      "step": 175
    },
    {
      "dpo_loss": 538.9683227539062,
      "epoch": 1.0203117619272555,
      "grad_norm": 58320.20497021165,
      "learning_rate": 2.8468388299726714e-06,
      "logits": -1.1605207920074463,
      "logps": -75.59226989746094,
      "loss": 511.6839,
      "objective": 538.9683227539062,
      "ranking_idealized": 0.5416666865348816,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.6333333253860474,
      "regularize": 538.9683227539062,
      "step": 180
    },
    {
      "dpo_loss": 314.388671875,
      "epoch": 1.0486537553141237,
      "grad_norm": 62603.73484971573,
      "learning_rate": 2.723390997059943e-06,
      "logits": -1.0448968410491943,
      "logps": -75.1429214477539,
      "loss": 380.8563,
      "objective": 314.388671875,
      "ranking_idealized": 0.4833333194255829,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.6041666865348816,
      "regularize": 314.388671875,
      "step": 185
    },
    {
      "dpo_loss": 386.7085876464844,
      "epoch": 1.076995748700992,
      "grad_norm": 57801.560782942805,
      "learning_rate": 2.599391287742315e-06,
      "logits": -1.1095143556594849,
      "logps": -75.83429718017578,
      "loss": 325.9453,
      "objective": 386.7085876464844,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.6291666626930237,
      "regularize": 386.7085876464844,
      "step": 190
    },
    {
      "dpo_loss": 269.2171936035156,
      "epoch": 1.10533774208786,
      "grad_norm": 53241.40122481533,
      "learning_rate": 2.4751460371661763e-06,
      "logits": -1.094388484954834,
      "logps": -73.99842071533203,
      "loss": 353.2301,
      "objective": 269.2171936035156,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.6625000238418579,
      "regularize": 269.2171936035156,
      "step": 195
    },
    {
      "dpo_loss": 328.60662841796875,
      "epoch": 1.1336797354747283,
      "grad_norm": 57521.06044941492,
      "learning_rate": 2.3509621870754505e-06,
      "logits": -1.1270310878753662,
      "logps": -72.87967681884766,
      "loss": 353.4232,
      "objective": 328.60662841796875,
      "ranking_idealized": 0.5083333253860474,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.625,
      "regularize": 328.60662841796875,
      "step": 200
    },
    {
      "epoch": 1.1336797354747283,
      "eval_dpo_loss": 1618.9881591796875,
      "eval_logits": -1.149292230606079,
      "eval_logps": -77.86053466796875,
      "eval_loss": 1601.4404296875,
      "eval_objective": 1618.9881591796875,
      "eval_ranking_idealized": 0.5092975497245789,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5278925895690918,
      "eval_regularize": 1618.9881591796875,
      "eval_runtime": 257.8744,
      "eval_samples_per_second": 22.453,
      "eval_steps_per_second": 0.938,
      "step": 200
    },
    {
      "dpo_loss": 364.3101501464844,
      "epoch": 1.1620217288615966,
      "grad_norm": 48119.667583862116,
      "learning_rate": 2.2271465275271985e-06,
      "logits": -1.0912747383117676,
      "logps": -73.03921508789062,
      "loss": 341.8208,
      "objective": 364.3101501464844,
      "ranking_idealized": 0.48750001192092896,
      "ranking_idealized_expo": 0.48750001192092896,
      "ranking_simple": 0.5791666507720947,
      "regularize": 364.3101501464844,
      "step": 205
    },
    {
      "dpo_loss": 422.2685241699219,
      "epoch": 1.1903637222484649,
      "grad_norm": 57977.90371330273,
      "learning_rate": 2.1040049389819628e-06,
      "logits": -1.084999442100525,
      "logps": -72.47502136230469,
      "loss": 382.6563,
      "objective": 422.2685241699219,
      "ranking_idealized": 0.4958333373069763,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.5708333253860474,
      "regularize": 422.2685241699219,
      "step": 210
    },
    {
      "dpo_loss": 400.0838928222656,
      "epoch": 1.2187057156353331,
      "grad_norm": 59470.53561937635,
      "learning_rate": 1.9818416366412277e-06,
      "logits": -1.0878026485443115,
      "logps": -71.41203308105469,
      "loss": 375.9584,
      "objective": 400.0838928222656,
      "ranking_idealized": 0.4833333194255829,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.6208333373069763,
      "regularize": 400.0838928222656,
      "step": 215
    },
    {
      "dpo_loss": 394.82720947265625,
      "epoch": 1.2470477090222012,
      "grad_norm": 62495.95003659264,
      "learning_rate": 1.8609584188988135e-06,
      "logits": -0.9512097239494324,
      "logps": -71.59188842773438,
      "loss": 378.8627,
      "objective": 394.82720947265625,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.6000000238418579,
      "ranking_simple": 0.6583333611488342,
      "regularize": 394.82720947265625,
      "step": 220
    },
    {
      "dpo_loss": 384.0245666503906,
      "epoch": 1.2753897024090695,
      "grad_norm": 59991.038303506364,
      "learning_rate": 1.7416539217628792e-06,
      "logits": -0.978052020072937,
      "logps": -73.61431121826172,
      "loss": 346.6835,
      "objective": 384.0245666503906,
      "ranking_idealized": 0.5291666388511658,
      "ranking_idealized_expo": 0.5291666388511658,
      "ranking_simple": 0.6541666388511658,
      "regularize": 384.0245666503906,
      "step": 225
    },
    {
      "dpo_loss": 477.4090881347656,
      "epoch": 1.3037316957959377,
      "grad_norm": 52558.251165534326,
      "learning_rate": 1.6242228810904393e-06,
      "logits": -0.9276587963104248,
      "logps": -73.25977325439453,
      "loss": 379.5923,
      "objective": 477.4090881347656,
      "ranking_idealized": 0.5083333253860474,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.5958333611488342,
      "regularize": 477.4090881347656,
      "step": 230
    },
    {
      "dpo_loss": 208.77500915527344,
      "epoch": 1.3320736891828058,
      "grad_norm": 54005.47450339259,
      "learning_rate": 1.508955404457015e-06,
      "logits": -1.0076844692230225,
      "logps": -70.91206359863281,
      "loss": 359.3852,
      "objective": 208.77500915527344,
      "ranking_idealized": 0.4833333194255829,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.6458333134651184,
      "regularize": 208.77500915527344,
      "step": 235
    },
    {
      "dpo_loss": 276.1641845703125,
      "epoch": 1.360415682569674,
      "grad_norm": 54946.545236512575,
      "learning_rate": 1.3961362544602215e-06,
      "logits": -0.9443127512931824,
      "logps": -73.09326934814453,
      "loss": 346.4313,
      "objective": 276.1641845703125,
      "ranking_idealized": 0.4791666567325592,
      "ranking_idealized_expo": 0.4791666567325592,
      "ranking_simple": 0.612500011920929,
      "regularize": 276.1641845703125,
      "step": 240
    },
    {
      "dpo_loss": 417.3522644042969,
      "epoch": 1.3887576759565423,
      "grad_norm": 56968.024166225034,
      "learning_rate": 1.2860441452278578e-06,
      "logits": -0.9423906207084656,
      "logps": -73.41870880126953,
      "loss": 411.8242,
      "objective": 417.3522644042969,
      "ranking_idealized": 0.5166666507720947,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.6541666388511658,
      "regularize": 417.3522644042969,
      "step": 245
    },
    {
      "dpo_loss": 195.7690887451172,
      "epoch": 1.4170996693434104,
      "grad_norm": 56033.321076391,
      "learning_rate": 1.1789510538684524e-06,
      "logits": -0.9399241209030151,
      "logps": -73.39521789550781,
      "loss": 363.333,
      "objective": 195.7690887451172,
      "ranking_idealized": 0.47083333134651184,
      "ranking_idealized_expo": 0.47083333134651184,
      "ranking_simple": 0.6166666746139526,
      "regularize": 195.7690887451172,
      "step": 250
    },
    {
      "epoch": 1.4170996693434104,
      "eval_dpo_loss": 1577.62451171875,
      "eval_logits": -1.0660799741744995,
      "eval_logps": -78.80528259277344,
      "eval_loss": 1571.6953125,
      "eval_objective": 1577.62451171875,
      "eval_ranking_idealized": 0.5092975497245789,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5227272510528564,
      "eval_regularize": 1577.62451171875,
      "eval_runtime": 258.9959,
      "eval_samples_per_second": 22.356,
      "eval_steps_per_second": 0.934,
      "step": 250
    },
    {
      "dpo_loss": 237.82786560058594,
      "epoch": 1.4454416627302786,
      "grad_norm": 52246.00686535072,
      "learning_rate": 1.0751215485652644e-06,
      "logits": -0.9620916247367859,
      "logps": -74.39702606201172,
      "loss": 312.183,
      "objective": 237.82786560058594,
      "ranking_idealized": 0.5458333492279053,
      "ranking_idealized_expo": 0.5458333492279053,
      "ranking_simple": 0.6416666507720947,
      "regularize": 237.82786560058594,
      "step": 255
    },
    {
      "dpo_loss": 380.9892578125,
      "epoch": 1.473783656117147,
      "grad_norm": 48045.018050204635,
      "learning_rate": 9.74812134973689e-07,
      "logits": -1.0035159587860107,
      "logps": -74.84737396240234,
      "loss": 310.9174,
      "objective": 380.9892578125,
      "ranking_idealized": 0.48750001192092896,
      "ranking_idealized_expo": 0.48750001192092896,
      "ranking_simple": 0.6083333492279053,
      "regularize": 380.9892578125,
      "step": 260
    },
    {
      "dpo_loss": 283.63079833984375,
      "epoch": 1.5021256495040152,
      "grad_norm": 61447.10899392111,
      "learning_rate": 8.78270622536716e-07,
      "logits": -0.998799204826355,
      "logps": -73.86261749267578,
      "loss": 345.8337,
      "objective": 283.63079833984375,
      "ranking_idealized": 0.5041666626930237,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.625,
      "regularize": 283.63079833984375,
      "step": 265
    },
    {
      "dpo_loss": 343.7718505859375,
      "epoch": 1.5304676428908834,
      "grad_norm": 61792.26148648853,
      "learning_rate": 7.857355122839674e-07,
      "logits": -0.9898220300674438,
      "logps": -75.11653137207031,
      "loss": 291.3343,
      "objective": 343.7718505859375,
      "ranking_idealized": 0.5041666626930237,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.5874999761581421,
      "regularize": 343.7718505859375,
      "step": 270
    },
    {
      "dpo_loss": 356.6233825683594,
      "epoch": 1.5588096362777515,
      "grad_norm": 50405.016409353884,
      "learning_rate": 6.974354076267081e-07,
      "logits": -0.9836587905883789,
      "logps": -75.24365997314453,
      "loss": 303.3459,
      "objective": 356.6233825683594,
      "ranking_idealized": 0.5249999761581421,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.6541666388511658,
      "regularize": 356.6233825683594,
      "step": 275
    },
    {
      "dpo_loss": 222.919921875,
      "epoch": 1.5871516296646198,
      "grad_norm": 52417.49896381969,
      "learning_rate": 6.135884496044245e-07,
      "logits": -0.9070791602134705,
      "logps": -73.96660614013672,
      "loss": 284.174,
      "objective": 222.919921875,
      "ranking_idealized": 0.5041666626930237,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.5958333611488342,
      "regularize": 222.919921875,
      "step": 280
    },
    {
      "dpo_loss": 252.81446838378906,
      "epoch": 1.615493623051488,
      "grad_norm": 52470.37814589712,
      "learning_rate": 5.344017779781835e-07,
      "logits": -1.0163558721542358,
      "logps": -75.34479522705078,
      "loss": 278.5324,
      "objective": 252.81446838378906,
      "ranking_idealized": 0.5166666507720947,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.6333333253860474,
      "regularize": 252.81446838378906,
      "step": 285
    },
    {
      "dpo_loss": 298.5639953613281,
      "epoch": 1.643835616438356,
      "grad_norm": 49456.57742469281,
      "learning_rate": 4.6007101950209827e-07,
      "logits": -1.0395054817199707,
      "logps": -73.3790283203125,
      "loss": 274.9208,
      "objective": 298.5639953613281,
      "ranking_idealized": 0.5625,
      "ranking_idealized_expo": 0.5625,
      "ranking_simple": 0.6291666626930237,
      "regularize": 298.5639953613281,
      "step": 290
    },
    {
      "dpo_loss": 278.822265625,
      "epoch": 1.6721776098252243,
      "grad_norm": 54115.8579067689,
      "learning_rate": 3.907798046371139e-07,
      "logits": -0.9579232931137085,
      "logps": -72.22044372558594,
      "loss": 266.663,
      "objective": 278.822265625,
      "ranking_idealized": 0.4791666567325592,
      "ranking_idealized_expo": 0.4791666567325592,
      "ranking_simple": 0.6083333492279053,
      "regularize": 278.822265625,
      "step": 295
    },
    {
      "dpo_loss": 236.7002716064453,
      "epoch": 1.7005196032120926,
      "grad_norm": 50618.31251453447,
      "learning_rate": 3.266993139010438e-07,
      "logits": -1.0140774250030518,
      "logps": -74.12470245361328,
      "loss": 267.0769,
      "objective": 236.7002716064453,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.574999988079071,
      "ranking_simple": 0.6791666746139526,
      "regularize": 236.7002716064453,
      "step": 300
    },
    {
      "epoch": 1.7005196032120926,
      "eval_dpo_loss": 1538.6409912109375,
      "eval_logits": -1.0586777925491333,
      "eval_logps": -79.39220428466797,
      "eval_loss": 1533.135009765625,
      "eval_objective": 1538.6409912109375,
      "eval_ranking_idealized": 0.5092975497245789,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5227272510528564,
      "eval_regularize": 1538.6409912109375,
      "eval_runtime": 258.9331,
      "eval_samples_per_second": 22.361,
      "eval_steps_per_second": 0.935,
      "step": 300
    },
    {
      "dpo_loss": 347.6080627441406,
      "epoch": 1.7288615965989607,
      "grad_norm": 54409.86939811849,
      "learning_rate": 2.6798785497556184e-07,
      "logits": -0.9848151803016663,
      "logps": -75.27159881591797,
      "loss": 263.9634,
      "objective": 347.6080627441406,
      "ranking_idealized": 0.5041666626930237,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.6166666746139526,
      "regularize": 347.6080627441406,
      "step": 305
    },
    {
      "dpo_loss": 236.26036071777344,
      "epoch": 1.7572035899858292,
      "grad_norm": 55045.39892986952,
      "learning_rate": 2.1479047161491351e-07,
      "logits": -1.0300458669662476,
      "logps": -72.95113372802734,
      "loss": 302.2823,
      "objective": 236.26036071777344,
      "ranking_idealized": 0.5208333134651184,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.6208333373069763,
      "regularize": 236.26036071777344,
      "step": 310
    },
    {
      "dpo_loss": 353.65228271484375,
      "epoch": 1.7855455833726972,
      "grad_norm": 53063.82376430783,
      "learning_rate": 1.6723858532249783e-07,
      "logits": -0.9826658964157104,
      "logps": -73.60942077636719,
      "loss": 282.9365,
      "objective": 353.65228271484375,
      "ranking_idealized": 0.5083333253860474,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.6041666865348816,
      "regularize": 353.65228271484375,
      "step": 315
    },
    {
      "dpo_loss": 279.9169006347656,
      "epoch": 1.8138875767595655,
      "grad_norm": 50212.55160450671,
      "learning_rate": 1.2544967068054332e-07,
      "logits": -1.03180992603302,
      "logps": -75.23188781738281,
      "loss": 276.6402,
      "objective": 279.9169006347656,
      "ranking_idealized": 0.5041666626930237,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.625,
      "regularize": 279.9169006347656,
      "step": 320
    },
    {
      "dpo_loss": 301.3766174316406,
      "epoch": 1.8422295701464337,
      "grad_norm": 58249.94058427391,
      "learning_rate": 8.952696513496756e-08,
      "logits": -1.0694580078125,
      "logps": -76.53679656982422,
      "loss": 311.5521,
      "objective": 301.3766174316406,
      "ranking_idealized": 0.5166666507720947,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.6541666388511658,
      "regularize": 301.3766174316406,
      "step": 325
    },
    {
      "dpo_loss": 271.7821044921875,
      "epoch": 1.8705715635333018,
      "grad_norm": 49890.30930096369,
      "learning_rate": 5.955921395237318e-08,
      "logits": -0.8758689761161804,
      "logps": -73.24466705322266,
      "loss": 278.1949,
      "objective": 271.7821044921875,
      "ranking_idealized": 0.47083333134651184,
      "ranking_idealized_expo": 0.47083333134651184,
      "ranking_simple": 0.6000000238418579,
      "regularize": 271.7821044921875,
      "step": 330
    },
    {
      "dpo_loss": 311.26239013671875,
      "epoch": 1.89891355692017,
      "grad_norm": 45583.78689582908,
      "learning_rate": 3.5620450979259034e-08,
      "logits": -1.0153381824493408,
      "logps": -73.08647155761719,
      "loss": 255.9677,
      "objective": 311.26239013671875,
      "ranking_idealized": 0.4958333373069763,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.5958333611488342,
      "regularize": 311.26239013671875,
      "step": 335
    },
    {
      "dpo_loss": 248.50962829589844,
      "epoch": 1.9272555503070383,
      "grad_norm": 56451.11973625406,
      "learning_rate": 1.7769815745066476e-08,
      "logits": -0.9851234555244446,
      "logps": -75.27400207519531,
      "loss": 311.7736,
      "objective": 248.50962829589844,
      "ranking_idealized": 0.5416666865348816,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.6541666388511658,
      "regularize": 248.50962829589844,
      "step": 340
    },
    {
      "dpo_loss": 199.6634979248047,
      "epoch": 1.9555975436939064,
      "grad_norm": 57100.11984573264,
      "learning_rate": 6.051407360895822e-09,
      "logits": -0.9951305985450745,
      "logps": -74.00302124023438,
      "loss": 260.0727,
      "objective": 199.6634979248047,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.6166666746139526,
      "regularize": 199.6634979248047,
      "step": 345
    },
    {
      "dpo_loss": 261.43450927734375,
      "epoch": 1.9839395370807746,
      "grad_norm": 49743.33740440382,
      "learning_rate": 4.941755748361088e-10,
      "logits": -0.9457499384880066,
      "logps": -74.98257446289062,
      "loss": 287.4463,
      "objective": 261.43450927734375,
      "ranking_idealized": 0.48750001192092896,
      "ranking_idealized_expo": 0.48750001192092896,
      "ranking_simple": 0.5958333611488342,
      "regularize": 261.43450927734375,
      "step": 350
    },
    {
      "epoch": 1.9839395370807746,
      "eval_dpo_loss": 1520.4884033203125,
      "eval_logits": -1.0707263946533203,
      "eval_logps": -79.11678314208984,
      "eval_loss": 1521.1865234375,
      "eval_objective": 1520.4884033203125,
      "eval_ranking_idealized": 0.5092975497245789,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5258264541625977,
      "eval_regularize": 1520.4884033203125,
      "eval_runtime": 261.9749,
      "eval_samples_per_second": 22.101,
      "eval_steps_per_second": 0.924,
      "step": 350
    },
    {
      "epoch": 1.995276334435522,
      "step": 352,
      "total_flos": 0.0,
      "train_loss": 584.4622564356436,
      "train_runtime": 14141.3958,
      "train_samples_per_second": 7.185,
      "train_steps_per_second": 0.025
    }
  ],
  "logging_steps": 5,
  "max_steps": 352,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}