{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.995276334435522,
  "eval_steps": 50,
  "global_step": 352,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "dpo_loss": 0.6931471824645996,
      "epoch": 0.005668398677373642,
      "grad_norm": 1341.3687944189992,
      "learning_rate": 1.3888888888888888e-07,
      "logits": -1.3147305250167847,
      "logps": -88.0877456665039,
      "loss": 0.6931,
      "objective": 0.6931471824645996,
      "ranking_idealized": 0.5208333134651184,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.5208333134651184,
      "regularize": 0.6931471824645996,
      "step": 1
    },
    {
      "dpo_loss": 1.0099722146987915,
      "epoch": 0.02834199338686821,
      "grad_norm": 2182.754930022205,
      "learning_rate": 6.944444444444446e-07,
      "logits": -1.367998719215393,
      "logps": -84.27975463867188,
      "loss": 0.9543,
      "objective": 1.0099722146987915,
      "ranking_idealized": 0.546875,
      "ranking_idealized_expo": 0.546875,
      "ranking_simple": 0.546875,
      "regularize": 1.0099722146987915,
      "step": 5
    },
    {
      "dpo_loss": 1.232261061668396,
      "epoch": 0.05668398677373642,
      "grad_norm": 1955.6998628854572,
      "learning_rate": 1.3888888888888892e-06,
      "logits": -1.4522252082824707,
      "logps": -83.40966796875,
      "loss": 1.4248,
      "objective": 1.232261061668396,
      "ranking_idealized": 0.512499988079071,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.512499988079071,
      "regularize": 1.232261061668396,
      "step": 10
    },
    {
      "dpo_loss": 3.4686033725738525,
      "epoch": 0.08502598016060463,
      "grad_norm": 2014.579294209837,
      "learning_rate": 2.0833333333333334e-06,
      "logits": -1.4239295721054077,
      "logps": -83.23849487304688,
      "loss": 2.5533,
      "objective": 3.4686033725738525,
      "ranking_idealized": 0.5041666626930237,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.5083333253860474,
      "regularize": 3.4686033725738525,
      "step": 15
    },
    {
      "dpo_loss": 4.306813716888428,
      "epoch": 0.11336797354747284,
      "grad_norm": 1483.6608319475595,
      "learning_rate": 2.7777777777777783e-06,
      "logits": -1.3964630365371704,
      "logps": -84.55473327636719,
      "loss": 3.9251,
      "objective": 4.306813716888428,
      "ranking_idealized": 0.5375000238418579,
      "ranking_idealized_expo": 0.5375000238418579,
      "ranking_simple": 0.550000011920929,
      "regularize": 4.306813716888428,
      "step": 20
    },
    {
      "dpo_loss": 4.465227127075195,
      "epoch": 0.14170996693434104,
      "grad_norm": 1726.5319247739817,
      "learning_rate": 3.4722222222222224e-06,
      "logits": -1.406508207321167,
      "logps": -83.8473129272461,
      "loss": 5.3642,
      "objective": 4.465227127075195,
      "ranking_idealized": 0.4958333373069763,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.5083333253860474,
      "regularize": 4.465227127075195,
      "step": 25
    },
    {
      "dpo_loss": 7.850214004516602,
      "epoch": 0.17005196032120926,
      "grad_norm": 1356.804024894421,
      "learning_rate": 4.166666666666667e-06,
      "logits": -1.386705756187439,
      "logps": -84.02658081054688,
      "loss": 6.5533,
      "objective": 7.850214004516602,
      "ranking_idealized": 0.5416666865348816,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.550000011920929,
      "regularize": 7.850214004516602,
      "step": 30
    },
    {
      "dpo_loss": 7.777320384979248,
      "epoch": 0.19839395370807747,
      "grad_norm": 1260.3392527439414,
      "learning_rate": 4.861111111111111e-06,
      "logits": -1.3726133108139038,
      "logps": -81.83206176757812,
      "loss": 6.6118,
      "objective": 7.777320384979248,
      "ranking_idealized": 0.5041666626930237,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.5291666388511658,
      "regularize": 7.777320384979248,
      "step": 35
    },
    {
      "dpo_loss": 6.831168174743652,
      "epoch": 0.22673594709494568,
      "grad_norm": 1252.0799000439422,
      "learning_rate": 4.998023493068255e-06,
      "logits": -1.359412431716919,
      "logps": -81.25328826904297,
      "loss": 7.7682,
      "objective": 6.831168174743652,
      "ranking_idealized": 0.5208333134651184,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.5625,
      "regularize": 6.831168174743652,
      "step": 40
    },
    {
      "dpo_loss": 8.811128616333008,
      "epoch": 0.25507794048181387,
      "grad_norm": 1307.005375082768,
      "learning_rate": 4.989999289644993e-06,
      "logits": -1.483437418937683,
      "logps": -85.30819702148438,
      "loss": 8.3899,
      "objective": 8.811128616333008,
      "ranking_idealized": 0.5375000238418579,
      "ranking_idealized_expo": 0.5375000238418579,
      "ranking_simple": 0.5666666626930237,
      "regularize": 8.811128616333008,
      "step": 45
    },
    {
      "dpo_loss": 8.527603149414062,
      "epoch": 0.2834199338686821,
      "grad_norm": 1144.3338187353374,
      "learning_rate": 4.975823666181256e-06,
      "logits": -1.4239095449447632,
      "logps": -83.47504425048828,
      "loss": 9.5723,
      "objective": 8.527603149414062,
      "ranking_idealized": 0.42500001192092896,
      "ranking_idealized_expo": 0.42500001192092896,
      "ranking_simple": 0.5041666626930237,
      "regularize": 8.527603149414062,
      "step": 50
    },
    {
      "epoch": 0.2834199338686821,
      "eval_dpo_loss": 9.650106430053711,
      "eval_logits": -1.4979068040847778,
      "eval_logps": -89.68624114990234,
      "eval_loss": 9.258649826049805,
      "eval_objective": 9.650106430053711,
      "eval_ranking_idealized": 0.5092975497245789,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5134297609329224,
      "eval_regularize": 9.650106430053711,
      "eval_runtime": 260.9367,
      "eval_samples_per_second": 22.189,
      "eval_steps_per_second": 0.927,
      "step": 50
    },
    {
      "dpo_loss": 13.282089233398438,
      "epoch": 0.3117619272555503,
      "grad_norm": 1044.8496740930686,
      "learning_rate": 4.955531642853404e-06,
      "logits": -1.527586579322815,
      "logps": -82.80742645263672,
      "loss": 10.597,
      "objective": 13.282089233398438,
      "ranking_idealized": 0.5416666865348816,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.5625,
      "regularize": 13.282089233398438,
      "step": 55
    },
    {
      "dpo_loss": 10.371745109558105,
      "epoch": 0.3401039206424185,
      "grad_norm": 1331.4227369121395,
      "learning_rate": 4.929173350101025e-06,
      "logits": -1.4083276987075806,
      "logps": -79.24844360351562,
      "loss": 10.3404,
      "objective": 10.371745109558105,
      "ranking_idealized": 0.5166666507720947,
      "ranking_idealized_expo": 0.512499988079071,
      "ranking_simple": 0.5625,
      "regularize": 10.371745109558105,
      "step": 60
    },
    {
      "dpo_loss": 10.49330997467041,
      "epoch": 0.3684459140292867,
      "grad_norm": 1137.902525031776,
      "learning_rate": 4.896813904782162e-06,
      "logits": -1.2618812322616577,
      "logps": -80.0319595336914,
      "loss": 11.0189,
      "objective": 10.49330997467041,
      "ranking_idealized": 0.5083333253860474,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.5541666746139526,
      "regularize": 10.49330997467041,
      "step": 65
    },
    {
      "dpo_loss": 9.181135177612305,
      "epoch": 0.39678790741615494,
      "grad_norm": 1088.944949936674,
      "learning_rate": 4.858533249305337e-06,
      "logits": -1.2879325151443481,
      "logps": -80.44699096679688,
      "loss": 11.3087,
      "objective": 9.181135177612305,
      "ranking_idealized": 0.5166666507720947,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5416666865348816,
      "regularize": 9.181135177612305,
      "step": 70
    },
    {
      "dpo_loss": 10.468655586242676,
      "epoch": 0.42512990080302315,
      "grad_norm": 999.1732460095175,
      "learning_rate": 4.814425954135786e-06,
      "logits": -1.2509528398513794,
      "logps": -79.77677154541016,
      "loss": 11.3742,
      "objective": 10.468655586242676,
      "ranking_idealized": 0.5458333492279053,
      "ranking_idealized_expo": 0.5458333492279053,
      "ranking_simple": 0.5791666507720947,
      "regularize": 10.468655586242676,
      "step": 75
    },
    {
      "dpo_loss": 12.168583869934082,
      "epoch": 0.45347189418989137,
      "grad_norm": 1008.7380345506941,
      "learning_rate": 4.764600984163809e-06,
      "logits": -1.282080888748169,
      "logps": -77.80094909667969,
      "loss": 11.9338,
      "objective": 12.168583869934082,
      "ranking_idealized": 0.5166666507720947,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.6000000238418579,
      "regularize": 12.168583869934082,
      "step": 80
    },
    {
      "dpo_loss": 8.932952880859375,
      "epoch": 0.4818138875767596,
      "grad_norm": 1016.915697188663,
      "learning_rate": 4.709181429512391e-06,
      "logits": -1.2718119621276855,
      "logps": -74.034912109375,
      "loss": 11.049,
      "objective": 8.932952880859375,
      "ranking_idealized": 0.5249999761581421,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.5791666507720947,
      "regularize": 8.932952880859375,
      "step": 85
    },
    {
      "dpo_loss": 13.262398719787598,
      "epoch": 0.5101558809636277,
      "grad_norm": 943.5896837416125,
      "learning_rate": 4.648304201449153e-06,
      "logits": -1.1908345222473145,
      "logps": -76.21371459960938,
      "loss": 11.6717,
      "objective": 13.262398719787598,
      "ranking_idealized": 0.4791666567325592,
      "ranking_idealized_expo": 0.4791666567325592,
      "ranking_simple": 0.49166667461395264,
      "regularize": 13.262398719787598,
      "step": 90
    },
    {
      "dpo_loss": 8.800294876098633,
      "epoch": 0.538497874350496,
      "grad_norm": 1052.3099264342789,
      "learning_rate": 4.582119694153834e-06,
      "logits": -1.3108305931091309,
      "logps": -74.5205078125,
      "loss": 9.5719,
      "objective": 8.800294876098633,
      "ranking_idealized": 0.5416666865348816,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.612500011920929,
      "regularize": 8.800294876098633,
      "step": 95
    },
    {
      "dpo_loss": 7.865393161773682,
      "epoch": 0.5668398677373642,
      "grad_norm": 888.472953862153,
      "learning_rate": 4.510791413176912e-06,
      "logits": -1.3763591051101685,
      "logps": -75.36319732666016,
      "loss": 9.8364,
      "objective": 7.865393161773682,
      "ranking_idealized": 0.5416666865348816,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.5958333611488342,
      "regularize": 7.865393161773682,
      "step": 100
    },
    {
      "epoch": 0.5668398677373642,
      "eval_dpo_loss": 15.540860176086426,
      "eval_logits": -1.3475213050842285,
      "eval_logps": -79.4201431274414,
      "eval_loss": 15.545284271240234,
      "eval_objective": 15.540860176086426,
      "eval_ranking_idealized": 0.5092975497245789,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5175619721412659,
      "eval_regularize": 15.540860176086426,
      "eval_runtime": 260.1572,
      "eval_samples_per_second": 22.256,
      "eval_steps_per_second": 0.93,
      "step": 100
    },
    {
      "dpo_loss": 9.041933059692383,
      "epoch": 0.5951818611242324,
      "grad_norm": 887.351501300793,
      "learning_rate": 4.4344955715072344e-06,
      "logits": -1.1800312995910645,
      "logps": -76.02098083496094,
      "loss": 10.6979,
      "objective": 9.041933059692383,
      "ranking_idealized": 0.5375000238418579,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.6083333492279053,
      "regularize": 9.041933059692383,
      "step": 105
    },
    {
      "dpo_loss": 10.477563858032227,
      "epoch": 0.6235238545111006,
      "grad_norm": 852.2946697576,
      "learning_rate": 4.353420654246546e-06,
      "logits": -1.2490488290786743,
      "logps": -74.80319213867188,
      "loss": 9.0671,
      "objective": 10.477563858032227,
      "ranking_idealized": 0.44999998807907104,
      "ranking_idealized_expo": 0.44999998807907104,
      "ranking_simple": 0.5375000238418579,
      "regularize": 10.477563858032227,
      "step": 110
    },
    {
      "dpo_loss": 9.403600692749023,
      "epoch": 0.6518658478979689,
      "grad_norm": 841.5806761052793,
      "learning_rate": 4.267766952966369e-06,
      "logits": -1.2325801849365234,
      "logps": -73.80712127685547,
      "loss": 9.6957,
      "objective": 9.403600692749023,
      "ranking_idealized": 0.4833333194255829,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.5625,
      "regularize": 9.403600692749023,
      "step": 115
    },
    {
      "dpo_loss": 12.634725570678711,
      "epoch": 0.680207841284837,
      "grad_norm": 843.1072296477777,
      "learning_rate": 4.177746070897593e-06,
      "logits": -1.159250259399414,
      "logps": -74.65933990478516,
      "loss": 10.2588,
      "objective": 12.634725570678711,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5958333611488342,
      "regularize": 12.634725570678711,
      "step": 120
    },
    {
      "dpo_loss": 13.12292766571045,
      "epoch": 0.7085498346717053,
      "grad_norm": 839.6378874859962,
      "learning_rate": 4.083580400175153e-06,
      "logits": -1.1041706800460815,
      "logps": -73.04627990722656,
      "loss": 9.9023,
      "objective": 13.12292766571045,
      "ranking_idealized": 0.550000011920929,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.6041666865348816,
      "regularize": 13.12292766571045,
      "step": 125
    },
    {
      "dpo_loss": 9.77008056640625,
      "epoch": 0.7368918280585735,
      "grad_norm": 932.0089885121237,
      "learning_rate": 3.985502572429277e-06,
      "logits": -1.1575837135314941,
      "logps": -72.78858184814453,
      "loss": 9.9485,
      "objective": 9.77008056640625,
      "ranking_idealized": 0.44999998807907104,
      "ranking_idealized_expo": 0.44999998807907104,
      "ranking_simple": 0.5249999761581421,
      "regularize": 9.77008056640625,
      "step": 130
    },
    {
      "dpo_loss": 9.381123542785645,
      "epoch": 0.7652338214454416,
      "grad_norm": 845.3231606025312,
      "learning_rate": 3.8837548840805395e-06,
      "logits": -1.137967824935913,
      "logps": -75.26256561279297,
      "loss": 9.4092,
      "objective": 9.381123542785645,
      "ranking_idealized": 0.5291666388511658,
      "ranking_idealized_expo": 0.5291666388511658,
      "ranking_simple": 0.5791666507720947,
      "regularize": 9.381123542785645,
      "step": 135
    },
    {
      "dpo_loss": 8.004253387451172,
      "epoch": 0.7935758148323099,
      "grad_norm": 733.7736305772362,
      "learning_rate": 3.7785886977585562e-06,
      "logits": -1.0919610261917114,
      "logps": -74.39104461669922,
      "loss": 8.376,
      "objective": 8.004253387451172,
      "ranking_idealized": 0.5041666626930237,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.5625,
      "regularize": 8.004253387451172,
      "step": 140
    },
    {
      "dpo_loss": 6.367016315460205,
      "epoch": 0.821917808219178,
      "grad_norm": 796.4959704194974,
      "learning_rate": 3.6702638213230344e-06,
      "logits": -1.115335464477539,
      "logps": -75.12439727783203,
      "loss": 7.6587,
      "objective": 6.367016315460205,
      "ranking_idealized": 0.4833333194255829,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.5541666746139526,
      "regularize": 6.367016315460205,
      "step": 145
    },
    {
      "dpo_loss": 8.81261920928955,
      "epoch": 0.8502598016060463,
      "grad_norm": 773.8278413807583,
      "learning_rate": 3.5590478660213214e-06,
      "logits": -1.0867894887924194,
      "logps": -74.4196548461914,
      "loss": 8.8451,
      "objective": 8.81261920928955,
      "ranking_idealized": 0.5416666865348816,
      "ranking_idealized_expo": 0.5375000238418579,
      "ranking_simple": 0.5791666507720947,
      "regularize": 8.81261920928955,
      "step": 150
    },
    {
      "epoch": 0.8502598016060463,
      "eval_dpo_loss": 16.562625885009766,
      "eval_logits": -1.1121788024902344,
      "eval_logps": -82.14590454101562,
      "eval_loss": 16.66257095336914,
      "eval_objective": 16.562625885009766,
      "eval_ranking_idealized": 0.5092975497245789,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5144628286361694,
      "eval_regularize": 16.562625885009766,
      "eval_runtime": 259.9617,
      "eval_samples_per_second": 22.273,
      "eval_steps_per_second": 0.931,
      "step": 150
    },
    {
      "dpo_loss": 7.545068264007568,
      "epoch": 0.8786017949929145,
      "grad_norm": 757.3947735379465,
      "learning_rate": 3.4452155853680457e-06,
      "logits": -1.0470577478408813,
      "logps": -75.96894836425781,
      "loss": 7.0262,
      "objective": 7.545068264007568,
      "ranking_idealized": 0.512499988079071,
      "ranking_idealized_expo": 0.512499988079071,
      "ranking_simple": 0.6208333373069763,
      "regularize": 7.545068264007568,
      "step": 155
    },
    {
      "dpo_loss": 6.935917854309082,
      "epoch": 0.9069437883797827,
      "grad_norm": 751.9625745316074,
      "learning_rate": 3.32904819638017e-06,
      "logits": -0.919123113155365,
      "logps": -78.4984359741211,
      "loss": 7.8293,
      "objective": 6.935917854309082,
      "ranking_idealized": 0.4958333373069763,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.5916666388511658,
      "regularize": 6.935917854309082,
      "step": 160
    },
    {
      "dpo_loss": 7.154492378234863,
      "epoch": 0.9352857817666509,
      "grad_norm": 658.8620194691712,
      "learning_rate": 3.2108326848442507e-06,
      "logits": -0.9658632278442383,
      "logps": -79.98240661621094,
      "loss": 7.7063,
      "objective": 7.154492378234863,
      "ranking_idealized": 0.5208333134651184,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.5958333611488342,
      "regularize": 7.154492378234863,
      "step": 165
    },
    {
      "dpo_loss": 7.305918216705322,
      "epoch": 0.9636277751535192,
      "grad_norm": 788.1879535129325,
      "learning_rate": 3.090861096332263e-06,
      "logits": -1.0152103900909424,
      "logps": -79.22311401367188,
      "loss": 7.4405,
      "objective": 7.305918216705322,
      "ranking_idealized": 0.512499988079071,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.5708333253860474,
      "regularize": 7.305918216705322,
      "step": 170
    },
    {
      "dpo_loss": 7.161056995391846,
      "epoch": 0.9919697685403873,
      "grad_norm": 703.4556255689452,
      "learning_rate": 2.9694298147174566e-06,
      "logits": -1.0000672340393066,
      "logps": -78.11359405517578,
      "loss": 6.5526,
      "objective": 7.161056995391846,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.574999988079071,
      "ranking_simple": 0.6791666746139526,
      "regularize": 7.161056995391846,
      "step": 175
    },
    {
      "dpo_loss": 5.952729225158691,
      "epoch": 1.0203117619272555,
      "grad_norm": 665.4146360676681,
      "learning_rate": 2.8468388299726714e-06,
      "logits": -1.1097006797790527,
      "logps": -78.40138244628906,
      "loss": 4.6537,
      "objective": 5.952729225158691,
      "ranking_idealized": 0.5416666865348816,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.637499988079071,
      "regularize": 5.952729225158691,
      "step": 180
    },
    {
      "dpo_loss": 4.262913703918457,
      "epoch": 1.0486537553141237,
      "grad_norm": 584.0991136785685,
      "learning_rate": 2.723390997059943e-06,
      "logits": -1.034575343132019,
      "logps": -76.80313110351562,
      "loss": 3.7788,
      "objective": 4.262913703918457,
      "ranking_idealized": 0.4833333194255829,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.5791666507720947,
      "regularize": 4.262913703918457,
      "step": 185
    },
    {
      "dpo_loss": 3.1785500049591064,
      "epoch": 1.076995748700992,
      "grad_norm": 544.0621151708413,
      "learning_rate": 2.599391287742315e-06,
      "logits": -1.0769795179367065,
      "logps": -77.34581756591797,
      "loss": 3.5653,
      "objective": 3.1785500049591064,
      "ranking_idealized": 0.5541666746139526,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.6458333134651184,
      "regularize": 3.1785500049591064,
      "step": 190
    },
    {
      "dpo_loss": 2.1757826805114746,
      "epoch": 1.10533774208786,
      "grad_norm": 485.2987165965692,
      "learning_rate": 2.4751460371661763e-06,
      "logits": -1.0281838178634644,
      "logps": -76.58784484863281,
      "loss": 3.4138,
      "objective": 2.1757826805114746,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.6666666865348816,
      "regularize": 2.1757826805114746,
      "step": 195
    },
    {
      "dpo_loss": 2.5434367656707764,
      "epoch": 1.1336797354747283,
      "grad_norm": 579.2034510566444,
      "learning_rate": 2.3509621870754505e-06,
      "logits": -1.0624656677246094,
      "logps": -76.0323715209961,
      "loss": 3.8083,
      "objective": 2.5434367656707764,
      "ranking_idealized": 0.5083333253860474,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.6208333373069763,
      "regularize": 2.5434367656707764,
      "step": 200
    },
    {
      "epoch": 1.1336797354747283,
      "eval_dpo_loss": 16.32398223876953,
      "eval_logits": -1.0874205827713013,
      "eval_logps": -81.67513275146484,
      "eval_loss": 16.051929473876953,
      "eval_objective": 16.32398223876953,
      "eval_ranking_idealized": 0.5092975497245789,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5185950398445129,
      "eval_regularize": 16.32398223876953,
      "eval_runtime": 260.3079,
      "eval_samples_per_second": 22.243,
      "eval_steps_per_second": 0.93,
      "step": 200
    },
    {
      "dpo_loss": 4.850390434265137,
      "epoch": 1.1620217288615966,
      "grad_norm": 492.7125988147346,
      "learning_rate": 2.2271465275271985e-06,
      "logits": -1.0401540994644165,
      "logps": -76.98418426513672,
      "loss": 3.4645,
      "objective": 4.850390434265137,
      "ranking_idealized": 0.48750001192092896,
      "ranking_idealized_expo": 0.48750001192092896,
      "ranking_simple": 0.6041666865348816,
      "regularize": 4.850390434265137,
      "step": 205
    },
    {
      "dpo_loss": 3.4442708492279053,
      "epoch": 1.1903637222484649,
      "grad_norm": 560.4210922655933,
      "learning_rate": 2.1040049389819628e-06,
      "logits": -1.0271131992340088,
      "logps": -76.39945220947266,
      "loss": 3.425,
      "objective": 3.4442708492279053,
      "ranking_idealized": 0.5,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.5874999761581421,
      "regularize": 3.4442708492279053,
      "step": 210
    },
    {
      "dpo_loss": 4.348363399505615,
      "epoch": 1.2187057156353331,
      "grad_norm": 570.4040655884006,
      "learning_rate": 1.9818416366412277e-06,
      "logits": -0.9843409657478333,
      "logps": -75.80220031738281,
      "loss": 3.8326,
      "objective": 4.348363399505615,
      "ranking_idealized": 0.4833333194255829,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.612500011920929,
      "regularize": 4.348363399505615,
      "step": 215
    },
    {
      "dpo_loss": 3.9427292346954346,
      "epoch": 1.2470477090222012,
      "grad_norm": 674.1094626776156,
      "learning_rate": 1.8609584188988135e-06,
      "logits": -0.8341898322105408,
      "logps": -75.6758804321289,
      "loss": 3.8234,
      "objective": 3.9427292346954346,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.6000000238418579,
      "ranking_simple": 0.675000011920929,
      "regularize": 3.9427292346954346,
      "step": 220
    },
    {
      "dpo_loss": 3.127484083175659,
      "epoch": 1.2753897024090695,
      "grad_norm": 586.9637333254279,
      "learning_rate": 1.7416539217628792e-06,
      "logits": -0.8649470806121826,
      "logps": -77.80410766601562,
      "loss": 3.7703,
      "objective": 3.127484083175659,
      "ranking_idealized": 0.5291666388511658,
      "ranking_idealized_expo": 0.5291666388511658,
      "ranking_simple": 0.6541666388511658,
      "regularize": 3.127484083175659,
      "step": 225
    },
    {
      "dpo_loss": 3.9277312755584717,
      "epoch": 1.3037316957959377,
      "grad_norm": 500.9237074632472,
      "learning_rate": 1.6242228810904393e-06,
      "logits": -0.806861162185669,
      "logps": -77.2365951538086,
      "loss": 3.2759,
      "objective": 3.9277312755584717,
      "ranking_idealized": 0.5083333253860474,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.5833333134651184,
      "regularize": 3.9277312755584717,
      "step": 230
    },
    {
      "dpo_loss": 2.1227903366088867,
      "epoch": 1.3320736891828058,
      "grad_norm": 529.1823240184154,
      "learning_rate": 1.508955404457015e-06,
      "logits": -0.8957360982894897,
      "logps": -74.15618133544922,
      "loss": 3.8217,
      "objective": 2.1227903366088867,
      "ranking_idealized": 0.4833333194255829,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.6416666507720947,
      "regularize": 2.1227903366088867,
      "step": 235
    },
    {
      "dpo_loss": 3.4101717472076416,
      "epoch": 1.360415682569674,
      "grad_norm": 527.5490363089041,
      "learning_rate": 1.3961362544602215e-06,
      "logits": -0.8434502482414246,
      "logps": -76.20458221435547,
      "loss": 3.1011,
      "objective": 3.4101717472076416,
      "ranking_idealized": 0.4791666567325592,
      "ranking_idealized_expo": 0.4791666567325592,
      "ranking_simple": 0.625,
      "regularize": 3.4101717472076416,
      "step": 240
    },
    {
      "dpo_loss": 3.476386070251465,
      "epoch": 1.3887576759565423,
      "grad_norm": 571.2307522066004,
      "learning_rate": 1.2860441452278578e-06,
      "logits": -0.8681662082672119,
      "logps": -76.64582061767578,
      "loss": 3.8625,
      "objective": 3.476386070251465,
      "ranking_idealized": 0.5208333134651184,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.612500011920929,
      "regularize": 3.476386070251465,
      "step": 245
    },
    {
      "dpo_loss": 1.8152854442596436,
      "epoch": 1.4170996693434104,
      "grad_norm": 506.7476965423747,
      "learning_rate": 1.1789510538684524e-06,
      "logits": -0.8697729110717773,
      "logps": -76.35972595214844,
      "loss": 3.6019,
      "objective": 1.8152854442596436,
      "ranking_idealized": 0.47083333134651184,
      "ranking_idealized_expo": 0.47083333134651184,
      "ranking_simple": 0.612500011920929,
      "regularize": 1.8152854442596436,
      "step": 250
    },
    {
      "epoch": 1.4170996693434104,
      "eval_dpo_loss": 15.767914772033691,
      "eval_logits": -0.993295431137085,
      "eval_logps": -81.56088256835938,
      "eval_loss": 15.814361572265625,
      "eval_objective": 15.767914772033691,
      "eval_ranking_idealized": 0.5092975497245789,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5175619721412659,
      "eval_regularize": 15.767914772033691,
      "eval_runtime": 259.6498,
      "eval_samples_per_second": 22.299,
      "eval_steps_per_second": 0.932,
      "step": 250
    },
    {
      "dpo_loss": 3.3358986377716064,
      "epoch": 1.4454416627302786,
      "grad_norm": 563.3612521264971,
      "learning_rate": 1.0751215485652644e-06,
      "logits": -0.891727089881897,
      "logps": -76.76223754882812,
      "loss": 3.0719,
      "objective": 3.3358986377716064,
      "ranking_idealized": 0.5458333492279053,
      "ranking_idealized_expo": 0.5458333492279053,
      "ranking_simple": 0.637499988079071,
      "regularize": 3.3358986377716064,
      "step": 255
    },
    {
      "dpo_loss": 2.244086265563965,
      "epoch": 1.473783656117147,
      "grad_norm": 518.9208628864491,
      "learning_rate": 9.74812134973689e-07,
      "logits": -0.9449579119682312,
      "logps": -76.72704315185547,
      "loss": 2.8837,
      "objective": 2.244086265563965,
      "ranking_idealized": 0.48750001192092896,
      "ranking_idealized_expo": 0.48750001192092896,
      "ranking_simple": 0.6041666865348816,
      "regularize": 2.244086265563965,
      "step": 260
    },
    {
      "dpo_loss": 3.1619741916656494,
      "epoch": 1.5021256495040152,
      "grad_norm": 583.760652002654,
      "learning_rate": 8.78270622536716e-07,
      "logits": -0.953507125377655,
      "logps": -75.11036682128906,
      "loss": 2.925,
      "objective": 3.1619741916656494,
      "ranking_idealized": 0.5041666626930237,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.6083333492279053,
      "regularize": 3.1619741916656494,
      "step": 265
    },
    {
      "dpo_loss": 2.98382830619812,
      "epoch": 1.5304676428908834,
      "grad_norm": 520.7932622156393,
      "learning_rate": 7.857355122839674e-07,
      "logits": -0.9570529460906982,
      "logps": -76.34257507324219,
      "loss": 2.3431,
      "objective": 2.98382830619812,
      "ranking_idealized": 0.5083333253860474,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.6041666865348816,
      "regularize": 2.98382830619812,
      "step": 270
    },
    {
      "dpo_loss": 2.821234941482544,
      "epoch": 1.5588096362777515,
      "grad_norm": 536.6516299894466,
      "learning_rate": 6.974354076267081e-07,
      "logits": -0.9497679471969604,
      "logps": -76.84500122070312,
      "loss": 3.0543,
      "objective": 2.821234941482544,
      "ranking_idealized": 0.5249999761581421,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.6625000238418579,
      "regularize": 2.821234941482544,
      "step": 275
    },
    {
      "dpo_loss": 1.6416581869125366,
      "epoch": 1.5871516296646198,
      "grad_norm": 473.79557869240267,
      "learning_rate": 6.135884496044245e-07,
      "logits": -0.8786503076553345,
      "logps": -75.35763549804688,
      "loss": 2.5903,
      "objective": 1.6416581869125366,
      "ranking_idealized": 0.5041666626930237,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.6000000238418579,
      "regularize": 1.6416581869125366,
      "step": 280
    },
    {
      "dpo_loss": 2.1852915287017822,
      "epoch": 1.615493623051488,
      "grad_norm": 530.1117274697609,
      "learning_rate": 5.344017779781835e-07,
      "logits": -0.9772613048553467,
      "logps": -76.81114196777344,
      "loss": 2.5659,
      "objective": 2.1852915287017822,
      "ranking_idealized": 0.5166666507720947,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.6166666746139526,
      "regularize": 2.1852915287017822,
      "step": 285
    },
    {
      "dpo_loss": 3.540595769882202,
      "epoch": 1.643835616438356,
      "grad_norm": 543.628954255078,
      "learning_rate": 4.6007101950209827e-07,
      "logits": -0.9982803463935852,
      "logps": -74.47586059570312,
      "loss": 3.1645,
      "objective": 3.540595769882202,
      "ranking_idealized": 0.5625,
      "ranking_idealized_expo": 0.5625,
      "ranking_simple": 0.625,
      "regularize": 3.540595769882202,
      "step": 290
    },
    {
      "dpo_loss": 1.856958031654358,
      "epoch": 1.6721776098252243,
      "grad_norm": 523.9882241509144,
      "learning_rate": 3.907798046371139e-07,
      "logits": -0.9143877029418945,
      "logps": -73.36082458496094,
      "loss": 2.4952,
      "objective": 1.856958031654358,
      "ranking_idealized": 0.4791666567325592,
      "ranking_idealized_expo": 0.4791666567325592,
      "ranking_simple": 0.612500011920929,
      "regularize": 1.856958031654358,
      "step": 295
    },
    {
      "dpo_loss": 1.6541905403137207,
      "epoch": 1.7005196032120926,
      "grad_norm": 474.76333707639907,
      "learning_rate": 3.266993139010438e-07,
      "logits": -0.9617911577224731,
      "logps": -74.96733093261719,
      "loss": 2.1682,
      "objective": 1.6541905403137207,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.574999988079071,
      "ranking_simple": 0.6708333492279053,
      "regularize": 1.6541905403137207,
      "step": 300
    },
    {
      "epoch": 1.7005196032120926,
      "eval_dpo_loss": 15.200382232666016,
      "eval_logits": -1.0035560131072998,
      "eval_logps": -80.33287811279297,
      "eval_loss": 15.382392883300781,
      "eval_objective": 15.200382232666016,
      "eval_ranking_idealized": 0.5092975497245789,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5113636255264282,
      "eval_regularize": 15.200382232666016,
      "eval_runtime": 264.8415,
      "eval_samples_per_second": 21.862,
      "eval_steps_per_second": 0.914,
      "step": 300
    },
    {
      "dpo_loss": 2.5656979084014893,
      "epoch": 1.7288615965989607,
      "grad_norm": 503.0202503507287,
      "learning_rate": 2.6798785497556184e-07,
      "logits": -0.933074414730072,
      "logps": -76.0098876953125,
      "loss": 2.728,
      "objective": 2.5656979084014893,
      "ranking_idealized": 0.5083333253860474,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.6166666746139526,
      "regularize": 2.5656979084014893,
      "step": 305
    },
    {
      "dpo_loss": 2.7561254501342773,
      "epoch": 1.7572035899858292,
      "grad_norm": 551.8103525795702,
      "learning_rate": 2.1479047161491351e-07,
      "logits": -0.9786698222160339,
      "logps": -73.98075103759766,
      "loss": 3.2082,
      "objective": 2.7561254501342773,
      "ranking_idealized": 0.5208333134651184,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.625,
      "regularize": 2.7561254501342773,
      "step": 310
    },
    {
      "dpo_loss": 2.8616750240325928,
      "epoch": 1.7855455833726972,
      "grad_norm": 481.44311587523885,
      "learning_rate": 1.6723858532249783e-07,
      "logits": -0.9176926016807556,
      "logps": -74.52711486816406,
      "loss": 2.6534,
      "objective": 2.8616750240325928,
      "ranking_idealized": 0.5083333253860474,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.6208333373069763,
      "regularize": 2.8616750240325928,
      "step": 315
    },
    {
      "dpo_loss": 2.875875949859619,
      "epoch": 1.8138875767595655,
      "grad_norm": 473.64902583279064,
      "learning_rate": 1.2544967068054332e-07,
      "logits": -0.968319833278656,
      "logps": -76.5433120727539,
      "loss": 2.4726,
      "objective": 2.875875949859619,
      "ranking_idealized": 0.5041666626930237,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.6083333492279053,
      "regularize": 2.875875949859619,
      "step": 320
    },
    {
      "dpo_loss": 2.392838716506958,
      "epoch": 1.8422295701464337,
      "grad_norm": 598.6261084207288,
      "learning_rate": 8.952696513496756e-08,
      "logits": -1.005333662033081,
      "logps": -77.68478393554688,
      "loss": 2.8906,
      "objective": 2.392838716506958,
      "ranking_idealized": 0.5166666507720947,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.6541666388511658,
      "regularize": 2.392838716506958,
      "step": 325
    },
    {
      "dpo_loss": 2.538822889328003,
      "epoch": 1.8705715635333018,
      "grad_norm": 468.16090350396775,
      "learning_rate": 5.955921395237318e-08,
      "logits": -0.8062960505485535,
      "logps": -74.2054443359375,
      "loss": 2.6483,
      "objective": 2.538822889328003,
      "ranking_idealized": 0.4749999940395355,
      "ranking_idealized_expo": 0.47083333134651184,
      "ranking_simple": 0.5791666507720947,
      "regularize": 2.538822889328003,
      "step": 330
    },
    {
      "dpo_loss": 2.293182849884033,
      "epoch": 1.89891355692017,
      "grad_norm": 457.5310635009536,
      "learning_rate": 3.5620450979259034e-08,
      "logits": -0.9471557140350342,
      "logps": -74.38040161132812,
      "loss": 2.5969,
      "objective": 2.293182849884033,
      "ranking_idealized": 0.4958333373069763,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.6333333253860474,
      "regularize": 2.293182849884033,
      "step": 335
    },
    {
      "dpo_loss": 2.192481279373169,
      "epoch": 1.9272555503070383,
      "grad_norm": 536.3353914926571,
      "learning_rate": 1.7769815745066476e-08,
      "logits": -0.9198692440986633,
      "logps": -76.65982055664062,
      "loss": 2.8401,
      "objective": 2.192481279373169,
      "ranking_idealized": 0.5416666865348816,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.6625000238418579,
      "regularize": 2.192481279373169,
      "step": 340
    },
    {
      "dpo_loss": 1.9803303480148315,
      "epoch": 1.9555975436939064,
      "grad_norm": 445.05103819649605,
      "learning_rate": 6.051407360895822e-09,
      "logits": -0.9369055032730103,
      "logps": -75.275634765625,
      "loss": 2.7774,
      "objective": 1.9803303480148315,
      "ranking_idealized": 0.5333333611488342,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.6333333253860474,
      "regularize": 1.9803303480148315,
      "step": 345
    },
    {
      "dpo_loss": 2.6546401977539062,
      "epoch": 1.9839395370807746,
      "grad_norm": 571.7789309975336,
      "learning_rate": 4.941755748361088e-10,
      "logits": -0.8779342770576477,
      "logps": -76.04293060302734,
      "loss": 2.703,
      "objective": 2.6546401977539062,
      "ranking_idealized": 0.48750001192092896,
      "ranking_idealized_expo": 0.48750001192092896,
      "ranking_simple": 0.5874999761581421,
      "regularize": 2.6546401977539062,
      "step": 350
    },
    {
      "epoch": 1.9839395370807746,
      "eval_dpo_loss": 15.144511222839355,
      "eval_logits": -1.0045602321624756,
      "eval_logps": -80.39806365966797,
      "eval_loss": 15.256567001342773,
      "eval_objective": 15.144511222839355,
      "eval_ranking_idealized": 0.5092975497245789,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5134297609329224,
      "eval_regularize": 15.144511222839355,
      "eval_runtime": 259.6567,
      "eval_samples_per_second": 22.299,
      "eval_steps_per_second": 0.932,
      "step": 350
    },
    {
      "epoch": 1.995276334435522,
      "step": 352,
      "total_flos": 0.0,
      "train_loss": 5.737799351865595,
      "train_runtime": 14206.9508,
      "train_samples_per_second": 7.152,
      "train_steps_per_second": 0.025
    }
  ],
  "logging_steps": 5,
  "max_steps": 352,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}