{
  "best_metric": -0.9551481008529663,
  "best_model_checkpoint": "./qwen2.5-0.5b/qwen2.5-0.5b-expo-DPO-ES-TRY/checkpoint-36",
  "epoch": 1.83656117146906,
  "eval_steps": 36,
  "global_step": 324,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "dpo_loss": 0.6931471824645996,
      "dpo_wo_beta": -0.6931471824645996,
      "epoch": 0.005668398677373642,
      "grad_norm": 13.433596947813236,
      "learning_rate": 1.3888888888888887e-08,
      "logits": -1.3874311447143555,
      "logps": -88.43561553955078,
      "loss": 0.6931,
      "objective": 0.6931471824645996,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.5208333134651184,
      "regularize": 0.6931471824645996,
      "step": 1
    },
    {
      "dpo_loss": 0.6931712031364441,
      "dpo_wo_beta": -0.6935088634490967,
      "epoch": 0.02834199338686821,
      "grad_norm": 13.470599581187907,
      "learning_rate": 6.944444444444444e-08,
      "logits": -1.408589482307434,
      "logps": -84.63459777832031,
      "loss": 0.6931,
      "objective": 0.6931712031364441,
      "ranking_idealized": 0.6197916865348816,
      "ranking_idealized_expo": 0.546875,
      "ranking_simple": 0.546875,
      "regularize": 0.6931712031364441,
      "step": 5
    },
    {
      "dpo_loss": 0.6912738680839539,
      "dpo_wo_beta": -0.6787264943122864,
      "epoch": 0.05668398677373642,
      "grad_norm": 12.891994338762009,
      "learning_rate": 1.3888888888888888e-07,
      "logits": -1.4766913652420044,
      "logps": -83.30558776855469,
      "loss": 0.6922,
      "objective": 0.6912738680839539,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.5083333253860474,
      "regularize": 0.6912738680839539,
      "step": 10
    },
    {
      "dpo_loss": 0.6917670369148254,
      "dpo_wo_beta": -0.6925874352455139,
      "epoch": 0.08502598016060463,
      "grad_norm": 12.616091674274701,
      "learning_rate": 2.0833333333333333e-07,
      "logits": -1.4627829790115356,
      "logps": -83.51612854003906,
      "loss": 0.691,
      "objective": 0.6917670369148254,
      "ranking_idealized": 0.5916666388511658,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.512499988079071,
      "regularize": 0.6917670369148254,
      "step": 15
    },
    {
      "dpo_loss": 0.6909351348876953,
      "dpo_wo_beta": -0.7156455516815186,
      "epoch": 0.11336797354747284,
      "grad_norm": 13.131384103632906,
      "learning_rate": 2.7777777777777776e-07,
      "logits": -1.4652808904647827,
      "logps": -83.81787109375,
      "loss": 0.6882,
      "objective": 0.6909351348876953,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.5375000238418579,
      "ranking_simple": 0.5375000238418579,
      "regularize": 0.6909351348876953,
      "step": 20
    },
    {
      "dpo_loss": 0.6833630800247192,
      "dpo_wo_beta": -0.6689243316650391,
      "epoch": 0.14170996693434104,
      "grad_norm": 12.965314552644642,
      "learning_rate": 3.472222222222222e-07,
      "logits": -1.5261986255645752,
      "logps": -83.36495971679688,
      "loss": 0.6842,
      "objective": 0.6833630800247192,
      "ranking_idealized": 0.5958333611488342,
      "ranking_idealized_expo": 0.4958333373069763,
      "ranking_simple": 0.5,
      "regularize": 0.6833630800247192,
      "step": 25
    },
    {
      "dpo_loss": 0.6835373044013977,
      "dpo_wo_beta": -0.7437445521354675,
      "epoch": 0.17005196032120926,
      "grad_norm": 13.693441774251875,
      "learning_rate": 4.1666666666666667e-07,
      "logits": -1.512542724609375,
      "logps": -83.833984375,
      "loss": 0.6812,
      "objective": 0.6835373044013977,
      "ranking_idealized": 0.6041666865348816,
      "ranking_idealized_expo": 0.5291666388511658,
      "ranking_simple": 0.5333333611488342,
      "regularize": 0.6835373044013977,
      "step": 30
    },
    {
      "dpo_loss": 0.6732779145240784,
      "dpo_wo_beta": -0.7419623136520386,
      "epoch": 0.19839395370807747,
      "grad_norm": 13.409062026931796,
      "learning_rate": 4.861111111111111e-07,
      "logits": -1.4930462837219238,
      "logps": -82.56928253173828,
      "loss": 0.672,
      "objective": 0.6732779145240784,
      "ranking_idealized": 0.612500011920929,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.512499988079071,
      "regularize": 0.6732779145240784,
      "step": 35
    },
    {
      "epoch": 0.20406235238545112,
      "eval_dpo_loss": 0.6925506591796875,
      "eval_dpo_wo_beta": -0.9551481008529663,
      "eval_logits": -1.5338705778121948,
      "eval_logps": -91.41157531738281,
      "eval_loss": 0.6866393685340881,
      "eval_objective": 0.6925506591796875,
      "eval_ranking_idealized": 0.5888429880142212,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5051652789115906,
      "eval_regularize": 0.6925506591796875,
      "eval_runtime": 211.8538,
      "eval_samples_per_second": 27.33,
      "eval_steps_per_second": 1.142,
      "step": 36
    },
    {
      "dpo_loss": 0.6632875800132751,
      "dpo_wo_beta": -0.7245739102363586,
      "epoch": 0.22673594709494568,
      "grad_norm": 15.824803800139307,
      "learning_rate": 4.998023493068254e-07,
      "logits": -1.4679725170135498,
      "logps": -83.45999908447266,
      "loss": 0.6728,
      "objective": 0.6632875800132751,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.5208333134651184,
      "regularize": 0.6632875800132751,
      "step": 40
    },
    {
      "dpo_loss": 0.6583938002586365,
      "dpo_wo_beta": -0.7636463642120361,
      "epoch": 0.25507794048181387,
      "grad_norm": 13.750505573599494,
      "learning_rate": 4.989999289644991e-07,
      "logits": -1.594675898551941,
      "logps": -84.99964141845703,
      "loss": 0.6655,
      "objective": 0.6583938002586365,
      "ranking_idealized": 0.6208333373069763,
      "ranking_idealized_expo": 0.5375000238418579,
      "ranking_simple": 0.5458333492279053,
      "regularize": 0.6583938002586365,
      "step": 45
    },
    {
      "dpo_loss": 0.6567264795303345,
      "dpo_wo_beta": -0.863508403301239,
      "epoch": 0.2834199338686821,
      "grad_norm": 13.413740152325811,
      "learning_rate": 4.975823666181255e-07,
      "logits": -1.5660439729690552,
      "logps": -82.82308959960938,
      "loss": 0.6644,
      "objective": 0.6567264795303345,
      "ranking_idealized": 0.5416666865348816,
      "ranking_idealized_expo": 0.42500001192092896,
      "ranking_simple": 0.4749999940395355,
      "regularize": 0.6567264795303345,
      "step": 50
    },
    {
      "dpo_loss": 0.6495455503463745,
      "dpo_wo_beta": -0.7752765417098999,
      "epoch": 0.3117619272555503,
      "grad_norm": 14.577952034074448,
      "learning_rate": 4.955531642853403e-07,
      "logits": -1.5825331211090088,
      "logps": -84.16141510009766,
      "loss": 0.653,
      "objective": 0.6495455503463745,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.6495455503463745,
      "step": 55
    },
    {
      "dpo_loss": 0.6498515605926514,
      "dpo_wo_beta": -0.736571729183197,
      "epoch": 0.3401039206424185,
      "grad_norm": 13.788809377020446,
      "learning_rate": 4.929173350101024e-07,
      "logits": -1.5849006175994873,
      "logps": -83.91230773925781,
      "loss": 0.6514,
      "objective": 0.6498515605926514,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.550000011920929,
      "regularize": 0.6498515605926514,
      "step": 60
    },
    {
      "dpo_loss": 0.6450857520103455,
      "dpo_wo_beta": -0.8086111545562744,
      "epoch": 0.3684459140292867,
      "grad_norm": 13.799818527893999,
      "learning_rate": 4.896813904782162e-07,
      "logits": -1.5256472826004028,
      "logps": -84.62181091308594,
      "loss": 0.6495,
      "objective": 0.6450857520103455,
      "ranking_idealized": 0.5708333253860474,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.5291666388511658,
      "regularize": 0.6450857520103455,
      "step": 65
    },
    {
      "dpo_loss": 0.644597053527832,
      "dpo_wo_beta": -0.891067385673523,
      "epoch": 0.39678790741615494,
      "grad_norm": 13.789576838493705,
      "learning_rate": 4.858533249305336e-07,
      "logits": -1.6211374998092651,
      "logps": -86.41259002685547,
      "loss": 0.6533,
      "objective": 0.644597053527832,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5208333134651184,
      "regularize": 0.644597053527832,
      "step": 70
    },
    {
      "epoch": 0.40812470477090224,
      "eval_dpo_loss": 0.6884610056877136,
      "eval_dpo_wo_beta": -1.1473097801208496,
      "eval_logits": -1.6310690641403198,
      "eval_logps": -92.47576904296875,
      "eval_loss": 0.6768966913223267,
      "eval_objective": 0.6884610056877136,
      "eval_ranking_idealized": 0.5888429880142212,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5175619721412659,
      "eval_regularize": 0.6884610056877136,
      "eval_runtime": 210.9564,
      "eval_samples_per_second": 27.446,
      "eval_steps_per_second": 1.147,
      "step": 72
    },
    {
      "dpo_loss": 0.6458336114883423,
      "dpo_wo_beta": -0.847459614276886,
      "epoch": 0.42512990080302315,
      "grad_norm": 13.350435766130868,
      "learning_rate": 4.814425954135785e-07,
      "logits": -1.6625837087631226,
      "logps": -86.4749755859375,
      "loss": 0.636,
      "objective": 0.6458336114883423,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.5458333492279053,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.6458336114883423,
      "step": 75
    },
    {
      "dpo_loss": 0.6464484930038452,
      "dpo_wo_beta": -0.9348036050796509,
      "epoch": 0.45347189418989137,
      "grad_norm": 13.762513071445667,
      "learning_rate": 4.764600984163808e-07,
      "logits": -1.7147225141525269,
      "logps": -86.26012420654297,
      "loss": 0.6326,
      "objective": 0.6464484930038452,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5166666507720947,
      "ranking_simple": 0.5458333492279053,
      "regularize": 0.6464484930038452,
      "step": 80
    },
    {
      "dpo_loss": 0.6432295441627502,
      "dpo_wo_beta": -1.0359190702438354,
      "epoch": 0.4818138875767596,
      "grad_norm": 13.87750405485222,
      "learning_rate": 4.70918142951239e-07,
      "logits": -1.6870180368423462,
      "logps": -85.32923889160156,
      "loss": 0.6336,
      "objective": 0.6432295441627502,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.5458333492279053,
      "regularize": 0.6432295441627502,
      "step": 85
    },
    {
      "dpo_loss": 0.6311187744140625,
      "dpo_wo_beta": -0.9493275284767151,
      "epoch": 0.5101558809636277,
      "grad_norm": 14.792020412361724,
      "learning_rate": 4.6483042014491527e-07,
      "logits": -1.5992980003356934,
      "logps": -88.2070083618164,
      "loss": 0.6216,
      "objective": 0.6311187744140625,
      "ranking_idealized": 0.5458333492279053,
      "ranking_idealized_expo": 0.4749999940395355,
      "ranking_simple": 0.5041666626930237,
      "regularize": 0.6311187744140625,
      "step": 90
    },
    {
      "dpo_loss": 0.595143735408783,
      "dpo_wo_beta": -0.8239250183105469,
      "epoch": 0.538497874350496,
      "grad_norm": 16.242182565245834,
      "learning_rate": 4.5821196941538334e-07,
      "logits": -1.6257656812667847,
      "logps": -87.32308197021484,
      "loss": 0.6157,
      "objective": 0.595143735408783,
      "ranking_idealized": 0.6291666626930237,
      "ranking_idealized_expo": 0.5458333492279053,
      "ranking_simple": 0.6041666865348816,
      "regularize": 0.595143735408783,
      "step": 95
    },
    {
      "dpo_loss": 0.6018479466438293,
      "dpo_wo_beta": -1.0667775869369507,
      "epoch": 0.5668398677373642,
      "grad_norm": 16.965716669559736,
      "learning_rate": 4.510791413176912e-07,
      "logits": -1.6872950792312622,
      "logps": -88.73619842529297,
      "loss": 0.623,
      "objective": 0.6018479466438293,
      "ranking_idealized": 0.612500011920929,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.5625,
      "regularize": 0.6018479466438293,
      "step": 100
    },
    {
      "dpo_loss": 0.5836849808692932,
      "dpo_wo_beta": -0.7914173007011414,
      "epoch": 0.5951818611242324,
      "grad_norm": 14.695711016472089,
      "learning_rate": 4.434495571507234e-07,
      "logits": -1.606242299079895,
      "logps": -87.9868392944336,
      "loss": 0.604,
      "objective": 0.5836849808692932,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.5836849808692932,
      "step": 105
    },
    {
      "epoch": 0.6121870571563534,
      "eval_dpo_loss": 0.6853068470954895,
      "eval_dpo_wo_beta": -1.3960636854171753,
      "eval_logits": -1.769094467163086,
      "eval_logps": -94.05838012695312,
      "eval_loss": 0.677278995513916,
      "eval_objective": 0.6853068470954895,
      "eval_ranking_idealized": 0.5888429880142212,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5185950398445129,
      "eval_regularize": 0.6853068470954895,
      "eval_runtime": 210.425,
      "eval_samples_per_second": 27.516,
      "eval_steps_per_second": 1.15,
      "step": 108
    },
    {
      "dpo_loss": 0.6007120013237,
      "dpo_wo_beta": -0.8422455191612244,
      "epoch": 0.6235238545111006,
      "grad_norm": 15.750957422421227,
      "learning_rate": 4.353420654246546e-07,
      "logits": -1.7943415641784668,
      "logps": -86.3438491821289,
      "loss": 0.6054,
      "objective": 0.6007120013237,
      "ranking_idealized": 0.5708333253860474,
      "ranking_idealized_expo": 0.4541666805744171,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.6007120013237,
      "step": 110
    },
    {
      "dpo_loss": 0.6109593510627747,
      "dpo_wo_beta": -1.0094329118728638,
      "epoch": 0.6518658478979689,
      "grad_norm": 16.233880140590145,
      "learning_rate": 4.2677669529663686e-07,
      "logits": -1.7633204460144043,
      "logps": -86.51942443847656,
      "loss": 0.6046,
      "objective": 0.6109593510627747,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.6109593510627747,
      "step": 115
    },
    {
      "dpo_loss": 0.5903018712997437,
      "dpo_wo_beta": -0.9045132994651794,
      "epoch": 0.680207841284837,
      "grad_norm": 17.133679937634657,
      "learning_rate": 4.177746070897592e-07,
      "logits": -1.7159924507141113,
      "logps": -87.81356811523438,
      "loss": 0.5999,
      "objective": 0.5903018712997437,
      "ranking_idealized": 0.5874999761581421,
      "ranking_idealized_expo": 0.5291666388511658,
      "ranking_simple": 0.5791666507720947,
      "regularize": 0.5903018712997437,
      "step": 120
    },
    {
      "dpo_loss": 0.6066262722015381,
      "dpo_wo_beta": -1.2728170156478882,
      "epoch": 0.7085498346717053,
      "grad_norm": 16.15790530794251,
      "learning_rate": 4.083580400175153e-07,
      "logits": -1.698543667793274,
      "logps": -88.70279693603516,
      "loss": 0.6139,
      "objective": 0.6066262722015381,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.550000011920929,
      "ranking_simple": 0.5874999761581421,
      "regularize": 0.6066262722015381,
      "step": 125
    },
    {
      "dpo_loss": 0.6065606474876404,
      "dpo_wo_beta": -1.1573857069015503,
      "epoch": 0.7368918280585735,
      "grad_norm": 16.556049529132082,
      "learning_rate": 3.9855025724292763e-07,
      "logits": -1.8358159065246582,
      "logps": -87.11023712158203,
      "loss": 0.6035,
      "objective": 0.6065606474876404,
      "ranking_idealized": 0.5416666865348816,
      "ranking_idealized_expo": 0.44999998807907104,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.6065606474876404,
      "step": 130
    },
    {
      "dpo_loss": 0.6158966422080994,
      "dpo_wo_beta": -1.1439249515533447,
      "epoch": 0.7652338214454416,
      "grad_norm": 14.197353075079507,
      "learning_rate": 3.883754884080539e-07,
      "logits": -1.7928814888000488,
      "logps": -89.14883422851562,
      "loss": 0.594,
      "objective": 0.6158966422080994,
      "ranking_idealized": 0.5874999761581421,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.5458333492279053,
      "regularize": 0.6158966422080994,
      "step": 135
    },
    {
      "dpo_loss": 0.61494380235672,
      "dpo_wo_beta": -1.0414694547653198,
      "epoch": 0.7935758148323099,
      "grad_norm": 15.670852477907466,
      "learning_rate": 3.7785886977585555e-07,
      "logits": -1.711374282836914,
      "logps": -88.56974792480469,
      "loss": 0.5911,
      "objective": 0.61494380235672,
      "ranking_idealized": 0.5791666507720947,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.5416666865348816,
      "regularize": 0.61494380235672,
      "step": 140
    },
    {
      "epoch": 0.8162494095418045,
      "eval_dpo_loss": 0.6885914206504822,
      "eval_dpo_wo_beta": -1.5153368711471558,
      "eval_logits": -1.816495418548584,
      "eval_logps": -95.75775146484375,
      "eval_loss": 0.6774819493293762,
      "eval_objective": 0.6885914206504822,
      "eval_ranking_idealized": 0.5888429880142212,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5185950398445129,
      "eval_regularize": 0.6885914206504822,
      "eval_runtime": 210.8957,
      "eval_samples_per_second": 27.454,
      "eval_steps_per_second": 1.147,
      "step": 144
    },
    {
      "dpo_loss": 0.5598483085632324,
      "dpo_wo_beta": -0.7996322512626648,
      "epoch": 0.821917808219178,
      "grad_norm": 14.122746042521586,
      "learning_rate": 3.670263821323034e-07,
      "logits": -1.738029956817627,
      "logps": -87.6530532836914,
      "loss": 0.5822,
      "objective": 0.5598483085632324,
      "ranking_idealized": 0.5416666865348816,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5208333134651184,
      "regularize": 0.5598483085632324,
      "step": 145
    },
    {
      "dpo_loss": 0.5972100496292114,
      "dpo_wo_beta": -0.8620439171791077,
      "epoch": 0.8502598016060463,
      "grad_norm": 14.65905297499526,
      "learning_rate": 3.5590478660213206e-07,
      "logits": -1.7470637559890747,
      "logps": -85.3743667602539,
      "loss": 0.5857,
      "objective": 0.5972100496292114,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.5791666507720947,
      "regularize": 0.5972100496292114,
      "step": 150
    },
    {
      "dpo_loss": 0.5853369235992432,
      "dpo_wo_beta": -1.0713670253753662,
      "epoch": 0.8786017949929145,
      "grad_norm": 14.449237235852513,
      "learning_rate": 3.4452155853680454e-07,
      "logits": -1.7700653076171875,
      "logps": -85.30365753173828,
      "loss": 0.5778,
      "objective": 0.5853369235992432,
      "ranking_idealized": 0.612500011920929,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.6000000238418579,
      "regularize": 0.5853369235992432,
      "step": 155
    },
    {
      "dpo_loss": 0.579626202583313,
      "dpo_wo_beta": -1.0907135009765625,
      "epoch": 0.9069437883797827,
      "grad_norm": 16.128315234232378,
      "learning_rate": 3.3290481963801696e-07,
      "logits": -1.7416605949401855,
      "logps": -87.19976043701172,
      "loss": 0.5926,
      "objective": 0.579626202583313,
      "ranking_idealized": 0.5708333253860474,
      "ranking_idealized_expo": 0.49166667461395264,
      "ranking_simple": 0.5625,
      "regularize": 0.579626202583313,
      "step": 160
    },
    {
      "dpo_loss": 0.561370313167572,
      "dpo_wo_beta": -1.004237174987793,
      "epoch": 0.9352857817666509,
      "grad_norm": 19.041287196836052,
      "learning_rate": 3.2108326848442503e-07,
      "logits": -1.71802818775177,
      "logps": -89.13147735595703,
      "loss": 0.574,
      "objective": 0.561370313167572,
      "ranking_idealized": 0.6041666865348816,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.5916666388511658,
      "regularize": 0.561370313167572,
      "step": 165
    },
    {
      "dpo_loss": 0.5719048976898193,
      "dpo_wo_beta": -0.9809625148773193,
      "epoch": 0.9636277751535192,
      "grad_norm": 18.87203490676025,
      "learning_rate": 3.0908610963322626e-07,
      "logits": -1.7574745416641235,
      "logps": -92.2130126953125,
      "loss": 0.5857,
      "objective": 0.5719048976898193,
      "ranking_idealized": 0.5791666507720947,
      "ranking_idealized_expo": 0.512499988079071,
      "ranking_simple": 0.5916666388511658,
      "regularize": 0.5719048976898193,
      "step": 170
    },
    {
      "dpo_loss": 0.5651528239250183,
      "dpo_wo_beta": -0.7982299327850342,
      "epoch": 0.9919697685403873,
      "grad_norm": 15.136769327688043,
      "learning_rate": 2.969429814717456e-07,
      "logits": -1.6923960447311401,
      "logps": -93.54885864257812,
      "loss": 0.5728,
      "objective": 0.5651528239250183,
      "ranking_idealized": 0.6541666388511658,
      "ranking_idealized_expo": 0.574999988079071,
      "ranking_simple": 0.6166666746139526,
      "regularize": 0.5651528239250183,
      "step": 175
    },
    {
      "dpo_loss": 0.541616678237915,
      "dpo_wo_beta": -0.9241905212402344,
      "epoch": 1.0203117619272555,
      "grad_norm": 14.869186889382515,
      "learning_rate": 2.846838829972671e-07,
      "logits": -1.8582412004470825,
      "logps": -90.80587005615234,
      "loss": 0.5482,
      "objective": 0.541616678237915,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5416666865348816,
      "ranking_simple": 0.612500011920929,
      "regularize": 0.541616678237915,
      "step": 180
    },
    {
      "epoch": 1.0203117619272555,
      "eval_dpo_loss": 0.6998419761657715,
      "eval_dpo_wo_beta": -1.868453025817871,
      "eval_logits": -1.8457019329071045,
      "eval_logps": -98.13460540771484,
      "eval_loss": 0.6859121918678284,
      "eval_objective": 0.6998419761657715,
      "eval_ranking_idealized": 0.5888429880142212,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5237603187561035,
      "eval_regularize": 0.6998419761657715,
      "eval_runtime": 211.2462,
      "eval_samples_per_second": 27.409,
      "eval_steps_per_second": 1.146,
      "step": 180
    },
    {
      "dpo_loss": 0.5197845697402954,
      "dpo_wo_beta": -0.6562149524688721,
      "epoch": 1.0486537553141237,
      "grad_norm": 15.717261963609046,
      "learning_rate": 2.7233909970599426e-07,
      "logits": -1.735451579093933,
      "logps": -90.08293914794922,
      "loss": 0.5338,
      "objective": 0.5197845697402954,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.49166667461395264,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.5197845697402954,
      "step": 185
    },
    {
      "dpo_loss": 0.5209748148918152,
      "dpo_wo_beta": -0.7339054942131042,
      "epoch": 1.076995748700992,
      "grad_norm": 15.852085328699776,
      "learning_rate": 2.5993912877423147e-07,
      "logits": -1.7829344272613525,
      "logps": -91.87959289550781,
      "loss": 0.5286,
      "objective": 0.5209748148918152,
      "ranking_idealized": 0.625,
      "ranking_idealized_expo": 0.5541666746139526,
      "ranking_simple": 0.6166666746139526,
      "regularize": 0.5209748148918152,
      "step": 190
    },
    {
      "dpo_loss": 0.5032804608345032,
      "dpo_wo_beta": -0.56788170337677,
      "epoch": 1.10533774208786,
      "grad_norm": 18.497871574750942,
      "learning_rate": 2.475146037166176e-07,
      "logits": -1.726409912109375,
      "logps": -91.2437744140625,
      "loss": 0.5213,
      "objective": 0.5032804608345032,
      "ranking_idealized": 0.6333333253860474,
      "ranking_idealized_expo": 0.5666666626930237,
      "ranking_simple": 0.6416666507720947,
      "regularize": 0.5032804608345032,
      "step": 195
    },
    {
      "dpo_loss": 0.5304009914398193,
      "dpo_wo_beta": -0.8262697458267212,
      "epoch": 1.1336797354747283,
      "grad_norm": 17.200667685474183,
      "learning_rate": 2.3509621870754504e-07,
      "logits": -1.8239718675613403,
      "logps": -90.6737289428711,
      "loss": 0.5178,
      "objective": 0.5304009914398193,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.574999988079071,
      "regularize": 0.5304009914398193,
      "step": 200
    },
    {
      "dpo_loss": 0.5203301310539246,
      "dpo_wo_beta": -0.938244640827179,
      "epoch": 1.1620217288615966,
      "grad_norm": 17.29293963157863,
      "learning_rate": 2.2271465275271983e-07,
      "logits": -1.799241304397583,
      "logps": -92.73731994628906,
      "loss": 0.5275,
      "objective": 0.5203301310539246,
      "ranking_idealized": 0.5708333253860474,
      "ranking_idealized_expo": 0.49166667461395264,
      "ranking_simple": 0.574999988079071,
      "regularize": 0.5203301310539246,
      "step": 205
    },
    {
      "dpo_loss": 0.5207778811454773,
      "dpo_wo_beta": -0.8990126252174377,
      "epoch": 1.1903637222484649,
      "grad_norm": 16.801298356601897,
      "learning_rate": 2.1040049389819624e-07,
      "logits": -1.8129061460494995,
      "logps": -92.11985778808594,
      "loss": 0.5172,
      "objective": 0.5207778811454773,
      "ranking_idealized": 0.5833333134651184,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.574999988079071,
      "regularize": 0.5207778811454773,
      "step": 210
    },
    {
      "dpo_loss": 0.5303699374198914,
      "dpo_wo_beta": -0.8046938180923462,
      "epoch": 1.2187057156353331,
      "grad_norm": 15.406676908767183,
      "learning_rate": 1.9818416366412275e-07,
      "logits": -1.757646918296814,
      "logps": -91.60792541503906,
      "loss": 0.5171,
      "objective": 0.5303699374198914,
      "ranking_idealized": 0.5583333373069763,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.5791666507720947,
      "regularize": 0.5303699374198914,
      "step": 215
    },
    {
      "epoch": 1.2243741143127067,
      "eval_dpo_loss": 0.7003704905509949,
      "eval_dpo_wo_beta": -2.0594191551208496,
      "eval_logits": -1.885943055152893,
      "eval_logps": -99.3935775756836,
      "eval_loss": 0.6891233325004578,
      "eval_objective": 0.7003704905509949,
      "eval_ranking_idealized": 0.5888429880142212,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5247933864593506,
      "eval_regularize": 0.7003704905509949,
      "eval_runtime": 210.3607,
      "eval_samples_per_second": 27.524,
      "eval_steps_per_second": 1.15,
      "step": 216
    },
    {
      "dpo_loss": 0.5014426112174988,
      "dpo_wo_beta": -0.6835492849349976,
      "epoch": 1.2470477090222012,
      "grad_norm": 17.734567216291715,
      "learning_rate": 1.8609584188988133e-07,
      "logits": -1.7417553663253784,
      "logps": -91.04163360595703,
      "loss": 0.5162,
      "objective": 0.5014426112174988,
      "ranking_idealized": 0.6541666388511658,
      "ranking_idealized_expo": 0.6041666865348816,
      "ranking_simple": 0.6583333611488342,
      "regularize": 0.5014426112174988,
      "step": 220
    },
    {
      "dpo_loss": 0.5014557242393494,
      "dpo_wo_beta": -0.8168562650680542,
      "epoch": 1.2753897024090695,
      "grad_norm": 16.95873203496136,
      "learning_rate": 1.741653921762879e-07,
      "logits": -1.8029441833496094,
      "logps": -92.49475860595703,
      "loss": 0.5135,
      "objective": 0.5014557242393494,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5333333611488342,
      "ranking_simple": 0.6291666626930237,
      "regularize": 0.5014557242393494,
      "step": 225
    },
    {
      "dpo_loss": 0.5178931355476379,
      "dpo_wo_beta": -0.9624936580657959,
      "epoch": 1.3037316957959377,
      "grad_norm": 16.74959646013195,
      "learning_rate": 1.624222881090439e-07,
      "logits": -1.7226511240005493,
      "logps": -91.97869110107422,
      "loss": 0.5168,
      "objective": 0.5178931355476379,
      "ranking_idealized": 0.5791666507720947,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.5178931355476379,
      "step": 230
    },
    {
      "dpo_loss": 0.504324197769165,
      "dpo_wo_beta": -0.7607553601264954,
      "epoch": 1.3320736891828058,
      "grad_norm": 16.27709469673636,
      "learning_rate": 1.5089554044570149e-07,
      "logits": -1.8560264110565186,
      "logps": -91.13007354736328,
      "loss": 0.5205,
      "objective": 0.504324197769165,
      "ranking_idealized": 0.6041666865348816,
      "ranking_idealized_expo": 0.4833333194255829,
      "ranking_simple": 0.6291666626930237,
      "regularize": 0.504324197769165,
      "step": 235
    },
    {
      "dpo_loss": 0.5133432745933533,
      "dpo_wo_beta": -0.8683770298957825,
      "epoch": 1.360415682569674,
      "grad_norm": 17.129996913619507,
      "learning_rate": 1.3961362544602212e-07,
      "logits": -1.8072270154953003,
      "logps": -92.90058898925781,
      "loss": 0.5032,
      "objective": 0.5133432745933533,
      "ranking_idealized": 0.5541666746139526,
      "ranking_idealized_expo": 0.4791666567325592,
      "ranking_simple": 0.574999988079071,
      "regularize": 0.5133432745933533,
      "step": 240
    },
    {
      "dpo_loss": 0.5124314427375793,
      "dpo_wo_beta": -0.7547214031219482,
      "epoch": 1.3887576759565423,
      "grad_norm": 16.459721846313148,
      "learning_rate": 1.2860441452278574e-07,
      "logits": -1.829980731010437,
      "logps": -92.37997436523438,
      "loss": 0.5115,
      "objective": 0.5124314427375793,
      "ranking_idealized": 0.5916666388511658,
      "ranking_idealized_expo": 0.5208333134651184,
      "ranking_simple": 0.6083333492279053,
      "regularize": 0.5124314427375793,
      "step": 245
    },
    {
      "dpo_loss": 0.5035055875778198,
      "dpo_wo_beta": -0.7095458507537842,
      "epoch": 1.4170996693434104,
      "grad_norm": 15.886363535231352,
      "learning_rate": 1.1789510538684522e-07,
      "logits": -1.8016122579574585,
      "logps": -92.37277221679688,
      "loss": 0.5093,
      "objective": 0.5035055875778198,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.47083333134651184,
      "ranking_simple": 0.5791666507720947,
      "regularize": 0.5035055875778198,
      "step": 250
    },
    {
      "epoch": 1.4284364666981577,
      "eval_dpo_loss": 0.7118967771530151,
      "eval_dpo_wo_beta": -2.3164827823638916,
      "eval_logits": -1.896795630455017,
      "eval_logps": -102.08467102050781,
      "eval_loss": 0.6999199986457825,
      "eval_objective": 0.7118967771530151,
      "eval_ranking_idealized": 0.5888429880142212,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5237603187561035,
      "eval_regularize": 0.7118967771530151,
      "eval_runtime": 210.1951,
      "eval_samples_per_second": 27.546,
      "eval_steps_per_second": 1.151,
      "step": 252
    },
    {
      "dpo_loss": 0.5017536282539368,
      "dpo_wo_beta": -0.8617421388626099,
      "epoch": 1.4454416627302786,
      "grad_norm": 18.01058733365972,
      "learning_rate": 1.0751215485652643e-07,
      "logits": -1.7449065446853638,
      "logps": -94.58292388916016,
      "loss": 0.5028,
      "objective": 0.5017536282539368,
      "ranking_idealized": 0.6166666746139526,
      "ranking_idealized_expo": 0.5458333492279053,
      "ranking_simple": 0.6291666626930237,
      "regularize": 0.5017536282539368,
      "step": 255
    },
    {
      "dpo_loss": 0.49517151713371277,
      "dpo_wo_beta": -0.7506776452064514,
      "epoch": 1.473783656117147,
      "grad_norm": 17.81035746121776,
      "learning_rate": 9.748121349736891e-08,
      "logits": -1.817198634147644,
      "logps": -95.67717742919922,
      "loss": 0.5086,
      "objective": 0.49517151713371277,
      "ranking_idealized": 0.5666666626930237,
      "ranking_idealized_expo": 0.48750001192092896,
      "ranking_simple": 0.5791666507720947,
      "regularize": 0.49517151713371277,
      "step": 260
    },
    {
      "dpo_loss": 0.5204591751098633,
      "dpo_wo_beta": -1.0298762321472168,
      "epoch": 1.5021256495040152,
      "grad_norm": 16.358665885069936,
      "learning_rate": 8.78270622536716e-08,
      "logits": -1.7978217601776123,
      "logps": -93.08860778808594,
      "loss": 0.5031,
      "objective": 0.5204591751098633,
      "ranking_idealized": 0.5791666507720947,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.5916666388511658,
      "regularize": 0.5204591751098633,
      "step": 265
    },
    {
      "dpo_loss": 0.5112512111663818,
      "dpo_wo_beta": -0.8745156526565552,
      "epoch": 1.5304676428908834,
      "grad_norm": 16.95418568294612,
      "learning_rate": 7.857355122839673e-08,
      "logits": -1.7797691822052002,
      "logps": -95.0974349975586,
      "loss": 0.5001,
      "objective": 0.5112512111663818,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.612500011920929,
      "regularize": 0.5112512111663818,
      "step": 270
    },
    {
      "dpo_loss": 0.4850163459777832,
      "dpo_wo_beta": -0.7579888105392456,
      "epoch": 1.5588096362777515,
      "grad_norm": 16.810103741459905,
      "learning_rate": 6.97435407626708e-08,
      "logits": -1.7828667163848877,
      "logps": -95.46377563476562,
      "loss": 0.5032,
      "objective": 0.4850163459777832,
      "ranking_idealized": 0.6208333373069763,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.6208333373069763,
      "regularize": 0.4850163459777832,
      "step": 275
    },
    {
      "dpo_loss": 0.500173032283783,
      "dpo_wo_beta": -0.9343259334564209,
      "epoch": 1.5871516296646198,
      "grad_norm": 19.77407532453596,
      "learning_rate": 6.135884496044244e-08,
      "logits": -1.7596685886383057,
      "logps": -94.32428741455078,
      "loss": 0.5082,
      "objective": 0.500173032283783,
      "ranking_idealized": 0.574999988079071,
      "ranking_idealized_expo": 0.5041666626930237,
      "ranking_simple": 0.5916666388511658,
      "regularize": 0.500173032283783,
      "step": 280
    },
    {
      "dpo_loss": 0.4930657148361206,
      "dpo_wo_beta": -0.8764402270317078,
      "epoch": 1.615493623051488,
      "grad_norm": 17.882115521129325,
      "learning_rate": 5.344017779781834e-08,
      "logits": -1.8836300373077393,
      "logps": -95.88809967041016,
      "loss": 0.4986,
      "objective": 0.4930657148361206,
      "ranking_idealized": 0.5874999761581421,
      "ranking_idealized_expo": 0.512499988079071,
      "ranking_simple": 0.6416666507720947,
      "regularize": 0.4930657148361206,
      "step": 285
    },
    {
      "epoch": 1.632498819083609,
      "eval_dpo_loss": 0.7089951038360596,
      "eval_dpo_wo_beta": -2.2741074562072754,
      "eval_logits": -1.9005335569381714,
      "eval_logps": -102.14405059814453,
      "eval_loss": 0.69806969165802,
      "eval_objective": 0.7089951038360596,
      "eval_ranking_idealized": 0.5888429880142212,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5278925895690918,
      "eval_regularize": 0.7089951038360596,
      "eval_runtime": 210.3642,
      "eval_samples_per_second": 27.524,
      "eval_steps_per_second": 1.15,
      "step": 288
    },
    {
      "dpo_loss": 0.48574885725975037,
      "dpo_wo_beta": -0.6394420266151428,
      "epoch": 1.643835616438356,
      "grad_norm": 18.69303873982025,
      "learning_rate": 4.600710195020982e-08,
      "logits": -1.8818236589431763,
      "logps": -93.7010726928711,
      "loss": 0.4998,
      "objective": 0.48574885725975037,
      "ranking_idealized": 0.612500011920929,
      "ranking_idealized_expo": 0.5625,
      "ranking_simple": 0.6666666865348816,
      "regularize": 0.48574885725975037,
      "step": 290
    },
    {
      "dpo_loss": 0.48394453525543213,
      "dpo_wo_beta": -0.8024871945381165,
      "epoch": 1.6721776098252243,
      "grad_norm": 16.956110275905026,
      "learning_rate": 3.9077980463711384e-08,
      "logits": -1.762637734413147,
      "logps": -92.77220153808594,
      "loss": 0.5044,
      "objective": 0.48394453525543213,
      "ranking_idealized": 0.5791666507720947,
      "ranking_idealized_expo": 0.4749999940395355,
      "ranking_simple": 0.6208333373069763,
      "regularize": 0.48394453525543213,
      "step": 295
    },
    {
      "dpo_loss": 0.49569427967071533,
      "dpo_wo_beta": -0.9217907190322876,
      "epoch": 1.7005196032120926,
      "grad_norm": 16.550791676698843,
      "learning_rate": 3.2669931390104374e-08,
      "logits": -1.8427069187164307,
      "logps": -94.1619873046875,
      "loss": 0.5001,
      "objective": 0.49569427967071533,
      "ranking_idealized": 0.6583333611488342,
      "ranking_idealized_expo": 0.574999988079071,
      "ranking_simple": 0.6499999761581421,
      "regularize": 0.49569427967071533,
      "step": 300
    },
    {
      "dpo_loss": 0.5322020053863525,
      "dpo_wo_beta": -1.2344766855239868,
      "epoch": 1.7288615965989607,
      "grad_norm": 18.336457660232934,
      "learning_rate": 2.679878549755618e-08,
      "logits": -1.769299864768982,
      "logps": -94.73688507080078,
      "loss": 0.5077,
      "objective": 0.5322020053863525,
      "ranking_idealized": 0.5625,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.6041666865348816,
      "regularize": 0.5322020053863525,
      "step": 305
    },
    {
      "dpo_loss": 0.5099295973777771,
      "dpo_wo_beta": -0.9262399077415466,
      "epoch": 1.7572035899858292,
      "grad_norm": 17.848858958468,
      "learning_rate": 2.147904716149135e-08,
      "logits": -1.8594757318496704,
      "logps": -91.70903015136719,
      "loss": 0.5042,
      "objective": 0.5099295973777771,
      "ranking_idealized": 0.6083333492279053,
      "ranking_idealized_expo": 0.5249999761581421,
      "ranking_simple": 0.6166666746139526,
      "regularize": 0.5099295973777771,
      "step": 310
    },
    {
      "dpo_loss": 0.5144280791282654,
      "dpo_wo_beta": -0.8178034424781799,
      "epoch": 1.7855455833726972,
      "grad_norm": 16.57904461321692,
      "learning_rate": 1.6723858532249778e-08,
      "logits": -1.753740906715393,
      "logps": -92.84646606445312,
      "loss": 0.5027,
      "objective": 0.5144280791282654,
      "ranking_idealized": 0.6041666865348816,
      "ranking_idealized_expo": 0.5083333253860474,
      "ranking_simple": 0.5874999761581421,
      "regularize": 0.5144280791282654,
      "step": 315
    },
    {
      "dpo_loss": 0.5112444162368774,
      "dpo_wo_beta": -0.9777346253395081,
      "epoch": 1.8138875767595655,
      "grad_norm": 18.774453505894844,
      "learning_rate": 1.254496706805433e-08,
      "logits": -1.8467546701431274,
      "logps": -95.57693481445312,
      "loss": 0.5055,
      "objective": 0.5112444162368774,
      "ranking_idealized": 0.6000000238418579,
      "ranking_idealized_expo": 0.5,
      "ranking_simple": 0.5958333611488342,
      "regularize": 0.5112444162368774,
      "step": 320
    },
    {
      "epoch": 1.83656117146906,
      "eval_dpo_loss": 0.7079721689224243,
      "eval_dpo_wo_beta": -2.264868974685669,
      "eval_logits": -1.8981069326400757,
      "eval_logps": -101.57964324951172,
      "eval_loss": 0.6964598298072815,
      "eval_objective": 0.7079721689224243,
      "eval_ranking_idealized": 0.5888429880142212,
      "eval_ranking_idealized_expo": 0.5092975497245789,
      "eval_ranking_simple": 0.5278925895690918,
      "eval_regularize": 0.7079721689224243,
      "eval_runtime": 210.0424,
      "eval_samples_per_second": 27.566,
      "eval_steps_per_second": 1.152,
      "step": 324
    },
    {
      "epoch": 1.83656117146906,
      "step": 324,
      "total_flos": 0.0,
      "train_loss": 0.29064991334338247,
      "train_runtime": 6418.0206,
      "train_samples_per_second": 15.831,
      "train_steps_per_second": 0.055
    }
  ],
  "logging_steps": 5,
  "max_steps": 352,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 36,
  "stateful_callbacks": {
    "EarlyStoppingCallback": {
      "args": {
        "early_stopping_patience": 3,
        "early_stopping_threshold": 0.0
      },
      "attributes": {
        "early_stopping_patience_counter": 0
      }
    },
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}