{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.882087502452423,
  "eval_steps": 50,
  "global_step": 1200,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.015695507161075144,
      "grad_norm": 0.04355761408805847,
      "learning_rate": 4.999451708687114e-06,
      "logits/chosen": 14.845781326293945,
      "logits/rejected": 14.576438903808594,
      "logps/chosen": -0.31864267587661743,
      "logps/rejected": -0.24545662105083466,
      "loss": 1.0492,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.47796401381492615,
      "rewards/margins": -0.10977902263402939,
      "rewards/rejected": -0.3681849539279938,
      "step": 10
    },
    {
      "epoch": 0.03139101432215029,
      "grad_norm": 0.04919258877635002,
      "learning_rate": 4.997807075247147e-06,
      "logits/chosen": 15.27595043182373,
      "logits/rejected": 14.872761726379395,
      "logps/chosen": -0.3344747722148895,
      "logps/rejected": -0.24258682131767273,
      "loss": 1.0487,
      "rewards/accuracies": 0.16249999403953552,
      "rewards/chosen": -0.5017121434211731,
      "rewards/margins": -0.1378319263458252,
      "rewards/rejected": -0.3638802468776703,
      "step": 20
    },
    {
      "epoch": 0.047086521483225424,
      "grad_norm": 0.049933061003685,
      "learning_rate": 4.9950668210706795e-06,
      "logits/chosen": 15.913165092468262,
      "logits/rejected": 15.607622146606445,
      "logps/chosen": -0.3440183997154236,
      "logps/rejected": -0.2831566333770752,
      "loss": 1.0405,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -0.516027569770813,
      "rewards/margins": -0.09129264950752258,
      "rewards/rejected": -0.4247349202632904,
      "step": 30
    },
    {
      "epoch": 0.06278202864430057,
      "grad_norm": 0.05693503096699715,
      "learning_rate": 4.9912321481237616e-06,
      "logits/chosen": 15.402900695800781,
      "logits/rejected": 14.99272632598877,
      "logps/chosen": -0.3297731578350067,
      "logps/rejected": -0.2746916711330414,
      "loss": 1.0369,
      "rewards/accuracies": 0.21250000596046448,
      "rewards/chosen": -0.49465981125831604,
      "rewards/margins": -0.08262218534946442,
      "rewards/rejected": -0.41203755140304565,
      "step": 40
    },
    {
      "epoch": 0.07847753580537571,
      "grad_norm": 0.05467928573489189,
      "learning_rate": 4.986304738420684e-06,
      "logits/chosen": 15.64543342590332,
      "logits/rejected": 15.632547378540039,
      "logps/chosen": -0.30952686071395874,
      "logps/rejected": -0.24847058951854706,
      "loss": 1.0367,
      "rewards/accuracies": 0.21250000596046448,
      "rewards/chosen": -0.46429023146629333,
      "rewards/margins": -0.09158438444137573,
      "rewards/rejected": -0.37270587682724,
      "step": 50
    },
    {
      "epoch": 0.07847753580537571,
      "eval_logits/chosen": 15.850138664245605,
      "eval_logits/rejected": 15.368529319763184,
      "eval_logps/chosen": -0.3222965598106384,
      "eval_logps/rejected": -0.26877468824386597,
      "eval_loss": 1.0326261520385742,
      "eval_rewards/accuracies": 0.26923078298568726,
      "eval_rewards/chosen": -0.4834447503089905,
      "eval_rewards/margins": -0.08028276264667511,
      "eval_rewards/rejected": -0.40316200256347656,
      "eval_runtime": 14.5044,
      "eval_samples_per_second": 28.405,
      "eval_steps_per_second": 3.585,
      "step": 50
    },
    {
      "epoch": 0.09417304296645085,
      "grad_norm": 0.06174452602863312,
      "learning_rate": 4.980286753286196e-06,
      "logits/chosen": 15.443066596984863,
      "logits/rejected": 15.192205429077148,
      "logps/chosen": -0.31090402603149414,
      "logps/rejected": -0.26281923055648804,
      "loss": 1.04,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.466356098651886,
      "rewards/margins": -0.07212716341018677,
      "rewards/rejected": -0.39422887563705444,
      "step": 60
    },
    {
      "epoch": 0.109868550127526,
      "grad_norm": 0.06952528655529022,
      "learning_rate": 4.973180832407471e-06,
      "logits/chosen": 16.024200439453125,
      "logits/rejected": 15.82934284210205,
      "logps/chosen": -0.348227322101593,
      "logps/rejected": -0.26220566034317017,
      "loss": 1.043,
      "rewards/accuracies": 0.21250000596046448,
      "rewards/chosen": -0.5223408937454224,
      "rewards/margins": -0.1290324479341507,
      "rewards/rejected": -0.39330852031707764,
      "step": 70
    },
    {
      "epoch": 0.12556405728860115,
      "grad_norm": 0.07572082430124283,
      "learning_rate": 4.964990092676263e-06,
      "logits/chosen": 15.884915351867676,
      "logits/rejected": 15.603845596313477,
      "logps/chosen": -0.34849274158477783,
      "logps/rejected": -0.26585355401039124,
      "loss": 1.0285,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -0.5227391719818115,
      "rewards/margins": -0.12395882606506348,
      "rewards/rejected": -0.39878037571907043,
      "step": 80
    },
    {
      "epoch": 0.14125956444967627,
      "grad_norm": 0.2423778474330902,
      "learning_rate": 4.9557181268217225e-06,
      "logits/chosen": 15.978216171264648,
      "logits/rejected": 15.76471996307373,
      "logps/chosen": -0.327436238527298,
      "logps/rejected": -0.25457051396369934,
      "loss": 1.03,
      "rewards/accuracies": 0.22499999403953552,
      "rewards/chosen": -0.49115434288978577,
      "rewards/margins": -0.10929858684539795,
      "rewards/rejected": -0.3818557560443878,
      "step": 90
    },
    {
      "epoch": 0.15695507161075142,
      "grad_norm": 0.1594536453485489,
      "learning_rate": 4.9453690018345144e-06,
      "logits/chosen": 16.307537078857422,
      "logits/rejected": 16.138330459594727,
      "logps/chosen": -0.3342314660549164,
      "logps/rejected": -0.27582648396492004,
      "loss": 1.0309,
      "rewards/accuracies": 0.26249998807907104,
      "rewards/chosen": -0.5013472437858582,
      "rewards/margins": -0.0876075029373169,
      "rewards/rejected": -0.41373974084854126,
      "step": 100
    },
    {
      "epoch": 0.15695507161075142,
      "eval_logits/chosen": 16.4310245513916,
      "eval_logits/rejected": 15.98912525177002,
      "eval_logps/chosen": -0.3239763677120209,
      "eval_logps/rejected": -0.28784558176994324,
      "eval_loss": 1.020836353302002,
      "eval_rewards/accuracies": 0.3076923191547394,
      "eval_rewards/chosen": -0.4859645664691925,
      "eval_rewards/margins": -0.054196178913116455,
      "eval_rewards/rejected": -0.43176835775375366,
      "eval_runtime": 14.5049,
      "eval_samples_per_second": 28.404,
      "eval_steps_per_second": 3.585,
      "step": 100
    },
    {
      "epoch": 0.17265057877182657,
      "grad_norm": 0.07431349903345108,
      "learning_rate": 4.933947257182901e-06,
      "logits/chosen": 16.56686782836914,
      "logits/rejected": 16.093189239501953,
      "logps/chosen": -0.34455060958862305,
      "logps/rejected": -0.2834388315677643,
      "loss": 1.0388,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.5168259739875793,
      "rewards/margins": -0.09166768193244934,
      "rewards/rejected": -0.4251582622528076,
      "step": 110
    },
    {
      "epoch": 0.1883460859329017,
      "grad_norm": 0.08802352845668793,
      "learning_rate": 4.921457902821578e-06,
      "logits/chosen": 16.50200843811035,
      "logits/rejected": 16.286388397216797,
      "logps/chosen": -0.30845317244529724,
      "logps/rejected": -0.2677682936191559,
      "loss": 1.0247,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.4626797139644623,
      "rewards/margins": -0.06102731078863144,
      "rewards/rejected": -0.40165242552757263,
      "step": 120
    },
    {
      "epoch": 0.20404159309397685,
      "grad_norm": 0.10464702546596527,
      "learning_rate": 4.907906416994146e-06,
      "logits/chosen": 16.163082122802734,
      "logits/rejected": 16.158031463623047,
      "logps/chosen": -0.3138599991798401,
      "logps/rejected": -0.28097471594810486,
      "loss": 1.0169,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.47078999876976013,
      "rewards/margins": -0.04932791367173195,
      "rewards/rejected": -0.4214620590209961,
      "step": 130
    },
    {
      "epoch": 0.219737100255052,
      "grad_norm": 0.16971275210380554,
      "learning_rate": 4.893298743830168e-06,
      "logits/chosen": 16.28864860534668,
      "logits/rejected": 16.151805877685547,
      "logps/chosen": -0.3283368945121765,
      "logps/rejected": -0.2850198745727539,
      "loss": 0.9964,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.49250537157058716,
      "rewards/margins": -0.06497551500797272,
      "rewards/rejected": -0.42752987146377563,
      "step": 140
    },
    {
      "epoch": 0.23543260741612712,
      "grad_norm": 0.18377964198589325,
      "learning_rate": 4.8776412907378845e-06,
      "logits/chosen": 16.890087127685547,
      "logits/rejected": 16.42388153076172,
      "logps/chosen": -0.33256903290748596,
      "logps/rejected": -0.2939595878124237,
      "loss": 1.0073,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.49885353446006775,
      "rewards/margins": -0.057914119213819504,
      "rewards/rejected": -0.44093936681747437,
      "step": 150
    },
    {
      "epoch": 0.23543260741612712,
      "eval_logits/chosen": 16.833438873291016,
      "eval_logits/rejected": 16.328977584838867,
      "eval_logps/chosen": -0.32567569613456726,
      "eval_logps/rejected": -0.35700783133506775,
      "eval_loss": 0.9802881479263306,
      "eval_rewards/accuracies": 0.42307692766189575,
      "eval_rewards/chosen": -0.4885135293006897,
      "eval_rewards/margins": 0.04699822515249252,
      "eval_rewards/rejected": -0.5355117321014404,
      "eval_runtime": 14.5005,
      "eval_samples_per_second": 28.413,
      "eval_steps_per_second": 3.586,
      "step": 150
    },
    {
      "epoch": 0.2511281145772023,
      "grad_norm": 0.12049826234579086,
      "learning_rate": 4.860940925593703e-06,
      "logits/chosen": 16.505878448486328,
      "logits/rejected": 16.178979873657227,
      "logps/chosen": -0.3397011458873749,
      "logps/rejected": -0.35640352964401245,
      "loss": 0.9795,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -0.5095517039299011,
      "rewards/margins": 0.0250535998493433,
      "rewards/rejected": -0.5346053242683411,
      "step": 160
    },
    {
      "epoch": 0.2668236217382774,
      "grad_norm": 0.09485407918691635,
      "learning_rate": 4.84320497372973e-06,
      "logits/chosen": 16.245588302612305,
      "logits/rejected": 15.922958374023438,
      "logps/chosen": -0.29733315110206604,
      "logps/rejected": -0.3461209237575531,
      "loss": 0.9694,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.44599977135658264,
      "rewards/margins": 0.0731816366314888,
      "rewards/rejected": -0.5191814303398132,
      "step": 170
    },
    {
      "epoch": 0.28251912889935255,
      "grad_norm": 0.155483216047287,
      "learning_rate": 4.824441214720629e-06,
      "logits/chosen": 16.339645385742188,
      "logits/rejected": 16.115110397338867,
      "logps/chosen": -0.3076801002025604,
      "logps/rejected": -0.3655286729335785,
      "loss": 0.9488,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.4615201950073242,
      "rewards/margins": 0.0867728441953659,
      "rewards/rejected": -0.5482929944992065,
      "step": 180
    },
    {
      "epoch": 0.2982146360604277,
      "grad_norm": 0.21345795691013336,
      "learning_rate": 4.804657878971252e-06,
      "logits/chosen": 16.491886138916016,
      "logits/rejected": 16.376684188842773,
      "logps/chosen": -0.32437095046043396,
      "logps/rejected": -0.39715105295181274,
      "loss": 0.9423,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.4865564703941345,
      "rewards/margins": 0.10917013883590698,
      "rewards/rejected": -0.5957266092300415,
      "step": 190
    },
    {
      "epoch": 0.31391014322150285,
      "grad_norm": 0.17633090913295746,
      "learning_rate": 4.783863644106502e-06,
      "logits/chosen": 16.6339168548584,
      "logits/rejected": 16.79404640197754,
      "logps/chosen": -0.33018192648887634,
      "logps/rejected": -0.384196400642395,
      "loss": 0.939,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.4952728748321533,
      "rewards/margins": 0.0810217633843422,
      "rewards/rejected": -0.5762946009635925,
      "step": 200
    },
    {
      "epoch": 0.31391014322150285,
      "eval_logits/chosen": 17.17803192138672,
      "eval_logits/rejected": 16.59328269958496,
      "eval_logps/chosen": -0.33600664138793945,
      "eval_logps/rejected": -0.47861453890800476,
      "eval_loss": 0.9303967356681824,
      "eval_rewards/accuracies": 0.4615384638309479,
      "eval_rewards/chosen": -0.5040098428726196,
      "eval_rewards/margins": 0.21391186118125916,
      "eval_rewards/rejected": -0.717921793460846,
      "eval_runtime": 14.5,
      "eval_samples_per_second": 28.414,
      "eval_steps_per_second": 3.586,
      "step": 200
    },
    {
      "epoch": 0.329605650382578,
      "grad_norm": 0.1562221795320511,
      "learning_rate": 4.762067631165049e-06,
      "logits/chosen": 16.597665786743164,
      "logits/rejected": 16.3507022857666,
      "logps/chosen": -0.33709320425987244,
      "logps/rejected": -0.43131333589553833,
      "loss": 0.9107,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.5056397914886475,
      "rewards/margins": 0.14133022725582123,
      "rewards/rejected": -0.6469700932502747,
      "step": 210
    },
    {
      "epoch": 0.34530115754365315,
      "grad_norm": 0.17680124938488007,
      "learning_rate": 4.7392794005985324e-06,
      "logits/chosen": 16.873676300048828,
      "logits/rejected": 16.61945152282715,
      "logps/chosen": -0.35083168745040894,
      "logps/rejected": -0.537697434425354,
      "loss": 0.9094,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.526247501373291,
      "rewards/margins": 0.2802986204624176,
      "rewards/rejected": -0.8065462112426758,
      "step": 220
    },
    {
      "epoch": 0.3609966647047283,
      "grad_norm": 0.2662070393562317,
      "learning_rate": 4.715508948078037e-06,
      "logits/chosen": 16.803974151611328,
      "logits/rejected": 16.698320388793945,
      "logps/chosen": -0.3500753343105316,
      "logps/rejected": -0.5399882793426514,
      "loss": 0.8871,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.5251129865646362,
      "rewards/margins": 0.2848694324493408,
      "rewards/rejected": -0.8099824786186218,
      "step": 230
    },
    {
      "epoch": 0.3766921718658034,
      "grad_norm": 0.17161667346954346,
      "learning_rate": 4.690766700109659e-06,
      "logits/chosen": 16.67904281616211,
      "logits/rejected": 16.410579681396484,
      "logps/chosen": -0.33549198508262634,
      "logps/rejected": -0.4875665605068207,
      "loss": 0.9037,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.5032380223274231,
      "rewards/margins": 0.22811183333396912,
      "rewards/rejected": -0.7313498258590698,
      "step": 240
    },
    {
      "epoch": 0.39238767902687854,
      "grad_norm": 1.106021761894226,
      "learning_rate": 4.665063509461098e-06,
      "logits/chosen": 17.208127975463867,
      "logits/rejected": 16.654085159301758,
      "logps/chosen": -0.3752726912498474,
      "logps/rejected": -0.4947708249092102,
      "loss": 0.8606,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.5629090070724487,
      "rewards/margins": 0.1792471706867218,
      "rewards/rejected": -0.7421562671661377,
      "step": 250
    },
    {
      "epoch": 0.39238767902687854,
      "eval_logits/chosen": 17.32963752746582,
      "eval_logits/rejected": 16.589412689208984,
      "eval_logps/chosen": -0.37825876474380493,
      "eval_logps/rejected": -0.9001243114471436,
      "eval_loss": 0.8168494701385498,
      "eval_rewards/accuracies": 0.5192307829856873,
      "eval_rewards/chosen": -0.5673881769180298,
      "eval_rewards/margins": 0.7827982306480408,
      "eval_rewards/rejected": -1.3501865863800049,
      "eval_runtime": 14.5053,
      "eval_samples_per_second": 28.403,
      "eval_steps_per_second": 3.585,
      "step": 250
    },
    {
      "epoch": 0.4080831861879537,
      "grad_norm": 0.29772138595581055,
      "learning_rate": 4.638410650401267e-06,
      "logits/chosen": 16.713775634765625,
      "logits/rejected": 16.35211944580078,
      "logps/chosen": -0.3877524733543396,
      "logps/rejected": -0.8511163592338562,
      "loss": 0.8138,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.581628680229187,
      "rewards/margins": 0.6950457692146301,
      "rewards/rejected": -1.276674509048462,
      "step": 260
    },
    {
      "epoch": 0.42377869334902885,
      "grad_norm": 0.31860050559043884,
      "learning_rate": 4.610819813755038e-06,
      "logits/chosen": 17.09469985961914,
      "logits/rejected": 16.5472412109375,
      "logps/chosen": -0.47504258155822754,
      "logps/rejected": -1.3266533613204956,
      "loss": 0.7318,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.7125638723373413,
      "rewards/margins": 1.2774161100387573,
      "rewards/rejected": -1.9899799823760986,
      "step": 270
    },
    {
      "epoch": 0.439474200510104,
      "grad_norm": 0.6508163809776306,
      "learning_rate": 4.582303101775249e-06,
      "logits/chosen": 17.11003303527832,
      "logits/rejected": 16.6564998626709,
      "logps/chosen": -0.49357643723487854,
      "logps/rejected": -1.4481580257415771,
      "loss": 0.7599,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.7403645515441895,
      "rewards/margins": 1.4318726062774658,
      "rewards/rejected": -2.1722371578216553,
      "step": 280
    },
    {
      "epoch": 0.45516970767117915,
      "grad_norm": 0.32430580258369446,
      "learning_rate": 4.55287302283426e-06,
      "logits/chosen": 16.830989837646484,
      "logits/rejected": 16.384944915771484,
      "logps/chosen": -0.593712329864502,
      "logps/rejected": -1.7630856037139893,
      "loss": 0.7336,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.8905684351921082,
      "rewards/margins": 1.7540600299835205,
      "rewards/rejected": -2.6446282863616943,
      "step": 290
    },
    {
      "epoch": 0.47086521483225424,
      "grad_norm": 0.8555932641029358,
      "learning_rate": 4.522542485937369e-06,
      "logits/chosen": 16.72231674194336,
      "logits/rejected": 16.28726577758789,
      "logps/chosen": -0.5670709609985352,
      "logps/rejected": -2.0420775413513184,
      "loss": 0.6861,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8506065607070923,
      "rewards/margins": 2.2125096321105957,
      "rewards/rejected": -3.0631160736083984,
      "step": 300
    },
    {
      "epoch": 0.47086521483225424,
      "eval_logits/chosen": 17.13121795654297,
      "eval_logits/rejected": 16.268341064453125,
      "eval_logps/chosen": -0.6842947602272034,
      "eval_logps/rejected": -2.119321584701538,
      "eval_loss": 0.7583853602409363,
      "eval_rewards/accuracies": 0.75,
      "eval_rewards/chosen": -1.026442289352417,
      "eval_rewards/margins": 2.1525399684906006,
      "eval_rewards/rejected": -3.1789822578430176,
      "eval_runtime": 14.5007,
      "eval_samples_per_second": 28.412,
      "eval_steps_per_second": 3.586,
      "step": 300
    },
    {
      "epoch": 0.4865607219933294,
      "grad_norm": 0.6059070825576782,
      "learning_rate": 4.491324795060491e-06,
      "logits/chosen": 17.147808074951172,
      "logits/rejected": 16.194652557373047,
      "logps/chosen": -0.8036400079727173,
      "logps/rejected": -2.289825201034546,
      "loss": 0.7163,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.2054599523544312,
      "rewards/margins": 2.229278087615967,
      "rewards/rejected": -3.4347376823425293,
      "step": 310
    },
    {
      "epoch": 0.5022562291544046,
      "grad_norm": 1.8073927164077759,
      "learning_rate": 4.4592336433146e-06,
      "logits/chosen": 16.94902992248535,
      "logits/rejected": 16.066068649291992,
      "logps/chosen": -1.198162317276001,
      "logps/rejected": -2.2922632694244385,
      "loss": 0.6894,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.7972434759140015,
      "rewards/margins": 1.6411516666412354,
      "rewards/rejected": -3.4383950233459473,
      "step": 320
    },
    {
      "epoch": 0.5179517363154797,
      "grad_norm": 3.746042490005493,
      "learning_rate": 4.426283106939474e-06,
      "logits/chosen": 16.31036376953125,
      "logits/rejected": 15.991762161254883,
      "logps/chosen": -1.6245781183242798,
      "logps/rejected": -2.553597927093506,
      "loss": 0.6607,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -2.4368672370910645,
      "rewards/margins": 1.3935294151306152,
      "rewards/rejected": -3.830397129058838,
      "step": 330
    },
    {
      "epoch": 0.5336472434765548,
      "grad_norm": 2.098111867904663,
      "learning_rate": 4.3924876391293915e-06,
      "logits/chosen": 16.59554672241211,
      "logits/rejected": 15.915553092956543,
      "logps/chosen": -2.183227062225342,
      "logps/rejected": -3.4911434650421143,
      "loss": 0.6381,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.274840831756592,
      "rewards/margins": 1.9618743658065796,
      "rewards/rejected": -5.236715316772461,
      "step": 340
    },
    {
      "epoch": 0.5493427506376299,
      "grad_norm": 2.153958320617676,
      "learning_rate": 4.357862063693486e-06,
      "logits/chosen": 16.332544326782227,
      "logits/rejected": 15.691922187805176,
      "logps/chosen": -2.673710346221924,
      "logps/rejected": -3.8687057495117188,
      "loss": 0.5752,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -4.010565757751465,
      "rewards/margins": 1.792493224143982,
      "rewards/rejected": -5.8030595779418945,
      "step": 350
    },
    {
      "epoch": 0.5493427506376299,
      "eval_logits/chosen": 16.272428512573242,
      "eval_logits/rejected": 15.381678581237793,
      "eval_logps/chosen": -3.0390572547912598,
      "eval_logps/rejected": -4.695068836212158,
      "eval_loss": 0.5928590893745422,
      "eval_rewards/accuracies": 0.9230769276618958,
      "eval_rewards/chosen": -4.5585856437683105,
      "eval_rewards/margins": 2.4840168952941895,
      "eval_rewards/rejected": -7.0426025390625,
      "eval_runtime": 14.5008,
      "eval_samples_per_second": 28.412,
      "eval_steps_per_second": 3.586,
      "step": 350
    },
    {
      "epoch": 0.5650382577987051,
      "grad_norm": 1.5755672454833984,
      "learning_rate": 4.322421568553529e-06,
      "logits/chosen": 15.589811325073242,
      "logits/rejected": 14.780921936035156,
      "logps/chosen": -3.069565534591675,
      "logps/rejected": -4.581957817077637,
      "loss": 0.5275,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.604348182678223,
      "rewards/margins": 2.268587827682495,
      "rewards/rejected": -6.872936248779297,
      "step": 360
    },
    {
      "epoch": 0.5807337649597802,
      "grad_norm": 1.8776415586471558,
      "learning_rate": 4.286181699082008e-06,
      "logits/chosen": 15.172673225402832,
      "logits/rejected": 14.400335311889648,
      "logps/chosen": -3.9551639556884766,
      "logps/rejected": -5.95252799987793,
      "loss": 0.5113,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -5.932745933532715,
      "rewards/margins": 2.996046543121338,
      "rewards/rejected": -8.928792953491211,
      "step": 370
    },
    {
      "epoch": 0.5964292721208554,
      "grad_norm": 1.5507289171218872,
      "learning_rate": 4.249158351283414e-06,
      "logits/chosen": 14.8423433303833,
      "logits/rejected": 14.592633247375488,
      "logps/chosen": -3.7946255207061768,
      "logps/rejected": -5.3750481605529785,
      "loss": 0.4862,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -5.691938877105713,
      "rewards/margins": 2.370633125305176,
      "rewards/rejected": -8.062570571899414,
      "step": 380
    },
    {
      "epoch": 0.6121247792819305,
      "grad_norm": 3.4510324001312256,
      "learning_rate": 4.211367764821722e-06,
      "logits/chosen": 14.586801528930664,
      "logits/rejected": 14.031987190246582,
      "logps/chosen": -4.391470432281494,
      "logps/rejected": -6.2107672691345215,
      "loss": 0.5117,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -6.587205410003662,
      "rewards/margins": 2.72894549369812,
      "rewards/rejected": -9.316150665283203,
      "step": 390
    },
    {
      "epoch": 0.6278202864430057,
      "grad_norm": 3.3816750049591064,
      "learning_rate": 4.172826515897146e-06,
      "logits/chosen": 14.523185729980469,
      "logits/rejected": 14.266815185546875,
      "logps/chosen": -4.090173244476318,
      "logps/rejected": -5.582955837249756,
      "loss": 0.5182,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -6.135260581970215,
      "rewards/margins": 2.2391738891601562,
      "rewards/rejected": -8.374434471130371,
      "step": 400
    },
    {
      "epoch": 0.6278202864430057,
      "eval_logits/chosen": 15.142684936523438,
      "eval_logits/rejected": 14.101082801818848,
      "eval_logps/chosen": -3.866588830947876,
      "eval_logps/rejected": -6.100707530975342,
      "eval_loss": 0.48699691891670227,
      "eval_rewards/accuracies": 0.9615384340286255,
      "eval_rewards/chosen": -5.7998833656311035,
      "eval_rewards/margins": 3.351177215576172,
      "eval_rewards/rejected": -9.15106201171875,
      "eval_runtime": 14.5024,
      "eval_samples_per_second": 28.409,
      "eval_steps_per_second": 3.586,
      "step": 400
    },
    {
      "epoch": 0.6435157936040808,
      "grad_norm": 1.845595359802246,
      "learning_rate": 4.133551509975264e-06,
      "logits/chosen": 14.54790210723877,
      "logits/rejected": 13.707855224609375,
      "logps/chosen": -4.112462043762207,
      "logps/rejected": -6.583975315093994,
      "loss": 0.5107,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -6.168692111968994,
      "rewards/margins": 3.707270383834839,
      "rewards/rejected": -9.875962257385254,
      "step": 410
    },
    {
      "epoch": 0.659211300765156,
      "grad_norm": 2.152916193008423,
      "learning_rate": 4.093559974371725e-06,
      "logits/chosen": 14.153576850891113,
      "logits/rejected": 13.553201675415039,
      "logps/chosen": -4.3408918380737305,
      "logps/rejected": -6.828585147857666,
      "loss": 0.41,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.511338233947754,
      "rewards/margins": 3.7315402030944824,
      "rewards/rejected": -10.242877960205078,
      "step": 420
    },
    {
      "epoch": 0.6749068079262311,
      "grad_norm": 2.305433750152588,
      "learning_rate": 4.052869450695776e-06,
      "logits/chosen": 13.992170333862305,
      "logits/rejected": 13.264738082885742,
      "logps/chosen": -4.259932518005371,
      "logps/rejected": -6.089646339416504,
      "loss": 0.4122,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -6.389898777008057,
      "rewards/margins": 2.7445709705352783,
      "rewards/rejected": -9.134469985961914,
      "step": 430
    },
    {
      "epoch": 0.6906023150873063,
      "grad_norm": 2.397674322128296,
      "learning_rate": 4.011497787155938e-06,
      "logits/chosen": 13.975759506225586,
      "logits/rejected": 13.173799514770508,
      "logps/chosen": -4.2483391761779785,
      "logps/rejected": -6.487355709075928,
      "loss": 0.4397,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -6.3725080490112305,
      "rewards/margins": 3.3585267066955566,
      "rewards/rejected": -9.731034278869629,
      "step": 440
    },
    {
      "epoch": 0.7062978222483814,
      "grad_norm": 2.3987627029418945,
      "learning_rate": 3.969463130731183e-06,
      "logits/chosen": 13.777560234069824,
      "logits/rejected": 12.936296463012695,
      "logps/chosen": -3.86059832572937,
      "logps/rejected": -6.495786190032959,
      "loss": 0.444,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -5.790897846221924,
      "rewards/margins": 3.952782392501831,
      "rewards/rejected": -9.743680000305176,
      "step": 450
    },
    {
      "epoch": 0.7062978222483814,
      "eval_logits/chosen": 14.397418975830078,
      "eval_logits/rejected": 13.271268844604492,
      "eval_logps/chosen": -3.9544241428375244,
      "eval_logps/rejected": -6.69989013671875,
      "eval_loss": 0.435256689786911,
      "eval_rewards/accuracies": 0.9615384340286255,
      "eval_rewards/chosen": -5.931635856628418,
      "eval_rewards/margins": 4.118198871612549,
      "eval_rewards/rejected": -10.049835205078125,
      "eval_runtime": 14.5029,
      "eval_samples_per_second": 28.408,
      "eval_steps_per_second": 3.585,
      "step": 450
    },
    {
      "epoch": 0.7219933294094566,
      "grad_norm": 2.3673465251922607,
      "learning_rate": 3.92678391921108e-06,
      "logits/chosen": 13.67004680633545,
      "logits/rejected": 12.689542770385742,
      "logps/chosen": -4.037893295288086,
      "logps/rejected": -6.727287292480469,
      "loss": 0.3858,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.056839942932129,
      "rewards/margins": 4.034090995788574,
      "rewards/rejected": -10.09093189239502,
      "step": 460
    },
    {
      "epoch": 0.7376888365705316,
      "grad_norm": 2.9868297576904297,
      "learning_rate": 3.88347887310836e-06,
      "logits/chosen": 13.124593734741211,
      "logits/rejected": 12.576837539672852,
      "logps/chosen": -4.324252605438232,
      "logps/rejected": -6.761715888977051,
      "loss": 0.363,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -6.486379146575928,
      "rewards/margins": 3.6561942100524902,
      "rewards/rejected": -10.142572402954102,
      "step": 470
    },
    {
      "epoch": 0.7533843437316068,
      "grad_norm": 2.613318920135498,
      "learning_rate": 3.839566987447492e-06,
      "logits/chosen": 13.514410018920898,
      "logits/rejected": 12.838116645812988,
      "logps/chosen": -4.333093166351318,
      "logps/rejected": -6.704646110534668,
      "loss": 0.3274,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.499639987945557,
      "rewards/margins": 3.5573298931121826,
      "rewards/rejected": -10.056970596313477,
      "step": 480
    },
    {
      "epoch": 0.7690798508926819,
      "grad_norm": 2.1568610668182373,
      "learning_rate": 3.795067523432826e-06,
      "logits/chosen": 13.456674575805664,
      "logits/rejected": 12.37132453918457,
      "logps/chosen": -4.630190849304199,
      "logps/rejected": -7.871635437011719,
      "loss": 0.3822,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -6.945285797119141,
      "rewards/margins": 4.862167835235596,
      "rewards/rejected": -11.807454109191895,
      "step": 490
    },
    {
      "epoch": 0.7847753580537571,
      "grad_norm": 2.516496419906616,
      "learning_rate": 3.7500000000000005e-06,
      "logits/chosen": 13.195734024047852,
      "logits/rejected": 12.557401657104492,
      "logps/chosen": -3.9327914714813232,
      "logps/rejected": -6.561183929443359,
      "loss": 0.352,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -5.899188041687012,
      "rewards/margins": 3.9425880908966064,
      "rewards/rejected": -9.841775894165039,
      "step": 500
    },
    {
      "epoch": 0.7847753580537571,
      "eval_logits/chosen": 13.891414642333984,
      "eval_logits/rejected": 12.734375,
      "eval_logps/chosen": -3.924809455871582,
      "eval_logps/rejected": -7.003909587860107,
      "eval_loss": 0.3888731002807617,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -5.887214183807373,
      "eval_rewards/margins": 4.618649482727051,
      "eval_rewards/rejected": -10.505864143371582,
      "eval_runtime": 14.5029,
      "eval_samples_per_second": 28.408,
      "eval_steps_per_second": 3.585,
      "step": 500
    },
    {
      "epoch": 0.8004708652148322,
      "grad_norm": 5.4663896560668945,
      "learning_rate": 3.7043841852542884e-06,
      "logits/chosen": 12.861404418945312,
      "logits/rejected": 11.9797945022583,
      "logps/chosen": -4.245055198669434,
      "logps/rejected": -7.063906669616699,
      "loss": 0.4009,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -6.36758279800415,
      "rewards/margins": 4.228276252746582,
      "rewards/rejected": -10.59585952758789,
      "step": 510
    },
    {
      "epoch": 0.8161663723759074,
      "grad_norm": 2.4403271675109863,
      "learning_rate": 3.658240087799655e-06,
      "logits/chosen": 12.84996223449707,
      "logits/rejected": 12.256246566772461,
      "logps/chosen": -4.2785773277282715,
      "logps/rejected": -7.003395080566406,
      "loss": 0.3327,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -6.417865753173828,
      "rewards/margins": 4.0872273445129395,
      "rewards/rejected": -10.505093574523926,
      "step": 520
    },
    {
      "epoch": 0.8318618795369825,
      "grad_norm": 3.596749782562256,
      "learning_rate": 3.611587947962319e-06,
      "logits/chosen": 13.3360595703125,
      "logits/rejected": 12.449459075927734,
      "logps/chosen": -4.8148322105407715,
      "logps/rejected": -7.221930503845215,
      "loss": 0.3255,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -7.222248077392578,
      "rewards/margins": 3.610647201538086,
      "rewards/rejected": -10.832895278930664,
      "step": 530
    },
    {
      "epoch": 0.8475573866980577,
      "grad_norm": 4.537969589233398,
      "learning_rate": 3.564448228912682e-06,
      "logits/chosen": 12.63983154296875,
      "logits/rejected": 12.042104721069336,
      "logps/chosen": -4.933573246002197,
      "logps/rejected": -7.656388282775879,
      "loss": 0.2941,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -7.400360107421875,
      "rewards/margins": 4.084224224090576,
      "rewards/rejected": -11.484583854675293,
      "step": 540
    },
    {
      "epoch": 0.8632528938591328,
      "grad_norm": 2.4124038219451904,
      "learning_rate": 3.516841607689501e-06,
      "logits/chosen": 13.053291320800781,
      "logits/rejected": 12.351530075073242,
      "logps/chosen": -5.196786403656006,
      "logps/rejected": -7.8461012840271,
      "loss": 0.3906,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -7.795179843902588,
      "rewards/margins": 3.9739716053009033,
      "rewards/rejected": -11.76915168762207,
      "step": 550
    },
    {
      "epoch": 0.8632528938591328,
      "eval_logits/chosen": 13.596405982971191,
      "eval_logits/rejected": 12.482269287109375,
      "eval_logps/chosen": -4.328949928283691,
      "eval_logps/rejected": -7.602427959442139,
      "eval_loss": 0.3577499985694885,
      "eval_rewards/accuracies": 0.9807692170143127,
      "eval_rewards/chosen": -6.493425369262695,
      "eval_rewards/margins": 4.910217761993408,
      "eval_rewards/rejected": -11.403642654418945,
      "eval_runtime": 14.5032,
      "eval_samples_per_second": 28.408,
      "eval_steps_per_second": 3.585,
      "step": 550
    },
    {
      "epoch": 0.878948401020208,
      "grad_norm": 1.42177414894104,
      "learning_rate": 3.4687889661302577e-06,
      "logits/chosen": 13.275362014770508,
      "logits/rejected": 12.30308723449707,
      "logps/chosen": -4.182652950286865,
      "logps/rejected": -7.163094997406006,
      "loss": 0.3692,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.273979663848877,
      "rewards/margins": 4.470663547515869,
      "rewards/rejected": -10.744643211364746,
      "step": 560
    },
    {
      "epoch": 0.8946439081812831,
      "grad_norm": 2.431758403778076,
      "learning_rate": 3.4203113817116955e-06,
      "logits/chosen": 13.407594680786133,
      "logits/rejected": 12.078115463256836,
      "logps/chosen": -4.193841457366943,
      "logps/rejected": -7.832788944244385,
      "loss": 0.3487,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.290762901306152,
      "rewards/margins": 5.458420753479004,
      "rewards/rejected": -11.749183654785156,
      "step": 570
    },
    {
      "epoch": 0.9103394153423583,
      "grad_norm": 2.549168348312378,
      "learning_rate": 3.3714301183045382e-06,
      "logits/chosen": 12.668423652648926,
      "logits/rejected": 11.851642608642578,
      "logps/chosen": -4.831603050231934,
      "logps/rejected": -7.4626054763793945,
      "loss": 0.3717,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -7.2474045753479,
      "rewards/margins": 3.946502685546875,
      "rewards/rejected": -11.193906784057617,
      "step": 580
    },
    {
      "epoch": 0.9260349225034334,
      "grad_norm": 3.3875763416290283,
      "learning_rate": 3.3221666168464584e-06,
      "logits/chosen": 12.595063209533691,
      "logits/rejected": 12.006935119628906,
      "logps/chosen": -4.957076072692871,
      "logps/rejected": -7.576946258544922,
      "loss": 0.3553,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -7.435614109039307,
      "rewards/margins": 3.9298064708709717,
      "rewards/rejected": -11.3654203414917,
      "step": 590
    },
    {
      "epoch": 0.9417304296645085,
      "grad_norm": 2.784954071044922,
      "learning_rate": 3.272542485937369e-06,
      "logits/chosen": 12.631284713745117,
      "logits/rejected": 11.908422470092773,
      "logps/chosen": -4.122859477996826,
      "logps/rejected": -7.009615421295166,
      "loss": 0.3086,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.18428897857666,
      "rewards/margins": 4.330134868621826,
      "rewards/rejected": -10.514424324035645,
      "step": 600
    },
    {
      "epoch": 0.9417304296645085,
      "eval_logits/chosen": 13.370320320129395,
      "eval_logits/rejected": 12.268073081970215,
      "eval_logps/chosen": -4.0377197265625,
      "eval_logps/rejected": -7.431212425231934,
      "eval_loss": 0.33698391914367676,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -6.056579113006592,
      "eval_rewards/margins": 5.090238094329834,
      "eval_rewards/rejected": -11.146818161010742,
      "eval_runtime": 14.5012,
      "eval_samples_per_second": 28.411,
      "eval_steps_per_second": 3.586,
      "step": 600
    },
    {
      "epoch": 0.9574259368255836,
      "grad_norm": 3.2864303588867188,
      "learning_rate": 3.222579492361179e-06,
      "logits/chosen": 12.638185501098633,
      "logits/rejected": 11.773119926452637,
      "logps/chosen": -4.101125240325928,
      "logps/rejected": -6.985491752624512,
      "loss": 0.2886,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.1516876220703125,
      "rewards/margins": 4.326549530029297,
      "rewards/rejected": -10.47823715209961,
      "step": 610
    },
    {
      "epoch": 0.9731214439866588,
      "grad_norm": 3.094020366668701,
      "learning_rate": 3.1722995515381644e-06,
      "logits/chosen": 12.71407413482666,
      "logits/rejected": 11.668571472167969,
      "logps/chosen": -4.560760498046875,
      "logps/rejected": -7.878456115722656,
      "loss": 0.294,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.841139793395996,
      "rewards/margins": 4.976544380187988,
      "rewards/rejected": -11.817686080932617,
      "step": 620
    },
    {
      "epoch": 0.9888169511477339,
      "grad_norm": 1.906849980354309,
      "learning_rate": 3.121724717912138e-06,
      "logits/chosen": 12.863229751586914,
      "logits/rejected": 11.391074180603027,
      "logps/chosen": -4.578230857849121,
      "logps/rejected": -8.216089248657227,
      "loss": 0.2579,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.86734676361084,
      "rewards/margins": 5.456785202026367,
      "rewards/rejected": -12.324131965637207,
      "step": 630
    },
    {
      "epoch": 1.003139101432215,
      "grad_norm": 2.699907064437866,
      "learning_rate": 3.0708771752766397e-06,
      "logits/chosen": 12.64507007598877,
      "logits/rejected": 12.148189544677734,
      "logps/chosen": -4.193073272705078,
      "logps/rejected": -7.2082905769348145,
      "loss": 0.2449,
      "rewards/accuracies": 0.931506872177124,
      "rewards/chosen": -6.289609909057617,
      "rewards/margins": 4.522825241088867,
      "rewards/rejected": -10.812435150146484,
      "step": 640
    },
    {
      "epoch": 1.0188346085932902,
      "grad_norm": 3.4014647006988525,
      "learning_rate": 3.019779227044398e-06,
      "logits/chosen": 12.921002388000488,
      "logits/rejected": 12.00908088684082,
      "logps/chosen": -4.350411415100098,
      "logps/rejected": -7.187474250793457,
      "loss": 0.2976,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -6.5256171226501465,
      "rewards/margins": 4.255594253540039,
      "rewards/rejected": -10.781211853027344,
      "step": 650
    },
    {
      "epoch": 1.0188346085932902,
      "eval_logits/chosen": 13.153624534606934,
      "eval_logits/rejected": 12.077543258666992,
      "eval_logps/chosen": -4.4063401222229,
      "eval_logps/rejected": -8.090719223022461,
      "eval_loss": 0.31872284412384033,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -6.6095099449157715,
      "eval_rewards/margins": 5.526568412780762,
      "eval_rewards/rejected": -12.136077880859375,
      "eval_runtime": 14.4992,
      "eval_samples_per_second": 28.415,
      "eval_steps_per_second": 3.586,
      "step": 650
    },
    {
      "epoch": 1.0345301157543654,
      "grad_norm": 3.4320549964904785,
      "learning_rate": 2.9684532864643123e-06,
      "logits/chosen": 12.755729675292969,
      "logits/rejected": 11.981709480285645,
      "logps/chosen": -4.497554302215576,
      "logps/rejected": -7.250922203063965,
      "loss": 0.3013,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -6.746331691741943,
      "rewards/margins": 4.13005256652832,
      "rewards/rejected": -10.876383781433105,
      "step": 660
    },
    {
      "epoch": 1.0502256229154405,
      "grad_norm": 5.130344867706299,
      "learning_rate": 2.9169218667902562e-06,
      "logits/chosen": 12.774751663208008,
      "logits/rejected": 12.063162803649902,
      "logps/chosen": -4.672727108001709,
      "logps/rejected": -7.972691535949707,
      "loss": 0.2677,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -7.009090423583984,
      "rewards/margins": 4.94994592666626,
      "rewards/rejected": -11.959035873413086,
      "step": 670
    },
    {
      "epoch": 1.0659211300765157,
      "grad_norm": 2.1107585430145264,
      "learning_rate": 2.8652075714060296e-06,
      "logits/chosen": 12.401437759399414,
      "logits/rejected": 11.329314231872559,
      "logps/chosen": -4.372309684753418,
      "logps/rejected": -7.968375205993652,
      "loss": 0.2321,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.558464050292969,
      "rewards/margins": 5.394099235534668,
      "rewards/rejected": -11.952563285827637,
      "step": 680
    },
    {
      "epoch": 1.0816166372375908,
      "grad_norm": 2.09533429145813,
      "learning_rate": 2.813333083910761e-06,
      "logits/chosen": 12.615015029907227,
      "logits/rejected": 11.613394737243652,
      "logps/chosen": -4.582761764526367,
      "logps/rejected": -7.808516502380371,
      "loss": 0.2593,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -6.874144077301025,
      "rewards/margins": 4.838631629943848,
      "rewards/rejected": -11.712774276733398,
      "step": 690
    },
    {
      "epoch": 1.097312144398666,
      "grad_norm": 2.652644157409668,
      "learning_rate": 2.761321158169134e-06,
      "logits/chosen": 12.447229385375977,
      "logits/rejected": 11.710171699523926,
      "logps/chosen": -4.722737789154053,
      "logps/rejected": -8.122079849243164,
      "loss": 0.263,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -7.084107398986816,
      "rewards/margins": 5.0990142822265625,
      "rewards/rejected": -12.183120727539062,
      "step": 700
    },
    {
      "epoch": 1.097312144398666,
      "eval_logits/chosen": 13.170889854431152,
      "eval_logits/rejected": 12.095331192016602,
      "eval_logps/chosen": -4.405147075653076,
      "eval_logps/rejected": -8.16016960144043,
      "eval_loss": 0.3023497760295868,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -6.607721328735352,
      "eval_rewards/margins": 5.632532119750977,
      "eval_rewards/rejected": -12.240253448486328,
      "eval_runtime": 14.5027,
      "eval_samples_per_second": 28.408,
      "eval_steps_per_second": 3.586,
      "step": 700
    },
    {
      "epoch": 1.113007651559741,
      "grad_norm": 2.3017961978912354,
      "learning_rate": 2.70919460833079e-06,
      "logits/chosen": 12.642290115356445,
      "logits/rejected": 11.817375183105469,
      "logps/chosen": -4.732531547546387,
      "logps/rejected": -8.388056755065918,
      "loss": 0.2707,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -7.098797798156738,
      "rewards/margins": 5.483288764953613,
      "rewards/rejected": -12.582086563110352,
      "step": 710
    },
    {
      "epoch": 1.1287031587208163,
      "grad_norm": 2.538917303085327,
      "learning_rate": 2.6569762988232838e-06,
      "logits/chosen": 12.873556137084961,
      "logits/rejected": 11.74571704864502,
      "logps/chosen": -5.0908284187316895,
      "logps/rejected": -8.69284439086914,
      "loss": 0.2443,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -7.636242866516113,
      "rewards/margins": 5.403023719787598,
      "rewards/rejected": -13.039266586303711,
      "step": 720
    },
    {
      "epoch": 1.1443986658818912,
      "grad_norm": 2.7106313705444336,
      "learning_rate": 2.604689134322999e-06,
      "logits/chosen": 12.663686752319336,
      "logits/rejected": 12.057671546936035,
      "logps/chosen": -4.6052398681640625,
      "logps/rejected": -7.64666748046875,
      "loss": 0.2657,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.907858848571777,
      "rewards/margins": 4.562141418457031,
      "rewards/rejected": -11.470001220703125,
      "step": 730
    },
    {
      "epoch": 1.1600941730429664,
      "grad_norm": 1.9821428060531616,
      "learning_rate": 2.5523560497083927e-06,
      "logits/chosen": 12.605002403259277,
      "logits/rejected": 11.786115646362305,
      "logps/chosen": -4.447022438049316,
      "logps/rejected": -8.232423782348633,
      "loss": 0.2234,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.670533180236816,
      "rewards/margins": 5.678102016448975,
      "rewards/rejected": -12.348634719848633,
      "step": 740
    },
    {
      "epoch": 1.1757896802040415,
      "grad_norm": 2.469222068786621,
      "learning_rate": 2.5e-06,
      "logits/chosen": 12.427464485168457,
      "logits/rejected": 11.602205276489258,
      "logps/chosen": -4.620213985443115,
      "logps/rejected": -8.116684913635254,
      "loss": 0.2895,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -6.93032169342041,
      "rewards/margins": 5.244706153869629,
      "rewards/rejected": -12.175026893615723,
      "step": 750
    },
    {
      "epoch": 1.1757896802040415,
      "eval_logits/chosen": 13.117497444152832,
      "eval_logits/rejected": 12.068766593933105,
      "eval_logps/chosen": -4.3247389793396,
      "eval_logps/rejected": -8.284286499023438,
      "eval_loss": 0.2885077893733978,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -6.48710823059082,
      "eval_rewards/margins": 5.939322471618652,
      "eval_rewards/rejected": -12.426431655883789,
      "eval_runtime": 14.5056,
      "eval_samples_per_second": 28.403,
      "eval_steps_per_second": 3.585,
      "step": 750
    },
    {
      "epoch": 1.1914851873651167,
      "grad_norm": 1.4759465456008911,
      "learning_rate": 2.447643950291608e-06,
      "logits/chosen": 12.716300010681152,
      "logits/rejected": 11.687711715698242,
      "logps/chosen": -4.740975379943848,
      "logps/rejected": -8.425516128540039,
      "loss": 0.2696,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -7.111462593078613,
      "rewards/margins": 5.526811122894287,
      "rewards/rejected": -12.638272285461426,
      "step": 760
    },
    {
      "epoch": 1.2071806945261918,
      "grad_norm": 4.352353572845459,
      "learning_rate": 2.3953108656770018e-06,
      "logits/chosen": 12.28320598602295,
      "logits/rejected": 11.589045524597168,
      "logps/chosen": -4.623471260070801,
      "logps/rejected": -7.964238166809082,
      "loss": 0.2692,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -6.935206413269043,
      "rewards/margins": 5.01115083694458,
      "rewards/rejected": -11.946358680725098,
      "step": 770
    },
    {
      "epoch": 1.222876201687267,
      "grad_norm": 3.407118558883667,
      "learning_rate": 2.3430237011767166e-06,
      "logits/chosen": 12.472453117370605,
      "logits/rejected": 11.568643569946289,
      "logps/chosen": -4.998484134674072,
      "logps/rejected": -9.068008422851562,
      "loss": 0.248,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -7.4977264404296875,
      "rewards/margins": 6.104286193847656,
      "rewards/rejected": -13.602012634277344,
      "step": 780
    },
    {
      "epoch": 1.2385717088483421,
      "grad_norm": 2.2443063259124756,
      "learning_rate": 2.290805391669212e-06,
      "logits/chosen": 12.315655708312988,
      "logits/rejected": 11.502891540527344,
      "logps/chosen": -4.696177959442139,
      "logps/rejected": -8.123588562011719,
      "loss": 0.2549,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.044266700744629,
      "rewards/margins": 5.141116142272949,
      "rewards/rejected": -12.185382843017578,
      "step": 790
    },
    {
      "epoch": 1.2542672160094173,
      "grad_norm": 3.419119358062744,
      "learning_rate": 2.238678841830867e-06,
      "logits/chosen": 12.070470809936523,
      "logits/rejected": 11.558723449707031,
      "logps/chosen": -4.473680019378662,
      "logps/rejected": -7.936869144439697,
      "loss": 0.3082,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.710520267486572,
      "rewards/margins": 5.1947832107543945,
      "rewards/rejected": -11.905302047729492,
      "step": 800
    },
    {
      "epoch": 1.2542672160094173,
      "eval_logits/chosen": 13.084930419921875,
      "eval_logits/rejected": 12.024311065673828,
      "eval_logps/chosen": -4.1232075691223145,
      "eval_logps/rejected": -8.072820663452148,
      "eval_loss": 0.27581357955932617,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -6.184810161590576,
      "eval_rewards/margins": 5.924418926239014,
      "eval_rewards/rejected": -12.10922908782959,
      "eval_runtime": 14.5062,
      "eval_samples_per_second": 28.402,
      "eval_steps_per_second": 3.585,
      "step": 800
    },
    {
      "epoch": 1.2699627231704924,
      "grad_norm": 3.637188196182251,
      "learning_rate": 2.186666916089239e-06,
      "logits/chosen": 12.425226211547852,
      "logits/rejected": 11.625535011291504,
      "logps/chosen": -4.465217590332031,
      "logps/rejected": -8.138174057006836,
      "loss": 0.2454,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.697826385498047,
      "rewards/margins": 5.509435653686523,
      "rewards/rejected": -12.20726203918457,
      "step": 810
    },
    {
      "epoch": 1.2856582303315676,
      "grad_norm": 2.836642265319824,
      "learning_rate": 2.134792428593971e-06,
      "logits/chosen": 12.305977821350098,
      "logits/rejected": 11.66655445098877,
      "logps/chosen": -4.680156707763672,
      "logps/rejected": -7.750007629394531,
      "loss": 0.2172,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -7.020236015319824,
      "rewards/margins": 4.604775905609131,
      "rewards/rejected": -11.625011444091797,
      "step": 820
    },
    {
      "epoch": 1.3013537374926427,
      "grad_norm": 3.5275964736938477,
      "learning_rate": 2.0830781332097446e-06,
      "logits/chosen": 12.902934074401855,
      "logits/rejected": 11.944432258605957,
      "logps/chosen": -4.6034321784973145,
      "logps/rejected": -8.786798477172852,
      "loss": 0.2231,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.905147552490234,
      "rewards/margins": 6.275050163269043,
      "rewards/rejected": -13.180196762084961,
      "step": 830
    },
    {
      "epoch": 1.3170492446537179,
      "grad_norm": 3.2921273708343506,
      "learning_rate": 2.031546713535688e-06,
      "logits/chosen": 12.8768892288208,
      "logits/rejected": 11.985623359680176,
      "logps/chosen": -4.825736045837402,
      "logps/rejected": -8.161986351013184,
      "loss": 0.2584,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -7.2386040687561035,
      "rewards/margins": 5.00437593460083,
      "rewards/rejected": -12.242980003356934,
      "step": 840
    },
    {
      "epoch": 1.332744751814793,
      "grad_norm": 3.1977927684783936,
      "learning_rate": 1.9802207729556023e-06,
      "logits/chosen": 12.695280075073242,
      "logits/rejected": 11.713974952697754,
      "logps/chosen": -4.446492671966553,
      "logps/rejected": -8.236989974975586,
      "loss": 0.2242,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -6.669739723205566,
      "rewards/margins": 5.685745716094971,
      "rewards/rejected": -12.355484008789062,
      "step": 850
    },
    {
      "epoch": 1.332744751814793,
      "eval_logits/chosen": 12.975180625915527,
      "eval_logits/rejected": 11.917383193969727,
      "eval_logps/chosen": -4.095187664031982,
      "eval_logps/rejected": -8.178255081176758,
      "eval_loss": 0.2621641755104065,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -6.1427812576293945,
      "eval_rewards/margins": 6.124600887298584,
      "eval_rewards/rejected": -12.26738166809082,
      "eval_runtime": 14.5043,
      "eval_samples_per_second": 28.405,
      "eval_steps_per_second": 3.585,
      "step": 850
    },
    {
      "epoch": 1.3484402589758682,
      "grad_norm": 4.474134922027588,
      "learning_rate": 1.9291228247233607e-06,
      "logits/chosen": 12.378435134887695,
      "logits/rejected": 11.085493087768555,
      "logps/chosen": -4.5196709632873535,
      "logps/rejected": -8.377789497375488,
      "loss": 0.2711,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -6.779506683349609,
      "rewards/margins": 5.787177085876465,
      "rewards/rejected": -12.566683769226074,
      "step": 860
    },
    {
      "epoch": 1.3641357661369433,
      "grad_norm": 3.2606918811798096,
      "learning_rate": 1.8782752820878636e-06,
      "logits/chosen": 11.961488723754883,
      "logits/rejected": 11.434579849243164,
      "logps/chosen": -4.525224208831787,
      "logps/rejected": -8.16456413269043,
      "loss": 0.2569,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.787835597991943,
      "rewards/margins": 5.459010124206543,
      "rewards/rejected": -12.246846199035645,
      "step": 870
    },
    {
      "epoch": 1.3798312732980185,
      "grad_norm": 3.297881603240967,
      "learning_rate": 1.827700448461836e-06,
      "logits/chosen": 12.268153190612793,
      "logits/rejected": 11.848055839538574,
      "logps/chosen": -3.979138135910034,
      "logps/rejected": -7.244141578674316,
      "loss": 0.2876,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.968707084655762,
      "rewards/margins": 4.897505760192871,
      "rewards/rejected": -10.866212844848633,
      "step": 880
    },
    {
      "epoch": 1.3955267804590936,
      "grad_norm": 5.355867862701416,
      "learning_rate": 1.7774205076388207e-06,
      "logits/chosen": 12.47008991241455,
      "logits/rejected": 11.850513458251953,
      "logps/chosen": -4.1252217292785645,
      "logps/rejected": -8.060781478881836,
      "loss": 0.2502,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.187832832336426,
      "rewards/margins": 5.9033403396606445,
      "rewards/rejected": -12.09117317199707,
      "step": 890
    },
    {
      "epoch": 1.4112222876201688,
      "grad_norm": 3.333230495452881,
      "learning_rate": 1.7274575140626318e-06,
      "logits/chosen": 12.16104793548584,
      "logits/rejected": 11.287874221801758,
      "logps/chosen": -3.827984571456909,
      "logps/rejected": -7.516768455505371,
      "loss": 0.2162,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.741976737976074,
      "rewards/margins": 5.533174991607666,
      "rewards/rejected": -11.275151252746582,
      "step": 900
    },
    {
      "epoch": 1.4112222876201688,
      "eval_logits/chosen": 12.950577735900879,
      "eval_logits/rejected": 11.89844799041748,
      "eval_logps/chosen": -4.105511665344238,
      "eval_logps/rejected": -8.293257713317871,
      "eval_loss": 0.2529076933860779,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -6.158267498016357,
      "eval_rewards/margins": 6.281619071960449,
      "eval_rewards/rejected": -12.439887046813965,
      "eval_runtime": 14.5023,
      "eval_samples_per_second": 28.409,
      "eval_steps_per_second": 3.586,
      "step": 900
    },
    {
      "epoch": 1.426917794781244,
      "grad_norm": 2.3288955688476562,
      "learning_rate": 1.677833383153542e-06,
      "logits/chosen": 12.872639656066895,
      "logits/rejected": 11.605780601501465,
      "logps/chosen": -4.383180141448975,
      "logps/rejected": -8.010499954223633,
      "loss": 0.212,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.574770927429199,
      "rewards/margins": 5.440978527069092,
      "rewards/rejected": -12.015748977661133,
      "step": 910
    },
    {
      "epoch": 1.442613301942319,
      "grad_norm": 3.773186206817627,
      "learning_rate": 1.6285698816954626e-06,
      "logits/chosen": 12.562664985656738,
      "logits/rejected": 11.542292594909668,
      "logps/chosen": -4.957005500793457,
      "logps/rejected": -9.080846786499023,
      "loss": 0.2343,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -7.435508728027344,
      "rewards/margins": 6.185763359069824,
      "rewards/rejected": -13.621271133422852,
      "step": 920
    },
    {
      "epoch": 1.4583088091033942,
      "grad_norm": 2.7567288875579834,
      "learning_rate": 1.5796886182883053e-06,
      "logits/chosen": 12.21378231048584,
      "logits/rejected": 11.380667686462402,
      "logps/chosen": -4.714855670928955,
      "logps/rejected": -8.239659309387207,
      "loss": 0.2487,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -7.072283744812012,
      "rewards/margins": 5.287206172943115,
      "rewards/rejected": -12.359490394592285,
      "step": 930
    },
    {
      "epoch": 1.4740043162644694,
      "grad_norm": 5.543617248535156,
      "learning_rate": 1.5312110338697427e-06,
      "logits/chosen": 11.844781875610352,
      "logits/rejected": 11.141448974609375,
      "logps/chosen": -4.98689079284668,
      "logps/rejected": -8.525138854980469,
      "loss": 0.2609,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -7.480336666107178,
      "rewards/margins": 5.307372570037842,
      "rewards/rejected": -12.78770923614502,
      "step": 940
    },
    {
      "epoch": 1.4896998234255445,
      "grad_norm": 4.39804220199585,
      "learning_rate": 1.4831583923105e-06,
      "logits/chosen": 12.365824699401855,
      "logits/rejected": 11.472888946533203,
      "logps/chosen": -4.707729816436768,
      "logps/rejected": -8.413005828857422,
      "loss": 0.2398,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -7.061594486236572,
      "rewards/margins": 5.557913780212402,
      "rewards/rejected": -12.619508743286133,
      "step": 950
    },
    {
      "epoch": 1.4896998234255445,
      "eval_logits/chosen": 12.97492790222168,
      "eval_logits/rejected": 11.935004234313965,
      "eval_logps/chosen": -4.231271743774414,
      "eval_logps/rejected": -8.475292205810547,
      "eval_loss": 0.24798962473869324,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -6.346907615661621,
      "eval_rewards/margins": 6.366030693054199,
      "eval_rewards/rejected": -12.71293830871582,
      "eval_runtime": 14.5034,
      "eval_samples_per_second": 28.407,
      "eval_steps_per_second": 3.585,
      "step": 950
    },
    {
      "epoch": 1.5053953305866195,
      "grad_norm": 2.8650925159454346,
      "learning_rate": 1.4355517710873184e-06,
      "logits/chosen": 12.443435668945312,
      "logits/rejected": 11.93461799621582,
      "logps/chosen": -4.258164405822754,
      "logps/rejected": -7.894736289978027,
      "loss": 0.1948,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.387246131896973,
      "rewards/margins": 5.454858779907227,
      "rewards/rejected": -11.8421049118042,
      "step": 960
    },
    {
      "epoch": 1.5210908377476948,
      "grad_norm": 3.400313377380371,
      "learning_rate": 1.388412052037682e-06,
      "logits/chosen": 12.361493110656738,
      "logits/rejected": 11.476273536682129,
      "logps/chosen": -4.457276344299316,
      "logps/rejected": -8.228893280029297,
      "loss": 0.1888,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.685914039611816,
      "rewards/margins": 5.657425880432129,
      "rewards/rejected": -12.343339920043945,
      "step": 970
    },
    {
      "epoch": 1.5367863449087698,
      "grad_norm": 3.8736472129821777,
      "learning_rate": 1.3417599122003464e-06,
      "logits/chosen": 12.806173324584961,
      "logits/rejected": 11.818658828735352,
      "logps/chosen": -4.827099800109863,
      "logps/rejected": -8.408552169799805,
      "loss": 0.231,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -7.240649223327637,
      "rewards/margins": 5.3721795082092285,
      "rewards/rejected": -12.612829208374023,
      "step": 980
    },
    {
      "epoch": 1.5524818520698451,
      "grad_norm": 3.6417574882507324,
      "learning_rate": 1.2956158147457116e-06,
      "logits/chosen": 12.428319931030273,
      "logits/rejected": 11.719505310058594,
      "logps/chosen": -5.147378444671631,
      "logps/rejected": -8.708709716796875,
      "loss": 0.2265,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -7.721067905426025,
      "rewards/margins": 5.341997146606445,
      "rewards/rejected": -13.06306266784668,
      "step": 990
    },
    {
      "epoch": 1.56817735923092,
      "grad_norm": 2.3367979526519775,
      "learning_rate": 1.2500000000000007e-06,
      "logits/chosen": 12.109533309936523,
      "logits/rejected": 11.499246597290039,
      "logps/chosen": -5.072439193725586,
      "logps/rejected": -9.098675727844238,
      "loss": 0.216,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -7.608658790588379,
      "rewards/margins": 6.039353847503662,
      "rewards/rejected": -13.648012161254883,
      "step": 1000
    },
    {
      "epoch": 1.56817735923092,
      "eval_logits/chosen": 12.952030181884766,
      "eval_logits/rejected": 11.900659561157227,
      "eval_logps/chosen": -4.205340385437012,
      "eval_logps/rejected": -8.569601058959961,
      "eval_loss": 0.2481767237186432,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -6.308011054992676,
      "eval_rewards/margins": 6.546389579772949,
      "eval_rewards/rejected": -12.854399681091309,
      "eval_runtime": 14.5049,
      "eval_samples_per_second": 28.404,
      "eval_steps_per_second": 3.585,
      "step": 1000
    },
    {
      "epoch": 1.5838728663919954,
      "grad_norm": 2.5456700325012207,
      "learning_rate": 1.204932476567175e-06,
      "logits/chosen": 11.960077285766602,
      "logits/rejected": 10.87277603149414,
      "logps/chosen": -4.433133125305176,
      "logps/rejected": -8.525662422180176,
      "loss": 0.2443,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.6496992111206055,
      "rewards/margins": 6.1387939453125,
      "rewards/rejected": -12.788493156433105,
      "step": 1010
    },
    {
      "epoch": 1.5995683735530704,
      "grad_norm": 2.834538698196411,
      "learning_rate": 1.160433012552508e-06,
      "logits/chosen": 12.570700645446777,
      "logits/rejected": 11.804948806762695,
      "logps/chosen": -3.698901414871216,
      "logps/rejected": -7.383004665374756,
      "loss": 0.2226,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.5483527183532715,
      "rewards/margins": 5.526154518127441,
      "rewards/rejected": -11.074506759643555,
      "step": 1020
    },
    {
      "epoch": 1.6152638807141457,
      "grad_norm": 3.336750030517578,
      "learning_rate": 1.11652112689164e-06,
      "logits/chosen": 12.501141548156738,
      "logits/rejected": 11.617207527160645,
      "logps/chosen": -4.3009490966796875,
      "logps/rejected": -8.17215347290039,
      "loss": 0.2303,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.451423645019531,
      "rewards/margins": 5.806807041168213,
      "rewards/rejected": -12.258230209350586,
      "step": 1030
    },
    {
      "epoch": 1.6309593878752207,
      "grad_norm": 2.519408941268921,
      "learning_rate": 1.073216080788921e-06,
      "logits/chosen": 12.614023208618164,
      "logits/rejected": 11.761343002319336,
      "logps/chosen": -4.564716815948486,
      "logps/rejected": -8.430013656616211,
      "loss": 0.2634,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -6.847075462341309,
      "rewards/margins": 5.797944068908691,
      "rewards/rejected": -12.645018577575684,
      "step": 1040
    },
    {
      "epoch": 1.6466548950362958,
      "grad_norm": 4.510939121246338,
      "learning_rate": 1.0305368692688175e-06,
      "logits/chosen": 12.286882400512695,
      "logits/rejected": 11.735586166381836,
      "logps/chosen": -4.190676212310791,
      "logps/rejected": -8.055787086486816,
      "loss": 0.1713,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.286014556884766,
      "rewards/margins": 5.797664642333984,
      "rewards/rejected": -12.08367919921875,
      "step": 1050
    },
    {
      "epoch": 1.6466548950362958,
      "eval_logits/chosen": 12.861144065856934,
      "eval_logits/rejected": 11.838491439819336,
      "eval_logps/chosen": -4.190127372741699,
      "eval_logps/rejected": -8.606769561767578,
      "eval_loss": 0.24250496923923492,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -6.285191535949707,
      "eval_rewards/margins": 6.624964237213135,
      "eval_rewards/rejected": -12.910155296325684,
      "eval_runtime": 14.4989,
      "eval_samples_per_second": 28.416,
      "eval_steps_per_second": 3.586,
      "step": 1050
    },
    {
      "epoch": 1.662350402197371,
      "grad_norm": 3.669426441192627,
      "learning_rate": 9.88502212844063e-07,
      "logits/chosen": 12.255383491516113,
      "logits/rejected": 11.495210647583008,
      "logps/chosen": -3.948782444000244,
      "logps/rejected": -8.229391098022461,
      "loss": 0.2143,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.923173427581787,
      "rewards/margins": 6.4209136962890625,
      "rewards/rejected": -12.344087600708008,
      "step": 1060
    },
    {
      "epoch": 1.678045909358446,
      "grad_norm": 4.005455493927002,
      "learning_rate": 9.471305493042243e-07,
      "logits/chosen": 12.1824312210083,
      "logits/rejected": 11.290962219238281,
      "logps/chosen": -4.578344821929932,
      "logps/rejected": -8.505768775939941,
      "loss": 0.2361,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.867516994476318,
      "rewards/margins": 5.89113712310791,
      "rewards/rejected": -12.75865364074707,
      "step": 1070
    },
    {
      "epoch": 1.6937414165195213,
      "grad_norm": 2.032545566558838,
      "learning_rate": 9.064400256282757e-07,
      "logits/chosen": 12.451539039611816,
      "logits/rejected": 11.746088027954102,
      "logps/chosen": -4.1952643394470215,
      "logps/rejected": -7.86816930770874,
      "loss": 0.1957,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.2928972244262695,
      "rewards/margins": 5.50935697555542,
      "rewards/rejected": -11.802255630493164,
      "step": 1080
    },
    {
      "epoch": 1.7094369236805964,
      "grad_norm": 4.466379642486572,
      "learning_rate": 8.664484900247363e-07,
      "logits/chosen": 12.144006729125977,
      "logits/rejected": 11.437246322631836,
      "logps/chosen": -4.593950271606445,
      "logps/rejected": -8.530038833618164,
      "loss": 0.186,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.890925407409668,
      "rewards/margins": 5.9041337966918945,
      "rewards/rejected": -12.795059204101562,
      "step": 1090
    },
    {
      "epoch": 1.7251324308416716,
      "grad_norm": 2.941102981567383,
      "learning_rate": 8.271734841028553e-07,
      "logits/chosen": 12.305268287658691,
      "logits/rejected": 11.327041625976562,
      "logps/chosen": -4.72153377532959,
      "logps/rejected": -8.988466262817383,
      "loss": 0.2264,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -7.082299709320068,
      "rewards/margins": 6.400399684906006,
      "rewards/rejected": -13.482701301574707,
      "step": 1100
    },
    {
      "epoch": 1.7251324308416716,
      "eval_logits/chosen": 12.896148681640625,
      "eval_logits/rejected": 11.868448257446289,
      "eval_logps/chosen": -4.222008228302002,
      "eval_logps/rejected": -8.642206192016602,
      "eval_loss": 0.2400711625814438,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -6.333012580871582,
      "eval_rewards/margins": 6.6302971839904785,
      "eval_rewards/rejected": -12.963308334350586,
      "eval_runtime": 14.5019,
      "eval_samples_per_second": 28.41,
      "eval_steps_per_second": 3.586,
      "step": 1100
    },
    {
      "epoch": 1.7408279380027467,
      "grad_norm": 4.001830101013184,
      "learning_rate": 7.886322351782782e-07,
      "logits/chosen": 12.858680725097656,
      "logits/rejected": 11.653299331665039,
      "logps/chosen": -4.239178657531738,
      "logps/rejected": -8.762260437011719,
      "loss": 0.2786,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.358767986297607,
      "rewards/margins": 6.784623622894287,
      "rewards/rejected": -13.143391609191895,
      "step": 1110
    },
    {
      "epoch": 1.7565234451638219,
      "grad_norm": 2.9689064025878906,
      "learning_rate": 7.508416487165862e-07,
      "logits/chosen": 12.076465606689453,
      "logits/rejected": 11.392438888549805,
      "logps/chosen": -4.560977458953857,
      "logps/rejected": -8.554190635681152,
      "loss": 0.1998,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.841466426849365,
      "rewards/margins": 5.98982048034668,
      "rewards/rejected": -12.831286430358887,
      "step": 1120
    },
    {
      "epoch": 1.772218952324897,
      "grad_norm": 1.6012306213378906,
      "learning_rate": 7.138183009179922e-07,
      "logits/chosen": 12.253329277038574,
      "logits/rejected": 11.584481239318848,
      "logps/chosen": -4.0213422775268555,
      "logps/rejected": -8.13912296295166,
      "loss": 0.1823,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.032013893127441,
      "rewards/margins": 6.176670074462891,
      "rewards/rejected": -12.208683967590332,
      "step": 1130
    },
    {
      "epoch": 1.7879144594859722,
      "grad_norm": 3.8666510581970215,
      "learning_rate": 6.775784314464717e-07,
      "logits/chosen": 11.831202507019043,
      "logits/rejected": 11.49097728729248,
      "logps/chosen": -4.555119037628174,
      "logps/rejected": -8.43702220916748,
      "loss": 0.2494,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.832678318023682,
      "rewards/margins": 5.822856426239014,
      "rewards/rejected": -12.655534744262695,
      "step": 1140
    },
    {
      "epoch": 1.8036099666470473,
      "grad_norm": 3.794703483581543,
      "learning_rate": 6.421379363065142e-07,
      "logits/chosen": 12.098966598510742,
      "logits/rejected": 11.32978343963623,
      "logps/chosen": -4.038215637207031,
      "logps/rejected": -7.658058166503906,
      "loss": 0.2365,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.057322978973389,
      "rewards/margins": 5.4297637939453125,
      "rewards/rejected": -11.487086296081543,
      "step": 1150
    },
    {
      "epoch": 1.8036099666470473,
      "eval_logits/chosen": 12.860236167907715,
      "eval_logits/rejected": 11.83856201171875,
      "eval_logps/chosen": -4.173120498657227,
      "eval_logps/rejected": -8.646206855773926,
      "eval_loss": 0.23765961825847626,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -6.2596821784973145,
      "eval_rewards/margins": 6.709629535675049,
      "eval_rewards/rejected": -12.96930980682373,
      "eval_runtime": 14.5141,
      "eval_samples_per_second": 28.386,
      "eval_steps_per_second": 3.583,
      "step": 1150
    },
    {
      "epoch": 1.8193054738081225,
      "grad_norm": 4.030630588531494,
      "learning_rate": 6.075123608706093e-07,
      "logits/chosen": 12.267806053161621,
      "logits/rejected": 11.600652694702148,
      "logps/chosen": -4.568865776062012,
      "logps/rejected": -7.858088493347168,
      "loss": 0.2043,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.853298187255859,
      "rewards/margins": 4.933834552764893,
      "rewards/rejected": -11.78713321685791,
      "step": 1160
    },
    {
      "epoch": 1.8350009809691976,
      "grad_norm": 3.0004727840423584,
      "learning_rate": 5.737168930605272e-07,
      "logits/chosen": 12.178464889526367,
      "logits/rejected": 11.31472110748291,
      "logps/chosen": -5.0792646408081055,
      "logps/rejected": -8.85279369354248,
      "loss": 0.1923,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -7.618897438049316,
      "rewards/margins": 5.6602935791015625,
      "rewards/rejected": -13.279190063476562,
      "step": 1170
    },
    {
      "epoch": 1.8506964881302728,
      "grad_norm": 3.7206344604492188,
      "learning_rate": 5.407663566854008e-07,
      "logits/chosen": 12.098569869995117,
      "logits/rejected": 11.365894317626953,
      "logps/chosen": -5.336371898651123,
      "logps/rejected": -9.606247901916504,
      "loss": 0.2206,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -8.004558563232422,
      "rewards/margins": 6.404813289642334,
      "rewards/rejected": -14.409372329711914,
      "step": 1180
    },
    {
      "epoch": 1.8663919952913477,
      "grad_norm": 4.599307537078857,
      "learning_rate": 5.086752049395094e-07,
      "logits/chosen": 12.062299728393555,
      "logits/rejected": 11.360358238220215,
      "logps/chosen": -5.103975772857666,
      "logps/rejected": -9.448270797729492,
      "loss": 0.2338,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -7.655962944030762,
      "rewards/margins": 6.516444206237793,
      "rewards/rejected": -14.172407150268555,
      "step": 1190
    },
    {
      "epoch": 1.882087502452423,
      "grad_norm": 2.590059757232666,
      "learning_rate": 4.774575140626317e-07,
      "logits/chosen": 12.54432201385498,
      "logits/rejected": 11.949193000793457,
      "logps/chosen": -4.637765884399414,
      "logps/rejected": -8.537626266479492,
      "loss": 0.2028,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.956648826599121,
      "rewards/margins": 5.849791526794434,
      "rewards/rejected": -12.806440353393555,
      "step": 1200
    },
    {
      "epoch": 1.882087502452423,
      "eval_logits/chosen": 12.827628135681152,
      "eval_logits/rejected": 11.809093475341797,
      "eval_logps/chosen": -4.21560001373291,
      "eval_logps/rejected": -8.739953994750977,
      "eval_loss": 0.23520438373088837,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -6.323400497436523,
      "eval_rewards/margins": 6.786531448364258,
      "eval_rewards/rejected": -13.109930992126465,
      "eval_runtime": 14.5066,
      "eval_samples_per_second": 28.401,
      "eval_steps_per_second": 3.585,
      "step": 1200
    }
  ],
  "logging_steps": 10,
  "max_steps": 1500,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 2.90612601452023e+18,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}