{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.0,
  "eval_steps": 20,
  "global_step": 206,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.01,
      "grad_norm": 8.9375,
      "learning_rate": 2.3809523809523808e-06,
      "logits/chosen": -2.7700600624084473,
      "logits/rejected": -2.8606302738189697,
      "logps/chosen": -421.64996337890625,
      "logps/rejected": -531.4378662109375,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.1,
      "grad_norm": 7.375,
      "learning_rate": 2.380952380952381e-05,
      "logits/chosen": -2.7684054374694824,
      "logits/rejected": -2.7337145805358887,
      "logps/chosen": -333.7870178222656,
      "logps/rejected": -312.4859313964844,
      "loss": 0.6852,
      "rewards/accuracies": 0.5277777910232544,
      "rewards/chosen": 0.030088074505329132,
      "rewards/margins": 0.01666567102074623,
      "rewards/rejected": 0.013422403484582901,
      "step": 10
    },
    {
      "epoch": 0.19,
      "grad_norm": 6.125,
      "learning_rate": 4.761904761904762e-05,
      "logits/chosen": -2.8010494709014893,
      "logits/rejected": -2.79127836227417,
      "logps/chosen": -331.8260498046875,
      "logps/rejected": -332.01409912109375,
      "loss": 0.6028,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": 0.41256317496299744,
      "rewards/margins": 0.21184520423412323,
      "rewards/rejected": 0.2007180005311966,
      "step": 20
    },
    {
      "epoch": 0.19,
      "eval_logits/chosen": -2.6401147842407227,
      "eval_logits/rejected": -2.614283800125122,
      "eval_logps/chosen": -324.9909973144531,
      "eval_logps/rejected": -327.9555969238281,
      "eval_loss": 0.5285959243774414,
      "eval_rewards/accuracies": 0.8125,
      "eval_rewards/chosen": 0.878866970539093,
      "eval_rewards/margins": 0.43177998065948486,
      "eval_rewards/rejected": 0.44708704948425293,
      "eval_runtime": 114.9886,
      "eval_samples_per_second": 1.6,
      "eval_steps_per_second": 0.104,
      "step": 20
    },
    {
      "epoch": 0.29,
      "grad_norm": 4.84375,
      "learning_rate": 4.9708589101037306e-05,
      "logits/chosen": -2.6763195991516113,
      "logits/rejected": -2.651015043258667,
      "logps/chosen": -356.8539733886719,
      "logps/rejected": -363.6021423339844,
      "loss": 0.4643,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.943565845489502,
      "rewards/margins": 0.708370566368103,
      "rewards/rejected": 0.23519524931907654,
      "step": 30
    },
    {
      "epoch": 0.39,
      "grad_norm": 5.75,
      "learning_rate": 4.870996167038154e-05,
      "logits/chosen": -2.655568838119507,
      "logits/rejected": -2.6175591945648193,
      "logps/chosen": -353.34619140625,
      "logps/rejected": -359.96832275390625,
      "loss": 0.3363,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.6268302798271179,
      "rewards/margins": 1.4118897914886475,
      "rewards/rejected": -0.7850595712661743,
      "step": 40
    },
    {
      "epoch": 0.39,
      "eval_logits/chosen": -2.535161256790161,
      "eval_logits/rejected": -2.5076351165771484,
      "eval_logps/chosen": -328.5650939941406,
      "eval_logps/rejected": -343.5235900878906,
      "eval_loss": 0.32319265604019165,
      "eval_rewards/accuracies": 0.859375,
      "eval_rewards/chosen": 0.5214586853981018,
      "eval_rewards/margins": 1.6311697959899902,
      "eval_rewards/rejected": -1.1097110509872437,
      "eval_runtime": 114.9563,
      "eval_samples_per_second": 1.601,
      "eval_steps_per_second": 0.104,
      "step": 40
    },
    {
      "epoch": 0.49,
      "grad_norm": 4.4375,
      "learning_rate": 4.7029241811087457e-05,
      "logits/chosen": -2.682722806930542,
      "logits/rejected": -2.627808094024658,
      "logps/chosen": -382.26690673828125,
      "logps/rejected": -376.25689697265625,
      "loss": 0.3043,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3637928366661072,
      "rewards/margins": 1.71381413936615,
      "rewards/rejected": -1.3500211238861084,
      "step": 50
    },
    {
      "epoch": 0.58,
      "grad_norm": 5.875,
      "learning_rate": 4.471478077342798e-05,
      "logits/chosen": -2.6791253089904785,
      "logits/rejected": -2.641322374343872,
      "logps/chosen": -344.8480529785156,
      "logps/rejected": -372.0831298828125,
      "loss": 0.2458,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.6151469349861145,
      "rewards/margins": 2.2607688903808594,
      "rewards/rejected": -1.6456218957901,
      "step": 60
    },
    {
      "epoch": 0.58,
      "eval_logits/chosen": -2.592442274093628,
      "eval_logits/rejected": -2.560177803039551,
      "eval_logps/chosen": -328.04132080078125,
      "eval_logps/rejected": -351.1114196777344,
      "eval_loss": 0.2501268982887268,
      "eval_rewards/accuracies": 0.9114583134651184,
      "eval_rewards/chosen": 0.5738345980644226,
      "eval_rewards/margins": 2.4423279762268066,
      "eval_rewards/rejected": -1.8684934377670288,
      "eval_runtime": 115.0094,
      "eval_samples_per_second": 1.6,
      "eval_steps_per_second": 0.104,
      "step": 60
    },
    {
      "epoch": 0.68,
      "grad_norm": 4.3125,
      "learning_rate": 4.1833161387527986e-05,
      "logits/chosen": -2.6796765327453613,
      "logits/rejected": -2.6534857749938965,
      "logps/chosen": -371.1224670410156,
      "logps/rejected": -358.3480529785156,
      "loss": 0.2487,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.5859086513519287,
      "rewards/margins": 2.399681329727173,
      "rewards/rejected": -3.9855899810791016,
      "step": 70
    },
    {
      "epoch": 0.78,
      "grad_norm": 4.4375,
      "learning_rate": 3.84672825965686e-05,
      "logits/chosen": -2.568530559539795,
      "logits/rejected": -2.5246570110321045,
      "logps/chosen": -354.64984130859375,
      "logps/rejected": -360.8916931152344,
      "loss": 0.2116,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.9132896661758423,
      "rewards/margins": 2.766739845275879,
      "rewards/rejected": -3.680030107498169,
      "step": 80
    },
    {
      "epoch": 0.78,
      "eval_logits/chosen": -2.3427236080169678,
      "eval_logits/rejected": -2.3129446506500244,
      "eval_logps/chosen": -340.5351257324219,
      "eval_logps/rejected": -370.7005920410156,
      "eval_loss": 0.19913233816623688,
      "eval_rewards/accuracies": 0.9166666865348816,
      "eval_rewards/chosen": -0.6755423545837402,
      "eval_rewards/margins": 3.1518704891204834,
      "eval_rewards/rejected": -3.8274126052856445,
      "eval_runtime": 114.9725,
      "eval_samples_per_second": 1.6,
      "eval_steps_per_second": 0.104,
      "step": 80
    },
    {
      "epoch": 0.87,
      "grad_norm": 2.0,
      "learning_rate": 3.471397460512563e-05,
      "logits/chosen": -2.428431272506714,
      "logits/rejected": -2.410618782043457,
      "logps/chosen": -361.0326232910156,
      "logps/rejected": -401.57269287109375,
      "loss": 0.1841,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.7255961298942566,
      "rewards/margins": 3.4665279388427734,
      "rewards/rejected": -4.192124366760254,
      "step": 90
    },
    {
      "epoch": 0.97,
      "grad_norm": 5.4375,
      "learning_rate": 3.0681213250482255e-05,
      "logits/chosen": -2.3709776401519775,
      "logits/rejected": -2.353501796722412,
      "logps/chosen": -331.63623046875,
      "logps/rejected": -377.9563903808594,
      "loss": 0.1386,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.6168140172958374,
      "rewards/margins": 3.699888229370117,
      "rewards/rejected": -4.316702365875244,
      "step": 100
    },
    {
      "epoch": 0.97,
      "eval_logits/chosen": -2.3535118103027344,
      "eval_logits/rejected": -2.3131775856018066,
      "eval_logps/chosen": -330.8599548339844,
      "eval_logps/rejected": -362.6181335449219,
      "eval_loss": 0.20019526779651642,
      "eval_rewards/accuracies": 0.9375,
      "eval_rewards/chosen": 0.29197368025779724,
      "eval_rewards/margins": 3.311133623123169,
      "eval_rewards/rejected": -3.019160032272339,
      "eval_runtime": 114.9013,
      "eval_samples_per_second": 1.601,
      "eval_steps_per_second": 0.104,
      "step": 100
    },
    {
      "epoch": 1.07,
      "grad_norm": 0.76953125,
      "learning_rate": 2.648501373438142e-05,
      "logits/chosen": -2.4437708854675293,
      "logits/rejected": -2.4320626258850098,
      "logps/chosen": -344.7204895019531,
      "logps/rejected": -410.47601318359375,
      "loss": 0.0711,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.051882706582546234,
      "rewards/margins": 5.166382789611816,
      "rewards/rejected": -5.218265533447266,
      "step": 110
    },
    {
      "epoch": 1.17,
      "grad_norm": 1.4375,
      "learning_rate": 2.2246093076900144e-05,
      "logits/chosen": -2.430386781692505,
      "logits/rejected": -2.34106183052063,
      "logps/chosen": -400.32452392578125,
      "logps/rejected": -425.37457275390625,
      "loss": 0.0458,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.0012158155441284,
      "rewards/margins": 6.266473293304443,
      "rewards/rejected": -7.2676897048950195,
      "step": 120
    },
    {
      "epoch": 1.17,
      "eval_logits/chosen": -2.2717294692993164,
      "eval_logits/rejected": -2.2290468215942383,
      "eval_logps/chosen": -347.5820007324219,
      "eval_logps/rejected": -391.1982727050781,
      "eval_loss": 0.17477566003799438,
      "eval_rewards/accuracies": 0.9479166865348816,
      "eval_rewards/chosen": -1.3802350759506226,
      "eval_rewards/margins": 4.496945858001709,
      "eval_rewards/rejected": -5.877180576324463,
      "eval_runtime": 114.9627,
      "eval_samples_per_second": 1.601,
      "eval_steps_per_second": 0.104,
      "step": 120
    },
    {
      "epoch": 1.26,
      "grad_norm": 1.0546875,
      "learning_rate": 1.8086397307570723e-05,
      "logits/chosen": -2.376091957092285,
      "logits/rejected": -2.3415114879608154,
      "logps/chosen": -337.0244140625,
      "logps/rejected": -408.39263916015625,
      "loss": 0.0283,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.006544408388435841,
      "rewards/margins": 6.594322204589844,
      "rewards/rejected": -6.600866794586182,
      "step": 130
    },
    {
      "epoch": 1.36,
      "grad_norm": 1.3515625,
      "learning_rate": 1.4125593300137766e-05,
      "logits/chosen": -2.364224672317505,
      "logits/rejected": -2.310997724533081,
      "logps/chosen": -343.6619567871094,
      "logps/rejected": -398.0953674316406,
      "loss": 0.0426,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": 0.7415364384651184,
      "rewards/margins": 6.200386047363281,
      "rewards/rejected": -5.4588494300842285,
      "step": 140
    },
    {
      "epoch": 1.36,
      "eval_logits/chosen": -2.240306854248047,
      "eval_logits/rejected": -2.195923328399658,
      "eval_logps/chosen": -334.4142761230469,
      "eval_logps/rejected": -375.5160217285156,
      "eval_loss": 0.17553412914276123,
      "eval_rewards/accuracies": 0.9375,
      "eval_rewards/chosen": -0.06346017122268677,
      "eval_rewards/margins": 4.2454915046691895,
      "eval_rewards/rejected": -4.3089518547058105,
      "eval_runtime": 115.021,
      "eval_samples_per_second": 1.6,
      "eval_steps_per_second": 0.104,
      "step": 140
    },
    {
      "epoch": 1.46,
      "grad_norm": 0.8515625,
      "learning_rate": 1.0477626174477404e-05,
      "logits/chosen": -2.3424103260040283,
      "logits/rejected": -2.2978971004486084,
      "logps/chosen": -347.8079528808594,
      "logps/rejected": -375.98260498046875,
      "loss": 0.0508,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.3227364122867584,
      "rewards/margins": 5.873551845550537,
      "rewards/rejected": -5.550815582275391,
      "step": 150
    },
    {
      "epoch": 1.55,
      "grad_norm": 1.96875,
      "learning_rate": 7.247441302957858e-06,
      "logits/chosen": -2.3495125770568848,
      "logits/rejected": -2.307555675506592,
      "logps/chosen": -331.29718017578125,
      "logps/rejected": -415.452392578125,
      "loss": 0.029,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": 0.16269809007644653,
      "rewards/margins": 6.475255012512207,
      "rewards/rejected": -6.312556266784668,
      "step": 160
    },
    {
      "epoch": 1.55,
      "eval_logits/chosen": -2.232851266860962,
      "eval_logits/rejected": -2.1893069744110107,
      "eval_logps/chosen": -341.7696533203125,
      "eval_logps/rejected": -387.3076171875,
      "eval_loss": 0.16915130615234375,
      "eval_rewards/accuracies": 0.9375,
      "eval_rewards/chosen": -0.7989979386329651,
      "eval_rewards/margins": 4.689115524291992,
      "eval_rewards/rejected": -5.4881134033203125,
      "eval_runtime": 114.9918,
      "eval_samples_per_second": 1.6,
      "eval_steps_per_second": 0.104,
      "step": 160
    },
    {
      "epoch": 1.65,
      "grad_norm": 0.345703125,
      "learning_rate": 4.527965223149957e-06,
      "logits/chosen": -2.407200336456299,
      "logits/rejected": -2.3430123329162598,
      "logps/chosen": -387.9550476074219,
      "logps/rejected": -445.9234313964844,
      "loss": 0.0175,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": 0.19500017166137695,
      "rewards/margins": 7.179248809814453,
      "rewards/rejected": -6.984248161315918,
      "step": 170
    },
    {
      "epoch": 1.75,
      "grad_norm": 3.015625,
      "learning_rate": 2.397432310532133e-06,
      "logits/chosen": -2.3570303916931152,
      "logits/rejected": -2.300320863723755,
      "logps/chosen": -367.35577392578125,
      "logps/rejected": -424.9029235839844,
      "loss": 0.0676,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": 0.1322220414876938,
      "rewards/margins": 6.755249977111816,
      "rewards/rejected": -6.623027801513672,
      "step": 180
    },
    {
      "epoch": 1.75,
      "eval_logits/chosen": -2.2314395904541016,
      "eval_logits/rejected": -2.1864326000213623,
      "eval_logps/chosen": -340.7237854003906,
      "eval_logps/rejected": -386.9397277832031,
      "eval_loss": 0.16764594614505768,
      "eval_rewards/accuracies": 0.9375,
      "eval_rewards/chosen": -0.6944115161895752,
      "eval_rewards/margins": 4.756911754608154,
      "eval_rewards/rejected": -5.45132303237915,
      "eval_runtime": 114.865,
      "eval_samples_per_second": 1.602,
      "eval_steps_per_second": 0.104,
      "step": 180
    },
    {
      "epoch": 1.84,
      "grad_norm": 0.56640625,
      "learning_rate": 9.171341179489034e-07,
      "logits/chosen": -2.3660504817962646,
      "logits/rejected": -2.2959539890289307,
      "logps/chosen": -335.60052490234375,
      "logps/rejected": -383.60040283203125,
      "loss": 0.0178,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19160650670528412,
      "rewards/margins": 6.720318794250488,
      "rewards/rejected": -6.528712272644043,
      "step": 190
    },
    {
      "epoch": 1.94,
      "grad_norm": 3.9375,
      "learning_rate": 1.296561292287446e-07,
      "logits/chosen": -2.3115243911743164,
      "logits/rejected": -2.281430959701538,
      "logps/chosen": -323.0104675292969,
      "logps/rejected": -385.94757080078125,
      "loss": 0.0517,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.006203270051628351,
      "rewards/margins": 6.517538547515869,
      "rewards/rejected": -6.523741722106934,
      "step": 200
    },
    {
      "epoch": 1.94,
      "eval_logits/chosen": -2.231421709060669,
      "eval_logits/rejected": -2.186391830444336,
      "eval_logps/chosen": -341.20733642578125,
      "eval_logps/rejected": -387.5655517578125,
      "eval_loss": 0.16659200191497803,
      "eval_rewards/accuracies": 0.9375,
      "eval_rewards/chosen": -0.7427660822868347,
      "eval_rewards/margins": 4.771137237548828,
      "eval_rewards/rejected": -5.513904094696045,
      "eval_runtime": 114.34,
      "eval_samples_per_second": 1.609,
      "eval_steps_per_second": 0.105,
      "step": 200
    },
    {
      "epoch": 2.0,
      "step": 206,
      "total_flos": 0.0,
      "train_loss": 0.1882365908726905,
      "train_runtime": 5068.0756,
      "train_samples_per_second": 0.65,
      "train_steps_per_second": 0.041
    }
  ],
  "logging_steps": 10,
  "max_steps": 206,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 500,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}