trainer_state.json

{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 478,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0020920502092050207,
      "grad_norm": 9.102261763693875,
      "learning_rate": 1.0416666666666666e-08,
      "logits/chosen": -2.598843574523926,
      "logits/rejected": -2.5780863761901855,
      "logps/chosen": -269.870849609375,
      "logps/rejected": -361.686767578125,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.02092050209205021,
      "grad_norm": 9.071970452275611,
      "learning_rate": 1.0416666666666667e-07,
      "logits/chosen": -2.4106454849243164,
      "logits/rejected": -2.3727071285247803,
      "logps/chosen": -269.8708801269531,
      "logps/rejected": -254.9169158935547,
      "loss": 0.6932,
      "rewards/accuracies": 0.4236111044883728,
      "rewards/chosen": 3.945813296013512e-05,
      "rewards/margins": -0.00021239757188595831,
      "rewards/rejected": 0.0002518556430004537,
      "step": 10
    },
    {
      "epoch": 0.04184100418410042,
      "grad_norm": 10.775263417671352,
      "learning_rate": 2.0833333333333333e-07,
      "logits/chosen": -2.4717860221862793,
      "logits/rejected": -2.407602310180664,
      "logps/chosen": -284.3338928222656,
      "logps/rejected": -297.76251220703125,
      "loss": 0.6925,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.004285941831767559,
      "rewards/margins": 0.0006065635825507343,
      "rewards/rejected": 0.00367937795817852,
      "step": 20
    },
    {
      "epoch": 0.06276150627615062,
      "grad_norm": 7.642238787668073,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": -2.5130527019500732,
      "logits/rejected": -2.4189674854278564,
      "logps/chosen": -302.9613952636719,
      "logps/rejected": -263.80633544921875,
      "loss": 0.6894,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.024250635877251625,
      "rewards/margins": 0.0054460749961435795,
      "rewards/rejected": 0.018804559484124184,
      "step": 30
    },
    {
      "epoch": 0.08368200836820083,
      "grad_norm": 7.769592061468885,
      "learning_rate": 4.1666666666666667e-07,
      "logits/chosen": -2.4025068283081055,
      "logits/rejected": -2.3595290184020996,
      "logps/chosen": -268.82928466796875,
      "logps/rejected": -243.6490936279297,
      "loss": 0.6827,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.050604142248630524,
      "rewards/margins": 0.028540844097733498,
      "rewards/rejected": 0.022063296288251877,
      "step": 40
    },
    {
      "epoch": 0.10460251046025104,
      "grad_norm": 7.836365186344556,
      "learning_rate": 4.999733114418725e-07,
      "logits/chosen": -2.3352060317993164,
      "logits/rejected": -2.3020994663238525,
      "logps/chosen": -283.0472717285156,
      "logps/rejected": -297.33740234375,
      "loss": 0.6733,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.014853554777801037,
      "rewards/margins": 0.029812458902597427,
      "rewards/rejected": -0.014958900399506092,
      "step": 50
    },
    {
      "epoch": 0.12552301255230125,
      "grad_norm": 15.966435105260237,
      "learning_rate": 4.990398100856366e-07,
      "logits/chosen": -2.396578311920166,
      "logits/rejected": -2.329003095626831,
      "logps/chosen": -274.9017333984375,
      "logps/rejected": -308.89105224609375,
      "loss": 0.6527,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.003318754257634282,
      "rewards/margins": 0.09059403091669083,
      "rewards/rejected": -0.08727528154850006,
      "step": 60
    },
    {
      "epoch": 0.14644351464435146,
      "grad_norm": 9.243487328642475,
      "learning_rate": 4.967775735898179e-07,
      "logits/chosen": -2.244589328765869,
      "logits/rejected": -2.24755859375,
      "logps/chosen": -278.61004638671875,
      "logps/rejected": -280.5500183105469,
      "loss": 0.6348,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.08607344329357147,
      "rewards/margins": 0.11931805312633514,
      "rewards/rejected": -0.20539149641990662,
      "step": 70
    },
    {
      "epoch": 0.16736401673640167,
      "grad_norm": 14.953704290427632,
      "learning_rate": 4.931986719649298e-07,
      "logits/chosen": -2.3784019947052,
      "logits/rejected": -2.2930545806884766,
      "logps/chosen": -330.93719482421875,
      "logps/rejected": -288.47344970703125,
      "loss": 0.6192,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.03501243516802788,
      "rewards/margins": 0.18967404961585999,
      "rewards/rejected": -0.22468650341033936,
      "step": 80
    },
    {
      "epoch": 0.18828451882845187,
      "grad_norm": 15.867488807966232,
      "learning_rate": 4.883222001996351e-07,
      "logits/chosen": -2.0707411766052246,
      "logits/rejected": -1.9940516948699951,
      "logps/chosen": -288.53009033203125,
      "logps/rejected": -311.29840087890625,
      "loss": 0.6004,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.2625655233860016,
      "rewards/margins": 0.3136993646621704,
      "rewards/rejected": -0.5762647986412048,
      "step": 90
    },
    {
      "epoch": 0.20920502092050208,
      "grad_norm": 13.25483251042058,
      "learning_rate": 4.821741763807186e-07,
      "logits/chosen": -1.913400411605835,
      "logits/rejected": -1.7971922159194946,
      "logps/chosen": -292.3438415527344,
      "logps/rejected": -316.77410888671875,
      "loss": 0.5999,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.2575649917125702,
      "rewards/margins": 0.323893278837204,
      "rewards/rejected": -0.5814582705497742,
      "step": 100
    },
    {
      "epoch": 0.20920502092050208,
      "eval_logits/chosen": -1.794881820678711,
      "eval_logits/rejected": -1.6762588024139404,
      "eval_logps/chosen": -302.7283935546875,
      "eval_logps/rejected": -339.30548095703125,
      "eval_loss": 0.5966345071792603,
      "eval_rewards/accuracies": 0.7109375,
      "eval_rewards/chosen": -0.37388285994529724,
      "eval_rewards/margins": 0.371126264333725,
      "eval_rewards/rejected": -0.7450091242790222,
      "eval_runtime": 53.5348,
      "eval_samples_per_second": 37.359,
      "eval_steps_per_second": 0.598,
      "step": 100
    },
    {
      "epoch": 0.2301255230125523,
      "grad_norm": 15.2172693582291,
      "learning_rate": 4.747874028753375e-07,
      "logits/chosen": -1.628220796585083,
      "logits/rejected": -1.3428412675857544,
      "logps/chosen": -351.1053466796875,
      "logps/rejected": -339.97393798828125,
      "loss": 0.5893,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.41141095757484436,
      "rewards/margins": 0.3850555717945099,
      "rewards/rejected": -0.7964665293693542,
      "step": 110
    },
    {
      "epoch": 0.2510460251046025,
      "grad_norm": 15.83081907349326,
      "learning_rate": 4.662012913161997e-07,
      "logits/chosen": -0.7602194547653198,
      "logits/rejected": -0.5894412994384766,
      "logps/chosen": -322.77508544921875,
      "logps/rejected": -345.4307861328125,
      "loss": 0.5779,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3538314998149872,
      "rewards/margins": 0.45046257972717285,
      "rewards/rejected": -0.8042941093444824,
      "step": 120
    },
    {
      "epoch": 0.2719665271966527,
      "grad_norm": 15.055402351260229,
      "learning_rate": 4.5646165232345103e-07,
      "logits/chosen": -0.357295423746109,
      "logits/rejected": -0.0887320414185524,
      "logps/chosen": -332.0826110839844,
      "logps/rejected": -368.31256103515625,
      "loss": 0.549,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.441201388835907,
      "rewards/margins": 0.5332251787185669,
      "rewards/rejected": -0.9744265675544739,
      "step": 130
    },
    {
      "epoch": 0.2928870292887029,
      "grad_norm": 18.351529760698416,
      "learning_rate": 4.456204510851956e-07,
      "logits/chosen": -0.05173628777265549,
      "logits/rejected": 0.2410668432712555,
      "logps/chosen": -356.73333740234375,
      "logps/rejected": -390.5004577636719,
      "loss": 0.5555,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.37852978706359863,
      "rewards/margins": 0.49712079763412476,
      "rewards/rejected": -0.8756505846977234,
      "step": 140
    },
    {
      "epoch": 0.3138075313807531,
      "grad_norm": 22.63597752112208,
      "learning_rate": 4.337355301007335e-07,
      "logits/chosen": 0.6089886426925659,
      "logits/rejected": 0.9238691329956055,
      "logps/chosen": -365.7376403808594,
      "logps/rejected": -402.1335144042969,
      "loss": 0.5568,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.7808293104171753,
      "rewards/margins": 0.449969619512558,
      "rewards/rejected": -1.2307989597320557,
      "step": 150
    },
    {
      "epoch": 0.33472803347280333,
      "grad_norm": 20.03187629205258,
      "learning_rate": 4.2087030056579986e-07,
      "logits/chosen": 0.2681460976600647,
      "logits/rejected": 0.93768310546875,
      "logps/chosen": -325.05755615234375,
      "logps/rejected": -369.85015869140625,
      "loss": 0.5599,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4498773515224457,
      "rewards/margins": 0.700762927532196,
      "rewards/rejected": -1.1506402492523193,
      "step": 160
    },
    {
      "epoch": 0.35564853556485354,
      "grad_norm": 25.495361660969547,
      "learning_rate": 4.070934040463998e-07,
      "logits/chosen": 0.556589663028717,
      "logits/rejected": 0.9764513969421387,
      "logps/chosen": -326.57281494140625,
      "logps/rejected": -349.85894775390625,
      "loss": 0.5672,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.7536547780036926,
      "rewards/margins": 0.4030492901802063,
      "rewards/rejected": -1.1567041873931885,
      "step": 170
    },
    {
      "epoch": 0.37656903765690375,
      "grad_norm": 24.47540687241319,
      "learning_rate": 3.9247834624635404e-07,
      "logits/chosen": 1.273604393005371,
      "logits/rejected": 1.7295089960098267,
      "logps/chosen": -344.38751220703125,
      "logps/rejected": -365.9759521484375,
      "loss": 0.5254,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.8270662426948547,
      "rewards/margins": 0.5501136779785156,
      "rewards/rejected": -1.3771798610687256,
      "step": 180
    },
    {
      "epoch": 0.39748953974895396,
      "grad_norm": 22.91459265715999,
      "learning_rate": 3.7710310482256523e-07,
      "logits/chosen": 1.0068578720092773,
      "logits/rejected": 1.2909595966339111,
      "logps/chosen": -328.5067138671875,
      "logps/rejected": -372.14324951171875,
      "loss": 0.5444,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.5530312061309814,
      "rewards/margins": 0.5563492178916931,
      "rewards/rejected": -1.1093804836273193,
      "step": 190
    },
    {
      "epoch": 0.41841004184100417,
      "grad_norm": 36.86200995874469,
      "learning_rate": 3.610497133404795e-07,
      "logits/chosen": 1.4111734628677368,
      "logits/rejected": 1.7813835144042969,
      "logps/chosen": -336.32794189453125,
      "logps/rejected": -386.16827392578125,
      "loss": 0.5606,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.7106108069419861,
      "rewards/margins": 0.5927971601486206,
      "rewards/rejected": -1.303407907485962,
      "step": 200
    },
    {
      "epoch": 0.41841004184100417,
      "eval_logits/chosen": 1.1101882457733154,
      "eval_logits/rejected": 1.5408776998519897,
      "eval_logps/chosen": -327.4931640625,
      "eval_logps/rejected": -387.65057373046875,
      "eval_loss": 0.5555490851402283,
      "eval_rewards/accuracies": 0.74609375,
      "eval_rewards/chosen": -0.6215302348136902,
      "eval_rewards/margins": 0.606930136680603,
      "eval_rewards/rejected": -1.228460431098938,
      "eval_runtime": 53.4692,
      "eval_samples_per_second": 37.405,
      "eval_steps_per_second": 0.598,
      "step": 200
    },
    {
      "epoch": 0.4393305439330544,
      "grad_norm": 19.538681466466336,
      "learning_rate": 3.4440382358952115e-07,
      "logits/chosen": 1.2745749950408936,
      "logits/rejected": 1.9181947708129883,
      "logps/chosen": -358.096435546875,
      "logps/rejected": -373.4177551269531,
      "loss": 0.5641,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5784404873847961,
      "rewards/margins": 0.6223729252815247,
      "rewards/rejected": -1.2008132934570312,
      "step": 210
    },
    {
      "epoch": 0.4602510460251046,
      "grad_norm": 20.86939524596463,
      "learning_rate": 3.272542485937368e-07,
      "logits/chosen": 0.8960593342781067,
      "logits/rejected": 1.4320571422576904,
      "logps/chosen": -332.91864013671875,
      "logps/rejected": -364.5188293457031,
      "loss": 0.5474,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5551973581314087,
      "rewards/margins": 0.6443191170692444,
      "rewards/rejected": -1.1995164155960083,
      "step": 220
    },
    {
      "epoch": 0.4811715481171548,
      "grad_norm": 19.099059937166764,
      "learning_rate": 3.096924887558854e-07,
      "logits/chosen": 0.6132632493972778,
      "logits/rejected": 0.9618092775344849,
      "logps/chosen": -323.7577209472656,
      "logps/rejected": -385.98565673828125,
      "loss": 0.552,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.7150660753250122,
      "rewards/margins": 0.6639611721038818,
      "rewards/rejected": -1.3790271282196045,
      "step": 230
    },
    {
      "epoch": 0.502092050209205,
      "grad_norm": 17.891063709919703,
      "learning_rate": 2.9181224366319943e-07,
      "logits/chosen": 0.4524998664855957,
      "logits/rejected": 0.8361547589302063,
      "logps/chosen": -321.6592102050781,
      "logps/rejected": -369.956298828125,
      "loss": 0.5196,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.551439642906189,
      "rewards/margins": 0.6203877925872803,
      "rewards/rejected": -1.1718274354934692,
      "step": 240
    },
    {
      "epoch": 0.5230125523012552,
      "grad_norm": 19.34076101834313,
      "learning_rate": 2.7370891215954565e-07,
      "logits/chosen": 1.31077241897583,
      "logits/rejected": 1.824054479598999,
      "logps/chosen": -356.10235595703125,
      "logps/rejected": -388.8703918457031,
      "loss": 0.5268,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.48947277665138245,
      "rewards/margins": 0.7497965097427368,
      "rewards/rejected": -1.2392691373825073,
      "step": 250
    },
    {
      "epoch": 0.5439330543933054,
      "grad_norm": 26.294415278038993,
      "learning_rate": 2.55479083351317e-07,
      "logits/chosen": 1.545461893081665,
      "logits/rejected": 2.077069044113159,
      "logps/chosen": -352.08587646484375,
      "logps/rejected": -382.9310607910156,
      "loss": 0.5298,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.5431629419326782,
      "rewards/margins": 0.7132429480552673,
      "rewards/rejected": -1.2564058303833008,
      "step": 260
    },
    {
      "epoch": 0.5648535564853556,
      "grad_norm": 16.790572679540336,
      "learning_rate": 2.3722002126275822e-07,
      "logits/chosen": 1.6584622859954834,
      "logits/rejected": 2.1149163246154785,
      "logps/chosen": -342.0724182128906,
      "logps/rejected": -371.71588134765625,
      "loss": 0.5367,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.615090548992157,
      "rewards/margins": 0.5203320980072021,
      "rewards/rejected": -1.1354224681854248,
      "step": 270
    },
    {
      "epoch": 0.5857740585774058,
      "grad_norm": 24.772008946191377,
      "learning_rate": 2.19029145890313e-07,
      "logits/chosen": 1.7721284627914429,
      "logits/rejected": 2.3066000938415527,
      "logps/chosen": -326.43603515625,
      "logps/rejected": -370.3632507324219,
      "loss": 0.5442,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5788434147834778,
      "rewards/margins": 0.684018075466156,
      "rewards/rejected": -1.2628614902496338,
      "step": 280
    },
    {
      "epoch": 0.606694560669456,
      "grad_norm": 19.443360004007754,
      "learning_rate": 2.0100351342479216e-07,
      "logits/chosen": 1.2106627225875854,
      "logits/rejected": 1.4834706783294678,
      "logps/chosen": -333.8368225097656,
      "logps/rejected": -384.6605224609375,
      "loss": 0.5396,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.8036863207817078,
      "rewards/margins": 0.5939573645591736,
      "rewards/rejected": -1.397643804550171,
      "step": 290
    },
    {
      "epoch": 0.6276150627615062,
      "grad_norm": 24.510951465909894,
      "learning_rate": 1.8323929841460178e-07,
      "logits/chosen": 0.9158282279968262,
      "logits/rejected": 1.5702682733535767,
      "logps/chosen": -382.8146057128906,
      "logps/rejected": -400.11474609375,
      "loss": 0.5211,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.7668856382369995,
      "rewards/margins": 0.6154419183731079,
      "rewards/rejected": -1.382327675819397,
      "step": 300
    },
    {
      "epoch": 0.6276150627615062,
      "eval_logits/chosen": 0.7847145199775696,
      "eval_logits/rejected": 1.2079328298568726,
      "eval_logps/chosen": -329.06488037109375,
      "eval_logps/rejected": -404.1211242675781,
      "eval_loss": 0.5284228324890137,
      "eval_rewards/accuracies": 0.76171875,
      "eval_rewards/chosen": -0.6372475028038025,
      "eval_rewards/margins": 0.7559184432029724,
      "eval_rewards/rejected": -1.393165946006775,
      "eval_runtime": 53.4499,
      "eval_samples_per_second": 37.418,
      "eval_steps_per_second": 0.599,
      "step": 300
    },
    {
      "epoch": 0.6485355648535565,
      "grad_norm": 21.649958560460085,
      "learning_rate": 1.6583128063291573e-07,
      "logits/chosen": 0.9612062573432922,
      "logits/rejected": 1.3238331079483032,
      "logps/chosen": -379.32904052734375,
      "logps/rejected": -410.33685302734375,
      "loss": 0.5135,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.7942469120025635,
      "rewards/margins": 0.6349788904190063,
      "rewards/rejected": -1.4292259216308594,
      "step": 310
    },
    {
      "epoch": 0.6694560669456067,
      "grad_norm": 21.732265089014117,
      "learning_rate": 1.488723393865766e-07,
      "logits/chosen": 0.9882587194442749,
      "logits/rejected": 1.442718744277954,
      "logps/chosen": -375.4773254394531,
      "logps/rejected": -389.82733154296875,
      "loss": 0.506,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7661019563674927,
      "rewards/margins": 0.6393279433250427,
      "rewards/rejected": -1.4054299592971802,
      "step": 320
    },
    {
      "epoch": 0.6903765690376569,
      "grad_norm": 25.995557260279178,
      "learning_rate": 1.3245295796480788e-07,
      "logits/chosen": 0.8205612897872925,
      "logits/rejected": 1.306123971939087,
      "logps/chosen": -333.33074951171875,
      "logps/rejected": -386.80072021484375,
      "loss": 0.5192,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.5908290147781372,
      "rewards/margins": 0.5889186859130859,
      "rewards/rejected": -1.1797475814819336,
      "step": 330
    },
    {
      "epoch": 0.7112970711297071,
      "grad_norm": 34.06994351015907,
      "learning_rate": 1.1666074087171627e-07,
      "logits/chosen": 0.8795354962348938,
      "logits/rejected": 1.3036845922470093,
      "logps/chosen": -349.6972961425781,
      "logps/rejected": -425.7892150878906,
      "loss": 0.5208,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.5422636270523071,
      "rewards/margins": 0.8381665945053101,
      "rewards/rejected": -1.3804302215576172,
      "step": 340
    },
    {
      "epoch": 0.7322175732217573,
      "grad_norm": 21.28540346436605,
      "learning_rate": 1.0157994641835734e-07,
      "logits/chosen": 1.0639841556549072,
      "logits/rejected": 1.6034936904907227,
      "logps/chosen": -329.88690185546875,
      "logps/rejected": -385.4267578125,
      "loss": 0.4889,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.6864292621612549,
      "rewards/margins": 0.7708432674407959,
      "rewards/rejected": -1.4572727680206299,
      "step": 350
    },
    {
      "epoch": 0.7531380753138075,
      "grad_norm": 23.66473954551423,
      "learning_rate": 8.729103716819111e-08,
      "logits/chosen": 1.199178695678711,
      "logits/rejected": 1.9472402334213257,
      "logps/chosen": -380.9803161621094,
      "logps/rejected": -419.5332946777344,
      "loss": 0.5316,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.7623031735420227,
      "rewards/margins": 0.7938345670700073,
      "rewards/rejected": -1.5561376810073853,
      "step": 360
    },
    {
      "epoch": 0.7740585774058577,
      "grad_norm": 22.814657698551834,
      "learning_rate": 7.387025063449081e-08,
      "logits/chosen": 1.5637246370315552,
      "logits/rejected": 2.090482234954834,
      "logps/chosen": -364.23126220703125,
      "logps/rejected": -389.804931640625,
      "loss": 0.5165,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.8549477458000183,
      "rewards/margins": 0.6235242486000061,
      "rewards/rejected": -1.4784719944000244,
      "step": 370
    },
    {
      "epoch": 0.7949790794979079,
      "grad_norm": 24.019268622369378,
      "learning_rate": 6.138919252022435e-08,
      "logits/chosen": 1.690513253211975,
      "logits/rejected": 1.896419882774353,
      "logps/chosen": -330.74395751953125,
      "logps/rejected": -429.8720703125,
      "loss": 0.5175,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.8654612302780151,
      "rewards/margins": 0.7997757196426392,
      "rewards/rejected": -1.6652368307113647,
      "step": 380
    },
    {
      "epoch": 0.8158995815899581,
      "grad_norm": 23.872152876539662,
      "learning_rate": 4.991445467064689e-08,
      "logits/chosen": 1.24801766872406,
      "logits/rejected": 1.5409824848175049,
      "logps/chosen": -374.94500732421875,
      "logps/rejected": -423.5159606933594,
      "loss": 0.5064,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.623989462852478,
      "rewards/margins": 0.7279406785964966,
      "rewards/rejected": -1.351930022239685,
      "step": 390
    },
    {
      "epoch": 0.8368200836820083,
      "grad_norm": 22.17989276400099,
      "learning_rate": 3.9507259776993954e-08,
      "logits/chosen": 1.4913052320480347,
      "logits/rejected": 1.9780304431915283,
      "logps/chosen": -344.193359375,
      "logps/rejected": -412.5020446777344,
      "loss": 0.5189,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.6954695582389832,
      "rewards/margins": 0.7355377078056335,
      "rewards/rejected": -1.4310071468353271,
      "step": 400
    },
    {
      "epoch": 0.8368200836820083,
      "eval_logits/chosen": 1.2049928903579712,
      "eval_logits/rejected": 1.687930941581726,
      "eval_logps/chosen": -323.4798889160156,
      "eval_logps/rejected": -403.6497802734375,
      "eval_loss": 0.5224906206130981,
      "eval_rewards/accuracies": 0.77734375,
      "eval_rewards/chosen": -0.5813976526260376,
      "eval_rewards/margins": 0.8070548176765442,
      "eval_rewards/rejected": -1.3884525299072266,
      "eval_runtime": 53.5131,
      "eval_samples_per_second": 37.374,
      "eval_steps_per_second": 0.598,
      "step": 400
    },
    {
      "epoch": 0.8577405857740585,
      "grad_norm": 24.78416106509579,
      "learning_rate": 3.022313472693447e-08,
      "logits/chosen": 1.401047945022583,
      "logits/rejected": 2.122811794281006,
      "logps/chosen": -370.56951904296875,
      "logps/rejected": -416.39080810546875,
      "loss": 0.5278,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.6760027408599854,
      "rewards/margins": 0.7805537581443787,
      "rewards/rejected": -1.4565564393997192,
      "step": 410
    },
    {
      "epoch": 0.8786610878661087,
      "grad_norm": 24.036813785272063,
      "learning_rate": 2.2111614344599684e-08,
      "logits/chosen": 1.3202136754989624,
      "logits/rejected": 1.759892225265503,
      "logps/chosen": -373.802734375,
      "logps/rejected": -416.2435607910156,
      "loss": 0.5136,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.7081748843193054,
      "rewards/margins": 0.6878335475921631,
      "rewards/rejected": -1.3960084915161133,
      "step": 420
    },
    {
      "epoch": 0.899581589958159,
      "grad_norm": 23.876888782252916,
      "learning_rate": 1.521597710086439e-08,
      "logits/chosen": 1.59340238571167,
      "logits/rejected": 2.011382579803467,
      "logps/chosen": -357.6289978027344,
      "logps/rejected": -414.678466796875,
      "loss": 0.4929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.7411682605743408,
      "rewards/margins": 0.7640897631645203,
      "rewards/rejected": -1.5052578449249268,
      "step": 430
    },
    {
      "epoch": 0.9205020920502092,
      "grad_norm": 23.73816215215032,
      "learning_rate": 9.57301420397924e-09,
      "logits/chosen": 1.240863561630249,
      "logits/rejected": 1.8725379705429077,
      "logps/chosen": -354.1866149902344,
      "logps/rejected": -410.48187255859375,
      "loss": 0.5066,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.6514572501182556,
      "rewards/margins": 0.7045157551765442,
      "rewards/rejected": -1.3559730052947998,
      "step": 440
    },
    {
      "epoch": 0.9414225941422594,
      "grad_norm": 23.69382783496658,
      "learning_rate": 5.212833302556258e-09,
      "logits/chosen": 1.4327576160430908,
      "logits/rejected": 1.8339331150054932,
      "logps/chosen": -374.9490661621094,
      "logps/rejected": -459.17169189453125,
      "loss": 0.51,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.7898253202438354,
      "rewards/margins": 0.6647788286209106,
      "rewards/rejected": -1.454604148864746,
      "step": 450
    },
    {
      "epoch": 0.9623430962343096,
      "grad_norm": 28.599243740532902,
      "learning_rate": 2.158697848236607e-09,
      "logits/chosen": 1.528529405593872,
      "logits/rejected": 2.007927179336548,
      "logps/chosen": -352.3992004394531,
      "logps/rejected": -387.2112121582031,
      "loss": 0.5217,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.7364510893821716,
      "rewards/margins": 0.685036838054657,
      "rewards/rejected": -1.4214880466461182,
      "step": 460
    },
    {
      "epoch": 0.9832635983263598,
      "grad_norm": 20.454762644290014,
      "learning_rate": 4.269029751107489e-10,
      "logits/chosen": 1.380399227142334,
      "logits/rejected": 1.8892757892608643,
      "logps/chosen": -350.90447998046875,
      "logps/rejected": -421.0154724121094,
      "loss": 0.5088,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.7164818644523621,
      "rewards/margins": 0.6821088194847107,
      "rewards/rejected": -1.3985908031463623,
      "step": 470
    },
    {
      "epoch": 1.0,
      "step": 478,
      "total_flos": 0.0,
      "train_loss": 0.5574632548886862,
      "train_runtime": 4290.8569,
      "train_samples_per_second": 14.248,
      "train_steps_per_second": 0.111
    }
  ],
  "logging_steps": 10,
  "max_steps": 478,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}