{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 1476,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.02,
      "grad_norm": 420.0,
      "learning_rate": 5.000000000000001e-07,
      "log_odds_chosen": 0.4207151532173157,
      "log_odds_ratio": -0.5201840400695801,
      "logits/chosen": -2.347763776779175,
      "logits/rejected": -2.3541953563690186,
      "logps/chosen": -1.7608455419540405,
      "logps/rejected": -2.1218907833099365,
      "loss": 3.9956,
      "nll_loss": 4.183717250823975,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.08804227411746979,
      "rewards/margins": 0.018052268773317337,
      "rewards/rejected": -0.10609455406665802,
      "step": 10
    },
    {
      "epoch": 0.04,
      "grad_norm": 44.25,
      "learning_rate": 1.0000000000000002e-06,
      "log_odds_chosen": 0.5392915606498718,
      "log_odds_ratio": -0.4769781231880188,
      "logits/chosen": -2.866349458694458,
      "logits/rejected": -2.866671323776245,
      "logps/chosen": -1.5143439769744873,
      "logps/rejected": -1.9585195779800415,
      "loss": 2.0453,
      "nll_loss": 1.9479516744613647,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.07571719586849213,
      "rewards/margins": 0.02220877632498741,
      "rewards/rejected": -0.09792597591876984,
      "step": 20
    },
    {
      "epoch": 0.06,
      "grad_norm": 7.5,
      "learning_rate": 1.5e-06,
      "log_odds_chosen": 0.9706680178642273,
      "log_odds_ratio": -0.3677331805229187,
      "logits/chosen": -2.6566712856292725,
      "logits/rejected": -2.660308361053467,
      "logps/chosen": -0.7488349676132202,
      "logps/rejected": -1.312819480895996,
      "loss": 1.0008,
      "nll_loss": 0.8921732902526855,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.03744174912571907,
      "rewards/margins": 0.028199229389429092,
      "rewards/rejected": -0.06564097106456757,
      "step": 30
    },
    {
      "epoch": 0.08,
      "grad_norm": 7.40625,
      "learning_rate": 2.0000000000000003e-06,
      "log_odds_chosen": 2.282021999359131,
      "log_odds_ratio": -0.20370522141456604,
      "logits/chosen": -2.1984050273895264,
      "logits/rejected": -2.202855110168457,
      "logps/chosen": -0.27147307991981506,
      "logps/rejected": -1.2435765266418457,
      "loss": 0.8107,
      "nll_loss": 0.683063268661499,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.013573653995990753,
      "rewards/margins": 0.04860517010092735,
      "rewards/rejected": -0.06217882037162781,
      "step": 40
    },
    {
      "epoch": 0.1,
      "grad_norm": 6.375,
      "learning_rate": 2.5e-06,
      "log_odds_chosen": 2.8677659034729004,
      "log_odds_ratio": -0.14154179394245148,
      "logits/chosen": -2.279298782348633,
      "logits/rejected": -2.2862045764923096,
      "logps/chosen": -0.216520756483078,
      "logps/rejected": -1.3591465950012207,
      "loss": 0.7978,
      "nll_loss": 0.828228771686554,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.01082603819668293,
      "rewards/margins": 0.057131290435791016,
      "rewards/rejected": -0.0679573267698288,
      "step": 50
    },
    {
      "epoch": 0.12,
      "grad_norm": 4.3125,
      "learning_rate": 3e-06,
      "log_odds_chosen": 3.2677154541015625,
      "log_odds_ratio": -0.12433552742004395,
      "logits/chosen": -2.2281601428985596,
      "logits/rejected": -2.2347497940063477,
      "logps/chosen": -0.20724907517433167,
      "logps/rejected": -1.5352025032043457,
      "loss": 0.7919,
      "nll_loss": 0.8021117448806763,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.010362453758716583,
      "rewards/margins": 0.06639767438173294,
      "rewards/rejected": -0.07676012814044952,
      "step": 60
    },
    {
      "epoch": 0.14,
      "grad_norm": 4.21875,
      "learning_rate": 3.5e-06,
      "log_odds_chosen": 3.9659018516540527,
      "log_odds_ratio": -0.11224009841680527,
      "logits/chosen": -2.0779483318328857,
      "logits/rejected": -2.0807433128356934,
      "logps/chosen": -0.1909479945898056,
      "logps/rejected": -1.9270519018173218,
      "loss": 0.7197,
      "nll_loss": 0.6386193037033081,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.00954739935696125,
      "rewards/margins": 0.08680519461631775,
      "rewards/rejected": -0.09635259956121445,
      "step": 70
    },
    {
      "epoch": 0.16,
      "grad_norm": 4.15625,
      "learning_rate": 4.000000000000001e-06,
      "log_odds_chosen": 3.9601542949676514,
      "log_odds_ratio": -0.11061631143093109,
      "logits/chosen": -2.1697168350219727,
      "logits/rejected": -2.1794517040252686,
      "logps/chosen": -0.185487762093544,
      "logps/rejected": -1.8811283111572266,
      "loss": 0.7033,
      "nll_loss": 0.7545000314712524,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.009274388663470745,
      "rewards/margins": 0.08478203415870667,
      "rewards/rejected": -0.09405642002820969,
      "step": 80
    },
    {
      "epoch": 0.18,
      "grad_norm": 4.71875,
      "learning_rate": 4.5e-06,
      "log_odds_chosen": 4.599364280700684,
      "log_odds_ratio": -0.07719282805919647,
      "logits/chosen": -2.150642156600952,
      "logits/rejected": -2.15342378616333,
      "logps/chosen": -0.15149493515491486,
      "logps/rejected": -2.1990609169006348,
      "loss": 0.6116,
      "nll_loss": 0.6107597947120667,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.007574746850878,
      "rewards/margins": 0.10237829387187958,
      "rewards/rejected": -0.10995303094387054,
      "step": 90
    },
    {
      "epoch": 0.2,
      "grad_norm": 4.53125,
      "learning_rate": 5e-06,
      "log_odds_chosen": 4.817965507507324,
      "log_odds_ratio": -0.07800901681184769,
      "logits/chosen": -2.1689393520355225,
      "logits/rejected": -2.1756036281585693,
      "logps/chosen": -0.1730736345052719,
      "logps/rejected": -2.246196746826172,
      "loss": 0.6043,
      "nll_loss": 0.6302344799041748,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.008653681725263596,
      "rewards/margins": 0.10365615785121918,
      "rewards/rejected": -0.11230983585119247,
      "step": 100
    },
    {
      "epoch": 0.22,
      "grad_norm": 5.625,
      "learning_rate": 4.767312946227961e-06,
      "log_odds_chosen": 4.97894287109375,
      "log_odds_ratio": -0.10877744108438492,
      "logits/chosen": -2.2094526290893555,
      "logits/rejected": -2.216020107269287,
      "logps/chosen": -0.18459849059581757,
      "logps/rejected": -2.573103904724121,
      "loss": 0.5724,
      "nll_loss": 0.6197156310081482,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.009229923598468304,
      "rewards/margins": 0.11942527443170547,
      "rewards/rejected": -0.12865519523620605,
      "step": 110
    },
    {
      "epoch": 0.24,
      "grad_norm": 5.8125,
      "learning_rate": 4.564354645876385e-06,
      "log_odds_chosen": 5.132683277130127,
      "log_odds_ratio": -0.13557349145412445,
      "logits/chosen": -2.178772449493408,
      "logits/rejected": -2.184910774230957,
      "logps/chosen": -0.154138445854187,
      "logps/rejected": -2.2600817680358887,
      "loss": 0.4748,
      "nll_loss": 0.459114134311676,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.0077069224789738655,
      "rewards/margins": 0.10529716312885284,
      "rewards/rejected": -0.11300408840179443,
      "step": 120
    },
    {
      "epoch": 0.26,
      "grad_norm": 5.0,
      "learning_rate": 4.385290096535147e-06,
      "log_odds_chosen": 5.768979072570801,
      "log_odds_ratio": -0.08052898198366165,
      "logits/chosen": -2.101267099380493,
      "logits/rejected": -2.104369878768921,
      "logps/chosen": -0.1255403310060501,
      "logps/rejected": -2.850447416305542,
      "loss": 0.4379,
      "nll_loss": 0.40499457716941833,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.006277016364037991,
      "rewards/margins": 0.13624534010887146,
      "rewards/rejected": -0.14252236485481262,
      "step": 130
    },
    {
      "epoch": 0.28,
      "grad_norm": 6.0,
      "learning_rate": 4.2257712736425835e-06,
      "log_odds_chosen": 5.547473907470703,
      "log_odds_ratio": -0.056151650846004486,
      "logits/chosen": -2.0569746494293213,
      "logits/rejected": -2.059227466583252,
      "logps/chosen": -0.12779685854911804,
      "logps/rejected": -2.394996166229248,
      "loss": 0.4122,
      "nll_loss": 0.33014988899230957,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.006389842834323645,
      "rewards/margins": 0.1133599653840065,
      "rewards/rejected": -0.11974982172250748,
      "step": 140
    },
    {
      "epoch": 0.3,
      "grad_norm": 5.21875,
      "learning_rate": 4.082482904638631e-06,
      "log_odds_chosen": 5.67286491394043,
      "log_odds_ratio": -0.06435154378414154,
      "logits/chosen": -2.166313886642456,
      "logits/rejected": -2.1680521965026855,
      "logps/chosen": -0.13122043013572693,
      "logps/rejected": -2.701577663421631,
      "loss": 0.4055,
      "nll_loss": 0.4321494996547699,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.0065610213205218315,
      "rewards/margins": 0.12851786613464355,
      "rewards/rejected": -0.13507887721061707,
      "step": 150
    },
    {
      "epoch": 0.33,
      "grad_norm": 6.5625,
      "learning_rate": 3.952847075210474e-06,
      "log_odds_chosen": 5.581357002258301,
      "log_odds_ratio": -0.094394750893116,
      "logits/chosen": -2.121497392654419,
      "logits/rejected": -2.1272549629211426,
      "logps/chosen": -0.15410242974758148,
      "logps/rejected": -2.376707077026367,
      "loss": 0.3695,
      "nll_loss": 0.43908724188804626,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.007705122232437134,
      "rewards/margins": 0.1111302375793457,
      "rewards/rejected": -0.11883536726236343,
      "step": 160
    },
    {
      "epoch": 0.35,
      "grad_norm": 6.0,
      "learning_rate": 3.834824944236852e-06,
      "log_odds_chosen": 6.281979560852051,
      "log_odds_ratio": -0.059505023062229156,
      "logits/chosen": -2.1019725799560547,
      "logits/rejected": -2.1044390201568604,
      "logps/chosen": -0.15183117985725403,
      "logps/rejected": -3.044116497039795,
      "loss": 0.3264,
      "nll_loss": 0.3386436998844147,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.007591559551656246,
      "rewards/margins": 0.14461426436901093,
      "rewards/rejected": -0.15220583975315094,
      "step": 170
    },
    {
      "epoch": 0.37,
      "grad_norm": 6.3125,
      "learning_rate": 3.72677996249965e-06,
      "log_odds_chosen": 6.111880302429199,
      "log_odds_ratio": -0.07030317932367325,
      "logits/chosen": -2.0706937313079834,
      "logits/rejected": -2.067783832550049,
      "logps/chosen": -0.14989063143730164,
      "logps/rejected": -3.172281265258789,
      "loss": 0.3324,
      "nll_loss": 0.38789159059524536,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.007494532503187656,
      "rewards/margins": 0.15111951529979706,
      "rewards/rejected": -0.15861406922340393,
      "step": 180
    },
    {
      "epoch": 0.39,
      "grad_norm": 6.0625,
      "learning_rate": 3.6273812505500587e-06,
      "log_odds_chosen": 6.445823669433594,
      "log_odds_ratio": -0.05266156792640686,
      "logits/chosen": -2.0448713302612305,
      "logits/rejected": -2.0481276512145996,
      "logps/chosen": -0.13432614505290985,
      "logps/rejected": -2.9074063301086426,
      "loss": 0.3063,
      "nll_loss": 0.28705888986587524,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.006716308183968067,
      "rewards/margins": 0.13865402340888977,
      "rewards/rejected": -0.14537033438682556,
      "step": 190
    },
    {
      "epoch": 0.41,
      "grad_norm": 6.0,
      "learning_rate": 3.5355339059327378e-06,
      "log_odds_chosen": 5.775099754333496,
      "log_odds_ratio": -0.08722616732120514,
      "logits/chosen": -2.087759256362915,
      "logits/rejected": -2.0876097679138184,
      "logps/chosen": -0.11597369611263275,
      "logps/rejected": -2.6853129863739014,
      "loss": 0.2962,
      "nll_loss": 0.31049779057502747,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.005798685364425182,
      "rewards/margins": 0.12846694886684418,
      "rewards/rejected": -0.13426566123962402,
      "step": 200
    },
    {
      "epoch": 0.43,
      "grad_norm": 5.90625,
      "learning_rate": 3.450327796711771e-06,
      "log_odds_chosen": 7.5723066329956055,
      "log_odds_ratio": -0.040629759430885315,
      "logits/chosen": -1.973818063735962,
      "logits/rejected": -1.976941704750061,
      "logps/chosen": -0.09340523928403854,
      "logps/rejected": -3.5397675037384033,
      "loss": 0.2637,
      "nll_loss": 0.20889191329479218,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.00467026187106967,
      "rewards/margins": 0.17231810092926025,
      "rewards/rejected": -0.1769883632659912,
      "step": 210
    },
    {
      "epoch": 0.45,
      "grad_norm": 6.25,
      "learning_rate": 3.3709993123162106e-06,
      "log_odds_chosen": 7.618361473083496,
      "log_odds_ratio": -0.074308380484581,
      "logits/chosen": -2.0433554649353027,
      "logits/rejected": -2.0466837882995605,
      "logps/chosen": -0.09071488678455353,
      "logps/rejected": -3.5269036293029785,
      "loss": 0.2727,
      "nll_loss": 0.24802179634571075,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.004535744432359934,
      "rewards/margins": 0.17180944979190826,
      "rewards/rejected": -0.17634519934654236,
      "step": 220
    },
    {
      "epoch": 0.47,
      "grad_norm": 5.875,
      "learning_rate": 3.296902366978936e-06,
      "log_odds_chosen": 6.698166847229004,
      "log_odds_ratio": -0.04654809832572937,
      "logits/chosen": -1.968505620956421,
      "logits/rejected": -1.9712941646575928,
      "logps/chosen": -0.1543513834476471,
      "logps/rejected": -3.186583995819092,
      "loss": 0.2015,
      "nll_loss": 0.1845930814743042,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.00771756935864687,
      "rewards/margins": 0.15161164104938507,
      "rewards/rejected": -0.15932922065258026,
      "step": 230
    },
    {
      "epoch": 0.49,
      "grad_norm": 5.28125,
      "learning_rate": 3.2274861218395142e-06,
      "log_odds_chosen": 7.480630397796631,
      "log_odds_ratio": -0.06602514535188675,
      "logits/chosen": -1.995495080947876,
      "logits/rejected": -1.9951270818710327,
      "logps/chosen": -0.12399391829967499,
      "logps/rejected": -3.7262351512908936,
      "loss": 0.2181,
      "nll_loss": 0.1835182011127472,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.006199696101248264,
      "rewards/margins": 0.1801120638847351,
      "rewards/rejected": -0.1863117516040802,
      "step": 240
    },
    {
      "epoch": 0.51,
      "grad_norm": 6.65625,
      "learning_rate": 3.1622776601683796e-06,
      "log_odds_chosen": 7.167886257171631,
      "log_odds_ratio": -0.020891310647130013,
      "logits/chosen": -1.9453086853027344,
      "logits/rejected": -1.9443309307098389,
      "logps/chosen": -0.07993815094232559,
      "logps/rejected": -3.2065398693084717,
      "loss": 0.1966,
      "nll_loss": 0.21496787667274475,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.003996907267719507,
      "rewards/margins": 0.15633007884025574,
      "rewards/rejected": -0.1603269875049591,
      "step": 250
    },
    {
      "epoch": 0.53,
      "grad_norm": 7.03125,
      "learning_rate": 3.1008683647302113e-06,
      "log_odds_chosen": 6.189971446990967,
      "log_odds_ratio": -0.06337615847587585,
      "logits/chosen": -1.983068823814392,
      "logits/rejected": -1.9870706796646118,
      "logps/chosen": -0.13599030673503876,
      "logps/rejected": -2.809053897857666,
      "loss": 0.1981,
      "nll_loss": 0.23529119789600372,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.0067995162680745125,
      "rewards/margins": 0.1336531937122345,
      "rewards/rejected": -0.14045271277427673,
      "step": 260
    },
    {
      "epoch": 0.55,
      "grad_norm": 5.125,
      "learning_rate": 3.0429030972509227e-06,
      "log_odds_chosen": 6.417479038238525,
      "log_odds_ratio": -0.05179642513394356,
      "logits/chosen": -1.9953663349151611,
      "logits/rejected": -1.9997583627700806,
      "logps/chosen": -0.12870252132415771,
      "logps/rejected": -2.9075326919555664,
      "loss": 0.1753,
      "nll_loss": 0.22951626777648926,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.006435126066207886,
      "rewards/margins": 0.13894154131412506,
      "rewards/rejected": -0.14537665247917175,
      "step": 270
    },
    {
      "epoch": 0.57,
      "grad_norm": 5.90625,
      "learning_rate": 2.988071523335984e-06,
      "log_odds_chosen": 6.808651924133301,
      "log_odds_ratio": -0.06405957788228989,
      "logits/chosen": -2.0137691497802734,
      "logits/rejected": -2.015270709991455,
      "logps/chosen": -0.11106196790933609,
      "logps/rejected": -3.0401816368103027,
      "loss": 0.1885,
      "nll_loss": 0.1923011988401413,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.005553097929805517,
      "rewards/margins": 0.14645597338676453,
      "rewards/rejected": -0.15200909972190857,
      "step": 280
    },
    {
      "epoch": 0.59,
      "grad_norm": 5.375,
      "learning_rate": 2.9361010975735177e-06,
      "log_odds_chosen": 8.074934959411621,
      "log_odds_ratio": -0.022188549861311913,
      "logits/chosen": -1.94329833984375,
      "logits/rejected": -1.943985939025879,
      "logps/chosen": -0.06887535750865936,
      "logps/rejected": -3.7694950103759766,
      "loss": 0.1729,
      "nll_loss": 0.15832534432411194,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.0034437670838087797,
      "rewards/margins": 0.185030996799469,
      "rewards/rejected": -0.18847474455833435,
      "step": 290
    },
    {
      "epoch": 0.61,
      "grad_norm": 4.875,
      "learning_rate": 2.8867513459481293e-06,
      "log_odds_chosen": 6.823982238769531,
      "log_odds_ratio": -0.04056983441114426,
      "logits/chosen": -1.9236243963241577,
      "logits/rejected": -1.9263416528701782,
      "logps/chosen": -0.15296845138072968,
      "logps/rejected": -3.20212984085083,
      "loss": 0.1766,
      "nll_loss": 0.1584191620349884,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0076484219171106815,
      "rewards/margins": 0.152458056807518,
      "rewards/rejected": -0.16010649502277374,
      "step": 300
    },
    {
      "epoch": 0.63,
      "grad_norm": 5.0,
      "learning_rate": 2.839809171235324e-06,
      "log_odds_chosen": 7.289086818695068,
      "log_odds_ratio": -0.07533205300569534,
      "logits/chosen": -2.0198066234588623,
      "logits/rejected": -2.022125720977783,
      "logps/chosen": -0.12243582308292389,
      "logps/rejected": -3.1500906944274902,
      "loss": 0.1636,
      "nll_loss": 0.1975417137145996,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.0061217909678816795,
      "rewards/margins": 0.15138274431228638,
      "rewards/rejected": -0.15750452876091003,
      "step": 310
    },
    {
      "epoch": 0.65,
      "grad_norm": 9.0,
      "learning_rate": 2.7950849718747376e-06,
      "log_odds_chosen": 6.9169487953186035,
      "log_odds_ratio": -0.03769087791442871,
      "logits/chosen": -1.9343087673187256,
      "logits/rejected": -1.9343109130859375,
      "logps/chosen": -0.10367435216903687,
      "logps/rejected": -3.0478665828704834,
      "loss": 0.1696,
      "nll_loss": 0.15094847977161407,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.005183717701584101,
      "rewards/margins": 0.14720964431762695,
      "rewards/rejected": -0.15239334106445312,
      "step": 320
    },
    {
      "epoch": 0.67,
      "grad_norm": 5.90625,
      "learning_rate": 2.752409412815902e-06,
      "log_odds_chosen": 6.86618185043335,
      "log_odds_ratio": -0.038672395050525665,
      "logits/chosen": -2.047116279602051,
      "logits/rejected": -2.044132947921753,
      "logps/chosen": -0.13211274147033691,
      "logps/rejected": -3.158360004425049,
      "loss": 0.1625,
      "nll_loss": 0.17262741923332214,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.006605637725442648,
      "rewards/margins": 0.15131238102912903,
      "rewards/rejected": -0.15791800618171692,
      "step": 330
    },
    {
      "epoch": 0.69,
      "grad_norm": 5.40625,
      "learning_rate": 2.711630722733202e-06,
      "log_odds_chosen": 7.859139442443848,
      "log_odds_ratio": -0.05241536349058151,
      "logits/chosen": -1.951807975769043,
      "logits/rejected": -1.9466063976287842,
      "logps/chosen": -0.08200397342443466,
      "logps/rejected": -3.5420470237731934,
      "loss": 0.1465,
      "nll_loss": 0.1444871723651886,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.004100198391824961,
      "rewards/margins": 0.17300216853618622,
      "rewards/rejected": -0.17710235714912415,
      "step": 340
    },
    {
      "epoch": 0.71,
      "grad_norm": 5.71875,
      "learning_rate": 2.6726124191242444e-06,
      "log_odds_chosen": 7.041175842285156,
      "log_odds_ratio": -0.03843230381608009,
      "logits/chosen": -1.9341367483139038,
      "logits/rejected": -1.9377237558364868,
      "logps/chosen": -0.105894073843956,
      "logps/rejected": -2.9890971183776855,
      "loss": 0.145,
      "nll_loss": 0.15412884950637817,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.005294703878462315,
      "rewards/margins": 0.14416013658046722,
      "rewards/rejected": -0.14945486187934875,
      "step": 350
    },
    {
      "epoch": 0.73,
      "grad_norm": 6.90625,
      "learning_rate": 2.6352313834736496e-06,
      "log_odds_chosen": 6.8190507888793945,
      "log_odds_ratio": -0.04403644800186157,
      "logits/chosen": -1.984197974205017,
      "logits/rejected": -1.987479567527771,
      "logps/chosen": -0.11883167922496796,
      "logps/rejected": -3.1214818954467773,
      "loss": 0.1577,
      "nll_loss": 0.12197007238864899,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.0059415847063064575,
      "rewards/margins": 0.15013250708580017,
      "rewards/rejected": -0.15607409179210663,
      "step": 360
    },
    {
      "epoch": 0.75,
      "grad_norm": 5.28125,
      "learning_rate": 2.599376224550182e-06,
      "log_odds_chosen": 6.533886909484863,
      "log_odds_ratio": -0.03479010984301567,
      "logits/chosen": -1.898244857788086,
      "logits/rejected": -1.8991117477416992,
      "logps/chosen": -0.10942695289850235,
      "logps/rejected": -2.946617841720581,
      "loss": 0.1199,
      "nll_loss": 0.11430631577968597,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.005471347831189632,
      "rewards/margins": 0.14185954630374908,
      "rewards/rejected": -0.14733090996742249,
      "step": 370
    },
    {
      "epoch": 0.77,
      "grad_norm": 5.6875,
      "learning_rate": 2.564945880212886e-06,
      "log_odds_chosen": 6.550747871398926,
      "log_odds_ratio": -0.0334211066365242,
      "logits/chosen": -1.9049135446548462,
      "logits/rejected": -1.9040100574493408,
      "logps/chosen": -0.12769648432731628,
      "logps/rejected": -3.1679463386535645,
      "loss": 0.1321,
      "nll_loss": 0.12015843391418457,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.006384824402630329,
      "rewards/margins": 0.15201251208782196,
      "rewards/rejected": -0.15839733183383942,
      "step": 380
    },
    {
      "epoch": 0.79,
      "grad_norm": 4.46875,
      "learning_rate": 2.5318484177091667e-06,
      "log_odds_chosen": 6.963167667388916,
      "log_odds_ratio": -0.07767531275749207,
      "logits/chosen": -1.9564456939697266,
      "logits/rejected": -1.957360029220581,
      "logps/chosen": -0.09807981550693512,
      "logps/rejected": -2.8958752155303955,
      "loss": 0.1334,
      "nll_loss": 0.13979199528694153,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.004903990775346756,
      "rewards/margins": 0.13988977670669556,
      "rewards/rejected": -0.1447937786579132,
      "step": 390
    },
    {
      "epoch": 0.81,
      "grad_norm": 6.46875,
      "learning_rate": 2.5e-06,
      "log_odds_chosen": 7.737925052642822,
      "log_odds_ratio": -0.011308287270367146,
      "logits/chosen": -1.8833544254302979,
      "logits/rejected": -1.8816306591033936,
      "logps/chosen": -0.07808590680360794,
      "logps/rejected": -3.517468214035034,
      "loss": 0.116,
      "nll_loss": 0.09308944642543793,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0039042953867465258,
      "rewards/margins": 0.17196913063526154,
      "rewards/rejected": -0.17587339878082275,
      "step": 400
    },
    {
      "epoch": 0.83,
      "grad_norm": 5.09375,
      "learning_rate": 2.4693239916239746e-06,
      "log_odds_chosen": 8.026070594787598,
      "log_odds_ratio": -0.009079935029149055,
      "logits/chosen": -1.8916471004486084,
      "logits/rejected": -1.8905131816864014,
      "logps/chosen": -0.08094353973865509,
      "logps/rejected": -3.705895185470581,
      "loss": 0.1292,
      "nll_loss": 0.10612641274929047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.00404717680066824,
      "rewards/margins": 0.18124757707118988,
      "rewards/rejected": -0.1852947473526001,
      "step": 410
    },
    {
      "epoch": 0.85,
      "grad_norm": 4.15625,
      "learning_rate": 2.4397501823713327e-06,
      "log_odds_chosen": 6.706048488616943,
      "log_odds_ratio": -0.04705270379781723,
      "logits/chosen": -1.9574733972549438,
      "logits/rejected": -1.9553906917572021,
      "logps/chosen": -0.13810355961322784,
      "logps/rejected": -3.244007110595703,
      "loss": 0.1116,
      "nll_loss": 0.13714413344860077,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.00690517807379365,
      "rewards/margins": 0.15529517829418182,
      "rewards/rejected": -0.16220036149024963,
      "step": 420
    },
    {
      "epoch": 0.87,
      "grad_norm": 4.9375,
      "learning_rate": 2.411214110852061e-06,
      "log_odds_chosen": 7.185335636138916,
      "log_odds_ratio": -0.049024853855371475,
      "logits/chosen": -1.9474496841430664,
      "logits/rejected": -1.94386887550354,
      "logps/chosen": -0.12190796434879303,
      "logps/rejected": -3.319507122039795,
      "loss": 0.1158,
      "nll_loss": 0.13795073330402374,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.0060953982174396515,
      "rewards/margins": 0.15987998247146606,
      "rewards/rejected": -0.16597537696361542,
      "step": 430
    },
    {
      "epoch": 0.89,
      "grad_norm": 4.8125,
      "learning_rate": 2.3836564731139807e-06,
      "log_odds_chosen": 7.590296268463135,
      "log_odds_ratio": -0.021134180948138237,
      "logits/chosen": -1.8886020183563232,
      "logits/rejected": -1.886425256729126,
      "logps/chosen": -0.09185922890901566,
      "logps/rejected": -3.507038116455078,
      "loss": 0.1033,
      "nll_loss": 0.06822694092988968,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.00459296116605401,
      "rewards/margins": 0.1707589477300644,
      "rewards/rejected": -0.17535191774368286,
      "step": 440
    },
    {
      "epoch": 0.91,
      "grad_norm": 6.28125,
      "learning_rate": 2.357022603955159e-06,
      "log_odds_chosen": 6.999331474304199,
      "log_odds_ratio": -0.03660160303115845,
      "logits/chosen": -1.861541748046875,
      "logits/rejected": -1.861534833908081,
      "logps/chosen": -0.12671752274036407,
      "logps/rejected": -3.262852430343628,
      "loss": 0.1048,
      "nll_loss": 0.09448351711034775,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.0063358754850924015,
      "rewards/margins": 0.15680675208568573,
      "rewards/rejected": -0.1631426364183426,
      "step": 450
    },
    {
      "epoch": 0.93,
      "grad_norm": 4.78125,
      "learning_rate": 2.3312620206007847e-06,
      "log_odds_chosen": 6.333981037139893,
      "log_odds_ratio": -0.049268852919340134,
      "logits/chosen": -1.8751580715179443,
      "logits/rejected": -1.8740007877349854,
      "logps/chosen": -0.13210389018058777,
      "logps/rejected": -2.7039647102355957,
      "loss": 0.1094,
      "nll_loss": 0.09672755748033524,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.006605193950235844,
      "rewards/margins": 0.12859304249286652,
      "rewards/rejected": -0.13519823551177979,
      "step": 460
    },
    {
      "epoch": 0.96,
      "grad_norm": 6.65625,
      "learning_rate": 2.3063280200722128e-06,
      "log_odds_chosen": 7.438237190246582,
      "log_odds_ratio": -0.02831345424056053,
      "logits/chosen": -1.9090907573699951,
      "logits/rejected": -1.9076954126358032,
      "logps/chosen": -0.07157810777425766,
      "logps/rejected": -3.022064447402954,
      "loss": 0.0996,
      "nll_loss": 0.09075023233890533,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.003578905714675784,
      "rewards/margins": 0.14752431213855743,
      "rewards/rejected": -0.151103213429451,
      "step": 470
    },
    {
      "epoch": 0.98,
      "grad_norm": 5.46875,
      "learning_rate": 2.2821773229381924e-06,
      "log_odds_chosen": 8.533849716186523,
      "log_odds_ratio": -0.010285028256475925,
      "logits/chosen": -1.958752989768982,
      "logits/rejected": -1.961133599281311,
      "logps/chosen": -0.07796324789524078,
      "logps/rejected": -3.159803628921509,
      "loss": 0.1074,
      "nll_loss": 0.10170219838619232,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0038981616962701082,
      "rewards/margins": 0.15409204363822937,
      "rewards/rejected": -0.15799018740653992,
      "step": 480
    },
    {
      "epoch": 1.0,
      "grad_norm": 4.40625,
      "learning_rate": 2.2587697572631284e-06,
      "log_odds_chosen": 7.246984958648682,
      "log_odds_ratio": -0.03945430740714073,
      "logits/chosen": -1.9564424753189087,
      "logits/rejected": -1.9600261449813843,
      "logps/chosen": -0.1045081838965416,
      "logps/rejected": -3.074450731277466,
      "loss": 0.1118,
      "nll_loss": 0.11936734616756439,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.00522540882229805,
      "rewards/margins": 0.14849711954593658,
      "rewards/rejected": -0.15372253954410553,
      "step": 490
    },
    {
      "epoch": 1.02,
      "grad_norm": 4.78125,
      "learning_rate": 2.23606797749979e-06,
      "log_odds_chosen": 7.313413143157959,
      "log_odds_ratio": -0.06336325407028198,
      "logits/chosen": -1.925005316734314,
      "logits/rejected": -1.9246110916137695,
      "logps/chosen": -0.06910906732082367,
      "logps/rejected": -3.088639974594116,
      "loss": 0.0748,
      "nll_loss": 0.06068875640630722,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.0034554533194750547,
      "rewards/margins": 0.15097656846046448,
      "rewards/rejected": -0.1544320285320282,
      "step": 500
    },
    {
      "epoch": 1.04,
      "grad_norm": 3.984375,
      "learning_rate": 2.2140372138502386e-06,
      "log_odds_chosen": 8.353876113891602,
      "log_odds_ratio": -0.011013227514922619,
      "logits/chosen": -1.9190175533294678,
      "logits/rejected": -1.9186252355575562,
      "logps/chosen": -0.060610998421907425,
      "logps/rejected": -3.5158791542053223,
      "loss": 0.073,
      "nll_loss": 0.0933486819267273,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0030305504333227873,
      "rewards/margins": 0.1727634072303772,
      "rewards/rejected": -0.17579396069049835,
      "step": 510
    },
    {
      "epoch": 1.06,
      "grad_norm": 4.6875,
      "learning_rate": 2.1926450482675734e-06,
      "log_odds_chosen": 8.398624420166016,
      "log_odds_ratio": -0.01034296303987503,
      "logits/chosen": -1.9005162715911865,
      "logits/rejected": -1.8965606689453125,
      "logps/chosen": -0.08098746836185455,
      "logps/rejected": -3.6455941200256348,
      "loss": 0.0753,
      "nll_loss": 0.0635688304901123,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.00404937332496047,
      "rewards/margins": 0.17823031544685364,
      "rewards/rejected": -0.18227970600128174,
      "step": 520
    },
    {
      "epoch": 1.08,
      "grad_norm": 5.4375,
      "learning_rate": 2.1718612138153473e-06,
      "log_odds_chosen": 8.60987663269043,
      "log_odds_ratio": -0.00940676499158144,
      "logits/chosen": -1.8967382907867432,
      "logits/rejected": -1.8937251567840576,
      "logps/chosen": -0.07309839874505997,
      "logps/rejected": -3.786154270172119,
      "loss": 0.0827,
      "nll_loss": 0.07813505828380585,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.003654920030385256,
      "rewards/margins": 0.18565279245376587,
      "rewards/rejected": -0.18930771946907043,
      "step": 530
    },
    {
      "epoch": 1.1,
      "grad_norm": 4.34375,
      "learning_rate": 2.151657414559676e-06,
      "log_odds_chosen": 9.486946105957031,
      "log_odds_ratio": -0.006830220576375723,
      "logits/chosen": -1.8446986675262451,
      "logits/rejected": -1.839769721031189,
      "logps/chosen": -0.045620597898960114,
      "logps/rejected": -4.278280258178711,
      "loss": 0.0759,
      "nll_loss": 0.061400435864925385,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.002281030174344778,
      "rewards/margins": 0.21163301169872284,
      "rewards/rejected": -0.21391403675079346,
      "step": 540
    },
    {
      "epoch": 1.12,
      "grad_norm": 3.328125,
      "learning_rate": 2.132007163556104e-06,
      "log_odds_chosen": 8.17650032043457,
      "log_odds_ratio": -0.015669699758291245,
      "logits/chosen": -1.83901047706604,
      "logits/rejected": -1.835837960243225,
      "logps/chosen": -0.08997470885515213,
      "logps/rejected": -3.697235584259033,
      "loss": 0.0766,
      "nll_loss": 0.05432655289769173,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.004498735070228577,
      "rewards/margins": 0.1803630292415619,
      "rewards/rejected": -0.18486176431179047,
      "step": 550
    },
    {
      "epoch": 1.14,
      "grad_norm": 4.75,
      "learning_rate": 2.1128856368212917e-06,
      "log_odds_chosen": 7.758215427398682,
      "log_odds_ratio": -0.027782147750258446,
      "logits/chosen": -1.887261152267456,
      "logits/rejected": -1.8827438354492188,
      "logps/chosen": -0.14876721799373627,
      "logps/rejected": -3.6743240356445312,
      "loss": 0.0828,
      "nll_loss": 0.11095689237117767,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.007438362576067448,
      "rewards/margins": 0.17627784609794617,
      "rewards/rejected": -0.18371620774269104,
      "step": 560
    },
    {
      "epoch": 1.16,
      "grad_norm": 4.5625,
      "learning_rate": 2.0942695414584777e-06,
      "log_odds_chosen": 8.34414291381836,
      "log_odds_ratio": -0.021350722759962082,
      "logits/chosen": -1.8923943042755127,
      "logits/rejected": -1.8904056549072266,
      "logps/chosen": -0.09869461506605148,
      "logps/rejected": -3.8127264976501465,
      "loss": 0.0633,
      "nll_loss": 0.06047019362449646,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.004934730939567089,
      "rewards/margins": 0.18570157885551453,
      "rewards/rejected": -0.19063633680343628,
      "step": 570
    },
    {
      "epoch": 1.18,
      "grad_norm": 4.09375,
      "learning_rate": 2.0761369963434992e-06,
      "log_odds_chosen": 8.360025405883789,
      "log_odds_ratio": -0.022461308166384697,
      "logits/chosen": -1.9567257165908813,
      "logits/rejected": -1.9553306102752686,
      "logps/chosen": -0.057451434433460236,
      "logps/rejected": -3.338317394256592,
      "loss": 0.0647,
      "nll_loss": 0.09119249880313873,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.0028725718148052692,
      "rewards/margins": 0.1640433371067047,
      "rewards/rejected": -0.1669158935546875,
      "step": 580
    },
    {
      "epoch": 1.2,
      "grad_norm": 4.53125,
      "learning_rate": 2.058467423981546e-06,
      "log_odds_chosen": 8.788118362426758,
      "log_odds_ratio": -0.014825952239334583,
      "logits/chosen": -2.0043745040893555,
      "logits/rejected": -2.00369930267334,
      "logps/chosen": -0.06359820067882538,
      "logps/rejected": -3.9866626262664795,
      "loss": 0.0737,
      "nll_loss": 0.10766579210758209,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.00317991035990417,
      "rewards/margins": 0.19615323841571808,
      "rewards/rejected": -0.19933313131332397,
      "step": 590
    },
    {
      "epoch": 1.22,
      "grad_norm": 4.53125,
      "learning_rate": 2.0412414523193154e-06,
      "log_odds_chosen": 8.787096977233887,
      "log_odds_ratio": -0.02094932086765766,
      "logits/chosen": -2.0005838871002197,
      "logits/rejected": -1.9980299472808838,
      "logps/chosen": -0.06464333832263947,
      "logps/rejected": -3.8292534351348877,
      "loss": 0.0665,
      "nll_loss": 0.07510735094547272,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0032321668695658445,
      "rewards/margins": 0.1882304847240448,
      "rewards/rejected": -0.1914626806974411,
      "step": 600
    },
    {
      "epoch": 1.24,
      "grad_norm": 5.0,
      "learning_rate": 2.0244408254472904e-06,
      "log_odds_chosen": 8.794069290161133,
      "log_odds_ratio": -0.0068406336940824986,
      "logits/chosen": -1.9200255870819092,
      "logits/rejected": -1.9168468713760376,
      "logps/chosen": -0.07637156546115875,
      "logps/rejected": -4.043463230133057,
      "loss": 0.0774,
      "nll_loss": 0.06419046223163605,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.00381857855245471,
      "rewards/margins": 0.1983545869588852,
      "rewards/rejected": -0.2021731585264206,
      "step": 610
    },
    {
      "epoch": 1.26,
      "grad_norm": 3.359375,
      "learning_rate": 2.0080483222562476e-06,
      "log_odds_chosen": 9.725520133972168,
      "log_odds_ratio": -0.010235416702926159,
      "logits/chosen": -1.8615529537200928,
      "logits/rejected": -1.8584880828857422,
      "logps/chosen": -0.07194074988365173,
      "logps/rejected": -4.24954891204834,
      "loss": 0.0592,
      "nll_loss": 0.045302603393793106,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0035970378667116165,
      "rewards/margins": 0.20888039469718933,
      "rewards/rejected": -0.212477445602417,
      "step": 620
    },
    {
      "epoch": 1.28,
      "grad_norm": 3.703125,
      "learning_rate": 1.9920476822239895e-06,
      "log_odds_chosen": 8.134096145629883,
      "log_odds_ratio": -0.02648916281759739,
      "logits/chosen": -1.987565279006958,
      "logits/rejected": -1.985645055770874,
      "logps/chosen": -0.07914531975984573,
      "logps/rejected": -3.5829226970672607,
      "loss": 0.076,
      "nll_loss": 0.06963127106428146,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.003957265987992287,
      "rewards/margins": 0.17518886923789978,
      "rewards/rejected": -0.17914614081382751,
      "step": 630
    },
    {
      "epoch": 1.3,
      "grad_norm": 4.65625,
      "learning_rate": 1.976423537605237e-06,
      "log_odds_chosen": 8.905769348144531,
      "log_odds_ratio": -0.0063996403478085995,
      "logits/chosen": -1.8973668813705444,
      "logits/rejected": -1.895581841468811,
      "logps/chosen": -0.04883747547864914,
      "logps/rejected": -3.7183055877685547,
      "loss": 0.0691,
      "nll_loss": 0.060373760759830475,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0024418740067631006,
      "rewards/margins": 0.1834733784198761,
      "rewards/rejected": -0.1859152615070343,
      "step": 640
    },
    {
      "epoch": 1.32,
      "grad_norm": 3.671875,
      "learning_rate": 1.961161351381841e-06,
      "log_odds_chosen": 8.213096618652344,
      "log_odds_ratio": -0.012629789300262928,
      "logits/chosen": -1.8932087421417236,
      "logits/rejected": -1.8902513980865479,
      "logps/chosen": -0.07331077754497528,
      "logps/rejected": -3.4578137397766113,
      "loss": 0.0703,
      "nll_loss": 0.060921620577573776,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0036655396688729525,
      "rewards/margins": 0.16922515630722046,
      "rewards/rejected": -0.17289067804813385,
      "step": 650
    },
    {
      "epoch": 1.34,
      "grad_norm": 4.1875,
      "learning_rate": 1.9462473604038077e-06,
      "log_odds_chosen": 8.2732572555542,
      "log_odds_ratio": -0.015530670061707497,
      "logits/chosen": -1.9228805303573608,
      "logits/rejected": -1.9188134670257568,
      "logps/chosen": -0.08276908099651337,
      "logps/rejected": -3.572401523590088,
      "loss": 0.0672,
      "nll_loss": 0.03623514249920845,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.004138454794883728,
      "rewards/margins": 0.1744816154241562,
      "rewards/rejected": -0.17862007021903992,
      "step": 660
    },
    {
      "epoch": 1.36,
      "grad_norm": 2.953125,
      "learning_rate": 1.9316685232156397e-06,
      "log_odds_chosen": 8.805624961853027,
      "log_odds_ratio": -0.0074573298916220665,
      "logits/chosen": -1.8942184448242188,
      "logits/rejected": -1.8891878128051758,
      "logps/chosen": -0.07351969927549362,
      "logps/rejected": -3.884014844894409,
      "loss": 0.0656,
      "nll_loss": 0.05570251867175102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0036759853828698397,
      "rewards/margins": 0.19052477180957794,
      "rewards/rejected": -0.1942007690668106,
      "step": 670
    },
    {
      "epoch": 1.38,
      "grad_norm": 3.328125,
      "learning_rate": 1.917412472118426e-06,
      "log_odds_chosen": 8.748679161071777,
      "log_odds_ratio": -0.024869108572602272,
      "logits/chosen": -1.9717075824737549,
      "logits/rejected": -1.9675525426864624,
      "logps/chosen": -0.06451079994440079,
      "logps/rejected": -3.9762165546417236,
      "loss": 0.073,
      "nll_loss": 0.11092261970043182,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.003225540043786168,
      "rewards/margins": 0.19558526575565338,
      "rewards/rejected": -0.19881080090999603,
      "step": 680
    },
    {
      "epoch": 1.4,
      "grad_norm": 4.90625,
      "learning_rate": 1.9034674690672024e-06,
      "log_odds_chosen": 8.910208702087402,
      "log_odds_ratio": -0.005920908413827419,
      "logits/chosen": -1.8858320713043213,
      "logits/rejected": -1.8857864141464233,
      "logps/chosen": -0.06332524120807648,
      "logps/rejected": -3.758251667022705,
      "loss": 0.0613,
      "nll_loss": 0.07212791591882706,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.003166262526065111,
      "rewards/margins": 0.18474632501602173,
      "rewards/rejected": -0.18791258335113525,
      "step": 690
    },
    {
      "epoch": 1.42,
      "grad_norm": 3.25,
      "learning_rate": 1.8898223650461362e-06,
      "log_odds_chosen": 8.65866756439209,
      "log_odds_ratio": -0.012265409342944622,
      "logits/chosen": -1.9048351049423218,
      "logits/rejected": -1.9031221866607666,
      "logps/chosen": -0.0720919519662857,
      "logps/rejected": -3.809000015258789,
      "loss": 0.0596,
      "nll_loss": 0.054995644837617874,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.003604597644880414,
      "rewards/margins": 0.18684543669223785,
      "rewards/rejected": -0.1904500126838684,
      "step": 700
    },
    {
      "epoch": 1.44,
      "grad_norm": 3.34375,
      "learning_rate": 1.876466562602004e-06,
      "log_odds_chosen": 8.907628059387207,
      "log_odds_ratio": -0.01111848745495081,
      "logits/chosen": -1.945853590965271,
      "logits/rejected": -1.942375898361206,
      "logps/chosen": -0.07798963785171509,
      "logps/rejected": -4.1663408279418945,
      "loss": 0.0662,
      "nll_loss": 0.08023327589035034,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.003899481613188982,
      "rewards/margins": 0.20441755652427673,
      "rewards/rejected": -0.20831704139709473,
      "step": 710
    },
    {
      "epoch": 1.46,
      "grad_norm": 5.28125,
      "learning_rate": 1.863389981249825e-06,
      "log_odds_chosen": 9.416102409362793,
      "log_odds_ratio": -0.007732940372079611,
      "logits/chosen": -1.8631290197372437,
      "logits/rejected": -1.8593025207519531,
      "logps/chosen": -0.06107889488339424,
      "logps/rejected": -4.503323554992676,
      "loss": 0.0585,
      "nll_loss": 0.05343944951891899,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0030539450235664845,
      "rewards/margins": 0.22211222350597382,
      "rewards/rejected": -0.2251661717891693,
      "step": 720
    },
    {
      "epoch": 1.48,
      "grad_norm": 2.78125,
      "learning_rate": 1.8505830254940132e-06,
      "log_odds_chosen": 9.134099960327148,
      "log_odds_ratio": -0.011219611391425133,
      "logits/chosen": -1.9615509510040283,
      "logits/rejected": -1.9607875347137451,
      "logps/chosen": -0.05891376733779907,
      "logps/rejected": -4.027497291564941,
      "loss": 0.0582,
      "nll_loss": 0.050385020673274994,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0029456880874931812,
      "rewards/margins": 0.1984291821718216,
      "rewards/rejected": -0.2013748586177826,
      "step": 730
    },
    {
      "epoch": 1.5,
      "grad_norm": 4.0625,
      "learning_rate": 1.8380365552345197e-06,
      "log_odds_chosen": 8.314208984375,
      "log_odds_ratio": -0.017240628600120544,
      "logits/chosen": -1.9306405782699585,
      "logits/rejected": -1.9276716709136963,
      "logps/chosen": -0.12811905145645142,
      "logps/rejected": -3.857971668243408,
      "loss": 0.0666,
      "nll_loss": 0.07230392098426819,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.006405952386558056,
      "rewards/margins": 0.18649262189865112,
      "rewards/rejected": -0.19289858639240265,
      "step": 740
    },
    {
      "epoch": 1.52,
      "grad_norm": 6.8125,
      "learning_rate": 1.8257418583505536e-06,
      "log_odds_chosen": 8.118074417114258,
      "log_odds_ratio": -0.04242696613073349,
      "logits/chosen": -1.951964020729065,
      "logits/rejected": -1.9483181238174438,
      "logps/chosen": -0.07924682646989822,
      "logps/rejected": -3.6926894187927246,
      "loss": 0.0608,
      "nll_loss": 0.05164622515439987,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.003962341230362654,
      "rewards/margins": 0.18067213892936707,
      "rewards/rejected": -0.1846344769001007,
      "step": 750
    },
    {
      "epoch": 1.54,
      "grad_norm": 2.890625,
      "learning_rate": 1.8136906252750293e-06,
      "log_odds_chosen": 8.884002685546875,
      "log_odds_ratio": -0.007163527421653271,
      "logits/chosen": -1.8522392511367798,
      "logits/rejected": -1.8525257110595703,
      "logps/chosen": -0.06406822055578232,
      "logps/rejected": -3.8873114585876465,
      "loss": 0.0508,
      "nll_loss": 0.052127063274383545,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.003203411353752017,
      "rewards/margins": 0.19116216897964478,
      "rewards/rejected": -0.19436559081077576,
      "step": 760
    },
    {
      "epoch": 1.57,
      "grad_norm": 4.8125,
      "learning_rate": 1.801874925391118e-06,
      "log_odds_chosen": 8.48697566986084,
      "log_odds_ratio": -0.026209736242890358,
      "logits/chosen": -1.9563274383544922,
      "logits/rejected": -1.9566619396209717,
      "logps/chosen": -0.075563944876194,
      "logps/rejected": -3.6374893188476562,
      "loss": 0.0665,
      "nll_loss": 0.05961776524782181,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.0037781973369419575,
      "rewards/margins": 0.17809626460075378,
      "rewards/rejected": -0.18187445402145386,
      "step": 770
    },
    {
      "epoch": 1.59,
      "grad_norm": 3.703125,
      "learning_rate": 1.7902871850985824e-06,
      "log_odds_chosen": 9.119174003601074,
      "log_odds_ratio": -0.024956868961453438,
      "logits/chosen": -1.918013572692871,
      "logits/rejected": -1.9134963750839233,
      "logps/chosen": -0.05534970015287399,
      "logps/rejected": -4.010322570800781,
      "loss": 0.0595,
      "nll_loss": 0.04103861376643181,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.0027674853336066008,
      "rewards/margins": 0.19774861633777618,
      "rewards/rejected": -0.20051613450050354,
      "step": 780
    },
    {
      "epoch": 1.61,
      "grad_norm": 3.6875,
      "learning_rate": 1.7789201674120502e-06,
      "log_odds_chosen": 8.733691215515137,
      "log_odds_ratio": -0.008985884487628937,
      "logits/chosen": -1.8730590343475342,
      "logits/rejected": -1.8695628643035889,
      "logps/chosen": -0.05646519735455513,
      "logps/rejected": -3.503816604614258,
      "loss": 0.0506,
      "nll_loss": 0.047934651374816895,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.002823260147124529,
      "rewards/margins": 0.17236760258674622,
      "rewards/rejected": -0.17519085109233856,
      "step": 790
    },
    {
      "epoch": 1.63,
      "grad_norm": 2.140625,
      "learning_rate": 1.7677669529663689e-06,
      "log_odds_chosen": 8.943314552307129,
      "log_odds_ratio": -0.00942598469555378,
      "logits/chosen": -1.8948551416397095,
      "logits/rejected": -1.892686128616333,
      "logps/chosen": -0.05206329748034477,
      "logps/rejected": -3.80363130569458,
      "loss": 0.0526,
      "nll_loss": 0.06757352501153946,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0026031648740172386,
      "rewards/margins": 0.18757839500904083,
      "rewards/rejected": -0.19018153846263885,
      "step": 800
    },
    {
      "epoch": 1.65,
      "grad_norm": 3.515625,
      "learning_rate": 1.7568209223157664e-06,
      "log_odds_chosen": 9.602978706359863,
      "log_odds_ratio": -0.0054091801866889,
      "logits/chosen": -1.8749698400497437,
      "logits/rejected": -1.8726646900177002,
      "logps/chosen": -0.049051497131586075,
      "logps/rejected": -4.167695045471191,
      "loss": 0.0527,
      "nll_loss": 0.08227180689573288,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0024525749031454325,
      "rewards/margins": 0.20593218505382538,
      "rewards/rejected": -0.20838478207588196,
      "step": 810
    },
    {
      "epoch": 1.67,
      "grad_norm": 4.1875,
      "learning_rate": 1.7460757394239458e-06,
      "log_odds_chosen": 8.731072425842285,
      "log_odds_ratio": -0.006553919520229101,
      "logits/chosen": -1.9533252716064453,
      "logits/rejected": -1.9529660940170288,
      "logps/chosen": -0.0727374479174614,
      "logps/rejected": -3.5265769958496094,
      "loss": 0.0659,
      "nll_loss": 0.0733790472149849,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.003636872861534357,
      "rewards/margins": 0.17269198596477509,
      "rewards/rejected": -0.1763288527727127,
      "step": 820
    },
    {
      "epoch": 1.69,
      "grad_norm": 2.984375,
      "learning_rate": 1.7355253362515584e-06,
      "log_odds_chosen": 9.135687828063965,
      "log_odds_ratio": -0.03666002303361893,
      "logits/chosen": -1.9465566873550415,
      "logits/rejected": -1.9425933361053467,
      "logps/chosen": -0.07385378330945969,
      "logps/rejected": -3.989952564239502,
      "loss": 0.0478,
      "nll_loss": 0.06535151600837708,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.003692689584568143,
      "rewards/margins": 0.1958049088716507,
      "rewards/rejected": -0.19949761033058167,
      "step": 830
    },
    {
      "epoch": 1.71,
      "grad_norm": 2.9375,
      "learning_rate": 1.7251638983558855e-06,
      "log_odds_chosen": 8.61412239074707,
      "log_odds_ratio": -0.006854639388620853,
      "logits/chosen": -1.8970463275909424,
      "logits/rejected": -1.8957879543304443,
      "logps/chosen": -0.09370087087154388,
      "logps/rejected": -3.841550827026367,
      "loss": 0.0587,
      "nll_loss": 0.046143922954797745,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.004685043357312679,
      "rewards/margins": 0.18739250302314758,
      "rewards/rejected": -0.19207754731178284,
      "step": 840
    },
    {
      "epoch": 1.73,
      "grad_norm": 3.40625,
      "learning_rate": 1.7149858514250883e-06,
      "log_odds_chosen": 9.044164657592773,
      "log_odds_ratio": -0.023376554250717163,
      "logits/chosen": -1.9251768589019775,
      "logits/rejected": -1.9241018295288086,
      "logps/chosen": -0.05019516497850418,
      "logps/rejected": -3.740339994430542,
      "loss": 0.0537,
      "nll_loss": 0.04762560874223709,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.0025097583420574665,
      "rewards/margins": 0.18450722098350525,
      "rewards/rejected": -0.18701700866222382,
      "step": 850
    },
    {
      "epoch": 1.75,
      "grad_norm": 2.703125,
      "learning_rate": 1.704985848676184e-06,
      "log_odds_chosen": 8.560470581054688,
      "log_odds_ratio": -0.015030615031719208,
      "logits/chosen": -1.893930435180664,
      "logits/rejected": -1.8902561664581299,
      "logps/chosen": -0.07954917848110199,
      "logps/rejected": -3.786511182785034,
      "loss": 0.0544,
      "nll_loss": 0.054652292281389236,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0039774589240550995,
      "rewards/margins": 0.18534812331199646,
      "rewards/rejected": -0.18932557106018066,
      "step": 860
    },
    {
      "epoch": 1.77,
      "grad_norm": 2.9375,
      "learning_rate": 1.6951587590520263e-06,
      "log_odds_chosen": 9.6688232421875,
      "log_odds_ratio": -0.010795338079333305,
      "logits/chosen": -1.9225839376449585,
      "logits/rejected": -1.9220905303955078,
      "logps/chosen": -0.048293761909008026,
      "logps/rejected": -4.323627948760986,
      "loss": 0.0488,
      "nll_loss": 0.043466441333293915,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.002414688002318144,
      "rewards/margins": 0.21376672387123108,
      "rewards/rejected": -0.2161814272403717,
      "step": 870
    },
    {
      "epoch": 1.79,
      "grad_norm": 3.234375,
      "learning_rate": 1.6854996561581053e-06,
      "log_odds_chosen": 9.363157272338867,
      "log_odds_ratio": -0.026759719476103783,
      "logits/chosen": -1.9485028982162476,
      "logits/rejected": -1.9452836513519287,
      "logps/chosen": -0.042739786207675934,
      "logps/rejected": -3.5607593059539795,
      "loss": 0.0555,
      "nll_loss": 0.04253282770514488,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.002136989263817668,
      "rewards/margins": 0.17590096592903137,
      "rewards/rejected": -0.17803797125816345,
      "step": 880
    },
    {
      "epoch": 1.81,
      "grad_norm": 4.1875,
      "learning_rate": 1.6760038078849776e-06,
      "log_odds_chosen": 8.319189071655273,
      "log_odds_ratio": -0.023240404203534126,
      "logits/chosen": -1.851123571395874,
      "logits/rejected": -1.849542260169983,
      "logps/chosen": -0.06788856536149979,
      "logps/rejected": -3.406628131866455,
      "loss": 0.0603,
      "nll_loss": 0.04571394622325897,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.003394428174942732,
      "rewards/margins": 0.1669369637966156,
      "rewards/rejected": -0.17033140361309052,
      "step": 890
    },
    {
      "epoch": 1.83,
      "grad_norm": 4.21875,
      "learning_rate": 1.6666666666666667e-06,
      "log_odds_chosen": 8.61945915222168,
      "log_odds_ratio": -0.017158176749944687,
      "logits/chosen": -1.9368407726287842,
      "logits/rejected": -1.9338048696517944,
      "logps/chosen": -0.05057498812675476,
      "logps/rejected": -3.4434471130371094,
      "loss": 0.0499,
      "nll_loss": 0.05269382521510124,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0025287498719990253,
      "rewards/margins": 0.1696436107158661,
      "rewards/rejected": -0.17217235267162323,
      "step": 900
    },
    {
      "epoch": 1.85,
      "grad_norm": 3.265625,
      "learning_rate": 1.6574838603294898e-06,
      "log_odds_chosen": 9.762075424194336,
      "log_odds_ratio": -0.0037377229891717434,
      "logits/chosen": -2.0028135776519775,
      "logits/rejected": -2.0014703273773193,
      "logps/chosen": -0.034680210053920746,
      "logps/rejected": -4.053231239318848,
      "loss": 0.0576,
      "nll_loss": 0.057746343314647675,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0017340105259791017,
      "rewards/margins": 0.20092757046222687,
      "rewards/rejected": -0.20266155898571014,
      "step": 910
    },
    {
      "epoch": 1.87,
      "grad_norm": 3.0,
      "learning_rate": 1.648451183489468e-06,
      "log_odds_chosen": 9.181684494018555,
      "log_odds_ratio": -0.007746308110654354,
      "logits/chosen": -1.935484528541565,
      "logits/rejected": -1.932363748550415,
      "logps/chosen": -0.08534862101078033,
      "logps/rejected": -4.044074535369873,
      "loss": 0.0526,
      "nll_loss": 0.06775641441345215,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.004267430864274502,
      "rewards/margins": 0.1979363113641739,
      "rewards/rejected": -0.20220375061035156,
      "step": 920
    },
    {
      "epoch": 1.89,
      "grad_norm": 1.609375,
      "learning_rate": 1.6395645894598825e-06,
      "log_odds_chosen": 9.165306091308594,
      "log_odds_ratio": -0.01449726801365614,
      "logits/chosen": -1.9488122463226318,
      "logits/rejected": -1.9472631216049194,
      "logps/chosen": -0.07069545984268188,
      "logps/rejected": -3.8849570751190186,
      "loss": 0.057,
      "nll_loss": 0.052025146782398224,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0035347728990018368,
      "rewards/margins": 0.1907130777835846,
      "rewards/rejected": -0.19424787163734436,
      "step": 930
    },
    {
      "epoch": 1.91,
      "grad_norm": 3.90625,
      "learning_rate": 1.6308201826336057e-06,
      "log_odds_chosen": 9.020658493041992,
      "log_odds_ratio": -0.00849425233900547,
      "logits/chosen": -1.916486144065857,
      "logits/rejected": -1.9109046459197998,
      "logps/chosen": -0.06480909883975983,
      "logps/rejected": -4.059305667877197,
      "loss": 0.0504,
      "nll_loss": 0.058377087116241455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0032404554076492786,
      "rewards/margins": 0.19972485303878784,
      "rewards/rejected": -0.20296530425548553,
      "step": 940
    },
    {
      "epoch": 1.93,
      "grad_norm": 3.75,
      "learning_rate": 1.6222142113076255e-06,
      "log_odds_chosen": 9.191986083984375,
      "log_odds_ratio": -0.007987757213413715,
      "logits/chosen": -1.8939701318740845,
      "logits/rejected": -1.892139196395874,
      "logps/chosen": -0.042362868785858154,
      "logps/rejected": -3.922192335128784,
      "loss": 0.0543,
      "nll_loss": 0.0392763689160347,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0021181434858590364,
      "rewards/margins": 0.19399148225784302,
      "rewards/rejected": -0.1961096078157425,
      "step": 950
    },
    {
      "epoch": 1.95,
      "grad_norm": 3.65625,
      "learning_rate": 1.6137430609197571e-06,
      "log_odds_chosen": 8.950517654418945,
      "log_odds_ratio": -0.01621365174651146,
      "logits/chosen": -1.9614036083221436,
      "logits/rejected": -1.9606664180755615,
      "logps/chosen": -0.07848796993494034,
      "logps/rejected": -4.038422584533691,
      "loss": 0.0554,
      "nll_loss": 0.0618586540222168,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.003924398683011532,
      "rewards/margins": 0.19799675047397614,
      "rewards/rejected": -0.20192115008831024,
      "step": 960
    },
    {
      "epoch": 1.97,
      "grad_norm": 2.875,
      "learning_rate": 1.605403247669839e-06,
      "log_odds_chosen": 8.41962718963623,
      "log_odds_ratio": -0.020061850547790527,
      "logits/chosen": -1.908539056777954,
      "logits/rejected": -1.9044193029403687,
      "logps/chosen": -0.07218164205551147,
      "logps/rejected": -3.6821017265319824,
      "loss": 0.0528,
      "nll_loss": 0.03915850818157196,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.003609082894399762,
      "rewards/margins": 0.1804960072040558,
      "rewards/rejected": -0.18410511314868927,
      "step": 970
    },
    {
      "epoch": 1.99,
      "grad_norm": 4.4375,
      "learning_rate": 1.59719141249985e-06,
      "log_odds_chosen": 8.960528373718262,
      "log_odds_ratio": -0.007182210683822632,
      "logits/chosen": -2.059950351715088,
      "logits/rejected": -2.058908224105835,
      "logps/chosen": -0.07827770709991455,
      "logps/rejected": -3.9805305004119873,
      "loss": 0.0589,
      "nll_loss": 0.08598147332668304,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.003913884982466698,
      "rewards/margins": 0.19511264562606812,
      "rewards/rejected": -0.19902652502059937,
      "step": 980
    },
    {
      "epoch": 2.01,
      "grad_norm": 2.421875,
      "learning_rate": 1.5891043154093205e-06,
      "log_odds_chosen": 8.75261402130127,
      "log_odds_ratio": -0.008246051147580147,
      "logits/chosen": -1.9300010204315186,
      "logits/rejected": -1.9284627437591553,
      "logps/chosen": -0.059024132788181305,
      "logps/rejected": -3.7112643718719482,
      "loss": 0.0501,
      "nll_loss": 0.0379130020737648,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0029512066394090652,
      "rewards/margins": 0.18261203169822693,
      "rewards/rejected": -0.18556323647499084,
      "step": 990
    },
    {
      "epoch": 2.03,
      "grad_norm": 2.4375,
      "learning_rate": 1.5811388300841898e-06,
      "log_odds_chosen": 9.132776260375977,
      "log_odds_ratio": -0.010855279862880707,
      "logits/chosen": -1.9048831462860107,
      "logits/rejected": -1.9021356105804443,
      "logps/chosen": -0.05851290374994278,
      "logps/rejected": -3.9342761039733887,
      "loss": 0.0399,
      "nll_loss": 0.045368582010269165,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.002925645327195525,
      "rewards/margins": 0.19378817081451416,
      "rewards/rejected": -0.19671380519866943,
      "step": 1000
    },
    {
      "epoch": 2.05,
      "grad_norm": 3.34375,
      "learning_rate": 1.5732919388188816e-06,
      "log_odds_chosen": 9.254007339477539,
      "log_odds_ratio": -0.004844398703426123,
      "logits/chosen": -1.906938910484314,
      "logits/rejected": -1.9027900695800781,
      "logps/chosen": -0.0639704093337059,
      "logps/rejected": -4.133778095245361,
      "loss": 0.0377,
      "nll_loss": 0.03271421417593956,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.003198520513251424,
      "rewards/margins": 0.20349040627479553,
      "rewards/rejected": -0.20668891072273254,
      "step": 1010
    },
    {
      "epoch": 2.07,
      "grad_norm": 2.34375,
      "learning_rate": 1.565560727712874e-06,
      "log_odds_chosen": 9.79780387878418,
      "log_odds_ratio": -0.0035797320306301117,
      "logits/chosen": -1.934853196144104,
      "logits/rejected": -1.9338003396987915,
      "logps/chosen": -0.05114441365003586,
      "logps/rejected": -4.095462799072266,
      "loss": 0.0456,
      "nll_loss": 0.055598996579647064,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.002557220868766308,
      "rewards/margins": 0.20221590995788574,
      "rewards/rejected": -0.20477311313152313,
      "step": 1020
    },
    {
      "epoch": 2.09,
      "grad_norm": 2.140625,
      "learning_rate": 1.5579423821243897e-06,
      "log_odds_chosen": 10.34232234954834,
      "log_odds_ratio": -0.002075557829812169,
      "logits/chosen": -1.908393144607544,
      "logits/rejected": -1.9075828790664673,
      "logps/chosen": -0.03475125879049301,
      "logps/rejected": -4.130476474761963,
      "loss": 0.0436,
      "nll_loss": 0.04652316868305206,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0017375629395246506,
      "rewards/margins": 0.2047862708568573,
      "rewards/rejected": -0.2065238207578659,
      "step": 1030
    },
    {
      "epoch": 2.11,
      "grad_norm": 3.796875,
      "learning_rate": 1.5504341823651056e-06,
      "log_odds_chosen": 9.534278869628906,
      "log_odds_ratio": -0.014330191537737846,
      "logits/chosen": -1.8937572240829468,
      "logits/rejected": -1.8925514221191406,
      "logps/chosen": -0.05281580239534378,
      "logps/rejected": -3.8376965522766113,
      "loss": 0.0506,
      "nll_loss": 0.04847393184900284,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0026407900732010603,
      "rewards/margins": 0.18924403190612793,
      "rewards/rejected": -0.1918848156929016,
      "step": 1040
    },
    {
      "epoch": 2.13,
      "grad_norm": 2.765625,
      "learning_rate": 1.5430334996209192e-06,
      "log_odds_chosen": 9.982309341430664,
      "log_odds_ratio": -0.0034835089463740587,
      "logits/chosen": -1.9073753356933594,
      "logits/rejected": -1.9033172130584717,
      "logps/chosen": -0.03734086453914642,
      "logps/rejected": -4.198154449462891,
      "loss": 0.0399,
      "nll_loss": 0.03343256562948227,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0018670432036742568,
      "rewards/margins": 0.20804063975811005,
      "rewards/rejected": -0.20990769565105438,
      "step": 1050
    },
    {
      "epoch": 2.15,
      "grad_norm": 2.015625,
      "learning_rate": 1.5357377920848783e-06,
      "log_odds_chosen": 9.932341575622559,
      "log_odds_ratio": -0.00664928276091814,
      "logits/chosen": -1.968000054359436,
      "logits/rejected": -1.9650490283966064,
      "logps/chosen": -0.053252361714839935,
      "logps/rejected": -4.3154191970825195,
      "loss": 0.0398,
      "nll_loss": 0.03708261623978615,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0026626181788742542,
      "rewards/margins": 0.2131083756685257,
      "rewards/rejected": -0.21577095985412598,
      "step": 1060
    },
    {
      "epoch": 2.17,
      "grad_norm": 3.203125,
      "learning_rate": 1.5285446012893579e-06,
      "log_odds_chosen": 10.412023544311523,
      "log_odds_ratio": -0.004382356535643339,
      "logits/chosen": -1.8630377054214478,
      "logits/rejected": -1.858724594116211,
      "logps/chosen": -0.03156786039471626,
      "logps/rejected": -4.467475414276123,
      "loss": 0.0435,
      "nll_loss": 0.043990008533000946,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0015783928101882339,
      "rewards/margins": 0.22179536521434784,
      "rewards/rejected": -0.22337380051612854,
      "step": 1070
    },
    {
      "epoch": 2.2,
      "grad_norm": 2.859375,
      "learning_rate": 1.5214515486254614e-06,
      "log_odds_chosen": 9.860450744628906,
      "log_odds_ratio": -0.00421870406717062,
      "logits/chosen": -1.9334264993667603,
      "logits/rejected": -1.9309759140014648,
      "logps/chosen": -0.04163174703717232,
      "logps/rejected": -4.278031349182129,
      "loss": 0.0399,
      "nll_loss": 0.03083736263215542,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0020815874449908733,
      "rewards/margins": 0.21182000637054443,
      "rewards/rejected": -0.2139015644788742,
      "step": 1080
    },
    {
      "epoch": 2.22,
      "grad_norm": 3.375,
      "learning_rate": 1.5144563320384566e-06,
      "log_odds_chosen": 9.61739444732666,
      "log_odds_ratio": -0.007051125168800354,
      "logits/chosen": -1.9654079675674438,
      "logits/rejected": -1.9627532958984375,
      "logps/chosen": -0.04155623912811279,
      "logps/rejected": -3.9517130851745605,
      "loss": 0.0419,
      "nll_loss": 0.0399235263466835,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.002077811863273382,
      "rewards/margins": 0.19550786912441254,
      "rewards/rejected": -0.19758570194244385,
      "step": 1090
    },
    {
      "epoch": 2.24,
      "grad_norm": 2.5625,
      "learning_rate": 1.5075567228888182e-06,
      "log_odds_chosen": 9.472233772277832,
      "log_odds_ratio": -0.004241319373250008,
      "logits/chosen": -1.8757314682006836,
      "logits/rejected": -1.875422477722168,
      "logps/chosen": -0.04146653413772583,
      "logps/rejected": -3.9626433849334717,
      "loss": 0.0388,
      "nll_loss": 0.044396962970495224,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.002073326613754034,
      "rewards/margins": 0.19605886936187744,
      "rewards/rejected": -0.19813218712806702,
      "step": 1100
    },
    {
      "epoch": 2.26,
      "grad_norm": 2.84375,
      "learning_rate": 1.5007505629691608e-06,
      "log_odds_chosen": 10.060752868652344,
      "log_odds_ratio": -0.005245304200798273,
      "logits/chosen": -1.884495735168457,
      "logits/rejected": -1.8812463283538818,
      "logps/chosen": -0.042206455022096634,
      "logps/rejected": -4.164919853210449,
      "loss": 0.0437,
      "nll_loss": 0.03563828393816948,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.002110322704538703,
      "rewards/margins": 0.20613567531108856,
      "rewards/rejected": -0.20824599266052246,
      "step": 1110
    },
    {
      "epoch": 2.28,
      "grad_norm": 1.875,
      "learning_rate": 1.494035761667992e-06,
      "log_odds_chosen": 10.209965705871582,
      "log_odds_ratio": -0.00312315602786839,
      "logits/chosen": -1.921362280845642,
      "logits/rejected": -1.9180587530136108,
      "logps/chosen": -0.04360593855381012,
      "logps/rejected": -4.336488723754883,
      "loss": 0.0425,
      "nll_loss": 0.050070591270923615,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.002180296927690506,
      "rewards/margins": 0.2146441489458084,
      "rewards/rejected": -0.21682441234588623,
      "step": 1120
    },
    {
      "epoch": 2.3,
      "grad_norm": 3.34375,
      "learning_rate": 1.487410293271824e-06,
      "log_odds_chosen": 9.571352005004883,
      "log_odds_ratio": -0.0124302227050066,
      "logits/chosen": -1.9245920181274414,
      "logits/rejected": -1.9231822490692139,
      "logps/chosen": -0.051782481372356415,
      "logps/rejected": -4.130411624908447,
      "loss": 0.042,
      "nll_loss": 0.04519005864858627,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.002589124022051692,
      "rewards/margins": 0.20393142104148865,
      "rewards/rejected": -0.20652055740356445,
      "step": 1130
    },
    {
      "epoch": 2.32,
      "grad_norm": 2.0625,
      "learning_rate": 1.480872194397731e-06,
      "log_odds_chosen": 9.836039543151855,
      "log_odds_ratio": -0.004647588822990656,
      "logits/chosen": -1.8972570896148682,
      "logits/rejected": -1.8939294815063477,
      "logps/chosen": -0.0661562830209732,
      "logps/rejected": -4.237457752227783,
      "loss": 0.0435,
      "nll_loss": 0.04812124744057655,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0033078144770115614,
      "rewards/margins": 0.20856507122516632,
      "rewards/rejected": -0.211872860789299,
      "step": 1140
    },
    {
      "epoch": 2.34,
      "grad_norm": 3.546875,
      "learning_rate": 1.4744195615489715e-06,
      "log_odds_chosen": 10.264336585998535,
      "log_odds_ratio": -0.0017655363772064447,
      "logits/chosen": -1.9511454105377197,
      "logits/rejected": -1.9473825693130493,
      "logps/chosen": -0.04254579171538353,
      "logps/rejected": -4.465078353881836,
      "loss": 0.0408,
      "nll_loss": 0.03954671695828438,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0021272895392030478,
      "rewards/margins": 0.22112664580345154,
      "rewards/rejected": -0.22325392067432404,
      "step": 1150
    },
    {
      "epoch": 2.36,
      "grad_norm": 2.203125,
      "learning_rate": 1.4680505487867589e-06,
      "log_odds_chosen": 9.718263626098633,
      "log_odds_ratio": -0.002675818046554923,
      "logits/chosen": -1.9049984216690063,
      "logits/rejected": -1.9013592004776,
      "logps/chosen": -0.04851164296269417,
      "logps/rejected": -4.104137420654297,
      "loss": 0.0396,
      "nll_loss": 0.03321639448404312,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0024255819153040648,
      "rewards/margins": 0.2027812898159027,
      "rewards/rejected": -0.20520684123039246,
      "step": 1160
    },
    {
      "epoch": 2.38,
      "grad_norm": 1.8828125,
      "learning_rate": 1.4617633655117156e-06,
      "log_odds_chosen": 9.280858993530273,
      "log_odds_ratio": -0.008265355601906776,
      "logits/chosen": -1.964536428451538,
      "logits/rejected": -1.9617042541503906,
      "logps/chosen": -0.06000961735844612,
      "logps/rejected": -4.104333400726318,
      "loss": 0.0435,
      "nll_loss": 0.04580981656908989,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.003000481054186821,
      "rewards/margins": 0.20221619307994843,
      "rewards/rejected": -0.2052166759967804,
      "step": 1170
    },
    {
      "epoch": 2.4,
      "grad_norm": 2.578125,
      "learning_rate": 1.4555562743489552e-06,
      "log_odds_chosen": 10.110807418823242,
      "log_odds_ratio": -0.003054619999602437,
      "logits/chosen": -1.925559401512146,
      "logits/rejected": -1.924526572227478,
      "logps/chosen": -0.03492622449994087,
      "logps/rejected": -4.048843860626221,
      "loss": 0.0415,
      "nll_loss": 0.040362752974033356,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.001746311434544623,
      "rewards/margins": 0.20069590210914612,
      "rewards/rejected": -0.2024422138929367,
      "step": 1180
    },
    {
      "epoch": 2.42,
      "grad_norm": 2.53125,
      "learning_rate": 1.4494275891311214e-06,
      "log_odds_chosen": 9.7469482421875,
      "log_odds_ratio": -0.012390781193971634,
      "logits/chosen": -1.8723634481430054,
      "logits/rejected": -1.8694031238555908,
      "logps/chosen": -0.045554567128419876,
      "logps/rejected": -4.15027379989624,
      "loss": 0.0354,
      "nll_loss": 0.03426050767302513,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.002277728170156479,
      "rewards/margins": 0.20523595809936523,
      "rewards/rejected": -0.2075137197971344,
      "step": 1190
    },
    {
      "epoch": 2.44,
      "grad_norm": 2.140625,
      "learning_rate": 1.4433756729740647e-06,
      "log_odds_chosen": 10.098226547241211,
      "log_odds_ratio": -0.0019627574365586042,
      "logits/chosen": -1.9441982507705688,
      "logits/rejected": -1.9426124095916748,
      "logps/chosen": -0.02355217933654785,
      "logps/rejected": -4.209158897399902,
      "loss": 0.0457,
      "nll_loss": 0.053135812282562256,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0011776090832427144,
      "rewards/margins": 0.20928029716014862,
      "rewards/rejected": -0.2104579210281372,
      "step": 1200
    },
    {
      "epoch": 2.46,
      "grad_norm": 3.609375,
      "learning_rate": 1.4373989364401727e-06,
      "log_odds_chosen": 9.691657066345215,
      "log_odds_ratio": -0.004629252012819052,
      "logits/chosen": -1.9337596893310547,
      "logits/rejected": -1.931654930114746,
      "logps/chosen": -0.06260120123624802,
      "logps/rejected": -4.166058540344238,
      "loss": 0.0411,
      "nll_loss": 0.03751087933778763,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.003130060387775302,
      "rewards/margins": 0.20517286658287048,
      "rewards/rejected": -0.20830294489860535,
      "step": 1210
    },
    {
      "epoch": 2.48,
      "grad_norm": 1.3984375,
      "learning_rate": 1.4314958357846706e-06,
      "log_odds_chosen": 9.472856521606445,
      "log_odds_ratio": -0.00650912756100297,
      "logits/chosen": -1.95502507686615,
      "logits/rejected": -1.951743721961975,
      "logps/chosen": -0.07376778870820999,
      "logps/rejected": -4.022158145904541,
      "loss": 0.0357,
      "nll_loss": 0.03547949343919754,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.003688389901071787,
      "rewards/margins": 0.19741952419281006,
      "rewards/rejected": -0.2011079341173172,
      "step": 1220
    },
    {
      "epoch": 2.5,
      "grad_norm": 2.59375,
      "learning_rate": 1.4256648712805027e-06,
      "log_odds_chosen": 10.319469451904297,
      "log_odds_ratio": -0.001875861780717969,
      "logits/chosen": -1.9282852411270142,
      "logits/rejected": -1.9243885278701782,
      "logps/chosen": -0.0374315045773983,
      "logps/rejected": -4.336482524871826,
      "loss": 0.0412,
      "nll_loss": 0.0424610860645771,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0018715756013989449,
      "rewards/margins": 0.21495255827903748,
      "rewards/rejected": -0.21682414412498474,
      "step": 1230
    },
    {
      "epoch": 2.52,
      "grad_norm": 3.78125,
      "learning_rate": 1.419904585617662e-06,
      "log_odds_chosen": 10.347482681274414,
      "log_odds_ratio": -0.0033975064288824797,
      "logits/chosen": -1.9401060342788696,
      "logits/rejected": -1.9385757446289062,
      "logps/chosen": -0.04415961354970932,
      "logps/rejected": -4.426904201507568,
      "loss": 0.0403,
      "nll_loss": 0.04613417014479637,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0022079809568822384,
      "rewards/margins": 0.21913722157478333,
      "rewards/rejected": -0.2213451862335205,
      "step": 1240
    },
    {
      "epoch": 2.54,
      "grad_norm": 2.53125,
      "learning_rate": 1.4142135623730952e-06,
      "log_odds_chosen": 10.331408500671387,
      "log_odds_ratio": -0.0018885558238252997,
      "logits/chosen": -1.940860390663147,
      "logits/rejected": -1.9384444952011108,
      "logps/chosen": -0.03379340097308159,
      "logps/rejected": -4.056138038635254,
      "loss": 0.0393,
      "nll_loss": 0.030756259337067604,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0016896702582016587,
      "rewards/margins": 0.20111723244190216,
      "rewards/rejected": -0.20280690491199493,
      "step": 1250
    },
    {
      "epoch": 2.56,
      "grad_norm": 2.109375,
      "learning_rate": 1.4085904245475275e-06,
      "log_odds_chosen": 10.41731071472168,
      "log_odds_ratio": -0.012277336791157722,
      "logits/chosen": -1.9542973041534424,
      "logits/rejected": -1.953238844871521,
      "logps/chosen": -0.030469322577118874,
      "logps/rejected": -4.482726097106934,
      "loss": 0.0407,
      "nll_loss": 0.04968537390232086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.001523466082289815,
      "rewards/margins": 0.22261285781860352,
      "rewards/rejected": -0.2241363227367401,
      "step": 1260
    },
    {
      "epoch": 2.58,
      "grad_norm": 2.75,
      "learning_rate": 1.4030338331657844e-06,
      "log_odds_chosen": 10.663671493530273,
      "log_odds_ratio": -0.005900475196540356,
      "logits/chosen": -1.8991870880126953,
      "logits/rejected": -1.8935034275054932,
      "logps/chosen": -0.036128245294094086,
      "logps/rejected": -4.496649742126465,
      "loss": 0.036,
      "nll_loss": 0.032302360981702805,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.00180641224142164,
      "rewards/margins": 0.2230260670185089,
      "rewards/rejected": -0.22483249008655548,
      "step": 1270
    },
    {
      "epoch": 2.6,
      "grad_norm": 1.984375,
      "learning_rate": 1.3975424859373688e-06,
      "log_odds_chosen": 9.953948020935059,
      "log_odds_ratio": -0.002804636722430587,
      "logits/chosen": -1.9552701711654663,
      "logits/rejected": -1.9517933130264282,
      "logps/chosen": -0.04314829781651497,
      "logps/rejected": -4.2899580001831055,
      "loss": 0.0389,
      "nll_loss": 0.03743810951709747,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.002157414797693491,
      "rewards/margins": 0.21234047412872314,
      "rewards/rejected": -0.21449792385101318,
      "step": 1280
    },
    {
      "epoch": 2.62,
      "grad_norm": 2.046875,
      "learning_rate": 1.3921151159742616e-06,
      "log_odds_chosen": 9.60466194152832,
      "log_odds_ratio": -0.021028626710176468,
      "logits/chosen": -1.8756396770477295,
      "logits/rejected": -1.8728761672973633,
      "logps/chosen": -0.043241944164037704,
      "logps/rejected": -4.110201835632324,
      "loss": 0.0374,
      "nll_loss": 0.03517068549990654,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.0021620974875986576,
      "rewards/margins": 0.20334799587726593,
      "rewards/rejected": -0.20551009476184845,
      "step": 1290
    },
    {
      "epoch": 2.64,
      "grad_norm": 2.171875,
      "learning_rate": 1.386750490563073e-06,
      "log_odds_chosen": 10.577958106994629,
      "log_odds_ratio": -0.0030032056383788586,
      "logits/chosen": -1.9832937717437744,
      "logits/rejected": -1.9783260822296143,
      "logps/chosen": -0.06359104067087173,
      "logps/rejected": -4.623613357543945,
      "loss": 0.0396,
      "nll_loss": 0.04861373454332352,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.003179552499204874,
      "rewards/margins": 0.22800114750862122,
      "rewards/rejected": -0.23118066787719727,
      "step": 1300
    },
    {
      "epoch": 2.66,
      "grad_norm": 4.1875,
      "learning_rate": 1.3814474099888442e-06,
      "log_odds_chosen": 10.487229347229004,
      "log_odds_ratio": -0.0037563578225672245,
      "logits/chosen": -1.9412161111831665,
      "logits/rejected": -1.9390010833740234,
      "logps/chosen": -0.04359542205929756,
      "logps/rejected": -4.6636810302734375,
      "loss": 0.0433,
      "nll_loss": 0.03972161188721657,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0021797711960971355,
      "rewards/margins": 0.2310042679309845,
      "rewards/rejected": -0.2331840544939041,
      "step": 1310
    },
    {
      "epoch": 2.68,
      "grad_norm": 1.796875,
      "learning_rate": 1.376204706407951e-06,
      "log_odds_chosen": 10.476730346679688,
      "log_odds_ratio": -0.002192417625337839,
      "logits/chosen": -1.9325330257415771,
      "logits/rejected": -1.930166244506836,
      "logps/chosen": -0.02953045442700386,
      "logps/rejected": -4.279176712036133,
      "loss": 0.0353,
      "nll_loss": 0.03177348151803017,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.001476522651501,
      "rewards/margins": 0.2124823033809662,
      "rewards/rejected": -0.21395882964134216,
      "step": 1320
    },
    {
      "epoch": 2.7,
      "grad_norm": 2.484375,
      "learning_rate": 1.3710212427677044e-06,
      "log_odds_chosen": 10.246335983276367,
      "log_odds_ratio": -0.0470537394285202,
      "logits/chosen": -1.9275156259536743,
      "logits/rejected": -1.9267303943634033,
      "logps/chosen": -0.07270321995019913,
      "logps/rejected": -4.131697177886963,
      "loss": 0.036,
      "nll_loss": 0.031709469854831696,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.0036351613234728575,
      "rewards/margins": 0.20294968783855438,
      "rewards/rejected": -0.2065848559141159,
      "step": 1330
    },
    {
      "epoch": 2.72,
      "grad_norm": 2.578125,
      "learning_rate": 1.3658959117703826e-06,
      "log_odds_chosen": 10.568267822265625,
      "log_odds_ratio": -0.003112158738076687,
      "logits/chosen": -1.948743462562561,
      "logits/rejected": -1.9471248388290405,
      "logps/chosen": -0.028313973918557167,
      "logps/rejected": -4.5648627281188965,
      "loss": 0.0378,
      "nll_loss": 0.041449058800935745,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.001415698672644794,
      "rewards/margins": 0.22682742774486542,
      "rewards/rejected": -0.2282431572675705,
      "step": 1340
    },
    {
      "epoch": 2.74,
      "grad_norm": 3.546875,
      "learning_rate": 1.3608276348795436e-06,
      "log_odds_chosen": 10.491666793823242,
      "log_odds_ratio": -0.0048934281803667545,
      "logits/chosen": -1.9433883428573608,
      "logits/rejected": -1.9420783519744873,
      "logps/chosen": -0.03949974104762077,
      "logps/rejected": -4.376409530639648,
      "loss": 0.0414,
      "nll_loss": 0.03402886912226677,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.001974987331777811,
      "rewards/margins": 0.21684548258781433,
      "rewards/rejected": -0.2188204824924469,
      "step": 1350
    },
    {
      "epoch": 2.76,
      "grad_norm": 2.265625,
      "learning_rate": 1.355815361366601e-06,
      "log_odds_chosen": 10.092164993286133,
      "log_odds_ratio": -0.008559630252420902,
      "logits/chosen": -1.945858359336853,
      "logits/rejected": -1.9418405294418335,
      "logps/chosen": -0.06214701011776924,
      "logps/rejected": -4.4873552322387695,
      "loss": 0.0359,
      "nll_loss": 0.03512818366289139,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0031073507852852345,
      "rewards/margins": 0.2212604284286499,
      "rewards/rejected": -0.22436778247356415,
      "step": 1360
    },
    {
      "epoch": 2.78,
      "grad_norm": 1.9296875,
      "learning_rate": 1.350858067395748e-06,
      "log_odds_chosen": 10.725711822509766,
      "log_odds_ratio": -0.020754020661115646,
      "logits/chosen": -1.9590380191802979,
      "logits/rejected": -1.953980803489685,
      "logps/chosen": -0.08093880116939545,
      "logps/rejected": -4.96276330947876,
      "loss": 0.0419,
      "nll_loss": 0.0406344048678875,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.004046940244734287,
      "rewards/margins": 0.2440912276506424,
      "rewards/rejected": -0.2481381893157959,
      "step": 1370
    },
    {
      "epoch": 2.8,
      "grad_norm": 3.359375,
      "learning_rate": 1.345954755145414e-06,
      "log_odds_chosen": 10.852327346801758,
      "log_odds_ratio": -0.0012642950750887394,
      "logits/chosen": -1.8603143692016602,
      "logits/rejected": -1.85476815700531,
      "logps/chosen": -0.03734128549695015,
      "logps/rejected": -4.723127841949463,
      "loss": 0.04,
      "nll_loss": 0.025891494005918503,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0018670640420168638,
      "rewards/margins": 0.23428933322429657,
      "rewards/rejected": -0.2361563891172409,
      "step": 1380
    },
    {
      "epoch": 2.83,
      "grad_norm": 2.390625,
      "learning_rate": 1.3411044519645502e-06,
      "log_odds_chosen": 10.230730056762695,
      "log_odds_ratio": -0.002351417439058423,
      "logits/chosen": -1.8683515787124634,
      "logits/rejected": -1.86406672000885,
      "logps/chosen": -0.044098712503910065,
      "logps/rejected": -4.59433126449585,
      "loss": 0.0331,
      "nll_loss": 0.03638423979282379,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0022049355320632458,
      "rewards/margins": 0.22751164436340332,
      "rewards/rejected": -0.22971658408641815,
      "step": 1390
    },
    {
      "epoch": 2.85,
      "grad_norm": 2.296875,
      "learning_rate": 1.3363062095621222e-06,
      "log_odds_chosen": 9.968645095825195,
      "log_odds_ratio": -0.004948216024786234,
      "logits/chosen": -1.9218571186065674,
      "logits/rejected": -1.9199250936508179,
      "logps/chosen": -0.06343747675418854,
      "logps/rejected": -4.219951629638672,
      "loss": 0.0353,
      "nll_loss": 0.03614867106080055,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0031718737445771694,
      "rewards/margins": 0.20782573521137238,
      "rewards/rejected": -0.21099761128425598,
      "step": 1400
    },
    {
      "epoch": 2.87,
      "grad_norm": 2.609375,
      "learning_rate": 1.3315591032282687e-06,
      "log_odds_chosen": 10.9727783203125,
      "log_odds_ratio": -0.004531107842922211,
      "logits/chosen": -1.9281492233276367,
      "logits/rejected": -1.9244182109832764,
      "logps/chosen": -0.03406985104084015,
      "logps/rejected": -4.806418418884277,
      "loss": 0.0409,
      "nll_loss": 0.04478253796696663,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0017034925986081362,
      "rewards/margins": 0.2386174201965332,
      "rewards/rejected": -0.24032095074653625,
      "step": 1410
    },
    {
      "epoch": 2.89,
      "grad_norm": 2.21875,
      "learning_rate": 1.3268622310856882e-06,
      "log_odds_chosen": 11.176974296569824,
      "log_odds_ratio": -0.003099204506725073,
      "logits/chosen": -1.9768095016479492,
      "logits/rejected": -1.9735969305038452,
      "logps/chosen": -0.03214428573846817,
      "logps/rejected": -4.844766616821289,
      "loss": 0.0368,
      "nll_loss": 0.03034246899187565,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0016072141006588936,
      "rewards/margins": 0.240631103515625,
      "rewards/rejected": -0.24223831295967102,
      "step": 1420
    },
    {
      "epoch": 2.91,
      "grad_norm": 3.953125,
      "learning_rate": 1.3222147133698626e-06,
      "log_odds_chosen": 9.971961975097656,
      "log_odds_ratio": -0.009039332158863544,
      "logits/chosen": -1.8924024105072021,
      "logits/rejected": -1.8878147602081299,
      "logps/chosen": -0.05458871275186539,
      "logps/rejected": -4.532398700714111,
      "loss": 0.041,
      "nll_loss": 0.033775750547647476,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.002729435684159398,
      "rewards/margins": 0.2238904982805252,
      "rewards/rejected": -0.2266199290752411,
      "step": 1430
    },
    {
      "epoch": 2.93,
      "grad_norm": 2.09375,
      "learning_rate": 1.3176156917368248e-06,
      "log_odds_chosen": 10.354188919067383,
      "log_odds_ratio": -0.0086409542709589,
      "logits/chosen": -1.871347427368164,
      "logits/rejected": -1.8664233684539795,
      "logps/chosen": -0.0295790396630764,
      "logps/rejected": -4.4826579093933105,
      "loss": 0.0313,
      "nll_loss": 0.030693087726831436,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0014789520064368844,
      "rewards/margins": 0.22265395522117615,
      "rewards/rejected": -0.22413286566734314,
      "step": 1440
    },
    {
      "epoch": 2.95,
      "grad_norm": 2.578125,
      "learning_rate": 1.3130643285972255e-06,
      "log_odds_chosen": 10.23839282989502,
      "log_odds_ratio": -0.008670750074088573,
      "logits/chosen": -1.9214890003204346,
      "logits/rejected": -1.9176349639892578,
      "logps/chosen": -0.04931662604212761,
      "logps/rejected": -4.222093105316162,
      "loss": 0.0354,
      "nll_loss": 0.03125245124101639,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.002465831348672509,
      "rewards/margins": 0.20863883197307587,
      "rewards/rejected": -0.21110467612743378,
      "step": 1450
    },
    {
      "epoch": 2.97,
      "grad_norm": 1.890625,
      "learning_rate": 1.3085598064755342e-06,
      "log_odds_chosen": 10.399072647094727,
      "log_odds_ratio": -0.003747476963326335,
      "logits/chosen": -1.8761295080184937,
      "logits/rejected": -1.8740062713623047,
      "logps/chosen": -0.048522982746362686,
      "logps/rejected": -4.560256004333496,
      "loss": 0.0359,
      "nll_loss": 0.042360566556453705,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0024261490907520056,
      "rewards/margins": 0.2255866527557373,
      "rewards/rejected": -0.2280128002166748,
      "step": 1460
    },
    {
      "epoch": 2.99,
      "grad_norm": 2.140625,
      "learning_rate": 1.3041013273932528e-06,
      "log_odds_chosen": 11.28456974029541,
      "log_odds_ratio": -0.0021586515940725803,
      "logits/chosen": -1.9435927867889404,
      "logits/rejected": -1.9409191608428955,
      "logps/chosen": -0.03155381605029106,
      "logps/rejected": -5.088595390319824,
      "loss": 0.0357,
      "nll_loss": 0.028978243470191956,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0015776908257976174,
      "rewards/margins": 0.25285208225250244,
      "rewards/rejected": -0.25442975759506226,
      "step": 1470
    },
    {
      "epoch": 3.0,
      "step": 1476,
      "total_flos": 0.0,
      "train_loss": 0.1731818498170311,
      "train_runtime": 10691.148,
      "train_samples_per_second": 8.831,
      "train_steps_per_second": 0.138
    }
  ],
  "logging_steps": 10,
  "max_steps": 1476,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}