{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9994767137624281,
  "eval_steps": 100,
  "global_step": 955,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 5.208333333333333e-09,
      "logits/chosen": -2.721794605255127,
      "logits/rejected": -2.6868748664855957,
      "logps/chosen": -229.75424194335938,
      "logps/rejected": -214.74229431152344,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "learning_rate": 5.208333333333333e-08,
      "logits/chosen": -2.6621577739715576,
      "logits/rejected": -2.5832412242889404,
      "logps/chosen": -295.84637451171875,
      "logps/rejected": -250.62530517578125,
      "loss": 0.6931,
      "rewards/accuracies": 0.4305555522441864,
      "rewards/chosen": -3.170721174683422e-05,
      "rewards/margins": 0.00016743924061302096,
      "rewards/rejected": -0.00019914642325602472,
      "step": 10
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.0416666666666667e-07,
      "logits/chosen": -2.5875017642974854,
      "logits/rejected": -2.5754282474517822,
      "logps/chosen": -271.33502197265625,
      "logps/rejected": -246.907470703125,
      "loss": 0.693,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.0004090077127330005,
      "rewards/margins": -0.00012708954454865307,
      "rewards/rejected": -0.0002819181536324322,
      "step": 20
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.5624999999999999e-07,
      "logits/chosen": -2.6605145931243896,
      "logits/rejected": -2.5901198387145996,
      "logps/chosen": -278.30572509765625,
      "logps/rejected": -254.51986694335938,
      "loss": 0.6924,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.001268151798285544,
      "rewards/margins": 0.001779665588401258,
      "rewards/rejected": -0.0005115137319080532,
      "step": 30
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.0833333333333333e-07,
      "logits/chosen": -2.631272792816162,
      "logits/rejected": -2.5783164501190186,
      "logps/chosen": -273.85552978515625,
      "logps/rejected": -237.7501220703125,
      "loss": 0.6907,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.003829254535958171,
      "rewards/margins": 0.006563174072653055,
      "rewards/rejected": -0.002733920468017459,
      "step": 40
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.604166666666667e-07,
      "logits/chosen": -2.634207248687744,
      "logits/rejected": -2.5876431465148926,
      "logps/chosen": -296.025146484375,
      "logps/rejected": -274.70440673828125,
      "loss": 0.6865,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.009410797618329525,
      "rewards/margins": 0.016297370195388794,
      "rewards/rejected": -0.006886570248752832,
      "step": 50
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": -2.598982334136963,
      "logits/rejected": -2.5420570373535156,
      "logps/chosen": -285.24249267578125,
      "logps/rejected": -274.27044677734375,
      "loss": 0.6802,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.025370482355356216,
      "rewards/margins": 0.02863493002951145,
      "rewards/rejected": -0.0032644483726471663,
      "step": 60
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.645833333333333e-07,
      "logits/chosen": -2.6306140422821045,
      "logits/rejected": -2.5784096717834473,
      "logps/chosen": -310.9300537109375,
      "logps/rejected": -290.3727111816406,
      "loss": 0.6673,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.03530525788664818,
      "rewards/margins": 0.053881775587797165,
      "rewards/rejected": -0.018576517701148987,
      "step": 70
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.1666666666666667e-07,
      "logits/chosen": -2.4795422554016113,
      "logits/rejected": -2.400930881500244,
      "logps/chosen": -304.25933837890625,
      "logps/rejected": -281.20159912109375,
      "loss": 0.644,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.03674982860684395,
      "rewards/margins": 0.11624778807163239,
      "rewards/rejected": -0.15299761295318604,
      "step": 80
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.6874999999999996e-07,
      "logits/chosen": -2.5025558471679688,
      "logits/rejected": -2.459231376647949,
      "logps/chosen": -291.841064453125,
      "logps/rejected": -283.4093322753906,
      "loss": 0.6346,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.000455733563285321,
      "rewards/margins": 0.17087821662425995,
      "rewards/rejected": -0.1704224795103073,
      "step": 90
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999732492681437e-07,
      "logits/chosen": -2.4846038818359375,
      "logits/rejected": -2.455655336380005,
      "logps/chosen": -335.92242431640625,
      "logps/rejected": -340.43475341796875,
      "loss": 0.6338,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.30497807264328003,
      "rewards/margins": 0.1776745617389679,
      "rewards/rejected": -0.4826526641845703,
      "step": 100
    },
    {
      "epoch": 0.1,
      "eval_logits/chosen": -2.5391652584075928,
      "eval_logits/rejected": -2.4857265949249268,
      "eval_logps/chosen": -325.942138671875,
      "eval_logps/rejected": -321.94073486328125,
      "eval_loss": 0.6332610249519348,
      "eval_rewards/accuracies": 0.6865079402923584,
      "eval_rewards/chosen": -0.4183884263038635,
      "eval_rewards/margins": 0.1832766830921173,
      "eval_rewards/rejected": -0.6016651391983032,
      "eval_runtime": 280.8039,
      "eval_samples_per_second": 7.122,
      "eval_steps_per_second": 0.224,
      "step": 100
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.996723692767926e-07,
      "logits/chosen": -2.335371255874634,
      "logits/rejected": -2.2089285850524902,
      "logps/chosen": -307.1370544433594,
      "logps/rejected": -291.34033203125,
      "loss": 0.618,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.46787112951278687,
      "rewards/margins": 0.27501681447029114,
      "rewards/rejected": -0.7428879737854004,
      "step": 110
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.990375746213598e-07,
      "logits/chosen": -1.776285171508789,
      "logits/rejected": -1.5675570964813232,
      "logps/chosen": -336.9173889160156,
      "logps/rejected": -321.92755126953125,
      "loss": 0.6025,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.31301265954971313,
      "rewards/margins": 0.37262943387031555,
      "rewards/rejected": -0.6856420636177063,
      "step": 120
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.980697142834314e-07,
      "logits/chosen": -1.6279542446136475,
      "logits/rejected": -1.286072015762329,
      "logps/chosen": -341.5347900390625,
      "logps/rejected": -326.0841369628906,
      "loss": 0.5902,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.21236875653266907,
      "rewards/margins": 0.35374048352241516,
      "rewards/rejected": -0.566109299659729,
      "step": 130
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.967700826904229e-07,
      "logits/chosen": -1.0164812803268433,
      "logits/rejected": -0.7005228996276855,
      "logps/chosen": -264.51629638671875,
      "logps/rejected": -297.2378234863281,
      "loss": 0.5579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.24701330065727234,
      "rewards/margins": 0.4705452024936676,
      "rewards/rejected": -0.7175585031509399,
      "step": 140
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.951404179843962e-07,
      "logits/chosen": -0.418118953704834,
      "logits/rejected": -0.1309668868780136,
      "logps/chosen": -362.30755615234375,
      "logps/rejected": -408.3031311035156,
      "loss": 0.6149,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.8524826765060425,
      "rewards/margins": 0.3839341700077057,
      "rewards/rejected": -1.2364168167114258,
      "step": 150
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.931828996974498e-07,
      "logits/chosen": -1.7524774074554443,
      "logits/rejected": -1.3019535541534424,
      "logps/chosen": -368.46270751953125,
      "logps/rejected": -398.49981689453125,
      "loss": 0.5535,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.7256969809532166,
      "rewards/margins": 0.35368454456329346,
      "rewards/rejected": -1.0793814659118652,
      "step": 160
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.909001458367866e-07,
      "logits/chosen": -1.5669044256210327,
      "logits/rejected": -1.0334669351577759,
      "logps/chosen": -347.9214782714844,
      "logps/rejected": -377.63446044921875,
      "loss": 0.5472,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.7482506632804871,
      "rewards/margins": 0.5563966631889343,
      "rewards/rejected": -1.3046473264694214,
      "step": 170
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.882952093833627e-07,
      "logits/chosen": -0.7614485025405884,
      "logits/rejected": 0.021318623796105385,
      "logps/chosen": -343.43798828125,
      "logps/rejected": -394.9954528808594,
      "loss": 0.5239,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9424430131912231,
      "rewards/margins": 0.6877145767211914,
      "rewards/rejected": -1.630157470703125,
      "step": 180
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.853715742087946e-07,
      "logits/chosen": -1.3107954263687134,
      "logits/rejected": -0.5814211368560791,
      "logps/chosen": -347.8511047363281,
      "logps/rejected": -398.18890380859375,
      "loss": 0.5298,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.6900913715362549,
      "rewards/margins": 0.7306533455848694,
      "rewards/rejected": -1.4207446575164795,
      "step": 190
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.821331504159906e-07,
      "logits/chosen": -0.4228581488132477,
      "logits/rejected": 0.10827291011810303,
      "logps/chosen": -387.1797180175781,
      "logps/rejected": -437.99200439453125,
      "loss": 0.5643,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.005066990852356,
      "rewards/margins": 0.6199973821640015,
      "rewards/rejected": -1.6250642538070679,
      "step": 200
    },
    {
      "epoch": 0.21,
      "eval_logits/chosen": -0.4671556353569031,
      "eval_logits/rejected": 0.11897020787000656,
      "eval_logps/chosen": -403.8748474121094,
      "eval_logps/rejected": -447.24224853515625,
      "eval_loss": 0.5547109246253967,
      "eval_rewards/accuracies": 0.7480158805847168,
      "eval_rewards/chosen": -1.1977157592773438,
      "eval_rewards/margins": 0.6569646000862122,
      "eval_rewards/rejected": -1.8546805381774902,
      "eval_runtime": 279.4406,
      "eval_samples_per_second": 7.157,
      "eval_steps_per_second": 0.225,
      "step": 200
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.785842691097342e-07,
      "logits/chosen": -0.3859473466873169,
      "logits/rejected": 0.4077689051628113,
      "logps/chosen": -418.86883544921875,
      "logps/rejected": -427.7865295410156,
      "loss": 0.5491,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.1351739168167114,
      "rewards/margins": 0.6033421754837036,
      "rewards/rejected": -1.738515853881836,
      "step": 210
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7472967660421603e-07,
      "logits/chosen": -1.136400580406189,
      "logits/rejected": -0.6079188585281372,
      "logps/chosen": -363.44915771484375,
      "logps/rejected": -404.9462585449219,
      "loss": 0.55,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.7393938302993774,
      "rewards/margins": 0.5468804836273193,
      "rewards/rejected": -1.2862741947174072,
      "step": 220
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.705745280752585e-07,
      "logits/chosen": -0.7287824153900146,
      "logits/rejected": 0.3289051949977875,
      "logps/chosen": -380.1632080078125,
      "logps/rejected": -415.7613220214844,
      "loss": 0.5434,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.8638499975204468,
      "rewards/margins": 0.8072270154953003,
      "rewards/rejected": -1.671076774597168,
      "step": 230
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.6612438066572555e-07,
      "logits/chosen": -1.221251130104065,
      "logits/rejected": -0.22343508899211884,
      "logps/chosen": -349.05084228515625,
      "logps/rejected": -355.9934997558594,
      "loss": 0.5308,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.7182748913764954,
      "rewards/margins": 0.5810624957084656,
      "rewards/rejected": -1.2993375062942505,
      "step": 240
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.6138518605333664e-07,
      "logits/chosen": -0.4631493091583252,
      "logits/rejected": 0.10230980068445206,
      "logps/chosen": -364.1564025878906,
      "logps/rejected": -439.31463623046875,
      "loss": 0.5407,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.9659140706062317,
      "rewards/margins": 0.6127485036849976,
      "rewards/rejected": -1.5786627531051636,
      "step": 250
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.5636328249082514e-07,
      "logits/chosen": -0.5042780637741089,
      "logits/rejected": 0.3967960774898529,
      "logps/chosen": -377.323974609375,
      "logps/rejected": -418.09661865234375,
      "loss": 0.5323,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.9326316714286804,
      "rewards/margins": 0.6699790358543396,
      "rewards/rejected": -1.6026105880737305,
      "step": 260
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.510653863290871e-07,
      "logits/chosen": -0.23900368809700012,
      "logits/rejected": 1.1343748569488525,
      "logps/chosen": -397.4126892089844,
      "logps/rejected": -434.02203369140625,
      "loss": 0.5028,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9560322761535645,
      "rewards/margins": 0.8822371363639832,
      "rewards/rejected": -1.8382694721221924,
      "step": 270
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.4549858303465737e-07,
      "logits/chosen": -0.15336796641349792,
      "logits/rejected": 0.8832536935806274,
      "logps/chosen": -387.30743408203125,
      "logps/rejected": -461.30084228515625,
      "loss": 0.5195,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.9608807563781738,
      "rewards/margins": 0.8189412355422974,
      "rewards/rejected": -1.779821753501892,
      "step": 280
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.396703177135261e-07,
      "logits/chosen": -1.3611189126968384,
      "logits/rejected": -0.8717398643493652,
      "logps/chosen": -381.35321044921875,
      "logps/rejected": -414.11279296875,
      "loss": 0.527,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9636392593383789,
      "rewards/margins": 0.6538799405097961,
      "rewards/rejected": -1.6175191402435303,
      "step": 290
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.335883851539693e-07,
      "logits/chosen": -1.2623040676116943,
      "logits/rejected": -0.05791844055056572,
      "logps/chosen": -380.7442626953125,
      "logps/rejected": -440.4820251464844,
      "loss": 0.5066,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.9736257791519165,
      "rewards/margins": 0.9214972257614136,
      "rewards/rejected": -1.8951228857040405,
      "step": 300
    },
    {
      "epoch": 0.31,
      "eval_logits/chosen": -1.4155113697052002,
      "eval_logits/rejected": -0.7390369772911072,
      "eval_logps/chosen": -379.7160949707031,
      "eval_logps/rejected": -440.3581848144531,
      "eval_loss": 0.5213961005210876,
      "eval_rewards/accuracies": 0.7777777910232544,
      "eval_rewards/chosen": -0.9561280608177185,
      "eval_rewards/margins": 0.8297119140625,
      "eval_rewards/rejected": -1.7858400344848633,
      "eval_runtime": 277.1592,
      "eval_samples_per_second": 7.216,
      "eval_steps_per_second": 0.227,
      "step": 300
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.272609194017105e-07,
      "logits/chosen": -1.365898847579956,
      "logits/rejected": -0.6500005722045898,
      "logps/chosen": -368.51959228515625,
      "logps/rejected": -451.77130126953125,
      "loss": 0.4823,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.8627009391784668,
      "rewards/margins": 0.9636882543563843,
      "rewards/rejected": -1.8263890743255615,
      "step": 310
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.2069638288135547e-07,
      "logits/chosen": -1.1361403465270996,
      "logits/rejected": -0.4094177186489105,
      "logps/chosen": -378.14337158203125,
      "logps/rejected": -442.03179931640625,
      "loss": 0.5231,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.9646435976028442,
      "rewards/margins": 0.7620329856872559,
      "rewards/rejected": -1.7266767024993896,
      "step": 320
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.139035550786494e-07,
      "logits/chosen": -1.151816487312317,
      "logits/rejected": -0.6147797703742981,
      "logps/chosen": -371.8530578613281,
      "logps/rejected": -387.37786865234375,
      "loss": 0.557,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.9230759739875793,
      "rewards/margins": 0.6179936528205872,
      "rewards/rejected": -1.541069507598877,
      "step": 330
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.0689152079869306e-07,
      "logits/chosen": -1.3167518377304077,
      "logits/rejected": -0.7884875535964966,
      "logps/chosen": -342.40643310546875,
      "logps/rejected": -372.789794921875,
      "loss": 0.5621,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.966681957244873,
      "rewards/margins": 0.5185996890068054,
      "rewards/rejected": -1.4852817058563232,
      "step": 340
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.99669658015821e-07,
      "logits/chosen": -0.2449747771024704,
      "logits/rejected": 0.10235898196697235,
      "logps/chosen": -384.5868225097656,
      "logps/rejected": -475.15606689453125,
      "loss": 0.5192,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.1246081590652466,
      "rewards/margins": 0.8372184038162231,
      "rewards/rejected": -1.9618265628814697,
      "step": 350
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.92247625331392e-07,
      "logits/chosen": -0.19500190019607544,
      "logits/rejected": 0.6360437273979187,
      "logps/chosen": -383.85845947265625,
      "logps/rejected": -425.1438903808594,
      "loss": 0.5123,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.011595606803894,
      "rewards/margins": 0.8127263188362122,
      "rewards/rejected": -1.8243221044540405,
      "step": 360
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.846353490562664e-07,
      "logits/chosen": -0.6418613791465759,
      "logits/rejected": 0.004262035712599754,
      "logps/chosen": -332.0767517089844,
      "logps/rejected": -436.4134826660156,
      "loss": 0.4874,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.919625461101532,
      "rewards/margins": 0.9047123789787292,
      "rewards/rejected": -1.8243379592895508,
      "step": 370
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.768430099352445e-07,
      "logits/chosen": -1.2811148166656494,
      "logits/rejected": -0.3615255355834961,
      "logps/chosen": -408.3636474609375,
      "logps/rejected": -467.22601318359375,
      "loss": 0.4952,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.1174873113632202,
      "rewards/margins": 0.9306632280349731,
      "rewards/rejected": -2.0481505393981934,
      "step": 380
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6888102953122304e-07,
      "logits/chosen": -0.6116107702255249,
      "logits/rejected": 0.6004475355148315,
      "logps/chosen": -408.0143737792969,
      "logps/rejected": -468.11602783203125,
      "loss": 0.5044,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.3232171535491943,
      "rewards/margins": 1.0072014331817627,
      "rewards/rejected": -2.330418348312378,
      "step": 390
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.607600562872785e-07,
      "logits/chosen": -0.746349036693573,
      "logits/rejected": 0.21193762123584747,
      "logps/chosen": -457.70703125,
      "logps/rejected": -493.6814880371094,
      "loss": 0.4941,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.5325865745544434,
      "rewards/margins": 0.7493036985397339,
      "rewards/rejected": -2.281890392303467,
      "step": 400
    },
    {
      "epoch": 0.42,
      "eval_logits/chosen": -0.766248881816864,
      "eval_logits/rejected": 0.168849915266037,
      "eval_logps/chosen": -409.9141540527344,
      "eval_logps/rejected": -475.0238037109375,
      "eval_loss": 0.5082493424415588,
      "eval_rewards/accuracies": 0.7599206566810608,
      "eval_rewards/chosen": -1.2581086158752441,
      "eval_rewards/margins": 0.8743875622749329,
      "eval_rewards/rejected": -2.1324961185455322,
      "eval_runtime": 277.9817,
      "eval_samples_per_second": 7.195,
      "eval_steps_per_second": 0.227,
      "step": 400
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5249095128531856e-07,
      "logits/chosen": -1.081780195236206,
      "logits/rejected": -0.14936906099319458,
      "logps/chosen": -433.837890625,
      "logps/rejected": -488.9054260253906,
      "loss": 0.5236,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.1999945640563965,
      "rewards/margins": 0.7789229154586792,
      "rewards/rejected": -1.9789173603057861,
      "step": 410
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4408477372034736e-07,
      "logits/chosen": -0.9525474309921265,
      "logits/rejected": -0.22369150817394257,
      "logps/chosen": -385.2050476074219,
      "logps/rejected": -414.0570373535156,
      "loss": 0.5265,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.2109800577163696,
      "rewards/margins": 0.6298609972000122,
      "rewards/rejected": -1.8408409357070923,
      "step": 420
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3555276610977276e-07,
      "logits/chosen": -1.1188991069793701,
      "logits/rejected": -0.6098810434341431,
      "logps/chosen": -366.67327880859375,
      "logps/rejected": -413.8002014160156,
      "loss": 0.5316,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.0450365543365479,
      "rewards/margins": 0.6988474726676941,
      "rewards/rejected": -1.7438838481903076,
      "step": 430
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.269063392575352e-07,
      "logits/chosen": -0.8952062726020813,
      "logits/rejected": -0.29640626907348633,
      "logps/chosen": -369.5331115722656,
      "logps/rejected": -437.5091857910156,
      "loss": 0.5019,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.1057814359664917,
      "rewards/margins": 0.7806274890899658,
      "rewards/rejected": -1.886409044265747,
      "step": 440
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.1815705699316964e-07,
      "logits/chosen": -0.5310872793197632,
      "logits/rejected": 0.35491126775741577,
      "logps/chosen": -383.77337646484375,
      "logps/rejected": -466.27252197265625,
      "loss": 0.5111,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.1957778930664062,
      "rewards/margins": 1.0301498174667358,
      "rewards/rejected": -2.2259278297424316,
      "step": 450
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.0931662070620794e-07,
      "logits/chosen": -0.14580872654914856,
      "logits/rejected": 1.0018532276153564,
      "logps/chosen": -406.70404052734375,
      "logps/rejected": -492.42333984375,
      "loss": 0.5107,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.4167234897613525,
      "rewards/margins": 0.9401258230209351,
      "rewards/rejected": -2.3568496704101562,
      "step": 460
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.003968536966078e-07,
      "logits/chosen": -0.4022013545036316,
      "logits/rejected": 0.9017314910888672,
      "logps/chosen": -438.36492919921875,
      "logps/rejected": -496.6222229003906,
      "loss": 0.506,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.4015130996704102,
      "rewards/margins": 0.9599510431289673,
      "rewards/rejected": -2.361464023590088,
      "step": 470
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9140968536213693e-07,
      "logits/chosen": -0.5105709433555603,
      "logits/rejected": 0.3333401083946228,
      "logps/chosen": -348.86138916015625,
      "logps/rejected": -420.00909423828125,
      "loss": 0.5345,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.1617661714553833,
      "rewards/margins": 0.749540388584137,
      "rewards/rejected": -1.911306619644165,
      "step": 480
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.823671352438608e-07,
      "logits/chosen": -0.5324286818504333,
      "logits/rejected": 0.6832448244094849,
      "logps/chosen": -373.8572692871094,
      "logps/rejected": -425.37188720703125,
      "loss": 0.5055,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.9815096855163574,
      "rewards/margins": 0.7788732051849365,
      "rewards/rejected": -1.760382890701294,
      "step": 490
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.73281296951072e-07,
      "logits/chosen": 0.2629791498184204,
      "logits/rejected": 1.6376205682754517,
      "logps/chosen": -394.38934326171875,
      "logps/rejected": -471.0135803222656,
      "loss": 0.506,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.2273036241531372,
      "rewards/margins": 0.9877279996871948,
      "rewards/rejected": -2.215031623840332,
      "step": 500
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": 0.08570393919944763,
      "eval_logits/rejected": 1.3982725143432617,
      "eval_logps/chosen": -394.77392578125,
      "eval_logps/rejected": -468.8965759277344,
      "eval_loss": 0.5090134739875793,
      "eval_rewards/accuracies": 0.7638888955116272,
      "eval_rewards/chosen": -1.1067068576812744,
      "eval_rewards/margins": 0.9645172357559204,
      "eval_rewards/rejected": -2.0712242126464844,
      "eval_runtime": 278.9033,
      "eval_samples_per_second": 7.171,
      "eval_steps_per_second": 0.226,
      "step": 500
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.641643219871597e-07,
      "logits/chosen": 0.26743632555007935,
      "logits/rejected": 1.9989964962005615,
      "logps/chosen": -418.77044677734375,
      "logps/rejected": -468.5576171875,
      "loss": 0.4874,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.1739251613616943,
      "rewards/margins": 0.9960535168647766,
      "rewards/rejected": -2.1699788570404053,
      "step": 510
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.550284034980507e-07,
      "logits/chosen": 0.5876134634017944,
      "logits/rejected": 1.631603479385376,
      "logps/chosen": -406.91351318359375,
      "logps/rejected": -484.844482421875,
      "loss": 0.5042,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.4775892496109009,
      "rewards/margins": 0.853603184223175,
      "rewards/rejected": -2.3311924934387207,
      "step": 520
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.4588575996495794e-07,
      "logits/chosen": 0.36718782782554626,
      "logits/rejected": 1.523099660873413,
      "logps/chosen": -433.95208740234375,
      "logps/rejected": -500.431396484375,
      "loss": 0.4987,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.4977753162384033,
      "rewards/margins": 0.9368091821670532,
      "rewards/rejected": -2.434584379196167,
      "step": 530
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.367486188632446e-07,
      "logits/chosen": -0.19433559477329254,
      "logits/rejected": 1.0568033456802368,
      "logps/chosen": -429.4280700683594,
      "logps/rejected": -540.71337890625,
      "loss": 0.5031,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.3522653579711914,
      "rewards/margins": 1.0526258945465088,
      "rewards/rejected": -2.4048912525177,
      "step": 540
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.276292003092593e-07,
      "logits/chosen": -0.5677329897880554,
      "logits/rejected": 0.8163228034973145,
      "logps/chosen": -383.25537109375,
      "logps/rejected": -460.65643310546875,
      "loss": 0.519,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1353278160095215,
      "rewards/margins": 0.9918322563171387,
      "rewards/rejected": -2.127159833908081,
      "step": 550
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.185397007170141e-07,
      "logits/chosen": -0.5527084469795227,
      "logits/rejected": 0.5254577398300171,
      "logps/chosen": -373.94189453125,
      "logps/rejected": -429.65283203125,
      "loss": 0.5028,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.071791410446167,
      "rewards/margins": 0.8905409574508667,
      "rewards/rejected": -1.962332010269165,
      "step": 560
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.094922764865619e-07,
      "logits/chosen": -0.08112873882055283,
      "logits/rejected": 1.072463035583496,
      "logps/chosen": -403.42254638671875,
      "logps/rejected": -467.67449951171875,
      "loss": 0.5061,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.2727501392364502,
      "rewards/margins": 0.8145562410354614,
      "rewards/rejected": -2.087306499481201,
      "step": 570
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.0049902774588797e-07,
      "logits/chosen": 0.4094156324863434,
      "logits/rejected": 1.793280005455017,
      "logps/chosen": -428.2859802246094,
      "logps/rejected": -489.5137634277344,
      "loss": 0.5007,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.5437203645706177,
      "rewards/margins": 0.9729151725769043,
      "rewards/rejected": -2.5166354179382324,
      "step": 580
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.9157198216806238e-07,
      "logits/chosen": -0.1933516561985016,
      "logits/rejected": 0.8015575408935547,
      "logps/chosen": -403.41021728515625,
      "logps/rejected": -485.6842346191406,
      "loss": 0.496,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.3032294511795044,
      "rewards/margins": 0.7608039379119873,
      "rewards/rejected": -2.0640335083007812,
      "step": 590
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8272307888529274e-07,
      "logits/chosen": -0.43585944175720215,
      "logits/rejected": 0.9283556938171387,
      "logps/chosen": -461.6996154785156,
      "logps/rejected": -543.3409423828125,
      "loss": 0.4893,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.3578338623046875,
      "rewards/margins": 0.9542685747146606,
      "rewards/rejected": -2.3121023178100586,
      "step": 600
    },
    {
      "epoch": 0.63,
      "eval_logits/chosen": -0.41807228326797485,
      "eval_logits/rejected": 0.961281418800354,
      "eval_logps/chosen": -431.0652160644531,
      "eval_logps/rejected": -511.40478515625,
      "eval_loss": 0.49531620740890503,
      "eval_rewards/accuracies": 0.7579365372657776,
      "eval_rewards/chosen": -1.469619870185852,
      "eval_rewards/margins": 1.0266858339309692,
      "eval_rewards/rejected": -2.4963057041168213,
      "eval_runtime": 279.0041,
      "eval_samples_per_second": 7.168,
      "eval_steps_per_second": 0.226,
      "step": 600
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7396415252139288e-07,
      "logits/chosen": -0.44998255372047424,
      "logits/rejected": 0.763337254524231,
      "logps/chosen": -419.0155334472656,
      "logps/rejected": -462.7840270996094,
      "loss": 0.4842,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.4401321411132812,
      "rewards/margins": 0.9432790875434875,
      "rewards/rejected": -2.383411407470703,
      "step": 610
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6530691736402316e-07,
      "logits/chosen": -0.37868770956993103,
      "logits/rejected": 1.2032816410064697,
      "logps/chosen": -442.1136169433594,
      "logps/rejected": -499.60504150390625,
      "loss": 0.5035,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7007478475570679,
      "rewards/margins": 0.9936334490776062,
      "rewards/rejected": -2.6943812370300293,
      "step": 620
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5676295169786864e-07,
      "logits/chosen": -0.10520428419113159,
      "logits/rejected": 1.3110377788543701,
      "logps/chosen": -421.3946838378906,
      "logps/rejected": -537.7481689453125,
      "loss": 0.4811,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.5981671810150146,
      "rewards/margins": 1.225342869758606,
      "rewards/rejected": -2.823509931564331,
      "step": 630
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.483436823197092e-07,
      "logits/chosen": -0.4932429790496826,
      "logits/rejected": 0.5939461588859558,
      "logps/chosen": -437.61572265625,
      "logps/rejected": -519.6408081054688,
      "loss": 0.496,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.5301989316940308,
      "rewards/margins": 1.0939643383026123,
      "rewards/rejected": -2.6241631507873535,
      "step": 640
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4006036925609243e-07,
      "logits/chosen": -0.7950553894042969,
      "logits/rejected": 0.526989221572876,
      "logps/chosen": -480.47161865234375,
      "logps/rejected": -521.6663208007812,
      "loss": 0.5013,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.7415778636932373,
      "rewards/margins": 0.8471624255180359,
      "rewards/rejected": -2.588740587234497,
      "step": 650
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.319240907040458e-07,
      "logits/chosen": -0.6145257353782654,
      "logits/rejected": 0.26546335220336914,
      "logps/chosen": -462.09527587890625,
      "logps/rejected": -522.7366943359375,
      "loss": 0.503,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.6060371398925781,
      "rewards/margins": 0.9507848024368286,
      "rewards/rejected": -2.556821823120117,
      "step": 660
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.239457282149695e-07,
      "logits/chosen": -0.6238754391670227,
      "logits/rejected": 0.2821710407733917,
      "logps/chosen": -436.31097412109375,
      "logps/rejected": -520.7191772460938,
      "loss": 0.4934,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.5321730375289917,
      "rewards/margins": 0.9870231747627258,
      "rewards/rejected": -2.5191962718963623,
      "step": 670
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1613595214152711e-07,
      "logits/chosen": -0.5683806538581848,
      "logits/rejected": 0.05843483284115791,
      "logps/chosen": -399.19952392578125,
      "logps/rejected": -444.88604736328125,
      "loss": 0.5183,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.3775449991226196,
      "rewards/margins": 0.6896665692329407,
      "rewards/rejected": -2.067211627960205,
      "step": 680
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.0850520736699362e-07,
      "logits/chosen": -0.8241173028945923,
      "logits/rejected": 0.25952425599098206,
      "logps/chosen": -385.21063232421875,
      "logps/rejected": -466.60809326171875,
      "loss": 0.4836,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.237121343612671,
      "rewards/margins": 1.0103009939193726,
      "rewards/rejected": -2.247422456741333,
      "step": 690
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0106369933615042e-07,
      "logits/chosen": -0.5171055793762207,
      "logits/rejected": 0.7539600133895874,
      "logps/chosen": -430.15069580078125,
      "logps/rejected": -530.7283935546875,
      "loss": 0.4558,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7596960067749023,
      "rewards/margins": 1.09982168674469,
      "rewards/rejected": -2.8595175743103027,
      "step": 700
    },
    {
      "epoch": 0.73,
      "eval_logits/chosen": -0.44453164935112,
      "eval_logits/rejected": 0.6945505738258362,
      "eval_logps/chosen": -465.3408508300781,
      "eval_logps/rejected": -550.7127685546875,
      "eval_loss": 0.49365857243537903,
      "eval_rewards/accuracies": 0.7698412537574768,
      "eval_rewards/chosen": -1.8123756647109985,
      "eval_rewards/margins": 1.0770103931427002,
      "eval_rewards/rejected": -2.889385938644409,
      "eval_runtime": 278.4646,
      "eval_samples_per_second": 7.182,
      "eval_steps_per_second": 0.226,
      "step": 700
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.382138040640714e-08,
      "logits/chosen": -0.43636417388916016,
      "logits/rejected": 0.8631266355514526,
      "logps/chosen": -496.98138427734375,
      "logps/rejected": -531.872314453125,
      "loss": 0.4688,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.92108154296875,
      "rewards/margins": 0.9923149943351746,
      "rewards/rejected": -2.9133965969085693,
      "step": 710
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.678793653740632e-08,
      "logits/chosen": -0.10378441959619522,
      "logits/rejected": 1.2110047340393066,
      "logps/chosen": -494.89990234375,
      "logps/rejected": -584.9482421875,
      "loss": 0.4768,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.034623146057129,
      "rewards/margins": 1.1489760875701904,
      "rewards/rejected": -3.1835992336273193,
      "step": 720
    },
    {
      "epoch": 0.76,
      "learning_rate": 7.997277433690983e-08,
      "logits/chosen": -0.11848801374435425,
      "logits/rejected": 1.194339632987976,
      "logps/chosen": -507.22760009765625,
      "logps/rejected": -562.0817260742188,
      "loss": 0.4949,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.0333375930786133,
      "rewards/margins": 1.0677392482757568,
      "rewards/rejected": -3.101076602935791,
      "step": 730
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.338500848029602e-08,
      "logits/chosen": 0.04791594296693802,
      "logits/rejected": 1.755448579788208,
      "logps/chosen": -520.539794921875,
      "logps/rejected": -599.8433837890625,
      "loss": 0.4849,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.0619003772735596,
      "rewards/margins": 1.345897912979126,
      "rewards/rejected": -3.4077982902526855,
      "step": 740
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.70334495204884e-08,
      "logits/chosen": -0.3863813579082489,
      "logits/rejected": 0.8495087623596191,
      "logps/chosen": -478.74835205078125,
      "logps/rejected": -582.2133178710938,
      "loss": 0.4878,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.953743577003479,
      "rewards/margins": 1.0831085443496704,
      "rewards/rejected": -3.0368518829345703,
      "step": 750
    },
    {
      "epoch": 0.8,
      "learning_rate": 6.092659210462231e-08,
      "logits/chosen": -0.1735175997018814,
      "logits/rejected": 0.8358144760131836,
      "logps/chosen": -487.7015075683594,
      "logps/rejected": -554.705810546875,
      "loss": 0.4576,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -2.09541392326355,
      "rewards/margins": 0.945289134979248,
      "rewards/rejected": -3.040703058242798,
      "step": 760
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.507260361320737e-08,
      "logits/chosen": -0.21100804209709167,
      "logits/rejected": 0.5948140621185303,
      "logps/chosen": -511.5699157714844,
      "logps/rejected": -609.0560302734375,
      "loss": 0.5073,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.175572156906128,
      "rewards/margins": 0.9202457666397095,
      "rewards/rejected": -3.095817804336548,
      "step": 770
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.947931323697982e-08,
      "logits/chosen": -0.36330968141555786,
      "logits/rejected": 0.8800643682479858,
      "logps/chosen": -465.767333984375,
      "logps/rejected": -530.8380737304688,
      "loss": 0.5019,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -2.095672130584717,
      "rewards/margins": 0.8483352661132812,
      "rewards/rejected": -2.944007158279419,
      "step": 780
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.415420150605398e-08,
      "logits/chosen": -0.35998591780662537,
      "logits/rejected": 0.594068706035614,
      "logps/chosen": -519.5208740234375,
      "logps/rejected": -622.3743896484375,
      "loss": 0.4857,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -2.191953182220459,
      "rewards/margins": 1.1867144107818604,
      "rewards/rejected": -3.3786678314208984,
      "step": 790
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.9104390285376374e-08,
      "logits/chosen": -0.4554923474788666,
      "logits/rejected": 0.7812598943710327,
      "logps/chosen": -540.5608520507812,
      "logps/rejected": -596.4019775390625,
      "loss": 0.4781,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.9934720993041992,
      "rewards/margins": 1.1518696546554565,
      "rewards/rejected": -3.1453421115875244,
      "step": 800
    },
    {
      "epoch": 0.84,
      "eval_logits/chosen": -0.45034220814704895,
      "eval_logits/rejected": 0.7310919761657715,
      "eval_logps/chosen": -483.78631591796875,
      "eval_logps/rejected": -571.6085815429688,
      "eval_loss": 0.4898362457752228,
      "eval_rewards/accuracies": 0.7698412537574768,
      "eval_rewards/chosen": -1.9968301057815552,
      "eval_rewards/margins": 1.1015136241912842,
      "eval_rewards/rejected": -3.09834361076355,
      "eval_runtime": 275.7757,
      "eval_samples_per_second": 7.252,
      "eval_steps_per_second": 0.228,
      "step": 800
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.433663324986208e-08,
      "logits/chosen": -0.3436442017555237,
      "logits/rejected": 1.0277094841003418,
      "logps/chosen": -494.46343994140625,
      "logps/rejected": -540.2312622070312,
      "loss": 0.4921,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -2.1496834754943848,
      "rewards/margins": 0.8621308207511902,
      "rewards/rejected": -3.0118138790130615,
      "step": 810
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.9857306851953897e-08,
      "logits/chosen": -0.10989487171173096,
      "logits/rejected": 0.818514347076416,
      "logps/chosen": -449.99346923828125,
      "logps/rejected": -534.0044555664062,
      "loss": 0.5095,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.9733314514160156,
      "rewards/margins": 1.014764428138733,
      "rewards/rejected": -2.988095760345459,
      "step": 820
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.567240179368185e-08,
      "logits/chosen": -0.16947147250175476,
      "logits/rejected": 0.5960814952850342,
      "logps/chosen": -450.3282165527344,
      "logps/rejected": -539.936767578125,
      "loss": 0.4804,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.0146961212158203,
      "rewards/margins": 0.9625784754753113,
      "rewards/rejected": -2.9772744178771973,
      "step": 830
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.1787515014630357e-08,
      "logits/chosen": -0.3402346968650818,
      "logits/rejected": 0.5662250518798828,
      "logps/chosen": -507.84747314453125,
      "logps/rejected": -563.3038940429688,
      "loss": 0.5113,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.9471012353897095,
      "rewards/margins": 0.8456957936286926,
      "rewards/rejected": -2.792797327041626,
      "step": 840
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.820784220652766e-08,
      "logits/chosen": -0.5917896032333374,
      "logits/rejected": 0.3662889003753662,
      "logps/chosen": -483.69134521484375,
      "logps/rejected": -513.99658203125,
      "loss": 0.494,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.8341186046600342,
      "rewards/margins": 0.8557528257369995,
      "rewards/rejected": -2.689871311187744,
      "step": 850
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.4938170864468636e-08,
      "logits/chosen": -0.317844957113266,
      "logits/rejected": 0.6835850477218628,
      "logps/chosen": -478.897705078125,
      "logps/rejected": -561.5941772460938,
      "loss": 0.4905,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.0050323009490967,
      "rewards/margins": 1.1022390127182007,
      "rewards/rejected": -3.107271432876587,
      "step": 860
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1982873884064465e-08,
      "logits/chosen": -0.48837724328041077,
      "logits/rejected": 0.6178911924362183,
      "logps/chosen": -414.78631591796875,
      "logps/rejected": -539.1580200195312,
      "loss": 0.4814,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.678101897239685,
      "rewards/margins": 1.251063346862793,
      "rewards/rejected": -2.9291653633117676,
      "step": 870
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.345903713082304e-09,
      "logits/chosen": -0.3579421043395996,
      "logits/rejected": 0.8588876724243164,
      "logps/chosen": -478.02728271484375,
      "logps/rejected": -552.3193359375,
      "loss": 0.4827,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -2.0388994216918945,
      "rewards/margins": 0.9511061906814575,
      "rewards/rejected": -2.9900057315826416,
      "step": 880
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.030787065396865e-09,
      "logits/chosen": -0.30088967084884644,
      "logits/rejected": 0.4045659601688385,
      "logps/chosen": -469.99920654296875,
      "logps/rejected": -559.8849487304688,
      "loss": 0.509,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -2.034852981567383,
      "rewards/margins": 0.909083366394043,
      "rewards/rejected": -2.943936586380005,
      "step": 890
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.04062020432286e-09,
      "logits/chosen": -0.4078744351863861,
      "logits/rejected": 0.6653724908828735,
      "logps/chosen": -496.38848876953125,
      "logps/rejected": -572.7337646484375,
      "loss": 0.495,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.979608178138733,
      "rewards/margins": 0.9050843119621277,
      "rewards/rejected": -2.8846921920776367,
      "step": 900
    },
    {
      "epoch": 0.94,
      "eval_logits/chosen": -0.46421733498573303,
      "eval_logits/rejected": 0.6757184267044067,
      "eval_logps/chosen": -477.7505187988281,
      "eval_logps/rejected": -563.537841796875,
      "eval_loss": 0.48935839533805847,
      "eval_rewards/accuracies": 0.7698412537574768,
      "eval_rewards/chosen": -1.9364722967147827,
      "eval_rewards/margins": 1.0811636447906494,
      "eval_rewards/rejected": -3.0176358222961426,
      "eval_runtime": 278.8433,
      "eval_samples_per_second": 7.172,
      "eval_steps_per_second": 0.226,
      "step": 900
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.3780648016376866e-09,
      "logits/chosen": -0.34233179688453674,
      "logits/rejected": 0.7931967973709106,
      "logps/chosen": -453.32513427734375,
      "logps/rejected": -545.7423095703125,
      "loss": 0.4928,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.057847023010254,
      "rewards/margins": 0.984235942363739,
      "rewards/rejected": -3.0420830249786377,
      "step": 910
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.0453443778310766e-09,
      "logits/chosen": -0.42071524262428284,
      "logits/rejected": 0.7710467576980591,
      "logps/chosen": -491.179931640625,
      "logps/rejected": -565.1658935546875,
      "loss": 0.4639,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.9547284841537476,
      "rewards/margins": 1.1154911518096924,
      "rewards/rejected": -3.0702195167541504,
      "step": 920
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.0442413283435758e-09,
      "logits/chosen": -0.39107799530029297,
      "logits/rejected": 0.5957974195480347,
      "logps/chosen": -469.1175231933594,
      "logps/rejected": -551.2074584960938,
      "loss": 0.4664,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.9638084173202515,
      "rewards/margins": 1.1078821420669556,
      "rewards/rejected": -3.071690797805786,
      "step": 930
    },
    {
      "epoch": 0.98,
      "learning_rate": 3.760945397705828e-10,
      "logits/chosen": -0.2911260724067688,
      "logits/rejected": 0.8524629473686218,
      "logps/chosen": -528.3253784179688,
      "logps/rejected": -595.6929931640625,
      "loss": 0.4725,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.9858026504516602,
      "rewards/margins": 1.0581741333007812,
      "rewards/rejected": -3.0439765453338623,
      "step": 940
    },
    {
      "epoch": 0.99,
      "learning_rate": 4.17975992204056e-11,
      "logits/chosen": -0.4992304742336273,
      "logits/rejected": 0.760162353515625,
      "logps/chosen": -487.5731506347656,
      "logps/rejected": -578.0675048828125,
      "loss": 0.5097,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.031038761138916,
      "rewards/margins": 1.11741042137146,
      "rewards/rejected": -3.148449182510376,
      "step": 950
    },
    {
      "epoch": 1.0,
      "step": 955,
      "total_flos": 0.0,
      "train_loss": 0.5274335609056563,
      "train_runtime": 22383.3635,
      "train_samples_per_second": 2.731,
      "train_steps_per_second": 0.043
    }
  ],
  "logging_steps": 10,
  "max_steps": 955,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}