{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 1912,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 2.6041666666666667e-08,
      "logits/chosen": -2.4267122745513916,
      "logits/rejected": -2.4661664962768555,
      "logps/chosen": -213.77220153808594,
      "logps/rejected": -156.72488403320312,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "learning_rate": 2.604166666666667e-07,
      "logits/chosen": -2.258631706237793,
      "logits/rejected": -2.2316627502441406,
      "logps/chosen": -232.25244140625,
      "logps/rejected": -223.98828125,
      "loss": 0.6931,
      "rewards/accuracies": 0.3888888955116272,
      "rewards/chosen": -0.00020581186981871724,
      "rewards/margins": 0.0007613838533870876,
      "rewards/rejected": -0.0009671957814134657,
      "step": 10
    },
    {
      "epoch": 0.01,
      "learning_rate": 5.208333333333334e-07,
      "logits/chosen": -2.3259849548339844,
      "logits/rejected": -2.3685474395751953,
      "logps/chosen": -245.956787109375,
      "logps/rejected": -231.7569122314453,
      "loss": 0.6931,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.0001782081089913845,
      "rewards/margins": -0.0010433623101562262,
      "rewards/rejected": 0.0008651542593725026,
      "step": 20
    },
    {
      "epoch": 0.02,
      "learning_rate": 7.8125e-07,
      "logits/chosen": -2.28281569480896,
      "logits/rejected": -2.1847293376922607,
      "logps/chosen": -256.9249572753906,
      "logps/rejected": -222.64596557617188,
      "loss": 0.6931,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0004837578162550926,
      "rewards/margins": 0.0007256327080540359,
      "rewards/rejected": -0.0002418749063508585,
      "step": 30
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.0416666666666667e-06,
      "logits/chosen": -2.2957146167755127,
      "logits/rejected": -2.221849203109741,
      "logps/chosen": -246.8458709716797,
      "logps/rejected": -216.1331024169922,
      "loss": 0.6932,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.001086801290512085,
      "rewards/margins": 0.0008543278090655804,
      "rewards/rejected": 0.00023247336503118277,
      "step": 40
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.3020833333333335e-06,
      "logits/chosen": -2.395200729370117,
      "logits/rejected": -2.1403450965881348,
      "logps/chosen": -230.1165313720703,
      "logps/rejected": -217.97817993164062,
      "loss": 0.6929,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.0015203278744593263,
      "rewards/margins": 0.0014944769209250808,
      "rewards/rejected": 2.5850749807432294e-05,
      "step": 50
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.5625e-06,
      "logits/chosen": -2.273294687271118,
      "logits/rejected": -2.337575912475586,
      "logps/chosen": -209.8964385986328,
      "logps/rejected": -216.16592407226562,
      "loss": 0.6928,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.001072435756213963,
      "rewards/margins": 0.0009590731933712959,
      "rewards/rejected": 0.0001133625628426671,
      "step": 60
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.8229166666666666e-06,
      "logits/chosen": -2.237490177154541,
      "logits/rejected": -2.1758182048797607,
      "logps/chosen": -215.27633666992188,
      "logps/rejected": -191.88693237304688,
      "loss": 0.6926,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0013392124092206359,
      "rewards/margins": 0.0011731551494449377,
      "rewards/rejected": 0.00016605725977569818,
      "step": 70
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.0833333333333334e-06,
      "logits/chosen": -2.281623363494873,
      "logits/rejected": -2.2088568210601807,
      "logps/chosen": -288.46343994140625,
      "logps/rejected": -255.1790313720703,
      "loss": 0.6926,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.00038608754402957857,
      "rewards/margins": 0.0004603867419064045,
      "rewards/rejected": -7.429932884406298e-05,
      "step": 80
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.3437500000000002e-06,
      "logits/chosen": -2.3208765983581543,
      "logits/rejected": -2.1739656925201416,
      "logps/chosen": -271.21282958984375,
      "logps/rejected": -223.1171112060547,
      "loss": 0.6924,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.00312628410756588,
      "rewards/margins": 0.003461697371676564,
      "rewards/rejected": -0.00033541349694132805,
      "step": 90
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.604166666666667e-06,
      "logits/chosen": -2.3325040340423584,
      "logits/rejected": -2.267913341522217,
      "logps/chosen": -277.1980895996094,
      "logps/rejected": -233.1110076904297,
      "loss": 0.6918,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.003763762768357992,
      "rewards/margins": 0.0024481036234647036,
      "rewards/rejected": 0.0013156586792320013,
      "step": 100
    },
    {
      "epoch": 0.05,
      "eval_logits/chosen": -2.228503942489624,
      "eval_logits/rejected": -2.1363625526428223,
      "eval_logps/chosen": -260.84576416015625,
      "eval_logps/rejected": -245.6464080810547,
      "eval_loss": 0.6914187669754028,
      "eval_rewards/accuracies": 0.7109375,
      "eval_rewards/chosen": 0.005916436668485403,
      "eval_rewards/margins": 0.004088809713721275,
      "eval_rewards/rejected": 0.001827626721933484,
      "eval_runtime": 99.0156,
      "eval_samples_per_second": 20.199,
      "eval_steps_per_second": 0.323,
      "step": 100
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.8645833333333334e-06,
      "logits/chosen": -2.2577621936798096,
      "logits/rejected": -2.3346638679504395,
      "logps/chosen": -229.360595703125,
      "logps/rejected": -291.27734375,
      "loss": 0.6908,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.007094097323715687,
      "rewards/margins": 0.002854767022654414,
      "rewards/rejected": 0.004239329136908054,
      "step": 110
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.125e-06,
      "logits/chosen": -2.253248453140259,
      "logits/rejected": -2.1254897117614746,
      "logps/chosen": -268.021240234375,
      "logps/rejected": -243.69796752929688,
      "loss": 0.6898,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.01565476693212986,
      "rewards/margins": 0.013339616358280182,
      "rewards/rejected": 0.002315149875357747,
      "step": 120
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.385416666666667e-06,
      "logits/chosen": -2.395129442214966,
      "logits/rejected": -2.2741332054138184,
      "logps/chosen": -256.48455810546875,
      "logps/rejected": -264.9955749511719,
      "loss": 0.6898,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.012440720573067665,
      "rewards/margins": -8.803028322290629e-05,
      "rewards/rejected": 0.012528751976788044,
      "step": 130
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.6458333333333333e-06,
      "logits/chosen": -2.275693416595459,
      "logits/rejected": -2.145570755004883,
      "logps/chosen": -327.93218994140625,
      "logps/rejected": -250.51492309570312,
      "loss": 0.6846,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03406423702836037,
      "rewards/margins": 0.022959694266319275,
      "rewards/rejected": 0.011104539968073368,
      "step": 140
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.90625e-06,
      "logits/chosen": -2.310237407684326,
      "logits/rejected": -2.2222559452056885,
      "logps/chosen": -269.9709167480469,
      "logps/rejected": -258.72406005859375,
      "loss": 0.6836,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.04501372575759888,
      "rewards/margins": 0.02766810730099678,
      "rewards/rejected": 0.017345618456602097,
      "step": 150
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.166666666666667e-06,
      "logits/chosen": -2.3985304832458496,
      "logits/rejected": -2.370260000228882,
      "logps/chosen": -269.9902038574219,
      "logps/rejected": -250.7908172607422,
      "loss": 0.6797,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.03560282289981842,
      "rewards/margins": 0.019300740212202072,
      "rewards/rejected": 0.01630208268761635,
      "step": 160
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.427083333333334e-06,
      "logits/chosen": -2.331279993057251,
      "logits/rejected": -2.2696022987365723,
      "logps/chosen": -250.06179809570312,
      "logps/rejected": -243.79971313476562,
      "loss": 0.6794,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.050360582768917084,
      "rewards/margins": 0.036857619881629944,
      "rewards/rejected": 0.013502964749932289,
      "step": 170
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.6875000000000004e-06,
      "logits/chosen": -2.127589464187622,
      "logits/rejected": -2.0284266471862793,
      "logps/chosen": -317.98809814453125,
      "logps/rejected": -264.03387451171875,
      "loss": 0.6688,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.05277144908905029,
      "rewards/margins": 0.060719214379787445,
      "rewards/rejected": -0.007947764359414577,
      "step": 180
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.947916666666667e-06,
      "logits/chosen": -2.0413432121276855,
      "logits/rejected": -2.025883913040161,
      "logps/chosen": -279.8322448730469,
      "logps/rejected": -278.4949035644531,
      "loss": 0.6663,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.014455978758633137,
      "rewards/margins": 0.019963031634688377,
      "rewards/rejected": -0.03441900759935379,
      "step": 190
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999733114418726e-06,
      "logits/chosen": -2.3885703086853027,
      "logits/rejected": -2.1402792930603027,
      "logps/chosen": -353.4009094238281,
      "logps/rejected": -268.4277648925781,
      "loss": 0.6619,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.021488813683390617,
      "rewards/margins": 0.1255442202091217,
      "rewards/rejected": -0.10405541956424713,
      "step": 200
    },
    {
      "epoch": 0.1,
      "eval_logits/chosen": -2.155834436416626,
      "eval_logits/rejected": -2.053650140762329,
      "eval_logps/chosen": -264.06280517578125,
      "eval_logps/rejected": -259.0110168457031,
      "eval_loss": 0.6497414708137512,
      "eval_rewards/accuracies": 0.70703125,
      "eval_rewards/chosen": -0.02625393494963646,
      "eval_rewards/margins": 0.10556431859731674,
      "eval_rewards/rejected": -0.1318182498216629,
      "eval_runtime": 100.0964,
      "eval_samples_per_second": 19.981,
      "eval_steps_per_second": 0.32,
      "step": 200
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.998648989404205e-06,
      "logits/chosen": -2.0490939617156982,
      "logits/rejected": -2.0286402702331543,
      "logps/chosen": -252.8923797607422,
      "logps/rejected": -319.3832092285156,
      "loss": 0.6513,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.011713042855262756,
      "rewards/margins": 0.11224497854709625,
      "rewards/rejected": -0.12395800650119781,
      "step": 210
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.996731305997416e-06,
      "logits/chosen": -2.139150381088257,
      "logits/rejected": -2.073772430419922,
      "logps/chosen": -235.38320922851562,
      "logps/rejected": -231.4700164794922,
      "loss": 0.6389,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06928001344203949,
      "rewards/margins": 0.14029313623905182,
      "rewards/rejected": -0.2095731794834137,
      "step": 220
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.9939807039445205e-06,
      "logits/chosen": -2.2183709144592285,
      "logits/rejected": -2.206679582595825,
      "logps/chosen": -294.1888732910156,
      "logps/rejected": -347.28173828125,
      "loss": 0.6245,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11882080882787704,
      "rewards/margins": 0.23026032745838165,
      "rewards/rejected": -0.3490811288356781,
      "step": 230
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.990398100856367e-06,
      "logits/chosen": -2.148719310760498,
      "logits/rejected": -2.125946044921875,
      "logps/chosen": -233.49478149414062,
      "logps/rejected": -280.72503662109375,
      "loss": 0.6275,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.15465135872364044,
      "rewards/margins": 0.1490645855665207,
      "rewards/rejected": -0.30371594429016113,
      "step": 240
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.985984691902379e-06,
      "logits/chosen": -2.243445873260498,
      "logits/rejected": -2.1190781593322754,
      "logps/chosen": -368.4538879394531,
      "logps/rejected": -323.40850830078125,
      "loss": 0.6479,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.38147231936454773,
      "rewards/margins": 0.18310242891311646,
      "rewards/rejected": -0.5645747780799866,
      "step": 250
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.980741949411839e-06,
      "logits/chosen": -2.0785181522369385,
      "logits/rejected": -1.9865573644638062,
      "logps/chosen": -249.36190795898438,
      "logps/rejected": -240.6489715576172,
      "loss": 0.6203,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.35873857140541077,
      "rewards/margins": 0.13610118627548218,
      "rewards/rejected": -0.49483975768089294,
      "step": 260
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.974671622382713e-06,
      "logits/chosen": -2.0679633617401123,
      "logits/rejected": -2.0502331256866455,
      "logps/chosen": -273.707763671875,
      "logps/rejected": -319.79541015625,
      "loss": 0.6132,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.385969340801239,
      "rewards/margins": 0.18665318191051483,
      "rewards/rejected": -0.572622537612915,
      "step": 270
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.967775735898179e-06,
      "logits/chosen": -2.0888559818267822,
      "logits/rejected": -2.04829478263855,
      "logps/chosen": -297.3575439453125,
      "logps/rejected": -322.2998352050781,
      "loss": 0.5982,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3168260455131531,
      "rewards/margins": 0.23598246276378632,
      "rewards/rejected": -0.5528085231781006,
      "step": 280
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.960056590451056e-06,
      "logits/chosen": -2.0440139770507812,
      "logits/rejected": -1.889760971069336,
      "logps/chosen": -281.42559814453125,
      "logps/rejected": -238.29434204101562,
      "loss": 0.6149,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.30001693964004517,
      "rewards/margins": 0.2756292521953583,
      "rewards/rejected": -0.5756462216377258,
      "step": 290
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.951516761176344e-06,
      "logits/chosen": -2.001333713531494,
      "logits/rejected": -1.9682204723358154,
      "logps/chosen": -310.86669921875,
      "logps/rejected": -276.2847595214844,
      "loss": 0.6077,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.3108038306236267,
      "rewards/margins": 0.14596325159072876,
      "rewards/rejected": -0.45676714181900024,
      "step": 300
    },
    {
      "epoch": 0.16,
      "eval_logits/chosen": -1.9869587421417236,
      "eval_logits/rejected": -1.850504994392395,
      "eval_logps/chosen": -287.5378723144531,
      "eval_logps/rejected": -300.8819580078125,
      "eval_loss": 0.6082866191864014,
      "eval_rewards/accuracies": 0.71875,
      "eval_rewards/chosen": -0.2610045373439789,
      "eval_rewards/margins": 0.289523184299469,
      "eval_rewards/rejected": -0.5505276918411255,
      "eval_runtime": 100.0694,
      "eval_samples_per_second": 19.986,
      "eval_steps_per_second": 0.32,
      "step": 300
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.942159096992154e-06,
      "logits/chosen": -2.2042763233184814,
      "logits/rejected": -2.090355157852173,
      "logps/chosen": -310.1408386230469,
      "logps/rejected": -309.30841064453125,
      "loss": 0.6141,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.3219727873802185,
      "rewards/margins": 0.24631533026695251,
      "rewards/rejected": -0.5682880878448486,
      "step": 310
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.931986719649298e-06,
      "logits/chosen": -2.02527117729187,
      "logits/rejected": -1.9362668991088867,
      "logps/chosen": -248.31161499023438,
      "logps/rejected": -276.61370849609375,
      "loss": 0.5962,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.362973153591156,
      "rewards/margins": 0.08302036672830582,
      "rewards/rejected": -0.445993572473526,
      "step": 320
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.921003022689865e-06,
      "logits/chosen": -1.9443562030792236,
      "logits/rejected": -1.8264896869659424,
      "logps/chosen": -310.73309326171875,
      "logps/rejected": -327.0270690917969,
      "loss": 0.6018,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.39779695868492126,
      "rewards/margins": 0.37337055802345276,
      "rewards/rejected": -0.771167516708374,
      "step": 330
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.909211670315115e-06,
      "logits/chosen": -2.0006632804870605,
      "logits/rejected": -1.8731825351715088,
      "logps/chosen": -346.4628601074219,
      "logps/rejected": -290.8958435058594,
      "loss": 0.6264,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.45741358399391174,
      "rewards/margins": 0.184947669506073,
      "rewards/rejected": -0.6423612833023071,
      "step": 340
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.8966165961630914e-06,
      "logits/chosen": -1.9583308696746826,
      "logits/rejected": -1.836646318435669,
      "logps/chosen": -295.84503173828125,
      "logps/rejected": -316.0587463378906,
      "loss": 0.6125,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.32640910148620605,
      "rewards/margins": 0.405341237783432,
      "rewards/rejected": -0.7317503094673157,
      "step": 350
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.883222001996352e-06,
      "logits/chosen": -1.839678406715393,
      "logits/rejected": -1.8322643041610718,
      "logps/chosen": -264.20538330078125,
      "logps/rejected": -310.7685852050781,
      "loss": 0.5648,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.49244385957717896,
      "rewards/margins": 0.33429765701293945,
      "rewards/rejected": -0.8267415165901184,
      "step": 360
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.869032356300234e-06,
      "logits/chosen": -1.939711570739746,
      "logits/rejected": -1.838923692703247,
      "logps/chosen": -346.75946044921875,
      "logps/rejected": -347.9893493652344,
      "loss": 0.5856,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.3741551339626312,
      "rewards/margins": 0.45344096422195435,
      "rewards/rejected": -0.8275960683822632,
      "step": 370
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.854052392792162e-06,
      "logits/chosen": -1.995643973350525,
      "logits/rejected": -1.8869165182113647,
      "logps/chosen": -270.1524658203125,
      "logps/rejected": -292.2115173339844,
      "loss": 0.6162,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.4847889840602875,
      "rewards/margins": 0.3497465252876282,
      "rewards/rejected": -0.8345354795455933,
      "step": 380
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.838287108842454e-06,
      "logits/chosen": -1.9855091571807861,
      "logits/rejected": -1.8763986825942993,
      "logps/chosen": -321.6457214355469,
      "logps/rejected": -359.28326416015625,
      "loss": 0.5772,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5840765237808228,
      "rewards/margins": 0.4493025839328766,
      "rewards/rejected": -1.033379077911377,
      "step": 390
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.821741763807186e-06,
      "logits/chosen": -1.8463340997695923,
      "logits/rejected": -1.870745062828064,
      "logps/chosen": -330.8719787597656,
      "logps/rejected": -393.6171875,
      "loss": 0.5813,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4910261034965515,
      "rewards/margins": 0.6333194375038147,
      "rewards/rejected": -1.1243455410003662,
      "step": 400
    },
    {
      "epoch": 0.21,
      "eval_logits/chosen": -1.9346712827682495,
      "eval_logits/rejected": -1.7833642959594727,
      "eval_logps/chosen": -311.629150390625,
      "eval_logps/rejected": -338.0690612792969,
      "eval_loss": 0.5857117772102356,
      "eval_rewards/accuracies": 0.734375,
      "eval_rewards/chosen": -0.5019174218177795,
      "eval_rewards/margins": 0.42048126459121704,
      "eval_rewards/rejected": -0.9223986864089966,
      "eval_runtime": 99.8761,
      "eval_samples_per_second": 20.025,
      "eval_steps_per_second": 0.32,
      "step": 400
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.804421877273649e-06,
      "logits/chosen": -1.9884185791015625,
      "logits/rejected": -1.923436164855957,
      "logps/chosen": -320.7406921386719,
      "logps/rejected": -325.28436279296875,
      "loss": 0.584,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.39559125900268555,
      "rewards/margins": 0.4437909722328186,
      "rewards/rejected": -0.8393821716308594,
      "step": 410
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.786333227218996e-06,
      "logits/chosen": -1.9525524377822876,
      "logits/rejected": -1.8964300155639648,
      "logps/chosen": -246.2771759033203,
      "logps/rejected": -283.88470458984375,
      "loss": 0.5985,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.3708118796348572,
      "rewards/margins": 0.24297749996185303,
      "rewards/rejected": -0.6137893795967102,
      "step": 420
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.7674818480826856e-06,
      "logits/chosen": -2.1766157150268555,
      "logits/rejected": -2.0494487285614014,
      "logps/chosen": -368.79644775390625,
      "logps/rejected": -343.870361328125,
      "loss": 0.5883,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.36435064673423767,
      "rewards/margins": 0.5354640483856201,
      "rewards/rejected": -0.8998147249221802,
      "step": 430
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.747874028753375e-06,
      "logits/chosen": -2.0296483039855957,
      "logits/rejected": -2.0344138145446777,
      "logps/chosen": -333.8695068359375,
      "logps/rejected": -346.3202209472656,
      "loss": 0.6011,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.47436419129371643,
      "rewards/margins": 0.3697816729545593,
      "rewards/rejected": -0.8441458940505981,
      "step": 440
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.72751631047092e-06,
      "logits/chosen": -1.8225123882293701,
      "logits/rejected": -1.6715686321258545,
      "logps/chosen": -314.07525634765625,
      "logps/rejected": -329.0157775878906,
      "loss": 0.5694,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.40629976987838745,
      "rewards/margins": 0.3972943127155304,
      "rewards/rejected": -0.8035939931869507,
      "step": 450
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.706415484644196e-06,
      "logits/chosen": -1.7603601217269897,
      "logits/rejected": -1.7641122341156006,
      "logps/chosen": -298.09197998046875,
      "logps/rejected": -359.8817138671875,
      "loss": 0.6036,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.6035802960395813,
      "rewards/margins": 0.2919658422470093,
      "rewards/rejected": -0.8955461382865906,
      "step": 460
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.684578590585457e-06,
      "logits/chosen": -1.7902147769927979,
      "logits/rejected": -1.5845298767089844,
      "logps/chosen": -339.6332092285156,
      "logps/rejected": -328.145263671875,
      "loss": 0.6001,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.355998158454895,
      "rewards/margins": 0.5849534869194031,
      "rewards/rejected": -0.9409516453742981,
      "step": 470
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.662012913161998e-06,
      "logits/chosen": -1.4573760032653809,
      "logits/rejected": -1.2839057445526123,
      "logps/chosen": -342.0000305175781,
      "logps/rejected": -344.8834228515625,
      "loss": 0.5528,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.44703975319862366,
      "rewards/margins": 0.5145122408866882,
      "rewards/rejected": -0.9615518450737,
      "step": 480
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.6387259803658945e-06,
      "logits/chosen": -1.5755952596664429,
      "logits/rejected": -1.6874421834945679,
      "logps/chosen": -310.2431640625,
      "logps/rejected": -330.83929443359375,
      "loss": 0.5653,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5772610306739807,
      "rewards/margins": 0.27001291513442993,
      "rewards/rejected": -0.8472738265991211,
      "step": 490
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.6147255608026395e-06,
      "logits/chosen": -1.265753149986267,
      "logits/rejected": -1.2227959632873535,
      "logps/chosen": -297.21771240234375,
      "logps/rejected": -325.6265869140625,
      "loss": 0.6033,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.6450126767158508,
      "rewards/margins": 0.21162936091423035,
      "rewards/rejected": -0.8566421270370483,
      "step": 500
    },
    {
      "epoch": 0.26,
      "eval_logits/chosen": -1.284441351890564,
      "eval_logits/rejected": -1.0646388530731201,
      "eval_logps/chosen": -326.2359924316406,
      "eval_logps/rejected": -359.0956726074219,
      "eval_loss": 0.5683714747428894,
      "eval_rewards/accuracies": 0.7578125,
      "eval_rewards/chosen": -0.6479859948158264,
      "eval_rewards/margins": 0.4846786856651306,
      "eval_rewards/rejected": -1.132664680480957,
      "eval_runtime": 99.87,
      "eval_samples_per_second": 20.026,
      "eval_steps_per_second": 0.32,
      "step": 500
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.590019661099507e-06,
      "logits/chosen": -1.3058087825775146,
      "logits/rejected": -0.954175591468811,
      "logps/chosen": -344.62689208984375,
      "logps/rejected": -351.04461669921875,
      "loss": 0.5816,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.686931312084198,
      "rewards/margins": 0.5147491693496704,
      "rewards/rejected": -1.2016804218292236,
      "step": 510
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.564616523234511e-06,
      "logits/chosen": -1.1599328517913818,
      "logits/rejected": -1.1522148847579956,
      "logps/chosen": -404.53692626953125,
      "logps/rejected": -452.5421447753906,
      "loss": 0.5824,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.0343563556671143,
      "rewards/margins": 0.4577902853488922,
      "rewards/rejected": -1.4921467304229736,
      "step": 520
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.538524621786851e-06,
      "logits/chosen": -1.2001521587371826,
      "logits/rejected": -0.9351218342781067,
      "logps/chosen": -339.4947814941406,
      "logps/rejected": -394.2229919433594,
      "loss": 0.5663,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8950597643852234,
      "rewards/margins": 0.6420807242393494,
      "rewards/rejected": -1.5371406078338623,
      "step": 530
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.5117526611097685e-06,
      "logits/chosen": -1.2330704927444458,
      "logits/rejected": -1.01473069190979,
      "logps/chosen": -337.0997314453125,
      "logps/rejected": -390.93597412109375,
      "loss": 0.5192,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.6719285249710083,
      "rewards/margins": 0.7142583131790161,
      "rewards/rejected": -1.3861868381500244,
      "step": 540
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.484309572426732e-06,
      "logits/chosen": -1.082319974899292,
      "logits/rejected": -0.7122712731361389,
      "logps/chosen": -314.4213562011719,
      "logps/rejected": -322.6642150878906,
      "loss": 0.5411,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.8003314733505249,
      "rewards/margins": 0.607052206993103,
      "rewards/rejected": -1.407383680343628,
      "step": 550
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.456204510851957e-06,
      "logits/chosen": -0.9956424832344055,
      "logits/rejected": -0.6910310983657837,
      "logps/chosen": -381.59820556640625,
      "logps/rejected": -378.14532470703125,
      "loss": 0.556,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.123124361038208,
      "rewards/margins": 0.35411447286605835,
      "rewards/rejected": -1.4772387742996216,
      "step": 560
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.427446852336223e-06,
      "logits/chosen": -0.6827043294906616,
      "logits/rejected": -0.7810491323471069,
      "logps/chosen": -299.59564208984375,
      "logps/rejected": -401.03204345703125,
      "loss": 0.538,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.8863980174064636,
      "rewards/margins": 0.5527952909469604,
      "rewards/rejected": -1.4391934871673584,
      "step": 570
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.398046190539025e-06,
      "logits/chosen": -1.026639461517334,
      "logits/rejected": -0.9812144041061401,
      "logps/chosen": -346.61846923828125,
      "logps/rejected": -431.44622802734375,
      "loss": 0.5472,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7535640001296997,
      "rewards/margins": 0.735582172870636,
      "rewards/rejected": -1.4891459941864014,
      "step": 580
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.368012333628089e-06,
      "logits/chosen": -0.5979812145233154,
      "logits/rejected": -0.34077292680740356,
      "logps/chosen": -342.59039306640625,
      "logps/rejected": -410.510498046875,
      "loss": 0.5475,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.9304134249687195,
      "rewards/margins": 0.619970440864563,
      "rewards/rejected": -1.5503838062286377,
      "step": 590
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.337355301007336e-06,
      "logits/chosen": -0.6573127508163452,
      "logits/rejected": -0.4596225321292877,
      "logps/chosen": -428.990966796875,
      "logps/rejected": -461.5831604003906,
      "loss": 0.5338,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.002087950706482,
      "rewards/margins": 0.5927420854568481,
      "rewards/rejected": -1.5948301553726196,
      "step": 600
    },
    {
      "epoch": 0.31,
      "eval_logits/chosen": -0.3462526202201843,
      "eval_logits/rejected": -0.005769494920969009,
      "eval_logps/chosen": -352.115234375,
      "eval_logps/rejected": -406.63665771484375,
      "eval_loss": 0.5431244969367981,
      "eval_rewards/accuracies": 0.75390625,
      "eval_rewards/chosen": -0.9067782163619995,
      "eval_rewards/margins": 0.7012960910797119,
      "eval_rewards/rejected": -1.6080741882324219,
      "eval_runtime": 100.0473,
      "eval_samples_per_second": 19.991,
      "eval_steps_per_second": 0.32,
      "step": 600
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.306085319974368e-06,
      "logits/chosen": -0.5207012891769409,
      "logits/rejected": -0.08656563609838486,
      "logps/chosen": -370.1058654785156,
      "logps/rejected": -396.0721435546875,
      "loss": 0.5403,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.8168034553527832,
      "rewards/margins": 0.7966574430465698,
      "rewards/rejected": -1.6134607791900635,
      "step": 610
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.274212822308612e-06,
      "logits/chosen": -0.7111207246780396,
      "logits/rejected": -0.2700180411338806,
      "logps/chosen": -391.42694091796875,
      "logps/rejected": -451.6090393066406,
      "loss": 0.594,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9661979675292969,
      "rewards/margins": 0.7674239277839661,
      "rewards/rejected": -1.7336218357086182,
      "step": 620
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.241748440791232e-06,
      "logits/chosen": 0.08613274991512299,
      "logits/rejected": 0.35309532284736633,
      "logps/chosen": -364.17803955078125,
      "logps/rejected": -411.7427673339844,
      "loss": 0.5565,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.0981857776641846,
      "rewards/margins": 0.6119104623794556,
      "rewards/rejected": -1.7100961208343506,
      "step": 630
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.208703005658e-06,
      "logits/chosen": -0.4963720440864563,
      "logits/rejected": -0.3326560854911804,
      "logps/chosen": -360.01678466796875,
      "logps/rejected": -418.79473876953125,
      "loss": 0.5304,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.9775301218032837,
      "rewards/margins": 0.6676076054573059,
      "rewards/rejected": -1.6451377868652344,
      "step": 640
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.175087540986283e-06,
      "logits/chosen": -0.41058802604675293,
      "logits/rejected": -0.10796304047107697,
      "logps/chosen": -394.26788330078125,
      "logps/rejected": -391.35235595703125,
      "loss": 0.5173,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0439192056655884,
      "rewards/margins": 0.5056353211402893,
      "rewards/rejected": -1.5495543479919434,
      "step": 650
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.140913261017382e-06,
      "logits/chosen": -0.2698209285736084,
      "logits/rejected": -0.15175169706344604,
      "logps/chosen": -360.52435302734375,
      "logps/rejected": -470.09796142578125,
      "loss": 0.5291,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0439623594284058,
      "rewards/margins": 0.9672080278396606,
      "rewards/rejected": -2.0111701488494873,
      "step": 660
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.106191566415412e-06,
      "logits/chosen": 0.04819805547595024,
      "logits/rejected": 0.37417465448379517,
      "logps/chosen": -363.9477233886719,
      "logps/rejected": -359.45098876953125,
      "loss": 0.525,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1165151596069336,
      "rewards/margins": 0.47697582840919495,
      "rewards/rejected": -1.5934909582138062,
      "step": 670
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.070934040463999e-06,
      "logits/chosen": -0.24051837623119354,
      "logits/rejected": -0.013500308617949486,
      "logps/chosen": -402.32135009765625,
      "logps/rejected": -470.900390625,
      "loss": 0.5275,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8973871469497681,
      "rewards/margins": 0.8212674260139465,
      "rewards/rejected": -1.7186546325683594,
      "step": 680
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.035152445202057e-06,
      "logits/chosen": -0.12294012308120728,
      "logits/rejected": 0.18862707912921906,
      "logps/chosen": -334.65838623046875,
      "logps/rejected": -339.1643981933594,
      "loss": 0.5649,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.870491623878479,
      "rewards/margins": 0.3955751061439514,
      "rewards/rejected": -1.2660666704177856,
      "step": 690
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.998858717499931e-06,
      "logits/chosen": -0.14695128798484802,
      "logits/rejected": -0.06862150132656097,
      "logps/chosen": -320.9615783691406,
      "logps/rejected": -379.9163513183594,
      "loss": 0.5235,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0008286237716675,
      "rewards/margins": 0.7716361880302429,
      "rewards/rejected": -1.7724647521972656,
      "step": 700
    },
    {
      "epoch": 0.37,
      "eval_logits/chosen": -0.13744021952152252,
      "eval_logits/rejected": 0.22457998991012573,
      "eval_logps/chosen": -364.7436218261719,
      "eval_logps/rejected": -428.6434326171875,
      "eval_loss": 0.5304214358329773,
      "eval_rewards/accuracies": 0.74609375,
      "eval_rewards/chosen": -1.0330617427825928,
      "eval_rewards/margins": 0.7950804233551025,
      "eval_rewards/rejected": -1.8281422853469849,
      "eval_runtime": 99.8419,
      "eval_samples_per_second": 20.032,
      "eval_steps_per_second": 0.321,
      "step": 700
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.9620649650772075e-06,
      "logits/chosen": -0.4862421154975891,
      "logits/rejected": -0.31390875577926636,
      "logps/chosen": -370.2496643066406,
      "logps/rejected": -415.6681213378906,
      "loss": 0.6052,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.104994535446167,
      "rewards/margins": 0.5711352229118347,
      "rewards/rejected": -1.6761295795440674,
      "step": 710
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.924783462463541e-06,
      "logits/chosen": -0.8888682126998901,
      "logits/rejected": -0.5112391710281372,
      "logps/chosen": -419.72259521484375,
      "logps/rejected": -429.8675842285156,
      "loss": 0.5374,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.8601617813110352,
      "rewards/margins": 0.8069065809249878,
      "rewards/rejected": -1.6670682430267334,
      "step": 720
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.887026646903824e-06,
      "logits/chosen": -0.5013810992240906,
      "logits/rejected": -0.2705232799053192,
      "logps/chosen": -308.744140625,
      "logps/rejected": -341.9669494628906,
      "loss": 0.5341,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.1102688312530518,
      "rewards/margins": 0.56825852394104,
      "rewards/rejected": -1.6785274744033813,
      "step": 730
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.8488071142090745e-06,
      "logits/chosen": -0.09252335131168365,
      "logits/rejected": 0.34569135308265686,
      "logps/chosen": -395.5841064453125,
      "logps/rejected": -367.30340576171875,
      "loss": 0.5889,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5202785730361938,
      "rewards/margins": 0.374620258808136,
      "rewards/rejected": -1.8948990106582642,
      "step": 740
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.8101376145544326e-06,
      "logits/chosen": -0.28637129068374634,
      "logits/rejected": 0.01174071989953518,
      "logps/chosen": -399.59100341796875,
      "logps/rejected": -440.2303161621094,
      "loss": 0.5645,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.3130629062652588,
      "rewards/margins": 0.7463555335998535,
      "rewards/rejected": -2.0594184398651123,
      "step": 750
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.771031048225653e-06,
      "logits/chosen": -0.419985294342041,
      "logits/rejected": 0.13889722526073456,
      "logps/chosen": -446.94354248046875,
      "logps/rejected": -477.18621826171875,
      "loss": 0.5789,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.3231090307235718,
      "rewards/margins": 0.6492261290550232,
      "rewards/rejected": -1.9723352193832397,
      "step": 760
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7315004613155203e-06,
      "logits/chosen": -0.1880003809928894,
      "logits/rejected": 0.058419276028871536,
      "logps/chosen": -360.2503356933594,
      "logps/rejected": -452.9226989746094,
      "loss": 0.5476,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.9840580821037292,
      "rewards/margins": 0.7168115377426147,
      "rewards/rejected": -1.7008697986602783,
      "step": 770
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6915590413716314e-06,
      "logits/chosen": -0.15248139202594757,
      "logits/rejected": -0.13186867535114288,
      "logps/chosen": -369.7059631347656,
      "logps/rejected": -392.6582946777344,
      "loss": 0.5676,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -1.1791012287139893,
      "rewards/margins": 0.19577212631702423,
      "rewards/rejected": -1.374873399734497,
      "step": 780
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.651220112996974e-06,
      "logits/chosen": -0.29559603333473206,
      "logits/rejected": 0.2896161377429962,
      "logps/chosen": -366.8363952636719,
      "logps/rejected": -398.7510070800781,
      "loss": 0.5805,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.1136764287948608,
      "rewards/margins": 0.7713703513145447,
      "rewards/rejected": -1.8850466012954712,
      "step": 790
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.6104971334047954e-06,
      "logits/chosen": -0.414069265127182,
      "logits/rejected": 0.013663103803992271,
      "logps/chosen": -333.1685791015625,
      "logps/rejected": -332.27154541015625,
      "loss": 0.5241,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.1102468967437744,
      "rewards/margins": 0.4005044400691986,
      "rewards/rejected": -1.510751485824585,
      "step": 800
    },
    {
      "epoch": 0.42,
      "eval_logits/chosen": -0.043191783130168915,
      "eval_logits/rejected": 0.3361242711544037,
      "eval_logps/chosen": -359.03619384765625,
      "eval_logps/rejected": -416.9325256347656,
      "eval_loss": 0.5276142954826355,
      "eval_rewards/accuracies": 0.7578125,
      "eval_rewards/chosen": -0.9759873151779175,
      "eval_rewards/margins": 0.7350461483001709,
      "eval_rewards/rejected": -1.711033582687378,
      "eval_runtime": 100.0001,
      "eval_samples_per_second": 20.0,
      "eval_steps_per_second": 0.32,
      "step": 800
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.5694036879292278e-06,
      "logits/chosen": 0.23451080918312073,
      "logits/rejected": 0.3333522379398346,
      "logps/chosen": -328.44879150390625,
      "logps/rejected": -390.59796142578125,
      "loss": 0.5168,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1335585117340088,
      "rewards/margins": 0.44246071577072144,
      "rewards/rejected": -1.576019525527954,
      "step": 810
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.527953485493168e-06,
      "logits/chosen": -0.14245302975177765,
      "logits/rejected": 0.7153403162956238,
      "logps/chosen": -433.4105529785156,
      "logps/rejected": -413.8798828125,
      "loss": 0.5277,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.0100504159927368,
      "rewards/margins": 0.694607138633728,
      "rewards/rejected": -1.7046575546264648,
      "step": 820
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.4861603540349343e-06,
      "logits/chosen": -0.15209975838661194,
      "logits/rejected": 0.1794484406709671,
      "logps/chosen": -386.0712890625,
      "logps/rejected": -396.65350341796875,
      "loss": 0.5623,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.0117709636688232,
      "rewards/margins": 0.4693912863731384,
      "rewards/rejected": -1.4811620712280273,
      "step": 830
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4440382358952116e-06,
      "logits/chosen": -0.18714885413646698,
      "logits/rejected": 0.25312191247940063,
      "logps/chosen": -389.5378723144531,
      "logps/rejected": -420.5859375,
      "loss": 0.5739,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.263262152671814,
      "rewards/margins": 0.639037013053894,
      "rewards/rejected": -1.902299165725708,
      "step": 840
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4016011831658413e-06,
      "logits/chosen": -0.19216255843639374,
      "logits/rejected": 0.2039567232131958,
      "logps/chosen": -401.4894104003906,
      "logps/rejected": -410.1803283691406,
      "loss": 0.5573,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.3493444919586182,
      "rewards/margins": 0.5228354334831238,
      "rewards/rejected": -1.8721797466278076,
      "step": 850
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3588633530019872e-06,
      "logits/chosen": 0.14342132210731506,
      "logits/rejected": 0.7742571234703064,
      "logps/chosen": -423.52508544921875,
      "logps/rejected": -430.2787170410156,
      "loss": 0.5323,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.1869566440582275,
      "rewards/margins": 0.7774726748466492,
      "rewards/rejected": -1.9644291400909424,
      "step": 860
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.3158390028992613e-06,
      "logits/chosen": 0.427659273147583,
      "logits/rejected": 0.7266635894775391,
      "logps/chosen": -395.55694580078125,
      "logps/rejected": -439.48974609375,
      "loss": 0.5392,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3343242406845093,
      "rewards/margins": 0.7467038035392761,
      "rewards/rejected": -2.0810282230377197,
      "step": 870
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.272542485937369e-06,
      "logits/chosen": 0.6910224556922913,
      "logits/rejected": 0.5784742832183838,
      "logps/chosen": -356.70440673828125,
      "logps/rejected": -424.77606201171875,
      "loss": 0.4842,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0763694047927856,
      "rewards/margins": 0.7257629632949829,
      "rewards/rejected": -1.8021323680877686,
      "step": 880
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.228988245991869e-06,
      "logits/chosen": 0.2895136773586273,
      "logits/rejected": 0.8819330930709839,
      "logps/chosen": -408.9416809082031,
      "logps/rejected": -470.29168701171875,
      "loss": 0.546,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4385627508163452,
      "rewards/margins": 0.8292133212089539,
      "rewards/rejected": -2.2677762508392334,
      "step": 890
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.185190812915646e-06,
      "logits/chosen": 0.31424444913864136,
      "logits/rejected": 1.1830999851226807,
      "logps/chosen": -399.9066162109375,
      "logps/rejected": -402.701171875,
      "loss": 0.5332,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3664418458938599,
      "rewards/margins": 0.7213462591171265,
      "rewards/rejected": -2.0877881050109863,
      "step": 900
    },
    {
      "epoch": 0.47,
      "eval_logits/chosen": 0.46808481216430664,
      "eval_logits/rejected": 0.89262855052948,
      "eval_logps/chosen": -385.51177978515625,
      "eval_logps/rejected": -452.3992919921875,
      "eval_loss": 0.5257142782211304,
      "eval_rewards/accuracies": 0.75,
      "eval_rewards/chosen": -1.2407435178756714,
      "eval_rewards/margins": 0.8249573707580566,
      "eval_rewards/rejected": -2.0657010078430176,
      "eval_runtime": 99.7208,
      "eval_samples_per_second": 20.056,
      "eval_steps_per_second": 0.321,
      "step": 900
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.141164797691697e-06,
      "logits/chosen": 0.3064689636230469,
      "logits/rejected": 1.0518977642059326,
      "logps/chosen": -415.59722900390625,
      "logps/rejected": -537.9058837890625,
      "loss": 0.5208,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.5367953777313232,
      "rewards/margins": 0.9906780123710632,
      "rewards/rejected": -2.527473211288452,
      "step": 910
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.0969248875588547e-06,
      "logits/chosen": 0.3721020817756653,
      "logits/rejected": 0.9606655836105347,
      "logps/chosen": -354.6772155761719,
      "logps/rejected": -410.63189697265625,
      "loss": 0.5864,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.3338547945022583,
      "rewards/margins": 0.5358750224113464,
      "rewards/rejected": -1.8697296380996704,
      "step": 920
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0524858411120745e-06,
      "logits/chosen": 0.24086757004261017,
      "logits/rejected": 0.5955575108528137,
      "logps/chosen": -439.4383239746094,
      "logps/rejected": -438.33892822265625,
      "loss": 0.4967,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.1810623407363892,
      "rewards/margins": 0.7811315059661865,
      "rewards/rejected": -1.9621938467025757,
      "step": 930
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.007862483378906e-06,
      "logits/chosen": -0.363764226436615,
      "logits/rejected": 0.19057665765285492,
      "logps/chosen": -405.48583984375,
      "logps/rejected": -428.91021728515625,
      "loss": 0.5547,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.2349156141281128,
      "rewards/margins": 0.433199018239975,
      "rewards/rejected": -1.6681146621704102,
      "step": 940
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.963069700873817e-06,
      "logits/chosen": -0.17086614668369293,
      "logits/rejected": 0.29652929306030273,
      "logps/chosen": -400.3623962402344,
      "logps/rejected": -419.65692138671875,
      "loss": 0.5162,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.1464965343475342,
      "rewards/margins": 0.6613754630088806,
      "rewards/rejected": -1.8078720569610596,
      "step": 950
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9181224366319947e-06,
      "logits/chosen": 0.455890953540802,
      "logits/rejected": 1.111419439315796,
      "logps/chosen": -390.26019287109375,
      "logps/rejected": -440.094970703125,
      "loss": 0.5123,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.5266087055206299,
      "rewards/margins": 0.7709134221076965,
      "rewards/rejected": -2.2975220680236816,
      "step": 960
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.873035685224297e-06,
      "logits/chosen": 0.41874760389328003,
      "logits/rejected": 0.5401151180267334,
      "logps/chosen": -409.41815185546875,
      "logps/rejected": -519.6390380859375,
      "loss": 0.5646,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.4578272104263306,
      "rewards/margins": 0.6119868159294128,
      "rewards/rejected": -2.0698142051696777,
      "step": 970
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.827824487755007e-06,
      "logits/chosen": 0.3823867440223694,
      "logits/rejected": 0.33489638566970825,
      "logps/chosen": -424.8614807128906,
      "logps/rejected": -423.9286193847656,
      "loss": 0.5896,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.6043908596038818,
      "rewards/margins": 0.10235877335071564,
      "rewards/rejected": -1.706749677658081,
      "step": 980
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7825039268440655e-06,
      "logits/chosen": 0.1400536596775055,
      "logits/rejected": 0.41318145394325256,
      "logps/chosen": -332.98907470703125,
      "logps/rejected": -411.08929443359375,
      "loss": 0.5605,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.362296462059021,
      "rewards/margins": 0.7367597818374634,
      "rewards/rejected": -2.0990562438964844,
      "step": 990
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7370891215954572e-06,
      "logits/chosen": 0.009750944562256336,
      "logits/rejected": 0.4523545205593109,
      "logps/chosen": -405.9288635253906,
      "logps/rejected": -448.93890380859375,
      "loss": 0.531,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.4209400415420532,
      "rewards/margins": 0.5459128618240356,
      "rewards/rejected": -1.9668527841567993,
      "step": 1000
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": -0.0765841156244278,
      "eval_logits/rejected": 0.2824910283088684,
      "eval_logps/chosen": -374.2120361328125,
      "eval_logps/rejected": -431.3622741699219,
      "eval_loss": 0.5232118964195251,
      "eval_rewards/accuracies": 0.74609375,
      "eval_rewards/chosen": -1.1277458667755127,
      "eval_rewards/margins": 0.7275847792625427,
      "eval_rewards/rejected": -1.8553305864334106,
      "eval_runtime": 99.9876,
      "eval_samples_per_second": 20.002,
      "eval_steps_per_second": 0.32,
      "step": 1000
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6915952225534187e-06,
      "logits/chosen": -0.0979020744562149,
      "logits/rejected": 0.344775527715683,
      "logps/chosen": -407.2764892578125,
      "logps/rejected": -420.5032653808594,
      "loss": 0.5258,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3287527561187744,
      "rewards/margins": 0.5185291171073914,
      "rewards/rejected": -1.847281813621521,
      "step": 1010
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6460374066481654e-06,
      "logits/chosen": 0.06318001449108124,
      "logits/rejected": 0.052066363394260406,
      "logps/chosen": -414.02142333984375,
      "logps/rejected": -435.2493591308594,
      "loss": 0.5507,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.5719435214996338,
      "rewards/margins": 0.11801069974899292,
      "rewards/rejected": -1.689954161643982,
      "step": 1020
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.6004308721328096e-06,
      "logits/chosen": -0.09392335265874863,
      "logits/rejected": 0.6331489682197571,
      "logps/chosen": -454.8587341308594,
      "logps/rejected": -456.37518310546875,
      "loss": 0.5531,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.4329391717910767,
      "rewards/margins": 0.73013836145401,
      "rewards/rejected": -2.1630773544311523,
      "step": 1030
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5547908335131704e-06,
      "logits/chosen": 0.029746657237410545,
      "logits/rejected": 0.5345848798751831,
      "logps/chosen": -442.5694274902344,
      "logps/rejected": -484.28472900390625,
      "loss": 0.5263,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.2879016399383545,
      "rewards/margins": 0.9445211291313171,
      "rewards/rejected": -2.2324228286743164,
      "step": 1040
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5091325164721574e-06,
      "logits/chosen": -0.13794167339801788,
      "logits/rejected": 0.2926728129386902,
      "logps/chosen": -406.499267578125,
      "logps/rejected": -446.9866638183594,
      "loss": 0.5208,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.321260690689087,
      "rewards/margins": 0.577609121799469,
      "rewards/rejected": -1.8988698720932007,
      "step": 1050
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.463471152790427e-06,
      "logits/chosen": 0.5882676243782043,
      "logits/rejected": 0.7257478833198547,
      "logps/chosen": -323.394287109375,
      "logps/rejected": -373.9036560058594,
      "loss": 0.5704,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.2941257953643799,
      "rewards/margins": 0.4104182720184326,
      "rewards/rejected": -1.7045438289642334,
      "step": 1060
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.417821975265009e-06,
      "logits/chosen": 0.3513713479042053,
      "logits/rejected": 0.9065437316894531,
      "logps/chosen": -369.2812194824219,
      "logps/rejected": -405.2218322753906,
      "loss": 0.529,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.17937433719635,
      "rewards/margins": 0.8810786008834839,
      "rewards/rejected": -2.060452699661255,
      "step": 1070
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.3722002126275826e-06,
      "logits/chosen": 0.16156932711601257,
      "logits/rejected": 0.3264058828353882,
      "logps/chosen": -362.4791564941406,
      "logps/rejected": -458.8309020996094,
      "loss": 0.5004,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.1431844234466553,
      "rewards/margins": 0.9749501347541809,
      "rewards/rejected": -2.1181344985961914,
      "step": 1080
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3266210844641242e-06,
      "logits/chosen": -0.016268134117126465,
      "logits/rejected": 0.5968390703201294,
      "logps/chosen": -462.67864990234375,
      "logps/rejected": -465.4684143066406,
      "loss": 0.5165,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.264756441116333,
      "rewards/margins": 0.684155285358429,
      "rewards/rejected": -1.9489116668701172,
      "step": 1090
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2810997961375943e-06,
      "logits/chosen": 0.6272622346878052,
      "logits/rejected": 1.0814670324325562,
      "logps/chosen": -317.378662109375,
      "logps/rejected": -359.07080078125,
      "loss": 0.4864,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.1075953245162964,
      "rewards/margins": 0.677285373210907,
      "rewards/rejected": -1.7848806381225586,
      "step": 1100
    },
    {
      "epoch": 0.58,
      "eval_logits/chosen": 0.740875244140625,
      "eval_logits/rejected": 1.181448221206665,
      "eval_logps/chosen": -378.1357727050781,
      "eval_logps/rejected": -444.7675476074219,
      "eval_loss": 0.5172202587127686,
      "eval_rewards/accuracies": 0.75,
      "eval_rewards/chosen": -1.1669834852218628,
      "eval_rewards/margins": 0.8224003314971924,
      "eval_rewards/rejected": -1.9893838167190552,
      "eval_runtime": 99.7722,
      "eval_samples_per_second": 20.046,
      "eval_steps_per_second": 0.321,
      "step": 1100
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2356515337153726e-06,
      "logits/chosen": 0.4388123154640198,
      "logits/rejected": 0.5722852349281311,
      "logps/chosen": -431.4241638183594,
      "logps/rejected": -424.3793029785156,
      "loss": 0.5682,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.5110528469085693,
      "rewards/margins": 0.26485976576805115,
      "rewards/rejected": -1.7759126424789429,
      "step": 1110
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.19029145890313e-06,
      "logits/chosen": 0.6199942827224731,
      "logits/rejected": 0.3805321156978607,
      "logps/chosen": -441.6695861816406,
      "logps/rejected": -530.5279541015625,
      "loss": 0.5617,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.2569580078125,
      "rewards/margins": 0.8591600656509399,
      "rewards/rejected": -2.1161179542541504,
      "step": 1120
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.145034703986834e-06,
      "logits/chosen": 0.03130330145359039,
      "logits/rejected": 0.9182190895080566,
      "logps/chosen": -422.25433349609375,
      "logps/rejected": -398.5935363769531,
      "loss": 0.5214,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.061409592628479,
      "rewards/margins": 0.7287064790725708,
      "rewards/rejected": -1.7901159524917603,
      "step": 1130
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.099896366784554e-06,
      "logits/chosen": 0.569893479347229,
      "logits/rejected": 0.5035176277160645,
      "logps/chosen": -370.8650817871094,
      "logps/rejected": -492.43206787109375,
      "loss": 0.519,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.164158582687378,
      "rewards/margins": 0.7605217695236206,
      "rewards/rejected": -1.9246803522109985,
      "step": 1140
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.054891505609782e-06,
      "logits/chosen": 0.6834603548049927,
      "logits/rejected": 0.6329541802406311,
      "logps/chosen": -374.13568115234375,
      "logps/rejected": -448.3985290527344,
      "loss": 0.5592,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.1955368518829346,
      "rewards/margins": 0.5105778574943542,
      "rewards/rejected": -1.7061145305633545,
      "step": 1150
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.010035134247922e-06,
      "logits/chosen": 0.7083349227905273,
      "logits/rejected": 1.068521499633789,
      "logps/chosen": -379.41778564453125,
      "logps/rejected": -438.62921142578125,
      "loss": 0.5196,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.0824288129806519,
      "rewards/margins": 0.8338676691055298,
      "rewards/rejected": -1.916296362876892,
      "step": 1160
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9653422169476354e-06,
      "logits/chosen": 0.44568267464637756,
      "logits/rejected": 0.703025221824646,
      "logps/chosen": -373.59710693359375,
      "logps/rejected": -427.1033630371094,
      "loss": 0.5485,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.987470269203186,
      "rewards/margins": 0.8238475918769836,
      "rewards/rejected": -1.811317801475525,
      "step": 1170
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.9208276634287143e-06,
      "logits/chosen": 0.3246023654937744,
      "logits/rejected": 1.222904920578003,
      "logps/chosen": -463.6178283691406,
      "logps/rejected": -413.4266662597656,
      "loss": 0.5061,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.2248636484146118,
      "rewards/margins": 0.7811811566352844,
      "rewards/rejected": -2.006044626235962,
      "step": 1180
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8765063239081442e-06,
      "logits/chosen": 0.8462961912155151,
      "logits/rejected": 1.0419355630874634,
      "logps/chosen": -394.54376220703125,
      "logps/rejected": -464.9222106933594,
      "loss": 0.5619,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.465216875076294,
      "rewards/margins": 0.712496817111969,
      "rewards/rejected": -2.177713632583618,
      "step": 1190
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.832392984146018e-06,
      "logits/chosen": 0.9549469947814941,
      "logits/rejected": 1.0832746028900146,
      "logps/chosen": -353.7349853515625,
      "logps/rejected": -435.2191467285156,
      "loss": 0.5467,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.5669246912002563,
      "rewards/margins": 0.64616858959198,
      "rewards/rejected": -2.2130932807922363,
      "step": 1200
    },
    {
      "epoch": 0.63,
      "eval_logits/chosen": 0.8593495488166809,
      "eval_logits/rejected": 1.3020421266555786,
      "eval_logps/chosen": -397.76275634765625,
      "eval_logps/rejected": -462.7305603027344,
      "eval_loss": 0.5195906758308411,
      "eval_rewards/accuracies": 0.73828125,
      "eval_rewards/chosen": -1.3632532358169556,
      "eval_rewards/margins": 0.8057605624198914,
      "eval_rewards/rejected": -2.169013738632202,
      "eval_runtime": 99.9113,
      "eval_samples_per_second": 20.018,
      "eval_steps_per_second": 0.32,
      "step": 1200
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.7885023605129532e-06,
      "logits/chosen": 0.4313087463378906,
      "logits/rejected": 0.9333802461624146,
      "logps/chosen": -355.1097717285156,
      "logps/rejected": -417.61810302734375,
      "loss": 0.534,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.317725419998169,
      "rewards/margins": 0.7560078501701355,
      "rewards/rejected": -2.073733329772949,
      "step": 1210
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7448490950806552e-06,
      "logits/chosen": 0.38834458589553833,
      "logits/rejected": 0.989336371421814,
      "logps/chosen": -413.79473876953125,
      "logps/rejected": -411.78973388671875,
      "loss": 0.5136,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.1249158382415771,
      "rewards/margins": 0.6000865697860718,
      "rewards/rejected": -1.7250025272369385,
      "step": 1220
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7014477507372712e-06,
      "logits/chosen": 0.537342369556427,
      "logits/rejected": 0.595427930355072,
      "logps/chosen": -365.5661315917969,
      "logps/rejected": -432.7490234375,
      "loss": 0.5302,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.1988743543624878,
      "rewards/margins": 0.6834364533424377,
      "rewards/rejected": -1.8823106288909912,
      "step": 1230
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6583128063291576e-06,
      "logits/chosen": -0.22250571846961975,
      "logits/rejected": 0.4327928125858307,
      "logps/chosen": -429.69677734375,
      "logps/rejected": -486.3809509277344,
      "loss": 0.4874,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.2788269519805908,
      "rewards/margins": 0.803261935710907,
      "rewards/rejected": -2.0820889472961426,
      "step": 1240
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6154586518306798e-06,
      "logits/chosen": 0.827475905418396,
      "logits/rejected": 1.162104845046997,
      "logps/chosen": -395.8812255859375,
      "logps/rejected": -423.1787109375,
      "loss": 0.5493,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.3758419752120972,
      "rewards/margins": 0.5010850429534912,
      "rewards/rejected": -1.876927137374878,
      "step": 1250
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5728995835436711e-06,
      "logits/chosen": 0.7995271682739258,
      "logits/rejected": 1.8586130142211914,
      "logps/chosen": -387.32623291015625,
      "logps/rejected": -397.79827880859375,
      "loss": 0.5026,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.1881077289581299,
      "rewards/margins": 1.0256298780441284,
      "rewards/rejected": -2.2137374877929688,
      "step": 1260
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5306497993281312e-06,
      "logits/chosen": 1.029513955116272,
      "logits/rejected": 1.2201731204986572,
      "logps/chosen": -383.57476806640625,
      "logps/rejected": -387.10015869140625,
      "loss": 0.5487,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.4437286853790283,
      "rewards/margins": 0.5175876021385193,
      "rewards/rejected": -1.9613163471221924,
      "step": 1270
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.488723393865766e-06,
      "logits/chosen": 0.6667646765708923,
      "logits/rejected": 1.68923020362854,
      "logps/chosen": -444.7820739746094,
      "logps/rejected": -444.03375244140625,
      "loss": 0.5775,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.6295793056488037,
      "rewards/margins": 0.6660442352294922,
      "rewards/rejected": -2.295623540878296,
      "step": 1280
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4471343539579562e-06,
      "logits/chosen": 0.24659457802772522,
      "logits/rejected": 1.137135624885559,
      "logps/chosen": -447.05303955078125,
      "logps/rejected": -455.335693359375,
      "loss": 0.5527,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.5161960124969482,
      "rewards/margins": 0.6156362295150757,
      "rewards/rejected": -2.1318325996398926,
      "step": 1290
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4058965538597033e-06,
      "logits/chosen": 0.5961114168167114,
      "logits/rejected": 0.9649273753166199,
      "logps/chosen": -414.1187438964844,
      "logps/rejected": -474.6805725097656,
      "loss": 0.5125,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1609492301940918,
      "rewards/margins": 0.921238124370575,
      "rewards/rejected": -2.0821871757507324,
      "step": 1300
    },
    {
      "epoch": 0.68,
      "eval_logits/chosen": 0.6638839840888977,
      "eval_logits/rejected": 1.1044923067092896,
      "eval_logps/chosen": -381.7656555175781,
      "eval_logps/rejected": -446.24365234375,
      "eval_loss": 0.5179304480552673,
      "eval_rewards/accuracies": 0.7421875,
      "eval_rewards/chosen": -1.203282117843628,
      "eval_rewards/margins": 0.8008626103401184,
      "eval_rewards/rejected": -2.0041446685791016,
      "eval_runtime": 99.7889,
      "eval_samples_per_second": 20.042,
      "eval_steps_per_second": 0.321,
      "step": 1300
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3650237506511333e-06,
      "logits/chosen": 0.17492257058620453,
      "logits/rejected": 0.6054025888442993,
      "logps/chosen": -375.28460693359375,
      "logps/rejected": -439.0665588378906,
      "loss": 0.4896,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.1908279657363892,
      "rewards/margins": 0.9128111600875854,
      "rewards/rejected": -2.1036391258239746,
      "step": 1310
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.324529579648079e-06,
      "logits/chosen": 0.2962098717689514,
      "logits/rejected": 0.7064357995986938,
      "logps/chosen": -435.1327209472656,
      "logps/rejected": -473.338623046875,
      "loss": 0.5331,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.3846468925476074,
      "rewards/margins": 0.5992780923843384,
      "rewards/rejected": -1.9839251041412354,
      "step": 1320
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2844275498532955e-06,
      "logits/chosen": 0.7400895953178406,
      "logits/rejected": 0.7335311770439148,
      "logps/chosen": -384.15252685546875,
      "logps/rejected": -455.39276123046875,
      "loss": 0.5029,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.3056604862213135,
      "rewards/margins": 0.7046310901641846,
      "rewards/rejected": -2.010291576385498,
      "step": 1330
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.244731039449802e-06,
      "logits/chosen": 0.6501943469047546,
      "logits/rejected": 1.1686570644378662,
      "logps/chosen": -431.3033142089844,
      "logps/rejected": -449.19207763671875,
      "loss": 0.4817,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.0331215858459473,
      "rewards/margins": 1.0517748594284058,
      "rewards/rejected": -2.0848963260650635,
      "step": 1340
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.2054532913378738e-06,
      "logits/chosen": 0.44238996505737305,
      "logits/rejected": 0.9898010492324829,
      "logps/chosen": -413.67388916015625,
      "logps/rejected": -463.6742248535156,
      "loss": 0.5009,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.4910118579864502,
      "rewards/margins": 0.8411634564399719,
      "rewards/rejected": -2.3321752548217773,
      "step": 1350
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1666074087171628e-06,
      "logits/chosen": 0.43889451026916504,
      "logits/rejected": 1.1597634553909302,
      "logps/chosen": -416.033203125,
      "logps/rejected": -421.3883361816406,
      "loss": 0.5046,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.1603151559829712,
      "rewards/margins": 0.6429589986801147,
      "rewards/rejected": -1.803274154663086,
      "step": 1360
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.1282063507154214e-06,
      "logits/chosen": 0.9281649589538574,
      "logits/rejected": 0.7970014810562134,
      "logps/chosen": -383.08203125,
      "logps/rejected": -429.7383728027344,
      "loss": 0.521,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6160444021224976,
      "rewards/margins": 0.5065132975578308,
      "rewards/rejected": -2.1225578784942627,
      "step": 1370
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.0902629280652933e-06,
      "logits/chosen": 0.7554650902748108,
      "logits/rejected": 1.4683620929718018,
      "logps/chosen": -424.9288024902344,
      "logps/rejected": -410.4168395996094,
      "loss": 0.5506,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.6847953796386719,
      "rewards/margins": 0.4181737005710602,
      "rewards/rejected": -2.102969169616699,
      "step": 1380
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.052789798830601e-06,
      "logits/chosen": 0.642483651638031,
      "logits/rejected": 0.7909359931945801,
      "logps/chosen": -461.81671142578125,
      "logps/rejected": -548.4481201171875,
      "loss": 0.4936,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.4545402526855469,
      "rewards/margins": 1.028944730758667,
      "rewards/rejected": -2.483484983444214,
      "step": 1390
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0157994641835737e-06,
      "logits/chosen": 0.5413963198661804,
      "logits/rejected": 1.2730069160461426,
      "logps/chosen": -385.44110107421875,
      "logps/rejected": -425.7725524902344,
      "loss": 0.4881,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.3209706544876099,
      "rewards/margins": 0.7255110144615173,
      "rewards/rejected": -2.0464818477630615,
      "step": 1400
    },
    {
      "epoch": 0.73,
      "eval_logits/chosen": 0.7444608807563782,
      "eval_logits/rejected": 1.1890881061553955,
      "eval_logps/chosen": -389.3553771972656,
      "eval_logps/rejected": -459.1728210449219,
      "eval_loss": 0.5158318877220154,
      "eval_rewards/accuracies": 0.75390625,
      "eval_rewards/chosen": -1.2791798114776611,
      "eval_rewards/margins": 0.8542567491531372,
      "eval_rewards/rejected": -2.133436679840088,
      "eval_runtime": 99.9692,
      "eval_samples_per_second": 20.006,
      "eval_steps_per_second": 0.32,
      "step": 1400
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.793042642344122e-07,
      "logits/chosen": 0.7017895579338074,
      "logits/rejected": 0.7292946577072144,
      "logps/chosen": -396.1260986328125,
      "logps/rejected": -448.18133544921875,
      "loss": 0.5094,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.3052157163619995,
      "rewards/margins": 0.5403749346733093,
      "rewards/rejected": -1.845590591430664,
      "step": 1410
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.433163739145773e-07,
      "logits/chosen": 0.32617324590682983,
      "logits/rejected": 0.8873109817504883,
      "logps/chosen": -385.48651123046875,
      "logps/rejected": -387.12322998046875,
      "loss": 0.5213,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2667211294174194,
      "rewards/margins": 0.7441008687019348,
      "rewards/rejected": -2.01082181930542,
      "step": 1420
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.078477989151957e-07,
      "logits/chosen": 0.4015222191810608,
      "logits/rejected": 0.17243003845214844,
      "logps/chosen": -413.64813232421875,
      "logps/rejected": -495.1240234375,
      "loss": 0.492,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3316669464111328,
      "rewards/margins": 0.6485150456428528,
      "rewards/rejected": -1.9801819324493408,
      "step": 1430
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.729103716819113e-07,
      "logits/chosen": 0.7524212598800659,
      "logits/rejected": 1.1642448902130127,
      "logps/chosen": -382.9913635253906,
      "logps/rejected": -450.04986572265625,
      "loss": 0.5076,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.3288081884384155,
      "rewards/margins": 0.8739243745803833,
      "rewards/rejected": -2.202732563018799,
      "step": 1440
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.385157474675415e-07,
      "logits/chosen": 0.31630071997642517,
      "logits/rejected": 0.6081461310386658,
      "logps/chosen": -395.0060119628906,
      "logps/rejected": -481.4752502441406,
      "loss": 0.5291,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.4594825506210327,
      "rewards/margins": 0.6843896508216858,
      "rewards/rejected": -2.1438724994659424,
      "step": 1450
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.046754004438428e-07,
      "logits/chosen": 0.766178548336029,
      "logits/rejected": 1.4351043701171875,
      "logps/chosen": -365.0591735839844,
      "logps/rejected": -402.4694519042969,
      "loss": 0.5532,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.0528379678726196,
      "rewards/margins": 0.6676140427589417,
      "rewards/rejected": -1.720452070236206,
      "step": 1460
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.714006198736837e-07,
      "logits/chosen": 1.2198323011398315,
      "logits/rejected": 1.0236773490905762,
      "logps/chosen": -340.1226501464844,
      "logps/rejected": -483.118896484375,
      "loss": 0.48,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.4166100025177002,
      "rewards/margins": 0.9187544584274292,
      "rewards/rejected": -2.33536434173584,
      "step": 1470
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.387025063449082e-07,
      "logits/chosen": 0.8669508099555969,
      "logits/rejected": 1.123145341873169,
      "logps/chosen": -364.0406799316406,
      "logps/rejected": -466.4491271972656,
      "loss": 0.4846,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.308837652206421,
      "rewards/margins": 0.9366034269332886,
      "rewards/rejected": -2.245441198348999,
      "step": 1480
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.065919680671357e-07,
      "logits/chosen": 1.0810492038726807,
      "logits/rejected": 1.4076943397521973,
      "logps/chosen": -371.6559753417969,
      "logps/rejected": -437.2684631347656,
      "loss": 0.5433,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.464137315750122,
      "rewards/margins": 0.6649857759475708,
      "rewards/rejected": -2.1291232109069824,
      "step": 1490
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.750797172327442e-07,
      "logits/chosen": 0.6836698651313782,
      "logits/rejected": 1.290857195854187,
      "logps/chosen": -459.5965270996094,
      "logps/rejected": -491.36322021484375,
      "loss": 0.5273,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.3482565879821777,
      "rewards/margins": 0.5898759365081787,
      "rewards/rejected": -1.9381325244903564,
      "step": 1500
    },
    {
      "epoch": 0.78,
      "eval_logits/chosen": 0.7972918748855591,
      "eval_logits/rejected": 1.2533293962478638,
      "eval_logps/chosen": -382.25054931640625,
      "eval_logps/rejected": -453.2860412597656,
      "eval_loss": 0.5135490298271179,
      "eval_rewards/accuracies": 0.75390625,
      "eval_rewards/chosen": -1.2081315517425537,
      "eval_rewards/margins": 0.8664370179176331,
      "eval_rewards/rejected": -2.074568271636963,
      "eval_runtime": 99.7477,
      "eval_samples_per_second": 20.051,
      "eval_steps_per_second": 0.321,
      "step": 1500
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.441762664432463e-07,
      "logits/chosen": 0.7760528922080994,
      "logits/rejected": 1.4465583562850952,
      "logps/chosen": -396.90496826171875,
      "logps/rejected": -484.97454833984375,
      "loss": 0.5245,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2378466129302979,
      "rewards/margins": 1.0589042901992798,
      "rewards/rejected": -2.296750545501709,
      "step": 1510
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.138919252022435e-07,
      "logits/chosen": 0.4587163031101227,
      "logits/rejected": 0.4502899646759033,
      "logps/chosen": -432.886962890625,
      "logps/rejected": -490.9667053222656,
      "loss": 0.5225,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.5580703020095825,
      "rewards/margins": 0.6701681613922119,
      "rewards/rejected": -2.228238582611084,
      "step": 1520
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.842367964761389e-07,
      "logits/chosen": 0.6432207226753235,
      "logits/rejected": 0.7316852807998657,
      "logps/chosen": -400.072265625,
      "logps/rejected": -470.08953857421875,
      "loss": 0.5145,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.3107836246490479,
      "rewards/margins": 0.6859927177429199,
      "rewards/rejected": -1.9967763423919678,
      "step": 1530
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.552207733237544e-07,
      "logits/chosen": 1.033753514289856,
      "logits/rejected": 1.2738745212554932,
      "logps/chosen": -349.98992919921875,
      "logps/rejected": -408.69586181640625,
      "loss": 0.52,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.2401503324508667,
      "rewards/margins": 0.8352483510971069,
      "rewards/rejected": -2.0753986835479736,
      "step": 1540
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.268535355959642e-07,
      "logits/chosen": 0.4373984932899475,
      "logits/rejected": 1.6638437509536743,
      "logps/chosen": -381.151123046875,
      "logps/rejected": -425.26373291015625,
      "loss": 0.5309,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.3500919342041016,
      "rewards/margins": 1.0549076795578003,
      "rewards/rejected": -2.4049997329711914,
      "step": 1550
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.99144546706469e-07,
      "logits/chosen": 0.5850018262863159,
      "logits/rejected": 0.9529399871826172,
      "logps/chosen": -376.8275451660156,
      "logps/rejected": -434.23651123046875,
      "loss": 0.5249,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.557494044303894,
      "rewards/margins": 0.6373199820518494,
      "rewards/rejected": -2.1948142051696777,
      "step": 1560
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.721030504747573e-07,
      "logits/chosen": 0.6524550914764404,
      "logits/rejected": 1.50775945186615,
      "logps/chosen": -485.05218505859375,
      "logps/rejected": -525.0234985351562,
      "loss": 0.5256,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.531062126159668,
      "rewards/margins": 1.007774829864502,
      "rewards/rejected": -2.53883695602417,
      "step": 1570
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.4573806804234344e-07,
      "logits/chosen": 1.0139309167861938,
      "logits/rejected": 1.3080246448516846,
      "logps/chosen": -372.17633056640625,
      "logps/rejected": -457.5221252441406,
      "loss": 0.5338,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.3809771537780762,
      "rewards/margins": 0.7371150851249695,
      "rewards/rejected": -2.1180920600891113,
      "step": 1580
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.2005839486327524e-07,
      "logits/chosen": 0.920538067817688,
      "logits/rejected": 1.3884646892547607,
      "logps/chosen": -408.55792236328125,
      "logps/rejected": -495.26007080078125,
      "loss": 0.4886,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.345349907875061,
      "rewards/margins": 0.9609087109565735,
      "rewards/rejected": -2.3062586784362793,
      "step": 1590
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.950725977699396e-07,
      "logits/chosen": 0.23017816245555878,
      "logits/rejected": 1.3223010301589966,
      "logps/chosen": -419.499755859375,
      "logps/rejected": -443.2064514160156,
      "loss": 0.5317,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.2144871950149536,
      "rewards/margins": 1.026813268661499,
      "rewards/rejected": -2.241300582885742,
      "step": 1600
    },
    {
      "epoch": 0.84,
      "eval_logits/chosen": 0.820225715637207,
      "eval_logits/rejected": 1.2751976251602173,
      "eval_logps/chosen": -389.5859069824219,
      "eval_logps/rejected": -461.7518310546875,
      "eval_loss": 0.5140097737312317,
      "eval_rewards/accuracies": 0.75,
      "eval_rewards/chosen": -1.281484842300415,
      "eval_rewards/margins": 0.8777413368225098,
      "eval_rewards/rejected": -2.159226179122925,
      "eval_runtime": 99.8683,
      "eval_samples_per_second": 20.026,
      "eval_steps_per_second": 0.32,
      "step": 1600
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.707890121151403e-07,
      "logits/chosen": 0.6467644572257996,
      "logits/rejected": 0.7336404323577881,
      "logps/chosen": -391.0705261230469,
      "logps/rejected": -510.64385986328125,
      "loss": 0.4993,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.2752177715301514,
      "rewards/margins": 0.9291207194328308,
      "rewards/rejected": -2.204338550567627,
      "step": 1610
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.4721573899138743e-07,
      "logits/chosen": 0.9288214445114136,
      "logits/rejected": 1.101698875427246,
      "logps/chosen": -416.335693359375,
      "logps/rejected": -487.194091796875,
      "loss": 0.5261,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3879741430282593,
      "rewards/margins": 0.7634307146072388,
      "rewards/rejected": -2.151404857635498,
      "step": 1620
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.2436064252834853e-07,
      "logits/chosen": 0.9459419250488281,
      "logits/rejected": 0.8354867696762085,
      "logps/chosen": -375.0196228027344,
      "logps/rejected": -481.24310302734375,
      "loss": 0.5144,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.3403657674789429,
      "rewards/margins": 0.9639253616333008,
      "rewards/rejected": -2.304291248321533,
      "step": 1630
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.022313472693447e-07,
      "logits/chosen": 0.7536389231681824,
      "logits/rejected": 0.9406282305717468,
      "logps/chosen": -388.83245849609375,
      "logps/rejected": -452.368896484375,
      "loss": 0.4979,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2452423572540283,
      "rewards/margins": 0.6904104948043823,
      "rewards/rejected": -1.9356529712677002,
      "step": 1640
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.808352356277738e-07,
      "logits/chosen": 0.5242574214935303,
      "logits/rejected": 1.6256099939346313,
      "logps/chosen": -419.309326171875,
      "logps/rejected": -446.4677734375,
      "loss": 0.5174,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.3458099365234375,
      "rewards/margins": 0.7937489748001099,
      "rewards/rejected": -2.139559030532837,
      "step": 1650
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.6017944542431394e-07,
      "logits/chosen": 0.4203415811061859,
      "logits/rejected": 0.6204716563224792,
      "logps/chosen": -407.8147277832031,
      "logps/rejected": -414.63201904296875,
      "loss": 0.5193,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.241703748703003,
      "rewards/margins": 0.5451580882072449,
      "rewards/rejected": -1.786861777305603,
      "step": 1660
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.4027086750571635e-07,
      "logits/chosen": 0.2778792977333069,
      "logits/rejected": 0.7795981168746948,
      "logps/chosen": -446.410888671875,
      "logps/rejected": -511.241455078125,
      "loss": 0.5197,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3088264465332031,
      "rewards/margins": 0.6983151435852051,
      "rewards/rejected": -2.007141590118408,
      "step": 1670
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.2111614344599686e-07,
      "logits/chosen": 0.31662610173225403,
      "logits/rejected": 0.2457505166530609,
      "logps/chosen": -361.7285461425781,
      "logps/rejected": -504.502197265625,
      "loss": 0.5237,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.1863205432891846,
      "rewards/margins": 0.8205347061157227,
      "rewards/rejected": -2.006855010986328,
      "step": 1680
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.027216633307813e-07,
      "logits/chosen": 0.7463894486427307,
      "logits/rejected": 0.9927177429199219,
      "logps/chosen": -337.95306396484375,
      "logps/rejected": -456.77288818359375,
      "loss": 0.5427,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2024390697479248,
      "rewards/margins": 0.9413339495658875,
      "rewards/rejected": -2.143772840499878,
      "step": 1690
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.8509356362554964e-07,
      "logits/chosen": 0.49471530318260193,
      "logits/rejected": 0.8219636678695679,
      "logps/chosen": -390.1451416015625,
      "logps/rejected": -428.46484375,
      "loss": 0.5384,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.300763726234436,
      "rewards/margins": 0.8397086262702942,
      "rewards/rejected": -2.140472173690796,
      "step": 1700
    },
    {
      "epoch": 0.89,
      "eval_logits/chosen": 0.8384195566177368,
      "eval_logits/rejected": 1.2938010692596436,
      "eval_logps/chosen": -386.92913818359375,
      "eval_logps/rejected": -458.70379638671875,
      "eval_loss": 0.5134327411651611,
      "eval_rewards/accuracies": 0.75390625,
      "eval_rewards/chosen": -1.2549172639846802,
      "eval_rewards/margins": 0.8738288283348083,
      "eval_rewards/rejected": -2.1287460327148438,
      "eval_runtime": 99.8361,
      "eval_samples_per_second": 20.033,
      "eval_steps_per_second": 0.321,
      "step": 1700
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.6823772512849086e-07,
      "logits/chosen": 1.2365819215774536,
      "logits/rejected": 1.28219473361969,
      "logps/chosen": -361.33197021484375,
      "logps/rejected": -429.3739318847656,
      "loss": 0.5435,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.4629837274551392,
      "rewards/margins": 0.5283857583999634,
      "rewards/rejected": -1.9913694858551025,
      "step": 1710
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.5215977100864394e-07,
      "logits/chosen": 0.44147777557373047,
      "logits/rejected": 0.7171396017074585,
      "logps/chosen": -452.79046630859375,
      "logps/rejected": -513.9752197265625,
      "loss": 0.516,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.2802356481552124,
      "rewards/margins": 0.8417502641677856,
      "rewards/rejected": -2.121985912322998,
      "step": 1720
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.3686506492999019e-07,
      "logits/chosen": 0.41590723395347595,
      "logits/rejected": 0.6705255508422852,
      "logps/chosen": -404.0863037109375,
      "logps/rejected": -468.7840270996094,
      "loss": 0.5693,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.4168977737426758,
      "rewards/margins": 0.6643761396408081,
      "rewards/rejected": -2.0812740325927734,
      "step": 1730
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.223587092621162e-07,
      "logits/chosen": 1.298346996307373,
      "logits/rejected": 1.2134599685668945,
      "logps/chosen": -342.03424072265625,
      "logps/rejected": -450.5323791503906,
      "loss": 0.4678,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.4358490705490112,
      "rewards/margins": 0.6444058418273926,
      "rewards/rejected": -2.0802547931671143,
      "step": 1740
    },
    {
      "epoch": 0.92,
      "learning_rate": 1.08645543378042e-07,
      "logits/chosen": 0.9801321029663086,
      "logits/rejected": 1.268592119216919,
      "logps/chosen": -363.4621887207031,
      "logps/rejected": -438.5384826660156,
      "loss": 0.4722,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.1044265031814575,
      "rewards/margins": 0.8860410451889038,
      "rewards/rejected": -1.9904674291610718,
      "step": 1750
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.573014203979241e-08,
      "logits/chosen": 0.8289960622787476,
      "logits/rejected": 1.2797130346298218,
      "logps/chosen": -406.34625244140625,
      "logps/rejected": -437.1571350097656,
      "loss": 0.4995,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.2909425497055054,
      "rewards/margins": 0.9118080139160156,
      "rewards/rejected": -2.2027504444122314,
      "step": 1760
    },
    {
      "epoch": 0.93,
      "learning_rate": 8.36168138722357e-08,
      "logits/chosen": 1.0189353227615356,
      "logits/rejected": 0.649911105632782,
      "logps/chosen": -410.6741638183594,
      "logps/rejected": -475.7628479003906,
      "loss": 0.5279,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.4780073165893555,
      "rewards/margins": 0.6910057067871094,
      "rewards/rejected": -2.169013023376465,
      "step": 1770
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.230959992571368e-08,
      "logits/chosen": 0.8552034497261047,
      "logits/rejected": 0.7838449478149414,
      "logps/chosen": -379.01898193359375,
      "logps/rejected": -456.55023193359375,
      "loss": 0.4994,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.3288278579711914,
      "rewards/margins": 0.6242281794548035,
      "rewards/rejected": -1.95305597782135,
      "step": 1780
    },
    {
      "epoch": 0.94,
      "learning_rate": 6.181227232793185e-08,
      "logits/chosen": 0.42763280868530273,
      "logits/rejected": 0.637231707572937,
      "logps/chosen": -440.8536071777344,
      "logps/rejected": -470.2457580566406,
      "loss": 0.513,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4187681674957275,
      "rewards/margins": 0.4330345690250397,
      "rewards/rejected": -1.8518028259277344,
      "step": 1790
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.212833302556258e-08,
      "logits/chosen": 0.6027169227600098,
      "logits/rejected": 1.1977932453155518,
      "logps/chosen": -382.1881408691406,
      "logps/rejected": -478.81787109375,
      "loss": 0.5619,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.3630785942077637,
      "rewards/margins": 0.7937374114990234,
      "rewards/rejected": -2.156816005706787,
      "step": 1800
    },
    {
      "epoch": 0.94,
      "eval_logits/chosen": 0.7986226677894592,
      "eval_logits/rejected": 1.2532446384429932,
      "eval_logps/chosen": -385.8194885253906,
      "eval_logps/rejected": -456.913330078125,
      "eval_loss": 0.5134571194648743,
      "eval_rewards/accuracies": 0.7578125,
      "eval_rewards/chosen": -1.2438209056854248,
      "eval_rewards/margins": 0.8670201301574707,
      "eval_rewards/rejected": -2.1108410358428955,
      "eval_runtime": 99.9232,
      "eval_samples_per_second": 20.015,
      "eval_steps_per_second": 0.32,
      "step": 1800
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.32610126159852e-08,
      "logits/chosen": 0.7875297665596008,
      "logits/rejected": 1.140857458114624,
      "logps/chosen": -437.805908203125,
      "logps/rejected": -488.8186950683594,
      "loss": 0.4932,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.306668996810913,
      "rewards/margins": 0.7567360997200012,
      "rewards/rejected": -2.0634050369262695,
      "step": 1810
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.521326926954533e-08,
      "logits/chosen": 0.038898296654224396,
      "logits/rejected": 0.8747750520706177,
      "logps/chosen": -430.21270751953125,
      "logps/rejected": -509.03338623046875,
      "loss": 0.5421,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.4582428932189941,
      "rewards/margins": 0.557195782661438,
      "rewards/rejected": -2.0154385566711426,
      "step": 1820
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.798778774269817e-08,
      "logits/chosen": 1.1741001605987549,
      "logits/rejected": 1.0272743701934814,
      "logps/chosen": -384.75341796875,
      "logps/rejected": -457.12432861328125,
      "loss": 0.5129,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.477851152420044,
      "rewards/margins": 0.7019861936569214,
      "rewards/rejected": -2.179837465286255,
      "step": 1830
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.1586978482366072e-08,
      "logits/chosen": 1.0403835773468018,
      "logits/rejected": 1.361781120300293,
      "logps/chosen": -325.6305847167969,
      "logps/rejected": -370.1999206542969,
      "loss": 0.4816,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.231935739517212,
      "rewards/margins": 0.4666922986507416,
      "rewards/rejected": -1.6986281871795654,
      "step": 1840
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.6012976821802583e-08,
      "logits/chosen": 0.4566507935523987,
      "logits/rejected": 0.8711694478988647,
      "logps/chosen": -389.9162902832031,
      "logps/rejected": -444.21783447265625,
      "loss": 0.543,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.235186219215393,
      "rewards/margins": 0.7044228315353394,
      "rewards/rejected": -1.939609169960022,
      "step": 1850
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.1267642268238121e-08,
      "logits/chosen": 0.705321192741394,
      "logits/rejected": 1.1006934642791748,
      "logps/chosen": -322.65911865234375,
      "logps/rejected": -389.2583312988281,
      "loss": 0.4922,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.2569407224655151,
      "rewards/margins": 0.7663857936859131,
      "rewards/rejected": -2.0233263969421387,
      "step": 1860
    },
    {
      "epoch": 0.98,
      "learning_rate": 7.352557882542022e-09,
      "logits/chosen": 0.11342964321374893,
      "logits/rejected": 1.1264240741729736,
      "logps/chosen": -491.3070373535156,
      "logps/rejected": -466.52423095703125,
      "loss": 0.475,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.4384621381759644,
      "rewards/margins": 0.7035077810287476,
      "rewards/rejected": -2.141970157623291,
      "step": 1870
    },
    {
      "epoch": 0.98,
      "learning_rate": 4.269029751107489e-09,
      "logits/chosen": 0.8203773498535156,
      "logits/rejected": 0.9573888778686523,
      "logps/chosen": -361.1534729003906,
      "logps/rejected": -452.228271484375,
      "loss": 0.5102,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.16557776927948,
      "rewards/margins": 0.9508386850357056,
      "rewards/rejected": -2.1164164543151855,
      "step": 1880
    },
    {
      "epoch": 0.99,
      "learning_rate": 2.0180865501362447e-09,
      "logits/chosen": 0.8782768249511719,
      "logits/rejected": 0.8547204732894897,
      "logps/chosen": -413.7078552246094,
      "logps/rejected": -529.6265869140625,
      "loss": 0.4938,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.4604012966156006,
      "rewards/margins": 0.9819992780685425,
      "rewards/rejected": -2.4424006938934326,
      "step": 1890
    },
    {
      "epoch": 0.99,
      "learning_rate": 6.004792024680295e-10,
      "logits/chosen": 0.9474574327468872,
      "logits/rejected": 0.7719841599464417,
      "logps/chosen": -407.2633972167969,
      "logps/rejected": -473.81585693359375,
      "loss": 0.5169,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.5017306804656982,
      "rewards/margins": 0.5851131081581116,
      "rewards/rejected": -2.086843967437744,
      "step": 1900
    },
    {
      "epoch": 0.99,
      "eval_logits/chosen": 0.7988529205322266,
      "eval_logits/rejected": 1.2523157596588135,
      "eval_logps/chosen": -385.9081726074219,
      "eval_logps/rejected": -457.0127868652344,
      "eval_loss": 0.5133188962936401,
      "eval_rewards/accuracies": 0.75390625,
      "eval_rewards/chosen": -1.244707703590393,
      "eval_rewards/margins": 0.8671280741691589,
      "eval_rewards/rejected": -2.1118357181549072,
      "eval_runtime": 99.8541,
      "eval_samples_per_second": 20.029,
      "eval_steps_per_second": 0.32,
      "step": 1900
    },
    {
      "epoch": 1.0,
      "learning_rate": 1.668062707155249e-11,
      "logits/chosen": 0.5608649253845215,
      "logits/rejected": 0.8075926899909973,
      "logps/chosen": -380.0560302734375,
      "logps/rejected": -421.47735595703125,
      "loss": 0.5332,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.450054407119751,
      "rewards/margins": 0.7468727827072144,
      "rewards/rejected": -2.196927309036255,
      "step": 1910
    },
    {
      "epoch": 1.0,
      "step": 1912,
      "total_flos": 0.0,
      "train_loss": 0.5577758540044768,
      "train_runtime": 7516.1301,
      "train_samples_per_second": 8.137,
      "train_steps_per_second": 0.254
    }
  ],
  "logging_steps": 10,
  "max_steps": 1912,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}