diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,10756 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 7000,
+  "global_step": 7642,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.535947712418301e-09,
+      "logits/chosen": -2.6548311710357666,
+      "logits/rejected": -2.625453472137451,
+      "logps/chosen": -241.47927856445312,
+      "logps/rejected": -255.76844787597656,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.535947712418302e-08,
+      "logits/chosen": -2.6198513507843018,
+      "logits/rejected": -2.599431037902832,
+      "logps/chosen": -295.6170959472656,
+      "logps/rejected": -264.95758056640625,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.3888888955116272,
+      "rewards/chosen": -0.00039648558595217764,
+      "rewards/margins": -0.00022628647275269032,
+      "rewards/rejected": -0.0001701990404399112,
+      "step": 10
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.3071895424836603e-07,
+      "logits/chosen": -2.641977310180664,
+      "logits/rejected": -2.625978946685791,
+      "logps/chosen": -285.5495300292969,
+      "logps/rejected": -288.0072937011719,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.0002985417377203703,
+      "rewards/margins": -0.00038930849405005574,
+      "rewards/rejected": 0.000687850231770426,
+      "step": 20
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.9607843137254904e-07,
+      "logits/chosen": -2.634305238723755,
+      "logits/rejected": -2.5763115882873535,
+      "logps/chosen": -252.57080078125,
+      "logps/rejected": -230.24319458007812,
+      "loss": 0.6934,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.00017283143824897707,
+      "rewards/margins": -0.0005285438382998109,
+      "rewards/rejected": 0.0003557124291546643,
+      "step": 30
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 2.6143790849673207e-07,
+      "logits/chosen": -2.6947145462036133,
+      "logits/rejected": -2.6161351203918457,
+      "logps/chosen": -283.332275390625,
+      "logps/rejected": -263.6659851074219,
+      "loss": 0.6934,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.00043302043923176825,
+      "rewards/margins": -0.00046260812086984515,
+      "rewards/rejected": 2.9587605240521953e-05,
+      "step": 40
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 3.267973856209151e-07,
+      "logits/chosen": -2.6638424396514893,
+      "logits/rejected": -2.6378073692321777,
+      "logps/chosen": -243.68295288085938,
+      "logps/rejected": -243.103271484375,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.0005886313738301396,
+      "rewards/margins": 0.00029269742663018405,
+      "rewards/rejected": 0.0002959339471999556,
+      "step": 50
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 3.921568627450981e-07,
+      "logits/chosen": -2.6192870140075684,
+      "logits/rejected": -2.6178603172302246,
+      "logps/chosen": -300.85760498046875,
+      "logps/rejected": -263.17523193359375,
+      "loss": 0.694,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.0010113344760611653,
+      "rewards/margins": -0.0017379462951794267,
+      "rewards/rejected": 0.0007266116444952786,
+      "step": 60
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 4.5751633986928105e-07,
+      "logits/chosen": -2.663377285003662,
+      "logits/rejected": -2.6458141803741455,
+      "logps/chosen": -264.3880615234375,
+      "logps/rejected": -221.2679443359375,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.0004358245641924441,
+      "rewards/margins": 0.00086634122999385,
+      "rewards/rejected": -0.0004305167240090668,
+      "step": 70
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.228758169934641e-07,
+      "logits/chosen": -2.6054091453552246,
+      "logits/rejected": -2.5989038944244385,
+      "logps/chosen": -311.854248046875,
+      "logps/rejected": -308.6831970214844,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.0007162230904214084,
+      "rewards/margins": 0.0005422987742349505,
+      "rewards/rejected": 0.00017392440349794924,
+      "step": 80
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.882352941176471e-07,
+      "logits/chosen": -2.702660322189331,
+      "logits/rejected": -2.663552761077881,
+      "logps/chosen": -275.44598388671875,
+      "logps/rejected": -243.2969970703125,
+      "loss": 0.6936,
+      "rewards/accuracies": 0.4124999940395355,
+      "rewards/chosen": -0.00016839444288052619,
+      "rewards/margins": -0.0009770167525857687,
+      "rewards/rejected": 0.000808622338809073,
+      "step": 90
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 6.535947712418302e-07,
+      "logits/chosen": -2.6264312267303467,
+      "logits/rejected": -2.6374175548553467,
+      "logps/chosen": -247.1935577392578,
+      "logps/rejected": -254.58740234375,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.0008388921851292253,
+      "rewards/margins": 0.0007487462717108428,
+      "rewards/rejected": 9.014595707412809e-05,
+      "step": 100
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 7.189542483660131e-07,
+      "logits/chosen": -2.6622517108917236,
+      "logits/rejected": -2.608503580093384,
+      "logps/chosen": -290.3687744140625,
+      "logps/rejected": -271.18170166015625,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.0006900489097461104,
+      "rewards/margins": 0.0007947798003442585,
+      "rewards/rejected": -0.0001047308323904872,
+      "step": 110
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 7.843137254901962e-07,
+      "logits/chosen": -2.660090446472168,
+      "logits/rejected": -2.6123783588409424,
+      "logps/chosen": -308.0357360839844,
+      "logps/rejected": -249.29544067382812,
+      "loss": 0.6934,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.00019220352987758815,
+      "rewards/margins": -0.0005792457377538085,
+      "rewards/rejected": 0.0003870422951877117,
+      "step": 120
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 8.496732026143792e-07,
+      "logits/chosen": -2.6670804023742676,
+      "logits/rejected": -2.6804003715515137,
+      "logps/chosen": -271.88018798828125,
+      "logps/rejected": -242.92874145507812,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.0007403042982332408,
+      "rewards/margins": 0.0005177179118618369,
+      "rewards/rejected": 0.00022258632816374302,
+      "step": 130
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.150326797385621e-07,
+      "logits/chosen": -2.6343283653259277,
+      "logits/rejected": -2.606320381164551,
+      "logps/chosen": -281.6643371582031,
+      "logps/rejected": -250.2583465576172,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 7.358348375419155e-05,
+      "rewards/margins": 0.000287846167339012,
+      "rewards/rejected": -0.00021426270541269332,
+      "step": 140
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.80392156862745e-07,
+      "logits/chosen": -2.7104547023773193,
+      "logits/rejected": -2.631685495376587,
+      "logps/chosen": -340.4215393066406,
+      "logps/rejected": -304.089599609375,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.0008256893488578498,
+      "rewards/margins": 0.0006710177985951304,
+      "rewards/rejected": 0.0001546714483993128,
+      "step": 150
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.0457516339869283e-06,
+      "logits/chosen": -2.6912295818328857,
+      "logits/rejected": -2.6192638874053955,
+      "logps/chosen": -263.2707214355469,
+      "logps/rejected": -241.91943359375,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.0006294053164310753,
+      "rewards/margins": -0.000346137210726738,
+      "rewards/rejected": -0.00028326810570433736,
+      "step": 160
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.111111111111111e-06,
+      "logits/chosen": -2.6141347885131836,
+      "logits/rejected": -2.589357852935791,
+      "logps/chosen": -247.6897430419922,
+      "logps/rejected": -230.26284790039062,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.00013610209862235934,
+      "rewards/margins": -0.00014482846017926931,
+      "rewards/rejected": 0.00028093045693822205,
+      "step": 170
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.1764705882352942e-06,
+      "logits/chosen": -2.7081546783447266,
+      "logits/rejected": -2.591892957687378,
+      "logps/chosen": -356.08349609375,
+      "logps/rejected": -272.71075439453125,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.0010515575995668769,
+      "rewards/margins": 0.00011522386921569705,
+      "rewards/rejected": 0.0009363336721435189,
+      "step": 180
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.2418300653594772e-06,
+      "logits/chosen": -2.612846851348877,
+      "logits/rejected": -2.5740773677825928,
+      "logps/chosen": -279.548583984375,
+      "logps/rejected": -228.6970672607422,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 0.0006955896387808025,
+      "rewards/margins": -0.00014482141705229878,
+      "rewards/rejected": 0.0008404110558331013,
+      "step": 190
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.3071895424836604e-06,
+      "logits/chosen": -2.6737027168273926,
+      "logits/rejected": -2.5836992263793945,
+      "logps/chosen": -264.43951416015625,
+      "logps/rejected": -266.1790466308594,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.0010568450670689344,
+      "rewards/margins": 0.0010566504206508398,
+      "rewards/rejected": 1.9475992019124533e-07,
+      "step": 200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.3725490196078434e-06,
+      "logits/chosen": -2.732459545135498,
+      "logits/rejected": -2.702866792678833,
+      "logps/chosen": -288.2290954589844,
+      "logps/rejected": -264.72613525390625,
+      "loss": 0.693,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.0013281672727316618,
+      "rewards/margins": 0.0002143843739759177,
+      "rewards/rejected": 0.0011137829860672355,
+      "step": 210
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.4379084967320261e-06,
+      "logits/chosen": -2.617934465408325,
+      "logits/rejected": -2.5537524223327637,
+      "logps/chosen": -289.5554504394531,
+      "logps/rejected": -263.4027404785156,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.001351059996522963,
+      "rewards/margins": 0.000558468047529459,
+      "rewards/rejected": 0.000792592007201165,
+      "step": 220
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.5032679738562091e-06,
+      "logits/chosen": -2.569349765777588,
+      "logits/rejected": -2.5612125396728516,
+      "logps/chosen": -237.69454956054688,
+      "logps/rejected": -280.23577880859375,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.0010062854271382093,
+      "rewards/margins": 3.270909655839205e-05,
+      "rewards/rejected": 0.0009735762141644955,
+      "step": 230
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.5686274509803923e-06,
+      "logits/chosen": -2.637214183807373,
+      "logits/rejected": -2.617504119873047,
+      "logps/chosen": -255.1501007080078,
+      "logps/rejected": -268.76898193359375,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.0017043532570824027,
+      "rewards/margins": 0.001065735355950892,
+      "rewards/rejected": 0.0006386177847161889,
+      "step": 240
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.6339869281045753e-06,
+      "logits/chosen": -2.6625113487243652,
+      "logits/rejected": -2.5959858894348145,
+      "logps/chosen": -323.64447021484375,
+      "logps/rejected": -278.6389465332031,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0023534377105534077,
+      "rewards/margins": 0.0016259655822068453,
+      "rewards/rejected": 0.000727472361177206,
+      "step": 250
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.6993464052287585e-06,
+      "logits/chosen": -2.689392566680908,
+      "logits/rejected": -2.63670015335083,
+      "logps/chosen": -301.657470703125,
+      "logps/rejected": -264.7417297363281,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.0035001803189516068,
+      "rewards/margins": 0.0018760201055556536,
+      "rewards/rejected": 0.0016241597477346659,
+      "step": 260
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.7647058823529414e-06,
+      "logits/chosen": -2.639979839324951,
+      "logits/rejected": -2.599606990814209,
+      "logps/chosen": -255.54641723632812,
+      "logps/rejected": -233.1532440185547,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.0029199840500950813,
+      "rewards/margins": 0.001584300072863698,
+      "rewards/rejected": 0.0013356839772313833,
+      "step": 270
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.8300653594771242e-06,
+      "logits/chosen": -2.7038564682006836,
+      "logits/rejected": -2.6199021339416504,
+      "logps/chosen": -325.8532409667969,
+      "logps/rejected": -290.75335693359375,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0041234977543354034,
+      "rewards/margins": 0.0016002748161554337,
+      "rewards/rejected": 0.0025232231710106134,
+      "step": 280
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.8954248366013072e-06,
+      "logits/chosen": -2.647611379623413,
+      "logits/rejected": -2.6253955364227295,
+      "logps/chosen": -296.51324462890625,
+      "logps/rejected": -272.4481506347656,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.004413033369928598,
+      "rewards/margins": 0.001867209910415113,
+      "rewards/rejected": 0.0025458228774368763,
+      "step": 290
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.96078431372549e-06,
+      "logits/chosen": -2.742915391921997,
+      "logits/rejected": -2.6598916053771973,
+      "logps/chosen": -306.30755615234375,
+      "logps/rejected": -253.3156280517578,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.005059394054114819,
+      "rewards/margins": 0.0029503516852855682,
+      "rewards/rejected": 0.002109042601659894,
+      "step": 300
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.0261437908496734e-06,
+      "logits/chosen": -2.6307213306427,
+      "logits/rejected": -2.6080451011657715,
+      "logps/chosen": -303.6937561035156,
+      "logps/rejected": -290.9927062988281,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.004248838871717453,
+      "rewards/margins": 0.001757316873408854,
+      "rewards/rejected": 0.0024915223475545645,
+      "step": 310
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.0915032679738565e-06,
+      "logits/chosen": -2.6685638427734375,
+      "logits/rejected": -2.5951087474823,
+      "logps/chosen": -280.6381530761719,
+      "logps/rejected": -276.507568359375,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.006356862839311361,
+      "rewards/margins": 0.0029213307425379753,
+      "rewards/rejected": 0.0034355330280959606,
+      "step": 320
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.1568627450980393e-06,
+      "logits/chosen": -2.6920762062072754,
+      "logits/rejected": -2.6318111419677734,
+      "logps/chosen": -307.9992370605469,
+      "logps/rejected": -256.708984375,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.005926724523305893,
+      "rewards/margins": 0.0028058711905032396,
+      "rewards/rejected": 0.0031208537984639406,
+      "step": 330
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.222222222222222e-06,
+      "logits/chosen": -2.6575253009796143,
+      "logits/rejected": -2.5525221824645996,
+      "logps/chosen": -278.5721740722656,
+      "logps/rejected": -232.0401611328125,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.007162533700466156,
+      "rewards/margins": 0.0030813287012279034,
+      "rewards/rejected": 0.004081204999238253,
+      "step": 340
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.2875816993464053e-06,
+      "logits/chosen": -2.7125134468078613,
+      "logits/rejected": -2.624390125274658,
+      "logps/chosen": -332.53369140625,
+      "logps/rejected": -271.7375793457031,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.008111939765512943,
+      "rewards/margins": 0.003727226285263896,
+      "rewards/rejected": 0.004384713713079691,
+      "step": 350
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.3529411764705885e-06,
+      "logits/chosen": -2.694873094558716,
+      "logits/rejected": -2.6326351165771484,
+      "logps/chosen": -266.7177734375,
+      "logps/rejected": -263.2748107910156,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.008503630757331848,
+      "rewards/margins": 0.0031798859126865864,
+      "rewards/rejected": 0.005323744844645262,
+      "step": 360
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.4183006535947716e-06,
+      "logits/chosen": -2.6473262310028076,
+      "logits/rejected": -2.641078472137451,
+      "logps/chosen": -270.0342712402344,
+      "logps/rejected": -247.8629913330078,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.00798349641263485,
+      "rewards/margins": 0.0034046161454170942,
+      "rewards/rejected": 0.004578880965709686,
+      "step": 370
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.4836601307189544e-06,
+      "logits/chosen": -2.6707968711853027,
+      "logits/rejected": -2.631885290145874,
+      "logps/chosen": -295.2742614746094,
+      "logps/rejected": -257.59832763671875,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.010473817586898804,
+      "rewards/margins": 0.0036211558617651463,
+      "rewards/rejected": 0.00685266125947237,
+      "step": 380
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.549019607843137e-06,
+      "logits/chosen": -2.6127049922943115,
+      "logits/rejected": -2.5472323894500732,
+      "logps/chosen": -311.64892578125,
+      "logps/rejected": -251.50717163085938,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.011281657963991165,
+      "rewards/margins": 0.0060841976664960384,
+      "rewards/rejected": 0.005197459366172552,
+      "step": 390
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.6143790849673208e-06,
+      "logits/chosen": -2.7016243934631348,
+      "logits/rejected": -2.702817440032959,
+      "logps/chosen": -295.6642150878906,
+      "logps/rejected": -304.6461486816406,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.010914499871432781,
+      "rewards/margins": 0.0038754392880946398,
+      "rewards/rejected": 0.00703906174749136,
+      "step": 400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.6797385620915036e-06,
+      "logits/chosen": -2.6141860485076904,
+      "logits/rejected": -2.561201572418213,
+      "logps/chosen": -268.1377258300781,
+      "logps/rejected": -220.4593505859375,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.011765848845243454,
+      "rewards/margins": 0.005084649659693241,
+      "rewards/rejected": 0.0066812001168727875,
+      "step": 410
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.7450980392156867e-06,
+      "logits/chosen": -2.6364364624023438,
+      "logits/rejected": -2.5971553325653076,
+      "logps/chosen": -289.3436584472656,
+      "logps/rejected": -284.9223327636719,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.013207967393100262,
+      "rewards/margins": 0.007489602081477642,
+      "rewards/rejected": 0.005718364380300045,
+      "step": 420
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.8104575163398695e-06,
+      "logits/chosen": -2.6831023693084717,
+      "logits/rejected": -2.6227285861968994,
+      "logps/chosen": -290.2114562988281,
+      "logps/rejected": -271.54486083984375,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.01368993241339922,
+      "rewards/margins": 0.006646145135164261,
+      "rewards/rejected": 0.007043786346912384,
+      "step": 430
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.8758169934640523e-06,
+      "logits/chosen": -2.7199249267578125,
+      "logits/rejected": -2.7110390663146973,
+      "logps/chosen": -279.10302734375,
+      "logps/rejected": -250.34030151367188,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.014295792207121849,
+      "rewards/margins": 0.005877715535461903,
+      "rewards/rejected": 0.008418076671659946,
+      "step": 440
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.9411764705882355e-06,
+      "logits/chosen": -2.684593439102173,
+      "logits/rejected": -2.6899209022521973,
+      "logps/chosen": -297.78277587890625,
+      "logps/rejected": -310.8353576660156,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.014938647858798504,
+      "rewards/margins": 0.006166353821754456,
+      "rewards/rejected": 0.008772294968366623,
+      "step": 450
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.0065359477124182e-06,
+      "logits/chosen": -2.615818977355957,
+      "logits/rejected": -2.5617904663085938,
+      "logps/chosen": -265.1422424316406,
+      "logps/rejected": -251.06689453125,
+      "loss": 0.69,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.017867058515548706,
+      "rewards/margins": 0.006379131227731705,
+      "rewards/rejected": 0.01148792915046215,
+      "step": 460
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.071895424836602e-06,
+      "logits/chosen": -2.648271083831787,
+      "logits/rejected": -2.64194917678833,
+      "logps/chosen": -285.4848937988281,
+      "logps/rejected": -263.63787841796875,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.02088136412203312,
+      "rewards/margins": 0.00808627251535654,
+      "rewards/rejected": 0.012795092537999153,
+      "step": 470
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.1372549019607846e-06,
+      "logits/chosen": -2.61929988861084,
+      "logits/rejected": -2.6027560234069824,
+      "logps/chosen": -292.32537841796875,
+      "logps/rejected": -243.3027801513672,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.022852329537272453,
+      "rewards/margins": 0.009881490841507912,
+      "rewards/rejected": 0.012970837764441967,
+      "step": 480
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.2026143790849674e-06,
+      "logits/chosen": -2.693856716156006,
+      "logits/rejected": -2.605499267578125,
+      "logps/chosen": -280.8228759765625,
+      "logps/rejected": -227.39205932617188,
+      "loss": 0.6875,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.01996048167347908,
+      "rewards/margins": 0.011412231251597404,
+      "rewards/rejected": 0.00854825135320425,
+      "step": 490
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.2679738562091506e-06,
+      "logits/chosen": -2.656646728515625,
+      "logits/rejected": -2.5914053916931152,
+      "logps/chosen": -259.90399169921875,
+      "logps/rejected": -237.15554809570312,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.017155682668089867,
+      "rewards/margins": 0.007095979992300272,
+      "rewards/rejected": 0.010059705004096031,
+      "step": 500
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.3333333333333333e-06,
+      "logits/chosen": -2.609172821044922,
+      "logits/rejected": -2.599658727645874,
+      "logps/chosen": -270.3688659667969,
+      "logps/rejected": -264.7857360839844,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.023332804441452026,
+      "rewards/margins": 0.0103679858148098,
+      "rewards/rejected": 0.012964817695319653,
+      "step": 510
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.398692810457517e-06,
+      "logits/chosen": -2.6225383281707764,
+      "logits/rejected": -2.5244414806365967,
+      "logps/chosen": -259.9064025878906,
+      "logps/rejected": -264.4110107421875,
+      "loss": 0.688,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.018774518743157387,
+      "rewards/margins": 0.010412798263132572,
+      "rewards/rejected": 0.00836171954870224,
+      "step": 520
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.4640522875816997e-06,
+      "logits/chosen": -2.6073412895202637,
+      "logits/rejected": -2.557931900024414,
+      "logps/chosen": -267.40045166015625,
+      "logps/rejected": -240.80545043945312,
+      "loss": 0.6859,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.02611006423830986,
+      "rewards/margins": 0.014706981368362904,
+      "rewards/rejected": 0.011403081007301807,
+      "step": 530
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.529411764705883e-06,
+      "logits/chosen": -2.656104803085327,
+      "logits/rejected": -2.5741233825683594,
+      "logps/chosen": -297.51910400390625,
+      "logps/rejected": -266.53363037109375,
+      "loss": 0.6859,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.02494840696454048,
+      "rewards/margins": 0.01465518493205309,
+      "rewards/rejected": 0.010293224826455116,
+      "step": 540
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.5947712418300657e-06,
+      "logits/chosen": -2.6402623653411865,
+      "logits/rejected": -2.60183048248291,
+      "logps/chosen": -259.5213317871094,
+      "logps/rejected": -248.22152709960938,
+      "loss": 0.686,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.02932048961520195,
+      "rewards/margins": 0.014470371417701244,
+      "rewards/rejected": 0.014850117266178131,
+      "step": 550
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.6601307189542484e-06,
+      "logits/chosen": -2.669956684112549,
+      "logits/rejected": -2.617684841156006,
+      "logps/chosen": -293.4594421386719,
+      "logps/rejected": -301.4708251953125,
+      "loss": 0.6858,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.027803082019090652,
+      "rewards/margins": 0.014931738376617432,
+      "rewards/rejected": 0.012871342711150646,
+      "step": 560
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.7254901960784316e-06,
+      "logits/chosen": -2.7117550373077393,
+      "logits/rejected": -2.6599695682525635,
+      "logps/chosen": -272.2671203613281,
+      "logps/rejected": -257.97760009765625,
+      "loss": 0.6863,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.027036011219024658,
+      "rewards/margins": 0.014011162333190441,
+      "rewards/rejected": 0.013024847023189068,
+      "step": 570
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.7908496732026144e-06,
+      "logits/chosen": -2.68557071685791,
+      "logits/rejected": -2.588757276535034,
+      "logps/chosen": -318.75567626953125,
+      "logps/rejected": -289.80572509765625,
+      "loss": 0.6833,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.03800877183675766,
+      "rewards/margins": 0.020139571279287338,
+      "rewards/rejected": 0.017869198694825172,
+      "step": 580
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.856209150326798e-06,
+      "logits/chosen": -2.68744158744812,
+      "logits/rejected": -2.597557544708252,
+      "logps/chosen": -297.6092529296875,
+      "logps/rejected": -259.01068115234375,
+      "loss": 0.683,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.03749191015958786,
+      "rewards/margins": 0.020818348973989487,
+      "rewards/rejected": 0.016673561185598373,
+      "step": 590
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.92156862745098e-06,
+      "logits/chosen": -2.6166014671325684,
+      "logits/rejected": -2.5153515338897705,
+      "logps/chosen": -265.6646728515625,
+      "logps/rejected": -248.9541473388672,
+      "loss": 0.686,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.03428620472550392,
+      "rewards/margins": 0.014840200543403625,
+      "rewards/rejected": 0.019446004182100296,
+      "step": 600
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.986928104575164e-06,
+      "logits/chosen": -2.6381916999816895,
+      "logits/rejected": -2.559558868408203,
+      "logps/chosen": -240.59716796875,
+      "logps/rejected": -205.5615692138672,
+      "loss": 0.6846,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.03674664348363876,
+      "rewards/margins": 0.017332863062620163,
+      "rewards/rejected": 0.0194137804210186,
+      "step": 610
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.052287581699347e-06,
+      "logits/chosen": -2.628159999847412,
+      "logits/rejected": -2.564661741256714,
+      "logps/chosen": -300.2850646972656,
+      "logps/rejected": -268.270263671875,
+      "loss": 0.6853,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.03930061683058739,
+      "rewards/margins": 0.01672186143696308,
+      "rewards/rejected": 0.022578757256269455,
+      "step": 620
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.11764705882353e-06,
+      "logits/chosen": -2.6141462326049805,
+      "logits/rejected": -2.5952517986297607,
+      "logps/chosen": -281.1327819824219,
+      "logps/rejected": -259.13372802734375,
+      "loss": 0.6856,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.03937515243887901,
+      "rewards/margins": 0.015657316893339157,
+      "rewards/rejected": 0.023717842996120453,
+      "step": 630
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.183006535947713e-06,
+      "logits/chosen": -2.684302806854248,
+      "logits/rejected": -2.613997220993042,
+      "logps/chosen": -273.54119873046875,
+      "logps/rejected": -260.19879150390625,
+      "loss": 0.683,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.03751297667622566,
+      "rewards/margins": 0.02131773717701435,
+      "rewards/rejected": 0.01619523949921131,
+      "step": 640
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.2483660130718954e-06,
+      "logits/chosen": -2.638551950454712,
+      "logits/rejected": -2.6129698753356934,
+      "logps/chosen": -271.42877197265625,
+      "logps/rejected": -257.5237731933594,
+      "loss": 0.6855,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.02996763028204441,
+      "rewards/margins": 0.01616152748465538,
+      "rewards/rejected": 0.01380610466003418,
+      "step": 650
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.313725490196079e-06,
+      "logits/chosen": -2.657040596008301,
+      "logits/rejected": -2.6005265712738037,
+      "logps/chosen": -317.86322021484375,
+      "logps/rejected": -262.32012939453125,
+      "loss": 0.6827,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.03117840550839901,
+      "rewards/margins": 0.021571574732661247,
+      "rewards/rejected": 0.009606831707060337,
+      "step": 660
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.379084967320262e-06,
+      "logits/chosen": -2.7391180992126465,
+      "logits/rejected": -2.6445021629333496,
+      "logps/chosen": -301.35858154296875,
+      "logps/rejected": -317.3902282714844,
+      "loss": 0.68,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.03976508975028992,
+      "rewards/margins": 0.02696283720433712,
+      "rewards/rejected": 0.012802252545952797,
+      "step": 670
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.444444444444444e-06,
+      "logits/chosen": -2.729236602783203,
+      "logits/rejected": -2.6563024520874023,
+      "logps/chosen": -289.63568115234375,
+      "logps/rejected": -266.2176513671875,
+      "loss": 0.6824,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.04282195121049881,
+      "rewards/margins": 0.022346172481775284,
+      "rewards/rejected": 0.020475778728723526,
+      "step": 680
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.509803921568628e-06,
+      "logits/chosen": -2.685117244720459,
+      "logits/rejected": -2.63557505607605,
+      "logps/chosen": -311.70892333984375,
+      "logps/rejected": -271.84857177734375,
+      "loss": 0.6824,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.03460826724767685,
+      "rewards/margins": 0.02228071726858616,
+      "rewards/rejected": 0.01232755184173584,
+      "step": 690
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.5751633986928105e-06,
+      "logits/chosen": -2.667707920074463,
+      "logits/rejected": -2.654254913330078,
+      "logps/chosen": -286.30767822265625,
+      "logps/rejected": -274.2806701660156,
+      "loss": 0.6803,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.02861057221889496,
+      "rewards/margins": 0.026673460379242897,
+      "rewards/rejected": 0.0019371159141883254,
+      "step": 700
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.640522875816994e-06,
+      "logits/chosen": -2.6690964698791504,
+      "logits/rejected": -2.618622303009033,
+      "logps/chosen": -293.69586181640625,
+      "logps/rejected": -277.69219970703125,
+      "loss": 0.677,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.034188129007816315,
+      "rewards/margins": 0.03403637558221817,
+      "rewards/rejected": 0.00015175435692071915,
+      "step": 710
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.705882352941177e-06,
+      "logits/chosen": -2.6516525745391846,
+      "logits/rejected": -2.6103947162628174,
+      "logps/chosen": -333.8087158203125,
+      "logps/rejected": -309.9309387207031,
+      "loss": 0.6782,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.03538156673312187,
+      "rewards/margins": 0.03180857375264168,
+      "rewards/rejected": 0.003572994377464056,
+      "step": 720
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.77124183006536e-06,
+      "logits/chosen": -2.6342673301696777,
+      "logits/rejected": -2.615257978439331,
+      "logps/chosen": -294.0147399902344,
+      "logps/rejected": -282.20562744140625,
+      "loss": 0.6768,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.03988462686538696,
+      "rewards/margins": 0.03406631201505661,
+      "rewards/rejected": 0.005818313919007778,
+      "step": 730
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.836601307189543e-06,
+      "logits/chosen": -2.627166271209717,
+      "logits/rejected": -2.5512778759002686,
+      "logps/chosen": -312.60919189453125,
+      "logps/rejected": -265.62127685546875,
+      "loss": 0.6762,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.048726730048656464,
+      "rewards/margins": 0.03582502156496048,
+      "rewards/rejected": 0.01290170568972826,
+      "step": 740
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.901960784313726e-06,
+      "logits/chosen": -2.6120457649230957,
+      "logits/rejected": -2.5613958835601807,
+      "logps/chosen": -252.6622772216797,
+      "logps/rejected": -243.0688934326172,
+      "loss": 0.6721,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.03125419467687607,
+      "rewards/margins": 0.04380708932876587,
+      "rewards/rejected": -0.01255289651453495,
+      "step": 750
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.967320261437909e-06,
+      "logits/chosen": -2.653477668762207,
+      "logits/rejected": -2.5420901775360107,
+      "logps/chosen": -284.51300048828125,
+      "logps/rejected": -243.9912109375,
+      "loss": 0.6685,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.033526089042425156,
+      "rewards/margins": 0.051363784819841385,
+      "rewards/rejected": -0.01783769577741623,
+      "step": 760
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.99999347843947e-06,
+      "logits/chosen": -2.6220593452453613,
+      "logits/rejected": -2.6276183128356934,
+      "logps/chosen": -297.4268493652344,
+      "logps/rejected": -277.671875,
+      "loss": 0.6758,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.026646548882126808,
+      "rewards/margins": 0.03750267252326012,
+      "rewards/rejected": -0.010856127366423607,
+      "step": 770
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.999941306159375e-06,
+      "logits/chosen": -2.6617603302001953,
+      "logits/rejected": -2.5644752979278564,
+      "logps/chosen": -310.10235595703125,
+      "logps/rejected": -277.78509521484375,
+      "loss": 0.661,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.016902543604373932,
+      "rewards/margins": 0.0686565712094307,
+      "rewards/rejected": -0.05175402760505676,
+      "step": 780
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.999836962687967e-06,
+      "logits/chosen": -2.5809264183044434,
+      "logits/rejected": -2.5831658840179443,
+      "logps/chosen": -296.1529541015625,
+      "logps/rejected": -299.0150146484375,
+      "loss": 0.6699,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.0022638884838670492,
+      "rewards/margins": 0.05030262470245361,
+      "rewards/rejected": -0.052566517144441605,
+      "step": 790
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.999680450202786e-06,
+      "logits/chosen": -2.4812557697296143,
+      "logits/rejected": -2.4974255561828613,
+      "logps/chosen": -275.08343505859375,
+      "logps/rejected": -295.24969482421875,
+      "loss": 0.6736,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.010703706182539463,
+      "rewards/margins": 0.04286975413560867,
+      "rewards/rejected": -0.05357345938682556,
+      "step": 800
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.999471771970087e-06,
+      "logits/chosen": -2.5986766815185547,
+      "logits/rejected": -2.5497448444366455,
+      "logps/chosen": -304.4241027832031,
+      "logps/rejected": -274.26422119140625,
+      "loss": 0.6694,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.009451848454773426,
+      "rewards/margins": 0.0511636957526207,
+      "rewards/rejected": -0.06061554700136185,
+      "step": 810
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.999210932344767e-06,
+      "logits/chosen": -2.683938503265381,
+      "logits/rejected": -2.6293458938598633,
+      "logps/chosen": -324.17730712890625,
+      "logps/rejected": -284.7932434082031,
+      "loss": 0.6633,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.0026789132971316576,
+      "rewards/margins": 0.06414036452770233,
+      "rewards/rejected": -0.06681927293539047,
+      "step": 820
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.998897936770281e-06,
+      "logits/chosen": -2.5737874507904053,
+      "logits/rejected": -2.6123242378234863,
+      "logps/chosen": -250.7256317138672,
+      "logps/rejected": -282.9770202636719,
+      "loss": 0.6741,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.033450886607170105,
+      "rewards/margins": 0.042477719485759735,
+      "rewards/rejected": -0.07592860609292984,
+      "step": 830
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.998532791778521e-06,
+      "logits/chosen": -2.5229029655456543,
+      "logits/rejected": -2.441638469696045,
+      "logps/chosen": -313.8226318359375,
+      "logps/rejected": -248.78286743164062,
+      "loss": 0.6595,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.03411540389060974,
+      "rewards/margins": 0.07376129925251007,
+      "rewards/rejected": -0.10787669569253922,
+      "step": 840
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.9981155049896885e-06,
+      "logits/chosen": -2.5809288024902344,
+      "logits/rejected": -2.529169797897339,
+      "logps/chosen": -299.9226379394531,
+      "logps/rejected": -322.46075439453125,
+      "loss": 0.6682,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.05599908158183098,
+      "rewards/margins": 0.05655177682638168,
+      "rewards/rejected": -0.11255085468292236,
+      "step": 850
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.997646085112126e-06,
+      "logits/chosen": -2.623964309692383,
+      "logits/rejected": -2.556356906890869,
+      "logps/chosen": -265.13909912109375,
+      "logps/rejected": -266.3385009765625,
+      "loss": 0.6642,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.05545296519994736,
+      "rewards/margins": 0.06523346155881882,
+      "rewards/rejected": -0.12068643420934677,
+      "step": 860
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.997124541942141e-06,
+      "logits/chosen": -2.5906269550323486,
+      "logits/rejected": -2.548358201980591,
+      "logps/chosen": -300.20611572265625,
+      "logps/rejected": -296.0473327636719,
+      "loss": 0.666,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.01703239232301712,
+      "rewards/margins": 0.060006629675626755,
+      "rewards/rejected": -0.07703902572393417,
+      "step": 870
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.996550886363801e-06,
+      "logits/chosen": -2.5821096897125244,
+      "logits/rejected": -2.533576726913452,
+      "logps/chosen": -270.1414794921875,
+      "logps/rejected": -289.24896240234375,
+      "loss": 0.6758,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.015453094616532326,
+      "rewards/margins": 0.040653444826602936,
+      "rewards/rejected": -0.05610654503107071,
+      "step": 880
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.995925130348706e-06,
+      "logits/chosen": -2.612670421600342,
+      "logits/rejected": -2.5754215717315674,
+      "logps/chosen": -270.81439208984375,
+      "logps/rejected": -278.97735595703125,
+      "loss": 0.6735,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.009032963775098324,
+      "rewards/margins": 0.0453304797410965,
+      "rewards/rejected": -0.0362975150346756,
+      "step": 890
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.995247286955734e-06,
+      "logits/chosen": -2.596879482269287,
+      "logits/rejected": -2.5179409980773926,
+      "logps/chosen": -263.36505126953125,
+      "logps/rejected": -231.7235565185547,
+      "loss": 0.666,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.026518285274505615,
+      "rewards/margins": 0.06129720062017441,
+      "rewards/rejected": -0.08781548589468002,
+      "step": 900
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.994517370330779e-06,
+      "logits/chosen": -2.5293946266174316,
+      "logits/rejected": -2.5589756965637207,
+      "logps/chosen": -290.03570556640625,
+      "logps/rejected": -323.95782470703125,
+      "loss": 0.6561,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.01835092343389988,
+      "rewards/margins": 0.0834575966000557,
+      "rewards/rejected": -0.10180851072072983,
+      "step": 910
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.993735395706446e-06,
+      "logits/chosen": -2.588839054107666,
+      "logits/rejected": -2.508923053741455,
+      "logps/chosen": -245.86978149414062,
+      "logps/rejected": -251.2750701904297,
+      "loss": 0.6497,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.008198278024792671,
+      "rewards/margins": 0.098419688642025,
+      "rewards/rejected": -0.10661796480417252,
+      "step": 920
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.992901379401737e-06,
+      "logits/chosen": -2.655158519744873,
+      "logits/rejected": -2.6311938762664795,
+      "logps/chosen": -325.3824157714844,
+      "logps/rejected": -301.60150146484375,
+      "loss": 0.657,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.017573771998286247,
+      "rewards/margins": 0.08263130486011505,
+      "rewards/rejected": -0.06505753099918365,
+      "step": 930
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.992015338821711e-06,
+      "logits/chosen": -2.5817184448242188,
+      "logits/rejected": -2.5037972927093506,
+      "logps/chosen": -297.3199768066406,
+      "logps/rejected": -289.2392578125,
+      "loss": 0.6555,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.008880408480763435,
+      "rewards/margins": 0.08480361104011536,
+      "rewards/rejected": -0.09368401765823364,
+      "step": 940
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.991077292457117e-06,
+      "logits/chosen": -2.609684705734253,
+      "logits/rejected": -2.5322821140289307,
+      "logps/chosen": -291.2019958496094,
+      "logps/rejected": -267.49969482421875,
+      "loss": 0.6496,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.008642762899398804,
+      "rewards/margins": 0.09803656488656998,
+      "rewards/rejected": -0.10667933523654938,
+      "step": 950
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.990087259884016e-06,
+      "logits/chosen": -2.6076533794403076,
+      "logits/rejected": -2.554643154144287,
+      "logps/chosen": -267.25732421875,
+      "logps/rejected": -248.2394561767578,
+      "loss": 0.6481,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.03241444751620293,
+      "rewards/margins": 0.10384710133075714,
+      "rewards/rejected": -0.13626155257225037,
+      "step": 960
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.989045261763362e-06,
+      "logits/chosen": -2.595069646835327,
+      "logits/rejected": -2.5618221759796143,
+      "logps/chosen": -297.8432922363281,
+      "logps/rejected": -288.5405578613281,
+      "loss": 0.6413,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.01923452876508236,
+      "rewards/margins": 0.11537112295627594,
+      "rewards/rejected": -0.13460564613342285,
+      "step": 970
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.98795131984058e-06,
+      "logits/chosen": -2.508185625076294,
+      "logits/rejected": -2.502739667892456,
+      "logps/chosen": -274.4787902832031,
+      "logps/rejected": -250.0324249267578,
+      "loss": 0.6503,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.030040761455893517,
+      "rewards/margins": 0.09917756170034409,
+      "rewards/rejected": -0.12921833992004395,
+      "step": 980
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.986805456945107e-06,
+      "logits/chosen": -2.5590319633483887,
+      "logits/rejected": -2.514545440673828,
+      "logps/chosen": -238.1454620361328,
+      "logps/rejected": -229.8460235595703,
+      "loss": 0.6393,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.031098058447241783,
+      "rewards/margins": 0.12257441133260727,
+      "rewards/rejected": -0.09147634357213974,
+      "step": 990
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.985607696989919e-06,
+      "logits/chosen": -2.6065621376037598,
+      "logits/rejected": -2.605027675628662,
+      "logps/chosen": -287.16986083984375,
+      "logps/rejected": -285.27789306640625,
+      "loss": 0.6501,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.012579170055687428,
+      "rewards/margins": 0.10108743607997894,
+      "rewards/rejected": -0.1136665940284729,
+      "step": 1000
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.984358064971026e-06,
+      "logits/chosen": -2.560281753540039,
+      "logits/rejected": -2.490305185317993,
+      "logps/chosen": -288.9957580566406,
+      "logps/rejected": -264.52166748046875,
+      "loss": 0.6591,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.08325138688087463,
+      "rewards/margins": 0.08632998168468475,
+      "rewards/rejected": -0.1695813685655594,
+      "step": 1010
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.983056586966958e-06,
+      "logits/chosen": -2.5651133060455322,
+      "logits/rejected": -2.4909870624542236,
+      "logps/chosen": -334.9750061035156,
+      "logps/rejected": -276.69317626953125,
+      "loss": 0.6293,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.04752253741025925,
+      "rewards/margins": 0.1456046998500824,
+      "rewards/rejected": -0.19312722980976105,
+      "step": 1020
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.981703290138215e-06,
+      "logits/chosen": -2.631713390350342,
+      "logits/rejected": -2.581732988357544,
+      "logps/chosen": -326.59320068359375,
+      "logps/rejected": -311.9397888183594,
+      "loss": 0.6382,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.05011386424303055,
+      "rewards/margins": 0.1374647617340088,
+      "rewards/rejected": -0.18757864832878113,
+      "step": 1030
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.980298202726706e-06,
+      "logits/chosen": -2.691251516342163,
+      "logits/rejected": -2.624222993850708,
+      "logps/chosen": -328.1868591308594,
+      "logps/rejected": -298.66473388671875,
+      "loss": 0.6088,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.007155093364417553,
+      "rewards/margins": 0.19580598175525665,
+      "rewards/rejected": -0.20296108722686768,
+      "step": 1040
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.978841354055148e-06,
+      "logits/chosen": -2.548856019973755,
+      "logits/rejected": -2.522235631942749,
+      "logps/chosen": -281.12835693359375,
+      "logps/rejected": -274.5442199707031,
+      "loss": 0.6485,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.0306741651147604,
+      "rewards/margins": 0.1061779260635376,
+      "rewards/rejected": -0.13685208559036255,
+      "step": 1050
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.977332774526471e-06,
+      "logits/chosen": -2.5694425106048584,
+      "logits/rejected": -2.4372596740722656,
+      "logps/chosen": -288.7796325683594,
+      "logps/rejected": -246.0713653564453,
+      "loss": 0.6226,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.016771093010902405,
+      "rewards/margins": 0.16713151335716248,
+      "rewards/rejected": -0.15036042034626007,
+      "step": 1060
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.97577249562317e-06,
+      "logits/chosen": -2.4907641410827637,
+      "logits/rejected": -2.469911813735962,
+      "logps/chosen": -273.491455078125,
+      "logps/rejected": -273.80584716796875,
+      "loss": 0.6279,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.045688532292842865,
+      "rewards/margins": 0.15459074079990387,
+      "rewards/rejected": -0.10890217870473862,
+      "step": 1070
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.974160549906652e-06,
+      "logits/chosen": -2.4853806495666504,
+      "logits/rejected": -2.428475856781006,
+      "logps/chosen": -265.21478271484375,
+      "logps/rejected": -256.92156982421875,
+      "loss": 0.6438,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.01522951852530241,
+      "rewards/margins": 0.12189844995737076,
+      "rewards/rejected": -0.13712796568870544,
+      "step": 1080
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.972496971016559e-06,
+      "logits/chosen": -2.506234884262085,
+      "logits/rejected": -2.49314546585083,
+      "logps/chosen": -270.284912109375,
+      "logps/rejected": -250.1904754638672,
+      "loss": 0.6173,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.011768155731260777,
+      "rewards/margins": 0.1889091283082962,
+      "rewards/rejected": -0.1771409809589386,
+      "step": 1090
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.9707817936700635e-06,
+      "logits/chosen": -2.5315027236938477,
+      "logits/rejected": -2.4613234996795654,
+      "logps/chosen": -289.7018737792969,
+      "logps/rejected": -279.943115234375,
+      "loss": 0.6455,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.04312612861394882,
+      "rewards/margins": 0.1247950941324234,
+      "rewards/rejected": -0.16792121529579163,
+      "step": 1100
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.969015053661142e-06,
+      "logits/chosen": -2.532196521759033,
+      "logits/rejected": -2.4918360710144043,
+      "logps/chosen": -304.68524169921875,
+      "logps/rejected": -285.6064453125,
+      "loss": 0.657,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.10881493240594864,
+      "rewards/margins": 0.09789443016052246,
+      "rewards/rejected": -0.2067093551158905,
+      "step": 1110
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.967196787859835e-06,
+      "logits/chosen": -2.560375928878784,
+      "logits/rejected": -2.490992784500122,
+      "logps/chosen": -307.4469299316406,
+      "logps/rejected": -310.51800537109375,
+      "loss": 0.644,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.09717439115047455,
+      "rewards/margins": 0.12193155288696289,
+      "rewards/rejected": -0.21910595893859863,
+      "step": 1120
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.965327034211469e-06,
+      "logits/chosen": -2.5642170906066895,
+      "logits/rejected": -2.4959988594055176,
+      "logps/chosen": -319.49249267578125,
+      "logps/rejected": -290.97015380859375,
+      "loss": 0.6031,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.07515348494052887,
+      "rewards/margins": 0.2159455269575119,
+      "rewards/rejected": -0.29109901189804077,
+      "step": 1130
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.96340583173587e-06,
+      "logits/chosen": -2.554924249649048,
+      "logits/rejected": -2.4227776527404785,
+      "logps/chosen": -267.262451171875,
+      "logps/rejected": -251.24148559570312,
+      "loss": 0.6181,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.12634123861789703,
+      "rewards/margins": 0.18214096128940582,
+      "rewards/rejected": -0.30848219990730286,
+      "step": 1140
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.96143322052655e-06,
+      "logits/chosen": -2.5284454822540283,
+      "logits/rejected": -2.573359251022339,
+      "logps/chosen": -309.14788818359375,
+      "logps/rejected": -300.38909912109375,
+      "loss": 0.6614,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.07307339459657669,
+      "rewards/margins": 0.09202548116445541,
+      "rewards/rejected": -0.1650988608598709,
+      "step": 1150
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.959409241749864e-06,
+      "logits/chosen": -2.555816650390625,
+      "logits/rejected": -2.461007595062256,
+      "logps/chosen": -291.8132629394531,
+      "logps/rejected": -305.55926513671875,
+      "loss": 0.6302,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.036139845848083496,
+      "rewards/margins": 0.15513625741004944,
+      "rewards/rejected": -0.19127610325813293,
+      "step": 1160
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.957333937644159e-06,
+      "logits/chosen": -2.5880494117736816,
+      "logits/rejected": -2.537034511566162,
+      "logps/chosen": -310.62689208984375,
+      "logps/rejected": -296.87628173828125,
+      "loss": 0.6264,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.07262597978115082,
+      "rewards/margins": 0.16844379901885986,
+      "rewards/rejected": -0.24106979370117188,
+      "step": 1170
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.955207351518885e-06,
+      "logits/chosen": -2.5874555110931396,
+      "logits/rejected": -2.574850559234619,
+      "logps/chosen": -288.90997314453125,
+      "logps/rejected": -264.8959045410156,
+      "loss": 0.641,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.11980124562978745,
+      "rewards/margins": 0.13190236687660217,
+      "rewards/rejected": -0.2517036199569702,
+      "step": 1180
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.953029527753699e-06,
+      "logits/chosen": -2.5432021617889404,
+      "logits/rejected": -2.5398898124694824,
+      "logps/chosen": -270.48175048828125,
+      "logps/rejected": -346.1780090332031,
+      "loss": 0.6798,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.2177748680114746,
+      "rewards/margins": 0.06020503118634224,
+      "rewards/rejected": -0.27797991037368774,
+      "step": 1190
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.95080051179753e-06,
+      "logits/chosen": -2.4899439811706543,
+      "logits/rejected": -2.48333740234375,
+      "logps/chosen": -287.114013671875,
+      "logps/rejected": -292.2990417480469,
+      "loss": 0.6561,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.24787192046642303,
+      "rewards/margins": 0.09571520984172821,
+      "rewards/rejected": -0.34358716011047363,
+      "step": 1200
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.948520350167637e-06,
+      "logits/chosen": -2.502089738845825,
+      "logits/rejected": -2.456737995147705,
+      "logps/chosen": -306.0370788574219,
+      "logps/rejected": -303.1842346191406,
+      "loss": 0.6316,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.2280178964138031,
+      "rewards/margins": 0.15330472588539124,
+      "rewards/rejected": -0.38132262229919434,
+      "step": 1210
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.946189090448639e-06,
+      "logits/chosen": -2.4915668964385986,
+      "logits/rejected": -2.4508776664733887,
+      "logps/chosen": -318.48333740234375,
+      "logps/rejected": -276.1875,
+      "loss": 0.6827,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.28770923614501953,
+      "rewards/margins": 0.04758835583925247,
+      "rewards/rejected": -0.3352976143360138,
+      "step": 1220
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.943806781291515e-06,
+      "logits/chosen": -2.53739070892334,
+      "logits/rejected": -2.490952491760254,
+      "logps/chosen": -289.3685607910156,
+      "logps/rejected": -271.5808410644531,
+      "loss": 0.5964,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.12012603133916855,
+      "rewards/margins": 0.23385348916053772,
+      "rewards/rejected": -0.35397952795028687,
+      "step": 1230
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.941373472412595e-06,
+      "logits/chosen": -2.5729668140411377,
+      "logits/rejected": -2.527350902557373,
+      "logps/chosen": -339.57049560546875,
+      "logps/rejected": -328.00103759765625,
+      "loss": 0.638,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.07913851737976074,
+      "rewards/margins": 0.15087315440177917,
+      "rewards/rejected": -0.23001165688037872,
+      "step": 1240
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.938889214592521e-06,
+      "logits/chosen": -2.572094678878784,
+      "logits/rejected": -2.5283236503601074,
+      "logps/chosen": -339.66009521484375,
+      "logps/rejected": -312.3247375488281,
+      "loss": 0.6597,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.002257655607536435,
+      "rewards/margins": 0.09324416518211365,
+      "rewards/rejected": -0.09550182521343231,
+      "step": 1250
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.936354059675186e-06,
+      "logits/chosen": -2.5886847972869873,
+      "logits/rejected": -2.5199790000915527,
+      "logps/chosen": -288.98883056640625,
+      "logps/rejected": -260.17230224609375,
+      "loss": 0.6458,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.006508971564471722,
+      "rewards/margins": 0.12008893489837646,
+      "rewards/rejected": -0.11357997357845306,
+      "step": 1260
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.933768060566654e-06,
+      "logits/chosen": -2.577394485473633,
+      "logits/rejected": -2.5192856788635254,
+      "logps/chosen": -280.8554992675781,
+      "logps/rejected": -268.8490905761719,
+      "loss": 0.6352,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.03389470651745796,
+      "rewards/margins": 0.13913270831108093,
+      "rewards/rejected": -0.10523799806833267,
+      "step": 1270
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.931131271234052e-06,
+      "logits/chosen": -2.534980535507202,
+      "logits/rejected": -2.440115451812744,
+      "logps/chosen": -279.3022155761719,
+      "logps/rejected": -248.8036651611328,
+      "loss": 0.6346,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.05754392594099045,
+      "rewards/margins": 0.14556458592414856,
+      "rewards/rejected": -0.2031085044145584,
+      "step": 1280
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.928443746704448e-06,
+      "logits/chosen": -2.5330698490142822,
+      "logits/rejected": -2.457183361053467,
+      "logps/chosen": -332.3924560546875,
+      "logps/rejected": -298.8351745605469,
+      "loss": 0.6147,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.10134587436914444,
+      "rewards/margins": 0.19787076115608215,
+      "rewards/rejected": -0.299216628074646,
+      "step": 1290
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.925705543063703e-06,
+      "logits/chosen": -2.470041513442993,
+      "logits/rejected": -2.4728071689605713,
+      "logps/chosen": -304.2667541503906,
+      "logps/rejected": -353.01348876953125,
+      "loss": 0.6143,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.10572239011526108,
+      "rewards/margins": 0.19867298007011414,
+      "rewards/rejected": -0.3043953776359558,
+      "step": 1300
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.922916717455297e-06,
+      "logits/chosen": -2.537431478500366,
+      "logits/rejected": -2.429407835006714,
+      "logps/chosen": -330.64190673828125,
+      "logps/rejected": -278.8482971191406,
+      "loss": 0.6321,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.17234036326408386,
+      "rewards/margins": 0.16186991333961487,
+      "rewards/rejected": -0.33421024680137634,
+      "step": 1310
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.920077328079136e-06,
+      "logits/chosen": -2.5637619495391846,
+      "logits/rejected": -2.483635425567627,
+      "logps/chosen": -323.4110412597656,
+      "logps/rejected": -288.7492370605469,
+      "loss": 0.6395,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.1858586072921753,
+      "rewards/margins": 0.14614257216453552,
+      "rewards/rejected": -0.3320012092590332,
+      "step": 1320
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.9171874341903445e-06,
+      "logits/chosen": -2.5309529304504395,
+      "logits/rejected": -2.465804100036621,
+      "logps/chosen": -304.7217102050781,
+      "logps/rejected": -324.9732360839844,
+      "loss": 0.6059,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.1965772658586502,
+      "rewards/margins": 0.21101978421211243,
+      "rewards/rejected": -0.40759706497192383,
+      "step": 1330
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.914247096098019e-06,
+      "logits/chosen": -2.5039734840393066,
+      "logits/rejected": -2.4910902976989746,
+      "logps/chosen": -294.86187744140625,
+      "logps/rejected": -305.4557189941406,
+      "loss": 0.6585,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.13758406043052673,
+      "rewards/margins": 0.10470825433731079,
+      "rewards/rejected": -0.24229232966899872,
+      "step": 1340
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.911256375163977e-06,
+      "logits/chosen": -2.415588855743408,
+      "logits/rejected": -2.4556922912597656,
+      "logps/chosen": -239.3975372314453,
+      "logps/rejected": -279.1402893066406,
+      "loss": 0.6237,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.030233344063162804,
+      "rewards/margins": 0.19135653972625732,
+      "rewards/rejected": -0.22158987820148468,
+      "step": 1350
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.908215333801474e-06,
+      "logits/chosen": -2.484656810760498,
+      "logits/rejected": -2.446765422821045,
+      "logps/chosen": -269.4073181152344,
+      "logps/rejected": -249.4739227294922,
+      "loss": 0.628,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.050433941185474396,
+      "rewards/margins": 0.16900701820850372,
+      "rewards/rejected": -0.21944093704223633,
+      "step": 1360
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.9051240354739004e-06,
+      "logits/chosen": -2.4662208557128906,
+      "logits/rejected": -2.3930366039276123,
+      "logps/chosen": -333.33831787109375,
+      "logps/rejected": -317.85369873046875,
+      "loss": 0.6583,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.16577494144439697,
+      "rewards/margins": 0.1024014800786972,
+      "rewards/rejected": -0.268176406621933,
+      "step": 1370
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.901982544693457e-06,
+      "logits/chosen": -2.519670248031616,
+      "logits/rejected": -2.4658124446868896,
+      "logps/chosen": -322.218017578125,
+      "logps/rejected": -315.02655029296875,
+      "loss": 0.5962,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.07793796062469482,
+      "rewards/margins": 0.24866263568401337,
+      "rewards/rejected": -0.3266006112098694,
+      "step": 1380
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.898790927019809e-06,
+      "logits/chosen": -2.404968500137329,
+      "logits/rejected": -2.3655800819396973,
+      "logps/chosen": -303.9695739746094,
+      "logps/rejected": -311.675048828125,
+      "loss": 0.643,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.19383570551872253,
+      "rewards/margins": 0.14913451671600342,
+      "rewards/rejected": -0.34297025203704834,
+      "step": 1390
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.895549249058718e-06,
+      "logits/chosen": -2.41062593460083,
+      "logits/rejected": -2.2885286808013916,
+      "logps/chosen": -290.38861083984375,
+      "logps/rejected": -284.9938659667969,
+      "loss": 0.6138,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.11517498642206192,
+      "rewards/margins": 0.2158602476119995,
+      "rewards/rejected": -0.33103522658348083,
+      "step": 1400
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.892257578460656e-06,
+      "logits/chosen": -2.48399019241333,
+      "logits/rejected": -2.440911054611206,
+      "logps/chosen": -288.1855163574219,
+      "logps/rejected": -292.6690979003906,
+      "loss": 0.6476,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.10677523910999298,
+      "rewards/margins": 0.14951439201831818,
+      "rewards/rejected": -0.25628960132598877,
+      "step": 1410
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.888915983919383e-06,
+      "logits/chosen": -2.519620656967163,
+      "logits/rejected": -2.4930050373077393,
+      "logps/chosen": -333.12139892578125,
+      "logps/rejected": -337.2662048339844,
+      "loss": 0.6077,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.0033730086870491505,
+      "rewards/margins": 0.21503393352031708,
+      "rewards/rejected": -0.21840694546699524,
+      "step": 1420
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.885524535170525e-06,
+      "logits/chosen": -2.4327306747436523,
+      "logits/rejected": -2.4226138591766357,
+      "logps/chosen": -300.2519836425781,
+      "logps/rejected": -272.81097412109375,
+      "loss": 0.6407,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.0004309698997531086,
+      "rewards/margins": 0.138811856508255,
+      "rewards/rejected": -0.13924282789230347,
+      "step": 1430
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.882083302990113e-06,
+      "logits/chosen": -2.5328216552734375,
+      "logits/rejected": -2.519892692565918,
+      "logps/chosen": -327.641845703125,
+      "logps/rejected": -308.94189453125,
+      "loss": 0.6132,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.06218453496694565,
+      "rewards/margins": 0.19737879931926727,
+      "rewards/rejected": -0.13519425690174103,
+      "step": 1440
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.878592359193104e-06,
+      "logits/chosen": -2.459606647491455,
+      "logits/rejected": -2.4827868938446045,
+      "logps/chosen": -277.9227294921875,
+      "logps/rejected": -268.23663330078125,
+      "loss": 0.6152,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.019242966547608376,
+      "rewards/margins": 0.19757792353630066,
+      "rewards/rejected": -0.2168208807706833,
+      "step": 1450
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.875051776631888e-06,
+      "logits/chosen": -2.4710533618927,
+      "logits/rejected": -2.410731554031372,
+      "logps/chosen": -368.6264343261719,
+      "logps/rejected": -335.9675598144531,
+      "loss": 0.6227,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.08454436808824539,
+      "rewards/margins": 0.19244490563869476,
+      "rewards/rejected": -0.27698928117752075,
+      "step": 1460
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.871461629194764e-06,
+      "logits/chosen": -2.4960079193115234,
+      "logits/rejected": -2.3759419918060303,
+      "logps/chosen": -280.95648193359375,
+      "logps/rejected": -256.5384216308594,
+      "loss": 0.6421,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.14747636020183563,
+      "rewards/margins": 0.1381666362285614,
+      "rewards/rejected": -0.28564298152923584,
+      "step": 1470
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.8678219918043984e-06,
+      "logits/chosen": -2.4740777015686035,
+      "logits/rejected": -2.385272979736328,
+      "logps/chosen": -293.96856689453125,
+      "logps/rejected": -260.2104797363281,
+      "loss": 0.634,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.10929224640130997,
+      "rewards/margins": 0.16633400321006775,
+      "rewards/rejected": -0.2756262421607971,
+      "step": 1480
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.864132940416262e-06,
+      "logits/chosen": -2.496289014816284,
+      "logits/rejected": -2.3923065662384033,
+      "logps/chosen": -326.35113525390625,
+      "logps/rejected": -289.10894775390625,
+      "loss": 0.5945,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.05443965643644333,
+      "rewards/margins": 0.24763624370098114,
+      "rewards/rejected": -0.30207592248916626,
+      "step": 1490
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.860394552017044e-06,
+      "logits/chosen": -2.4541478157043457,
+      "logits/rejected": -2.436887741088867,
+      "logps/chosen": -308.11859130859375,
+      "logps/rejected": -337.92633056640625,
+      "loss": 0.5989,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.14253541827201843,
+      "rewards/margins": 0.24313051998615265,
+      "rewards/rejected": -0.3856659531593323,
+      "step": 1500
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.856606904623047e-06,
+      "logits/chosen": -2.4310736656188965,
+      "logits/rejected": -2.4193150997161865,
+      "logps/chosen": -250.90640258789062,
+      "logps/rejected": -278.09735107421875,
+      "loss": 0.6456,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.1735648810863495,
+      "rewards/margins": 0.14540037512779236,
+      "rewards/rejected": -0.31896525621414185,
+      "step": 1510
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.852770077278557e-06,
+      "logits/chosen": -2.414815902709961,
+      "logits/rejected": -2.370915174484253,
+      "logps/chosen": -294.3868713378906,
+      "logps/rejected": -282.88751220703125,
+      "loss": 0.5838,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.11077413707971573,
+      "rewards/margins": 0.2743500769138336,
+      "rewards/rejected": -0.3851242661476135,
+      "step": 1520
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.848884150054196e-06,
+      "logits/chosen": -2.4590086936950684,
+      "logits/rejected": -2.4183783531188965,
+      "logps/chosen": -283.24932861328125,
+      "logps/rejected": -260.66949462890625,
+      "loss": 0.6576,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.1314813196659088,
+      "rewards/margins": 0.12110692262649536,
+      "rewards/rejected": -0.2525882422924042,
+      "step": 1530
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.8449492040452495e-06,
+      "logits/chosen": -2.4285058975219727,
+      "logits/rejected": -2.392300844192505,
+      "logps/chosen": -307.29901123046875,
+      "logps/rejected": -276.5957946777344,
+      "loss": 0.6042,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.09597836434841156,
+      "rewards/margins": 0.2392992526292801,
+      "rewards/rejected": -0.33527761697769165,
+      "step": 1540
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.840965321369973e-06,
+      "logits/chosen": -2.4373369216918945,
+      "logits/rejected": -2.3086016178131104,
+      "logps/chosen": -297.9757080078125,
+      "logps/rejected": -282.6000061035156,
+      "loss": 0.6129,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.10073225200176239,
+      "rewards/margins": 0.20788928866386414,
+      "rewards/rejected": -0.3086215853691101,
+      "step": 1550
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.8369325851678795e-06,
+      "logits/chosen": -2.4190502166748047,
+      "logits/rejected": -2.352062702178955,
+      "logps/chosen": -311.42169189453125,
+      "logps/rejected": -303.9734191894531,
+      "loss": 0.604,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.15880802273750305,
+      "rewards/margins": 0.22597630321979523,
+      "rewards/rejected": -0.3847842812538147,
+      "step": 1560
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.832851079598007e-06,
+      "logits/chosen": -2.4244778156280518,
+      "logits/rejected": -2.3742895126342773,
+      "logps/chosen": -269.66949462890625,
+      "logps/rejected": -303.1509704589844,
+      "loss": 0.6242,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.141389861702919,
+      "rewards/margins": 0.1891460120677948,
+      "rewards/rejected": -0.3305358290672302,
+      "step": 1570
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.828720889837158e-06,
+      "logits/chosen": -2.3640172481536865,
+      "logits/rejected": -2.316593647003174,
+      "logps/chosen": -301.21771240234375,
+      "logps/rejected": -269.7645568847656,
+      "loss": 0.6379,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.14845463633537292,
+      "rewards/margins": 0.16235341131687164,
+      "rewards/rejected": -0.3108080327510834,
+      "step": 1580
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.824542102078125e-06,
+      "logits/chosen": -2.3453826904296875,
+      "logits/rejected": -2.334388494491577,
+      "logps/chosen": -284.1679992675781,
+      "logps/rejected": -272.05841064453125,
+      "loss": 0.6336,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.1061963215470314,
+      "rewards/margins": 0.1646105945110321,
+      "rewards/rejected": -0.2708069086074829,
+      "step": 1590
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.820314803527888e-06,
+      "logits/chosen": -2.325477361679077,
+      "logits/rejected": -2.23325777053833,
+      "logps/chosen": -278.75384521484375,
+      "logps/rejected": -280.7860412597656,
+      "loss": 0.6183,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.20397250354290009,
+      "rewards/margins": 0.19710715115070343,
+      "rewards/rejected": -0.4010796546936035,
+      "step": 1600
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.816039082405799e-06,
+      "logits/chosen": -2.3431990146636963,
+      "logits/rejected": -2.311972141265869,
+      "logps/chosen": -300.4079895019531,
+      "logps/rejected": -313.13934326171875,
+      "loss": 0.6239,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.19499923288822174,
+      "rewards/margins": 0.19155170023441315,
+      "rewards/rejected": -0.3865508735179901,
+      "step": 1610
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.81171502794174e-06,
+      "logits/chosen": -2.419560194015503,
+      "logits/rejected": -2.391439914703369,
+      "logps/chosen": -334.3507995605469,
+      "logps/rejected": -320.0035095214844,
+      "loss": 0.6251,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.09380961209535599,
+      "rewards/margins": 0.1967724859714508,
+      "rewards/rejected": -0.2905820608139038,
+      "step": 1620
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.8073427303742584e-06,
+      "logits/chosen": -2.4514782428741455,
+      "logits/rejected": -2.3883414268493652,
+      "logps/chosen": -251.7395477294922,
+      "logps/rejected": -262.00921630859375,
+      "loss": 0.5929,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.031386177986860275,
+      "rewards/margins": 0.2520635724067688,
+      "rewards/rejected": -0.2834497392177582,
+      "step": 1630
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.802922280948685e-06,
+      "logits/chosen": -2.3628644943237305,
+      "logits/rejected": -2.358330011367798,
+      "logps/chosen": -255.8154296875,
+      "logps/rejected": -309.6222229003906,
+      "loss": 0.6305,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.105247363448143,
+      "rewards/margins": 0.1670643836259842,
+      "rewards/rejected": -0.2723117470741272,
+      "step": 1640
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.798453771915231e-06,
+      "logits/chosen": -2.4843735694885254,
+      "logits/rejected": -2.3918941020965576,
+      "logps/chosen": -300.1679992675781,
+      "logps/rejected": -294.6748962402344,
+      "loss": 0.6062,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.10460405051708221,
+      "rewards/margins": 0.22866663336753845,
+      "rewards/rejected": -0.33327072858810425,
+      "step": 1650
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.793937296527062e-06,
+      "logits/chosen": -2.4866888523101807,
+      "logits/rejected": -2.395613193511963,
+      "logps/chosen": -351.20257568359375,
+      "logps/rejected": -353.2774963378906,
+      "loss": 0.639,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.1539994478225708,
+      "rewards/margins": 0.17863675951957703,
+      "rewards/rejected": -0.33263617753982544,
+      "step": 1660
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.78937294903835e-06,
+      "logits/chosen": -2.437178134918213,
+      "logits/rejected": -2.33906888961792,
+      "logps/chosen": -280.81884765625,
+      "logps/rejected": -280.7358093261719,
+      "loss": 0.5956,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.07330270111560822,
+      "rewards/margins": 0.2512477934360504,
+      "rewards/rejected": -0.32455044984817505,
+      "step": 1670
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.78476082470231e-06,
+      "logits/chosen": -2.445133686065674,
+      "logits/rejected": -2.3581621646881104,
+      "logps/chosen": -282.26776123046875,
+      "logps/rejected": -274.84368896484375,
+      "loss": 0.6238,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.15945813059806824,
+      "rewards/margins": 0.19288024306297302,
+      "rewards/rejected": -0.35233837366104126,
+      "step": 1680
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.780101019769212e-06,
+      "logits/chosen": -2.411268949508667,
+      "logits/rejected": -2.3567655086517334,
+      "logps/chosen": -329.9833984375,
+      "logps/rejected": -304.2483825683594,
+      "loss": 0.6298,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.16943173110485077,
+      "rewards/margins": 0.19025549292564392,
+      "rewards/rejected": -0.3596872091293335,
+      "step": 1690
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.775393631484368e-06,
+      "logits/chosen": -2.3584446907043457,
+      "logits/rejected": -2.2917559146881104,
+      "logps/chosen": -315.8598937988281,
+      "logps/rejected": -333.73553466796875,
+      "loss": 0.5791,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.11365552991628647,
+      "rewards/margins": 0.29867830872535706,
+      "rewards/rejected": -0.4123338758945465,
+      "step": 1700
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.770638758086105e-06,
+      "logits/chosen": -2.3838050365448,
+      "logits/rejected": -2.3302836418151855,
+      "logps/chosen": -302.39459228515625,
+      "logps/rejected": -289.2746887207031,
+      "loss": 0.632,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.17028506100177765,
+      "rewards/margins": 0.17936842143535614,
+      "rewards/rejected": -0.3496534526348114,
+      "step": 1710
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.7658364988037184e-06,
+      "logits/chosen": -2.439105987548828,
+      "logits/rejected": -2.3976969718933105,
+      "logps/chosen": -337.58843994140625,
+      "logps/rejected": -325.3658142089844,
+      "loss": 0.6464,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.26347967982292175,
+      "rewards/margins": 0.14311228692531586,
+      "rewards/rejected": -0.4065919816493988,
+      "step": 1720
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.760986953855395e-06,
+      "logits/chosen": -2.4409725666046143,
+      "logits/rejected": -2.438054323196411,
+      "logps/chosen": -294.93121337890625,
+      "logps/rejected": -326.00799560546875,
+      "loss": 0.6005,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.2294178009033203,
+      "rewards/margins": 0.24227671325206757,
+      "rewards/rejected": -0.4716945290565491,
+      "step": 1730
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.756090224446127e-06,
+      "logits/chosen": -2.4490959644317627,
+      "logits/rejected": -2.367088794708252,
+      "logps/chosen": -332.90606689453125,
+      "logps/rejected": -326.7022705078125,
+      "loss": 0.6089,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.08716649562120438,
+      "rewards/margins": 0.22671692073345184,
+      "rewards/rejected": -0.3138834536075592,
+      "step": 1740
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.7511464127655945e-06,
+      "logits/chosen": -2.4698281288146973,
+      "logits/rejected": -2.319673538208008,
+      "logps/chosen": -297.69561767578125,
+      "logps/rejected": -289.238525390625,
+      "loss": 0.5916,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.1407977044582367,
+      "rewards/margins": 0.27011579275131226,
+      "rewards/rejected": -0.41091352701187134,
+      "step": 1750
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.74615562198604e-06,
+      "logits/chosen": -2.367678165435791,
+      "logits/rejected": -2.2320752143859863,
+      "logps/chosen": -279.7342529296875,
+      "logps/rejected": -272.38128662109375,
+      "loss": 0.5787,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.07772514969110489,
+      "rewards/margins": 0.30095797777175903,
+      "rewards/rejected": -0.3786831200122833,
+      "step": 1760
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.741117956260107e-06,
+      "logits/chosen": -2.333146333694458,
+      "logits/rejected": -2.2740590572357178,
+      "logps/chosen": -286.67266845703125,
+      "logps/rejected": -306.8995361328125,
+      "loss": 0.6015,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.05592029169201851,
+      "rewards/margins": 0.24995534121990204,
+      "rewards/rejected": -0.30587562918663025,
+      "step": 1770
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.736033520718672e-06,
+      "logits/chosen": -2.433627128601074,
+      "logits/rejected": -2.277806043624878,
+      "logps/chosen": -322.6488952636719,
+      "logps/rejected": -315.2213134765625,
+      "loss": 0.5985,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.0807700827717781,
+      "rewards/margins": 0.254264771938324,
+      "rewards/rejected": -0.33503487706184387,
+      "step": 1780
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.730902421468652e-06,
+      "logits/chosen": -2.4281609058380127,
+      "logits/rejected": -2.336761474609375,
+      "logps/chosen": -283.50677490234375,
+      "logps/rejected": -310.46502685546875,
+      "loss": 0.5985,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.17131593823432922,
+      "rewards/margins": 0.2610486149787903,
+      "rewards/rejected": -0.4323646128177643,
+      "step": 1790
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.7257247655907854e-06,
+      "logits/chosen": -2.3688364028930664,
+      "logits/rejected": -2.3433785438537598,
+      "logps/chosen": -262.29986572265625,
+      "logps/rejected": -294.90924072265625,
+      "loss": 0.6086,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.15793509781360626,
+      "rewards/margins": 0.23940256237983704,
+      "rewards/rejected": -0.3973376154899597,
+      "step": 1800
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.720500661137397e-06,
+      "logits/chosen": -2.392461061477661,
+      "logits/rejected": -2.333200454711914,
+      "logps/chosen": -284.260498046875,
+      "logps/rejected": -301.0771484375,
+      "loss": 0.6467,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.16365382075309753,
+      "rewards/margins": 0.15784037113189697,
+      "rewards/rejected": -0.3214941620826721,
+      "step": 1810
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.71523021713015e-06,
+      "logits/chosen": -2.4759411811828613,
+      "logits/rejected": -2.4524388313293457,
+      "logps/chosen": -392.81787109375,
+      "logps/rejected": -414.74249267578125,
+      "loss": 0.6457,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.17052128911018372,
+      "rewards/margins": 0.17021805047988892,
+      "rewards/rejected": -0.34073930978775024,
+      "step": 1820
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.709913543557761e-06,
+      "logits/chosen": -2.418494701385498,
+      "logits/rejected": -2.372173547744751,
+      "logps/chosen": -339.1895751953125,
+      "logps/rejected": -339.2401123046875,
+      "loss": 0.6153,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.1340826153755188,
+      "rewards/margins": 0.2253703773021698,
+      "rewards/rejected": -0.359453022480011,
+      "step": 1830
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.704550751373715e-06,
+      "logits/chosen": -2.375180244445801,
+      "logits/rejected": -2.283752679824829,
+      "logps/chosen": -379.5498046875,
+      "logps/rejected": -350.1661071777344,
+      "loss": 0.5828,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.08494368940591812,
+      "rewards/margins": 0.30550602078437805,
+      "rewards/rejected": -0.3904497027397156,
+      "step": 1840
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.699141952493941e-06,
+      "logits/chosen": -2.389798641204834,
+      "logits/rejected": -2.350865602493286,
+      "logps/chosen": -288.4355773925781,
+      "logps/rejected": -306.73883056640625,
+      "loss": 0.635,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.1179732233285904,
+      "rewards/margins": 0.1771775782108307,
+      "rewards/rejected": -0.2951507866382599,
+      "step": 1850
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.6936872597944814e-06,
+      "logits/chosen": -2.4748127460479736,
+      "logits/rejected": -2.3655686378479004,
+      "logps/chosen": -321.76165771484375,
+      "logps/rejected": -273.697509765625,
+      "loss": 0.5639,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.04060537368059158,
+      "rewards/margins": 0.3436310291290283,
+      "rewards/rejected": -0.3842363953590393,
+      "step": 1860
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.688186787109136e-06,
+      "logits/chosen": -2.412353515625,
+      "logits/rejected": -2.365421772003174,
+      "logps/chosen": -324.1878662109375,
+      "logps/rejected": -338.0693359375,
+      "loss": 0.5989,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.10340994596481323,
+      "rewards/margins": 0.2609453797340393,
+      "rewards/rejected": -0.3643553555011749,
+      "step": 1870
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.682640649227085e-06,
+      "logits/chosen": -2.4154398441314697,
+      "logits/rejected": -2.353245973587036,
+      "logps/chosen": -287.70440673828125,
+      "logps/rejected": -274.77886962890625,
+      "loss": 0.5911,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.026734206825494766,
+      "rewards/margins": 0.2650344967842102,
+      "rewards/rejected": -0.2917686700820923,
+      "step": 1880
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.677048961890492e-06,
+      "logits/chosen": -2.302427291870117,
+      "logits/rejected": -2.2373576164245605,
+      "logps/chosen": -300.24896240234375,
+      "logps/rejected": -291.5721130371094,
+      "loss": 0.6059,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.10195889323949814,
+      "rewards/margins": 0.23982873558998108,
+      "rewards/rejected": -0.3417876362800598,
+      "step": 1890
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.671411841792096e-06,
+      "logits/chosen": -2.3422927856445312,
+      "logits/rejected": -2.354828357696533,
+      "logps/chosen": -332.606201171875,
+      "logps/rejected": -353.5418395996094,
+      "loss": 0.6258,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.1744678020477295,
+      "rewards/margins": 0.19613692164421082,
+      "rewards/rejected": -0.3706046938896179,
+      "step": 1900
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.665729406572764e-06,
+      "logits/chosen": -2.350491523742676,
+      "logits/rejected": -2.3193857669830322,
+      "logps/chosen": -270.14398193359375,
+      "logps/rejected": -286.48931884765625,
+      "loss": 0.6241,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.2020377218723297,
+      "rewards/margins": 0.2181723564863205,
+      "rewards/rejected": -0.42021018266677856,
+      "step": 1910
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.660001774819048e-06,
+      "logits/chosen": -2.4870200157165527,
+      "logits/rejected": -2.4069786071777344,
+      "logps/chosen": -337.8989562988281,
+      "logps/rejected": -291.6131591796875,
+      "loss": 0.611,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.15691247582435608,
+      "rewards/margins": 0.23032662272453308,
+      "rewards/rejected": -0.38723909854888916,
+      "step": 1920
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.654229066060702e-06,
+      "logits/chosen": -2.3907480239868164,
+      "logits/rejected": -2.2987189292907715,
+      "logps/chosen": -294.33331298828125,
+      "logps/rejected": -267.0952453613281,
+      "loss": 0.6379,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.13681192696094513,
+      "rewards/margins": 0.15825247764587402,
+      "rewards/rejected": -0.29506438970565796,
+      "step": 1930
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.648411400768193e-06,
+      "logits/chosen": -2.471407890319824,
+      "logits/rejected": -2.3333897590637207,
+      "logps/chosen": -313.52508544921875,
+      "logps/rejected": -257.5264892578125,
+      "loss": 0.5945,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.13640491664409637,
+      "rewards/margins": 0.24762806296348572,
+      "rewards/rejected": -0.3840329647064209,
+      "step": 1940
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.642548900350182e-06,
+      "logits/chosen": -2.3417105674743652,
+      "logits/rejected": -2.3160438537597656,
+      "logps/chosen": -264.9716491699219,
+      "logps/rejected": -276.3714599609375,
+      "loss": 0.6235,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.16017001867294312,
+      "rewards/margins": 0.20394861698150635,
+      "rewards/rejected": -0.36411863565444946,
+      "step": 1950
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.636641687150994e-06,
+      "logits/chosen": -2.4112541675567627,
+      "logits/rejected": -2.403289794921875,
+      "logps/chosen": -330.2745056152344,
+      "logps/rejected": -314.89324951171875,
+      "loss": 0.6105,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.09830652922391891,
+      "rewards/margins": 0.237725168466568,
+      "rewards/rejected": -0.3360317051410675,
+      "step": 1960
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.6306898844480615e-06,
+      "logits/chosen": -2.3712351322174072,
+      "logits/rejected": -2.327747344970703,
+      "logps/chosen": -335.15362548828125,
+      "logps/rejected": -307.4205017089844,
+      "loss": 0.6308,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.07365657389163971,
+      "rewards/margins": 0.17441605031490326,
+      "rewards/rejected": -0.24807262420654297,
+      "step": 1970
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.624693616449358e-06,
+      "logits/chosen": -2.3876092433929443,
+      "logits/rejected": -2.3262836933135986,
+      "logps/chosen": -246.8582000732422,
+      "logps/rejected": -252.2131805419922,
+      "loss": 0.6357,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.0633595809340477,
+      "rewards/margins": 0.17477674782276154,
+      "rewards/rejected": -0.23813633620738983,
+      "step": 1980
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.6186530082908e-06,
+      "logits/chosen": -2.3876285552978516,
+      "logits/rejected": -2.316649913787842,
+      "logps/chosen": -334.5726318359375,
+      "logps/rejected": -336.7871398925781,
+      "loss": 0.6257,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.11085058748722076,
+      "rewards/margins": 0.2251889705657959,
+      "rewards/rejected": -0.33603957295417786,
+      "step": 1990
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.612568186033633e-06,
+      "logits/chosen": -2.4370970726013184,
+      "logits/rejected": -2.4089818000793457,
+      "logps/chosen": -277.29840087890625,
+      "logps/rejected": -299.14544677734375,
+      "loss": 0.6438,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.16993489861488342,
+      "rewards/margins": 0.1625920981168747,
+      "rewards/rejected": -0.3325270116329193,
+      "step": 2000
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.6064392766618125e-06,
+      "logits/chosen": -2.2922611236572266,
+      "logits/rejected": -2.2452197074890137,
+      "logps/chosen": -318.24725341796875,
+      "logps/rejected": -313.4471740722656,
+      "loss": 0.6207,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.13722370564937592,
+      "rewards/margins": 0.21375449001789093,
+      "rewards/rejected": -0.35097816586494446,
+      "step": 2010
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.60026640807934e-06,
+      "logits/chosen": -2.378413438796997,
+      "logits/rejected": -2.32692289352417,
+      "logps/chosen": -306.7994079589844,
+      "logps/rejected": -286.31939697265625,
+      "loss": 0.6027,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.07194646447896957,
+      "rewards/margins": 0.23152203857898712,
+      "rewards/rejected": -0.3034684956073761,
+      "step": 2020
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.594049709107604e-06,
+      "logits/chosen": -2.423443078994751,
+      "logits/rejected": -2.2880520820617676,
+      "logps/chosen": -334.21600341796875,
+      "logps/rejected": -334.7265625,
+      "loss": 0.6122,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.14375853538513184,
+      "rewards/margins": 0.24089816212654114,
+      "rewards/rejected": -0.38465672731399536,
+      "step": 2030
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.587789309482687e-06,
+      "logits/chosen": -2.3012590408325195,
+      "logits/rejected": -2.27238130569458,
+      "logps/chosen": -275.12115478515625,
+      "logps/rejected": -334.7784118652344,
+      "loss": 0.5961,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.1960037648677826,
+      "rewards/margins": 0.2675577998161316,
+      "rewards/rejected": -0.4635615944862366,
+      "step": 2040
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.581485339852659e-06,
+      "logits/chosen": -2.305781364440918,
+      "logits/rejected": -2.244325637817383,
+      "logps/chosen": -337.34027099609375,
+      "logps/rejected": -359.4820861816406,
+      "loss": 0.6366,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.18363210558891296,
+      "rewards/margins": 0.20024652779102325,
+      "rewards/rejected": -0.383878618478775,
+      "step": 2050
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.5751379317748514e-06,
+      "logits/chosen": -2.362804889678955,
+      "logits/rejected": -2.304513692855835,
+      "logps/chosen": -295.06793212890625,
+      "logps/rejected": -303.84881591796875,
+      "loss": 0.6507,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.22672036290168762,
+      "rewards/margins": 0.13963665068149567,
+      "rewards/rejected": -0.3663569986820221,
+      "step": 2060
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.56874721771311e-06,
+      "logits/chosen": -2.4059696197509766,
+      "logits/rejected": -2.3265323638916016,
+      "logps/chosen": -321.29754638671875,
+      "logps/rejected": -305.83929443359375,
+      "loss": 0.6215,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.25780874490737915,
+      "rewards/margins": 0.2031063586473465,
+      "rewards/rejected": -0.46091508865356445,
+      "step": 2070
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.562313331035032e-06,
+      "logits/chosen": -2.3071093559265137,
+      "logits/rejected": -2.2454941272735596,
+      "logps/chosen": -297.61773681640625,
+      "logps/rejected": -331.1474304199219,
+      "loss": 0.6275,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.35569310188293457,
+      "rewards/margins": 0.18869885802268982,
+      "rewards/rejected": -0.5443919897079468,
+      "step": 2080
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.555836406009183e-06,
+      "logits/chosen": -2.3374013900756836,
+      "logits/rejected": -2.3034305572509766,
+      "logps/chosen": -285.54107666015625,
+      "logps/rejected": -282.3943786621094,
+      "loss": 0.6324,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.3528543710708618,
+      "rewards/margins": 0.17680703103542328,
+      "rewards/rejected": -0.5296614170074463,
+      "step": 2090
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.5493165778022945e-06,
+      "logits/chosen": -2.3471615314483643,
+      "logits/rejected": -2.2503392696380615,
+      "logps/chosen": -281.76812744140625,
+      "logps/rejected": -268.25872802734375,
+      "loss": 0.6143,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.2267017811536789,
+      "rewards/margins": 0.2128499299287796,
+      "rewards/rejected": -0.4395516812801361,
+      "step": 2100
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.542753982476443e-06,
+      "logits/chosen": -2.360699415206909,
+      "logits/rejected": -2.2485642433166504,
+      "logps/chosen": -327.87591552734375,
+      "logps/rejected": -306.8153991699219,
+      "loss": 0.5964,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.08724816143512726,
+      "rewards/margins": 0.26559630036354065,
+      "rewards/rejected": -0.3528444468975067,
+      "step": 2110
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.53614875698621e-06,
+      "logits/chosen": -2.4491329193115234,
+      "logits/rejected": -2.346787214279175,
+      "logps/chosen": -337.62335205078125,
+      "logps/rejected": -348.6912536621094,
+      "loss": 0.6047,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.12890265882015228,
+      "rewards/margins": 0.24755427241325378,
+      "rewards/rejected": -0.3764569163322449,
+      "step": 2120
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.529501039175824e-06,
+      "logits/chosen": -2.3807685375213623,
+      "logits/rejected": -2.352710247039795,
+      "logps/chosen": -259.227294921875,
+      "logps/rejected": -269.9205322265625,
+      "loss": 0.6442,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.16931673884391785,
+      "rewards/margins": 0.17628124356269836,
+      "rewards/rejected": -0.3455979824066162,
+      "step": 2130
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.522810967776287e-06,
+      "logits/chosen": -2.3062472343444824,
+      "logits/rejected": -2.1800289154052734,
+      "logps/chosen": -315.9835510253906,
+      "logps/rejected": -264.95050048828125,
+      "loss": 0.5979,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.18467222154140472,
+      "rewards/margins": 0.2567806541919708,
+      "rewards/rejected": -0.44145289063453674,
+      "step": 2140
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.516078682402473e-06,
+      "logits/chosen": -2.3751935958862305,
+      "logits/rejected": -2.2700018882751465,
+      "logps/chosen": -330.53070068359375,
+      "logps/rejected": -322.2341003417969,
+      "loss": 0.595,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.24924437701702118,
+      "rewards/margins": 0.28462594747543335,
+      "rewards/rejected": -0.5338703393936157,
+      "step": 2150
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.509304323550221e-06,
+      "logits/chosen": -2.3613998889923096,
+      "logits/rejected": -2.3095054626464844,
+      "logps/chosen": -354.7084655761719,
+      "logps/rejected": -343.9185485839844,
+      "loss": 0.6226,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.20070073008537292,
+      "rewards/margins": 0.2098993957042694,
+      "rewards/rejected": -0.41060012578964233,
+      "step": 2160
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.502488032593398e-06,
+      "logits/chosen": -2.422234058380127,
+      "logits/rejected": -2.3208489418029785,
+      "logps/chosen": -316.1452941894531,
+      "logps/rejected": -304.63330078125,
+      "loss": 0.6185,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.18992137908935547,
+      "rewards/margins": 0.2037736475467682,
+      "rewards/rejected": -0.39369502663612366,
+      "step": 2170
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.495629951780951e-06,
+      "logits/chosen": -2.3926033973693848,
+      "logits/rejected": -2.2609989643096924,
+      "logps/chosen": -369.9411315917969,
+      "logps/rejected": -305.66607666015625,
+      "loss": 0.5797,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.07531694322824478,
+      "rewards/margins": 0.2898343801498413,
+      "rewards/rejected": -0.3651513159275055,
+      "step": 2180
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.488730224233941e-06,
+      "logits/chosen": -2.316122531890869,
+      "logits/rejected": -2.250302314758301,
+      "logps/chosen": -248.08547973632812,
+      "logps/rejected": -300.73223876953125,
+      "loss": 0.5419,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.08749908953905106,
+      "rewards/margins": 0.3842558264732361,
+      "rewards/rejected": -0.47175493836402893,
+      "step": 2190
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.481788993942547e-06,
+      "logits/chosen": -2.3757452964782715,
+      "logits/rejected": -2.2821907997131348,
+      "logps/chosen": -303.1983947753906,
+      "logps/rejected": -289.0472106933594,
+      "loss": 0.5995,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.12067332118749619,
+      "rewards/margins": 0.26477959752082825,
+      "rewards/rejected": -0.38545292615890503,
+      "step": 2200
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.474806405763076e-06,
+      "logits/chosen": -2.3512368202209473,
+      "logits/rejected": -2.3097450733184814,
+      "logps/chosen": -301.57452392578125,
+      "logps/rejected": -299.31671142578125,
+      "loss": 0.6031,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.11587338149547577,
+      "rewards/margins": 0.25784656405448914,
+      "rewards/rejected": -0.3737199306488037,
+      "step": 2210
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.4677826054149235e-06,
+      "logits/chosen": -2.338404417037964,
+      "logits/rejected": -2.253673553466797,
+      "logps/chosen": -340.2875671386719,
+      "logps/rejected": -330.8304748535156,
+      "loss": 0.6391,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.10927800834178925,
+      "rewards/margins": 0.17066845297813416,
+      "rewards/rejected": -0.2799464762210846,
+      "step": 2220
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.460717739477543e-06,
+      "logits/chosen": -2.355332612991333,
+      "logits/rejected": -2.3418946266174316,
+      "logps/chosen": -287.7314758300781,
+      "logps/rejected": -341.2978515625,
+      "loss": 0.6354,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.12638050317764282,
+      "rewards/margins": 0.18259502947330475,
+      "rewards/rejected": -0.30897554755210876,
+      "step": 2230
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.4536119553873866e-06,
+      "logits/chosen": -2.311370372772217,
+      "logits/rejected": -2.35019588470459,
+      "logps/chosen": -278.7158203125,
+      "logps/rejected": -300.1242370605469,
+      "loss": 0.5766,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.039584096521139145,
+      "rewards/margins": 0.3132474422454834,
+      "rewards/rejected": -0.35283154249191284,
+      "step": 2240
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.446465401434824e-06,
+      "logits/chosen": -2.3198676109313965,
+      "logits/rejected": -2.2937235832214355,
+      "logps/chosen": -269.0504455566406,
+      "logps/rejected": -327.19183349609375,
+      "loss": 0.6296,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.15721771121025085,
+      "rewards/margins": 0.2044394314289093,
+      "rewards/rejected": -0.36165714263916016,
+      "step": 2250
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.43927822676105e-06,
+      "logits/chosen": -2.3916709423065186,
+      "logits/rejected": -2.2888083457946777,
+      "logps/chosen": -322.9589538574219,
+      "logps/rejected": -301.18609619140625,
+      "loss": 0.6161,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.116670623421669,
+      "rewards/margins": 0.22785842418670654,
+      "rewards/rejected": -0.34452906250953674,
+      "step": 2260
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.432050581354972e-06,
+      "logits/chosen": -2.3398289680480957,
+      "logits/rejected": -2.2326717376708984,
+      "logps/chosen": -295.10894775390625,
+      "logps/rejected": -317.585693359375,
+      "loss": 0.6182,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.139591783285141,
+      "rewards/margins": 0.2277776300907135,
+      "rewards/rejected": -0.36736947298049927,
+      "step": 2270
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.424782616050078e-06,
+      "logits/chosen": -2.2997286319732666,
+      "logits/rejected": -2.237948417663574,
+      "logps/chosen": -307.8941650390625,
+      "logps/rejected": -298.88006591796875,
+      "loss": 0.614,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.11191853135824203,
+      "rewards/margins": 0.23675408959388733,
+      "rewards/rejected": -0.34867262840270996,
+      "step": 2280
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.4174744825212954e-06,
+      "logits/chosen": -2.381026268005371,
+      "logits/rejected": -2.3788371086120605,
+      "logps/chosen": -274.66656494140625,
+      "logps/rejected": -307.1249084472656,
+      "loss": 0.6717,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.15815982222557068,
+      "rewards/margins": 0.12555231153964996,
+      "rewards/rejected": -0.28371211886405945,
+      "step": 2290
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.410126333281815e-06,
+      "logits/chosen": -2.3236286640167236,
+      "logits/rejected": -2.2463207244873047,
+      "logps/chosen": -351.5771484375,
+      "logps/rejected": -385.1914978027344,
+      "loss": 0.6002,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.16838675737380981,
+      "rewards/margins": 0.2537475526332855,
+      "rewards/rejected": -0.42213425040245056,
+      "step": 2300
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.402738321679918e-06,
+      "logits/chosen": -2.3222060203552246,
+      "logits/rejected": -2.245678186416626,
+      "logps/chosen": -319.8814392089844,
+      "logps/rejected": -279.41656494140625,
+      "loss": 0.5771,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.15020090341567993,
+      "rewards/margins": 0.31123560667037964,
+      "rewards/rejected": -0.4614364504814148,
+      "step": 2310
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.395310601895772e-06,
+      "logits/chosen": -2.3784894943237305,
+      "logits/rejected": -2.3047304153442383,
+      "logps/chosen": -309.1102294921875,
+      "logps/rejected": -321.6300354003906,
+      "loss": 0.5812,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.11590411514043808,
+      "rewards/margins": 0.30960220098495483,
+      "rewards/rejected": -0.42550626397132874,
+      "step": 2320
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.38784332893821e-06,
+      "logits/chosen": -2.3468894958496094,
+      "logits/rejected": -2.311635971069336,
+      "logps/chosen": -226.0956268310547,
+      "logps/rejected": -260.84552001953125,
+      "loss": 0.5672,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.12173782289028168,
+      "rewards/margins": 0.33285272121429443,
+      "rewards/rejected": -0.4545905590057373,
+      "step": 2330
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.380336658641503e-06,
+      "logits/chosen": -2.4134764671325684,
+      "logits/rejected": -2.252642869949341,
+      "logps/chosen": -314.12115478515625,
+      "logps/rejected": -291.8180236816406,
+      "loss": 0.6069,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.1728154420852661,
+      "rewards/margins": 0.2426457703113556,
+      "rewards/rejected": -0.4154612123966217,
+      "step": 2340
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.372790747662101e-06,
+      "logits/chosen": -2.280484199523926,
+      "logits/rejected": -2.2489004135131836,
+      "logps/chosen": -306.0282287597656,
+      "logps/rejected": -303.01141357421875,
+      "loss": 0.6283,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.2237296998500824,
+      "rewards/margins": 0.19994047284126282,
+      "rewards/rejected": -0.4236702024936676,
+      "step": 2350
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.365205753475367e-06,
+      "logits/chosen": -2.366082191467285,
+      "logits/rejected": -2.2484962940216064,
+      "logps/chosen": -368.59075927734375,
+      "logps/rejected": -318.4290466308594,
+      "loss": 0.5859,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.11698893457651138,
+      "rewards/margins": 0.29912567138671875,
+      "rewards/rejected": -0.41611456871032715,
+      "step": 2360
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.35758183437229e-06,
+      "logits/chosen": -2.410271167755127,
+      "logits/rejected": -2.3593297004699707,
+      "logps/chosen": -354.11431884765625,
+      "logps/rejected": -354.12945556640625,
+      "loss": 0.6349,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.022561874240636826,
+      "rewards/margins": 0.17640873789787292,
+      "rewards/rejected": -0.19897060096263885,
+      "step": 2370
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.3499191494561835e-06,
+      "logits/chosen": -2.2106945514678955,
+      "logits/rejected": -2.1773438453674316,
+      "logps/chosen": -296.61639404296875,
+      "logps/rejected": -269.124755859375,
+      "loss": 0.6266,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.06345901638269424,
+      "rewards/margins": 0.19572456181049347,
+      "rewards/rejected": -0.2591835856437683,
+      "step": 2380
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.3422178586393615e-06,
+      "logits/chosen": -2.4216809272766113,
+      "logits/rejected": -2.2997002601623535,
+      "logps/chosen": -341.6378173828125,
+      "logps/rejected": -307.9349670410156,
+      "loss": 0.5915,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.06652353703975677,
+      "rewards/margins": 0.2856021821498871,
+      "rewards/rejected": -0.35212570428848267,
+      "step": 2390
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.334478122639804e-06,
+      "logits/chosen": -2.429971933364868,
+      "logits/rejected": -2.2178616523742676,
+      "logps/chosen": -286.44097900390625,
+      "logps/rejected": -248.7102813720703,
+      "loss": 0.5218,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.10870865732431412,
+      "rewards/margins": 0.4377427101135254,
+      "rewards/rejected": -0.5464513897895813,
+      "step": 2400
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.3267001029778015e-06,
+      "logits/chosen": -2.367393732070923,
+      "logits/rejected": -2.246732711791992,
+      "logps/chosen": -330.3119201660156,
+      "logps/rejected": -277.81982421875,
+      "loss": 0.6332,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.17828808724880219,
+      "rewards/margins": 0.18477049469947815,
+      "rewards/rejected": -0.36305850744247437,
+      "step": 2410
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.318883961972585e-06,
+      "logits/chosen": -2.2722887992858887,
+      "logits/rejected": -2.2594714164733887,
+      "logps/chosen": -294.3458251953125,
+      "logps/rejected": -307.031982421875,
+      "loss": 0.5982,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.17498214542865753,
+      "rewards/margins": 0.2718316614627838,
+      "rewards/rejected": -0.44681382179260254,
+      "step": 2420
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.311029862738942e-06,
+      "logits/chosen": -2.3691306114196777,
+      "logits/rejected": -2.286330223083496,
+      "logps/chosen": -321.17352294921875,
+      "logps/rejected": -314.225830078125,
+      "loss": 0.6092,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.2745895981788635,
+      "rewards/margins": 0.22876712679862976,
+      "rewards/rejected": -0.5033567547798157,
+      "step": 2430
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.303137969183804e-06,
+      "logits/chosen": -2.261221408843994,
+      "logits/rejected": -2.218248128890991,
+      "logps/chosen": -304.7114562988281,
+      "logps/rejected": -303.0655212402344,
+      "loss": 0.5703,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.22714805603027344,
+      "rewards/margins": 0.32687151432037354,
+      "rewards/rejected": -0.554019570350647,
+      "step": 2440
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.295208446002832e-06,
+      "logits/chosen": -2.2834646701812744,
+      "logits/rejected": -2.1622109413146973,
+      "logps/chosen": -264.4088439941406,
+      "logps/rejected": -263.5638427734375,
+      "loss": 0.6082,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.2190243899822235,
+      "rewards/margins": 0.25770148634910583,
+      "rewards/rejected": -0.47672590613365173,
+      "step": 2450
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.287241458676981e-06,
+      "logits/chosen": -2.2780747413635254,
+      "logits/rejected": -2.2258801460266113,
+      "logps/chosen": -355.08612060546875,
+      "logps/rejected": -348.1125793457031,
+      "loss": 0.5692,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.19551631808280945,
+      "rewards/margins": 0.35570698976516724,
+      "rewards/rejected": -0.5512233376502991,
+      "step": 2460
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.279237173469043e-06,
+      "logits/chosen": -2.3363661766052246,
+      "logits/rejected": -2.1859169006347656,
+      "logps/chosen": -296.65423583984375,
+      "logps/rejected": -286.93377685546875,
+      "loss": 0.5738,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.12626975774765015,
+      "rewards/margins": 0.3321135342121124,
+      "rewards/rejected": -0.4583832621574402,
+      "step": 2470
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.271195757420177e-06,
+      "logits/chosen": -2.2018187046051025,
+      "logits/rejected": -2.1672024726867676,
+      "logps/chosen": -309.4765319824219,
+      "logps/rejected": -344.6891174316406,
+      "loss": 0.6229,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.24267332255840302,
+      "rewards/margins": 0.23746450245380402,
+      "rewards/rejected": -0.48013782501220703,
+      "step": 2480
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.263117378346425e-06,
+      "logits/chosen": -2.28141450881958,
+      "logits/rejected": -2.1525015830993652,
+      "logps/chosen": -332.60894775390625,
+      "logps/rejected": -308.6076354980469,
+      "loss": 0.5955,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.1525574028491974,
+      "rewards/margins": 0.30104440450668335,
+      "rewards/rejected": -0.4536018371582031,
+      "step": 2490
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.255002204835208e-06,
+      "logits/chosen": -2.2987420558929443,
+      "logits/rejected": -2.2409260272979736,
+      "logps/chosen": -328.59521484375,
+      "logps/rejected": -330.57098388671875,
+      "loss": 0.578,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.12822741270065308,
+      "rewards/margins": 0.3138895630836487,
+      "rewards/rejected": -0.44211697578430176,
+      "step": 2500
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.246850406241812e-06,
+      "logits/chosen": -2.283529758453369,
+      "logits/rejected": -2.196633815765381,
+      "logps/chosen": -248.55825805664062,
+      "logps/rejected": -302.5537109375,
+      "loss": 0.5715,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.17185014486312866,
+      "rewards/margins": 0.3294577896595001,
+      "rewards/rejected": -0.5013079047203064,
+      "step": 2510
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.2386621526858465e-06,
+      "logits/chosen": -2.2872681617736816,
+      "logits/rejected": -2.2363412380218506,
+      "logps/chosen": -260.451171875,
+      "logps/rejected": -248.4807891845703,
+      "loss": 0.5905,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.1857258677482605,
+      "rewards/margins": 0.2745230197906494,
+      "rewards/rejected": -0.4602488577365875,
+      "step": 2520
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.2304376150477015e-06,
+      "logits/chosen": -2.3132100105285645,
+      "logits/rejected": -2.2969963550567627,
+      "logps/chosen": -313.3431701660156,
+      "logps/rejected": -330.83404541015625,
+      "loss": 0.5747,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.1469540297985077,
+      "rewards/margins": 0.33261358737945557,
+      "rewards/rejected": -0.47956761717796326,
+      "step": 2530
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.222176964964977e-06,
+      "logits/chosen": -2.196845531463623,
+      "logits/rejected": -2.145677089691162,
+      "logps/chosen": -312.68353271484375,
+      "logps/rejected": -317.6732482910156,
+      "loss": 0.6325,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.2049153745174408,
+      "rewards/margins": 0.18959246575832367,
+      "rewards/rejected": -0.39450788497924805,
+      "step": 2540
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.213880374828903e-06,
+      "logits/chosen": -2.3408188819885254,
+      "logits/rejected": -2.2571680545806885,
+      "logps/chosen": -268.51995849609375,
+      "logps/rejected": -301.3713073730469,
+      "loss": 0.5891,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.17139479517936707,
+      "rewards/margins": 0.2834908962249756,
+      "rewards/rejected": -0.45488566160202026,
+      "step": 2550
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.2055480177807406e-06,
+      "logits/chosen": -2.170337200164795,
+      "logits/rejected": -2.16973876953125,
+      "logps/chosen": -252.72805786132812,
+      "logps/rejected": -257.45928955078125,
+      "loss": 0.6601,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.18463487923145294,
+      "rewards/margins": 0.1279645711183548,
+      "rewards/rejected": -0.3125994801521301,
+      "step": 2560
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.1971800677081696e-06,
+      "logits/chosen": -2.2201130390167236,
+      "logits/rejected": -2.2000250816345215,
+      "logps/chosen": -306.4814147949219,
+      "logps/rejected": -351.8539123535156,
+      "loss": 0.6612,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.1903466135263443,
+      "rewards/margins": 0.14342021942138672,
+      "rewards/rejected": -0.3337668478488922,
+      "step": 2570
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.188776699241661e-06,
+      "logits/chosen": -2.365649700164795,
+      "logits/rejected": -2.2430615425109863,
+      "logps/chosen": -313.85162353515625,
+      "logps/rejected": -308.8529357910156,
+      "loss": 0.6126,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.11727418005466461,
+      "rewards/margins": 0.23288528621196747,
+      "rewards/rejected": -0.3501594662666321,
+      "step": 2580
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.180338087750827e-06,
+      "logits/chosen": -2.283839702606201,
+      "logits/rejected": -2.2854695320129395,
+      "logps/chosen": -288.41754150390625,
+      "logps/rejected": -333.4906311035156,
+      "loss": 0.5952,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.1470477283000946,
+      "rewards/margins": 0.2957322895526886,
+      "rewards/rejected": -0.4427799582481384,
+      "step": 2590
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.1718644093407704e-06,
+      "logits/chosen": -2.215942859649658,
+      "logits/rejected": -2.168015241622925,
+      "logps/chosen": -300.6781311035156,
+      "logps/rejected": -312.72198486328125,
+      "loss": 0.5495,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.11697547137737274,
+      "rewards/margins": 0.38940751552581787,
+      "rewards/rejected": -0.5063830614089966,
+      "step": 2600
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.163355840848401e-06,
+      "logits/chosen": -2.3301100730895996,
+      "logits/rejected": -2.21736478805542,
+      "logps/chosen": -333.5172119140625,
+      "logps/rejected": -344.1437072753906,
+      "loss": 0.6089,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.2591416835784912,
+      "rewards/margins": 0.27335575222969055,
+      "rewards/rejected": -0.5324974656105042,
+      "step": 2610
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.154812559838748e-06,
+      "logits/chosen": -2.299487590789795,
+      "logits/rejected": -2.2091782093048096,
+      "logps/chosen": -278.13702392578125,
+      "logps/rejected": -286.05548095703125,
+      "loss": 0.5407,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.0895254835486412,
+      "rewards/margins": 0.4064970016479492,
+      "rewards/rejected": -0.49602246284484863,
+      "step": 2620
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.146234744601259e-06,
+      "logits/chosen": -2.2448086738586426,
+      "logits/rejected": -2.1447150707244873,
+      "logps/chosen": -272.74371337890625,
+      "logps/rejected": -309.2991638183594,
+      "loss": 0.5772,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.15012232959270477,
+      "rewards/margins": 0.323434978723526,
+      "rewards/rejected": -0.47355732321739197,
+      "step": 2630
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.137622574146071e-06,
+      "logits/chosen": -2.321953296661377,
+      "logits/rejected": -2.2377140522003174,
+      "logps/chosen": -271.127197265625,
+      "logps/rejected": -257.08197021484375,
+      "loss": 0.5904,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.04710864648222923,
+      "rewards/margins": 0.2709490954875946,
+      "rewards/rejected": -0.3180577754974365,
+      "step": 2640
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.12897622820028e-06,
+      "logits/chosen": -2.323284387588501,
+      "logits/rejected": -2.216844081878662,
+      "logps/chosen": -327.4410400390625,
+      "logps/rejected": -322.1961975097656,
+      "loss": 0.5974,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.12447996437549591,
+      "rewards/margins": 0.26364922523498535,
+      "rewards/rejected": -0.38812920451164246,
+      "step": 2650
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.120295887204191e-06,
+      "logits/chosen": -2.2313876152038574,
+      "logits/rejected": -2.1354150772094727,
+      "logps/chosen": -286.85870361328125,
+      "logps/rejected": -324.2660827636719,
+      "loss": 0.6137,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.21455173194408417,
+      "rewards/margins": 0.26458728313446045,
+      "rewards/rejected": -0.4791390299797058,
+      "step": 2660
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.111581732307548e-06,
+      "logits/chosen": -2.239713668823242,
+      "logits/rejected": -2.2120895385742188,
+      "logps/chosen": -273.3136901855469,
+      "logps/rejected": -301.74273681640625,
+      "loss": 0.6098,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.2157611846923828,
+      "rewards/margins": 0.26702648401260376,
+      "rewards/rejected": -0.4827876687049866,
+      "step": 2670
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.1028339453657595e-06,
+      "logits/chosen": -2.2836809158325195,
+      "logits/rejected": -2.0904288291931152,
+      "logps/chosen": -344.65362548828125,
+      "logps/rejected": -314.09149169921875,
+      "loss": 0.6117,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.12366046011447906,
+      "rewards/margins": 0.24424687027931213,
+      "rewards/rejected": -0.36790731549263,
+      "step": 2680
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.094052708936096e-06,
+      "logits/chosen": -2.309826612472534,
+      "logits/rejected": -2.2577881813049316,
+      "logps/chosen": -316.89727783203125,
+      "logps/rejected": -366.1764221191406,
+      "loss": 0.5631,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.08284427970647812,
+      "rewards/margins": 0.36028558015823364,
+      "rewards/rejected": -0.44312983751296997,
+      "step": 2690
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.0852382062738874e-06,
+      "logits/chosen": -2.305793285369873,
+      "logits/rejected": -2.2325210571289062,
+      "logps/chosen": -323.45245361328125,
+      "logps/rejected": -338.0271301269531,
+      "loss": 0.6095,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.22808530926704407,
+      "rewards/margins": 0.24091526865959167,
+      "rewards/rejected": -0.46900051832199097,
+      "step": 2700
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.076390621328693e-06,
+      "logits/chosen": -2.338980197906494,
+      "logits/rejected": -2.2027366161346436,
+      "logps/chosen": -348.8518981933594,
+      "logps/rejected": -330.1253967285156,
+      "loss": 0.5792,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.20264069736003876,
+      "rewards/margins": 0.3276185095310211,
+      "rewards/rejected": -0.5302591323852539,
+      "step": 2710
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.067510138740467e-06,
+      "logits/chosen": -2.2085351943969727,
+      "logits/rejected": -2.179727077484131,
+      "logps/chosen": -337.2236022949219,
+      "logps/rejected": -348.954833984375,
+      "loss": 0.5933,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.2019720822572708,
+      "rewards/margins": 0.29704728722572327,
+      "rewards/rejected": -0.4990193843841553,
+      "step": 2720
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.058596943835703e-06,
+      "logits/chosen": -2.2849490642547607,
+      "logits/rejected": -2.255769729614258,
+      "logps/chosen": -313.56585693359375,
+      "logps/rejected": -291.57489013671875,
+      "loss": 0.627,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.16551968455314636,
+      "rewards/margins": 0.2221347838640213,
+      "rewards/rejected": -0.38765448331832886,
+      "step": 2730
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.049651222623568e-06,
+      "logits/chosen": -2.3381128311157227,
+      "logits/rejected": -2.279048204421997,
+      "logps/chosen": -303.53460693359375,
+      "logps/rejected": -284.5528869628906,
+      "loss": 0.6096,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.17010393738746643,
+      "rewards/margins": 0.2440837174654007,
+      "rewards/rejected": -0.4141876697540283,
+      "step": 2740
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.040673161792014e-06,
+      "logits/chosen": -2.297952651977539,
+      "logits/rejected": -2.2086384296417236,
+      "logps/chosen": -343.0641174316406,
+      "logps/rejected": -349.12786865234375,
+      "loss": 0.608,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.19907456636428833,
+      "rewards/margins": 0.2641879916191101,
+      "rewards/rejected": -0.46326255798339844,
+      "step": 2750
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.031662948703896e-06,
+      "logits/chosen": -2.2400102615356445,
+      "logits/rejected": -2.1351914405822754,
+      "logps/chosen": -308.60504150390625,
+      "logps/rejected": -325.3629455566406,
+      "loss": 0.5625,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.1372842639684677,
+      "rewards/margins": 0.3827325701713562,
+      "rewards/rejected": -0.5200168490409851,
+      "step": 2760
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.022620771393047e-06,
+      "logits/chosen": -2.3050320148468018,
+      "logits/rejected": -2.191411256790161,
+      "logps/chosen": -324.8857116699219,
+      "logps/rejected": -308.9805908203125,
+      "loss": 0.61,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.254262775182724,
+      "rewards/margins": 0.25628453493118286,
+      "rewards/rejected": -0.5105473399162292,
+      "step": 2770
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.013546818560362e-06,
+      "logits/chosen": -2.2292933464050293,
+      "logits/rejected": -2.0870003700256348,
+      "logps/chosen": -292.7982177734375,
+      "logps/rejected": -288.9156188964844,
+      "loss": 0.6229,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.24121983349323273,
+      "rewards/margins": 0.21186479926109314,
+      "rewards/rejected": -0.45308464765548706,
+      "step": 2780
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 4.00444127956986e-06,
+      "logits/chosen": -2.3071656227111816,
+      "logits/rejected": -2.1425764560699463,
+      "logps/chosen": -282.1644592285156,
+      "logps/rejected": -288.5877990722656,
+      "loss": 0.5743,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.1937316209077835,
+      "rewards/margins": 0.3192608952522278,
+      "rewards/rejected": -0.5129925012588501,
+      "step": 2790
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.9953043444447255e-06,
+      "logits/chosen": -2.3431785106658936,
+      "logits/rejected": -2.2949059009552,
+      "logps/chosen": -337.19873046875,
+      "logps/rejected": -321.8359375,
+      "loss": 0.6295,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.14888326823711395,
+      "rewards/margins": 0.20381757616996765,
+      "rewards/rejected": -0.3527008593082428,
+      "step": 2800
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.986136203863355e-06,
+      "logits/chosen": -2.269035816192627,
+      "logits/rejected": -2.258761167526245,
+      "logps/chosen": -334.3866271972656,
+      "logps/rejected": -335.5721435546875,
+      "loss": 0.5786,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.1394425928592682,
+      "rewards/margins": 0.3137834966182709,
+      "rewards/rejected": -0.45322608947753906,
+      "step": 2810
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.976937049155365e-06,
+      "logits/chosen": -2.244459629058838,
+      "logits/rejected": -2.1737873554229736,
+      "logps/chosen": -295.0531005859375,
+      "logps/rejected": -328.18231201171875,
+      "loss": 0.5907,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.1519058644771576,
+      "rewards/margins": 0.28696325421333313,
+      "rewards/rejected": -0.43886908888816833,
+      "step": 2820
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.967707072297608e-06,
+      "logits/chosen": -2.15556001663208,
+      "logits/rejected": -2.0936989784240723,
+      "logps/chosen": -322.49542236328125,
+      "logps/rejected": -345.54522705078125,
+      "loss": 0.5865,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.18620704114437103,
+      "rewards/margins": 0.29388242959976196,
+      "rewards/rejected": -0.4800894856452942,
+      "step": 2830
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.958446465910159e-06,
+      "logits/chosen": -2.356562852859497,
+      "logits/rejected": -2.3032262325286865,
+      "logps/chosen": -305.7391662597656,
+      "logps/rejected": -330.58135986328125,
+      "loss": 0.5959,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.2515236735343933,
+      "rewards/margins": 0.27204760909080505,
+      "rewards/rejected": -0.5235711932182312,
+      "step": 2840
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.9491554232523066e-06,
+      "logits/chosen": -2.154636859893799,
+      "logits/rejected": -2.174771785736084,
+      "logps/chosen": -259.4274597167969,
+      "logps/rejected": -301.49591064453125,
+      "loss": 0.6512,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.2738287150859833,
+      "rewards/margins": 0.16962867975234985,
+      "rewards/rejected": -0.4434574246406555,
+      "step": 2850
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.939834138218505e-06,
+      "logits/chosen": -2.259753704071045,
+      "logits/rejected": -2.2131614685058594,
+      "logps/chosen": -316.70062255859375,
+      "logps/rejected": -321.30499267578125,
+      "loss": 0.5859,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.22981849312782288,
+      "rewards/margins": 0.32320156693458557,
+      "rewards/rejected": -0.5530200004577637,
+      "step": 2860
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.930482805334339e-06,
+      "logits/chosen": -2.2335028648376465,
+      "logits/rejected": -2.1571171283721924,
+      "logps/chosen": -287.03802490234375,
+      "logps/rejected": -297.360107421875,
+      "loss": 0.6134,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.24314312636852264,
+      "rewards/margins": 0.2636451721191406,
+      "rewards/rejected": -0.5067883133888245,
+      "step": 2870
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.921101619752464e-06,
+      "logits/chosen": -2.2365384101867676,
+      "logits/rejected": -2.118043899536133,
+      "logps/chosen": -315.28839111328125,
+      "logps/rejected": -292.41571044921875,
+      "loss": 0.6048,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.2544742524623871,
+      "rewards/margins": 0.25104787945747375,
+      "rewards/rejected": -0.5055221319198608,
+      "step": 2880
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.911690777248525e-06,
+      "logits/chosen": -2.279517650604248,
+      "logits/rejected": -2.1619081497192383,
+      "logps/chosen": -331.37713623046875,
+      "logps/rejected": -343.8858337402344,
+      "loss": 0.6157,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.28338202834129333,
+      "rewards/margins": 0.22550642490386963,
+      "rewards/rejected": -0.5088884234428406,
+      "step": 2890
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.902250474217079e-06,
+      "logits/chosen": -2.249533176422119,
+      "logits/rejected": -2.1728508472442627,
+      "logps/chosen": -290.2388000488281,
+      "logps/rejected": -288.2944030761719,
+      "loss": 0.5999,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.212762713432312,
+      "rewards/margins": 0.2684236168861389,
+      "rewards/rejected": -0.4811863303184509,
+      "step": 2900
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.892780907667495e-06,
+      "logits/chosen": -2.254171848297119,
+      "logits/rejected": -2.1818957328796387,
+      "logps/chosen": -290.6761169433594,
+      "logps/rejected": -308.3992614746094,
+      "loss": 0.5687,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.10069400072097778,
+      "rewards/margins": 0.34916025400161743,
+      "rewards/rejected": -0.4498542845249176,
+      "step": 2910
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.883282275219837e-06,
+      "logits/chosen": -2.3390860557556152,
+      "logits/rejected": -2.244075059890747,
+      "logps/chosen": -319.77874755859375,
+      "logps/rejected": -308.5218200683594,
+      "loss": 0.5906,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.23237352073192596,
+      "rewards/margins": 0.3077944219112396,
+      "rewards/rejected": -0.5401679873466492,
+      "step": 2920
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.873754775100751e-06,
+      "logits/chosen": -2.332498073577881,
+      "logits/rejected": -2.244035482406616,
+      "logps/chosen": -316.33770751953125,
+      "logps/rejected": -314.38739013671875,
+      "loss": 0.5997,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.1779693067073822,
+      "rewards/margins": 0.3119404911994934,
+      "rewards/rejected": -0.48990970849990845,
+      "step": 2930
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.8641986061393145e-06,
+      "logits/chosen": -2.3143229484558105,
+      "logits/rejected": -2.261376142501831,
+      "logps/chosen": -287.69525146484375,
+      "logps/rejected": -336.7581787109375,
+      "loss": 0.5772,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.08865178376436234,
+      "rewards/margins": 0.3206256926059723,
+      "rewards/rejected": -0.40927743911743164,
+      "step": 2940
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.854613967762898e-06,
+      "logits/chosen": -2.2918875217437744,
+      "logits/rejected": -2.1621129512786865,
+      "logps/chosen": -347.6598815917969,
+      "logps/rejected": -332.0389709472656,
+      "loss": 0.5778,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.14384640753269196,
+      "rewards/margins": 0.33262744545936584,
+      "rewards/rejected": -0.476473867893219,
+      "step": 2950
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.845001059992999e-06,
+      "logits/chosen": -2.2963719367980957,
+      "logits/rejected": -2.2243804931640625,
+      "logps/chosen": -321.4737854003906,
+      "logps/rejected": -306.0363464355469,
+      "loss": 0.5931,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.2161785364151001,
+      "rewards/margins": 0.285264253616333,
+      "rewards/rejected": -0.5014427900314331,
+      "step": 2960
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.835360083441067e-06,
+      "logits/chosen": -2.1902008056640625,
+      "logits/rejected": -2.1949942111968994,
+      "logps/chosen": -284.7296447753906,
+      "logps/rejected": -296.80828857421875,
+      "loss": 0.559,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.12577447295188904,
+      "rewards/margins": 0.37559282779693604,
+      "rewards/rejected": -0.5013672113418579,
+      "step": 2970
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.825691239304318e-06,
+      "logits/chosen": -2.2690746784210205,
+      "logits/rejected": -2.194706439971924,
+      "logps/chosen": -297.1219177246094,
+      "logps/rejected": -316.86395263671875,
+      "loss": 0.6104,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.19138354063034058,
+      "rewards/margins": 0.24811045825481415,
+      "rewards/rejected": -0.43949398398399353,
+      "step": 2980
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.8159947293615385e-06,
+      "logits/chosen": -2.3479580879211426,
+      "logits/rejected": -2.2541697025299072,
+      "logps/chosen": -332.5345764160156,
+      "logps/rejected": -315.8721618652344,
+      "loss": 0.5872,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.10395509004592896,
+      "rewards/margins": 0.29379114508628845,
+      "rewards/rejected": -0.3977462351322174,
+      "step": 2990
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.806270755968866e-06,
+      "logits/chosen": -2.34431791305542,
+      "logits/rejected": -2.254798412322998,
+      "logps/chosen": -364.5201721191406,
+      "logps/rejected": -317.5235290527344,
+      "loss": 0.5968,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.18700167536735535,
+      "rewards/margins": 0.317229688167572,
+      "rewards/rejected": -0.504231333732605,
+      "step": 3000
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.7965195220555784e-06,
+      "logits/chosen": -2.1608643531799316,
+      "logits/rejected": -2.1240034103393555,
+      "logps/chosen": -289.5939025878906,
+      "logps/rejected": -296.3310241699219,
+      "loss": 0.6677,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.3151804506778717,
+      "rewards/margins": 0.11579956114292145,
+      "rewards/rejected": -0.43098002672195435,
+      "step": 3010
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.786741231119847e-06,
+      "logits/chosen": -2.3387038707733154,
+      "logits/rejected": -2.179028272628784,
+      "logps/chosen": -307.25726318359375,
+      "logps/rejected": -297.0262756347656,
+      "loss": 0.6305,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.12563619017601013,
+      "rewards/margins": 0.220838263630867,
+      "rewards/rejected": -0.34647443890571594,
+      "step": 3020
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7769360872244992e-06,
+      "logits/chosen": -2.262904405593872,
+      "logits/rejected": -2.238494634628296,
+      "logps/chosen": -280.38232421875,
+      "logps/rejected": -277.2829284667969,
+      "loss": 0.6257,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.11235074698925018,
+      "rewards/margins": 0.2052387297153473,
+      "rewards/rejected": -0.3175894618034363,
+      "step": 3030
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.767104294992754e-06,
+      "logits/chosen": -2.3239455223083496,
+      "logits/rejected": -2.2261970043182373,
+      "logps/chosen": -333.36273193359375,
+      "logps/rejected": -263.41082763671875,
+      "loss": 0.621,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.1215156763792038,
+      "rewards/margins": 0.2308710515499115,
+      "rewards/rejected": -0.3523866832256317,
+      "step": 3040
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7572460596039524e-06,
+      "logits/chosen": -2.2785532474517822,
+      "logits/rejected": -2.289318084716797,
+      "logps/chosen": -303.37408447265625,
+      "logps/rejected": -311.6347351074219,
+      "loss": 0.64,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.20798878371715546,
+      "rewards/margins": 0.19912748038768768,
+      "rewards/rejected": -0.40711626410484314,
+      "step": 3050
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.74736158678928e-06,
+      "logits/chosen": -2.177031993865967,
+      "logits/rejected": -2.113590717315674,
+      "logps/chosen": -389.597412109375,
+      "logps/rejected": -324.69122314453125,
+      "loss": 0.5969,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.13935567438602448,
+      "rewards/margins": 0.30283206701278687,
+      "rewards/rejected": -0.4421877861022949,
+      "step": 3060
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7374510828274673e-06,
+      "logits/chosen": -2.3563344478607178,
+      "logits/rejected": -2.245603561401367,
+      "logps/chosen": -370.6936340332031,
+      "logps/rejected": -324.93255615234375,
+      "loss": 0.6248,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.1752164363861084,
+      "rewards/margins": 0.24540957808494568,
+      "rewards/rejected": -0.4206259846687317,
+      "step": 3070
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.72751475454049e-06,
+      "logits/chosen": -2.223323345184326,
+      "logits/rejected": -2.1870429515838623,
+      "logps/chosen": -312.41802978515625,
+      "logps/rejected": -312.8993225097656,
+      "loss": 0.6326,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.13783670961856842,
+      "rewards/margins": 0.20678336918354034,
+      "rewards/rejected": -0.3446200489997864,
+      "step": 3080
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7175528092892503e-06,
+      "logits/chosen": -2.192528009414673,
+      "logits/rejected": -2.089958667755127,
+      "logps/chosen": -296.0037536621094,
+      "logps/rejected": -298.6204528808594,
+      "loss": 0.6075,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.1506446897983551,
+      "rewards/margins": 0.2501722574234009,
+      "rewards/rejected": -0.4008169174194336,
+      "step": 3090
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.7075654549692498e-06,
+      "logits/chosen": -2.3563451766967773,
+      "logits/rejected": -2.253929853439331,
+      "logps/chosen": -373.33721923828125,
+      "logps/rejected": -308.59918212890625,
+      "loss": 0.5507,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.03195354714989662,
+      "rewards/margins": 0.36858731508255005,
+      "rewards/rejected": -0.40054088830947876,
+      "step": 3100
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.697552900006249e-06,
+      "logits/chosen": -2.2431931495666504,
+      "logits/rejected": -2.239027500152588,
+      "logps/chosen": -294.050048828125,
+      "logps/rejected": -288.5823669433594,
+      "loss": 0.6181,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.15926004946231842,
+      "rewards/margins": 0.2313537299633026,
+      "rewards/rejected": -0.3906137943267822,
+      "step": 3110
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6875153533519244e-06,
+      "logits/chosen": -2.153017520904541,
+      "logits/rejected": -2.1035304069519043,
+      "logps/chosen": -333.7864685058594,
+      "logps/rejected": -318.5226135253906,
+      "loss": 0.6047,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.09763199090957642,
+      "rewards/margins": 0.26820889115333557,
+      "rewards/rejected": -0.3658409118652344,
+      "step": 3120
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6774530244794992e-06,
+      "logits/chosen": -2.2834887504577637,
+      "logits/rejected": -2.151353597640991,
+      "logps/chosen": -315.78375244140625,
+      "logps/rejected": -276.2959289550781,
+      "loss": 0.6187,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.14533431828022003,
+      "rewards/margins": 0.24671044945716858,
+      "rewards/rejected": -0.3920448124408722,
+      "step": 3130
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.667366123379378e-06,
+      "logits/chosen": -2.2238709926605225,
+      "logits/rejected": -2.1253607273101807,
+      "logps/chosen": -255.31527709960938,
+      "logps/rejected": -252.7228240966797,
+      "loss": 0.6183,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.22929397225379944,
+      "rewards/margins": 0.2054261714220047,
+      "rewards/rejected": -0.4347201883792877,
+      "step": 3140
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6572548605547607e-06,
+      "logits/chosen": -2.3579158782958984,
+      "logits/rejected": -2.274282217025757,
+      "logps/chosen": -282.13775634765625,
+      "logps/rejected": -323.9889221191406,
+      "loss": 0.5839,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.15901920199394226,
+      "rewards/margins": 0.33014026284217834,
+      "rewards/rejected": -0.4891594350337982,
+      "step": 3150
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6471194470172538e-06,
+      "logits/chosen": -2.29618501663208,
+      "logits/rejected": -2.2774035930633545,
+      "logps/chosen": -303.40814208984375,
+      "logps/rejected": -333.0376892089844,
+      "loss": 0.5978,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.17918583750724792,
+      "rewards/margins": 0.28591203689575195,
+      "rewards/rejected": -0.46509790420532227,
+      "step": 3160
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.636960094282461e-06,
+      "logits/chosen": -2.196422576904297,
+      "logits/rejected": -2.139151096343994,
+      "logps/chosen": -320.49676513671875,
+      "logps/rejected": -305.7890319824219,
+      "loss": 0.5728,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.1928102970123291,
+      "rewards/margins": 0.34613439440727234,
+      "rewards/rejected": -0.538944661617279,
+      "step": 3170
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.6267770143655743e-06,
+      "logits/chosen": -2.4047622680664062,
+      "logits/rejected": -2.283409595489502,
+      "logps/chosen": -320.8899841308594,
+      "logps/rejected": -308.2359924316406,
+      "loss": 0.6233,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.23850727081298828,
+      "rewards/margins": 0.23102286458015442,
+      "rewards/rejected": -0.4695301949977875,
+      "step": 3180
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.6165704197769484e-06,
+      "logits/chosen": -2.2761237621307373,
+      "logits/rejected": -2.2387335300445557,
+      "logps/chosen": -295.4513854980469,
+      "logps/rejected": -303.45062255859375,
+      "loss": 0.6285,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.14914533495903015,
+      "rewards/margins": 0.2022182047367096,
+      "rewards/rejected": -0.35136356949806213,
+      "step": 3190
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.606340523517663e-06,
+      "logits/chosen": -2.248460054397583,
+      "logits/rejected": -2.194171667098999,
+      "logps/chosen": -284.29449462890625,
+      "logps/rejected": -288.8624572753906,
+      "loss": 0.6104,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.07691571861505508,
+      "rewards/margins": 0.2391362488269806,
+      "rewards/rejected": -0.3160519599914551,
+      "step": 3200
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.5960875390750793e-06,
+      "logits/chosen": -2.325347900390625,
+      "logits/rejected": -2.1235616207122803,
+      "logps/chosen": -360.14239501953125,
+      "logps/rejected": -321.9942932128906,
+      "loss": 0.5561,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.04620830714702606,
+      "rewards/margins": 0.38283371925354004,
+      "rewards/rejected": -0.4290420114994049,
+      "step": 3210
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.585811680418386e-06,
+      "logits/chosen": -2.24568510055542,
+      "logits/rejected": -2.1574959754943848,
+      "logps/chosen": -310.50616455078125,
+      "logps/rejected": -308.9969177246094,
+      "loss": 0.5897,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.08727524429559708,
+      "rewards/margins": 0.29882222414016724,
+      "rewards/rejected": -0.3860974907875061,
+      "step": 3220
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.5755131619941347e-06,
+      "logits/chosen": -2.201096296310425,
+      "logits/rejected": -2.157496452331543,
+      "logps/chosen": -288.51776123046875,
+      "logps/rejected": -278.84356689453125,
+      "loss": 0.611,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.13764691352844238,
+      "rewards/margins": 0.2472527027130127,
+      "rewards/rejected": -0.3848995864391327,
+      "step": 3230
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.565192198721759e-06,
+      "logits/chosen": -2.207019567489624,
+      "logits/rejected": -2.1581149101257324,
+      "logps/chosen": -274.2110290527344,
+      "logps/rejected": -330.1119079589844,
+      "loss": 0.5841,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.17945356667041779,
+      "rewards/margins": 0.3046710789203644,
+      "rewards/rejected": -0.4841246008872986,
+      "step": 3240
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5548490059890965e-06,
+      "logits/chosen": -2.2094249725341797,
+      "logits/rejected": -2.1251931190490723,
+      "logps/chosen": -338.31134033203125,
+      "logps/rejected": -347.03143310546875,
+      "loss": 0.5808,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.14935877919197083,
+      "rewards/margins": 0.34563302993774414,
+      "rewards/rejected": -0.4949917793273926,
+      "step": 3250
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5444837996478903e-06,
+      "logits/chosen": -2.287841558456421,
+      "logits/rejected": -2.161818027496338,
+      "logps/chosen": -342.252685546875,
+      "logps/rejected": -380.78582763671875,
+      "loss": 0.6197,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.20302172005176544,
+      "rewards/margins": 0.259146511554718,
+      "rewards/rejected": -0.4621681571006775,
+      "step": 3260
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.534096796009282e-06,
+      "logits/chosen": -2.1436102390289307,
+      "logits/rejected": -2.121990203857422,
+      "logps/chosen": -290.8398742675781,
+      "logps/rejected": -306.20050048828125,
+      "loss": 0.6164,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.12765438854694366,
+      "rewards/margins": 0.23902735114097595,
+      "rewards/rejected": -0.3666817545890808,
+      "step": 3270
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5236882118393046e-06,
+      "logits/chosen": -2.2807412147521973,
+      "logits/rejected": -2.20418119430542,
+      "logps/chosen": -330.96759033203125,
+      "logps/rejected": -313.71533203125,
+      "loss": 0.6203,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.08276918530464172,
+      "rewards/margins": 0.22776314616203308,
+      "rewards/rejected": -0.3105323016643524,
+      "step": 3280
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5132582643543513e-06,
+      "logits/chosen": -2.199484348297119,
+      "logits/rejected": -2.11887264251709,
+      "logps/chosen": -308.62176513671875,
+      "logps/rejected": -303.9732666015625,
+      "loss": 0.613,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.10497989505529404,
+      "rewards/margins": 0.22992253303527832,
+      "rewards/rejected": -0.33490240573883057,
+      "step": 3290
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5028071712166456e-06,
+      "logits/chosen": -2.2492740154266357,
+      "logits/rejected": -2.0946688652038574,
+      "logps/chosen": -314.8434143066406,
+      "logps/rejected": -283.57696533203125,
+      "loss": 0.5991,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.07174123823642731,
+      "rewards/margins": 0.2797015309333801,
+      "rewards/rejected": -0.35144275426864624,
+      "step": 3300
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.4923351505297008e-06,
+      "logits/chosen": -2.3448548316955566,
+      "logits/rejected": -2.2648229598999023,
+      "logps/chosen": -326.4363098144531,
+      "logps/rejected": -330.6997375488281,
+      "loss": 0.5816,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.10261914879083633,
+      "rewards/margins": 0.299100786447525,
+      "rewards/rejected": -0.4017198979854584,
+      "step": 3310
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.481842420833766e-06,
+      "logits/chosen": -2.3306705951690674,
+      "logits/rejected": -2.2149271965026855,
+      "logps/chosen": -368.263916015625,
+      "logps/rejected": -313.85577392578125,
+      "loss": 0.5753,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.1251881867647171,
+      "rewards/margins": 0.32386136054992676,
+      "rewards/rejected": -0.4490494728088379,
+      "step": 3320
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4713292011012645e-06,
+      "logits/chosen": -2.1722724437713623,
+      "logits/rejected": -2.05879282951355,
+      "logps/chosen": -327.58026123046875,
+      "logps/rejected": -323.82806396484375,
+      "loss": 0.6109,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.16994589567184448,
+      "rewards/margins": 0.2712971270084381,
+      "rewards/rejected": -0.4412430226802826,
+      "step": 3330
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4607957107322277e-06,
+      "logits/chosen": -2.2359910011291504,
+      "logits/rejected": -2.0647006034851074,
+      "logps/chosen": -275.21966552734375,
+      "logps/rejected": -279.6451110839844,
+      "loss": 0.5917,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.22481286525726318,
+      "rewards/margins": 0.28406020998954773,
+      "rewards/rejected": -0.5088731050491333,
+      "step": 3340
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4502421695497112e-06,
+      "logits/chosen": -2.1291165351867676,
+      "logits/rejected": -2.114325523376465,
+      "logps/chosen": -304.02459716796875,
+      "logps/rejected": -307.4547424316406,
+      "loss": 0.6274,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.1552276909351349,
+      "rewards/margins": 0.20688049495220184,
+      "rewards/rejected": -0.3621082007884979,
+      "step": 3350
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4396687977952137e-06,
+      "logits/chosen": -2.171285629272461,
+      "logits/rejected": -2.0632450580596924,
+      "logps/chosen": -295.75616455078125,
+      "logps/rejected": -300.7738342285156,
+      "loss": 0.5886,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.18257856369018555,
+      "rewards/margins": 0.3184903562068939,
+      "rewards/rejected": -0.5010689496994019,
+      "step": 3360
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.429075816124075e-06,
+      "logits/chosen": -2.2236578464508057,
+      "logits/rejected": -2.1825549602508545,
+      "logps/chosen": -332.3364562988281,
+      "logps/rejected": -344.52398681640625,
+      "loss": 0.5825,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.23878760635852814,
+      "rewards/margins": 0.3120969235897064,
+      "rewards/rejected": -0.5508845448493958,
+      "step": 3370
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.418463445600874e-06,
+      "logits/chosen": -2.254668712615967,
+      "logits/rejected": -2.173424243927002,
+      "logps/chosen": -275.3207092285156,
+      "logps/rejected": -298.79443359375,
+      "loss": 0.606,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.33824622631073,
+      "rewards/margins": 0.2654719650745392,
+      "rewards/rejected": -0.6037181615829468,
+      "step": 3380
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4078319076948173e-06,
+      "logits/chosen": -2.1863226890563965,
+      "logits/rejected": -2.1001029014587402,
+      "logps/chosen": -281.54888916015625,
+      "logps/rejected": -294.13311767578125,
+      "loss": 0.5669,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.18318873643875122,
+      "rewards/margins": 0.3577658236026764,
+      "rewards/rejected": -0.5409545302391052,
+      "step": 3390
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.3971814242751123e-06,
+      "logits/chosen": -2.2467076778411865,
+      "logits/rejected": -2.1341750621795654,
+      "logps/chosen": -327.49407958984375,
+      "logps/rejected": -304.19384765625,
+      "loss": 0.6118,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.07793440669775009,
+      "rewards/margins": 0.24999889731407166,
+      "rewards/rejected": -0.32793331146240234,
+      "step": 3400
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.386512217606339e-06,
+      "logits/chosen": -2.256131410598755,
+      "logits/rejected": -2.2085914611816406,
+      "logps/chosen": -301.0712890625,
+      "logps/rejected": -293.40728759765625,
+      "loss": 0.5734,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.15557502210140228,
+      "rewards/margins": 0.33370381593704224,
+      "rewards/rejected": -0.4892788529396057,
+      "step": 3410
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.375824510343816e-06,
+      "logits/chosen": -2.2942771911621094,
+      "logits/rejected": -2.183180570602417,
+      "logps/chosen": -329.44525146484375,
+      "logps/rejected": -359.43402099609375,
+      "loss": 0.5946,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.1809997260570526,
+      "rewards/margins": 0.2896903157234192,
+      "rewards/rejected": -0.4706901013851166,
+      "step": 3420
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3651185255289466e-06,
+      "logits/chosen": -2.295954465866089,
+      "logits/rejected": -2.2029337882995605,
+      "logps/chosen": -350.3366394042969,
+      "logps/rejected": -360.73248291015625,
+      "loss": 0.5818,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.23058481514453888,
+      "rewards/margins": 0.31552547216415405,
+      "rewards/rejected": -0.5461102724075317,
+      "step": 3430
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.354394486584568e-06,
+      "logits/chosen": -2.2945914268493652,
+      "logits/rejected": -2.179950714111328,
+      "logps/chosen": -354.335205078125,
+      "logps/rejected": -368.111328125,
+      "loss": 0.5819,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.21776917576789856,
+      "rewards/margins": 0.302031934261322,
+      "rewards/rejected": -0.519801139831543,
+      "step": 3440
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3436526173102913e-06,
+      "logits/chosen": -2.2314987182617188,
+      "logits/rejected": -2.1264474391937256,
+      "logps/chosen": -274.8760986328125,
+      "logps/rejected": -247.54135131835938,
+      "loss": 0.5977,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.21922855079174042,
+      "rewards/margins": 0.2775750160217285,
+      "rewards/rejected": -0.4968036115169525,
+      "step": 3450
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3328931418778254e-06,
+      "logits/chosen": -2.2393717765808105,
+      "logits/rejected": -2.203644275665283,
+      "logps/chosen": -310.0876770019531,
+      "logps/rejected": -333.08660888671875,
+      "loss": 0.5576,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.1818048506975174,
+      "rewards/margins": 0.3758290708065033,
+      "rewards/rejected": -0.5576339960098267,
+      "step": 3460
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3221162848263028e-06,
+      "logits/chosen": -2.255612850189209,
+      "logits/rejected": -2.183682680130005,
+      "logps/chosen": -335.07220458984375,
+      "logps/rejected": -319.34234619140625,
+      "loss": 0.5432,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.18285012245178223,
+      "rewards/margins": 0.40342992544174194,
+      "rewards/rejected": -0.5862800478935242,
+      "step": 3470
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.3113222710575914e-06,
+      "logits/chosen": -2.288682460784912,
+      "logits/rejected": -2.2233240604400635,
+      "logps/chosen": -342.3880310058594,
+      "logps/rejected": -330.94049072265625,
+      "loss": 0.6524,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.2884979248046875,
+      "rewards/margins": 0.15531443059444427,
+      "rewards/rejected": -0.44381237030029297,
+      "step": 3480
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.300511325831603e-06,
+      "logits/chosen": -2.1967105865478516,
+      "logits/rejected": -2.1920440196990967,
+      "logps/chosen": -306.7802429199219,
+      "logps/rejected": -332.65289306640625,
+      "loss": 0.6126,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.17921099066734314,
+      "rewards/margins": 0.2644205093383789,
+      "rewards/rejected": -0.44363147020339966,
+      "step": 3490
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.289683674761592e-06,
+      "logits/chosen": -2.21291184425354,
+      "logits/rejected": -2.155174493789673,
+      "logps/chosen": -310.1589660644531,
+      "logps/rejected": -302.49102783203125,
+      "loss": 0.5816,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.19234074652194977,
+      "rewards/margins": 0.31977444887161255,
+      "rewards/rejected": -0.5121151804924011,
+      "step": 3500
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.2788395438094444e-06,
+      "logits/chosen": -2.2448463439941406,
+      "logits/rejected": -2.154503583908081,
+      "logps/chosen": -322.6603698730469,
+      "logps/rejected": -321.0233154296875,
+      "loss": 0.603,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.22834105789661407,
+      "rewards/margins": 0.29928749799728394,
+      "rewards/rejected": -0.5276285409927368,
+      "step": 3510
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.2679791592809653e-06,
+      "logits/chosen": -2.2332160472869873,
+      "logits/rejected": -2.134450912475586,
+      "logps/chosen": -271.142822265625,
+      "logps/rejected": -309.9905090332031,
+      "loss": 0.5777,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.07708028703927994,
+      "rewards/margins": 0.30101966857910156,
+      "rewards/rejected": -0.3780999183654785,
+      "step": 3520
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.257102747821157e-06,
+      "logits/chosen": -2.184156894683838,
+      "logits/rejected": -2.0749685764312744,
+      "logps/chosen": -334.580322265625,
+      "logps/rejected": -338.4678955078125,
+      "loss": 0.5468,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.09229175001382828,
+      "rewards/margins": 0.4070521295070648,
+      "rewards/rejected": -0.49934396147727966,
+      "step": 3530
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.246210536409484e-06,
+      "logits/chosen": -2.1600894927978516,
+      "logits/rejected": -2.0873522758483887,
+      "logps/chosen": -318.50640869140625,
+      "logps/rejected": -305.25787353515625,
+      "loss": 0.6145,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.20227620005607605,
+      "rewards/margins": 0.24205967783927917,
+      "rewards/rejected": -0.44433584809303284,
+      "step": 3540
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.235302752355142e-06,
+      "logits/chosen": -2.2977747917175293,
+      "logits/rejected": -2.1594138145446777,
+      "logps/chosen": -337.0153503417969,
+      "logps/rejected": -287.40789794921875,
+      "loss": 0.5581,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.0762878805398941,
+      "rewards/margins": 0.37493664026260376,
+      "rewards/rejected": -0.45122450590133667,
+      "step": 3550
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.2243796232923097e-06,
+      "logits/chosen": -2.3173668384552,
+      "logits/rejected": -2.1900227069854736,
+      "logps/chosen": -362.87994384765625,
+      "logps/rejected": -369.14581298828125,
+      "loss": 0.6038,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.2711828351020813,
+      "rewards/margins": 0.27714723348617554,
+      "rewards/rejected": -0.5483301877975464,
+      "step": 3560
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.2134413771754037e-06,
+      "logits/chosen": -2.3003287315368652,
+      "logits/rejected": -2.2316155433654785,
+      "logps/chosen": -307.1923828125,
+      "logps/rejected": -319.0071105957031,
+      "loss": 0.5999,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.2506977617740631,
+      "rewards/margins": 0.2686203718185425,
+      "rewards/rejected": -0.519318163394928,
+      "step": 3570
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.2024882422743118e-06,
+      "logits/chosen": -2.206987142562866,
+      "logits/rejected": -2.154073476791382,
+      "logps/chosen": -273.85357666015625,
+      "logps/rejected": -308.89007568359375,
+      "loss": 0.5649,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.20727932453155518,
+      "rewards/margins": 0.34459999203681946,
+      "rewards/rejected": -0.5518792867660522,
+      "step": 3580
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.1915204471696425e-06,
+      "logits/chosen": -2.298491954803467,
+      "logits/rejected": -2.126559019088745,
+      "logps/chosen": -301.7465515136719,
+      "logps/rejected": -271.80535888671875,
+      "loss": 0.5882,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.13729017972946167,
+      "rewards/margins": 0.3090158998966217,
+      "rewards/rejected": -0.4463060796260834,
+      "step": 3590
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.180538220747943e-06,
+      "logits/chosen": -2.240255355834961,
+      "logits/rejected": -2.1766419410705566,
+      "logps/chosen": -330.14813232421875,
+      "logps/rejected": -320.18756103515625,
+      "loss": 0.6079,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.24947145581245422,
+      "rewards/margins": 0.28124094009399414,
+      "rewards/rejected": -0.530712366104126,
+      "step": 3600
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.1695417921969287e-06,
+      "logits/chosen": -2.3270153999328613,
+      "logits/rejected": -2.2739806175231934,
+      "logps/chosen": -294.16253662109375,
+      "logps/rejected": -328.5826721191406,
+      "loss": 0.6349,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.14072641730308533,
+      "rewards/margins": 0.1998475342988968,
+      "rewards/rejected": -0.3405739665031433,
+      "step": 3610
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.158531391000697e-06,
+      "logits/chosen": -2.2572436332702637,
+      "logits/rejected": -2.1108286380767822,
+      "logps/chosen": -315.2557373046875,
+      "logps/rejected": -303.2156982421875,
+      "loss": 0.582,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.14727096259593964,
+      "rewards/margins": 0.34550607204437256,
+      "rewards/rejected": -0.4927770495414734,
+      "step": 3620
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.147507246934943e-06,
+      "logits/chosen": -2.273355722427368,
+      "logits/rejected": -2.232914447784424,
+      "logps/chosen": -340.02618408203125,
+      "logps/rejected": -342.54217529296875,
+      "loss": 0.5948,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.0910380631685257,
+      "rewards/margins": 0.2883290648460388,
+      "rewards/rejected": -0.3793671727180481,
+      "step": 3630
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.136469590062158e-06,
+      "logits/chosen": -2.1601414680480957,
+      "logits/rejected": -2.0744059085845947,
+      "logps/chosen": -277.2970275878906,
+      "logps/rejected": -271.2074890136719,
+      "loss": 0.5807,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.1494699865579605,
+      "rewards/margins": 0.32256048917770386,
+      "rewards/rejected": -0.4720304608345032,
+      "step": 3640
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.1254186507268354e-06,
+      "logits/chosen": -2.272613763809204,
+      "logits/rejected": -2.1787149906158447,
+      "logps/chosen": -267.9743347167969,
+      "logps/rejected": -291.6812438964844,
+      "loss": 0.6001,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.07630626112222672,
+      "rewards/margins": 0.2765544354915619,
+      "rewards/rejected": -0.3528607189655304,
+      "step": 3650
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.114354659550656e-06,
+      "logits/chosen": -2.1525888442993164,
+      "logits/rejected": -2.1212618350982666,
+      "logps/chosen": -322.45074462890625,
+      "logps/rejected": -308.0700378417969,
+      "loss": 0.5766,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.24668438732624054,
+      "rewards/margins": 0.34969207644462585,
+      "rewards/rejected": -0.5963764190673828,
+      "step": 3660
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.1032778474276816e-06,
+      "logits/chosen": -2.366992473602295,
+      "logits/rejected": -2.2086708545684814,
+      "logps/chosen": -319.9839782714844,
+      "logps/rejected": -298.745361328125,
+      "loss": 0.5549,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.2000807523727417,
+      "rewards/margins": 0.3787020444869995,
+      "rewards/rejected": -0.5787827968597412,
+      "step": 3670
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.092188445519532e-06,
+      "logits/chosen": -2.26643967628479,
+      "logits/rejected": -2.128983974456787,
+      "logps/chosen": -326.4763488769531,
+      "logps/rejected": -297.0374755859375,
+      "loss": 0.6034,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.24593958258628845,
+      "rewards/margins": 0.30430930852890015,
+      "rewards/rejected": -0.550248920917511,
+      "step": 3680
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.081086685250565e-06,
+      "logits/chosen": -2.2758965492248535,
+      "logits/rejected": -2.1524596214294434,
+      "logps/chosen": -292.1910400390625,
+      "logps/rejected": -272.1326904296875,
+      "loss": 0.5854,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.2058788239955902,
+      "rewards/margins": 0.31988587975502014,
+      "rewards/rejected": -0.5257646441459656,
+      "step": 3690
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.0699727983030434e-06,
+      "logits/chosen": -2.3243601322174072,
+      "logits/rejected": -2.2180685997009277,
+      "logps/chosen": -332.9953308105469,
+      "logps/rejected": -295.5639953613281,
+      "loss": 0.6358,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.23166723549365997,
+      "rewards/margins": 0.2067691832780838,
+      "rewards/rejected": -0.4384364187717438,
+      "step": 3700
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.058847016612301e-06,
+      "logits/chosen": -2.23081636428833,
+      "logits/rejected": -2.207549571990967,
+      "logps/chosen": -353.3510437011719,
+      "logps/rejected": -366.8516540527344,
+      "loss": 0.6399,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.19398830831050873,
+      "rewards/margins": 0.19586136937141418,
+      "rewards/rejected": -0.3898496925830841,
+      "step": 3710
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0477095723619034e-06,
+      "logits/chosen": -2.3202261924743652,
+      "logits/rejected": -2.171468734741211,
+      "logps/chosen": -286.7763366699219,
+      "logps/rejected": -291.25006103515625,
+      "loss": 0.5881,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.18975184857845306,
+      "rewards/margins": 0.3185967803001404,
+      "rewards/rejected": -0.5083485841751099,
+      "step": 3720
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0365606979788003e-06,
+      "logits/chosen": -2.29597544670105,
+      "logits/rejected": -2.201775074005127,
+      "logps/chosen": -308.6613464355469,
+      "logps/rejected": -297.501220703125,
+      "loss": 0.589,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.16128845512866974,
+      "rewards/margins": 0.29569411277770996,
+      "rewards/rejected": -0.4569825530052185,
+      "step": 3730
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0254006261284786e-06,
+      "logits/chosen": -2.2917492389678955,
+      "logits/rejected": -2.20141339302063,
+      "logps/chosen": -305.4287109375,
+      "logps/rejected": -313.76934814453125,
+      "loss": 0.5959,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.22600774466991425,
+      "rewards/margins": 0.28799906373023987,
+      "rewards/rejected": -0.5140067934989929,
+      "step": 3740
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0142295897101032e-06,
+      "logits/chosen": -2.1403932571411133,
+      "logits/rejected": -1.9929752349853516,
+      "logps/chosen": -326.08441162109375,
+      "logps/rejected": -297.84808349609375,
+      "loss": 0.5601,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.1513725221157074,
+      "rewards/margins": 0.3802407681941986,
+      "rewards/rejected": -0.5316132307052612,
+      "step": 3750
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0030478218516578e-06,
+      "logits/chosen": -2.2112417221069336,
+      "logits/rejected": -2.1325557231903076,
+      "logps/chosen": -309.48089599609375,
+      "logps/rejected": -310.97509765625,
+      "loss": 0.588,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.14635725319385529,
+      "rewards/margins": 0.3053987920284271,
+      "rewards/rejected": -0.4517560601234436,
+      "step": 3760
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 2.9918555559050826e-06,
+      "logits/chosen": -2.1812379360198975,
+      "logits/rejected": -2.1582610607147217,
+      "logps/chosen": -297.3588562011719,
+      "logps/rejected": -288.1904296875,
+      "loss": 0.5738,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.13978731632232666,
+      "rewards/margins": 0.3152059316635132,
+      "rewards/rejected": -0.45499324798583984,
+      "step": 3770
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 2.980653025441399e-06,
+      "logits/chosen": -2.294768810272217,
+      "logits/rejected": -2.217179775238037,
+      "logps/chosen": -335.85235595703125,
+      "logps/rejected": -327.75543212890625,
+      "loss": 0.5307,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.053152166306972504,
+      "rewards/margins": 0.4419528841972351,
+      "rewards/rejected": -0.4951050877571106,
+      "step": 3780
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.969440464245841e-06,
+      "logits/chosen": -2.312551975250244,
+      "logits/rejected": -2.216581344604492,
+      "logps/chosen": -335.02862548828125,
+      "logps/rejected": -301.836181640625,
+      "loss": 0.6369,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.15713906288146973,
+      "rewards/margins": 0.18496128916740417,
+      "rewards/rejected": -0.3421003222465515,
+      "step": 3790
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.95821810631297e-06,
+      "logits/chosen": -2.155395030975342,
+      "logits/rejected": -2.148796796798706,
+      "logps/chosen": -260.44122314453125,
+      "logps/rejected": -276.7882080078125,
+      "loss": 0.5771,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.0877716988325119,
+      "rewards/margins": 0.322742760181427,
+      "rewards/rejected": -0.4105144441127777,
+      "step": 3800
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.946986185841801e-06,
+      "logits/chosen": -2.1743924617767334,
+      "logits/rejected": -2.0896475315093994,
+      "logps/chosen": -299.45391845703125,
+      "logps/rejected": -304.2263488769531,
+      "loss": 0.6052,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.12790082395076752,
+      "rewards/margins": 0.24636109173297882,
+      "rewards/rejected": -0.37426191568374634,
+      "step": 3810
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.935744937230903e-06,
+      "logits/chosen": -2.1896514892578125,
+      "logits/rejected": -2.161184310913086,
+      "logps/chosen": -324.8854675292969,
+      "logps/rejected": -304.50994873046875,
+      "loss": 0.6399,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.06740345805883408,
+      "rewards/margins": 0.1831163913011551,
+      "rewards/rejected": -0.25051984190940857,
+      "step": 3820
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.924494595073517e-06,
+      "logits/chosen": -2.2635974884033203,
+      "logits/rejected": -2.2178092002868652,
+      "logps/chosen": -305.7368469238281,
+      "logps/rejected": -304.89007568359375,
+      "loss": 0.5615,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.032348960638046265,
+      "rewards/margins": 0.38179102540016174,
+      "rewards/rejected": -0.41413992643356323,
+      "step": 3830
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.9132353941526575e-06,
+      "logits/chosen": -2.2265233993530273,
+      "logits/rejected": -2.157034158706665,
+      "logps/chosen": -296.8994445800781,
+      "logps/rejected": -320.8399963378906,
+      "loss": 0.6032,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.1254245936870575,
+      "rewards/margins": 0.27609962224960327,
+      "rewards/rejected": -0.40152424573898315,
+      "step": 3840
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.901967569436209e-06,
+      "logits/chosen": -2.2328522205352783,
+      "logits/rejected": -2.1735124588012695,
+      "logps/chosen": -250.29086303710938,
+      "logps/rejected": -267.7702941894531,
+      "loss": 0.5602,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.062219761312007904,
+      "rewards/margins": 0.36623015999794006,
+      "rewards/rejected": -0.42844992876052856,
+      "step": 3850
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.89069135607203e-06,
+      "logits/chosen": -2.1895928382873535,
+      "logits/rejected": -2.1317780017852783,
+      "logps/chosen": -311.4351806640625,
+      "logps/rejected": -313.31988525390625,
+      "loss": 0.6074,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.10760854184627533,
+      "rewards/margins": 0.24679633975028992,
+      "rewards/rejected": -0.35440492630004883,
+      "step": 3860
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8794069893830386e-06,
+      "logits/chosen": -2.198737382888794,
+      "logits/rejected": -2.1195740699768066,
+      "logps/chosen": -287.1357116699219,
+      "logps/rejected": -308.64666748046875,
+      "loss": 0.5917,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.1630997508764267,
+      "rewards/margins": 0.29942062497138977,
+      "rewards/rejected": -0.46252042055130005,
+      "step": 3870
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8681147048623038e-06,
+      "logits/chosen": -2.2363996505737305,
+      "logits/rejected": -2.2083020210266113,
+      "logps/chosen": -323.7630310058594,
+      "logps/rejected": -343.34991455078125,
+      "loss": 0.5567,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.06575950235128403,
+      "rewards/margins": 0.38548240065574646,
+      "rewards/rejected": -0.4512419104576111,
+      "step": 3880
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8568147381681333e-06,
+      "logits/chosen": -2.246504068374634,
+      "logits/rejected": -2.189868688583374,
+      "logps/chosen": -288.006591796875,
+      "logps/rejected": -303.1795959472656,
+      "loss": 0.6026,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.1896587759256363,
+      "rewards/margins": 0.28125303983688354,
+      "rewards/rejected": -0.47091180086135864,
+      "step": 3890
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8455073251191533e-06,
+      "logits/chosen": -2.279578447341919,
+      "logits/rejected": -2.235502243041992,
+      "logps/chosen": -310.79351806640625,
+      "logps/rejected": -327.28033447265625,
+      "loss": 0.6095,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.2548527121543884,
+      "rewards/margins": 0.2784276306629181,
+      "rewards/rejected": -0.5332802534103394,
+      "step": 3900
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8341927016893887e-06,
+      "logits/chosen": -2.243419647216797,
+      "logits/rejected": -2.213569164276123,
+      "logps/chosen": -282.7080993652344,
+      "logps/rejected": -317.5572509765625,
+      "loss": 0.591,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.28008633852005005,
+      "rewards/margins": 0.30951592326164246,
+      "rewards/rejected": -0.5896022915840149,
+      "step": 3910
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.822871104003335e-06,
+      "logits/chosen": -2.1288013458251953,
+      "logits/rejected": -2.0994820594787598,
+      "logps/chosen": -268.18475341796875,
+      "logps/rejected": -329.8453063964844,
+      "loss": 0.5971,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.19896329939365387,
+      "rewards/margins": 0.28691792488098145,
+      "rewards/rejected": -0.48588117957115173,
+      "step": 3920
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8115427683310355e-06,
+      "logits/chosen": -2.309434413909912,
+      "logits/rejected": -2.18902850151062,
+      "logps/chosen": -343.51385498046875,
+      "logps/rejected": -341.13525390625,
+      "loss": 0.5761,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.19033750891685486,
+      "rewards/margins": 0.3401159346103668,
+      "rewards/rejected": -0.5304533839225769,
+      "step": 3930
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.8002079310831477e-06,
+      "logits/chosen": -2.25565242767334,
+      "logits/rejected": -2.157276153564453,
+      "logps/chosen": -297.09967041015625,
+      "logps/rejected": -317.29083251953125,
+      "loss": 0.6199,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.21050819754600525,
+      "rewards/margins": 0.23793046176433563,
+      "rewards/rejected": -0.4484386444091797,
+      "step": 3940
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7888668288060095e-06,
+      "logits/chosen": -2.258208751678467,
+      "logits/rejected": -2.150857925415039,
+      "logps/chosen": -326.5715637207031,
+      "logps/rejected": -338.069580078125,
+      "loss": 0.5258,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.1631345897912979,
+      "rewards/margins": 0.44891101121902466,
+      "rewards/rejected": -0.6120456457138062,
+      "step": 3950
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7775196981767044e-06,
+      "logits/chosen": -2.1642158031463623,
+      "logits/rejected": -2.065582275390625,
+      "logps/chosen": -278.7409973144531,
+      "logps/rejected": -314.7583923339844,
+      "loss": 0.5873,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.19401176273822784,
+      "rewards/margins": 0.34037813544273376,
+      "rewards/rejected": -0.5343899726867676,
+      "step": 3960
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7661667759981213e-06,
+      "logits/chosen": -2.245973587036133,
+      "logits/rejected": -2.2456717491149902,
+      "logps/chosen": -271.9554138183594,
+      "logps/rejected": -294.5726623535156,
+      "loss": 0.6176,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.22680599987506866,
+      "rewards/margins": 0.2192671298980713,
+      "rewards/rejected": -0.44607311487197876,
+      "step": 3970
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7548082991940137e-06,
+      "logits/chosen": -2.23134183883667,
+      "logits/rejected": -2.1554768085479736,
+      "logps/chosen": -308.91314697265625,
+      "logps/rejected": -320.1424865722656,
+      "loss": 0.6458,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.20446991920471191,
+      "rewards/margins": 0.19974420964717865,
+      "rewards/rejected": -0.40421414375305176,
+      "step": 3980
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.743444504804051e-06,
+      "logits/chosen": -2.1393723487854004,
+      "logits/rejected": -2.0969605445861816,
+      "logps/chosen": -254.2072296142578,
+      "logps/rejected": -263.64495849609375,
+      "loss": 0.6221,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.21768343448638916,
+      "rewards/margins": 0.21589119732379913,
+      "rewards/rejected": -0.4335746169090271,
+      "step": 3990
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7320756299788788e-06,
+      "logits/chosen": -2.199575901031494,
+      "logits/rejected": -2.108891010284424,
+      "logps/chosen": -316.3847961425781,
+      "logps/rejected": -306.77606201171875,
+      "loss": 0.6127,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.16566872596740723,
+      "rewards/margins": 0.25379401445388794,
+      "rewards/rejected": -0.41946274042129517,
+      "step": 4000
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7207019119751644e-06,
+      "logits/chosen": -2.2674145698547363,
+      "logits/rejected": -2.1696324348449707,
+      "logps/chosen": -276.8014221191406,
+      "logps/rejected": -265.08660888671875,
+      "loss": 0.5671,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.1784783899784088,
+      "rewards/margins": 0.3498672842979431,
+      "rewards/rejected": -0.5283456444740295,
+      "step": 4010
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.7093235881506474e-06,
+      "logits/chosen": -2.3143670558929443,
+      "logits/rejected": -2.181687831878662,
+      "logps/chosen": -314.69378662109375,
+      "logps/rejected": -334.4461975097656,
+      "loss": 0.5555,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.1640726625919342,
+      "rewards/margins": 0.4091968536376953,
+      "rewards/rejected": -0.5732695460319519,
+      "step": 4020
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6979408959591863e-06,
+      "logits/chosen": -2.280393123626709,
+      "logits/rejected": -2.2724785804748535,
+      "logps/chosen": -272.15374755859375,
+      "logps/rejected": -299.1646728515625,
+      "loss": 0.6125,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.13850513100624084,
+      "rewards/margins": 0.2782619595527649,
+      "rewards/rejected": -0.41676706075668335,
+      "step": 4030
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6865540729458034e-06,
+      "logits/chosen": -2.2067208290100098,
+      "logits/rejected": -2.104708194732666,
+      "logps/chosen": -318.2445373535156,
+      "logps/rejected": -334.36676025390625,
+      "loss": 0.5605,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.09667529910802841,
+      "rewards/margins": 0.3725287914276123,
+      "rewards/rejected": -0.4692040979862213,
+      "step": 4040
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.675163356741726e-06,
+      "logits/chosen": -2.2189674377441406,
+      "logits/rejected": -2.1255784034729004,
+      "logps/chosen": -267.58892822265625,
+      "logps/rejected": -300.7120666503906,
+      "loss": 0.5771,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.03780367970466614,
+      "rewards/margins": 0.3484765887260437,
+      "rewards/rejected": -0.38628023862838745,
+      "step": 4050
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6637689850594285e-06,
+      "logits/chosen": -2.2345590591430664,
+      "logits/rejected": -2.1449902057647705,
+      "logps/chosen": -273.7065124511719,
+      "logps/rejected": -324.08111572265625,
+      "loss": 0.6185,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.11849425733089447,
+      "rewards/margins": 0.24025535583496094,
+      "rewards/rejected": -0.3587496876716614,
+      "step": 4060
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.652371195687671e-06,
+      "logits/chosen": -2.2655909061431885,
+      "logits/rejected": -2.1847379207611084,
+      "logps/chosen": -287.54351806640625,
+      "logps/rejected": -315.51531982421875,
+      "loss": 0.5583,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.022312898188829422,
+      "rewards/margins": 0.3931361138820648,
+      "rewards/rejected": -0.41544899344444275,
+      "step": 4070
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.64097022648654e-06,
+      "logits/chosen": -2.220186710357666,
+      "logits/rejected": -2.1098077297210693,
+      "logps/chosen": -325.7334289550781,
+      "logps/rejected": -305.6304016113281,
+      "loss": 0.5655,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.08791736513376236,
+      "rewards/margins": 0.38575097918510437,
+      "rewards/rejected": -0.4736683964729309,
+      "step": 4080
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.6295663153824774e-06,
+      "logits/chosen": -2.2417075634002686,
+      "logits/rejected": -2.1070821285247803,
+      "logps/chosen": -322.048828125,
+      "logps/rejected": -293.63092041015625,
+      "loss": 0.5736,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.11551324278116226,
+      "rewards/margins": 0.3490517735481262,
+      "rewards/rejected": -0.4645650386810303,
+      "step": 4090
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.6181597003633218e-06,
+      "logits/chosen": -2.284554958343506,
+      "logits/rejected": -2.167095899581909,
+      "logps/chosen": -289.63763427734375,
+      "logps/rejected": -300.6242370605469,
+      "loss": 0.5748,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.05315490812063217,
+      "rewards/margins": 0.34837380051612854,
+      "rewards/rejected": -0.4015286862850189,
+      "step": 4100
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.606750619473342e-06,
+      "logits/chosen": -2.1678881645202637,
+      "logits/rejected": -2.1289010047912598,
+      "logps/chosen": -276.3775634765625,
+      "logps/rejected": -320.9483337402344,
+      "loss": 0.5601,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.14479228854179382,
+      "rewards/margins": 0.36358359456062317,
+      "rewards/rejected": -0.508375883102417,
+      "step": 4110
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.595339310808262e-06,
+      "logits/chosen": -2.2610087394714355,
+      "logits/rejected": -2.250720500946045,
+      "logps/chosen": -279.8154296875,
+      "logps/rejected": -314.9268493652344,
+      "loss": 0.5925,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.12273641675710678,
+      "rewards/margins": 0.29857784509658813,
+      "rewards/rejected": -0.4213142395019531,
+      "step": 4120
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5839260125103004e-06,
+      "logits/chosen": -2.2299766540527344,
+      "logits/rejected": -2.088897228240967,
+      "logps/chosen": -290.0578308105469,
+      "logps/rejected": -304.0356750488281,
+      "loss": 0.5816,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.1891263872385025,
+      "rewards/margins": 0.3399195075035095,
+      "rewards/rejected": -0.5290459394454956,
+      "step": 4130
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5725109627631984e-06,
+      "logits/chosen": -2.2458443641662598,
+      "logits/rejected": -2.2014641761779785,
+      "logps/chosen": -317.4651794433594,
+      "logps/rejected": -290.9559020996094,
+      "loss": 0.5728,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.15963537991046906,
+      "rewards/margins": 0.34964263439178467,
+      "rewards/rejected": -0.5092780590057373,
+      "step": 4140
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5610943997872443e-06,
+      "logits/chosen": -2.21856951713562,
+      "logits/rejected": -2.118227481842041,
+      "logps/chosen": -304.394775390625,
+      "logps/rejected": -321.1172790527344,
+      "loss": 0.6039,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.26354244351387024,
+      "rewards/margins": 0.2619742751121521,
+      "rewards/rejected": -0.5255167484283447,
+      "step": 4150
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5496765618343096e-06,
+      "logits/chosen": -2.2561728954315186,
+      "logits/rejected": -2.2816646099090576,
+      "logps/chosen": -235.952880859375,
+      "logps/rejected": -308.1616516113281,
+      "loss": 0.5897,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.049363039433956146,
+      "rewards/margins": 0.29978281259536743,
+      "rewards/rejected": -0.3491458594799042,
+      "step": 4160
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.538257687182871e-06,
+      "logits/chosen": -2.2296934127807617,
+      "logits/rejected": -2.24413800239563,
+      "logps/chosen": -312.1509094238281,
+      "logps/rejected": -329.59002685546875,
+      "loss": 0.5784,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.21270079910755157,
+      "rewards/margins": 0.3697042167186737,
+      "rewards/rejected": -0.5824050903320312,
+      "step": 4170
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.526838014133041e-06,
+      "logits/chosen": -2.300278663635254,
+      "logits/rejected": -2.1520462036132812,
+      "logps/chosen": -295.089111328125,
+      "logps/rejected": -299.8688049316406,
+      "loss": 0.6147,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.17557980120182037,
+      "rewards/margins": 0.2640655040740967,
+      "rewards/rejected": -0.43964534997940063,
+      "step": 4180
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.515417781001594e-06,
+      "logits/chosen": -2.202958583831787,
+      "logits/rejected": -2.1037344932556152,
+      "logps/chosen": -315.9857482910156,
+      "logps/rejected": -307.81396484375,
+      "loss": 0.5757,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.24192318320274353,
+      "rewards/margins": 0.34862837195396423,
+      "rewards/rejected": -0.5905515551567078,
+      "step": 4190
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.503997226116992e-06,
+      "logits/chosen": -2.2134172916412354,
+      "logits/rejected": -2.1476619243621826,
+      "logps/chosen": -292.0931091308594,
+      "logps/rejected": -327.3158874511719,
+      "loss": 0.589,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.20902717113494873,
+      "rewards/margins": 0.29169192910194397,
+      "rewards/rejected": -0.5007191300392151,
+      "step": 4200
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.4925765878144115e-06,
+      "logits/chosen": -2.272026777267456,
+      "logits/rejected": -2.180445909500122,
+      "logps/chosen": -282.70440673828125,
+      "logps/rejected": -305.32659912109375,
+      "loss": 0.5539,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.1601114571094513,
+      "rewards/margins": 0.39398443698883057,
+      "rewards/rejected": -0.5540959239006042,
+      "step": 4210
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.4811561044307727e-06,
+      "logits/chosen": -2.174419403076172,
+      "logits/rejected": -2.157695770263672,
+      "logps/chosen": -276.0654296875,
+      "logps/rejected": -327.058837890625,
+      "loss": 0.5467,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.1962500512599945,
+      "rewards/margins": 0.4340830445289612,
+      "rewards/rejected": -0.6303331255912781,
+      "step": 4220
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.469736014299758e-06,
+      "logits/chosen": -2.3431780338287354,
+      "logits/rejected": -2.2919249534606934,
+      "logps/chosen": -320.96673583984375,
+      "logps/rejected": -338.6656494140625,
+      "loss": 0.6172,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.2039066106081009,
+      "rewards/margins": 0.24904990196228027,
+      "rewards/rejected": -0.45295652747154236,
+      "step": 4230
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.458316555746846e-06,
+      "logits/chosen": -2.3326783180236816,
+      "logits/rejected": -2.273663282394409,
+      "logps/chosen": -365.83270263671875,
+      "logps/rejected": -353.1064758300781,
+      "loss": 0.5618,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.17347517609596252,
+      "rewards/margins": 0.3686824440956116,
+      "rewards/rejected": -0.5421576499938965,
+      "step": 4240
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.446897967084334e-06,
+      "logits/chosen": -2.271059274673462,
+      "logits/rejected": -2.1506505012512207,
+      "logps/chosen": -344.97119140625,
+      "logps/rejected": -309.66107177734375,
+      "loss": 0.5966,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.25669288635253906,
+      "rewards/margins": 0.31914716958999634,
+      "rewards/rejected": -0.5758401155471802,
+      "step": 4250
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.4354804866063684e-06,
+      "logits/chosen": -2.248983860015869,
+      "logits/rejected": -2.182692289352417,
+      "logps/chosen": -312.4232177734375,
+      "logps/rejected": -328.446533203125,
+      "loss": 0.5894,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.22014348208904266,
+      "rewards/margins": 0.3003430664539337,
+      "rewards/rejected": -0.5204865336418152,
+      "step": 4260
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.424064352583964e-06,
+      "logits/chosen": -2.297269344329834,
+      "logits/rejected": -2.1917881965637207,
+      "logps/chosen": -320.69378662109375,
+      "logps/rejected": -361.626220703125,
+      "loss": 0.5803,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.2521662414073944,
+      "rewards/margins": 0.34581252932548523,
+      "rewards/rejected": -0.5979788303375244,
+      "step": 4270
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.4126498032600403e-06,
+      "logits/chosen": -2.2667434215545654,
+      "logits/rejected": -2.2319869995117188,
+      "logps/chosen": -316.8856201171875,
+      "logps/rejected": -350.779052734375,
+      "loss": 0.5954,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.17394258081912994,
+      "rewards/margins": 0.3134852349758148,
+      "rewards/rejected": -0.48742786049842834,
+      "step": 4280
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.401237076844445e-06,
+      "logits/chosen": -2.2061569690704346,
+      "logits/rejected": -2.1190149784088135,
+      "logps/chosen": -285.7115173339844,
+      "logps/rejected": -270.5001525878906,
+      "loss": 0.6478,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.25898075103759766,
+      "rewards/margins": 0.17670731246471405,
+      "rewards/rejected": -0.4356880187988281,
+      "step": 4290
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.38982641150898e-06,
+      "logits/chosen": -2.128105878829956,
+      "logits/rejected": -2.07566499710083,
+      "logps/chosen": -292.49517822265625,
+      "logps/rejected": -288.88555908203125,
+      "loss": 0.5645,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.18100666999816895,
+      "rewards/margins": 0.35543978214263916,
+      "rewards/rejected": -0.5364464521408081,
+      "step": 4300
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.3784180453824414e-06,
+      "logits/chosen": -2.196662425994873,
+      "logits/rejected": -2.1289172172546387,
+      "logps/chosen": -286.9634094238281,
+      "logps/rejected": -309.788330078125,
+      "loss": 0.5923,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.18550142645835876,
+      "rewards/margins": 0.31308630108833313,
+      "rewards/rejected": -0.4985876977443695,
+      "step": 4310
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.367012216545638e-06,
+      "logits/chosen": -2.244236469268799,
+      "logits/rejected": -2.163097620010376,
+      "logps/chosen": -284.93682861328125,
+      "logps/rejected": -281.2056884765625,
+      "loss": 0.6113,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.2157663106918335,
+      "rewards/margins": 0.2603696882724762,
+      "rewards/rejected": -0.4761359691619873,
+      "step": 4320
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3556091630264294e-06,
+      "logits/chosen": -2.242006778717041,
+      "logits/rejected": -2.1157474517822266,
+      "logps/chosen": -292.1373596191406,
+      "logps/rejected": -280.36322021484375,
+      "loss": 0.5863,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.1108798012137413,
+      "rewards/margins": 0.3140495717525482,
+      "rewards/rejected": -0.4249293804168701,
+      "step": 4330
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.344209122794757e-06,
+      "logits/chosen": -2.278151512145996,
+      "logits/rejected": -2.2355151176452637,
+      "logps/chosen": -307.44207763671875,
+      "logps/rejected": -342.2320556640625,
+      "loss": 0.5696,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.1577954888343811,
+      "rewards/margins": 0.3458147644996643,
+      "rewards/rejected": -0.5036102533340454,
+      "step": 4340
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3328123337576787e-06,
+      "logits/chosen": -2.1961028575897217,
+      "logits/rejected": -2.231940507888794,
+      "logps/chosen": -270.5798645019531,
+      "logps/rejected": -328.88958740234375,
+      "loss": 0.5705,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.14266864955425262,
+      "rewards/margins": 0.35884302854537964,
+      "rewards/rejected": -0.5015116930007935,
+      "step": 4350
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3214190337544017e-06,
+      "logits/chosen": -2.3711869716644287,
+      "logits/rejected": -2.2435050010681152,
+      "logps/chosen": -292.3756103515625,
+      "logps/rejected": -303.8966369628906,
+      "loss": 0.5751,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.16369611024856567,
+      "rewards/margins": 0.3327244818210602,
+      "rewards/rejected": -0.49642062187194824,
+      "step": 4360
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.310029460551323e-06,
+      "logits/chosen": -2.2210006713867188,
+      "logits/rejected": -2.092313289642334,
+      "logps/chosen": -336.9834899902344,
+      "logps/rejected": -296.81842041015625,
+      "loss": 0.6183,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.1385176181793213,
+      "rewards/margins": 0.26691168546676636,
+      "rewards/rejected": -0.40542930364608765,
+      "step": 4370
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.2986438518370645e-06,
+      "logits/chosen": -2.3118929862976074,
+      "logits/rejected": -2.172315835952759,
+      "logps/chosen": -293.66888427734375,
+      "logps/rejected": -308.866455078125,
+      "loss": 0.5572,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.20712819695472717,
+      "rewards/margins": 0.38452520966529846,
+      "rewards/rejected": -0.5916533470153809,
+      "step": 4380
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.2872624452175123e-06,
+      "logits/chosen": -2.1337263584136963,
+      "logits/rejected": -2.096621036529541,
+      "logps/chosen": -317.05853271484375,
+      "logps/rejected": -306.6132507324219,
+      "loss": 0.5822,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.17859305441379547,
+      "rewards/margins": 0.31152716279029846,
+      "rewards/rejected": -0.4901202321052551,
+      "step": 4390
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2758854782108584e-06,
+      "logits/chosen": -2.2928977012634277,
+      "logits/rejected": -2.213534116744995,
+      "logps/chosen": -344.09814453125,
+      "logps/rejected": -357.140380859375,
+      "loss": 0.576,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.2843039333820343,
+      "rewards/margins": 0.3630077540874481,
+      "rewards/rejected": -0.6473116278648376,
+      "step": 4400
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2645131882426458e-06,
+      "logits/chosen": -2.24771785736084,
+      "logits/rejected": -2.217566728591919,
+      "logps/chosen": -296.27117919921875,
+      "logps/rejected": -318.70465087890625,
+      "loss": 0.6186,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.13453179597854614,
+      "rewards/margins": 0.254404753446579,
+      "rewards/rejected": -0.3889365792274475,
+      "step": 4410
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2531458126408154e-06,
+      "logits/chosen": -2.211513042449951,
+      "logits/rejected": -2.1245293617248535,
+      "logps/chosen": -298.334716796875,
+      "logps/rejected": -299.1788635253906,
+      "loss": 0.5585,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.22218851745128632,
+      "rewards/margins": 0.36935141682624817,
+      "rewards/rejected": -0.5915398597717285,
+      "step": 4420
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2417835886307452e-06,
+      "logits/chosen": -2.2878329753875732,
+      "logits/rejected": -2.191943645477295,
+      "logps/chosen": -295.5746154785156,
+      "logps/rejected": -303.7256774902344,
+      "loss": 0.5662,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.23160219192504883,
+      "rewards/margins": 0.3590404689311981,
+      "rewards/rejected": -0.5906426906585693,
+      "step": 4430
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2304267533303075e-06,
+      "logits/chosen": -2.198486089706421,
+      "logits/rejected": -2.1466586589813232,
+      "logps/chosen": -330.70770263671875,
+      "logps/rejected": -326.08502197265625,
+      "loss": 0.5688,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.1812635213136673,
+      "rewards/margins": 0.3645995259284973,
+      "rewards/rejected": -0.5458630323410034,
+      "step": 4440
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.219075543744918e-06,
+      "logits/chosen": -2.313459634780884,
+      "logits/rejected": -2.2000479698181152,
+      "logps/chosen": -310.2533874511719,
+      "logps/rejected": -302.20257568359375,
+      "loss": 0.6248,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.2181706428527832,
+      "rewards/margins": 0.23994271457195282,
+      "rewards/rejected": -0.45811334252357483,
+      "step": 4450
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.207730196762589e-06,
+      "logits/chosen": -2.1994261741638184,
+      "logits/rejected": -2.086202621459961,
+      "logps/chosen": -330.8420715332031,
+      "logps/rejected": -322.92034912109375,
+      "loss": 0.5502,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.2075098752975464,
+      "rewards/margins": 0.392425000667572,
+      "rewards/rejected": -0.5999349355697632,
+      "step": 4460
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.1963909491489846e-06,
+      "logits/chosen": -2.270793914794922,
+      "logits/rejected": -2.2250256538391113,
+      "logps/chosen": -316.1448059082031,
+      "logps/rejected": -365.9324035644531,
+      "loss": 0.6155,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.19968464970588684,
+      "rewards/margins": 0.2766803205013275,
+      "rewards/rejected": -0.47636494040489197,
+      "step": 4470
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.185058037542486e-06,
+      "logits/chosen": -2.3094911575317383,
+      "logits/rejected": -2.185676336288452,
+      "logps/chosen": -322.5010070800781,
+      "logps/rejected": -314.72601318359375,
+      "loss": 0.5884,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.19121722877025604,
+      "rewards/margins": 0.31124967336654663,
+      "rewards/rejected": -0.5024669170379639,
+      "step": 4480
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.173731698449244e-06,
+      "logits/chosen": -2.2886509895324707,
+      "logits/rejected": -2.221942186355591,
+      "logps/chosen": -298.2712707519531,
+      "logps/rejected": -320.44842529296875,
+      "loss": 0.5943,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.17132337391376495,
+      "rewards/margins": 0.3021024167537689,
+      "rewards/rejected": -0.4734257757663727,
+      "step": 4490
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1624121682382495e-06,
+      "logits/chosen": -2.312828302383423,
+      "logits/rejected": -2.1678576469421387,
+      "logps/chosen": -290.67218017578125,
+      "logps/rejected": -297.09759521484375,
+      "loss": 0.5425,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.18776679039001465,
+      "rewards/margins": 0.4041364789009094,
+      "rewards/rejected": -0.5919032692909241,
+      "step": 4500
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1510996831363993e-06,
+      "logits/chosen": -2.2624051570892334,
+      "logits/rejected": -2.1440157890319824,
+      "logps/chosen": -303.673583984375,
+      "logps/rejected": -276.68798828125,
+      "loss": 0.6176,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3465026021003723,
+      "rewards/margins": 0.2668396532535553,
+      "rewards/rejected": -0.61334228515625,
+      "step": 4510
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.139794479223565e-06,
+      "logits/chosen": -2.177112579345703,
+      "logits/rejected": -2.1620678901672363,
+      "logps/chosen": -357.629638671875,
+      "logps/rejected": -324.1454162597656,
+      "loss": 0.5798,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.206990048289299,
+      "rewards/margins": 0.34105998277664185,
+      "rewards/rejected": -0.548050045967102,
+      "step": 4520
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.128496792427669e-06,
+      "logits/chosen": -2.2875874042510986,
+      "logits/rejected": -2.273015260696411,
+      "logps/chosen": -324.6013488769531,
+      "logps/rejected": -359.5809326171875,
+      "loss": 0.5796,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.1470945179462433,
+      "rewards/margins": 0.33035987615585327,
+      "rewards/rejected": -0.47745442390441895,
+      "step": 4530
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.117206858519758e-06,
+      "logits/chosen": -2.3104758262634277,
+      "logits/rejected": -2.3068227767944336,
+      "logps/chosen": -288.22686767578125,
+      "logps/rejected": -338.008056640625,
+      "loss": 0.616,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.23997744917869568,
+      "rewards/margins": 0.23832137882709503,
+      "rewards/rejected": -0.47829875349998474,
+      "step": 4540
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.1059249131090844e-06,
+      "logits/chosen": -2.1803038120269775,
+      "logits/rejected": -2.1118412017822266,
+      "logps/chosen": -271.5584411621094,
+      "logps/rejected": -260.3912658691406,
+      "loss": 0.634,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.32383498549461365,
+      "rewards/margins": 0.23475320637226105,
+      "rewards/rejected": -0.5585881471633911,
+      "step": 4550
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.094651191638189e-06,
+      "logits/chosen": -2.2164502143859863,
+      "logits/rejected": -2.1599996089935303,
+      "logps/chosen": -299.9985046386719,
+      "logps/rejected": -292.871826171875,
+      "loss": 0.6067,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.25585514307022095,
+      "rewards/margins": 0.26472902297973633,
+      "rewards/rejected": -0.520584225654602,
+      "step": 4560
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0833859293779867e-06,
+      "logits/chosen": -2.2270455360412598,
+      "logits/rejected": -2.142152786254883,
+      "logps/chosen": -311.3333435058594,
+      "logps/rejected": -325.640869140625,
+      "loss": 0.6001,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.21837690472602844,
+      "rewards/margins": 0.28140920400619507,
+      "rewards/rejected": -0.4997861385345459,
+      "step": 4570
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0721293614228568e-06,
+      "logits/chosen": -2.239487409591675,
+      "logits/rejected": -2.190485715866089,
+      "logps/chosen": -271.17694091796875,
+      "logps/rejected": -347.658447265625,
+      "loss": 0.5628,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.18228408694267273,
+      "rewards/margins": 0.365995854139328,
+      "rewards/rejected": -0.548279881477356,
+      "step": 4580
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.060881722685742e-06,
+      "logits/chosen": -2.3012619018554688,
+      "logits/rejected": -2.3104777336120605,
+      "logps/chosen": -295.18853759765625,
+      "logps/rejected": -333.760986328125,
+      "loss": 0.5775,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.2056879997253418,
+      "rewards/margins": 0.32939228415489197,
+      "rewards/rejected": -0.5350803136825562,
+      "step": 4590
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.049643247893235e-06,
+      "logits/chosen": -2.270509958267212,
+      "logits/rejected": -2.230644464492798,
+      "logps/chosen": -306.8868103027344,
+      "logps/rejected": -314.2762756347656,
+      "loss": 0.5663,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.2096973955631256,
+      "rewards/margins": 0.36910349130630493,
+      "rewards/rejected": -0.5788008570671082,
+      "step": 4600
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0384141715806903e-06,
+      "logits/chosen": -2.3155465126037598,
+      "logits/rejected": -2.258345365524292,
+      "logps/chosen": -343.16064453125,
+      "logps/rejected": -335.7472229003906,
+      "loss": 0.6168,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.23217563331127167,
+      "rewards/margins": 0.23074308037757874,
+      "rewards/rejected": -0.462918758392334,
+      "step": 4610
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0271947280873255e-06,
+      "logits/chosen": -2.323875904083252,
+      "logits/rejected": -2.2460594177246094,
+      "logps/chosen": -294.6390075683594,
+      "logps/rejected": -310.1608581542969,
+      "loss": 0.5651,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.13543495535850525,
+      "rewards/margins": 0.37415453791618347,
+      "rewards/rejected": -0.5095894932746887,
+      "step": 4620
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.0159851515513302e-06,
+      "logits/chosen": -2.1863014698028564,
+      "logits/rejected": -2.179434299468994,
+      "logps/chosen": -251.57015991210938,
+      "logps/rejected": -300.84246826171875,
+      "loss": 0.6143,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.1544230431318283,
+      "rewards/margins": 0.2654711604118347,
+      "rewards/rejected": -0.4198942184448242,
+      "step": 4630
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.004785675904982e-06,
+      "logits/chosen": -2.2885677814483643,
+      "logits/rejected": -2.2793614864349365,
+      "logps/chosen": -320.08502197265625,
+      "logps/rejected": -323.58355712890625,
+      "loss": 0.5914,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.14112447202205658,
+      "rewards/margins": 0.3043881356716156,
+      "rewards/rejected": -0.445512592792511,
+      "step": 4640
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.9935965348697624e-06,
+      "logits/chosen": -2.28725266456604,
+      "logits/rejected": -2.18363094329834,
+      "logps/chosen": -309.8697814941406,
+      "logps/rejected": -277.9781799316406,
+      "loss": 0.5836,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.09367014467716217,
+      "rewards/margins": 0.32573920488357544,
+      "rewards/rejected": -0.4194093644618988,
+      "step": 4650
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.9824179619514807e-06,
+      "logits/chosen": -2.2646570205688477,
+      "logits/rejected": -2.215155839920044,
+      "logps/chosen": -325.0639953613281,
+      "logps/rejected": -358.20281982421875,
+      "loss": 0.6222,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.236891508102417,
+      "rewards/margins": 0.2578909397125244,
+      "rewards/rejected": -0.4947824478149414,
+      "step": 4660
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.9712501904354004e-06,
+      "logits/chosen": -2.2292869091033936,
+      "logits/rejected": -2.1636569499969482,
+      "logps/chosen": -292.38629150390625,
+      "logps/rejected": -309.7817687988281,
+      "loss": 0.5976,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.20154981315135956,
+      "rewards/margins": 0.2922808825969696,
+      "rewards/rejected": -0.49383068084716797,
+      "step": 4670
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.960093453381369e-06,
+      "logits/chosen": -2.2603418827056885,
+      "logits/rejected": -2.2402851581573486,
+      "logps/chosen": -299.7186584472656,
+      "logps/rejected": -327.2197570800781,
+      "loss": 0.6625,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.2904617488384247,
+      "rewards/margins": 0.16700880229473114,
+      "rewards/rejected": -0.45747050642967224,
+      "step": 4680
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.948947983618962e-06,
+      "logits/chosen": -2.2831361293792725,
+      "logits/rejected": -2.186596155166626,
+      "logps/chosen": -331.02001953125,
+      "logps/rejected": -323.8455810546875,
+      "loss": 0.5772,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.2344851940870285,
+      "rewards/margins": 0.37284547090530396,
+      "rewards/rejected": -0.6073306798934937,
+      "step": 4690
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.937814013742611e-06,
+      "logits/chosen": -2.192678928375244,
+      "logits/rejected": -2.1581597328186035,
+      "logps/chosen": -288.61614990234375,
+      "logps/rejected": -313.3251647949219,
+      "loss": 0.6163,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.24112264811992645,
+      "rewards/margins": 0.270085871219635,
+      "rewards/rejected": -0.5112084746360779,
+      "step": 4700
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.9266917761067617e-06,
+      "logits/chosen": -2.252155065536499,
+      "logits/rejected": -2.150637149810791,
+      "logps/chosen": -274.75201416015625,
+      "logps/rejected": -309.49407958984375,
+      "loss": 0.5682,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.1783822923898697,
+      "rewards/margins": 0.39890679717063904,
+      "rewards/rejected": -0.5772891044616699,
+      "step": 4710
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.915581502821017e-06,
+      "logits/chosen": -2.2457118034362793,
+      "logits/rejected": -2.202882766723633,
+      "logps/chosen": -311.6221618652344,
+      "logps/rejected": -326.28167724609375,
+      "loss": 0.6551,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.29532963037490845,
+      "rewards/margins": 0.16430024802684784,
+      "rewards/rejected": -0.4596298336982727,
+      "step": 4720
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.9044834257452997e-06,
+      "logits/chosen": -2.262570381164551,
+      "logits/rejected": -2.277740001678467,
+      "logps/chosen": -322.4426574707031,
+      "logps/rejected": -324.764892578125,
+      "loss": 0.5724,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.19570091366767883,
+      "rewards/margins": 0.3290325105190277,
+      "rewards/rejected": -0.5247333645820618,
+      "step": 4730
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.893397776485006e-06,
+      "logits/chosen": -2.195594072341919,
+      "logits/rejected": -2.083705425262451,
+      "logps/chosen": -264.0104064941406,
+      "logps/rejected": -286.9299621582031,
+      "loss": 0.5795,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.24587281048297882,
+      "rewards/margins": 0.34859025478363037,
+      "rewards/rejected": -0.594463050365448,
+      "step": 4740
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.8823247863861804e-06,
+      "logits/chosen": -2.1810431480407715,
+      "logits/rejected": -2.1117537021636963,
+      "logps/chosen": -292.3841857910156,
+      "logps/rejected": -286.7529296875,
+      "loss": 0.5537,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.17687204480171204,
+      "rewards/margins": 0.40910300612449646,
+      "rewards/rejected": -0.5859750509262085,
+      "step": 4750
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.8712646865306822e-06,
+      "logits/chosen": -2.2291183471679688,
+      "logits/rejected": -2.2378058433532715,
+      "logps/chosen": -306.16046142578125,
+      "logps/rejected": -347.13543701171875,
+      "loss": 0.5795,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.21971885859966278,
+      "rewards/margins": 0.3429061472415924,
+      "rewards/rejected": -0.562624990940094,
+      "step": 4760
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.8602177077313631e-06,
+      "logits/chosen": -2.245903253555298,
+      "logits/rejected": -2.2234671115875244,
+      "logps/chosen": -287.87542724609375,
+      "logps/rejected": -343.1726989746094,
+      "loss": 0.597,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.1752200871706009,
+      "rewards/margins": 0.31497079133987427,
+      "rewards/rejected": -0.4901908040046692,
+      "step": 4770
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8491840805272546e-06,
+      "logits/chosen": -2.2528433799743652,
+      "logits/rejected": -2.20115327835083,
+      "logps/chosen": -285.2230529785156,
+      "logps/rejected": -294.3558044433594,
+      "loss": 0.5773,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.2560577988624573,
+      "rewards/margins": 0.3361159861087799,
+      "rewards/rejected": -0.5921737551689148,
+      "step": 4780
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8381640351787516e-06,
+      "logits/chosen": -2.1432268619537354,
+      "logits/rejected": -2.0488507747650146,
+      "logps/chosen": -305.54290771484375,
+      "logps/rejected": -316.4336242675781,
+      "loss": 0.5747,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.24591200053691864,
+      "rewards/margins": 0.3427841067314148,
+      "rewards/rejected": -0.5886961221694946,
+      "step": 4790
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8271578016628122e-06,
+      "logits/chosen": -2.3278307914733887,
+      "logits/rejected": -2.226720094680786,
+      "logps/chosen": -320.807373046875,
+      "logps/rejected": -316.5043640136719,
+      "loss": 0.6094,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.24610686302185059,
+      "rewards/margins": 0.2771647572517395,
+      "rewards/rejected": -0.5232715606689453,
+      "step": 4800
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8161656096681546e-06,
+      "logits/chosen": -2.283215284347534,
+      "logits/rejected": -2.1816093921661377,
+      "logps/chosen": -301.87738037109375,
+      "logps/rejected": -300.78631591796875,
+      "loss": 0.5584,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.18695667386054993,
+      "rewards/margins": 0.4141852855682373,
+      "rewards/rejected": -0.6011419296264648,
+      "step": 4810
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8051876885904645e-06,
+      "logits/chosen": -2.160775661468506,
+      "logits/rejected": -2.090919017791748,
+      "logps/chosen": -263.7298889160156,
+      "logps/rejected": -277.83697509765625,
+      "loss": 0.6028,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.2347753793001175,
+      "rewards/margins": 0.30403420329093933,
+      "rewards/rejected": -0.538809597492218,
+      "step": 4820
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.7942242675276098e-06,
+      "logits/chosen": -2.222561836242676,
+      "logits/rejected": -2.1378979682922363,
+      "logps/chosen": -311.54071044921875,
+      "logps/rejected": -351.65911865234375,
+      "loss": 0.5569,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.2278716266155243,
+      "rewards/margins": 0.4089781641960144,
+      "rewards/rejected": -0.6368497610092163,
+      "step": 4830
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.783275575274856e-06,
+      "logits/chosen": -2.077249050140381,
+      "logits/rejected": -2.02012300491333,
+      "logps/chosen": -311.983154296875,
+      "logps/rejected": -328.8287658691406,
+      "loss": 0.541,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.2819608747959137,
+      "rewards/margins": 0.4067571759223938,
+      "rewards/rejected": -0.6887180209159851,
+      "step": 4840
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.7723418403200943e-06,
+      "logits/chosen": -2.2702884674072266,
+      "logits/rejected": -2.1757566928863525,
+      "logps/chosen": -306.5168762207031,
+      "logps/rejected": -339.5025939941406,
+      "loss": 0.5797,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.22507329285144806,
+      "rewards/margins": 0.38953521847724915,
+      "rewards/rejected": -0.6146085262298584,
+      "step": 4850
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7614232908390748e-06,
+      "logits/chosen": -2.199558973312378,
+      "logits/rejected": -2.0967612266540527,
+      "logps/chosen": -322.54193115234375,
+      "logps/rejected": -318.96319580078125,
+      "loss": 0.5785,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.31979161500930786,
+      "rewards/margins": 0.35457512736320496,
+      "rewards/rejected": -0.6743667721748352,
+      "step": 4860
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7505201546906398e-06,
+      "logits/chosen": -2.3104164600372314,
+      "logits/rejected": -2.1747078895568848,
+      "logps/chosen": -321.0475158691406,
+      "logps/rejected": -316.3406677246094,
+      "loss": 0.5938,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.24703386425971985,
+      "rewards/margins": 0.32056108117103577,
+      "rewards/rejected": -0.5675949454307556,
+      "step": 4870
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7396326594119717e-06,
+      "logits/chosen": -2.1867446899414062,
+      "logits/rejected": -2.2044179439544678,
+      "logps/chosen": -318.033447265625,
+      "logps/rejected": -376.8894958496094,
+      "loss": 0.549,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.21440622210502625,
+      "rewards/margins": 0.4122908115386963,
+      "rewards/rejected": -0.6266970038414001,
+      "step": 4880
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7287610322138449e-06,
+      "logits/chosen": -2.2811503410339355,
+      "logits/rejected": -2.2072041034698486,
+      "logps/chosen": -287.03448486328125,
+      "logps/rejected": -296.5589904785156,
+      "loss": 0.5992,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.19398990273475647,
+      "rewards/margins": 0.2939120829105377,
+      "rewards/rejected": -0.4879019856452942,
+      "step": 4890
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7179054999758817e-06,
+      "logits/chosen": -2.2527401447296143,
+      "logits/rejected": -2.2034683227539062,
+      "logps/chosen": -322.36529541015625,
+      "logps/rejected": -313.5167541503906,
+      "loss": 0.604,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.22369468212127686,
+      "rewards/margins": 0.2823079228401184,
+      "rewards/rejected": -0.50600266456604,
+      "step": 4900
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7070662892418225e-06,
+      "logits/chosen": -2.2799415588378906,
+      "logits/rejected": -2.2384002208709717,
+      "logps/chosen": -292.792236328125,
+      "logps/rejected": -323.26556396484375,
+      "loss": 0.5765,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.17639829218387604,
+      "rewards/margins": 0.34047171473503113,
+      "rewards/rejected": -0.5168699622154236,
+      "step": 4910
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.6962436262147913e-06,
+      "logits/chosen": -2.3212456703186035,
+      "logits/rejected": -2.231661319732666,
+      "logps/chosen": -346.7903137207031,
+      "logps/rejected": -369.8257751464844,
+      "loss": 0.6044,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3193613588809967,
+      "rewards/margins": 0.2776809334754944,
+      "rewards/rejected": -0.5970422625541687,
+      "step": 4920
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6854377367525814e-06,
+      "logits/chosen": -2.229032516479492,
+      "logits/rejected": -2.1569199562072754,
+      "logps/chosen": -288.3301086425781,
+      "logps/rejected": -323.98944091796875,
+      "loss": 0.5499,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.1581147015094757,
+      "rewards/margins": 0.41132253408432007,
+      "rewards/rejected": -0.5694373250007629,
+      "step": 4930
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6746488463629362e-06,
+      "logits/chosen": -2.251732349395752,
+      "logits/rejected": -2.1786773204803467,
+      "logps/chosen": -331.1020812988281,
+      "logps/rejected": -317.1491394042969,
+      "loss": 0.5869,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.12536010146141052,
+      "rewards/margins": 0.3522440493106842,
+      "rewards/rejected": -0.47760409116744995,
+      "step": 4940
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6638771801988483e-06,
+      "logits/chosen": -2.2855029106140137,
+      "logits/rejected": -2.182276964187622,
+      "logps/chosen": -339.62530517578125,
+      "logps/rejected": -338.089111328125,
+      "loss": 0.6111,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.18541690707206726,
+      "rewards/margins": 0.2663753926753998,
+      "rewards/rejected": -0.45179229974746704,
+      "step": 4950
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.653122963053857e-06,
+      "logits/chosen": -2.2749080657958984,
+      "logits/rejected": -2.250270366668701,
+      "logps/chosen": -281.3585510253906,
+      "logps/rejected": -291.1257629394531,
+      "loss": 0.5941,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.13789398968219757,
+      "rewards/margins": 0.2909575402736664,
+      "rewards/rejected": -0.42885151505470276,
+      "step": 4960
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6423864193573606e-06,
+      "logits/chosen": -2.2902145385742188,
+      "logits/rejected": -2.168369770050049,
+      "logps/chosen": -302.56756591796875,
+      "logps/rejected": -324.24639892578125,
+      "loss": 0.6049,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.179901584982872,
+      "rewards/margins": 0.27662497758865356,
+      "rewards/rejected": -0.45652657747268677,
+      "step": 4970
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6316677731699286e-06,
+      "logits/chosen": -2.2854247093200684,
+      "logits/rejected": -2.1676154136657715,
+      "logps/chosen": -315.88287353515625,
+      "logps/rejected": -309.0321350097656,
+      "loss": 0.5703,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.16615870594978333,
+      "rewards/margins": 0.34920066595077515,
+      "rewards/rejected": -0.5153594017028809,
+      "step": 4980
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6209672481786302e-06,
+      "logits/chosen": -2.227534770965576,
+      "logits/rejected": -2.1882176399230957,
+      "logps/chosen": -306.0838317871094,
+      "logps/rejected": -336.73150634765625,
+      "loss": 0.6091,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.2127208709716797,
+      "rewards/margins": 0.2690984606742859,
+      "rewards/rejected": -0.4818193316459656,
+      "step": 4990
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6102850676923616e-06,
+      "logits/chosen": -2.1546339988708496,
+      "logits/rejected": -2.2065951824188232,
+      "logps/chosen": -233.6758575439453,
+      "logps/rejected": -282.8135986328125,
+      "loss": 0.5674,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.12460102885961533,
+      "rewards/margins": 0.33533772826194763,
+      "rewards/rejected": -0.4599387049674988,
+      "step": 5000
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5996214546371888e-06,
+      "logits/chosen": -2.2153162956237793,
+      "logits/rejected": -2.1397464275360107,
+      "logps/chosen": -300.18560791015625,
+      "logps/rejected": -345.79058837890625,
+      "loss": 0.5581,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.09821708500385284,
+      "rewards/margins": 0.3663147985935211,
+      "rewards/rejected": -0.46453189849853516,
+      "step": 5010
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.588976631551697e-06,
+      "logits/chosen": -2.229888439178467,
+      "logits/rejected": -2.027146339416504,
+      "logps/chosen": -322.75372314453125,
+      "logps/rejected": -313.32061767578125,
+      "loss": 0.5698,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.17541874945163727,
+      "rewards/margins": 0.34946078062057495,
+      "rewards/rejected": -0.5248795747756958,
+      "step": 5020
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5783508205823412e-06,
+      "logits/chosen": -2.2798500061035156,
+      "logits/rejected": -2.1787333488464355,
+      "logps/chosen": -289.74658203125,
+      "logps/rejected": -309.55584716796875,
+      "loss": 0.5514,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.11370769888162613,
+      "rewards/margins": 0.42184892296791077,
+      "rewards/rejected": -0.5355566143989563,
+      "step": 5030
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5677442434788143e-06,
+      "logits/chosen": -2.241020679473877,
+      "logits/rejected": -2.1790409088134766,
+      "logps/chosen": -347.45867919921875,
+      "logps/rejected": -309.4788513183594,
+      "loss": 0.6261,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.2387397736310959,
+      "rewards/margins": 0.20377786457538605,
+      "rewards/rejected": -0.4425176680088043,
+      "step": 5040
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5571571215894181e-06,
+      "logits/chosen": -2.144336223602295,
+      "logits/rejected": -2.112370491027832,
+      "logps/chosen": -271.4165954589844,
+      "logps/rejected": -289.98236083984375,
+      "loss": 0.57,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.16046592593193054,
+      "rewards/margins": 0.3745737373828888,
+      "rewards/rejected": -0.5350396633148193,
+      "step": 5050
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5465896758564452e-06,
+      "logits/chosen": -2.269864559173584,
+      "logits/rejected": -2.20538592338562,
+      "logps/chosen": -308.1907958984375,
+      "logps/rejected": -300.5450439453125,
+      "loss": 0.6381,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.1570548266172409,
+      "rewards/margins": 0.22571484744548798,
+      "rewards/rejected": -0.3827696442604065,
+      "step": 5060
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5360421268115653e-06,
+      "logits/chosen": -2.3499755859375,
+      "logits/rejected": -2.2601470947265625,
+      "logps/chosen": -296.19677734375,
+      "logps/rejected": -309.92144775390625,
+      "loss": 0.6088,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.1885501593351364,
+      "rewards/margins": 0.2522444427013397,
+      "rewards/rejected": -0.4407946467399597,
+      "step": 5070
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5255146945712267e-06,
+      "logits/chosen": -2.318502902984619,
+      "logits/rejected": -2.317073345184326,
+      "logps/chosen": -264.9830017089844,
+      "logps/rejected": -285.3392639160156,
+      "loss": 0.5724,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.04835155978798866,
+      "rewards/margins": 0.33312511444091797,
+      "rewards/rejected": -0.38147664070129395,
+      "step": 5080
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.5150075988320594e-06,
+      "logits/chosen": -2.3264520168304443,
+      "logits/rejected": -2.158167600631714,
+      "logps/chosen": -272.5290222167969,
+      "logps/rejected": -264.8362731933594,
+      "loss": 0.5598,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.13282960653305054,
+      "rewards/margins": 0.40845727920532227,
+      "rewards/rejected": -0.5412868857383728,
+      "step": 5090
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.5045210588662929e-06,
+      "logits/chosen": -2.243901491165161,
+      "logits/rejected": -2.2242465019226074,
+      "logps/chosen": -333.20440673828125,
+      "logps/rejected": -331.9303894042969,
+      "loss": 0.6221,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.2723377048969269,
+      "rewards/margins": 0.2254316508769989,
+      "rewards/rejected": -0.497769296169281,
+      "step": 5100
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4940552935171781e-06,
+      "logits/chosen": -2.185523509979248,
+      "logits/rejected": -2.1053202152252197,
+      "logps/chosen": -306.14935302734375,
+      "logps/rejected": -293.40130615234375,
+      "loss": 0.6166,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.19349311292171478,
+      "rewards/margins": 0.28787046670913696,
+      "rewards/rejected": -0.48136359453201294,
+      "step": 5110
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.483610521194419e-06,
+      "logits/chosen": -2.1743483543395996,
+      "logits/rejected": -2.123823642730713,
+      "logps/chosen": -253.65994262695312,
+      "logps/rejected": -274.9971618652344,
+      "loss": 0.5659,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.13903601467609406,
+      "rewards/margins": 0.3621757924556732,
+      "rewards/rejected": -0.5012118220329285,
+      "step": 5120
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4731869598696226e-06,
+      "logits/chosen": -2.3371212482452393,
+      "logits/rejected": -2.2577626705169678,
+      "logps/chosen": -348.0635681152344,
+      "logps/rejected": -335.85894775390625,
+      "loss": 0.5872,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.15545479953289032,
+      "rewards/margins": 0.3256588876247406,
+      "rewards/rejected": -0.4811136722564697,
+      "step": 5130
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4627848270717387e-06,
+      "logits/chosen": -2.359225034713745,
+      "logits/rejected": -2.271225929260254,
+      "logps/chosen": -300.64215087890625,
+      "logps/rejected": -313.2225341796875,
+      "loss": 0.5655,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.1491098403930664,
+      "rewards/margins": 0.3669964671134949,
+      "rewards/rejected": -0.5161063075065613,
+      "step": 5140
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4524043398825277e-06,
+      "logits/chosen": -2.3282556533813477,
+      "logits/rejected": -2.105178117752075,
+      "logps/chosen": -325.2800598144531,
+      "logps/rejected": -251.1908416748047,
+      "loss": 0.5857,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.13584375381469727,
+      "rewards/margins": 0.3080797791481018,
+      "rewards/rejected": -0.44392356276512146,
+      "step": 5150
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4420457149320299e-06,
+      "logits/chosen": -2.3180902004241943,
+      "logits/rejected": -2.1738173961639404,
+      "logps/chosen": -325.9745178222656,
+      "logps/rejected": -324.75347900390625,
+      "loss": 0.5682,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.14252132177352905,
+      "rewards/margins": 0.35515937209129333,
+      "rewards/rejected": -0.49768075346946716,
+      "step": 5160
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.431709168394042e-06,
+      "logits/chosen": -2.2476959228515625,
+      "logits/rejected": -2.187673568725586,
+      "logps/chosen": -276.9595031738281,
+      "logps/rejected": -304.9107360839844,
+      "loss": 0.6158,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.18238958716392517,
+      "rewards/margins": 0.2684926986694336,
+      "rewards/rejected": -0.45088228583335876,
+      "step": 5170
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4213949159816059e-06,
+      "logits/chosen": -2.0993666648864746,
+      "logits/rejected": -2.034672498703003,
+      "logps/chosen": -291.9742736816406,
+      "logps/rejected": -333.3852233886719,
+      "loss": 0.6012,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.2016802579164505,
+      "rewards/margins": 0.3297204375267029,
+      "rewards/rejected": -0.531400740146637,
+      "step": 5180
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4111031729425103e-06,
+      "logits/chosen": -2.280778169631958,
+      "logits/rejected": -2.2158453464508057,
+      "logps/chosen": -312.0046081542969,
+      "logps/rejected": -329.3743591308594,
+      "loss": 0.5698,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.11512058973312378,
+      "rewards/margins": 0.3418727219104767,
+      "rewards/rejected": -0.45699334144592285,
+      "step": 5190
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4008341540547965e-06,
+      "logits/chosen": -2.2735283374786377,
+      "logits/rejected": -2.2086353302001953,
+      "logps/chosen": -275.8941650390625,
+      "logps/rejected": -257.28326416015625,
+      "loss": 0.6039,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.2713925838470459,
+      "rewards/margins": 0.27395159006118774,
+      "rewards/rejected": -0.5453441143035889,
+      "step": 5200
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.3905880736222737e-06,
+      "logits/chosen": -2.2202446460723877,
+      "logits/rejected": -2.152207851409912,
+      "logps/chosen": -274.55718994140625,
+      "logps/rejected": -284.71246337890625,
+      "loss": 0.5776,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.2355513572692871,
+      "rewards/margins": 0.34016019105911255,
+      "rewards/rejected": -0.5757116079330444,
+      "step": 5210
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.3803651454700531e-06,
+      "logits/chosen": -2.291025400161743,
+      "logits/rejected": -2.220370292663574,
+      "logps/chosen": -323.9850769042969,
+      "logps/rejected": -318.9848327636719,
+      "loss": 0.5719,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.15749236941337585,
+      "rewards/margins": 0.3263644278049469,
+      "rewards/rejected": -0.48385676741600037,
+      "step": 5220
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.3701655829400773e-06,
+      "logits/chosen": -2.2412068843841553,
+      "logits/rejected": -2.1757941246032715,
+      "logps/chosen": -282.99298095703125,
+      "logps/rejected": -296.9221496582031,
+      "loss": 0.614,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.17335820198059082,
+      "rewards/margins": 0.25958600640296936,
+      "rewards/rejected": -0.4329442083835602,
+      "step": 5230
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3599895988866756e-06,
+      "logits/chosen": -2.22521710395813,
+      "logits/rejected": -2.205357074737549,
+      "logps/chosen": -273.3722229003906,
+      "logps/rejected": -316.802001953125,
+      "loss": 0.5615,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.22788643836975098,
+      "rewards/margins": 0.37798845767974854,
+      "rewards/rejected": -0.6058748960494995,
+      "step": 5240
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3498374056721198e-06,
+      "logits/chosen": -2.236832857131958,
+      "logits/rejected": -2.1135387420654297,
+      "logps/chosen": -347.28631591796875,
+      "logps/rejected": -304.4608459472656,
+      "loss": 0.6088,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.18296435475349426,
+      "rewards/margins": 0.2971632182598114,
+      "rewards/rejected": -0.48012763261795044,
+      "step": 5250
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3397092151621883e-06,
+      "logits/chosen": -2.181856632232666,
+      "logits/rejected": -2.118957042694092,
+      "logps/chosen": -314.9497985839844,
+      "logps/rejected": -348.48602294921875,
+      "loss": 0.6031,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3234806954860687,
+      "rewards/margins": 0.2929115891456604,
+      "rewards/rejected": -0.6163923740386963,
+      "step": 5260
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3296052387217484e-06,
+      "logits/chosen": -2.3140857219696045,
+      "logits/rejected": -2.269150495529175,
+      "logps/chosen": -335.5849609375,
+      "logps/rejected": -337.6166687011719,
+      "loss": 0.5836,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.21779537200927734,
+      "rewards/margins": 0.3222483992576599,
+      "rewards/rejected": -0.5400437116622925,
+      "step": 5270
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3195256872103476e-06,
+      "logits/chosen": -2.371598720550537,
+      "logits/rejected": -2.2018089294433594,
+      "logps/chosen": -370.02215576171875,
+      "logps/rejected": -298.6385498046875,
+      "loss": 0.5612,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.17238309979438782,
+      "rewards/margins": 0.3724508583545685,
+      "rewards/rejected": -0.5448338985443115,
+      "step": 5280
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3094707709778068e-06,
+      "logits/chosen": -2.275477886199951,
+      "logits/rejected": -2.2253832817077637,
+      "logps/chosen": -335.6798400878906,
+      "logps/rejected": -341.5209655761719,
+      "loss": 0.6059,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.2835708260536194,
+      "rewards/margins": 0.2779124677181244,
+      "rewards/rejected": -0.5614833235740662,
+      "step": 5290
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.2994406998598364e-06,
+      "logits/chosen": -2.320047378540039,
+      "logits/rejected": -2.2582650184631348,
+      "logps/chosen": -311.9765930175781,
+      "logps/rejected": -303.0313415527344,
+      "loss": 0.5961,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.1751178801059723,
+      "rewards/margins": 0.2839520573616028,
+      "rewards/rejected": -0.45906996726989746,
+      "step": 5300
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.2894356831736558e-06,
+      "logits/chosen": -2.330653429031372,
+      "logits/rejected": -2.2683331966400146,
+      "logps/chosen": -347.83404541015625,
+      "logps/rejected": -331.4981994628906,
+      "loss": 0.6062,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.17227540910243988,
+      "rewards/margins": 0.2967098355293274,
+      "rewards/rejected": -0.4689852297306061,
+      "step": 5310
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2794559297136203e-06,
+      "logits/chosen": -2.1872544288635254,
+      "logits/rejected": -2.121631622314453,
+      "logps/chosen": -332.4129333496094,
+      "logps/rejected": -350.60662841796875,
+      "loss": 0.6709,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.31739237904548645,
+      "rewards/margins": 0.20196351408958435,
+      "rewards/rejected": -0.5193558931350708,
+      "step": 5320
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2695016477468724e-06,
+      "logits/chosen": -2.210880994796753,
+      "logits/rejected": -2.2127017974853516,
+      "logps/chosen": -269.5990295410156,
+      "logps/rejected": -329.0955810546875,
+      "loss": 0.541,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.14754262566566467,
+      "rewards/margins": 0.46367454528808594,
+      "rewards/rejected": -0.6112171411514282,
+      "step": 5330
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2595730450089874e-06,
+      "logits/chosen": -2.3433351516723633,
+      "logits/rejected": -2.269415855407715,
+      "logps/chosen": -343.03265380859375,
+      "logps/rejected": -328.55035400390625,
+      "loss": 0.538,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.07410895079374313,
+      "rewards/margins": 0.4277653694152832,
+      "rewards/rejected": -0.5018743276596069,
+      "step": 5340
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2496703286996433e-06,
+      "logits/chosen": -2.379992961883545,
+      "logits/rejected": -2.249950408935547,
+      "logps/chosen": -323.057373046875,
+      "logps/rejected": -333.5340270996094,
+      "loss": 0.6141,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.17264093458652496,
+      "rewards/margins": 0.26445716619491577,
+      "rewards/rejected": -0.43709808588027954,
+      "step": 5350
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2397937054782961e-06,
+      "logits/chosen": -2.3674094676971436,
+      "logits/rejected": -2.261693000793457,
+      "logps/chosen": -296.53131103515625,
+      "logps/rejected": -300.84710693359375,
+      "loss": 0.6406,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.2015656679868698,
+      "rewards/margins": 0.22162330150604248,
+      "rewards/rejected": -0.4231889843940735,
+      "step": 5360
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2299433814598635e-06,
+      "logits/chosen": -2.311593770980835,
+      "logits/rejected": -2.267444610595703,
+      "logps/chosen": -339.8316955566406,
+      "logps/rejected": -333.5635681152344,
+      "loss": 0.6398,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.2215697318315506,
+      "rewards/margins": 0.17241211235523224,
+      "rewards/rejected": -0.39398181438446045,
+      "step": 5370
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2201195622104265e-06,
+      "logits/chosen": -2.2464287281036377,
+      "logits/rejected": -2.13893723487854,
+      "logps/chosen": -310.2950744628906,
+      "logps/rejected": -321.4704895019531,
+      "loss": 0.5879,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.20835845172405243,
+      "rewards/margins": 0.3142760992050171,
+      "rewards/rejected": -0.5226346254348755,
+      "step": 5380
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.2103224527429417e-06,
+      "logits/chosen": -2.287074565887451,
+      "logits/rejected": -2.287384510040283,
+      "logps/chosen": -328.0266418457031,
+      "logps/rejected": -336.390380859375,
+      "loss": 0.6092,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.18557676672935486,
+      "rewards/margins": 0.281202107667923,
+      "rewards/rejected": -0.46677881479263306,
+      "step": 5390
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.2005522575129559e-06,
+      "logits/chosen": -2.3571369647979736,
+      "logits/rejected": -2.2702527046203613,
+      "logps/chosen": -328.0036926269531,
+      "logps/rejected": -348.81451416015625,
+      "loss": 0.5618,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.12667694687843323,
+      "rewards/margins": 0.3967793583869934,
+      "rewards/rejected": -0.5234562754631042,
+      "step": 5400
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1908091804143469e-06,
+      "logits/chosen": -2.169992208480835,
+      "logits/rejected": -2.131552219390869,
+      "logps/chosen": -289.8910827636719,
+      "logps/rejected": -305.1105651855469,
+      "loss": 0.5734,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.14728592336177826,
+      "rewards/margins": 0.3489544093608856,
+      "rewards/rejected": -0.4962403178215027,
+      "step": 5410
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1810934247750649e-06,
+      "logits/chosen": -2.2190515995025635,
+      "logits/rejected": -2.098783016204834,
+      "logps/chosen": -337.6979675292969,
+      "logps/rejected": -306.4912109375,
+      "loss": 0.6128,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.20069074630737305,
+      "rewards/margins": 0.27513587474823,
+      "rewards/rejected": -0.47582656145095825,
+      "step": 5420
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1714051933528881e-06,
+      "logits/chosen": -2.259046792984009,
+      "logits/rejected": -2.174419403076172,
+      "logps/chosen": -307.9107971191406,
+      "logps/rejected": -321.9892578125,
+      "loss": 0.5619,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.17964601516723633,
+      "rewards/margins": 0.44221797585487366,
+      "rewards/rejected": -0.6218639612197876,
+      "step": 5430
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.161744688331192e-06,
+      "logits/chosen": -2.2144017219543457,
+      "logits/rejected": -2.104426383972168,
+      "logps/chosen": -315.6021423339844,
+      "logps/rejected": -317.8662109375,
+      "loss": 0.5714,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.1677999347448349,
+      "rewards/margins": 0.3393593430519104,
+      "rewards/rejected": -0.5071592926979065,
+      "step": 5440
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.152112111314733e-06,
+      "logits/chosen": -2.2710747718811035,
+      "logits/rejected": -2.3048489093780518,
+      "logps/chosen": -310.8573913574219,
+      "logps/rejected": -316.53350830078125,
+      "loss": 0.6252,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.22738580405712128,
+      "rewards/margins": 0.23117342591285706,
+      "rewards/rejected": -0.45855921506881714,
+      "step": 5450
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.142507663325439e-06,
+      "logits/chosen": -2.2273240089416504,
+      "logits/rejected": -2.180992603302002,
+      "logps/chosen": -311.2198791503906,
+      "logps/rejected": -304.9374084472656,
+      "loss": 0.5482,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.12583717703819275,
+      "rewards/margins": 0.4427891671657562,
+      "rewards/rejected": -0.5686262845993042,
+      "step": 5460
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.132931544798211e-06,
+      "logits/chosen": -2.2862751483917236,
+      "logits/rejected": -2.169010639190674,
+      "logps/chosen": -320.9517517089844,
+      "logps/rejected": -279.1144104003906,
+      "loss": 0.5979,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.2036980837583542,
+      "rewards/margins": 0.29568424820899963,
+      "rewards/rejected": -0.499382346868515,
+      "step": 5470
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.1233839555767482e-06,
+      "logits/chosen": -2.2294743061065674,
+      "logits/rejected": -2.21972918510437,
+      "logps/chosen": -264.0962829589844,
+      "logps/rejected": -317.7840270996094,
+      "loss": 0.5615,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.1876416802406311,
+      "rewards/margins": 0.3944639563560486,
+      "rewards/rejected": -0.5821055769920349,
+      "step": 5480
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.1138650949093668e-06,
+      "logits/chosen": -2.2792954444885254,
+      "logits/rejected": -2.237781047821045,
+      "logps/chosen": -279.8136291503906,
+      "logps/rejected": -312.3643798828125,
+      "loss": 0.6221,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.22569458186626434,
+      "rewards/margins": 0.24749894440174103,
+      "rewards/rejected": -0.47319355607032776,
+      "step": 5490
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.1043751614448543e-06,
+      "logits/chosen": -2.191702365875244,
+      "logits/rejected": -2.1757311820983887,
+      "logps/chosen": -306.8429260253906,
+      "logps/rejected": -328.25262451171875,
+      "loss": 0.6302,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.2591175138950348,
+      "rewards/margins": 0.23401156067848206,
+      "rewards/rejected": -0.49312910437583923,
+      "step": 5500
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.0949143532283107e-06,
+      "logits/chosen": -2.2734391689300537,
+      "logits/rejected": -2.1344447135925293,
+      "logps/chosen": -287.9913635253906,
+      "logps/rejected": -288.32305908203125,
+      "loss": 0.5483,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.16467280685901642,
+      "rewards/margins": 0.41381826996803284,
+      "rewards/rejected": -0.5784910917282104,
+      "step": 5510
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.0854828676970275e-06,
+      "logits/chosen": -2.1989667415618896,
+      "logits/rejected": -2.140624523162842,
+      "logps/chosen": -338.8399658203125,
+      "logps/rejected": -291.35333251953125,
+      "loss": 0.6008,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.23939767479896545,
+      "rewards/margins": 0.2871435582637787,
+      "rewards/rejected": -0.5265411734580994,
+      "step": 5520
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.076080901676361e-06,
+      "logits/chosen": -2.2054896354675293,
+      "logits/rejected": -2.294689655303955,
+      "logps/chosen": -301.1555480957031,
+      "logps/rejected": -356.1802978515625,
+      "loss": 0.6367,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.27497249841690063,
+      "rewards/margins": 0.19801412522792816,
+      "rewards/rejected": -0.4729866087436676,
+      "step": 5530
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.0667086513756234e-06,
+      "logits/chosen": -2.163573980331421,
+      "logits/rejected": -2.1624486446380615,
+      "logps/chosen": -258.3441162109375,
+      "logps/rejected": -276.3139343261719,
+      "loss": 0.581,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.22899429500102997,
+      "rewards/margins": 0.32109665870666504,
+      "rewards/rejected": -0.5500909090042114,
+      "step": 5540
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0573663123839912e-06,
+      "logits/chosen": -2.2281603813171387,
+      "logits/rejected": -2.146318197250366,
+      "logps/chosen": -252.2932891845703,
+      "logps/rejected": -278.700439453125,
+      "loss": 0.5812,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.21522720158100128,
+      "rewards/margins": 0.31266722083091736,
+      "rewards/rejected": -0.5278943777084351,
+      "step": 5550
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0480540796664251e-06,
+      "logits/chosen": -2.295581817626953,
+      "logits/rejected": -2.2284538745880127,
+      "logps/chosen": -358.788818359375,
+      "logps/rejected": -363.5187072753906,
+      "loss": 0.6118,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.22466108202934265,
+      "rewards/margins": 0.2881178855895996,
+      "rewards/rejected": -0.5127789974212646,
+      "step": 5560
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0387721475595978e-06,
+      "logits/chosen": -2.3117308616638184,
+      "logits/rejected": -2.255199909210205,
+      "logps/chosen": -343.64739990234375,
+      "logps/rejected": -330.49798583984375,
+      "loss": 0.622,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.2619249224662781,
+      "rewards/margins": 0.2441365271806717,
+      "rewards/rejected": -0.5060614347457886,
+      "step": 5570
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0295207097678378e-06,
+      "logits/chosen": -2.321887969970703,
+      "logits/rejected": -2.1694681644439697,
+      "logps/chosen": -303.7414855957031,
+      "logps/rejected": -286.21697998046875,
+      "loss": 0.6026,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.19890199601650238,
+      "rewards/margins": 0.3013504445552826,
+      "rewards/rejected": -0.5002524852752686,
+      "step": 5580
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0202999593590924e-06,
+      "logits/chosen": -2.1839795112609863,
+      "logits/rejected": -2.1160731315612793,
+      "logps/chosen": -280.7303161621094,
+      "logps/rejected": -253.0699005126953,
+      "loss": 0.6294,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.24383839964866638,
+      "rewards/margins": 0.2163550853729248,
+      "rewards/rejected": -0.4601934850215912,
+      "step": 5590
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.011110088760891e-06,
+      "logits/chosen": -2.3758280277252197,
+      "logits/rejected": -2.244551181793213,
+      "logps/chosen": -311.04730224609375,
+      "logps/rejected": -292.23748779296875,
+      "loss": 0.5873,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.18764840066432953,
+      "rewards/margins": 0.3283648192882538,
+      "rewards/rejected": -0.5160131454467773,
+      "step": 5600
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0019512897563347e-06,
+      "logits/chosen": -2.1539273262023926,
+      "logits/rejected": -2.155287504196167,
+      "logps/chosen": -266.65643310546875,
+      "logps/rejected": -317.4012145996094,
+      "loss": 0.5188,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.14052417874336243,
+      "rewards/margins": 0.497483491897583,
+      "rewards/rejected": -0.638007640838623,
+      "step": 5610
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.928237534800935e-07,
+      "logits/chosen": -2.3350729942321777,
+      "logits/rejected": -2.17598295211792,
+      "logps/chosen": -313.20025634765625,
+      "logps/rejected": -303.25201416015625,
+      "loss": 0.5729,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.22488637268543243,
+      "rewards/margins": 0.347454696893692,
+      "rewards/rejected": -0.5723410844802856,
+      "step": 5620
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.837276704144174e-07,
+      "logits/chosen": -2.2017126083374023,
+      "logits/rejected": -2.249338388442993,
+      "logps/chosen": -263.5722961425781,
+      "logps/rejected": -320.7875671386719,
+      "loss": 0.6455,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.29073411226272583,
+      "rewards/margins": 0.1826065182685852,
+      "rewards/rejected": -0.47334060072898865,
+      "step": 5630
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.746632303851569e-07,
+      "logits/chosen": -2.246596574783325,
+      "logits/rejected": -2.208622455596924,
+      "logps/chosen": -319.1954040527344,
+      "logps/rejected": -301.9804382324219,
+      "loss": 0.6029,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.14036308228969574,
+      "rewards/margins": 0.2734617292881012,
+      "rewards/rejected": -0.4138248562812805,
+      "step": 5640
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.65630622557809e-07,
+      "logits/chosen": -2.1633477210998535,
+      "logits/rejected": -2.1405282020568848,
+      "logps/chosen": -287.43878173828125,
+      "logps/rejected": -305.71551513671875,
+      "loss": 0.6005,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.22302007675170898,
+      "rewards/margins": 0.29547959566116333,
+      "rewards/rejected": -0.5184996724128723,
+      "step": 5650
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.56630035433561e-07,
+      "logits/chosen": -2.3428664207458496,
+      "logits/rejected": -2.2034668922424316,
+      "logps/chosen": -312.30511474609375,
+      "logps/rejected": -310.9969177246094,
+      "loss": 0.5626,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.20427866280078888,
+      "rewards/margins": 0.35165926814079285,
+      "rewards/rejected": -0.5559378862380981,
+      "step": 5660
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.476616568453659e-07,
+      "logits/chosen": -2.236656904220581,
+      "logits/rejected": -2.1467082500457764,
+      "logps/chosen": -307.81842041015625,
+      "logps/rejected": -295.35455322265625,
+      "loss": 0.619,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.2767907679080963,
+      "rewards/margins": 0.26458436250686646,
+      "rewards/rejected": -0.5413750410079956,
+      "step": 5670
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.387256739540162e-07,
+      "logits/chosen": -2.2104694843292236,
+      "logits/rejected": -2.100861072540283,
+      "logps/chosen": -258.29876708984375,
+      "logps/rejected": -254.37704467773438,
+      "loss": 0.5961,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.1629006713628769,
+      "rewards/margins": 0.29869306087493896,
+      "rewards/rejected": -0.46159371733665466,
+      "step": 5680
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.298222732442377e-07,
+      "logits/chosen": -2.2853808403015137,
+      "logits/rejected": -2.258769989013672,
+      "logps/chosen": -291.1142883300781,
+      "logps/rejected": -339.2998352050781,
+      "loss": 0.5793,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.22501727938652039,
+      "rewards/margins": 0.332058846950531,
+      "rewards/rejected": -0.5570761561393738,
+      "step": 5690
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 9.20951640520803e-07,
+      "logits/chosen": -2.382042407989502,
+      "logits/rejected": -2.300607919692993,
+      "logps/chosen": -333.5317077636719,
+      "logps/rejected": -361.81109619140625,
+      "loss": 0.553,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.13352376222610474,
+      "rewards/margins": 0.39602312445640564,
+      "rewards/rejected": -0.529546856880188,
+      "step": 5700
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 9.121139609046484e-07,
+      "logits/chosen": -2.3873419761657715,
+      "logits/rejected": -2.246689796447754,
+      "logps/chosen": -295.42633056640625,
+      "logps/rejected": -291.67498779296875,
+      "loss": 0.5954,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.21414339542388916,
+      "rewards/margins": 0.28452545404434204,
+      "rewards/rejected": -0.4986688494682312,
+      "step": 5710
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 9.033094188290121e-07,
+      "logits/chosen": -2.345534563064575,
+      "logits/rejected": -2.2926785945892334,
+      "logps/chosen": -300.8721008300781,
+      "logps/rejected": -347.0675048828125,
+      "loss": 0.5706,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.18854479491710663,
+      "rewards/margins": 0.3862067759037018,
+      "rewards/rejected": -0.5747515559196472,
+      "step": 5720
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.945381980355889e-07,
+      "logits/chosen": -2.328420877456665,
+      "logits/rejected": -2.2627224922180176,
+      "logps/chosen": -318.6661376953125,
+      "logps/rejected": -330.42950439453125,
+      "loss": 0.6339,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.2633594572544098,
+      "rewards/margins": 0.20379576086997986,
+      "rewards/rejected": -0.4671551585197449,
+      "step": 5730
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.858004815706919e-07,
+      "logits/chosen": -2.222224712371826,
+      "logits/rejected": -2.224364757537842,
+      "logps/chosen": -259.622802734375,
+      "logps/rejected": -296.14923095703125,
+      "loss": 0.5829,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.11960119009017944,
+      "rewards/margins": 0.35548511147499084,
+      "rewards/rejected": -0.4750862717628479,
+      "step": 5740
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.77096451781432e-07,
+      "logits/chosen": -2.1508302688598633,
+      "logits/rejected": -2.1575701236724854,
+      "logps/chosen": -295.2816162109375,
+      "logps/rejected": -298.4773864746094,
+      "loss": 0.5663,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.16687360405921936,
+      "rewards/margins": 0.37276214361190796,
+      "rewards/rejected": -0.5396358370780945,
+      "step": 5750
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.684262903119165e-07,
+      "logits/chosen": -2.2715201377868652,
+      "logits/rejected": -2.1461901664733887,
+      "logps/chosen": -334.562744140625,
+      "logps/rejected": -293.47265625,
+      "loss": 0.5984,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.23921248316764832,
+      "rewards/margins": 0.2952956557273865,
+      "rewards/rejected": -0.5345081090927124,
+      "step": 5760
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.597901780994525e-07,
+      "logits/chosen": -2.140129804611206,
+      "logits/rejected": -2.0952887535095215,
+      "logps/chosen": -320.7549133300781,
+      "logps/rejected": -348.1116943359375,
+      "loss": 0.5804,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.16557027399539948,
+      "rewards/margins": 0.34092384576797485,
+      "rewards/rejected": -0.5064941048622131,
+      "step": 5770
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.511882953707773e-07,
+      "logits/chosen": -2.194216012954712,
+      "logits/rejected": -2.2444090843200684,
+      "logps/chosen": -309.2511901855469,
+      "logps/rejected": -334.2317199707031,
+      "loss": 0.6026,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.23426124453544617,
+      "rewards/margins": 0.3184870779514313,
+      "rewards/rejected": -0.5527483224868774,
+      "step": 5780
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.426208216382944e-07,
+      "logits/chosen": -2.069161891937256,
+      "logits/rejected": -2.0820701122283936,
+      "logps/chosen": -249.37881469726562,
+      "logps/rejected": -305.1940002441406,
+      "loss": 0.5999,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.19658584892749786,
+      "rewards/margins": 0.3157762289047241,
+      "rewards/rejected": -0.5123620629310608,
+      "step": 5790
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.340879356963245e-07,
+      "logits/chosen": -2.275988817214966,
+      "logits/rejected": -2.204143524169922,
+      "logps/chosen": -266.86895751953125,
+      "logps/rejected": -306.22491455078125,
+      "loss": 0.5774,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.20006637275218964,
+      "rewards/margins": 0.33580079674720764,
+      "rewards/rejected": -0.5358671545982361,
+      "step": 5800
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.255898156173777e-07,
+      "logits/chosen": -2.194227695465088,
+      "logits/rejected": -2.169670820236206,
+      "logps/chosen": -362.6389465332031,
+      "logps/rejected": -331.9048767089844,
+      "loss": 0.6589,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.358928382396698,
+      "rewards/margins": 0.15927556157112122,
+      "rewards/rejected": -0.5182039141654968,
+      "step": 5810
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.171266387484389e-07,
+      "logits/chosen": -2.19954252243042,
+      "logits/rejected": -2.181100368499756,
+      "logps/chosen": -310.19866943359375,
+      "logps/rejected": -348.6551208496094,
+      "loss": 0.6145,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.24491801857948303,
+      "rewards/margins": 0.2709752917289734,
+      "rewards/rejected": -0.5158933401107788,
+      "step": 5820
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.086985817072604e-07,
+      "logits/chosen": -2.2362565994262695,
+      "logits/rejected": -2.1750736236572266,
+      "logps/chosen": -279.22943115234375,
+      "logps/rejected": -297.84405517578125,
+      "loss": 0.5367,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.15669438242912292,
+      "rewards/margins": 0.4486146569252014,
+      "rewards/rejected": -0.605309009552002,
+      "step": 5830
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.003058203786835e-07,
+      "logits/chosen": -2.2802937030792236,
+      "logits/rejected": -2.206637144088745,
+      "logps/chosen": -301.95965576171875,
+      "logps/rejected": -281.61566162109375,
+      "loss": 0.6613,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.2820906341075897,
+      "rewards/margins": 0.1556374877691269,
+      "rewards/rejected": -0.4377281069755554,
+      "step": 5840
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.91948529910963e-07,
+      "logits/chosen": -2.3338422775268555,
+      "logits/rejected": -2.2146682739257812,
+      "logps/chosen": -379.86395263671875,
+      "logps/rejected": -342.8544006347656,
+      "loss": 0.6284,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.2647390067577362,
+      "rewards/margins": 0.25852879881858826,
+      "rewards/rejected": -0.5232678055763245,
+      "step": 5850
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.836268847121126e-07,
+      "logits/chosen": -2.2316813468933105,
+      "logits/rejected": -2.1332125663757324,
+      "logps/chosen": -321.942626953125,
+      "logps/rejected": -333.6553039550781,
+      "loss": 0.5478,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.1956327110528946,
+      "rewards/margins": 0.3980453312397003,
+      "rewards/rejected": -0.5936779975891113,
+      "step": 5860
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.753410584462681e-07,
+      "logits/chosen": -2.2441978454589844,
+      "logits/rejected": -2.209455966949463,
+      "logps/chosen": -319.6392517089844,
+      "logps/rejected": -367.57415771484375,
+      "loss": 0.5529,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.23563961684703827,
+      "rewards/margins": 0.39463010430336,
+      "rewards/rejected": -0.6302697062492371,
+      "step": 5870
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.670912240300596e-07,
+      "logits/chosen": -2.261707067489624,
+      "logits/rejected": -2.168673038482666,
+      "logps/chosen": -306.07293701171875,
+      "logps/rejected": -354.01513671875,
+      "loss": 0.6089,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.24367579817771912,
+      "rewards/margins": 0.2957931160926819,
+      "rewards/rejected": -0.5394688844680786,
+      "step": 5880
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.588775536290035e-07,
+      "logits/chosen": -2.3636014461517334,
+      "logits/rejected": -2.2855453491210938,
+      "logps/chosen": -313.52703857421875,
+      "logps/rejected": -322.87322998046875,
+      "loss": 0.5993,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.18039445579051971,
+      "rewards/margins": 0.3048645853996277,
+      "rewards/rejected": -0.4852589964866638,
+      "step": 5890
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.507002186539147e-07,
+      "logits/chosen": -2.3298370838165283,
+      "logits/rejected": -2.264897584915161,
+      "logps/chosen": -329.05426025390625,
+      "logps/rejected": -335.554443359375,
+      "loss": 0.6083,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.2601746916770935,
+      "rewards/margins": 0.29222458600997925,
+      "rewards/rejected": -0.5523992776870728,
+      "step": 5900
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.425593897573216e-07,
+      "logits/chosen": -2.230464458465576,
+      "logits/rejected": -2.0954625606536865,
+      "logps/chosen": -293.1115417480469,
+      "logps/rejected": -318.01263427734375,
+      "loss": 0.586,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.20525069534778595,
+      "rewards/margins": 0.3247843086719513,
+      "rewards/rejected": -0.5300350189208984,
+      "step": 5910
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.344552368299088e-07,
+      "logits/chosen": -2.1736879348754883,
+      "logits/rejected": -2.1197733879089355,
+      "logps/chosen": -285.8775634765625,
+      "logps/rejected": -314.8118896484375,
+      "loss": 0.6024,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.3129540681838989,
+      "rewards/margins": 0.27547574043273926,
+      "rewards/rejected": -0.5884298086166382,
+      "step": 5920
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.26387928996973e-07,
+      "logits/chosen": -2.2027297019958496,
+      "logits/rejected": -2.091571092605591,
+      "logps/chosen": -306.52398681640625,
+      "logps/rejected": -293.25689697265625,
+      "loss": 0.6152,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.2530812919139862,
+      "rewards/margins": 0.25087738037109375,
+      "rewards/rejected": -0.5039586424827576,
+      "step": 5930
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.183576346148899e-07,
+      "logits/chosen": -2.2879271507263184,
+      "logits/rejected": -2.2315640449523926,
+      "logps/chosen": -338.07135009765625,
+      "logps/rejected": -343.3145751953125,
+      "loss": 0.5948,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.2316274642944336,
+      "rewards/margins": 0.30058106780052185,
+      "rewards/rejected": -0.5322085022926331,
+      "step": 5940
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.103645212676044e-07,
+      "logits/chosen": -2.2760863304138184,
+      "logits/rejected": -2.2098329067230225,
+      "logps/chosen": -335.37030029296875,
+      "logps/rejected": -331.3144836425781,
+      "loss": 0.5758,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.167093425989151,
+      "rewards/margins": 0.3470625579357147,
+      "rewards/rejected": -0.5141559839248657,
+      "step": 5950
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.024087557631318e-07,
+      "logits/chosen": -2.3430886268615723,
+      "logits/rejected": -2.2137045860290527,
+      "logps/chosen": -315.588623046875,
+      "logps/rejected": -324.27435302734375,
+      "loss": 0.5913,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.26683956384658813,
+      "rewards/margins": 0.3276747763156891,
+      "rewards/rejected": -0.5945142507553101,
+      "step": 5960
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.944905041300739e-07,
+      "logits/chosen": -2.354771375656128,
+      "logits/rejected": -2.235982894897461,
+      "logps/chosen": -329.1302185058594,
+      "logps/rejected": -307.24176025390625,
+      "loss": 0.5805,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.13809750974178314,
+      "rewards/margins": 0.343759685754776,
+      "rewards/rejected": -0.48185721039772034,
+      "step": 5970
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.866099316141606e-07,
+      "logits/chosen": -2.2135233879089355,
+      "logits/rejected": -2.17932391166687,
+      "logps/chosen": -323.75799560546875,
+      "logps/rejected": -317.0902404785156,
+      "loss": 0.5982,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.18628045916557312,
+      "rewards/margins": 0.316842257976532,
+      "rewards/rejected": -0.5031227469444275,
+      "step": 5980
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.787672026747946e-07,
+      "logits/chosen": -2.1737163066864014,
+      "logits/rejected": -2.0860915184020996,
+      "logps/chosen": -288.33111572265625,
+      "logps/rejected": -328.392578125,
+      "loss": 0.5433,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.11049487441778183,
+      "rewards/margins": 0.43303370475769043,
+      "rewards/rejected": -0.54352867603302,
+      "step": 5990
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.709624809816223e-07,
+      "logits/chosen": -2.2856686115264893,
+      "logits/rejected": -2.143397092819214,
+      "logps/chosen": -293.72772216796875,
+      "logps/rejected": -281.6180419921875,
+      "loss": 0.6046,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.23028914630413055,
+      "rewards/margins": 0.2749117314815521,
+      "rewards/rejected": -0.5052008628845215,
+      "step": 6000
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.6319592941112e-07,
+      "logits/chosen": -2.276402711868286,
+      "logits/rejected": -2.199864149093628,
+      "logps/chosen": -315.3931579589844,
+      "logps/rejected": -298.86737060546875,
+      "loss": 0.587,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.23889532685279846,
+      "rewards/margins": 0.31959256529808044,
+      "rewards/rejected": -0.5584878921508789,
+      "step": 6010
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.554677100431927e-07,
+      "logits/chosen": -2.239607572555542,
+      "logits/rejected": -2.155582904815674,
+      "logps/chosen": -269.63873291015625,
+      "logps/rejected": -292.4139709472656,
+      "loss": 0.5285,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.15219341218471527,
+      "rewards/margins": 0.4786094129085541,
+      "rewards/rejected": -0.6308028101921082,
+      "step": 6020
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.4777798415779e-07,
+      "logits/chosen": -2.416595935821533,
+      "logits/rejected": -2.2838592529296875,
+      "logps/chosen": -318.8462829589844,
+      "logps/rejected": -312.1090393066406,
+      "loss": 0.5785,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.12302907556295395,
+      "rewards/margins": 0.3575192987918854,
+      "rewards/rejected": -0.4805483818054199,
+      "step": 6030
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.401269122315451e-07,
+      "logits/chosen": -2.225639820098877,
+      "logits/rejected": -2.2105183601379395,
+      "logps/chosen": -317.6982727050781,
+      "logps/rejected": -369.0693359375,
+      "loss": 0.6053,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.2531086504459381,
+      "rewards/margins": 0.2957480847835541,
+      "rewards/rejected": -0.5488567352294922,
+      "step": 6040
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.325146539344196e-07,
+      "logits/chosen": -2.209933280944824,
+      "logits/rejected": -2.0943663120269775,
+      "logps/chosen": -409.1436462402344,
+      "logps/rejected": -398.69842529296875,
+      "loss": 0.5813,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.2911561131477356,
+      "rewards/margins": 0.36323466897010803,
+      "rewards/rejected": -0.654390811920166,
+      "step": 6050
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.249413681263782e-07,
+      "logits/chosen": -2.232851982116699,
+      "logits/rejected": -2.155423879623413,
+      "logps/chosen": -250.1781768798828,
+      "logps/rejected": -332.4837951660156,
+      "loss": 0.5884,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.24195115268230438,
+      "rewards/margins": 0.35050472617149353,
+      "rewards/rejected": -0.5924558639526367,
+      "step": 6060
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.174072128540686e-07,
+      "logits/chosen": -2.3279008865356445,
+      "logits/rejected": -2.26503586769104,
+      "logps/chosen": -337.4497985839844,
+      "logps/rejected": -307.052734375,
+      "loss": 0.6443,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.21359160542488098,
+      "rewards/margins": 0.2172565460205078,
+      "rewards/rejected": -0.4308481812477112,
+      "step": 6070
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 6.099123453475245e-07,
+      "logits/chosen": -2.245133399963379,
+      "logits/rejected": -2.1051242351531982,
+      "logps/chosen": -337.2713623046875,
+      "logps/rejected": -332.1095275878906,
+      "loss": 0.5822,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.23662367463111877,
+      "rewards/margins": 0.3294454514980316,
+      "rewards/rejected": -0.5660691261291504,
+      "step": 6080
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 6.024569220168836e-07,
+      "logits/chosen": -2.2474308013916016,
+      "logits/rejected": -2.104712724685669,
+      "logps/chosen": -322.6059265136719,
+      "logps/rejected": -311.0595397949219,
+      "loss": 0.5344,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.11445900052785873,
+      "rewards/margins": 0.4638898968696594,
+      "rewards/rejected": -0.5783488750457764,
+      "step": 6090
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.950410984491268e-07,
+      "logits/chosen": -2.231515645980835,
+      "logits/rejected": -2.23624849319458,
+      "logps/chosen": -334.6339111328125,
+      "logps/rejected": -355.5390319824219,
+      "loss": 0.58,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.2133444845676422,
+      "rewards/margins": 0.3178350329399109,
+      "rewards/rejected": -0.5311795473098755,
+      "step": 6100
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.876650294048262e-07,
+      "logits/chosen": -2.2630536556243896,
+      "logits/rejected": -2.2580368518829346,
+      "logps/chosen": -270.99591064453125,
+      "logps/rejected": -303.1621398925781,
+      "loss": 0.6118,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.19207841157913208,
+      "rewards/margins": 0.2853357791900635,
+      "rewards/rejected": -0.47741419076919556,
+      "step": 6110
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.8032886881492e-07,
+      "logits/chosen": -2.174412250518799,
+      "logits/rejected": -2.121708393096924,
+      "logps/chosen": -249.01132202148438,
+      "logps/rejected": -254.09683227539062,
+      "loss": 0.6319,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.22126099467277527,
+      "rewards/margins": 0.22934678196907043,
+      "rewards/rejected": -0.4506077170372009,
+      "step": 6120
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.730327697774988e-07,
+      "logits/chosen": -2.2567834854125977,
+      "logits/rejected": -2.223865270614624,
+      "logps/chosen": -280.71051025390625,
+      "logps/rejected": -335.3533630371094,
+      "loss": 0.5546,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.1094474345445633,
+      "rewards/margins": 0.37788665294647217,
+      "rewards/rejected": -0.48733407258987427,
+      "step": 6130
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.657768845546068e-07,
+      "logits/chosen": -2.209315061569214,
+      "logits/rejected": -2.1145834922790527,
+      "logps/chosen": -242.4571533203125,
+      "logps/rejected": -268.93524169921875,
+      "loss": 0.5958,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.22153480350971222,
+      "rewards/margins": 0.29737162590026855,
+      "rewards/rejected": -0.5189064741134644,
+      "step": 6140
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.585613645690713e-07,
+      "logits/chosen": -2.23826003074646,
+      "logits/rejected": -2.1479594707489014,
+      "logps/chosen": -283.0222473144531,
+      "logps/rejected": -324.4845886230469,
+      "loss": 0.548,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.18606118857860565,
+      "rewards/margins": 0.42992424964904785,
+      "rewards/rejected": -0.6159854531288147,
+      "step": 6150
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.513863604013355e-07,
+      "logits/chosen": -2.3211870193481445,
+      "logits/rejected": -2.1886870861053467,
+      "logps/chosen": -334.17950439453125,
+      "logps/rejected": -308.362548828125,
+      "loss": 0.6228,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.24311676621437073,
+      "rewards/margins": 0.24683444201946259,
+      "rewards/rejected": -0.4899512231349945,
+      "step": 6160
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.442520217863215e-07,
+      "logits/chosen": -2.227276563644409,
+      "logits/rejected": -2.1782267093658447,
+      "logps/chosen": -287.66583251953125,
+      "logps/rejected": -291.90948486328125,
+      "loss": 0.6353,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.2558891177177429,
+      "rewards/margins": 0.2297544926404953,
+      "rewards/rejected": -0.48564356565475464,
+      "step": 6170
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.371584976103034e-07,
+      "logits/chosen": -2.28275990486145,
+      "logits/rejected": -2.173872470855713,
+      "logps/chosen": -303.195068359375,
+      "logps/rejected": -327.93853759765625,
+      "loss": 0.5368,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.16327349841594696,
+      "rewards/margins": 0.44478267431259155,
+      "rewards/rejected": -0.6080561876296997,
+      "step": 6180
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.301059359077987e-07,
+      "logits/chosen": -2.2108511924743652,
+      "logits/rejected": -2.1684250831604004,
+      "logps/chosen": -270.13397216796875,
+      "logps/rejected": -290.10211181640625,
+      "loss": 0.6143,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.24181067943572998,
+      "rewards/margins": 0.2641329765319824,
+      "rewards/rejected": -0.5059436559677124,
+      "step": 6190
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.230944838584806e-07,
+      "logits/chosen": -2.293006181716919,
+      "logits/rejected": -2.262521266937256,
+      "logps/chosen": -325.39898681640625,
+      "logps/rejected": -333.6467590332031,
+      "loss": 0.6232,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.1768663227558136,
+      "rewards/margins": 0.2859518826007843,
+      "rewards/rejected": -0.4628182053565979,
+      "step": 6200
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.161242877841083e-07,
+      "logits/chosen": -2.361013412475586,
+      "logits/rejected": -2.2488346099853516,
+      "logps/chosen": -310.595458984375,
+      "logps/rejected": -313.78564453125,
+      "loss": 0.5627,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.18735520541667938,
+      "rewards/margins": 0.36751627922058105,
+      "rewards/rejected": -0.5548714399337769,
+      "step": 6210
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.091954931454682e-07,
+      "logits/chosen": -2.1743760108947754,
+      "logits/rejected": -2.0993740558624268,
+      "logps/chosen": -299.3758239746094,
+      "logps/rejected": -320.70550537109375,
+      "loss": 0.5427,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.09906675666570663,
+      "rewards/margins": 0.4197840690612793,
+      "rewards/rejected": -0.5188508629798889,
+      "step": 6220
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 5.023082445393446e-07,
+      "logits/chosen": -2.222538948059082,
+      "logits/rejected": -2.1469168663024902,
+      "logps/chosen": -296.52783203125,
+      "logps/rejected": -293.89630126953125,
+      "loss": 0.531,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.14179904758930206,
+      "rewards/margins": 0.4603515565395355,
+      "rewards/rejected": -0.602150559425354,
+      "step": 6230
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.95462685695498e-07,
+      "logits/chosen": -2.299201488494873,
+      "logits/rejected": -2.2109179496765137,
+      "logps/chosen": -308.81817626953125,
+      "logps/rejected": -290.07586669921875,
+      "loss": 0.616,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.21971330046653748,
+      "rewards/margins": 0.26939094066619873,
+      "rewards/rejected": -0.4891042113304138,
+      "step": 6240
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.88658959473666e-07,
+      "logits/chosen": -2.2667698860168457,
+      "logits/rejected": -2.178948402404785,
+      "logps/chosen": -315.88116455078125,
+      "logps/rejected": -302.3547668457031,
+      "loss": 0.5547,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.15843792259693146,
+      "rewards/margins": 0.40256786346435547,
+      "rewards/rejected": -0.5610057711601257,
+      "step": 6250
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.818972078605821e-07,
+      "logits/chosen": -2.2810981273651123,
+      "logits/rejected": -2.12772274017334,
+      "logps/chosen": -371.85943603515625,
+      "logps/rejected": -299.86651611328125,
+      "loss": 0.6056,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.22580282390117645,
+      "rewards/margins": 0.3250388205051422,
+      "rewards/rejected": -0.5508416891098022,
+      "step": 6260
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.7517757196701514e-07,
+      "logits/chosen": -2.3292527198791504,
+      "logits/rejected": -2.2128889560699463,
+      "logps/chosen": -277.4884338378906,
+      "logps/rejected": -277.57928466796875,
+      "loss": 0.5827,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.23419201374053955,
+      "rewards/margins": 0.32151398062705994,
+      "rewards/rejected": -0.5557060241699219,
+      "step": 6270
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.6850019202482193e-07,
+      "logits/chosen": -2.2545182704925537,
+      "logits/rejected": -2.1212382316589355,
+      "logps/chosen": -278.05474853515625,
+      "logps/rejected": -286.8658447265625,
+      "loss": 0.6044,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.25792375206947327,
+      "rewards/margins": 0.3005824685096741,
+      "rewards/rejected": -0.5585063099861145,
+      "step": 6280
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.618652073840188e-07,
+      "logits/chosen": -2.354627847671509,
+      "logits/rejected": -2.2349581718444824,
+      "logps/chosen": -331.49822998046875,
+      "logps/rejected": -317.4371337890625,
+      "loss": 0.5817,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.17250117659568787,
+      "rewards/margins": 0.3626854121685028,
+      "rewards/rejected": -0.5351866483688354,
+      "step": 6290
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.5527275650987965e-07,
+      "logits/chosen": -2.2923061847686768,
+      "logits/rejected": -2.209237575531006,
+      "logps/chosen": -310.1547546386719,
+      "logps/rejected": -298.08636474609375,
+      "loss": 0.6077,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.25109967589378357,
+      "rewards/margins": 0.26999661326408386,
+      "rewards/rejected": -0.5210963487625122,
+      "step": 6300
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.487229769800394e-07,
+      "logits/chosen": -2.2456047534942627,
+      "logits/rejected": -2.1408562660217285,
+      "logps/chosen": -261.54779052734375,
+      "logps/rejected": -265.80511474609375,
+      "loss": 0.6225,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.24030759930610657,
+      "rewards/margins": 0.2347225695848465,
+      "rewards/rejected": -0.4750301241874695,
+      "step": 6310
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.422160054816285e-07,
+      "logits/chosen": -2.1729722023010254,
+      "logits/rejected": -2.0807228088378906,
+      "logps/chosen": -344.05810546875,
+      "logps/rejected": -352.92376708984375,
+      "loss": 0.6316,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.24153423309326172,
+      "rewards/margins": 0.21908824145793915,
+      "rewards/rejected": -0.4606224596500397,
+      "step": 6320
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.35751977808416e-07,
+      "logits/chosen": -2.284636974334717,
+      "logits/rejected": -2.201692581176758,
+      "logps/chosen": -297.3533935546875,
+      "logps/rejected": -308.90155029296875,
+      "loss": 0.5235,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.12625914812088013,
+      "rewards/margins": 0.4717814326286316,
+      "rewards/rejected": -0.5980406403541565,
+      "step": 6330
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.293310288579794e-07,
+      "logits/chosen": -2.0978596210479736,
+      "logits/rejected": -2.1035284996032715,
+      "logps/chosen": -291.7745056152344,
+      "logps/rejected": -306.15411376953125,
+      "loss": 0.6196,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.24561206996440887,
+      "rewards/margins": 0.26010841131210327,
+      "rewards/rejected": -0.5057204961776733,
+      "step": 6340
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.2295329262888733e-07,
+      "logits/chosen": -2.2204253673553467,
+      "logits/rejected": -2.11824893951416,
+      "logps/chosen": -297.88568115234375,
+      "logps/rejected": -290.4247131347656,
+      "loss": 0.6025,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.26537007093429565,
+      "rewards/margins": 0.2906087636947632,
+      "rewards/rejected": -0.5559788942337036,
+      "step": 6350
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.1661890221790316e-07,
+      "logits/chosen": -2.2358736991882324,
+      "logits/rejected": -2.2458510398864746,
+      "logps/chosen": -281.9924621582031,
+      "logps/rejected": -350.58941650390625,
+      "loss": 0.5342,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.15237683057785034,
+      "rewards/margins": 0.4450441002845764,
+      "rewards/rejected": -0.5974208116531372,
+      "step": 6360
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.103279898172072e-07,
+      "logits/chosen": -2.322620153427124,
+      "logits/rejected": -2.168433427810669,
+      "logps/chosen": -366.12786865234375,
+      "logps/rejected": -325.152587890625,
+      "loss": 0.6199,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.29785770177841187,
+      "rewards/margins": 0.26100578904151917,
+      "rewards/rejected": -0.5588634610176086,
+      "step": 6370
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.040806867116401e-07,
+      "logits/chosen": -2.250920057296753,
+      "logits/rejected": -2.2237958908081055,
+      "logps/chosen": -321.1445007324219,
+      "logps/rejected": -321.01239013671875,
+      "loss": 0.6241,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.22857800126075745,
+      "rewards/margins": 0.23127146065235138,
+      "rewards/rejected": -0.4598495066165924,
+      "step": 6380
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.978771232759615e-07,
+      "logits/chosen": -2.2364799976348877,
+      "logits/rejected": -2.212056875228882,
+      "logps/chosen": -269.04595947265625,
+      "logps/rejected": -298.14642333984375,
+      "loss": 0.5804,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.1761271059513092,
+      "rewards/margins": 0.32325392961502075,
+      "rewards/rejected": -0.49938106536865234,
+      "step": 6390
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.917174289721276e-07,
+      "logits/chosen": -2.2611424922943115,
+      "logits/rejected": -2.2784390449523926,
+      "logps/chosen": -315.00732421875,
+      "logps/rejected": -305.6526184082031,
+      "loss": 0.6236,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.2230949103832245,
+      "rewards/margins": 0.21592876315116882,
+      "rewards/rejected": -0.4390237331390381,
+      "step": 6400
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.856017323465938e-07,
+      "logits/chosen": -2.191107749938965,
+      "logits/rejected": -2.1177146434783936,
+      "logps/chosen": -274.64263916015625,
+      "logps/rejected": -298.37091064453125,
+      "loss": 0.5915,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.2880602478981018,
+      "rewards/margins": 0.30864188075065613,
+      "rewards/rejected": -0.5967021584510803,
+      "step": 6410
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.7953016102762695e-07,
+      "logits/chosen": -2.2297253608703613,
+      "logits/rejected": -2.2038657665252686,
+      "logps/chosen": -305.0720520019531,
+      "logps/rejected": -304.5875549316406,
+      "loss": 0.6349,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.19702813029289246,
+      "rewards/margins": 0.25031501054763794,
+      "rewards/rejected": -0.4473431706428528,
+      "step": 6420
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.7350284172264493e-07,
+      "logits/chosen": -2.242567300796509,
+      "logits/rejected": -2.2194294929504395,
+      "logps/chosen": -260.15185546875,
+      "logps/rejected": -306.4600524902344,
+      "loss": 0.6072,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.2087719887495041,
+      "rewards/margins": 0.27035030722618103,
+      "rewards/rejected": -0.4791223108768463,
+      "step": 6430
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.67519900215573e-07,
+      "logits/chosen": -2.237037420272827,
+      "logits/rejected": -2.1576311588287354,
+      "logps/chosen": -295.35089111328125,
+      "logps/rejected": -292.58917236328125,
+      "loss": 0.5995,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.20834775269031525,
+      "rewards/margins": 0.3026237487792969,
+      "rewards/rejected": -0.5109714865684509,
+      "step": 6440
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.615814613642174e-07,
+      "logits/chosen": -2.2904200553894043,
+      "logits/rejected": -2.1129608154296875,
+      "logps/chosen": -343.78662109375,
+      "logps/rejected": -301.4750671386719,
+      "loss": 0.6,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.22187814116477966,
+      "rewards/margins": 0.28146690130233765,
+      "rewards/rejected": -0.5033451318740845,
+      "step": 6450
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.5568764909765795e-07,
+      "logits/chosen": -2.2265584468841553,
+      "logits/rejected": -2.1241002082824707,
+      "logps/chosen": -330.27655029296875,
+      "logps/rejected": -335.1883239746094,
+      "loss": 0.5565,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.16477003693580627,
+      "rewards/margins": 0.3972473740577698,
+      "rewards/rejected": -0.5620173811912537,
+      "step": 6460
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.498385864136672e-07,
+      "logits/chosen": -2.2943968772888184,
+      "logits/rejected": -2.254265308380127,
+      "logps/chosen": -307.8959655761719,
+      "logps/rejected": -323.3202819824219,
+      "loss": 0.6178,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.2675125002861023,
+      "rewards/margins": 0.26192277669906616,
+      "rewards/rejected": -0.5294352769851685,
+      "step": 6470
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.440343953761363e-07,
+      "logits/chosen": -2.2510826587677,
+      "logits/rejected": -2.151731014251709,
+      "logps/chosen": -297.1509094238281,
+      "logps/rejected": -293.01715087890625,
+      "loss": 0.6359,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.2728116512298584,
+      "rewards/margins": 0.22913222014904022,
+      "rewards/rejected": -0.5019438862800598,
+      "step": 6480
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.382751971125345e-07,
+      "logits/chosen": -2.225860834121704,
+      "logits/rejected": -2.175529956817627,
+      "logps/chosen": -287.80706787109375,
+      "logps/rejected": -287.97802734375,
+      "loss": 0.6297,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.24230583012104034,
+      "rewards/margins": 0.23965951800346375,
+      "rewards/rejected": -0.4819653630256653,
+      "step": 6490
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.3256111181137753e-07,
+      "logits/chosen": -2.3095192909240723,
+      "logits/rejected": -2.2175674438476562,
+      "logps/chosen": -286.6315612792969,
+      "logps/rejected": -287.605224609375,
+      "loss": 0.5935,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.18542449176311493,
+      "rewards/margins": 0.2997186779975891,
+      "rewards/rejected": -0.48514318466186523,
+      "step": 6500
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.2689225871971905e-07,
+      "logits/chosen": -2.213831663131714,
+      "logits/rejected": -2.195160150527954,
+      "logps/chosen": -316.28851318359375,
+      "logps/rejected": -342.26446533203125,
+      "loss": 0.6135,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.23785743117332458,
+      "rewards/margins": 0.2976071238517761,
+      "rewards/rejected": -0.5354645848274231,
+      "step": 6510
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.2126875614066523e-07,
+      "logits/chosen": -2.273104190826416,
+      "logits/rejected": -2.214599609375,
+      "logps/chosen": -287.70855712890625,
+      "logps/rejected": -285.56036376953125,
+      "loss": 0.6276,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.2149469405412674,
+      "rewards/margins": 0.21491274237632751,
+      "rewards/rejected": -0.4298597276210785,
+      "step": 6520
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.156907214309024e-07,
+      "logits/chosen": -2.1992926597595215,
+      "logits/rejected": -2.141411066055298,
+      "logps/chosen": -274.2710876464844,
+      "logps/rejected": -306.6649169921875,
+      "loss": 0.5484,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.1834583729505539,
+      "rewards/margins": 0.38944798707962036,
+      "rewards/rejected": -0.5729063749313354,
+      "step": 6530
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 3.1015827099824923e-07,
+      "logits/chosen": -2.3543479442596436,
+      "logits/rejected": -2.218076229095459,
+      "logps/chosen": -315.0438537597656,
+      "logps/rejected": -313.69757080078125,
+      "loss": 0.5693,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.19667963683605194,
+      "rewards/margins": 0.34850531816482544,
+      "rewards/rejected": -0.5451849102973938,
+      "step": 6540
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 3.0467152029922926e-07,
+      "logits/chosen": -2.1841235160827637,
+      "logits/rejected": -2.1445069313049316,
+      "logps/chosen": -273.4917907714844,
+      "logps/rejected": -334.92669677734375,
+      "loss": 0.6072,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.2173459827899933,
+      "rewards/margins": 0.28793925046920776,
+      "rewards/rejected": -0.5052852630615234,
+      "step": 6550
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.992305838366591e-07,
+      "logits/chosen": -2.3001112937927246,
+      "logits/rejected": -2.259284496307373,
+      "logps/chosen": -362.2890930175781,
+      "logps/rejected": -344.4498596191406,
+      "loss": 0.5852,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.22456571459770203,
+      "rewards/margins": 0.3367841839790344,
+      "rewards/rejected": -0.5613499283790588,
+      "step": 6560
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.938355751572583e-07,
+      "logits/chosen": -2.2215077877044678,
+      "logits/rejected": -2.122424364089966,
+      "logps/chosen": -305.6630859375,
+      "logps/rejected": -292.48687744140625,
+      "loss": 0.6135,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.2404574155807495,
+      "rewards/margins": 0.25619426369667053,
+      "rewards/rejected": -0.49665170907974243,
+      "step": 6570
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.8848660684928307e-07,
+      "logits/chosen": -2.212378978729248,
+      "logits/rejected": -2.1459293365478516,
+      "logps/chosen": -328.45721435546875,
+      "logps/rejected": -319.005126953125,
+      "loss": 0.5897,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.24658489227294922,
+      "rewards/margins": 0.32633015513420105,
+      "rewards/rejected": -0.5729150772094727,
+      "step": 6580
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.8318379054017383e-07,
+      "logits/chosen": -2.2369682788848877,
+      "logits/rejected": -2.1657543182373047,
+      "logps/chosen": -320.33563232421875,
+      "logps/rejected": -298.6103820800781,
+      "loss": 0.5521,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.17626380920410156,
+      "rewards/margins": 0.4276823103427887,
+      "rewards/rejected": -0.6039462089538574,
+      "step": 6590
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.779272368942246e-07,
+      "logits/chosen": -2.3483896255493164,
+      "logits/rejected": -2.2070584297180176,
+      "logps/chosen": -327.89691162109375,
+      "logps/rejected": -339.7986755371094,
+      "loss": 0.599,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.2045535296201706,
+      "rewards/margins": 0.3108772337436676,
+      "rewards/rejected": -0.5154308080673218,
+      "step": 6600
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.7271705561027986e-07,
+      "logits/chosen": -2.2898526191711426,
+      "logits/rejected": -2.179412603378296,
+      "logps/chosen": -350.47845458984375,
+      "logps/rejected": -316.5538330078125,
+      "loss": 0.5658,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.1713074892759323,
+      "rewards/margins": 0.39373883605003357,
+      "rewards/rejected": -0.5650463104248047,
+      "step": 6610
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.6755335541943677e-07,
+      "logits/chosen": -2.2905704975128174,
+      "logits/rejected": -2.2754604816436768,
+      "logps/chosen": -309.1745300292969,
+      "logps/rejected": -333.67938232421875,
+      "loss": 0.6222,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.283588171005249,
+      "rewards/margins": 0.24496212601661682,
+      "rewards/rejected": -0.5285503268241882,
+      "step": 6620
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.62436244082781e-07,
+      "logits/chosen": -2.350175142288208,
+      "logits/rejected": -2.3214735984802246,
+      "logps/chosen": -322.21685791015625,
+      "logps/rejected": -337.0072326660156,
+      "loss": 0.5259,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.15177544951438904,
+      "rewards/margins": 0.4634989798069,
+      "rewards/rejected": -0.6152744293212891,
+      "step": 6630
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.5736582838913836e-07,
+      "logits/chosen": -2.2806878089904785,
+      "logits/rejected": -2.1992342472076416,
+      "logps/chosen": -307.2468566894531,
+      "logps/rejected": -306.5870666503906,
+      "loss": 0.5171,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.12974882125854492,
+      "rewards/margins": 0.5071049928665161,
+      "rewards/rejected": -0.6368538737297058,
+      "step": 6640
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.5234221415284363e-07,
+      "logits/chosen": -2.2767059803009033,
+      "logits/rejected": -2.210958957672119,
+      "logps/chosen": -314.1791076660156,
+      "logps/rejected": -352.72564697265625,
+      "loss": 0.5694,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.1622014045715332,
+      "rewards/margins": 0.35927143692970276,
+      "rewards/rejected": -0.5214728713035583,
+      "step": 6650
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.4736550621153375e-07,
+      "logits/chosen": -2.2079639434814453,
+      "logits/rejected": -2.1122167110443115,
+      "logps/chosen": -277.1203308105469,
+      "logps/rejected": -299.25885009765625,
+      "loss": 0.5394,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.2556740939617157,
+      "rewards/margins": 0.44593340158462524,
+      "rewards/rejected": -0.7016074657440186,
+      "step": 6660
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.424358084239609e-07,
+      "logits/chosen": -2.2848687171936035,
+      "logits/rejected": -2.2923455238342285,
+      "logps/chosen": -297.78179931640625,
+      "logps/rejected": -319.9772033691406,
+      "loss": 0.6469,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.23676368594169617,
+      "rewards/margins": 0.20353002846240997,
+      "rewards/rejected": -0.44029369950294495,
+      "step": 6670
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.3755322366782158e-07,
+      "logits/chosen": -2.2076756954193115,
+      "logits/rejected": -2.1373071670532227,
+      "logps/chosen": -310.51495361328125,
+      "logps/rejected": -308.01654052734375,
+      "loss": 0.5738,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.16560694575309753,
+      "rewards/margins": 0.35412055253982544,
+      "rewards/rejected": -0.5197274684906006,
+      "step": 6680
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.3271785383761431e-07,
+      "logits/chosen": -2.2977707386016846,
+      "logits/rejected": -2.253223419189453,
+      "logps/chosen": -306.7425231933594,
+      "logps/rejected": -299.0722351074219,
+      "loss": 0.6451,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.2154873162508011,
+      "rewards/margins": 0.17152908444404602,
+      "rewards/rejected": -0.3870163559913635,
+      "step": 6690
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.2792979984250978e-07,
+      "logits/chosen": -2.304884672164917,
+      "logits/rejected": -2.1841373443603516,
+      "logps/chosen": -325.85693359375,
+      "logps/rejected": -301.67694091796875,
+      "loss": 0.6013,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.16651174426078796,
+      "rewards/margins": 0.27157658338546753,
+      "rewards/rejected": -0.4380883276462555,
+      "step": 6700
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.231891616042453e-07,
+      "logits/chosen": -2.3625378608703613,
+      "logits/rejected": -2.2966275215148926,
+      "logps/chosen": -317.78643798828125,
+      "logps/rejected": -343.6407470703125,
+      "loss": 0.558,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.21526435017585754,
+      "rewards/margins": 0.3885418772697449,
+      "rewards/rejected": -0.60380619764328,
+      "step": 6710
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.1849603805504328e-07,
+      "logits/chosen": -2.3497023582458496,
+      "logits/rejected": -2.25719952583313,
+      "logps/chosen": -319.1162109375,
+      "logps/rejected": -331.0961608886719,
+      "loss": 0.5468,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.13552118837833405,
+      "rewards/margins": 0.41164129972457886,
+      "rewards/rejected": -0.5471624732017517,
+      "step": 6720
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.1385052713554066e-07,
+      "logits/chosen": -2.1155714988708496,
+      "logits/rejected": -2.0738768577575684,
+      "logps/chosen": -286.7259521484375,
+      "logps/rejected": -323.79388427734375,
+      "loss": 0.5892,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.22641244530677795,
+      "rewards/margins": 0.31793564558029175,
+      "rewards/rejected": -0.5443481206893921,
+      "step": 6730
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.0925272579274873e-07,
+      "logits/chosen": -2.2623276710510254,
+      "logits/rejected": -2.278806686401367,
+      "logps/chosen": -297.44622802734375,
+      "logps/rejected": -334.06915283203125,
+      "loss": 0.5437,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.14218921959400177,
+      "rewards/margins": 0.4180937707424164,
+      "rewards/rejected": -0.5602830052375793,
+      "step": 6740
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.047027299780302e-07,
+      "logits/chosen": -2.2847766876220703,
+      "logits/rejected": -2.237086534500122,
+      "logps/chosen": -337.4761657714844,
+      "logps/rejected": -318.3897399902344,
+      "loss": 0.6143,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.17803582549095154,
+      "rewards/margins": 0.24196699261665344,
+      "rewards/rejected": -0.42000284790992737,
+      "step": 6750
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.0020063464509492e-07,
+      "logits/chosen": -2.1941092014312744,
+      "logits/rejected": -2.1769931316375732,
+      "logps/chosen": -299.5726013183594,
+      "logps/rejected": -310.5265197753906,
+      "loss": 0.5557,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.15381093323230743,
+      "rewards/margins": 0.3809550404548645,
+      "rewards/rejected": -0.534765899181366,
+      "step": 6760
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.957465337480191e-07,
+      "logits/chosen": -2.1837964057922363,
+      "logits/rejected": -2.1292548179626465,
+      "logps/chosen": -345.9893798828125,
+      "logps/rejected": -344.40460205078125,
+      "loss": 0.59,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.2072792500257492,
+      "rewards/margins": 0.3229365646839142,
+      "rewards/rejected": -0.530215859413147,
+      "step": 6770
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.9134052023928622e-07,
+      "logits/chosen": -2.2238945960998535,
+      "logits/rejected": -2.1773924827575684,
+      "logps/chosen": -284.38330078125,
+      "logps/rejected": -311.63232421875,
+      "loss": 0.6043,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.22153577208518982,
+      "rewards/margins": 0.26465216279029846,
+      "rewards/rejected": -0.4861879348754883,
+      "step": 6780
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.8698268606784392e-07,
+      "logits/chosen": -2.234797239303589,
+      "logits/rejected": -2.236541986465454,
+      "logps/chosen": -274.5113220214844,
+      "logps/rejected": -309.7568359375,
+      "loss": 0.5625,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.20977351069450378,
+      "rewards/margins": 0.41913795471191406,
+      "rewards/rejected": -0.6289114952087402,
+      "step": 6790
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.826731221771866e-07,
+      "logits/chosen": -2.2946181297302246,
+      "logits/rejected": -2.1501200199127197,
+      "logps/chosen": -328.7485046386719,
+      "logps/rejected": -303.0035095214844,
+      "loss": 0.5848,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.22563433647155762,
+      "rewards/margins": 0.35804134607315063,
+      "rewards/rejected": -0.583675742149353,
+      "step": 6800
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.7841191850345967e-07,
+      "logits/chosen": -2.17826509475708,
+      "logits/rejected": -2.160841464996338,
+      "logps/chosen": -316.8827819824219,
+      "logps/rejected": -346.2579650878906,
+      "loss": 0.5618,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.17746450006961823,
+      "rewards/margins": 0.3708667457103729,
+      "rewards/rejected": -0.5483312606811523,
+      "step": 6810
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.7419916397357905e-07,
+      "logits/chosen": -2.2968761920928955,
+      "logits/rejected": -2.1889562606811523,
+      "logps/chosen": -332.37384033203125,
+      "logps/rejected": -314.2474670410156,
+      "loss": 0.6169,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.252440482378006,
+      "rewards/margins": 0.2617506682872772,
+      "rewards/rejected": -0.5141911506652832,
+      "step": 6820
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.700349465033782e-07,
+      "logits/chosen": -2.2410268783569336,
+      "logits/rejected": -2.1834325790405273,
+      "logps/chosen": -312.6735534667969,
+      "logps/rejected": -321.71875,
+      "loss": 0.5946,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.23761789500713348,
+      "rewards/margins": 0.30250900983810425,
+      "rewards/rejected": -0.5401269197463989,
+      "step": 6830
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.6591935299577227e-07,
+      "logits/chosen": -2.190650224685669,
+      "logits/rejected": -2.1460845470428467,
+      "logps/chosen": -296.0951232910156,
+      "logps/rejected": -275.2724609375,
+      "loss": 0.5734,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.18607425689697266,
+      "rewards/margins": 0.3766199052333832,
+      "rewards/rejected": -0.5626941919326782,
+      "step": 6840
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.6185246933894338e-07,
+      "logits/chosen": -2.3041493892669678,
+      "logits/rejected": -2.2566075325012207,
+      "logps/chosen": -303.43121337890625,
+      "logps/rejected": -318.13983154296875,
+      "loss": 0.6156,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.20772568881511688,
+      "rewards/margins": 0.28159254789352417,
+      "rewards/rejected": -0.48931822180747986,
+      "step": 6850
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.5783438040455097e-07,
+      "logits/chosen": -2.304779529571533,
+      "logits/rejected": -2.19102144241333,
+      "logps/chosen": -327.00958251953125,
+      "logps/rejected": -284.0123596191406,
+      "loss": 0.5961,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.2807047367095947,
+      "rewards/margins": 0.28873246908187866,
+      "rewards/rejected": -0.5694371461868286,
+      "step": 6860
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.538651700459576e-07,
+      "logits/chosen": -2.248351573944092,
+      "logits/rejected": -2.1588175296783447,
+      "logps/chosen": -336.6203918457031,
+      "logps/rejected": -411.03387451171875,
+      "loss": 0.5578,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.15604709088802338,
+      "rewards/margins": 0.36909565329551697,
+      "rewards/rejected": -0.5251427888870239,
+      "step": 6870
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.4994492109648151e-07,
+      "logits/chosen": -2.207669973373413,
+      "logits/rejected": -2.1326980590820312,
+      "logps/chosen": -282.05950927734375,
+      "logps/rejected": -282.82220458984375,
+      "loss": 0.569,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.2813078761100769,
+      "rewards/margins": 0.37337321043014526,
+      "rewards/rejected": -0.6546810865402222,
+      "step": 6880
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.4607371536766695e-07,
+      "logits/chosen": -2.261211395263672,
+      "logits/rejected": -2.1927568912506104,
+      "logps/chosen": -261.5356750488281,
+      "logps/rejected": -281.796630859375,
+      "loss": 0.5787,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.2290767878293991,
+      "rewards/margins": 0.351195752620697,
+      "rewards/rejected": -0.5802726149559021,
+      "step": 6890
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.4225163364757655e-07,
+      "logits/chosen": -2.213453769683838,
+      "logits/rejected": -2.0844531059265137,
+      "logps/chosen": -322.1564636230469,
+      "logps/rejected": -316.3904724121094,
+      "loss": 0.6334,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.307486891746521,
+      "rewards/margins": 0.20499345660209656,
+      "rewards/rejected": -0.5124803781509399,
+      "step": 6900
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.3847875569910462e-07,
+      "logits/chosen": -2.2584121227264404,
+      "logits/rejected": -2.168458938598633,
+      "logps/chosen": -301.62115478515625,
+      "logps/rejected": -313.1435852050781,
+      "loss": 0.585,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.16885143518447876,
+      "rewards/margins": 0.32763198018074036,
+      "rewards/rejected": -0.4964834749698639,
+      "step": 6910
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.3475516025831552e-07,
+      "logits/chosen": -2.291088819503784,
+      "logits/rejected": -2.2780730724334717,
+      "logps/chosen": -296.73297119140625,
+      "logps/rejected": -327.2522888183594,
+      "loss": 0.5713,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.1908951699733734,
+      "rewards/margins": 0.3621283173561096,
+      "rewards/rejected": -0.5530234575271606,
+      "step": 6920
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.310809250327974e-07,
+      "logits/chosen": -2.0948023796081543,
+      "logits/rejected": -2.055344581604004,
+      "logps/chosen": -280.61175537109375,
+      "logps/rejected": -291.4289245605469,
+      "loss": 0.5726,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.19594351947307587,
+      "rewards/margins": 0.3705883324146271,
+      "rewards/rejected": -0.5665318369865417,
+      "step": 6930
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.2745612670004153e-07,
+      "logits/chosen": -2.2199532985687256,
+      "logits/rejected": -2.161428928375244,
+      "logps/chosen": -286.59930419921875,
+      "logps/rejected": -384.32720947265625,
+      "loss": 0.5807,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.18531376123428345,
+      "rewards/margins": 0.3441842794418335,
+      "rewards/rejected": -0.5294980406761169,
+      "step": 6940
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.2388084090584395e-07,
+      "logits/chosen": -2.2351741790771484,
+      "logits/rejected": -2.1834254264831543,
+      "logps/chosen": -285.2957458496094,
+      "logps/rejected": -326.4273986816406,
+      "loss": 0.6217,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.3242362439632416,
+      "rewards/margins": 0.27158844470977783,
+      "rewards/rejected": -0.5958247780799866,
+      "step": 6950
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.2035514226272305e-07,
+      "logits/chosen": -2.201180934906006,
+      "logits/rejected": -2.1012425422668457,
+      "logps/chosen": -273.3121032714844,
+      "logps/rejected": -305.42120361328125,
+      "loss": 0.579,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.20711271464824677,
+      "rewards/margins": 0.35867878794670105,
+      "rewards/rejected": -0.5657914876937866,
+      "step": 6960
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.1687910434836607e-07,
+      "logits/chosen": -2.195547580718994,
+      "logits/rejected": -2.1885972023010254,
+      "logps/chosen": -290.4573669433594,
+      "logps/rejected": -315.23175048828125,
+      "loss": 0.5884,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.23133409023284912,
+      "rewards/margins": 0.3339126706123352,
+      "rewards/rejected": -0.5652468204498291,
+      "step": 6970
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.1345279970409128e-07,
+      "logits/chosen": -2.1727585792541504,
+      "logits/rejected": -2.179666519165039,
+      "logps/chosen": -260.8642883300781,
+      "logps/rejected": -298.676513671875,
+      "loss": 0.652,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.23396344482898712,
+      "rewards/margins": 0.17201301455497742,
+      "rewards/rejected": -0.40597644448280334,
+      "step": 6980
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.1007629983333629e-07,
+      "logits/chosen": -2.186631679534912,
+      "logits/rejected": -2.111750364303589,
+      "logps/chosen": -292.9834289550781,
+      "logps/rejected": -299.646240234375,
+      "loss": 0.5754,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.18006864190101624,
+      "rewards/margins": 0.340657502412796,
+      "rewards/rejected": -0.5207261443138123,
+      "step": 6990
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 1.067496752001626e-07,
+      "logits/chosen": -2.21193790435791,
+      "logits/rejected": -2.147188663482666,
+      "logps/chosen": -309.00604248046875,
+      "logps/rejected": -298.1776123046875,
+      "loss": 0.5796,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.2621694505214691,
+      "rewards/margins": 0.3532871603965759,
+      "rewards/rejected": -0.6154566407203674,
+      "step": 7000
+    },
+    {
+      "epoch": 0.92,
+      "eval_logits/chosen": -2.2260730266571045,
+      "eval_logits/rejected": -2.1487460136413574,
+      "eval_logps/chosen": -313.4948425292969,
+      "eval_logps/rejected": -320.43585205078125,
+      "eval_loss": 0.5940207242965698,
+      "eval_rewards/accuracies": 0.6940000057220459,
+      "eval_rewards/chosen": -0.21513542532920837,
+      "eval_rewards/margins": 0.30914872884750366,
+      "eval_rewards/rejected": -0.5242841243743896,
+      "eval_runtime": 1172.4889,
+      "eval_samples_per_second": 1.706,
+      "eval_steps_per_second": 0.213,
+      "step": 7000
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 1.0347299522778909e-07,
+      "logits/chosen": -2.280367374420166,
+      "logits/rejected": -2.2171788215637207,
+      "logps/chosen": -338.8251953125,
+      "logps/rejected": -333.30853271484375,
+      "loss": 0.557,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.21220973134040833,
+      "rewards/margins": 0.3927744925022125,
+      "rewards/rejected": -0.6049842238426208,
+      "step": 7010
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 1.0024632829713971e-07,
+      "logits/chosen": -2.219359874725342,
+      "logits/rejected": -2.2551379203796387,
+      "logps/chosen": -278.66119384765625,
+      "logps/rejected": -319.22418212890625,
+      "loss": 0.6177,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.2532063126564026,
+      "rewards/margins": 0.2454725205898285,
+      "rewards/rejected": -0.49867886304855347,
+      "step": 7020
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 9.706974174541889e-08,
+      "logits/chosen": -2.2002689838409424,
+      "logits/rejected": -2.1216237545013428,
+      "logps/chosen": -328.0563049316406,
+      "logps/rejected": -313.7506103515625,
+      "loss": 0.5491,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.1435789167881012,
+      "rewards/margins": 0.4156878888607025,
+      "rewards/rejected": -0.5592668652534485,
+      "step": 7030
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 9.39433018647043e-08,
+      "logits/chosen": -2.3666234016418457,
+      "logits/rejected": -2.3080554008483887,
+      "logps/chosen": -304.62506103515625,
+      "logps/rejected": -350.1346435546875,
+      "loss": 0.5554,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.13446173071861267,
+      "rewards/margins": 0.3920600116252899,
+      "rewards/rejected": -0.5265216827392578,
+      "step": 7040
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 9.086707390056543e-08,
+      "logits/chosen": -2.3087592124938965,
+      "logits/rejected": -2.227196216583252,
+      "logps/chosen": -320.02374267578125,
+      "logps/rejected": -315.66668701171875,
+      "loss": 0.6072,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.18638381361961365,
+      "rewards/margins": 0.2937110960483551,
+      "rewards/rejected": -0.48009490966796875,
+      "step": 7050
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.784112205070083e-08,
+      "logits/chosen": -2.2482821941375732,
+      "logits/rejected": -2.247669219970703,
+      "logps/chosen": -281.7313537597656,
+      "logps/rejected": -324.09564208984375,
+      "loss": 0.6207,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.22902318835258484,
+      "rewards/margins": 0.2263072431087494,
+      "rewards/rejected": -0.45533037185668945,
+      "step": 7060
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 8.486550946359779e-08,
+      "logits/chosen": -2.293630599975586,
+      "logits/rejected": -2.1961753368377686,
+      "logps/chosen": -329.4664611816406,
+      "logps/rejected": -332.4908447265625,
+      "loss": 0.6724,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.2469293624162674,
+      "rewards/margins": 0.12247602641582489,
+      "rewards/rejected": -0.3694053888320923,
+      "step": 7070
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 8.194029823721556e-08,
+      "logits/chosen": -2.2293028831481934,
+      "logits/rejected": -2.113828659057617,
+      "logps/chosen": -293.24163818359375,
+      "logps/rejected": -302.811279296875,
+      "loss": 0.5748,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.19101731479167938,
+      "rewards/margins": 0.3309341371059418,
+      "rewards/rejected": -0.5219514966011047,
+      "step": 7080
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.906554941768896e-08,
+      "logits/chosen": -2.2355294227600098,
+      "logits/rejected": -2.204115390777588,
+      "logps/chosen": -277.62774658203125,
+      "logps/rejected": -302.0038146972656,
+      "loss": 0.6123,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.2697092890739441,
+      "rewards/margins": 0.25456854701042175,
+      "rewards/rejected": -0.5242778062820435,
+      "step": 7090
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.624132299805575e-08,
+      "logits/chosen": -2.2374064922332764,
+      "logits/rejected": -2.2581026554107666,
+      "logps/chosen": -345.123046875,
+      "logps/rejected": -358.58941650390625,
+      "loss": 0.553,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.20718857645988464,
+      "rewards/margins": 0.4173831045627594,
+      "rewards/rejected": -0.624571681022644,
+      "step": 7100
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.346767791700127e-08,
+      "logits/chosen": -2.214167833328247,
+      "logits/rejected": -2.1587700843811035,
+      "logps/chosen": -344.684814453125,
+      "logps/rejected": -337.58135986328125,
+      "loss": 0.5272,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.1494571715593338,
+      "rewards/margins": 0.465663343667984,
+      "rewards/rejected": -0.615120530128479,
+      "step": 7110
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.07446720576327e-08,
+      "logits/chosen": -2.194397449493408,
+      "logits/rejected": -2.1149685382843018,
+      "logps/chosen": -282.08489990234375,
+      "logps/rejected": -291.74896240234375,
+      "loss": 0.6077,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.27410179376602173,
+      "rewards/margins": 0.2866399884223938,
+      "rewards/rejected": -0.5607417821884155,
+      "step": 7120
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.807236224626701e-08,
+      "logits/chosen": -2.2754781246185303,
+      "logits/rejected": -2.2205283641815186,
+      "logps/chosen": -315.891845703125,
+      "logps/rejected": -316.5805969238281,
+      "loss": 0.6225,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.20273521542549133,
+      "rewards/margins": 0.2261861264705658,
+      "rewards/rejected": -0.42892131209373474,
+      "step": 7130
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.545080425124888e-08,
+      "logits/chosen": -2.2018957138061523,
+      "logits/rejected": -2.1935436725616455,
+      "logps/chosen": -297.48065185546875,
+      "logps/rejected": -327.8349609375,
+      "loss": 0.6065,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.18633002042770386,
+      "rewards/margins": 0.31622928380966187,
+      "rewards/rejected": -0.5025593042373657,
+      "step": 7140
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.288005278178382e-08,
+      "logits/chosen": -2.3038198947906494,
+      "logits/rejected": -2.2294774055480957,
+      "logps/chosen": -314.5107421875,
+      "logps/rejected": -312.3115234375,
+      "loss": 0.6334,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.17556893825531006,
+      "rewards/margins": 0.23685109615325928,
+      "rewards/rejected": -0.41242003440856934,
+      "step": 7150
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.036016148679825e-08,
+      "logits/chosen": -2.3269429206848145,
+      "logits/rejected": -2.221116065979004,
+      "logps/chosen": -311.7928161621094,
+      "logps/rejected": -280.10498046875,
+      "loss": 0.5777,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.2013367861509323,
+      "rewards/margins": 0.3149644136428833,
+      "rewards/rejected": -0.5163012742996216,
+      "step": 7160
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.7891182953819235e-08,
+      "logits/chosen": -2.309523105621338,
+      "logits/rejected": -2.2230565547943115,
+      "logps/chosen": -264.33099365234375,
+      "logps/rejected": -320.1153869628906,
+      "loss": 0.5605,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.16378042101860046,
+      "rewards/margins": 0.37534573674201965,
+      "rewards/rejected": -0.5391262173652649,
+      "step": 7170
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.547316870787689e-08,
+      "logits/chosen": -2.3665127754211426,
+      "logits/rejected": -2.2079923152923584,
+      "logps/chosen": -327.3066101074219,
+      "logps/rejected": -321.58447265625,
+      "loss": 0.5633,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.20733241736888885,
+      "rewards/margins": 0.3998747169971466,
+      "rewards/rejected": -0.607207179069519,
+      "step": 7180
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.310616921042927e-08,
+      "logits/chosen": -2.3350491523742676,
+      "logits/rejected": -2.193040370941162,
+      "logps/chosen": -350.3499450683594,
+      "logps/rejected": -335.13385009765625,
+      "loss": 0.5699,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.2007473260164261,
+      "rewards/margins": 0.3822272717952728,
+      "rewards/rejected": -0.5829745531082153,
+      "step": 7190
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.079023385830939e-08,
+      "logits/chosen": -2.1871137619018555,
+      "logits/rejected": -2.092442274093628,
+      "logps/chosen": -355.10369873046875,
+      "logps/rejected": -324.44525146484375,
+      "loss": 0.5941,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.27507293224334717,
+      "rewards/margins": 0.32030484080314636,
+      "rewards/rejected": -0.5953778028488159,
+      "step": 7200
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 4.8525410982695476e-08,
+      "logits/chosen": -2.265181541442871,
+      "logits/rejected": -2.1195521354675293,
+      "logps/chosen": -343.2086486816406,
+      "logps/rejected": -350.94854736328125,
+      "loss": 0.5739,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.15476097166538239,
+      "rewards/margins": 0.37012749910354614,
+      "rewards/rejected": -0.5248885154724121,
+      "step": 7210
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 4.6311747848099e-08,
+      "logits/chosen": -2.2479965686798096,
+      "logits/rejected": -2.113255023956299,
+      "logps/chosen": -281.4270324707031,
+      "logps/rejected": -274.7620544433594,
+      "loss": 0.6068,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.237498477101326,
+      "rewards/margins": 0.29350370168685913,
+      "rewards/rejected": -0.5310021638870239,
+      "step": 7220
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.4149290651382405e-08,
+      "logits/chosen": -2.2208240032196045,
+      "logits/rejected": -2.2298367023468018,
+      "logps/chosen": -249.2593994140625,
+      "logps/rejected": -297.3638916015625,
+      "loss": 0.6534,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.23134097456932068,
+      "rewards/margins": 0.19426584243774414,
+      "rewards/rejected": -0.4256068170070648,
+      "step": 7230
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.203808452079211e-08,
+      "logits/chosen": -2.2425343990325928,
+      "logits/rejected": -2.193664073944092,
+      "logps/chosen": -315.44464111328125,
+      "logps/rejected": -329.19805908203125,
+      "loss": 0.5699,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.1318148672580719,
+      "rewards/margins": 0.3500699996948242,
+      "rewards/rejected": -0.48188480734825134,
+      "step": 7240
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.9978173515018427e-08,
+      "logits/chosen": -2.3699262142181396,
+      "logits/rejected": -2.2451012134552,
+      "logps/chosen": -357.103271484375,
+      "logps/rejected": -330.6025695800781,
+      "loss": 0.5719,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.20406512916088104,
+      "rewards/margins": 0.3894132673740387,
+      "rewards/rejected": -0.5934784412384033,
+      "step": 7250
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.7969600622274614e-08,
+      "logits/chosen": -2.3619580268859863,
+      "logits/rejected": -2.3217098712921143,
+      "logps/chosen": -355.99786376953125,
+      "logps/rejected": -307.1109924316406,
+      "loss": 0.5836,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.1927039921283722,
+      "rewards/margins": 0.3221895396709442,
+      "rewards/rejected": -0.5148935317993164,
+      "step": 7260
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.601240775940151e-08,
+      "logits/chosen": -2.2095894813537598,
+      "logits/rejected": -2.203479051589966,
+      "logps/chosen": -285.00537109375,
+      "logps/rejected": -307.1541748046875,
+      "loss": 0.5907,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.22207343578338623,
+      "rewards/margins": 0.3408282399177551,
+      "rewards/rejected": -0.5629016160964966,
+      "step": 7270
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.410663577099071e-08,
+      "logits/chosen": -2.352631092071533,
+      "logits/rejected": -2.2568843364715576,
+      "logps/chosen": -367.5334167480469,
+      "logps/rejected": -351.14337158203125,
+      "loss": 0.6094,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.2421077936887741,
+      "rewards/margins": 0.2984387278556824,
+      "rewards/rejected": -0.5405465364456177,
+      "step": 7280
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.2252324428534986e-08,
+      "logits/chosen": -2.25034499168396,
+      "logits/rejected": -2.159763813018799,
+      "logps/chosen": -340.13385009765625,
+      "logps/rejected": -316.416259765625,
+      "loss": 0.5969,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.12953811883926392,
+      "rewards/margins": 0.31046968698501587,
+      "rewards/rejected": -0.4400077760219574,
+      "step": 7290
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.0449512429594486e-08,
+      "logits/chosen": -2.182349681854248,
+      "logits/rejected": -2.095487356185913,
+      "logps/chosen": -312.82342529296875,
+      "logps/rejected": -324.8750915527344,
+      "loss": 0.5346,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.13655495643615723,
+      "rewards/margins": 0.45154494047164917,
+      "rewards/rejected": -0.5880998373031616,
+      "step": 7300
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.8698237396992956e-08,
+      "logits/chosen": -2.3066210746765137,
+      "logits/rejected": -2.314364194869995,
+      "logps/chosen": -305.68328857421875,
+      "logps/rejected": -333.3356018066406,
+      "loss": 0.5677,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.22517363727092743,
+      "rewards/margins": 0.3729446232318878,
+      "rewards/rejected": -0.598118245601654,
+      "step": 7310
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.6998535878030584e-08,
+      "logits/chosen": -2.2450592517852783,
+      "logits/rejected": -2.1315126419067383,
+      "logps/chosen": -239.87503051757812,
+      "logps/rejected": -227.39389038085938,
+      "loss": 0.5835,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.17774894833564758,
+      "rewards/margins": 0.33111661672592163,
+      "rewards/rejected": -0.5088655352592468,
+      "step": 7320
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.535044334372072e-08,
+      "logits/chosen": -2.214465618133545,
+      "logits/rejected": -2.114131212234497,
+      "logps/chosen": -373.511962890625,
+      "logps/rejected": -330.72393798828125,
+      "loss": 0.5968,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.21916863322257996,
+      "rewards/margins": 0.3310352861881256,
+      "rewards/rejected": -0.5502039194107056,
+      "step": 7330
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.3753994188051853e-08,
+      "logits/chosen": -2.2956297397613525,
+      "logits/rejected": -2.1181273460388184,
+      "logps/chosen": -329.48443603515625,
+      "logps/rejected": -315.2552795410156,
+      "loss": 0.6199,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.2757745385169983,
+      "rewards/margins": 0.2727423906326294,
+      "rewards/rejected": -0.5485169291496277,
+      "step": 7340
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.220922172726764e-08,
+      "logits/chosen": -2.2476680278778076,
+      "logits/rejected": -2.197944164276123,
+      "logps/chosen": -306.67291259765625,
+      "logps/rejected": -296.61871337890625,
+      "loss": 0.6074,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.19910797476768494,
+      "rewards/margins": 0.2877027094364166,
+      "rewards/rejected": -0.4868106245994568,
+      "step": 7350
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.071615819917244e-08,
+      "logits/chosen": -2.3216476440429688,
+      "logits/rejected": -2.3249900341033936,
+      "logps/chosen": -312.84942626953125,
+      "logps/rejected": -339.9403076171875,
+      "loss": 0.5956,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.19821664690971375,
+      "rewards/margins": 0.271074503660202,
+      "rewards/rejected": -0.4692911207675934,
+      "step": 7360
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.9274834762459393e-08,
+      "logits/chosen": -2.2404820919036865,
+      "logits/rejected": -2.132887363433838,
+      "logps/chosen": -333.1466064453125,
+      "logps/rejected": -303.07623291015625,
+      "loss": 0.5646,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.22520729899406433,
+      "rewards/margins": 0.3837920129299164,
+      "rewards/rejected": -0.6089993715286255,
+      "step": 7370
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.7885281496058947e-08,
+      "logits/chosen": -2.348447322845459,
+      "logits/rejected": -2.2594656944274902,
+      "logps/chosen": -327.52490234375,
+      "logps/rejected": -389.01995849609375,
+      "loss": 0.5819,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.22132794559001923,
+      "rewards/margins": 0.32333776354789734,
+      "rewards/rejected": -0.5446656942367554,
+      "step": 7380
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.654752739851134e-08,
+      "logits/chosen": -2.352313280105591,
+      "logits/rejected": -2.190316677093506,
+      "logps/chosen": -320.6366271972656,
+      "logps/rejected": -305.8868408203125,
+      "loss": 0.5708,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.18807712197303772,
+      "rewards/margins": 0.3853645324707031,
+      "rewards/rejected": -0.5734416842460632,
+      "step": 7390
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.526160038736235e-08,
+      "logits/chosen": -2.3725545406341553,
+      "logits/rejected": -2.2139534950256348,
+      "logps/chosen": -362.9305419921875,
+      "logps/rejected": -325.1018371582031,
+      "loss": 0.5774,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.149748295545578,
+      "rewards/margins": 0.3587859570980072,
+      "rewards/rejected": -0.5085342526435852,
+      "step": 7400
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.402752729857959e-08,
+      "logits/chosen": -2.2219464778900146,
+      "logits/rejected": -2.208585262298584,
+      "logps/chosen": -288.1156005859375,
+      "logps/rejected": -335.0963439941406,
+      "loss": 0.5739,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.15629379451274872,
+      "rewards/margins": 0.34737879037857056,
+      "rewards/rejected": -0.5036724805831909,
+      "step": 7410
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.2845333885992683e-08,
+      "logits/chosen": -2.2705419063568115,
+      "logits/rejected": -2.1903064250946045,
+      "logps/chosen": -314.6650390625,
+      "logps/rejected": -300.46478271484375,
+      "loss": 0.5884,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.1653841882944107,
+      "rewards/margins": 0.3278418183326721,
+      "rewards/rejected": -0.49322596192359924,
+      "step": 7420
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.171504482075675e-08,
+      "logits/chosen": -2.343583583831787,
+      "logits/rejected": -2.254533290863037,
+      "logps/chosen": -351.45062255859375,
+      "logps/rejected": -369.24066162109375,
+      "loss": 0.5625,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.1713610142469406,
+      "rewards/margins": 0.397853285074234,
+      "rewards/rejected": -0.5692143440246582,
+      "step": 7430
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.0636683690836147e-08,
+      "logits/chosen": -2.3275885581970215,
+      "logits/rejected": -2.2382407188415527,
+      "logps/chosen": -296.9913024902344,
+      "logps/rejected": -306.6391906738281,
+      "loss": 0.5994,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.17596189677715302,
+      "rewards/margins": 0.29039308428764343,
+      "rewards/rejected": -0.46635502576828003,
+      "step": 7440
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 9.610273000513203e-09,
+      "logits/chosen": -2.3288352489471436,
+      "logits/rejected": -2.228902578353882,
+      "logps/chosen": -351.21826171875,
+      "logps/rejected": -350.92034912109375,
+      "loss": 0.6418,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.2834011912345886,
+      "rewards/margins": 0.19712185859680176,
+      "rewards/rejected": -0.480523020029068,
+      "step": 7450
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.635834169918312e-09,
+      "logits/chosen": -2.24247407913208,
+      "logits/rejected": -2.2252285480499268,
+      "logps/chosen": -343.86529541015625,
+      "logps/rejected": -359.72381591796875,
+      "loss": 0.6262,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.2396010458469391,
+      "rewards/margins": 0.2377980500459671,
+      "rewards/rejected": -0.4773990511894226,
+      "step": 7460
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 7.713387534582506e-09,
+      "logits/chosen": -2.22129487991333,
+      "logits/rejected": -2.1426587104797363,
+      "logps/chosen": -288.47857666015625,
+      "logps/rejected": -334.2923889160156,
+      "loss": 0.5616,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.15719589591026306,
+      "rewards/margins": 0.3799470365047455,
+      "rewards/rejected": -0.5371429920196533,
+      "step": 7470
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 6.84295234501392e-09,
+      "logits/chosen": -2.261748790740967,
+      "logits/rejected": -2.1660752296447754,
+      "logps/chosen": -312.7343444824219,
+      "logps/rejected": -301.38897705078125,
+      "loss": 0.6069,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.220415398478508,
+      "rewards/margins": 0.27050694823265076,
+      "rewards/rejected": -0.49092239141464233,
+      "step": 7480
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 6.024546766295325e-09,
+      "logits/chosen": -2.2253947257995605,
+      "logits/rejected": -2.0965373516082764,
+      "logps/chosen": -341.40618896484375,
+      "logps/rejected": -308.0964050292969,
+      "loss": 0.5498,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.17060282826423645,
+      "rewards/margins": 0.4203481078147888,
+      "rewards/rejected": -0.5909509658813477,
+      "step": 7490
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 5.2581878777049895e-09,
+      "logits/chosen": -2.297600030899048,
+      "logits/rejected": -2.2106003761291504,
+      "logps/chosen": -320.5754089355469,
+      "logps/rejected": -340.50433349609375,
+      "loss": 0.5427,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.16367055475711823,
+      "rewards/margins": 0.4177732467651367,
+      "rewards/rejected": -0.5814438462257385,
+      "step": 7500
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 4.543891672361411e-09,
+      "logits/chosen": -2.223428726196289,
+      "logits/rejected": -2.1965279579162598,
+      "logps/chosen": -319.80853271484375,
+      "logps/rejected": -334.6192321777344,
+      "loss": 0.6028,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.21012039482593536,
+      "rewards/margins": 0.2811013162136078,
+      "rewards/rejected": -0.49122172594070435,
+      "step": 7510
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 3.881673056887747e-09,
+      "logits/chosen": -2.3167202472686768,
+      "logits/rejected": -2.2619433403015137,
+      "logps/chosen": -325.6050720214844,
+      "logps/rejected": -329.0150451660156,
+      "loss": 0.5812,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.18179884552955627,
+      "rewards/margins": 0.3695027232170105,
+      "rewards/rejected": -0.5513015985488892,
+      "step": 7520
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.2715458511023425e-09,
+      "logits/chosen": -2.24223256111145,
+      "logits/rejected": -2.1757986545562744,
+      "logps/chosen": -318.0364685058594,
+      "logps/rejected": -343.64984130859375,
+      "loss": 0.6106,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.2036343812942505,
+      "rewards/margins": 0.2790573239326477,
+      "rewards/rejected": -0.48269176483154297,
+      "step": 7530
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 2.7135227877289617e-09,
+      "logits/chosen": -2.2901463508605957,
+      "logits/rejected": -2.2843945026397705,
+      "logps/chosen": -294.41400146484375,
+      "logps/rejected": -338.856689453125,
+      "loss": 0.5917,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.15333044528961182,
+      "rewards/margins": 0.30261334776878357,
+      "rewards/rejected": -0.4559438228607178,
+      "step": 7540
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 2.2076155121328326e-09,
+      "logits/chosen": -2.155514717102051,
+      "logits/rejected": -2.0566611289978027,
+      "logps/chosen": -282.373046875,
+      "logps/rejected": -292.1627502441406,
+      "loss": 0.5583,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.16147629916667938,
+      "rewards/margins": 0.39047032594680786,
+      "rewards/rejected": -0.5519465804100037,
+      "step": 7550
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.7538345820755641e-09,
+      "logits/chosen": -2.2953784465789795,
+      "logits/rejected": -2.1645030975341797,
+      "logps/chosen": -347.6931457519531,
+      "logps/rejected": -315.5379638671875,
+      "loss": 0.5725,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.19179323315620422,
+      "rewards/margins": 0.33790022134780884,
+      "rewards/rejected": -0.5296934843063354,
+      "step": 7560
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.3521894674961567e-09,
+      "logits/chosen": -2.23071551322937,
+      "logits/rejected": -2.198906421661377,
+      "logps/chosen": -329.29095458984375,
+      "logps/rejected": -332.01507568359375,
+      "loss": 0.6182,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.2797185778617859,
+      "rewards/margins": 0.24416124820709229,
+      "rewards/rejected": -0.5238798260688782,
+      "step": 7570
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.0026885503131023e-09,
+      "logits/chosen": -2.2943203449249268,
+      "logits/rejected": -2.2038662433624268,
+      "logps/chosen": -325.7265930175781,
+      "logps/rejected": -290.61712646484375,
+      "loss": 0.5624,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.28593507409095764,
+      "rewards/margins": 0.3805255591869354,
+      "rewards/rejected": -0.6664606332778931,
+      "step": 7580
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.053391242492491e-10,
+      "logits/chosen": -2.140082836151123,
+      "logits/rejected": -2.0679380893707275,
+      "logps/chosen": -318.735107421875,
+      "logps/rejected": -327.1845397949219,
+      "loss": 0.6501,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3730572760105133,
+      "rewards/margins": 0.20047850906848907,
+      "rewards/rejected": -0.5735357999801636,
+      "step": 7590
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.6014739467997725e-10,
+      "logits/chosen": -2.221858501434326,
+      "logits/rejected": -2.188326358795166,
+      "logps/chosen": -307.19915771484375,
+      "logps/rejected": -311.16961669921875,
+      "loss": 0.6322,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.2230357676744461,
+      "rewards/margins": 0.23147812485694885,
+      "rewards/rejected": -0.45451393723487854,
+      "step": 7600
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.671184785033032e-10,
+      "logits/chosen": -2.3816375732421875,
+      "logits/rejected": -2.2962653636932373,
+      "logps/chosen": -336.73284912109375,
+      "logps/rejected": -351.965576171875,
+      "loss": 0.5929,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.20788922905921936,
+      "rewards/margins": 0.30553072690963745,
+      "rewards/rejected": -0.5134199857711792,
+      "step": 7610
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.2625640403302054e-10,
+      "logits/chosen": -2.318190813064575,
+      "logits/rejected": -2.2393438816070557,
+      "logps/chosen": -313.6557922363281,
+      "logps/rejected": -356.33367919921875,
+      "loss": 0.5491,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.13495388627052307,
+      "rewards/margins": 0.42312994599342346,
+      "rewards/rejected": -0.5580838322639465,
+      "step": 7620
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 3.756411091515588e-11,
+      "logits/chosen": -2.2948296070098877,
+      "logits/rejected": -2.2274951934814453,
+      "logps/chosen": -351.2091064453125,
+      "logps/rejected": -353.2593688964844,
+      "loss": 0.5941,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.17898276448249817,
+      "rewards/margins": 0.32828378677368164,
+      "rewards/rejected": -0.5072665810585022,
+      "step": 7630
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.0434500657963143e-12,
+      "logits/chosen": -2.1720404624938965,
+      "logits/rejected": -2.13431715965271,
+      "logps/chosen": -302.1468505859375,
+      "logps/rejected": -350.88897705078125,
+      "loss": 0.5555,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.21987417340278625,
+      "rewards/margins": 0.3745797574520111,
+      "rewards/rejected": -0.5944539308547974,
+      "step": 7640
+    },
+    {
+      "epoch": 1.0,
+      "step": 7642,
+      "total_flos": 0.0,
+      "train_loss": 0.6084560017191406,
+      "train_runtime": 59476.7244,
+      "train_samples_per_second": 1.028,
+      "train_steps_per_second": 0.128
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 7642,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}