Training in progress, step 1726, checkpoint

last-checkpoint/global_step1726/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab8153b1f364ce8143dd8ca6a91c02d9b732a7f246e193af8418363fb39c323e
+size 24090788996

last-checkpoint/global_step1726/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c33f67b21fac802a7b9f40879e4213877fdadc621faf76516815fbfb899b0182
+size 24090788996

last-checkpoint/global_step1726/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e7ad1f2e6b5d52de51c9142c253e4504b14cb96ba450e1d677982e176eb40e6
+size 24090788996

last-checkpoint/global_step1726/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3d4e1f3390a9b43c817745dc7aacca53a7f480440f86642baacca43ef2b6cbf
+size 24090788996

last-checkpoint/global_step1726/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db3e44efac1a79db42ada5e189c9066027261b7fb0012fcb570344c3f83140e7
+size 150693

last-checkpoint/global_step1726/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0503a2119d3a7637d504d6bee881b7a97f77742d4a2e4dcc7a27a1dd1b027a5
+size 150693

last-checkpoint/global_step1726/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:361b7e8e42f99ecbc4792177905ee85e8dffc028b9a1b65f86bf348be14d0039
+size 150693

last-checkpoint/global_step1726/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd50bc52fa7d9aa5ccff56846315efd88d3dbf255e3f23d479db9461ebdfd302
+size 150693

last-checkpoint/latest

@@ -1 +1 @@
-global_step1500
+global_step1726

last-checkpoint/model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:99de85720481c98cc093f3faf5805a4ff05d5df419d49b8575ed63ce236d5815
+oid sha256:d06d89588d2c5a6b7c30a35a96b4705ea4a256222a1999601f01382cead91d16
 size 4976698672

last-checkpoint/model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:368f761161ebad7292a8dbdeca4656fb602262d1f2495446f32f49896062f7dc
+oid sha256:195f937f5574012744471b0d6769d312a0af52820de2cb76093f28e08f193898
 size 4999802720

last-checkpoint/model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:306982a5a2f0fc8003fdb3eebf34d6850d83379bebc04fbe40d7a6bb9f8b6a5c
+oid sha256:590d0c8a9ae18db231102e3d95cf9330b9873b37d3e0d44bb2af348b5368d23d
 size 4915916176

last-checkpoint/model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fa2c5d55d66df705380544c1b8076cf199a0a6e6da3583e847a01a69fbf8edb4
+oid sha256:b71c6b037e8207917d33c1313e1bd133b2aed3cfcf9087dc27c9683c5ff38c99
 size 1168138808

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b4f003069486a57c6ac033f30cf4c4213eb6b7d659bab68a5a50fdb8da7c4118
+oid sha256:b8f22ced19e790cc864cefe3b7c711d9ae631c44f95d42fb4829688cc3de0153
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a016ef89b4392d083b2c15a7cf06a39bc61a759f648cf6dc03f1c32b89a526aa
+oid sha256:3e0407513eba77d34cbf3adf0e59a58bd80716f4f00f414854253637e82be43d
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9b56fe0893036dc052d18d90feba4328b90ea71561942150b07406ac3d7a700e
+oid sha256:6060636c023258ce9b965e244b8a58b4c99d5784dde4405b39737550ef50cd4f
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c0c203d12c2c308dab785ed672c9ca27fb6a2f72acd1e1552d1516c7b0006013
+oid sha256:c24ccdfdcde39cb2265c82c50c36ffdfcc670f757aba4bcf4bb0fdc6d1373c4c
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b0eed8fec4102664205a804b0fbc28ba65f44e3fb811cdaf695f0e9321c6fe0b
+oid sha256:b56a0f1c3322e3323ab0de90511453e2d705194cbbc2d4c04dd46c593fd07065
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.7372421281216068,
+  "epoch": 1.9989866087585957,
   "eval_steps": 100,
-  "global_step": 1500,
+  "global_step": 1726,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -11497,6 +11497,1733 @@
       "eval_samples_per_second": 4.338,
       "eval_steps_per_second": 1.085,
       "step": 1500
+    },
+    {
+      "epoch": 1.7395584509591024,
+      "grad_norm": 55.81693817141226,
+      "learning_rate": 9.220091139554887e-09,
+      "logits/chosen": -1.1932331323623657,
+      "logits/rejected": -1.1756948232650757,
+      "logps/chosen": -103.76750183105469,
+      "logps/rejected": -129.5191650390625,
+      "loss": 0.4117,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.2089885175228119,
+      "rewards/margins": 0.9426325559616089,
+      "rewards/rejected": -1.1516211032867432,
+      "step": 1502
+    },
+    {
+      "epoch": 1.741874773796598,
+      "grad_norm": 62.56378010964504,
+      "learning_rate": 9.05870280610117e-09,
+      "logits/chosen": -1.2296499013900757,
+      "logits/rejected": -1.1992714405059814,
+      "logps/chosen": -123.05607604980469,
+      "logps/rejected": -130.20932006835938,
+      "loss": 0.4251,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.41981515288352966,
+      "rewards/margins": 0.8822442293167114,
+      "rewards/rejected": -1.3020594120025635,
+      "step": 1504
+    },
+    {
+      "epoch": 1.7441910966340934,
+      "grad_norm": 81.02596584931305,
+      "learning_rate": 8.898672408511553e-09,
+      "logits/chosen": -1.2401373386383057,
+      "logits/rejected": -1.17184317111969,
+      "logps/chosen": -168.21986389160156,
+      "logps/rejected": -162.71383666992188,
+      "loss": 0.4402,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.5058491230010986,
+      "rewards/margins": 1.3156105279922485,
+      "rewards/rejected": -1.8214595317840576,
+      "step": 1506
+    },
+    {
+      "epoch": 1.7465074194715888,
+      "grad_norm": 56.282435640432055,
+      "learning_rate": 8.740002336360686e-09,
+      "logits/chosen": -1.351073980331421,
+      "logits/rejected": -1.4128607511520386,
+      "logps/chosen": -152.2001190185547,
+      "logps/rejected": -191.74932861328125,
+      "loss": 0.4226,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.2529861629009247,
+      "rewards/margins": 1.520397424697876,
+      "rewards/rejected": -1.773383617401123,
+      "step": 1508
+    },
+    {
+      "epoch": 1.7488237423090842,
+      "grad_norm": 49.81866700382731,
+      "learning_rate": 8.582694958910807e-09,
+      "logits/chosen": -1.2254369258880615,
+      "logits/rejected": -1.2749468088150024,
+      "logps/chosen": -182.1556854248047,
+      "logps/rejected": -216.18411254882812,
+      "loss": 0.3902,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.6539211869239807,
+      "rewards/margins": 1.5710985660552979,
+      "rewards/rejected": -2.225019693374634,
+      "step": 1510
+    },
+    {
+      "epoch": 1.7511400651465798,
+      "grad_norm": 52.34711654194476,
+      "learning_rate": 8.426752625076373e-09,
+      "logits/chosen": -1.2552261352539062,
+      "logits/rejected": -1.3420953750610352,
+      "logps/chosen": -186.13934326171875,
+      "logps/rejected": -285.68804931640625,
+      "loss": 0.3499,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.5289927124977112,
+      "rewards/margins": 3.851708173751831,
+      "rewards/rejected": -4.380701065063477,
+      "step": 1512
+    },
+    {
+      "epoch": 1.7534563879840754,
+      "grad_norm": 52.389521034020916,
+      "learning_rate": 8.272177663389046e-09,
+      "logits/chosen": -1.1967260837554932,
+      "logits/rejected": -1.1967551708221436,
+      "logps/chosen": -217.95095825195312,
+      "logps/rejected": -236.05380249023438,
+      "loss": 0.3782,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.3343973159790039,
+      "rewards/margins": 1.9475483894348145,
+      "rewards/rejected": -2.2819457054138184,
+      "step": 1514
+    },
+    {
+      "epoch": 1.7557727108215708,
+      "grad_norm": 55.99063639865476,
+      "learning_rate": 8.118972381962851e-09,
+      "logits/chosen": -1.1716736555099487,
+      "logits/rejected": -1.2387813329696655,
+      "logps/chosen": -161.53382873535156,
+      "logps/rejected": -189.86782836914062,
+      "loss": 0.3571,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.7027201652526855,
+      "rewards/margins": 1.415562629699707,
+      "rewards/rejected": -2.1182827949523926,
+      "step": 1516
+    },
+    {
+      "epoch": 1.7580890336590662,
+      "grad_norm": 60.98846930904024,
+      "learning_rate": 7.967139068459726e-09,
+      "logits/chosen": -1.1493229866027832,
+      "logits/rejected": -1.146936297416687,
+      "logps/chosen": -128.60189819335938,
+      "logps/rejected": -160.15321350097656,
+      "loss": 0.3883,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.36255598068237305,
+      "rewards/margins": 1.3770678043365479,
+      "rewards/rejected": -1.7396236658096313,
+      "step": 1518
+    },
+    {
+      "epoch": 1.7604053564965616,
+      "grad_norm": 176.68417193498476,
+      "learning_rate": 7.81667999005543e-09,
+      "logits/chosen": -1.3078656196594238,
+      "logits/rejected": -1.3120546340942383,
+      "logps/chosen": -179.33438110351562,
+      "logps/rejected": -186.1118621826172,
+      "loss": 0.4993,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.22156819701194763,
+      "rewards/margins": 0.9435240030288696,
+      "rewards/rejected": -1.16509211063385,
+      "step": 1520
+    },
+    {
+      "epoch": 1.7627216793340572,
+      "grad_norm": 51.91509618352603,
+      "learning_rate": 7.6675973934056e-09,
+      "logits/chosen": -1.0760035514831543,
+      "logits/rejected": -1.1664559841156006,
+      "logps/chosen": -136.77081298828125,
+      "logps/rejected": -175.25926208496094,
+      "loss": 0.4397,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.2903676927089691,
+      "rewards/margins": 1.1137360334396362,
+      "rewards/rejected": -1.4041036367416382,
+      "step": 1522
+    },
+    {
+      "epoch": 1.7650380021715528,
+      "grad_norm": 59.67977888278656,
+      "learning_rate": 7.51989350461224e-09,
+      "logits/chosen": -1.2789033651351929,
+      "logits/rejected": -1.3050099611282349,
+      "logps/chosen": -145.60400390625,
+      "logps/rejected": -156.1199951171875,
+      "loss": 0.4456,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.3298056125640869,
+      "rewards/margins": 0.7980384826660156,
+      "rewards/rejected": -1.1278440952301025,
+      "step": 1524
+    },
+    {
+      "epoch": 1.7673543250090482,
+      "grad_norm": 50.021597145613505,
+      "learning_rate": 7.373570529190498e-09,
+      "logits/chosen": -1.2145639657974243,
+      "logits/rejected": -1.2063783407211304,
+      "logps/chosen": -162.4467315673828,
+      "logps/rejected": -196.65577697753906,
+      "loss": 0.3705,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.2843925654888153,
+      "rewards/margins": 1.6666440963745117,
+      "rewards/rejected": -1.9510366916656494,
+      "step": 1526
+    },
+    {
+      "epoch": 1.7696706478465436,
+      "grad_norm": 62.64810414938643,
+      "learning_rate": 7.228630652035717e-09,
+      "logits/chosen": -1.2518867254257202,
+      "logits/rejected": -1.1622406244277954,
+      "logps/chosen": -169.1246337890625,
+      "logps/rejected": -189.02169799804688,
+      "loss": 0.3845,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.23387570679187775,
+      "rewards/margins": 1.9417215585708618,
+      "rewards/rejected": -2.1755971908569336,
+      "step": 1528
+    },
+    {
+      "epoch": 1.771986970684039,
+      "grad_norm": 56.92633456964183,
+      "learning_rate": 7.08507603739078e-09,
+      "logits/chosen": -1.2512166500091553,
+      "logits/rejected": -1.2901430130004883,
+      "logps/chosen": -162.3385772705078,
+      "logps/rejected": -193.41940307617188,
+      "loss": 0.4376,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.40012550354003906,
+      "rewards/margins": 1.2529363632202148,
+      "rewards/rejected": -1.653061866760254,
+      "step": 1530
+    },
+    {
+      "epoch": 1.7743032935215346,
+      "grad_norm": 76.55205063599432,
+      "learning_rate": 6.942908828813876e-09,
+      "logits/chosen": -1.323652982711792,
+      "logits/rejected": -1.2494463920593262,
+      "logps/chosen": -192.47853088378906,
+      "logps/rejected": -190.863037109375,
+      "loss": 0.3828,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.46822619438171387,
+      "rewards/margins": 1.0589056015014648,
+      "rewards/rejected": -1.5271317958831787,
+      "step": 1532
+    },
+    {
+      "epoch": 1.77661961635903,
+      "grad_norm": 58.83800639947019,
+      "learning_rate": 6.802131149146373e-09,
+      "logits/chosen": -1.254701018333435,
+      "logits/rejected": -1.2828840017318726,
+      "logps/chosen": -144.4114990234375,
+      "logps/rejected": -155.79037475585938,
+      "loss": 0.374,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.3679081201553345,
+      "rewards/margins": 1.2586240768432617,
+      "rewards/rejected": -1.6265323162078857,
+      "step": 1534
+    },
+    {
+      "epoch": 1.7789359391965256,
+      "grad_norm": 68.11680500501693,
+      "learning_rate": 6.662745100481271e-09,
+      "logits/chosen": -1.1942329406738281,
+      "logits/rejected": -1.289471983909607,
+      "logps/chosen": -103.48456573486328,
+      "logps/rejected": -110.4488754272461,
+      "loss": 0.3996,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.294292151927948,
+      "rewards/margins": 0.796977162361145,
+      "rewards/rejected": -1.0912692546844482,
+      "step": 1536
+    },
+    {
+      "epoch": 1.781252262034021,
+      "grad_norm": 60.49404420207855,
+      "learning_rate": 6.5247527641316465e-09,
+      "logits/chosen": -1.2070562839508057,
+      "logits/rejected": -1.2548003196716309,
+      "logps/chosen": -156.2920379638672,
+      "logps/rejected": -177.38890075683594,
+      "loss": 0.4381,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.03995545208454132,
+      "rewards/margins": 1.5764446258544922,
+      "rewards/rejected": -1.616400122642517,
+      "step": 1538
+    },
+    {
+      "epoch": 1.7835685848715164,
+      "grad_norm": 58.839873216052546,
+      "learning_rate": 6.388156200599726e-09,
+      "logits/chosen": -1.1764907836914062,
+      "logits/rejected": -1.2363911867141724,
+      "logps/chosen": -138.08792114257812,
+      "logps/rejected": -156.8811492919922,
+      "loss": 0.4437,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.5578911304473877,
+      "rewards/margins": 0.9130861163139343,
+      "rewards/rejected": -1.4709770679473877,
+      "step": 1540
+    },
+    {
+      "epoch": 1.785884907709012,
+      "grad_norm": 54.830012236677426,
+      "learning_rate": 6.2529574495459815e-09,
+      "logits/chosen": -1.2464194297790527,
+      "logits/rejected": -1.2431282997131348,
+      "logps/chosen": -146.96051025390625,
+      "logps/rejected": -175.07481384277344,
+      "loss": 0.386,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.2085748016834259,
+      "rewards/margins": 1.4820951223373413,
+      "rewards/rejected": -1.6906698942184448,
+      "step": 1542
+    },
+    {
+      "epoch": 1.7882012305465074,
+      "grad_norm": 53.964612199236846,
+      "learning_rate": 6.119158529758817e-09,
+      "logits/chosen": -1.2010880708694458,
+      "logits/rejected": -1.252152681350708,
+      "logps/chosen": -125.40618896484375,
+      "logps/rejected": -147.0822296142578,
+      "loss": 0.4294,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.6274422407150269,
+      "rewards/margins": 1.2626943588256836,
+      "rewards/rejected": -1.8901365995407104,
+      "step": 1544
+    },
+    {
+      "epoch": 1.790517553384003,
+      "grad_norm": 61.93429459296764,
+      "learning_rate": 5.986761439124288e-09,
+      "logits/chosen": -1.0499889850616455,
+      "logits/rejected": -1.0637288093566895,
+      "logps/chosen": -145.4034881591797,
+      "logps/rejected": -170.8926239013672,
+      "loss": 0.4097,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -1.0535945892333984,
+      "rewards/margins": 1.4541335105895996,
+      "rewards/rejected": -2.507727861404419,
+      "step": 1546
+    },
+    {
+      "epoch": 1.7928338762214984,
+      "grad_norm": 63.39263653013202,
+      "learning_rate": 5.855768154596363e-09,
+      "logits/chosen": -1.2247127294540405,
+      "logits/rejected": -1.287811040878296,
+      "logps/chosen": -128.2520751953125,
+      "logps/rejected": -145.6575927734375,
+      "loss": 0.4182,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.20227603614330292,
+      "rewards/margins": 0.8022910356521606,
+      "rewards/rejected": -1.0045669078826904,
+      "step": 1548
+    },
+    {
+      "epoch": 1.7951501990589938,
+      "grad_norm": 53.1979380263347,
+      "learning_rate": 5.726180632167354e-09,
+      "logits/chosen": -1.2052092552185059,
+      "logits/rejected": -1.2159252166748047,
+      "logps/chosen": -151.31918334960938,
+      "logps/rejected": -184.94479370117188,
+      "loss": 0.4075,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.06491108983755112,
+      "rewards/margins": 1.7111616134643555,
+      "rewards/rejected": -1.7760728597640991,
+      "step": 1550
+    },
+    {
+      "epoch": 1.7974665218964894,
+      "grad_norm": 53.36509041975899,
+      "learning_rate": 5.5980008068387655e-09,
+      "logits/chosen": -1.192318081855774,
+      "logits/rejected": -1.2422665357589722,
+      "logps/chosen": -158.2563934326172,
+      "logps/rejected": -208.37709045410156,
+      "loss": 0.3783,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.3992021381855011,
+      "rewards/margins": 1.9673078060150146,
+      "rewards/rejected": -2.3665099143981934,
+      "step": 1552
+    },
+    {
+      "epoch": 1.7997828447339848,
+      "grad_norm": 57.43435085106451,
+      "learning_rate": 5.471230592592313e-09,
+      "logits/chosen": -1.2281129360198975,
+      "logits/rejected": -1.1943424940109253,
+      "logps/chosen": -132.93118286132812,
+      "logps/rejected": -142.4124755859375,
+      "loss": 0.3999,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.2500740587711334,
+      "rewards/margins": 0.9634323120117188,
+      "rewards/rejected": -1.2135063409805298,
+      "step": 1554
+    },
+    {
+      "epoch": 1.8020991675714804,
+      "grad_norm": 116.95703791110742,
+      "learning_rate": 5.345871882361397e-09,
+      "logits/chosen": -1.222663402557373,
+      "logits/rejected": -1.2307226657867432,
+      "logps/chosen": -195.47381591796875,
+      "logps/rejected": -213.84588623046875,
+      "loss": 0.5455,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.06570303440094,
+      "rewards/margins": 1.080770492553711,
+      "rewards/rejected": -2.1464734077453613,
+      "step": 1556
+    },
+    {
+      "epoch": 1.8044154904089758,
+      "grad_norm": 54.00119490171407,
+      "learning_rate": 5.221926548002875e-09,
+      "logits/chosen": -1.1924062967300415,
+      "logits/rejected": -1.269582748413086,
+      "logps/chosen": -165.26943969726562,
+      "logps/rejected": -179.38568115234375,
+      "loss": 0.4258,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.21030552685260773,
+      "rewards/margins": 1.1077656745910645,
+      "rewards/rejected": -0.8974601030349731,
+      "step": 1558
+    },
+    {
+      "epoch": 1.8067318132464711,
+      "grad_norm": 47.637201993987425,
+      "learning_rate": 5.099396440269033e-09,
+      "logits/chosen": -1.1668461561203003,
+      "logits/rejected": -1.1675832271575928,
+      "logps/chosen": -132.747314453125,
+      "logps/rejected": -197.4693603515625,
+      "loss": 0.3714,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.2991000711917877,
+      "rewards/margins": 2.7344629764556885,
+      "rewards/rejected": -3.0335628986358643,
+      "step": 1560
+    },
+    {
+      "epoch": 1.8090481360839668,
+      "grad_norm": 73.3924537450436,
+      "learning_rate": 4.978283388780002e-09,
+      "logits/chosen": -1.2106759548187256,
+      "logits/rejected": -1.3471499681472778,
+      "logps/chosen": -172.1467742919922,
+      "logps/rejected": -206.36143493652344,
+      "loss": 0.3895,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.703016996383667,
+      "rewards/margins": 1.2060117721557617,
+      "rewards/rejected": -1.9090288877487183,
+      "step": 1562
+    },
+    {
+      "epoch": 1.8113644589214621,
+      "grad_norm": 90.51012356320436,
+      "learning_rate": 4.858589201996432e-09,
+      "logits/chosen": -1.0378146171569824,
+      "logits/rejected": -1.1732603311538696,
+      "logps/chosen": -141.2643280029297,
+      "logps/rejected": -164.4271697998047,
+      "loss": 0.4848,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.8795535564422607,
+      "rewards/margins": 0.9949630498886108,
+      "rewards/rejected": -1.874516487121582,
+      "step": 1564
+    },
+    {
+      "epoch": 1.8136807817589577,
+      "grad_norm": 56.01168000132923,
+      "learning_rate": 4.740315667192441e-09,
+      "logits/chosen": -1.1176464557647705,
+      "logits/rejected": -1.2511212825775146,
+      "logps/chosen": -103.09518432617188,
+      "logps/rejected": -135.39122009277344,
+      "loss": 0.4393,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.2941249907016754,
+      "rewards/margins": 0.9569557905197144,
+      "rewards/rejected": -1.2510807514190674,
+      "step": 1566
+    },
+    {
+      "epoch": 1.8159971045964531,
+      "grad_norm": 48.16666589487942,
+      "learning_rate": 4.623464550429002e-09,
+      "logits/chosen": -1.102777361869812,
+      "logits/rejected": -1.1394641399383545,
+      "logps/chosen": -111.80138397216797,
+      "logps/rejected": -145.2130126953125,
+      "loss": 0.4561,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.7176414728164673,
+      "rewards/margins": 1.2812902927398682,
+      "rewards/rejected": -1.998931646347046,
+      "step": 1568
+    },
+    {
+      "epoch": 1.8183134274339485,
+      "grad_norm": 50.97749085021057,
+      "learning_rate": 4.508037596527525e-09,
+      "logits/chosen": -1.1966917514801025,
+      "logits/rejected": -1.2247413396835327,
+      "logps/chosen": -114.48523712158203,
+      "logps/rejected": -128.622802734375,
+      "loss": 0.3797,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.19608543813228607,
+      "rewards/margins": 0.9697508215904236,
+      "rewards/rejected": -1.1658360958099365,
+      "step": 1570
+    },
+    {
+      "epoch": 1.8206297502714441,
+      "grad_norm": 54.387837657286084,
+      "learning_rate": 4.39403652904381e-09,
+      "logits/chosen": -1.1147388219833374,
+      "logits/rejected": -1.1594665050506592,
+      "logps/chosen": -114.78770446777344,
+      "logps/rejected": -155.918701171875,
+      "loss": 0.3961,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.18746113777160645,
+      "rewards/margins": 1.8321788311004639,
+      "rewards/rejected": -2.0196399688720703,
+      "step": 1572
+    },
+    {
+      "epoch": 1.8229460731089395,
+      "grad_norm": 62.271451803387365,
+      "learning_rate": 4.2814630502422845e-09,
+      "logits/chosen": -1.1847018003463745,
+      "logits/rejected": -1.1410635709762573,
+      "logps/chosen": -178.85458374023438,
+      "logps/rejected": -211.72219848632812,
+      "loss": 0.4279,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.22784435749053955,
+      "rewards/margins": 2.5258147716522217,
+      "rewards/rejected": -2.7536590099334717,
+      "step": 1574
+    },
+    {
+      "epoch": 1.8252623959464351,
+      "grad_norm": 57.174518848316346,
+      "learning_rate": 4.170318841070708e-09,
+      "logits/chosen": -1.120819330215454,
+      "logits/rejected": -1.1634063720703125,
+      "logps/chosen": -140.26319885253906,
+      "logps/rejected": -207.86880493164062,
+      "loss": 0.4442,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.25584009289741516,
+      "rewards/margins": 1.6964097023010254,
+      "rewards/rejected": -1.9522497653961182,
+      "step": 1576
+    },
+    {
+      "epoch": 1.8275787187839305,
+      "grad_norm": 57.29425789262467,
+      "learning_rate": 4.060605561134889e-09,
+      "logits/chosen": -1.3027273416519165,
+      "logits/rejected": -1.2673333883285522,
+      "logps/chosen": -170.17152404785156,
+      "logps/rejected": -188.33880615234375,
+      "loss": 0.4311,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.42000892758369446,
+      "rewards/margins": 1.3922333717346191,
+      "rewards/rejected": -1.8122422695159912,
+      "step": 1578
+    },
+    {
+      "epoch": 1.829895041621426,
+      "grad_norm": 65.96165236575662,
+      "learning_rate": 3.952324848674004e-09,
+      "logits/chosen": -1.1435868740081787,
+      "logits/rejected": -1.2456907033920288,
+      "logps/chosen": -118.90472412109375,
+      "logps/rejected": -160.82818603515625,
+      "loss": 0.3963,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.47032859921455383,
+      "rewards/margins": 1.457180142402649,
+      "rewards/rejected": -1.9275087118148804,
+      "step": 1580
+    },
+    {
+      "epoch": 1.8322113644589213,
+      "grad_norm": 62.96392877654251,
+      "learning_rate": 3.8454783205361774e-09,
+      "logits/chosen": -1.2181570529937744,
+      "logits/rejected": -1.2495853900909424,
+      "logps/chosen": -195.74876403808594,
+      "logps/rejected": -292.0626220703125,
+      "loss": 0.4112,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.3267236649990082,
+      "rewards/margins": 4.259873390197754,
+      "rewards/rejected": -4.586597442626953,
+      "step": 1582
+    },
+    {
+      "epoch": 1.834527687296417,
+      "grad_norm": 58.298377548314235,
+      "learning_rate": 3.740067572154238e-09,
+      "logits/chosen": -1.292594075202942,
+      "logits/rejected": -1.3315826654434204,
+      "logps/chosen": -154.32740783691406,
+      "logps/rejected": -175.98606872558594,
+      "loss": 0.4088,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.2856728136539459,
+      "rewards/margins": 1.166110634803772,
+      "rewards/rejected": -1.4517834186553955,
+      "step": 1584
+    },
+    {
+      "epoch": 1.8368440101339125,
+      "grad_norm": 59.34894292485851,
+      "learning_rate": 3.6360941775219534e-09,
+      "logits/chosen": -1.2552549839019775,
+      "logits/rejected": -1.3246078491210938,
+      "logps/chosen": -165.2515869140625,
+      "logps/rejected": -189.0300750732422,
+      "loss": 0.3893,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.31748124957084656,
+      "rewards/margins": 1.6454672813415527,
+      "rewards/rejected": -1.9629485607147217,
+      "step": 1586
+    },
+    {
+      "epoch": 1.839160332971408,
+      "grad_norm": 72.01635850106565,
+      "learning_rate": 3.53355968917054e-09,
+      "logits/chosen": -1.1828457117080688,
+      "logits/rejected": -1.1595231294631958,
+      "logps/chosen": -188.94688415527344,
+      "logps/rejected": -214.75,
+      "loss": 0.3615,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.9493909478187561,
+      "rewards/margins": 1.7555177211761475,
+      "rewards/rejected": -2.704908609390259,
+      "step": 1588
+    },
+    {
+      "epoch": 1.8414766558089033,
+      "grad_norm": 62.6493622965171,
+      "learning_rate": 3.432465638145443e-09,
+      "logits/chosen": -1.2264246940612793,
+      "logits/rejected": -1.2353841066360474,
+      "logps/chosen": -195.12002563476562,
+      "logps/rejected": -207.8896942138672,
+      "loss": 0.3766,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.44329333305358887,
+      "rewards/margins": 1.4592864513397217,
+      "rewards/rejected": -1.9025800228118896,
+      "step": 1590
+    },
+    {
+      "epoch": 1.8437929786463987,
+      "grad_norm": 90.93634487708707,
+      "learning_rate": 3.3328135339834917e-09,
+      "logits/chosen": -1.2629611492156982,
+      "logits/rejected": -1.249568223953247,
+      "logps/chosen": -190.23126220703125,
+      "logps/rejected": -229.70721435546875,
+      "loss": 0.4445,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.7300775647163391,
+      "rewards/margins": 1.7166606187820435,
+      "rewards/rejected": -2.4467382431030273,
+      "step": 1592
+    },
+    {
+      "epoch": 1.8461093014838943,
+      "grad_norm": 70.40594487298823,
+      "learning_rate": 3.234604864690349e-09,
+      "logits/chosen": -1.1563414335250854,
+      "logits/rejected": -1.1132121086120605,
+      "logps/chosen": -112.93782806396484,
+      "logps/rejected": -115.19876861572266,
+      "loss": 0.4635,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.16091413795948029,
+      "rewards/margins": 1.0079346895217896,
+      "rewards/rejected": -1.1688487529754639,
+      "step": 1594
+    },
+    {
+      "epoch": 1.84842562432139,
+      "grad_norm": 70.46959430003976,
+      "learning_rate": 3.13784109671833e-09,
+      "logits/chosen": -1.1287944316864014,
+      "logits/rejected": -1.1954846382141113,
+      "logps/chosen": -145.52749633789062,
+      "logps/rejected": -168.68896484375,
+      "loss": 0.4428,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.435101717710495,
+      "rewards/margins": 1.1136534214019775,
+      "rewards/rejected": -1.548755168914795,
+      "step": 1596
+    },
+    {
+      "epoch": 1.8507419471588853,
+      "grad_norm": 63.53152924846159,
+      "learning_rate": 3.0425236749444307e-09,
+      "logits/chosen": -1.1079940795898438,
+      "logits/rejected": -1.1838057041168213,
+      "logps/chosen": -104.38517761230469,
+      "logps/rejected": -126.13815307617188,
+      "loss": 0.4119,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.20333430171012878,
+      "rewards/margins": 1.2202249765396118,
+      "rewards/rejected": -1.4235591888427734,
+      "step": 1598
+    },
+    {
+      "epoch": 1.8530582699963807,
+      "grad_norm": 66.75495278471351,
+      "learning_rate": 2.9486540226488555e-09,
+      "logits/chosen": -1.1984293460845947,
+      "logits/rejected": -1.1811829805374146,
+      "logps/chosen": -107.63018035888672,
+      "logps/rejected": -130.51141357421875,
+      "loss": 0.4185,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.44386693835258484,
+      "rewards/margins": 1.2538187503814697,
+      "rewards/rejected": -1.697685718536377,
+      "step": 1600
+    },
+    {
+      "epoch": 1.8530582699963807,
+      "eval_logits/chosen": -1.2096275091171265,
+      "eval_logits/rejected": -1.2041908502578735,
+      "eval_logps/chosen": -144.02456665039062,
+      "eval_logps/rejected": -149.35797119140625,
+      "eval_loss": 0.59078049659729,
+      "eval_rewards/accuracies": 0.7599999904632568,
+      "eval_rewards/chosen": -0.8393388986587524,
+      "eval_rewards/margins": 0.7011021375656128,
+      "eval_rewards/rejected": -1.5404411554336548,
+      "eval_runtime": 26.1157,
+      "eval_samples_per_second": 3.829,
+      "eval_steps_per_second": 0.957,
+      "step": 1600
+    },
+    {
+      "epoch": 1.855374592833876,
+      "grad_norm": 51.25806551926982,
+      "learning_rate": 2.856233541493691e-09,
+      "logits/chosen": -1.1180177927017212,
+      "logits/rejected": -1.153393268585205,
+      "logps/chosen": -144.27127075195312,
+      "logps/rejected": -172.8219757080078,
+      "loss": 0.4103,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.5550628900527954,
+      "rewards/margins": 1.737399697303772,
+      "rewards/rejected": -2.2924625873565674,
+      "step": 1602
+    },
+    {
+      "epoch": 1.8576909156713717,
+      "grad_norm": 44.574245908096394,
+      "learning_rate": 2.7652636115019554e-09,
+      "logits/chosen": -1.2830660343170166,
+      "logits/rejected": -1.3382513523101807,
+      "logps/chosen": -186.8584747314453,
+      "logps/rejected": -214.39341735839844,
+      "loss": 0.336,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.011630617082118988,
+      "rewards/margins": 2.068033218383789,
+      "rewards/rejected": -2.0564029216766357,
+      "step": 1604
+    },
+    {
+      "epoch": 1.8600072385088673,
+      "grad_norm": 88.24929156836725,
+      "learning_rate": 2.6757455910370487e-09,
+      "logits/chosen": -1.1889640092849731,
+      "logits/rejected": -1.178146481513977,
+      "logps/chosen": -169.42417907714844,
+      "logps/rejected": -202.3689422607422,
+      "loss": 0.453,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.4166257083415985,
+      "rewards/margins": 1.3977904319763184,
+      "rewards/rejected": -1.8144161701202393,
+      "step": 1606
+    },
+    {
+      "epoch": 1.8623235613463627,
+      "grad_norm": 54.823245822280576,
+      "learning_rate": 2.5876808167825005e-09,
+      "logits/chosen": -1.2597419023513794,
+      "logits/rejected": -1.193768858909607,
+      "logps/chosen": -93.74658203125,
+      "logps/rejected": -92.6616439819336,
+      "loss": 0.3924,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.25879502296447754,
+      "rewards/margins": 0.4778652489185333,
+      "rewards/rejected": -0.7366602420806885,
+      "step": 1608
+    },
+    {
+      "epoch": 1.864639884183858,
+      "grad_norm": 74.02926136780609,
+      "learning_rate": 2.5010706037218885e-09,
+      "logits/chosen": -1.2314317226409912,
+      "logits/rejected": -1.2886399030685425,
+      "logps/chosen": -163.94842529296875,
+      "logps/rejected": -195.11390686035156,
+      "loss": 0.3971,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.2574860155582428,
+      "rewards/margins": 1.3789194822311401,
+      "rewards/rejected": -1.6364054679870605,
+      "step": 1610
+    },
+    {
+      "epoch": 1.8669562070213535,
+      "grad_norm": 54.30860117915708,
+      "learning_rate": 2.4159162451193094e-09,
+      "logits/chosen": -1.0902681350708008,
+      "logits/rejected": -1.112775206565857,
+      "logps/chosen": -140.3288116455078,
+      "logps/rejected": -188.05210876464844,
+      "loss": 0.4003,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.1892092376947403,
+      "rewards/margins": 1.8315831422805786,
+      "rewards/rejected": -2.0207924842834473,
+      "step": 1612
+    },
+    {
+      "epoch": 1.869272529858849,
+      "grad_norm": 62.18685455386846,
+      "learning_rate": 2.3322190125000475e-09,
+      "logits/chosen": -1.0966382026672363,
+      "logits/rejected": -1.1658515930175781,
+      "logps/chosen": -116.89921569824219,
+      "logps/rejected": -151.35015869140625,
+      "loss": 0.4112,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.4243711233139038,
+      "rewards/margins": 1.4443333148956299,
+      "rewards/rejected": -1.8687043190002441,
+      "step": 1614
+    },
+    {
+      "epoch": 1.8715888526963447,
+      "grad_norm": 55.591911669551806,
+      "learning_rate": 2.24998015563157e-09,
+      "logits/chosen": -1.1172374486923218,
+      "logits/rejected": -1.1134474277496338,
+      "logps/chosen": -106.15010070800781,
+      "logps/rejected": -122.0849609375,
+      "loss": 0.42,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.5767372250556946,
+      "rewards/margins": 0.9947463274002075,
+      "rewards/rejected": -1.5714833736419678,
+      "step": 1616
+    },
+    {
+      "epoch": 1.87390517553384,
+      "grad_norm": 69.67114883544231,
+      "learning_rate": 2.169200902504842e-09,
+      "logits/chosen": -1.291989803314209,
+      "logits/rejected": -1.3668596744537354,
+      "logps/chosen": -140.99050903320312,
+      "logps/rejected": -164.55636596679688,
+      "loss": 0.4102,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.1543090045452118,
+      "rewards/margins": 1.0028785467147827,
+      "rewards/rejected": -1.1571874618530273,
+      "step": 1618
+    },
+    {
+      "epoch": 1.8762214983713354,
+      "grad_norm": 49.47929313355962,
+      "learning_rate": 2.0898824593160503e-09,
+      "logits/chosen": -1.1244778633117676,
+      "logits/rejected": -1.1484088897705078,
+      "logps/chosen": -115.74507904052734,
+      "logps/rejected": -133.63916015625,
+      "loss": 0.3943,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.4605577886104584,
+      "rewards/margins": 0.8462937474250793,
+      "rewards/rejected": -1.3068513870239258,
+      "step": 1620
+    },
+    {
+      "epoch": 1.8785378212088308,
+      "grad_norm": 51.80324368762919,
+      "learning_rate": 2.012026010448542e-09,
+      "logits/chosen": -1.0954941511154175,
+      "logits/rejected": -1.160184383392334,
+      "logps/chosen": -131.16983032226562,
+      "logps/rejected": -178.2694549560547,
+      "loss": 0.4011,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.1698540300130844,
+      "rewards/margins": 1.7133314609527588,
+      "rewards/rejected": -1.8831853866577148,
+      "step": 1622
+    },
+    {
+      "epoch": 1.8808541440463264,
+      "grad_norm": 61.62870922068943,
+      "learning_rate": 1.935632718455171e-09,
+      "logits/chosen": -1.167246699333191,
+      "logits/rejected": -1.2080024480819702,
+      "logps/chosen": -154.55758666992188,
+      "logps/rejected": -188.28201293945312,
+      "loss": 0.3673,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.5035812854766846,
+      "rewards/margins": 1.6042046546936035,
+      "rewards/rejected": -2.107785701751709,
+      "step": 1624
+    },
+    {
+      "epoch": 1.883170466883822,
+      "grad_norm": 66.67930280370663,
+      "learning_rate": 1.860703724040935e-09,
+      "logits/chosen": -1.1037479639053345,
+      "logits/rejected": -1.0890851020812988,
+      "logps/chosen": -151.0048370361328,
+      "logps/rejected": -170.82943725585938,
+      "loss": 0.4514,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.4693964123725891,
+      "rewards/margins": 0.7652498483657837,
+      "rewards/rejected": -1.2346463203430176,
+      "step": 1626
+    },
+    {
+      "epoch": 1.8854867897213174,
+      "grad_norm": 64.45068485207041,
+      "learning_rate": 1.7872401460458874e-09,
+      "logits/chosen": -1.1636077165603638,
+      "logits/rejected": -1.2113772630691528,
+      "logps/chosen": -128.23899841308594,
+      "logps/rejected": -151.98800659179688,
+      "loss": 0.4074,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.15583369135856628,
+      "rewards/margins": 1.3744523525238037,
+      "rewards/rejected": -1.5302859544754028,
+      "step": 1628
+    },
+    {
+      "epoch": 1.8878031125588128,
+      "grad_norm": 60.44416949725557,
+      "learning_rate": 1.7152430814285302e-09,
+      "logits/chosen": -1.2152669429779053,
+      "logits/rejected": -1.225684404373169,
+      "logps/chosen": -145.33447265625,
+      "logps/rejected": -170.64059448242188,
+      "loss": 0.4299,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.09620651602745056,
+      "rewards/margins": 1.3490362167358398,
+      "rewards/rejected": -1.4452428817749023,
+      "step": 1630
+    },
+    {
+      "epoch": 1.8901194353963082,
+      "grad_norm": 46.81333988402518,
+      "learning_rate": 1.6447136052493704e-09,
+      "logits/chosen": -1.169386863708496,
+      "logits/rejected": -1.248286247253418,
+      "logps/chosen": -163.55990600585938,
+      "logps/rejected": -216.13929748535156,
+      "loss": 0.3547,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5737725496292114,
+      "rewards/margins": 1.9429514408111572,
+      "rewards/rejected": -2.516724109649658,
+      "step": 1632
+    },
+    {
+      "epoch": 1.8924357582338038,
+      "grad_norm": 76.34596117355929,
+      "learning_rate": 1.5756527706548561e-09,
+      "logits/chosen": -1.2912683486938477,
+      "logits/rejected": -1.2929950952529907,
+      "logps/chosen": -192.79061889648438,
+      "logps/rejected": -216.2474365234375,
+      "loss": 0.4647,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.6607008576393127,
+      "rewards/margins": 1.0471830368041992,
+      "rewards/rejected": -1.7078838348388672,
+      "step": 1634
+    },
+    {
+      "epoch": 1.8947520810712994,
+      "grad_norm": 50.49082900430503,
+      "learning_rate": 1.5080616088616882e-09,
+      "logits/chosen": -1.1574630737304688,
+      "logits/rejected": -1.2174662351608276,
+      "logps/chosen": -105.63571166992188,
+      "logps/rejected": -138.35476684570312,
+      "loss": 0.4099,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.4468805193901062,
+      "rewards/margins": 1.2918894290924072,
+      "rewards/rejected": -1.7387701272964478,
+      "step": 1636
+    },
+    {
+      "epoch": 1.8970684039087948,
+      "grad_norm": 75.9093198815468,
+      "learning_rate": 1.4419411291413885e-09,
+      "logits/chosen": -1.1982598304748535,
+      "logits/rejected": -1.1942592859268188,
+      "logps/chosen": -191.63914489746094,
+      "logps/rejected": -217.94439697265625,
+      "loss": 0.4268,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.6322917938232422,
+      "rewards/margins": 1.844104528427124,
+      "rewards/rejected": -2.4763965606689453,
+      "step": 1638
+    },
+    {
+      "epoch": 1.8993847267462902,
+      "grad_norm": 58.34682402089141,
+      "learning_rate": 1.3772923188052787e-09,
+      "logits/chosen": -1.0590007305145264,
+      "logits/rejected": -1.0922439098358154,
+      "logps/chosen": -90.13923645019531,
+      "logps/rejected": -109.63917541503906,
+      "loss": 0.4401,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.1902827024459839,
+      "rewards/margins": 1.1345347166061401,
+      "rewards/rejected": -1.3248172998428345,
+      "step": 1640
+    },
+    {
+      "epoch": 1.9017010495837856,
+      "grad_norm": 60.17667883042894,
+      "learning_rate": 1.3141161431896808e-09,
+      "logits/chosen": -1.292588472366333,
+      "logits/rejected": -1.2948338985443115,
+      "logps/chosen": -141.24307250976562,
+      "logps/rejected": -183.1881866455078,
+      "loss": 0.4093,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.06378068029880524,
+      "rewards/margins": 1.4755961894989014,
+      "rewards/rejected": -1.539376974105835,
+      "step": 1642
+    },
+    {
+      "epoch": 1.9040173724212812,
+      "grad_norm": 46.419475472935424,
+      "learning_rate": 1.2524135456415286e-09,
+      "logits/chosen": -1.3303675651550293,
+      "logits/rejected": -1.3957011699676514,
+      "logps/chosen": -197.4314727783203,
+      "logps/rejected": -230.6079559326172,
+      "loss": 0.3864,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.5181460380554199,
+      "rewards/margins": 1.6645830869674683,
+      "rewards/rejected": -2.1827290058135986,
+      "step": 1644
+    },
+    {
+      "epoch": 1.9063336952587768,
+      "grad_norm": 63.51262338219536,
+      "learning_rate": 1.1921854475043125e-09,
+      "logits/chosen": -1.3057444095611572,
+      "logits/rejected": -1.3598231077194214,
+      "logps/chosen": -140.8748779296875,
+      "logps/rejected": -162.47158813476562,
+      "loss": 0.438,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.24470748007297516,
+      "rewards/margins": 1.0919952392578125,
+      "rewards/rejected": -1.3367027044296265,
+      "step": 1646
+    },
+    {
+      "epoch": 1.9086500180962722,
+      "grad_norm": 59.16346811149136,
+      "learning_rate": 1.133432748104257e-09,
+      "logits/chosen": -1.2664871215820312,
+      "logits/rejected": -1.2415859699249268,
+      "logps/chosen": -168.35910034179688,
+      "logps/rejected": -186.53665161132812,
+      "loss": 0.3824,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3128247559070587,
+      "rewards/margins": 1.2026112079620361,
+      "rewards/rejected": -1.5154359340667725,
+      "step": 1648
+    },
+    {
+      "epoch": 1.9109663409337676,
+      "grad_norm": 72.05923858294557,
+      "learning_rate": 1.0761563247369322e-09,
+      "logits/chosen": -1.2224782705307007,
+      "logits/rejected": -1.2964767217636108,
+      "logps/chosen": -149.58824157714844,
+      "logps/rejected": -172.06092834472656,
+      "loss": 0.4327,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.10623270273208618,
+      "rewards/margins": 1.0352814197540283,
+      "rewards/rejected": -1.1415140628814697,
+      "step": 1650
+    },
+    {
+      "epoch": 1.913282663771263,
+      "grad_norm": 72.25189940099223,
+      "learning_rate": 1.0203570326541622e-09,
+      "logits/chosen": -1.0865113735198975,
+      "logits/rejected": -1.1271172761917114,
+      "logps/chosen": -103.49948120117188,
+      "logps/rejected": -108.95231628417969,
+      "loss": 0.4442,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.29463884234428406,
+      "rewards/margins": 0.5380735397338867,
+      "rewards/rejected": -0.8327123522758484,
+      "step": 1652
+    },
+    {
+      "epoch": 1.9155989866087586,
+      "grad_norm": 58.87468402600475,
+      "learning_rate": 9.660357050512158e-10,
+      "logits/chosen": -1.1705281734466553,
+      "logits/rejected": -1.213219165802002,
+      "logps/chosen": -151.6047821044922,
+      "logps/rejected": -167.16241455078125,
+      "loss": 0.4755,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.2385808229446411,
+      "rewards/margins": 0.9532268047332764,
+      "rewards/rejected": -2.191807746887207,
+      "step": 1654
+    },
+    {
+      "epoch": 1.9179153094462542,
+      "grad_norm": 61.13297680975046,
+      "learning_rate": 9.131931530544146e-10,
+      "logits/chosen": -1.1430917978286743,
+      "logits/rejected": -1.1427133083343506,
+      "logps/chosen": -152.1170654296875,
+      "logps/rejected": -184.9088592529297,
+      "loss": 0.4595,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.958507239818573,
+      "rewards/margins": 1.5942294597625732,
+      "rewards/rejected": -2.552736759185791,
+      "step": 1656
+    },
+    {
+      "epoch": 1.9202316322837496,
+      "grad_norm": 52.83655360025748,
+      "learning_rate": 8.618301657089877e-10,
+      "logits/chosen": -1.190130352973938,
+      "logits/rejected": -1.2142915725708008,
+      "logps/chosen": -161.54466247558594,
+      "logps/rejected": -182.72946166992188,
+      "loss": 0.3894,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.6212272047996521,
+      "rewards/margins": 1.291230320930481,
+      "rewards/rejected": -1.9124574661254883,
+      "step": 1658
+    },
+    {
+      "epoch": 1.922547955121245,
+      "grad_norm": 110.57140073148696,
+      "learning_rate": 8.119475099673035e-10,
+      "logits/chosen": -1.1131267547607422,
+      "logits/rejected": -1.2174605131149292,
+      "logps/chosen": -157.0382843017578,
+      "logps/rejected": -187.213134765625,
+      "loss": 0.4364,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.5017825961112976,
+      "rewards/margins": 0.9430520534515381,
+      "rewards/rejected": -1.444834589958191,
+      "step": 1660
+    },
+    {
+      "epoch": 1.9248642779587404,
+      "grad_norm": 53.02147974570211,
+      "learning_rate": 7.635459306773784e-10,
+      "logits/chosen": -1.1875760555267334,
+      "logits/rejected": -1.2432739734649658,
+      "logps/chosen": -147.28131103515625,
+      "logps/rejected": -169.49566650390625,
+      "loss": 0.43,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.40067267417907715,
+      "rewards/margins": 1.1304292678833008,
+      "rewards/rejected": -1.531101942062378,
+      "step": 1662
+    },
+    {
+      "epoch": 1.927180600796236,
+      "grad_norm": 64.80466315551176,
+      "learning_rate": 7.166261505718418e-10,
+      "logits/chosen": -1.222117304801941,
+      "logits/rejected": -1.2081456184387207,
+      "logps/chosen": -157.27780151367188,
+      "logps/rejected": -173.7355499267578,
+      "loss": 0.3734,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.33614563941955566,
+      "rewards/margins": 1.0949398279190063,
+      "rewards/rejected": -1.431085467338562,
+      "step": 1664
+    },
+    {
+      "epoch": 1.9294969236337316,
+      "grad_norm": 53.26534606993647,
+      "learning_rate": 6.711888702570556e-10,
+      "logits/chosen": -1.2844552993774414,
+      "logits/rejected": -1.3053499460220337,
+      "logps/chosen": -167.5966033935547,
+      "logps/rejected": -168.5198974609375,
+      "loss": 0.4155,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.15978145599365234,
+      "rewards/margins": 1.0049140453338623,
+      "rewards/rejected": -1.1646955013275146,
+      "step": 1666
+    },
+    {
+      "epoch": 1.931813246471227,
+      "grad_norm": 44.85242610407527,
+      "learning_rate": 6.272347682026779e-10,
+      "logits/chosen": -1.0309640169143677,
+      "logits/rejected": -1.134310007095337,
+      "logps/chosen": -111.43970489501953,
+      "logps/rejected": -144.99891662597656,
+      "loss": 0.3869,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.24206304550170898,
+      "rewards/margins": 1.2624578475952148,
+      "rewards/rejected": -1.5045208930969238,
+      "step": 1668
+    },
+    {
+      "epoch": 1.9341295693087224,
+      "grad_norm": 47.25828490220509,
+      "learning_rate": 5.847645007315937e-10,
+      "logits/chosen": -1.1614665985107422,
+      "logits/rejected": -1.254847764968872,
+      "logps/chosen": -137.72381591796875,
+      "logps/rejected": -143.80300903320312,
+      "loss": 0.4304,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.6075265407562256,
+      "rewards/margins": 0.8250117301940918,
+      "rewards/rejected": -1.4325382709503174,
+      "step": 1670
+    },
+    {
+      "epoch": 1.9364458921462178,
+      "grad_norm": 47.60965224972502,
+      "learning_rate": 5.437787020100115e-10,
+      "logits/chosen": -1.2086517810821533,
+      "logits/rejected": -1.1928253173828125,
+      "logps/chosen": -162.096923828125,
+      "logps/rejected": -178.24951171875,
+      "loss": 0.3871,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.37640607357025146,
+      "rewards/margins": 1.857001781463623,
+      "rewards/rejected": -2.233407974243164,
+      "step": 1672
+    },
+    {
+      "epoch": 1.9387622149837134,
+      "grad_norm": 51.270284022735154,
+      "learning_rate": 5.042779840380595e-10,
+      "logits/chosen": -1.2238942384719849,
+      "logits/rejected": -1.195109486579895,
+      "logps/chosen": -119.4543685913086,
+      "logps/rejected": -133.65127563476562,
+      "loss": 0.4054,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.3657826781272888,
+      "rewards/margins": 0.8839918375015259,
+      "rewards/rejected": -1.249774694442749,
+      "step": 1674
+    },
+    {
+      "epoch": 1.941078537821209,
+      "grad_norm": 58.278892064452975,
+      "learning_rate": 4.662629366406601e-10,
+      "logits/chosen": -1.2001346349716187,
+      "logits/rejected": -1.1202467679977417,
+      "logps/chosen": -128.72256469726562,
+      "logps/rejected": -139.1036376953125,
+      "loss": 0.4051,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.27061766386032104,
+      "rewards/margins": 1.1671736240386963,
+      "rewards/rejected": -1.437791347503662,
+      "step": 1676
+    },
+    {
+      "epoch": 1.9433948606587044,
+      "grad_norm": 46.614126047623216,
+      "learning_rate": 4.2973412745864744e-10,
+      "logits/chosen": -1.167816162109375,
+      "logits/rejected": -1.1992610692977905,
+      "logps/chosen": -129.0093536376953,
+      "logps/rejected": -169.50445556640625,
+      "loss": 0.3959,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.2619994878768921,
+      "rewards/margins": 1.6201242208480835,
+      "rewards/rejected": -1.8821238279342651,
+      "step": 1678
+    },
+    {
+      "epoch": 1.9457111834961998,
+      "grad_norm": 75.8267220593344,
+      "learning_rate": 3.946921019403859e-10,
+      "logits/chosen": -1.2351601123809814,
+      "logits/rejected": -1.3069424629211426,
+      "logps/chosen": -130.69961547851562,
+      "logps/rejected": -150.25662231445312,
+      "loss": 0.4755,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.33375632762908936,
+      "rewards/margins": 0.7270826697349548,
+      "rewards/rejected": -1.0608389377593994,
+      "step": 1680
+    },
+    {
+      "epoch": 1.9480275063336951,
+      "grad_norm": 57.21603903329357,
+      "learning_rate": 3.61137383333554e-10,
+      "logits/chosen": -1.1741724014282227,
+      "logits/rejected": -1.158752679824829,
+      "logps/chosen": -158.3549346923828,
+      "logps/rejected": -179.17391967773438,
+      "loss": 0.393,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.5816279649734497,
+      "rewards/margins": 1.2226191759109497,
+      "rewards/rejected": -1.8042471408843994,
+      "step": 1682
+    },
+    {
+      "epoch": 1.9503438291711908,
+      "grad_norm": 56.808299639154505,
+      "learning_rate": 3.2907047267736186e-10,
+      "logits/chosen": -1.240709900856018,
+      "logits/rejected": -1.223825454711914,
+      "logps/chosen": -163.18728637695312,
+      "logps/rejected": -198.8317413330078,
+      "loss": 0.3877,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.28957706689834595,
+      "rewards/margins": 1.6465396881103516,
+      "rewards/rejected": -1.9361168146133423,
+      "step": 1684
+    },
+    {
+      "epoch": 1.9526601520086864,
+      "grad_norm": 52.80591879977487,
+      "learning_rate": 2.9849184879506827e-10,
+      "logits/chosen": -1.1730728149414062,
+      "logits/rejected": -1.1454265117645264,
+      "logps/chosen": -136.7293701171875,
+      "logps/rejected": -151.21914672851562,
+      "loss": 0.3944,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.25426185131073,
+      "rewards/margins": 1.232313871383667,
+      "rewards/rejected": -1.4865756034851074,
+      "step": 1686
+    },
+    {
+      "epoch": 1.9549764748461818,
+      "grad_norm": 75.29247959139558,
+      "learning_rate": 2.6940196828681983e-10,
+      "logits/chosen": -1.093564510345459,
+      "logits/rejected": -1.0963504314422607,
+      "logps/chosen": -189.71131896972656,
+      "logps/rejected": -234.08973693847656,
+      "loss": 0.4931,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.0700315237045288,
+      "rewards/margins": 1.948075532913208,
+      "rewards/rejected": -3.0181069374084473,
+      "step": 1688
+    },
+    {
+      "epoch": 1.9572927976836771,
+      "grad_norm": 70.12384040466765,
+      "learning_rate": 2.418012655228452e-10,
+      "logits/chosen": -1.2431126832962036,
+      "logits/rejected": -1.2657066583633423,
+      "logps/chosen": -99.85934448242188,
+      "logps/rejected": -121.7491226196289,
+      "loss": 0.426,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3242354393005371,
+      "rewards/margins": 0.8404097557067871,
+      "rewards/rejected": -1.1646450757980347,
+      "step": 1690
+    },
+    {
+      "epoch": 1.9596091205211725,
+      "grad_norm": 50.23873116829206,
+      "learning_rate": 2.1569015263697143e-10,
+      "logits/chosen": -1.2286624908447266,
+      "logits/rejected": -1.2570605278015137,
+      "logps/chosen": -145.4255828857422,
+      "logps/rejected": -194.1035919189453,
+      "loss": 0.4198,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.5209024548530579,
+      "rewards/margins": 1.7652302980422974,
+      "rewards/rejected": -2.286133050918579,
+      "step": 1692
+    },
+    {
+      "epoch": 1.9619254433586681,
+      "grad_norm": 54.25222586547325,
+      "learning_rate": 1.9106901952045119e-10,
+      "logits/chosen": -1.2050321102142334,
+      "logits/rejected": -1.2619915008544922,
+      "logps/chosen": -179.3286590576172,
+      "logps/rejected": -227.4953155517578,
+      "loss": 0.4315,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.5577185750007629,
+      "rewards/margins": 1.80524742603302,
+      "rewards/rejected": -2.3629660606384277,
+      "step": 1694
+    },
+    {
+      "epoch": 1.9642417661961638,
+      "grad_norm": 61.997046360221496,
+      "learning_rate": 1.6793823381614501e-10,
+      "logits/chosen": -1.294581651687622,
+      "logits/rejected": -1.247463583946228,
+      "logps/chosen": -144.93246459960938,
+      "logps/rejected": -168.00628662109375,
+      "loss": 0.444,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.0931825190782547,
+      "rewards/margins": 1.4015557765960693,
+      "rewards/rejected": -1.4947383403778076,
+      "step": 1696
+    },
+    {
+      "epoch": 1.9665580890336591,
+      "grad_norm": 82.5016461876686,
+      "learning_rate": 1.4629814091307036e-10,
+      "logits/chosen": -1.2317255735397339,
+      "logits/rejected": -1.2456412315368652,
+      "logps/chosen": -156.1422576904297,
+      "logps/rejected": -151.87107849121094,
+      "loss": 0.417,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.4730032682418823,
+      "rewards/margins": 1.0971145629882812,
+      "rewards/rejected": -1.570117712020874,
+      "step": 1698
+    },
+    {
+      "epoch": 1.9688744118711545,
+      "grad_norm": 58.642096283997354,
+      "learning_rate": 1.261490639411833e-10,
+      "logits/chosen": -1.1768873929977417,
+      "logits/rejected": -1.2999684810638428,
+      "logps/chosen": -109.31826782226562,
+      "logps/rejected": -134.15371704101562,
+      "loss": 0.3986,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.2187974750995636,
+      "rewards/margins": 1.001755714416504,
+      "rewards/rejected": -1.2205531597137451,
+      "step": 1700
+    },
+    {
+      "epoch": 1.9688744118711545,
+      "eval_logits/chosen": -1.215119481086731,
+      "eval_logits/rejected": -1.2099292278289795,
+      "eval_logps/chosen": -144.09429931640625,
+      "eval_logps/rejected": -149.15176391601562,
+      "eval_loss": 0.5949785113334656,
+      "eval_rewards/accuracies": 0.7599999904632568,
+      "eval_rewards/chosen": -0.8463126420974731,
+      "eval_rewards/margins": 0.6735073924064636,
+      "eval_rewards/rejected": -1.5198200941085815,
+      "eval_runtime": 22.9886,
+      "eval_samples_per_second": 4.35,
+      "eval_steps_per_second": 1.087,
+      "step": 1700
+    },
+    {
+      "epoch": 1.97119073470865,
+      "grad_norm": 55.677396344114726,
+      "learning_rate": 1.0749130376659366e-10,
+      "logits/chosen": -1.2230623960494995,
+      "logits/rejected": -1.163780689239502,
+      "logps/chosen": -164.1004638671875,
+      "logps/rejected": -179.64088439941406,
+      "loss": 0.3997,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.1451684981584549,
+      "rewards/margins": 1.5081610679626465,
+      "rewards/rejected": -1.653329610824585,
+      "step": 1702
+    },
+    {
+      "epoch": 1.9735070575461455,
+      "grad_norm": 71.96257699524784,
+      "learning_rate": 9.032513898705741e-11,
+      "logits/chosen": -1.2779675722122192,
+      "logits/rejected": -1.2883471250534058,
+      "logps/chosen": -133.36380004882812,
+      "logps/rejected": -153.81741333007812,
+      "loss": 0.467,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.13987727463245392,
+      "rewards/margins": 1.16806161403656,
+      "rewards/rejected": -1.307938814163208,
+      "step": 1704
+    },
+    {
+      "epoch": 1.975823380383641,
+      "grad_norm": 49.123608283002156,
+      "learning_rate": 7.465082592782445e-11,
+      "logits/chosen": -1.207802414894104,
+      "logits/rejected": -1.1620241403579712,
+      "logps/chosen": -164.59759521484375,
+      "logps/rejected": -208.24905395507812,
+      "loss": 0.3896,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.5080645084381104,
+      "rewards/margins": 2.8074049949645996,
+      "rewards/rejected": -3.315469980239868,
+      "step": 1706
+    },
+    {
+      "epoch": 1.9781397032211365,
+      "grad_norm": 51.95036099086183,
+      "learning_rate": 6.04685986378195e-11,
+      "logits/chosen": -1.25301992893219,
+      "logits/rejected": -1.2303074598312378,
+      "logps/chosen": -160.34934997558594,
+      "logps/rejected": -162.81515502929688,
+      "loss": 0.3678,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.21462872624397278,
+      "rewards/margins": 1.2838810682296753,
+      "rewards/rejected": -1.4985097646713257,
+      "step": 1708
+    },
+    {
+      "epoch": 1.980456026058632,
+      "grad_norm": 55.67392221664361,
+      "learning_rate": 4.777866888611148e-11,
+      "logits/chosen": -1.1367592811584473,
+      "logits/rejected": -1.2029287815093994,
+      "logps/chosen": -170.26568603515625,
+      "logps/rejected": -203.4815216064453,
+      "loss": 0.3303,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.21711598336696625,
+      "rewards/margins": 1.6773165464401245,
+      "rewards/rejected": -1.8944324254989624,
+      "step": 1710
+    },
+    {
+      "epoch": 1.9827723488961273,
+      "grad_norm": 77.61747497941103,
+      "learning_rate": 3.658122615880499e-11,
+      "logits/chosen": -1.198671817779541,
+      "logits/rejected": -1.1844216585159302,
+      "logps/chosen": -188.25486755371094,
+      "logps/rejected": -188.4522247314453,
+      "loss": 0.4122,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.35006386041641235,
+      "rewards/margins": 1.136628270149231,
+      "rewards/rejected": -1.4866920709609985,
+      "step": 1712
+    },
+    {
+      "epoch": 1.985088671733623,
+      "grad_norm": 72.68985609286901,
+      "learning_rate": 2.687643765615366e-11,
+      "logits/chosen": -1.2630504369735718,
+      "logits/rejected": -1.1126054525375366,
+      "logps/chosen": -170.22010803222656,
+      "logps/rejected": -162.9827880859375,
+      "loss": 0.4154,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.1824585199356079,
+      "rewards/margins": 1.2966349124908447,
+      "rewards/rejected": -1.479093313217163,
+      "step": 1714
+    },
+    {
+      "epoch": 1.9874049945711183,
+      "grad_norm": 57.324360587243646,
+      "learning_rate": 1.8664448290106606e-11,
+      "logits/chosen": -1.1106977462768555,
+      "logits/rejected": -1.1625827550888062,
+      "logps/chosen": -127.45535278320312,
+      "logps/rejected": -168.37313842773438,
+      "loss": 0.4271,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.41478192806243896,
+      "rewards/margins": 1.4449265003204346,
+      "rewards/rejected": -1.8597084283828735,
+      "step": 1716
+    },
+    {
+      "epoch": 1.989721317408614,
+      "grad_norm": 64.91297510204477,
+      "learning_rate": 1.1945380682132355e-11,
+      "logits/chosen": -1.3344897031784058,
+      "logits/rejected": -1.3637080192565918,
+      "logps/chosen": -155.83399963378906,
+      "logps/rejected": -174.8062744140625,
+      "loss": 0.4455,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.2664361596107483,
+      "rewards/margins": 1.3320696353912354,
+      "rewards/rejected": -1.5985058546066284,
+      "step": 1718
+    },
+    {
+      "epoch": 1.9920376402461093,
+      "grad_norm": 55.133928870906985,
+      "learning_rate": 6.719335161364803e-12,
+      "logits/chosen": -1.210727334022522,
+      "logits/rejected": -1.2122191190719604,
+      "logps/chosen": -138.89447021484375,
+      "logps/rejected": -185.5248565673828,
+      "loss": 0.407,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.43058472871780396,
+      "rewards/margins": 2.1915080547332764,
+      "rewards/rejected": -2.6220927238464355,
+      "step": 1720
+    },
+    {
+      "epoch": 1.9943539630836047,
+      "grad_norm": 59.1945232431432,
+      "learning_rate": 2.9863897631488e-12,
+      "logits/chosen": -1.1495387554168701,
+      "logits/rejected": -1.2423110008239746,
+      "logps/chosen": -168.22439575195312,
+      "logps/rejected": -205.48471069335938,
+      "loss": 0.3821,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.808469295501709,
+      "rewards/margins": 1.811312198638916,
+      "rewards/rejected": -2.619781494140625,
+      "step": 1722
+    },
+    {
+      "epoch": 1.9966702859211003,
+      "grad_norm": 50.35911703570131,
+      "learning_rate": 7.466002278522232e-13,
+      "logits/chosen": -1.1888153553009033,
+      "logits/rejected": -1.1903085708618164,
+      "logps/chosen": -183.95315551757812,
+      "logps/rejected": -213.3731689453125,
+      "loss": 0.3585,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.34876811504364014,
+      "rewards/margins": 1.8841259479522705,
+      "rewards/rejected": -2.232893943786621,
+      "step": 1724
+    },
+    {
+      "epoch": 1.9989866087585957,
+      "grad_norm": 63.15894144617586,
+      "learning_rate": 0.0,
+      "logits/chosen": -1.270525574684143,
+      "logits/rejected": -1.1932576894760132,
+      "logps/chosen": -133.14906311035156,
+      "logps/rejected": -160.24928283691406,
+      "loss": 0.4361,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.38358262181282043,
+      "rewards/margins": 1.584211826324463,
+      "rewards/rejected": -1.967794418334961,
+      "step": 1726
     }
   ],
   "logging_steps": 2,
@@ -11511,7 +13238,7 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }