Training in progress, step 1752, checkpoint

last-checkpoint/global_step1752/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38c93c1a87f13462b5e0a5655712f0b5c4c012fffd291dcc75382e0879b36d9e
+size 24090788996

last-checkpoint/global_step1752/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab31a681deaa44cca1b5ed9bc3093e52b27f87ad098476a6f765ed1702235565
+size 24090788996

last-checkpoint/global_step1752/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db1c7f5fed1fb99aea9d6c75635912272ce175d4bebe42e8ad47b5e85730f8f5
+size 24090788996

last-checkpoint/global_step1752/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f69a4fa888baac249d427aa2ac2dc127a91fd0f7bee11e1b0be999f15e4438b7
+size 24090788996

last-checkpoint/global_step1752/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c7d2aa6a6d4c0a30e10c39ed299af9b8aef9d66b6ff8d7a84c0a25a45701cd9
+size 150693

last-checkpoint/global_step1752/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bc324cb078fc2735eda0f2ec2e2afdefed8e49fbf43a1c59c83521b42cc9657
+size 150693

last-checkpoint/global_step1752/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37f1bc8bdd8263b75afd7b99b3c7ae1cf493a40191c5c0ed770efb3243c0528c
+size 150693

last-checkpoint/global_step1752/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0aef5cf0e0dc06562d25766e6f1e03bd601dbe432d4bbd0b9eb3fbc4b6fdac0
+size 150693

last-checkpoint/latest

@@ -1 +1 @@
-global_step1500
+global_step1752

last-checkpoint/model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ec16c0b98fcebf6052a9aa927090800759b4fb6700367ad2c29354ecbf45f9f7
+oid sha256:09a78ea9ea6e0e17f21e9463d421205fee33f06a038f692cdbc15cc5da5406e6
 size 4976698672

last-checkpoint/model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cd85acac0b1cf3d6b603028d0abef6bbae49730ebe45add6807617156b350d1c
+oid sha256:67ebff0a52efc66dec3162b74b95953b162147486a84d2998d089213d5696860
 size 4999802720

last-checkpoint/model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2df1f85a263fda844c3a5170fff9df97853e8127b0a5eddf0ad7744a2325916c
+oid sha256:d9fbac7a8745fc91c66963c503ba1c883ddde4758539b7301fb550afd9e30274
 size 4915916176

last-checkpoint/model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3e480518cebd078a58ebf6e0cea1f57aa4919ad9372aba8cc8a04682ef0e504f
+oid sha256:254f93f00f3d0d8c2a810b3561ea09850b051a412761bbe113d75573d3f92b1b
 size 1168138808

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b4f003069486a57c6ac033f30cf4c4213eb6b7d659bab68a5a50fdb8da7c4118
+oid sha256:b8f22ced19e790cc864cefe3b7c711d9ae631c44f95d42fb4829688cc3de0153
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a016ef89b4392d083b2c15a7cf06a39bc61a759f648cf6dc03f1c32b89a526aa
+oid sha256:3e0407513eba77d34cbf3adf0e59a58bd80716f4f00f414854253637e82be43d
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9b56fe0893036dc052d18d90feba4328b90ea71561942150b07406ac3d7a700e
+oid sha256:6060636c023258ce9b965e244b8a58b4c99d5784dde4405b39737550ef50cd4f
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c0c203d12c2c308dab785ed672c9ca27fb6a2f72acd1e1552d1516c7b0006013
+oid sha256:c24ccdfdcde39cb2265c82c50c36ffdfcc670f757aba4bcf4bb0fdc6d1373c4c
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3ace6290c890a8d1e173a6da04a3c0a74aa055e1dc2c0b019def7feb7e061c29
+oid sha256:2800f1191a8da3fdb8a578f3e45335b90e0bd680c7897d41e35ad73896db01cd
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.710498182595681,
+  "epoch": 1.9978618772717553,
   "eval_steps": 100,
-  "global_step": 1500,
+  "global_step": 1752,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -11497,6 +11497,1928 @@
       "eval_samples_per_second": 4.648,
       "eval_steps_per_second": 1.162,
       "step": 1500
+    },
+    {
+      "epoch": 1.712778846839142,
+      "grad_norm": 59.337175090307106,
+      "learning_rate": 1.1090063682116202e-08,
+      "logits/chosen": -1.2650439739227295,
+      "logits/rejected": -1.3502650260925293,
+      "logps/chosen": -183.25546264648438,
+      "logps/rejected": -224.42698669433594,
+      "loss": 0.4722,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.49561411142349243,
+      "rewards/margins": 1.1255251169204712,
+      "rewards/rejected": -1.6211390495300293,
+      "step": 1502
+    },
+    {
+      "epoch": 1.7150595110826028,
+      "grad_norm": 65.4693723853712,
+      "learning_rate": 1.091662101338714e-08,
+      "logits/chosen": -1.198454737663269,
+      "logits/rejected": -1.1747905015945435,
+      "logps/chosen": -176.22344970703125,
+      "logps/rejected": -209.0,
+      "loss": 0.428,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.4550539553165436,
+      "rewards/margins": 1.0402750968933105,
+      "rewards/rejected": -1.4953290224075317,
+      "step": 1504
+    },
+    {
+      "epoch": 1.7173401753260638,
+      "grad_norm": 56.55232069175886,
+      "learning_rate": 1.0744466997123425e-08,
+      "logits/chosen": -1.131535291671753,
+      "logits/rejected": -1.2650810480117798,
+      "logps/chosen": -149.54214477539062,
+      "logps/rejected": -207.89288330078125,
+      "loss": 0.4043,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.31441810727119446,
+      "rewards/margins": 1.342848777770996,
+      "rewards/rejected": -1.6572668552398682,
+      "step": 1506
+    },
+    {
+      "epoch": 1.7196208395695245,
+      "grad_norm": 51.8349753003821,
+      "learning_rate": 1.0573604123651503e-08,
+      "logits/chosen": -1.3305891752243042,
+      "logits/rejected": -1.357151985168457,
+      "logps/chosen": -193.94952392578125,
+      "logps/rejected": -216.302490234375,
+      "loss": 0.3987,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.3934152126312256,
+      "rewards/margins": 1.1823620796203613,
+      "rewards/rejected": -1.575777292251587,
+      "step": 1508
+    },
+    {
+      "epoch": 1.7219015038129855,
+      "grad_norm": 63.60872727771506,
+      "learning_rate": 1.0404034864620603e-08,
+      "logits/chosen": -1.174712061882019,
+      "logits/rejected": -1.2022875547409058,
+      "logps/chosen": -148.9698486328125,
+      "logps/rejected": -179.8749542236328,
+      "loss": 0.4382,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.2437065690755844,
+      "rewards/margins": 1.0777262449264526,
+      "rewards/rejected": -1.3214329481124878,
+      "step": 1510
+    },
+    {
+      "epoch": 1.7241821680564464,
+      "grad_norm": 56.263603035763026,
+      "learning_rate": 1.0235761672966902e-08,
+      "logits/chosen": -1.2788983583450317,
+      "logits/rejected": -1.2880148887634277,
+      "logps/chosen": -158.33840942382812,
+      "logps/rejected": -187.15737915039062,
+      "loss": 0.397,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.34705379605293274,
+      "rewards/margins": 1.0756170749664307,
+      "rewards/rejected": -1.422670841217041,
+      "step": 1512
+    },
+    {
+      "epoch": 1.7264628322999074,
+      "grad_norm": 54.16757407624544,
+      "learning_rate": 1.0068786982878087e-08,
+      "logits/chosen": -1.3718278408050537,
+      "logits/rejected": -1.337512731552124,
+      "logps/chosen": -133.3784942626953,
+      "logps/rejected": -142.33865356445312,
+      "loss": 0.3983,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.2702075242996216,
+      "rewards/margins": 0.9396376013755798,
+      "rewards/rejected": -1.2098451852798462,
+      "step": 1514
+    },
+    {
+      "epoch": 1.7287434965433683,
+      "grad_norm": 65.43992152706379,
+      "learning_rate": 9.903113209758097e-09,
+      "logits/chosen": -1.174783706665039,
+      "logits/rejected": -1.1753756999969482,
+      "logps/chosen": -141.90370178222656,
+      "logps/rejected": -165.7357940673828,
+      "loss": 0.3958,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.39702969789505005,
+      "rewards/margins": 1.0974403619766235,
+      "rewards/rejected": -1.494470238685608,
+      "step": 1516
+    },
+    {
+      "epoch": 1.7310241607868293,
+      "grad_norm": 53.07155795474272,
+      "learning_rate": 9.73874275019223e-09,
+      "logits/chosen": -1.2876778841018677,
+      "logits/rejected": -1.2978581190109253,
+      "logps/chosen": -172.82862854003906,
+      "logps/rejected": -199.86740112304688,
+      "loss": 0.4196,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.360828161239624,
+      "rewards/margins": 0.7934844493865967,
+      "rewards/rejected": -1.1543126106262207,
+      "step": 1518
+    },
+    {
+      "epoch": 1.73330482503029,
+      "grad_norm": 56.60655829896917,
+      "learning_rate": 9.575677981912456e-09,
+      "logits/chosen": -1.3419064283370972,
+      "logits/rejected": -1.4197977781295776,
+      "logps/chosen": -215.0706787109375,
+      "logps/rejected": -234.6818084716797,
+      "loss": 0.4216,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.36836767196655273,
+      "rewards/margins": 0.9383161067962646,
+      "rewards/rejected": -1.3066837787628174,
+      "step": 1520
+    },
+    {
+      "epoch": 1.735585489273751,
+      "grad_norm": 60.4365358347784,
+      "learning_rate": 9.413921263763024e-09,
+      "logits/chosen": -1.1901061534881592,
+      "logits/rejected": -1.1732714176177979,
+      "logps/chosen": -148.09774780273438,
+      "logps/rejected": -138.72329711914062,
+      "loss": 0.4147,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.24259831011295319,
+      "rewards/margins": 0.738941490650177,
+      "rewards/rejected": -0.9815397262573242,
+      "step": 1522
+    },
+    {
+      "epoch": 1.7378661535172117,
+      "grad_norm": 61.21131572090355,
+      "learning_rate": 9.253474935666338e-09,
+      "logits/chosen": -1.319941520690918,
+      "logits/rejected": -1.32881498336792,
+      "logps/chosen": -186.58230590820312,
+      "logps/rejected": -211.57525634765625,
+      "loss": 0.5162,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.4148866832256317,
+      "rewards/margins": 1.0180983543395996,
+      "rewards/rejected": -1.4329850673675537,
+      "step": 1524
+    },
+    {
+      "epoch": 1.7401468177606727,
+      "grad_norm": 55.57275725271543,
+      "learning_rate": 9.094341318589071e-09,
+      "logits/chosen": -1.1223199367523193,
+      "logits/rejected": -1.2205249071121216,
+      "logps/chosen": -123.37957763671875,
+      "logps/rejected": -155.87530517578125,
+      "loss": 0.4639,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.4156235158443451,
+      "rewards/margins": 0.8936042785644531,
+      "rewards/rejected": -1.309227705001831,
+      "step": 1526
+    },
+    {
+      "epoch": 1.7424274820041337,
+      "grad_norm": 75.32187086371678,
+      "learning_rate": 8.936522714508676e-09,
+      "logits/chosen": -1.1820390224456787,
+      "logits/rejected": -1.227403163909912,
+      "logps/chosen": -240.14578247070312,
+      "logps/rejected": -284.03839111328125,
+      "loss": 0.4268,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.9300968647003174,
+      "rewards/margins": 1.1027811765670776,
+      "rewards/rejected": -2.0328779220581055,
+      "step": 1528
+    },
+    {
+      "epoch": 1.7447081462475946,
+      "grad_norm": 56.43871330540775,
+      "learning_rate": 8.780021406380011e-09,
+      "logits/chosen": -1.1268565654754639,
+      "logits/rejected": -1.2307937145233154,
+      "logps/chosen": -176.69204711914062,
+      "logps/rejected": -203.42315673828125,
+      "loss": 0.433,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.7230830788612366,
+      "rewards/margins": 1.0807398557662964,
+      "rewards/rejected": -1.8038227558135986,
+      "step": 1530
+    },
+    {
+      "epoch": 1.7469888104910556,
+      "grad_norm": 53.44401452377932,
+      "learning_rate": 8.624839658102345e-09,
+      "logits/chosen": -1.172848105430603,
+      "logits/rejected": -1.2623870372772217,
+      "logps/chosen": -172.2826385498047,
+      "logps/rejected": -218.0788116455078,
+      "loss": 0.4167,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.4317859411239624,
+      "rewards/margins": 1.20097017288208,
+      "rewards/rejected": -1.632756233215332,
+      "step": 1532
+    },
+    {
+      "epoch": 1.7492694747345165,
+      "grad_norm": 44.672996526234876,
+      "learning_rate": 8.470979714486626e-09,
+      "logits/chosen": -1.2581678628921509,
+      "logits/rejected": -1.3436583280563354,
+      "logps/chosen": -164.71560668945312,
+      "logps/rejected": -197.1131134033203,
+      "loss": 0.3891,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.48163390159606934,
+      "rewards/margins": 1.20646333694458,
+      "rewards/rejected": -1.688097357749939,
+      "step": 1534
+    },
+    {
+      "epoch": 1.7515501389779773,
+      "grad_norm": 59.7183121930943,
+      "learning_rate": 8.318443801222918e-09,
+      "logits/chosen": -1.1442248821258545,
+      "logits/rejected": -1.2501918077468872,
+      "logps/chosen": -155.9255828857422,
+      "logps/rejected": -185.97073364257812,
+      "loss": 0.4332,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.4522356390953064,
+      "rewards/margins": 0.7766329050064087,
+      "rewards/rejected": -1.2288686037063599,
+      "step": 1536
+    },
+    {
+      "epoch": 1.7538308032214383,
+      "grad_norm": 52.4727599794761,
+      "learning_rate": 8.167234124848343e-09,
+      "logits/chosen": -1.2721006870269775,
+      "logits/rejected": -1.3032159805297852,
+      "logps/chosen": -213.73348999023438,
+      "logps/rejected": -231.73304748535156,
+      "loss": 0.4188,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.38457486033439636,
+      "rewards/margins": 1.214958667755127,
+      "rewards/rejected": -1.5995336771011353,
+      "step": 1538
+    },
+    {
+      "epoch": 1.756111467464899,
+      "grad_norm": 56.28899819514317,
+      "learning_rate": 8.017352872715077e-09,
+      "logits/chosen": -1.246867299079895,
+      "logits/rejected": -1.321312665939331,
+      "logps/chosen": -166.28347778320312,
+      "logps/rejected": -189.37855529785156,
+      "loss": 0.3864,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.6479480266571045,
+      "rewards/margins": 0.916532039642334,
+      "rewards/rejected": -1.5644800662994385,
+      "step": 1540
+    },
+    {
+      "epoch": 1.75839213170836,
+      "grad_norm": 65.96019718362385,
+      "learning_rate": 7.868802212958703e-09,
+      "logits/chosen": -1.2408630847930908,
+      "logits/rejected": -1.3144943714141846,
+      "logps/chosen": -148.76805114746094,
+      "logps/rejected": -169.2152099609375,
+      "loss": 0.4679,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.313798189163208,
+      "rewards/margins": 0.8195222020149231,
+      "rewards/rejected": -1.1333203315734863,
+      "step": 1542
+    },
+    {
+      "epoch": 1.760672795951821,
+      "grad_norm": 67.11700647965242,
+      "learning_rate": 7.721584294466899e-09,
+      "logits/chosen": -1.3456957340240479,
+      "logits/rejected": -1.3798415660858154,
+      "logps/chosen": -180.2974090576172,
+      "logps/rejected": -203.2277374267578,
+      "loss": 0.4066,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.5308500528335571,
+      "rewards/margins": 0.968846321105957,
+      "rewards/rejected": -1.4996963739395142,
+      "step": 1544
+    },
+    {
+      "epoch": 1.7629534601952819,
+      "grad_norm": 59.467637180664155,
+      "learning_rate": 7.575701246848299e-09,
+      "logits/chosen": -1.2550122737884521,
+      "logits/rejected": -1.4017306566238403,
+      "logps/chosen": -176.01060485839844,
+      "logps/rejected": -221.2160186767578,
+      "loss": 0.4151,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.6037588715553284,
+      "rewards/margins": 1.1545339822769165,
+      "rewards/rejected": -1.7582929134368896,
+      "step": 1546
+    },
+    {
+      "epoch": 1.7652341244387428,
+      "grad_norm": 59.34463361306739,
+      "learning_rate": 7.431155180401704e-09,
+      "logits/chosen": -1.1449761390686035,
+      "logits/rejected": -1.2112215757369995,
+      "logps/chosen": -175.2537384033203,
+      "logps/rejected": -213.36187744140625,
+      "loss": 0.3787,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.547699511051178,
+      "rewards/margins": 1.1472948789596558,
+      "rewards/rejected": -1.694994568824768,
+      "step": 1548
+    },
+    {
+      "epoch": 1.7675147886822038,
+      "grad_norm": 107.3697379891581,
+      "learning_rate": 7.287948186085613e-09,
+      "logits/chosen": -1.1891409158706665,
+      "logits/rejected": -1.2180662155151367,
+      "logps/chosen": -101.96951293945312,
+      "logps/rejected": -131.66488647460938,
+      "loss": 0.5419,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.35958331823349,
+      "rewards/margins": 0.8678293228149414,
+      "rewards/rejected": -1.2274125814437866,
+      "step": 1550
+    },
+    {
+      "epoch": 1.7697954529256648,
+      "grad_norm": 58.46793417479867,
+      "learning_rate": 7.146082335487824e-09,
+      "logits/chosen": -1.2643113136291504,
+      "logits/rejected": -1.2983956336975098,
+      "logps/chosen": -184.83563232421875,
+      "logps/rejected": -210.6478729248047,
+      "loss": 0.417,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.5568149089813232,
+      "rewards/margins": 0.8875846862792969,
+      "rewards/rejected": -1.4443995952606201,
+      "step": 1552
+    },
+    {
+      "epoch": 1.7720761171691255,
+      "grad_norm": 68.29885233859635,
+      "learning_rate": 7.005559680795658e-09,
+      "logits/chosen": -1.255906105041504,
+      "logits/rejected": -1.2899165153503418,
+      "logps/chosen": -123.41840362548828,
+      "logps/rejected": -138.2176055908203,
+      "loss": 0.4641,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.31046992540359497,
+      "rewards/margins": 0.6126350164413452,
+      "rewards/rejected": -0.9231049418449402,
+      "step": 1554
+    },
+    {
+      "epoch": 1.7743567814125865,
+      "grad_norm": 54.33302324585405,
+      "learning_rate": 6.866382254766156e-09,
+      "logits/chosen": -1.330174207687378,
+      "logits/rejected": -1.363855242729187,
+      "logps/chosen": -196.70680236816406,
+      "logps/rejected": -213.6434326171875,
+      "loss": 0.4372,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.5205326676368713,
+      "rewards/margins": 1.0018987655639648,
+      "rewards/rejected": -1.5224316120147705,
+      "step": 1556
+    },
+    {
+      "epoch": 1.7766374456560472,
+      "grad_norm": 66.94592762976625,
+      "learning_rate": 6.7285520706966914e-09,
+      "logits/chosen": -1.3225196599960327,
+      "logits/rejected": -1.3494971990585327,
+      "logps/chosen": -185.50836181640625,
+      "logps/rejected": -207.73159790039062,
+      "loss": 0.4265,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.5292472839355469,
+      "rewards/margins": 0.988534152507782,
+      "rewards/rejected": -1.5177814960479736,
+      "step": 1558
+    },
+    {
+      "epoch": 1.7789181098995082,
+      "grad_norm": 66.14902165695068,
+      "learning_rate": 6.592071122395848e-09,
+      "logits/chosen": -1.2869586944580078,
+      "logits/rejected": -1.2955509424209595,
+      "logps/chosen": -203.05213928222656,
+      "logps/rejected": -221.87176513671875,
+      "loss": 0.4555,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.4785195589065552,
+      "rewards/margins": 0.8982763886451721,
+      "rewards/rejected": -1.3767958879470825,
+      "step": 1560
+    },
+    {
+      "epoch": 1.7811987741429691,
+      "grad_norm": 56.82906531266912,
+      "learning_rate": 6.4569413841546124e-09,
+      "logits/chosen": -1.3752797842025757,
+      "logits/rejected": -1.342197060585022,
+      "logps/chosen": -213.2560577392578,
+      "logps/rejected": -226.26138305664062,
+      "loss": 0.3949,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.6319646835327148,
+      "rewards/margins": 0.7403106093406677,
+      "rewards/rejected": -1.3722753524780273,
+      "step": 1562
+    },
+    {
+      "epoch": 1.78347943838643,
+      "grad_norm": 57.01107575636888,
+      "learning_rate": 6.323164810717751e-09,
+      "logits/chosen": -1.4058406352996826,
+      "logits/rejected": -1.4284158945083618,
+      "logps/chosen": -164.4686279296875,
+      "logps/rejected": -175.3655548095703,
+      "loss": 0.4426,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.2919442057609558,
+      "rewards/margins": 0.8540188074111938,
+      "rewards/rejected": -1.1459629535675049,
+      "step": 1564
+    },
+    {
+      "epoch": 1.785760102629891,
+      "grad_norm": 53.78725622915069,
+      "learning_rate": 6.1907433372555885e-09,
+      "logits/chosen": -1.227329969406128,
+      "logits/rejected": -1.2803211212158203,
+      "logps/chosen": -215.01806640625,
+      "logps/rejected": -240.11415100097656,
+      "loss": 0.4856,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.37019410729408264,
+      "rewards/margins": 1.2054839134216309,
+      "rewards/rejected": -1.5756779909133911,
+      "step": 1566
+    },
+    {
+      "epoch": 1.788040766873352,
+      "grad_norm": 50.06277787185665,
+      "learning_rate": 6.0596788793360055e-09,
+      "logits/chosen": -1.2019214630126953,
+      "logits/rejected": -1.167099952697754,
+      "logps/chosen": -128.18609619140625,
+      "logps/rejected": -156.6576690673828,
+      "loss": 0.3964,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.24505016207695007,
+      "rewards/margins": 0.7954214811325073,
+      "rewards/rejected": -1.0404715538024902,
+      "step": 1568
+    },
+    {
+      "epoch": 1.7903214311168127,
+      "grad_norm": 58.820202526748965,
+      "learning_rate": 5.929973332896676e-09,
+      "logits/chosen": -1.295718789100647,
+      "logits/rejected": -1.3617008924484253,
+      "logps/chosen": -177.0591583251953,
+      "logps/rejected": -229.13153076171875,
+      "loss": 0.3765,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.3693293035030365,
+      "rewards/margins": 1.0683492422103882,
+      "rewards/rejected": -1.437678575515747,
+      "step": 1570
+    },
+    {
+      "epoch": 1.7926020953602737,
+      "grad_norm": 69.22569134675261,
+      "learning_rate": 5.801628574217732e-09,
+      "logits/chosen": -1.231567621231079,
+      "logits/rejected": -1.2869716882705688,
+      "logps/chosen": -115.6909408569336,
+      "logps/rejected": -169.88333129882812,
+      "loss": 0.4175,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.2287057340145111,
+      "rewards/margins": 0.9424973130226135,
+      "rewards/rejected": -1.1712028980255127,
+      "step": 1572
+    },
+    {
+      "epoch": 1.7948827596037344,
+      "grad_norm": 57.45741658297443,
+      "learning_rate": 5.674646459894539e-09,
+      "logits/chosen": -1.3439488410949707,
+      "logits/rejected": -1.3449054956436157,
+      "logps/chosen": -169.16116333007812,
+      "logps/rejected": -214.2422332763672,
+      "loss": 0.4044,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.29708918929100037,
+      "rewards/margins": 1.2709224224090576,
+      "rewards/rejected": -1.5680116415023804,
+      "step": 1574
+    },
+    {
+      "epoch": 1.7971634238471954,
+      "grad_norm": 53.78313839896124,
+      "learning_rate": 5.549028826810886e-09,
+      "logits/chosen": -1.3002756834030151,
+      "logits/rejected": -1.305463433265686,
+      "logps/chosen": -191.0225067138672,
+      "logps/rejected": -210.54293823242188,
+      "loss": 0.4336,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.6976014971733093,
+      "rewards/margins": 1.1452404260635376,
+      "rewards/rejected": -1.8428419828414917,
+      "step": 1576
+    },
+    {
+      "epoch": 1.7994440880906564,
+      "grad_norm": 54.73230632912025,
+      "learning_rate": 5.42477749211242e-09,
+      "logits/chosen": -1.1779245138168335,
+      "logits/rejected": -1.1519317626953125,
+      "logps/chosen": -162.1126708984375,
+      "logps/rejected": -177.6374053955078,
+      "loss": 0.4314,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.23180986940860748,
+      "rewards/margins": 1.1439672708511353,
+      "rewards/rejected": -1.375777244567871,
+      "step": 1578
+    },
+    {
+      "epoch": 1.8017247523341173,
+      "grad_norm": 72.74484543771715,
+      "learning_rate": 5.301894253180295e-09,
+      "logits/chosen": -1.179969310760498,
+      "logits/rejected": -1.2561529874801636,
+      "logps/chosen": -142.0485382080078,
+      "logps/rejected": -183.1311492919922,
+      "loss": 0.4323,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3770468533039093,
+      "rewards/margins": 0.8587576150894165,
+      "rewards/rejected": -1.2358046770095825,
+      "step": 1580
+    },
+    {
+      "epoch": 1.8040054165775783,
+      "grad_norm": 52.220335591750064,
+      "learning_rate": 5.180380887605252e-09,
+      "logits/chosen": -1.292273998260498,
+      "logits/rejected": -1.3601034879684448,
+      "logps/chosen": -198.21466064453125,
+      "logps/rejected": -246.2793731689453,
+      "loss": 0.3947,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.4539499282836914,
+      "rewards/margins": 1.7145434617996216,
+      "rewards/rejected": -2.1684935092926025,
+      "step": 1582
+    },
+    {
+      "epoch": 1.8062860808210393,
+      "grad_norm": 53.485899864483656,
+      "learning_rate": 5.060239153161872e-09,
+      "logits/chosen": -1.1770296096801758,
+      "logits/rejected": -1.2744455337524414,
+      "logps/chosen": -188.80868530273438,
+      "logps/rejected": -251.5846405029297,
+      "loss": 0.4114,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.5241818428039551,
+      "rewards/margins": 1.110929250717163,
+      "rewards/rejected": -1.6351109743118286,
+      "step": 1584
+    },
+    {
+      "epoch": 1.8085667450645,
+      "grad_norm": 71.54349228655005,
+      "learning_rate": 4.941470787783131e-09,
+      "logits/chosen": -1.2420802116394043,
+      "logits/rejected": -1.2955926656723022,
+      "logps/chosen": -153.46408081054688,
+      "logps/rejected": -185.44371032714844,
+      "loss": 0.4323,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.4371834695339203,
+      "rewards/margins": 1.0911630392074585,
+      "rewards/rejected": -1.5283464193344116,
+      "step": 1586
+    },
+    {
+      "epoch": 1.810847409307961,
+      "grad_norm": 43.02051086457159,
+      "learning_rate": 4.8240775095352515e-09,
+      "logits/chosen": -1.2829195261001587,
+      "logits/rejected": -1.3175885677337646,
+      "logps/chosen": -147.54412841796875,
+      "logps/rejected": -172.52725219726562,
+      "loss": 0.3904,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.4066880941390991,
+      "rewards/margins": 0.901046633720398,
+      "rewards/rejected": -1.307734727859497,
+      "step": 1588
+    },
+    {
+      "epoch": 1.8131280735514217,
+      "grad_norm": 56.502783078071204,
+      "learning_rate": 4.708061016592923e-09,
+      "logits/chosen": -1.327852725982666,
+      "logits/rejected": -1.3697575330734253,
+      "logps/chosen": -178.9647216796875,
+      "logps/rejected": -206.71951293945312,
+      "loss": 0.4688,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.5031104683876038,
+      "rewards/margins": 1.0050235986709595,
+      "rewards/rejected": -1.5081340074539185,
+      "step": 1590
+    },
+    {
+      "epoch": 1.8154087377948827,
+      "grad_norm": 52.124874452064724,
+      "learning_rate": 4.593422987214668e-09,
+      "logits/chosen": -1.2299569845199585,
+      "logits/rejected": -1.276241421699524,
+      "logps/chosen": -146.35475158691406,
+      "logps/rejected": -174.0493621826172,
+      "loss": 0.4263,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.01635855622589588,
+      "rewards/margins": 1.0491864681243896,
+      "rewards/rejected": -1.0328278541564941,
+      "step": 1592
+    },
+    {
+      "epoch": 1.8176894020383436,
+      "grad_norm": 52.42542929704426,
+      "learning_rate": 4.480165079718568e-09,
+      "logits/chosen": -1.2522296905517578,
+      "logits/rejected": -1.2631944417953491,
+      "logps/chosen": -195.42518615722656,
+      "logps/rejected": -221.43161010742188,
+      "loss": 0.4216,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.45845019817352295,
+      "rewards/margins": 1.135830044746399,
+      "rewards/rejected": -1.5942802429199219,
+      "step": 1594
+    },
+    {
+      "epoch": 1.8199700662818046,
+      "grad_norm": 64.83240480126588,
+      "learning_rate": 4.368288932458308e-09,
+      "logits/chosen": -1.3056426048278809,
+      "logits/rejected": -1.2827690839767456,
+      "logps/chosen": -172.3594207763672,
+      "logps/rejected": -187.4986114501953,
+      "loss": 0.4437,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.4361146092414856,
+      "rewards/margins": 0.8268535733222961,
+      "rewards/rejected": -1.2629680633544922,
+      "step": 1596
+    },
+    {
+      "epoch": 1.8222507305252655,
+      "grad_norm": 52.10599638998205,
+      "learning_rate": 4.257796163799454e-09,
+      "logits/chosen": -1.1653319597244263,
+      "logits/rejected": -1.2118003368377686,
+      "logps/chosen": -109.62616729736328,
+      "logps/rejected": -133.0435028076172,
+      "loss": 0.4041,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.14802826941013336,
+      "rewards/margins": 0.935161828994751,
+      "rewards/rejected": -1.0831902027130127,
+      "step": 1598
+    },
+    {
+      "epoch": 1.8245313947687265,
+      "grad_norm": 55.52964788045627,
+      "learning_rate": 4.1486883720960435e-09,
+      "logits/chosen": -1.1933330297470093,
+      "logits/rejected": -1.228639006614685,
+      "logps/chosen": -149.2159423828125,
+      "logps/rejected": -200.82742309570312,
+      "loss": 0.3988,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.32056179642677307,
+      "rewards/margins": 1.0430424213409424,
+      "rewards/rejected": -1.3636044263839722,
+      "step": 1600
+    },
+    {
+      "epoch": 1.8245313947687265,
+      "eval_logits/chosen": -1.3431406021118164,
+      "eval_logits/rejected": -1.3253653049468994,
+      "eval_logps/chosen": -132.28553771972656,
+      "eval_logps/rejected": -139.2881317138672,
+      "eval_loss": 0.5448750853538513,
+      "eval_rewards/accuracies": 0.7200000286102295,
+      "eval_rewards/chosen": -0.23875679075717926,
+      "eval_rewards/margins": 0.48378121852874756,
+      "eval_rewards/rejected": -0.7225379943847656,
+      "eval_runtime": 20.9101,
+      "eval_samples_per_second": 4.782,
+      "eval_steps_per_second": 1.196,
+      "step": 1600
+    },
+    {
+      "epoch": 1.8268120590121875,
+      "grad_norm": 52.449005735628376,
+      "learning_rate": 4.040967135667472e-09,
+      "logits/chosen": -1.3316993713378906,
+      "logits/rejected": -1.4035625457763672,
+      "logps/chosen": -134.9228057861328,
+      "logps/rejected": -165.58567810058594,
+      "loss": 0.408,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.16615627706050873,
+      "rewards/margins": 0.7779840230941772,
+      "rewards/rejected": -0.9441402554512024,
+      "step": 1602
+    },
+    {
+      "epoch": 1.8290927232556482,
+      "grad_norm": 58.59433738136844,
+      "learning_rate": 3.9346340127756616e-09,
+      "logits/chosen": -1.3039021492004395,
+      "logits/rejected": -1.3089298009872437,
+      "logps/chosen": -196.00987243652344,
+      "logps/rejected": -218.62310791015625,
+      "loss": 0.4161,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.46055513620376587,
+      "rewards/margins": 0.8772752285003662,
+      "rewards/rejected": -1.3378304243087769,
+      "step": 1604
+    },
+    {
+      "epoch": 1.8313733874991092,
+      "grad_norm": 63.586529609086426,
+      "learning_rate": 3.829690541602504e-09,
+      "logits/chosen": -1.173471450805664,
+      "logits/rejected": -1.27016282081604,
+      "logps/chosen": -162.02359008789062,
+      "logps/rejected": -195.39901733398438,
+      "loss": 0.3966,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.4807528853416443,
+      "rewards/margins": 1.1842964887619019,
+      "rewards/rejected": -1.665049433708191,
+      "step": 1606
+    },
+    {
+      "epoch": 1.83365405174257,
+      "grad_norm": 59.32975592376087,
+      "learning_rate": 3.726138240227628e-09,
+      "logits/chosen": -1.255246639251709,
+      "logits/rejected": -1.3199628591537476,
+      "logps/chosen": -132.19796752929688,
+      "logps/rejected": -187.007568359375,
+      "loss": 0.433,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.2542805075645447,
+      "rewards/margins": 1.1002681255340576,
+      "rewards/rejected": -1.3545485734939575,
+      "step": 1608
+    },
+    {
+      "epoch": 1.8359347159860309,
+      "grad_norm": 53.62231825347675,
+      "learning_rate": 3.623978606606426e-09,
+      "logits/chosen": -1.165490984916687,
+      "logits/rejected": -1.1475247144699097,
+      "logps/chosen": -173.6094970703125,
+      "logps/rejected": -202.46153259277344,
+      "loss": 0.3824,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.47014373540878296,
+      "rewards/margins": 0.9737652540206909,
+      "rewards/rejected": -1.4439090490341187,
+      "step": 1610
+    },
+    {
+      "epoch": 1.8382153802294918,
+      "grad_norm": 54.19282245857111,
+      "learning_rate": 3.523213118548407e-09,
+      "logits/chosen": -1.2416139841079712,
+      "logits/rejected": -1.278630256652832,
+      "logps/chosen": -152.09185791015625,
+      "logps/rejected": -186.2200164794922,
+      "loss": 0.4421,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.47047847509384155,
+      "rewards/margins": 0.9149271845817566,
+      "rewards/rejected": -1.3854056596755981,
+      "step": 1612
+    },
+    {
+      "epoch": 1.8404960444729528,
+      "grad_norm": 66.35928406172948,
+      "learning_rate": 3.423843233695789e-09,
+      "logits/chosen": -1.2137271165847778,
+      "logits/rejected": -1.209067940711975,
+      "logps/chosen": -162.6099853515625,
+      "logps/rejected": -168.80848693847656,
+      "loss": 0.5097,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.4765735864639282,
+      "rewards/margins": 0.7680255174636841,
+      "rewards/rejected": -1.2445989847183228,
+      "step": 1614
+    },
+    {
+      "epoch": 1.8427767087164137,
+      "grad_norm": 76.5553780804754,
+      "learning_rate": 3.3258703895024386e-09,
+      "logits/chosen": -1.2277370691299438,
+      "logits/rejected": -1.2694729566574097,
+      "logps/chosen": -167.89581298828125,
+      "logps/rejected": -190.85592651367188,
+      "loss": 0.448,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.586859941482544,
+      "rewards/margins": 0.8319023847579956,
+      "rewards/rejected": -1.4187625646591187,
+      "step": 1616
+    },
+    {
+      "epoch": 1.8450573729598747,
+      "grad_norm": 62.481075662440986,
+      "learning_rate": 3.2292960032130557e-09,
+      "logits/chosen": -1.3022971153259277,
+      "logits/rejected": -1.3909885883331299,
+      "logps/chosen": -138.24668884277344,
+      "logps/rejected": -144.76327514648438,
+      "loss": 0.5064,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.29410520195961,
+      "rewards/margins": 0.5049476623535156,
+      "rewards/rejected": -0.7990528345108032,
+      "step": 1618
+    },
+    {
+      "epoch": 1.8473380372033354,
+      "grad_norm": 49.40037975448036,
+      "learning_rate": 3.134121471842688e-09,
+      "logits/chosen": -1.3413515090942383,
+      "logits/rejected": -1.3459126949310303,
+      "logps/chosen": -168.7082977294922,
+      "logps/rejected": -210.79649353027344,
+      "loss": 0.3703,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.37254247069358826,
+      "rewards/margins": 1.4622108936309814,
+      "rewards/rejected": -1.8347532749176025,
+      "step": 1620
+    },
+    {
+      "epoch": 1.8496187014467964,
+      "grad_norm": 64.64467682395964,
+      "learning_rate": 3.0403481721565373e-09,
+      "logits/chosen": -1.165281891822815,
+      "logits/rejected": -1.2723007202148438,
+      "logps/chosen": -136.62118530273438,
+      "logps/rejected": -162.6814422607422,
+      "loss": 0.4646,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.21550993621349335,
+      "rewards/margins": 0.7950283288955688,
+      "rewards/rejected": -1.0105382204055786,
+      "step": 1622
+    },
+    {
+      "epoch": 1.8518993656902571,
+      "grad_norm": 52.728229173624364,
+      "learning_rate": 2.947977460649975e-09,
+      "logits/chosen": -1.2271634340286255,
+      "logits/rejected": -1.2183120250701904,
+      "logps/chosen": -120.3056640625,
+      "logps/rejected": -152.6262664794922,
+      "loss": 0.4505,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.4195891320705414,
+      "rewards/margins": 0.9882142543792725,
+      "rewards/rejected": -1.4078034162521362,
+      "step": 1624
+    },
+    {
+      "epoch": 1.854180029933718,
+      "grad_norm": 69.89183140204983,
+      "learning_rate": 2.8570106735290144e-09,
+      "logits/chosen": -1.2754062414169312,
+      "logits/rejected": -1.285915732383728,
+      "logps/chosen": -148.9326171875,
+      "logps/rejected": -176.26077270507812,
+      "loss": 0.4184,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.42058467864990234,
+      "rewards/margins": 1.0723397731781006,
+      "rewards/rejected": -1.492924451828003,
+      "step": 1626
+    },
+    {
+      "epoch": 1.856460694177179,
+      "grad_norm": 78.54847866654863,
+      "learning_rate": 2.7674491266909016e-09,
+      "logits/chosen": -1.2117140293121338,
+      "logits/rejected": -1.2298004627227783,
+      "logps/chosen": -187.1058807373047,
+      "logps/rejected": -215.0411376953125,
+      "loss": 0.4243,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.9483327865600586,
+      "rewards/margins": 1.1213573217391968,
+      "rewards/rejected": -2.069690227508545,
+      "step": 1628
+    },
+    {
+      "epoch": 1.85874135842064,
+      "grad_norm": 55.582996135186875,
+      "learning_rate": 2.679294115705144e-09,
+      "logits/chosen": -1.2989763021469116,
+      "logits/rejected": -1.425642967224121,
+      "logps/chosen": -191.18502807617188,
+      "logps/rejected": -235.31173706054688,
+      "loss": 0.3977,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.30440258979797363,
+      "rewards/margins": 1.0316964387893677,
+      "rewards/rejected": -1.3360989093780518,
+      "step": 1630
+    },
+    {
+      "epoch": 1.861022022664101,
+      "grad_norm": 51.611066982113556,
+      "learning_rate": 2.5925469157947135e-09,
+      "logits/chosen": -1.2190608978271484,
+      "logits/rejected": -1.2017196416854858,
+      "logps/chosen": -209.8916778564453,
+      "logps/rejected": -251.41043090820312,
+      "loss": 0.3837,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.5788516402244568,
+      "rewards/margins": 1.5164175033569336,
+      "rewards/rejected": -2.095268964767456,
+      "step": 1632
+    },
+    {
+      "epoch": 1.863302686907562,
+      "grad_norm": 61.654874361770986,
+      "learning_rate": 2.507208781817638e-09,
+      "logits/chosen": -1.2814161777496338,
+      "logits/rejected": -1.4046311378479004,
+      "logps/chosen": -190.91163635253906,
+      "logps/rejected": -240.64810180664062,
+      "loss": 0.4435,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.6178247928619385,
+      "rewards/margins": 1.0759724378585815,
+      "rewards/rejected": -1.6937971115112305,
+      "step": 1634
+    },
+    {
+      "epoch": 1.8655833511510227,
+      "grad_norm": 65.59685553528632,
+      "learning_rate": 2.4232809482488403e-09,
+      "logits/chosen": -1.204872965812683,
+      "logits/rejected": -1.2012598514556885,
+      "logps/chosen": -180.18150329589844,
+      "logps/rejected": -206.04872131347656,
+      "loss": 0.4425,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.6783896684646606,
+      "rewards/margins": 1.0068707466125488,
+      "rewards/rejected": -1.68526029586792,
+      "step": 1636
+    },
+    {
+      "epoch": 1.8678640153944837,
+      "grad_norm": 60.54718272304763,
+      "learning_rate": 2.340764629162284e-09,
+      "logits/chosen": -1.1621663570404053,
+      "logits/rejected": -1.2637563943862915,
+      "logps/chosen": -178.81527709960938,
+      "logps/rejected": -228.64007568359375,
+      "loss": 0.3834,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.3456554114818573,
+      "rewards/margins": 0.9540256261825562,
+      "rewards/rejected": -1.2996809482574463,
+      "step": 1638
+    },
+    {
+      "epoch": 1.8701446796379444,
+      "grad_norm": 55.533431370620015,
+      "learning_rate": 2.2596610182133325e-09,
+      "logits/chosen": -1.2759058475494385,
+      "logits/rejected": -1.3463444709777832,
+      "logps/chosen": -156.00009155273438,
+      "logps/rejected": -192.7170867919922,
+      "loss": 0.4157,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.003083046991378069,
+      "rewards/margins": 1.3031508922576904,
+      "rewards/rejected": -1.3062340021133423,
+      "step": 1640
+    },
+    {
+      "epoch": 1.8724253438814054,
+      "grad_norm": 62.10012065390903,
+      "learning_rate": 2.1799712886216628e-09,
+      "logits/chosen": -1.1718287467956543,
+      "logits/rejected": -1.1970500946044922,
+      "logps/chosen": -146.75628662109375,
+      "logps/rejected": -208.56558227539062,
+      "loss": 0.3858,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.1122959554195404,
+      "rewards/margins": 1.5379095077514648,
+      "rewards/rejected": -1.650205373764038,
+      "step": 1642
+    },
+    {
+      "epoch": 1.8747060081248663,
+      "grad_norm": 56.04768532231553,
+      "learning_rate": 2.1016965931541007e-09,
+      "logits/chosen": -1.253339171409607,
+      "logits/rejected": -1.3260321617126465,
+      "logps/chosen": -207.69583129882812,
+      "logps/rejected": -243.12586975097656,
+      "loss": 0.4223,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.5657081604003906,
+      "rewards/margins": 1.0796537399291992,
+      "rewards/rejected": -1.6453620195388794,
+      "step": 1644
+    },
+    {
+      "epoch": 1.8769866723683273,
+      "grad_norm": 60.14746373631622,
+      "learning_rate": 2.02483806410807e-09,
+      "logits/chosen": -1.114426851272583,
+      "logits/rejected": -1.2585283517837524,
+      "logps/chosen": -144.21237182617188,
+      "logps/rejected": -216.4067840576172,
+      "loss": 0.4165,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.6168836951255798,
+      "rewards/margins": 1.211737036705017,
+      "rewards/rejected": -1.8286206722259521,
+      "step": 1646
+    },
+    {
+      "epoch": 1.8792673366117882,
+      "grad_norm": 58.35041588601004,
+      "learning_rate": 1.9493968132951455e-09,
+      "logits/chosen": -1.2065017223358154,
+      "logits/rejected": -1.2185966968536377,
+      "logps/chosen": -144.52651977539062,
+      "logps/rejected": -182.16015625,
+      "loss": 0.4119,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.5021862387657166,
+      "rewards/margins": 1.0205962657928467,
+      "rewards/rejected": -1.522782564163208,
+      "step": 1648
+    },
+    {
+      "epoch": 1.8815480008552492,
+      "grad_norm": 61.66500286457481,
+      "learning_rate": 1.875373932025015e-09,
+      "logits/chosen": -1.113441824913025,
+      "logits/rejected": -1.2277448177337646,
+      "logps/chosen": -88.60232543945312,
+      "logps/rejected": -123.64103698730469,
+      "loss": 0.5025,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.3603760600090027,
+      "rewards/margins": 0.6354212760925293,
+      "rewards/rejected": -0.9957974553108215,
+      "step": 1650
+    },
+    {
+      "epoch": 1.8838286650987102,
+      "grad_norm": 66.53236233989348,
+      "learning_rate": 1.8027704910896668e-09,
+      "logits/chosen": -1.2696802616119385,
+      "logits/rejected": -1.293856143951416,
+      "logps/chosen": -178.1632843017578,
+      "logps/rejected": -208.1569061279297,
+      "loss": 0.4141,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.4311864674091339,
+      "rewards/margins": 1.18040931224823,
+      "rewards/rejected": -1.611595869064331,
+      "step": 1652
+    },
+    {
+      "epoch": 1.886109329342171,
+      "grad_norm": 73.576105680566,
+      "learning_rate": 1.731587540747903e-09,
+      "logits/chosen": -1.3243728876113892,
+      "logits/rejected": -1.3624733686447144,
+      "logps/chosen": -178.9629669189453,
+      "logps/rejected": -203.73611450195312,
+      "loss": 0.4112,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.35923391580581665,
+      "rewards/margins": 1.0950078964233398,
+      "rewards/rejected": -1.4542417526245117,
+      "step": 1654
+    },
+    {
+      "epoch": 1.8883899935856319,
+      "grad_norm": 60.917071878292,
+      "learning_rate": 1.6618261107101628e-09,
+      "logits/chosen": -1.2426798343658447,
+      "logits/rejected": -1.299846887588501,
+      "logps/chosen": -166.9674530029297,
+      "logps/rejected": -200.53256225585938,
+      "loss": 0.4225,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.36187708377838135,
+      "rewards/margins": 1.0805463790893555,
+      "rewards/rejected": -1.4424233436584473,
+      "step": 1656
+    },
+    {
+      "epoch": 1.8906706578290926,
+      "grad_norm": 58.315768026639354,
+      "learning_rate": 1.5934872101235785e-09,
+      "logits/chosen": -1.1988379955291748,
+      "logits/rejected": -1.276864767074585,
+      "logps/chosen": -117.3199691772461,
+      "logps/rejected": -166.17037963867188,
+      "loss": 0.4669,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.37138426303863525,
+      "rewards/margins": 0.7784909009933472,
+      "rewards/rejected": -1.1498751640319824,
+      "step": 1658
+    },
+    {
+      "epoch": 1.8929513220725536,
+      "grad_norm": 73.00783189642374,
+      "learning_rate": 1.5265718275574656e-09,
+      "logits/chosen": -1.199881672859192,
+      "logits/rejected": -1.294306755065918,
+      "logps/chosen": -163.18499755859375,
+      "logps/rejected": -240.10389709472656,
+      "loss": 0.4091,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.5089311599731445,
+      "rewards/margins": 1.5384039878845215,
+      "rewards/rejected": -2.047335147857666,
+      "step": 1660
+    },
+    {
+      "epoch": 1.8952319863160145,
+      "grad_norm": 59.87965880073438,
+      "learning_rate": 1.4610809309889338e-09,
+      "logits/chosen": -1.1384081840515137,
+      "logits/rejected": -1.2086718082427979,
+      "logps/chosen": -228.3030548095703,
+      "logps/rejected": -291.9572448730469,
+      "loss": 0.3708,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.6747040748596191,
+      "rewards/margins": 1.962480068206787,
+      "rewards/rejected": -2.6371843814849854,
+      "step": 1662
+    },
+    {
+      "epoch": 1.8975126505594755,
+      "grad_norm": 65.04049810626964,
+      "learning_rate": 1.3970154677889312e-09,
+      "logits/chosen": -1.2462139129638672,
+      "logits/rejected": -1.3531625270843506,
+      "logps/chosen": -166.90621948242188,
+      "logps/rejected": -219.62258911132812,
+      "loss": 0.4477,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.6883362531661987,
+      "rewards/margins": 1.3010790348052979,
+      "rewards/rejected": -1.989415168762207,
+      "step": 1664
+    },
+    {
+      "epoch": 1.8997933148029365,
+      "grad_norm": 67.17708620355747,
+      "learning_rate": 1.3343763647085337e-09,
+      "logits/chosen": -1.149139404296875,
+      "logits/rejected": -1.2173185348510742,
+      "logps/chosen": -155.94863891601562,
+      "logps/rejected": -230.28384399414062,
+      "loss": 0.4712,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.5960929989814758,
+      "rewards/margins": 1.4794337749481201,
+      "rewards/rejected": -2.075526714324951,
+      "step": 1666
+    },
+    {
+      "epoch": 1.9020739790463974,
+      "grad_norm": 56.97928801975319,
+      "learning_rate": 1.2731645278655445e-09,
+      "logits/chosen": -1.2667725086212158,
+      "logits/rejected": -1.3436360359191895,
+      "logps/chosen": -160.5927734375,
+      "logps/rejected": -186.63018798828125,
+      "loss": 0.4321,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.48528626561164856,
+      "rewards/margins": 0.931164562702179,
+      "rewards/rejected": -1.4164507389068604,
+      "step": 1668
+    },
+    {
+      "epoch": 1.9043546432898582,
+      "grad_norm": 63.8162468661416,
+      "learning_rate": 1.2133808427313485e-09,
+      "logits/chosen": -1.2804149389266968,
+      "logits/rejected": -1.3368322849273682,
+      "logps/chosen": -131.45181274414062,
+      "logps/rejected": -180.07037353515625,
+      "loss": 0.4551,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.5746269226074219,
+      "rewards/margins": 0.8660170435905457,
+      "rewards/rejected": -1.4406440258026123,
+      "step": 1670
+    },
+    {
+      "epoch": 1.9066353075333191,
+      "grad_norm": 52.46121205171607,
+      "learning_rate": 1.1550261741181565e-09,
+      "logits/chosen": -1.2648401260375977,
+      "logits/rejected": -1.239923119544983,
+      "logps/chosen": -163.7249298095703,
+      "logps/rejected": -190.42611694335938,
+      "loss": 0.3885,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.3933155834674835,
+      "rewards/margins": 1.0268510580062866,
+      "rewards/rejected": -1.4201666116714478,
+      "step": 1672
+    },
+    {
+      "epoch": 1.9089159717767799,
+      "grad_norm": 64.22449427271324,
+      "learning_rate": 1.0981013661664706e-09,
+      "logits/chosen": -1.293068289756775,
+      "logits/rejected": -1.4030743837356567,
+      "logps/chosen": -174.76348876953125,
+      "logps/rejected": -219.83175659179688,
+      "loss": 0.4548,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3287160098552704,
+      "rewards/margins": 0.9871301651000977,
+      "rewards/rejected": -1.3158462047576904,
+      "step": 1674
+    },
+    {
+      "epoch": 1.9111966360202408,
+      "grad_norm": 55.77569297347896,
+      "learning_rate": 1.042607242332838e-09,
+      "logits/chosen": -1.2760488986968994,
+      "logits/rejected": -1.31367027759552,
+      "logps/chosen": -148.86257934570312,
+      "logps/rejected": -173.52359008789062,
+      "loss": 0.4349,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.24598023295402527,
+      "rewards/margins": 0.7863931655883789,
+      "rewards/rejected": -1.0323734283447266,
+      "step": 1676
+    },
+    {
+      "epoch": 1.9134773002637018,
+      "grad_norm": 57.697067279665426,
+      "learning_rate": 9.885446053780277e-10,
+      "logits/chosen": -1.3217543363571167,
+      "logits/rejected": -1.3588594198226929,
+      "logps/chosen": -234.18218994140625,
+      "logps/rejected": -244.6886444091797,
+      "loss": 0.4099,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.6677453517913818,
+      "rewards/margins": 0.864335834980011,
+      "rewards/rejected": -1.5320810079574585,
+      "step": 1678
+    },
+    {
+      "epoch": 1.9157579645071627,
+      "grad_norm": 52.93198869808111,
+      "learning_rate": 9.359142373553286e-10,
+      "logits/chosen": -1.2812589406967163,
+      "logits/rejected": -1.374354362487793,
+      "logps/chosen": -181.33377075195312,
+      "logps/rejected": -226.33584594726562,
+      "loss": 0.3969,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.40316393971443176,
+      "rewards/margins": 1.0113019943237305,
+      "rewards/rejected": -1.4144660234451294,
+      "step": 1680
+    },
+    {
+      "epoch": 1.9180386287506237,
+      "grad_norm": 56.81221018582624,
+      "learning_rate": 8.847168995992915e-10,
+      "logits/chosen": -1.3222296237945557,
+      "logits/rejected": -1.3470890522003174,
+      "logps/chosen": -144.92401123046875,
+      "logps/rejected": -160.66360473632812,
+      "loss": 0.4557,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.2944653630256653,
+      "rewards/margins": 0.691783607006073,
+      "rewards/rejected": -0.9862489104270935,
+      "step": 1682
+    },
+    {
+      "epoch": 1.9203192929940847,
+      "grad_norm": 59.96217669380799,
+      "learning_rate": 8.349533327146719e-10,
+      "logits/chosen": -1.1826375722885132,
+      "logits/rejected": -1.2797571420669556,
+      "logps/chosen": -173.0347442626953,
+      "logps/rejected": -217.1268768310547,
+      "loss": 0.421,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.6681329011917114,
+      "rewards/margins": 1.3352155685424805,
+      "rewards/rejected": -2.0033483505249023,
+      "step": 1684
+    },
+    {
+      "epoch": 1.9225999572375454,
+      "grad_norm": 53.71192157200429,
+      "learning_rate": 7.866242565657599e-10,
+      "logits/chosen": -1.133514642715454,
+      "logits/rejected": -1.2035727500915527,
+      "logps/chosen": -123.30015563964844,
+      "logps/rejected": -165.7233123779297,
+      "loss": 0.4164,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.38910388946533203,
+      "rewards/margins": 1.0756738185882568,
+      "rewards/rejected": -1.4647778272628784,
+      "step": 1686
+    },
+    {
+      "epoch": 1.9248806214810064,
+      "grad_norm": 84.94214636470907,
+      "learning_rate": 7.397303702659674e-10,
+      "logits/chosen": -1.2931269407272339,
+      "logits/rejected": -1.3728893995285034,
+      "logps/chosen": -153.67669677734375,
+      "logps/rejected": -198.1190185546875,
+      "loss": 0.4512,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.3215530216693878,
+      "rewards/margins": 1.3663368225097656,
+      "rewards/rejected": -1.6878899335861206,
+      "step": 1688
+    },
+    {
+      "epoch": 1.927161285724467,
+      "grad_norm": 60.620511783810954,
+      "learning_rate": 6.942723521676464e-10,
+      "logits/chosen": -1.2509236335754395,
+      "logits/rejected": -1.3288919925689697,
+      "logps/chosen": -171.9304962158203,
+      "logps/rejected": -200.7103729248047,
+      "loss": 0.406,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.2758704423904419,
+      "rewards/margins": 0.8382859230041504,
+      "rewards/rejected": -1.1141563653945923,
+      "step": 1690
+    },
+    {
+      "epoch": 1.929441949967928,
+      "grad_norm": 61.96268516778484,
+      "learning_rate": 6.502508598523748e-10,
+      "logits/chosen": -1.1139042377471924,
+      "logits/rejected": -1.113561987876892,
+      "logps/chosen": -145.94715881347656,
+      "logps/rejected": -177.67091369628906,
+      "loss": 0.4153,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.3293991684913635,
+      "rewards/margins": 0.9399027824401855,
+      "rewards/rejected": -1.2693020105361938,
+      "step": 1692
+    },
+    {
+      "epoch": 1.931722614211389,
+      "grad_norm": 64.38956995843415,
+      "learning_rate": 6.076665301213646e-10,
+      "logits/chosen": -1.2710590362548828,
+      "logits/rejected": -1.3398025035858154,
+      "logps/chosen": -258.7908935546875,
+      "logps/rejected": -270.1191711425781,
+      "loss": 0.4663,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.183610200881958,
+      "rewards/margins": 1.2962732315063477,
+      "rewards/rejected": -2.4798836708068848,
+      "step": 1694
+    },
+    {
+      "epoch": 1.93400327845485,
+      "grad_norm": 70.46099460182324,
+      "learning_rate": 5.665199789862907e-10,
+      "logits/chosen": -1.3359112739562988,
+      "logits/rejected": -1.3275290727615356,
+      "logps/chosen": -183.69839477539062,
+      "logps/rejected": -183.45762634277344,
+      "loss": 0.5587,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.5863617658615112,
+      "rewards/margins": 0.39982593059539795,
+      "rewards/rejected": -0.9861876368522644,
+      "step": 1696
+    },
+    {
+      "epoch": 1.936283942698311,
+      "grad_norm": 59.14480772543764,
+      "learning_rate": 5.268118016603651e-10,
+      "logits/chosen": -1.2939709424972534,
+      "logits/rejected": -1.2879596948623657,
+      "logps/chosen": -242.49627685546875,
+      "logps/rejected": -255.47325134277344,
+      "loss": 0.5263,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.6082537174224854,
+      "rewards/margins": 0.8644734621047974,
+      "rewards/rejected": -1.4727270603179932,
+      "step": 1698
+    },
+    {
+      "epoch": 1.938564606941772,
+      "grad_norm": 65.14033380217037,
+      "learning_rate": 4.88542572549755e-10,
+      "logits/chosen": -1.258131742477417,
+      "logits/rejected": -1.341509461402893,
+      "logps/chosen": -280.3218688964844,
+      "logps/rejected": -349.66766357421875,
+      "loss": 0.4044,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.9879501461982727,
+      "rewards/margins": 1.8015985488891602,
+      "rewards/rejected": -2.789548873901367,
+      "step": 1700
+    },
+    {
+      "epoch": 1.938564606941772,
+      "eval_logits/chosen": -1.3368662595748901,
+      "eval_logits/rejected": -1.3193824291229248,
+      "eval_logps/chosen": -132.35743713378906,
+      "eval_logps/rejected": -139.43797302246094,
+      "eval_loss": 0.5376756191253662,
+      "eval_rewards/accuracies": 0.7200000286102295,
+      "eval_rewards/chosen": -0.2459474503993988,
+      "eval_rewards/margins": 0.49157509207725525,
+      "eval_rewards/rejected": -0.7375224828720093,
+      "eval_runtime": 20.9786,
+      "eval_samples_per_second": 4.767,
+      "eval_steps_per_second": 1.192,
+      "step": 1700
+    },
+    {
+      "epoch": 1.9408452711852326,
+      "grad_norm": 67.5831820584933,
+      "learning_rate": 4.5171284524521127e-10,
+      "logits/chosen": -1.1273996829986572,
+      "logits/rejected": -1.200268030166626,
+      "logps/chosen": -141.1074981689453,
+      "logps/rejected": -161.90756225585938,
+      "loss": 0.3986,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.13401609659194946,
+      "rewards/margins": 0.9844987392425537,
+      "rewards/rejected": -1.118514895439148,
+      "step": 1702
+    },
+    {
+      "epoch": 1.9431259354286936,
+      "grad_norm": 64.49222478839803,
+      "learning_rate": 4.163231525141309e-10,
+      "logits/chosen": -1.4064816236495972,
+      "logits/rejected": -1.4361658096313477,
+      "logps/chosen": -240.72976684570312,
+      "logps/rejected": -262.8630676269531,
+      "loss": 0.4409,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.6487561464309692,
+      "rewards/margins": 1.2385480403900146,
+      "rewards/rejected": -1.8873043060302734,
+      "step": 1704
+    },
+    {
+      "epoch": 1.9454065996721546,
+      "grad_norm": 50.83672179405874,
+      "learning_rate": 3.8237400629280714e-10,
+      "logits/chosen": -1.0193315744400024,
+      "logits/rejected": -0.9887692928314209,
+      "logps/chosen": -120.17151641845703,
+      "logps/rejected": -136.94674682617188,
+      "loss": 0.4543,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.27739959955215454,
+      "rewards/margins": 0.6725433468818665,
+      "rewards/rejected": -0.949942946434021,
+      "step": 1706
+    },
+    {
+      "epoch": 1.9476872639156153,
+      "grad_norm": 53.5967901694209,
+      "learning_rate": 3.4986589767902476e-10,
+      "logits/chosen": -1.2411226034164429,
+      "logits/rejected": -1.3345215320587158,
+      "logps/chosen": -90.0027084350586,
+      "logps/rejected": -116.62909698486328,
+      "loss": 0.4495,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.078713558614254,
+      "rewards/margins": 0.6798267364501953,
+      "rewards/rejected": -0.7585403323173523,
+      "step": 1708
+    },
+    {
+      "epoch": 1.9499679281590763,
+      "grad_norm": 59.266256355757655,
+      "learning_rate": 3.187992969249875e-10,
+      "logits/chosen": -1.2389315366744995,
+      "logits/rejected": -1.2326477766036987,
+      "logps/chosen": -127.89494323730469,
+      "logps/rejected": -154.4901580810547,
+      "loss": 0.4366,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.4514698088169098,
+      "rewards/margins": 0.6477700471878052,
+      "rewards/rejected": -1.0992399454116821,
+      "step": 1710
+    },
+    {
+      "epoch": 1.9522485924025372,
+      "grad_norm": 62.77597760270175,
+      "learning_rate": 2.8917465343047954e-10,
+      "logits/chosen": -1.3058414459228516,
+      "logits/rejected": -1.374163269996643,
+      "logps/chosen": -171.95448303222656,
+      "logps/rejected": -207.97914123535156,
+      "loss": 0.3709,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.32011568546295166,
+      "rewards/margins": 1.2897684574127197,
+      "rewards/rejected": -1.6098840236663818,
+      "step": 1712
+    },
+    {
+      "epoch": 1.9545292566459982,
+      "grad_norm": 56.41697287299428,
+      "learning_rate": 2.609923957363702e-10,
+      "logits/chosen": -1.2662739753723145,
+      "logits/rejected": -1.3142364025115967,
+      "logps/chosen": -149.61746215820312,
+      "logps/rejected": -184.44931030273438,
+      "loss": 0.4197,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.28848132491111755,
+      "rewards/margins": 1.1091067790985107,
+      "rewards/rejected": -1.3975881338119507,
+      "step": 1714
+    },
+    {
+      "epoch": 1.9568099208894592,
+      "grad_norm": 58.90785637715505,
+      "learning_rate": 2.3425293151845273e-10,
+      "logits/chosen": -1.2464115619659424,
+      "logits/rejected": -1.2890043258666992,
+      "logps/chosen": -146.8938446044922,
+      "logps/rejected": -162.07830810546875,
+      "loss": 0.4128,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.2892056405544281,
+      "rewards/margins": 0.7288376092910767,
+      "rewards/rejected": -1.0180431604385376,
+      "step": 1716
+    },
+    {
+      "epoch": 1.9590905851329201,
+      "grad_norm": 66.05988715134325,
+      "learning_rate": 2.0895664758151521e-10,
+      "logits/chosen": -1.1935899257659912,
+      "logits/rejected": -1.254826545715332,
+      "logps/chosen": -194.64492797851562,
+      "logps/rejected": -225.4056854248047,
+      "loss": 0.4007,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.525715172290802,
+      "rewards/margins": 1.1838706731796265,
+      "rewards/rejected": -1.7095859050750732,
+      "step": 1718
+    },
+    {
+      "epoch": 1.9613712493763809,
+      "grad_norm": 64.18176403648182,
+      "learning_rate": 1.8510390985371216e-10,
+      "logits/chosen": -1.1969249248504639,
+      "logits/rejected": -1.2043269872665405,
+      "logps/chosen": -201.9025115966797,
+      "logps/rejected": -233.6451873779297,
+      "loss": 0.4386,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.5428147315979004,
+      "rewards/margins": 0.8293758630752563,
+      "rewards/rejected": -1.3721905946731567,
+      "step": 1720
+    },
+    {
+      "epoch": 1.9636519136198418,
+      "grad_norm": 58.86675845104934,
+      "learning_rate": 1.626950633813351e-10,
+      "logits/chosen": -1.2230490446090698,
+      "logits/rejected": -1.2853928804397583,
+      "logps/chosen": -160.97621154785156,
+      "logps/rejected": -223.6080780029297,
+      "loss": 0.4377,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.3992640972137451,
+      "rewards/margins": 1.2427630424499512,
+      "rewards/rejected": -1.6420272588729858,
+      "step": 1722
+    },
+    {
+      "epoch": 1.9659325778633026,
+      "grad_norm": 63.84847943868757,
+      "learning_rate": 1.4173043232380554e-10,
+      "logits/chosen": -1.1713950634002686,
+      "logits/rejected": -1.2190814018249512,
+      "logps/chosen": -160.047119140625,
+      "logps/rejected": -184.2767791748047,
+      "loss": 0.4372,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.4601638913154602,
+      "rewards/margins": 0.9584896564483643,
+      "rewards/rejected": -1.4186536073684692,
+      "step": 1724
+    },
+    {
+      "epoch": 1.9682132421067635,
+      "grad_norm": 64.3589769132475,
+      "learning_rate": 1.222103199489455e-10,
+      "logits/chosen": -1.3236342668533325,
+      "logits/rejected": -1.3482894897460938,
+      "logps/chosen": -182.65269470214844,
+      "logps/rejected": -217.88861083984375,
+      "loss": 0.4358,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.43886712193489075,
+      "rewards/margins": 1.474854588508606,
+      "rewards/rejected": -1.9137215614318848,
+      "step": 1726
+    },
+    {
+      "epoch": 1.9704939063502245,
+      "grad_norm": 71.97812711207425,
+      "learning_rate": 1.0413500862864743e-10,
+      "logits/chosen": -1.377000093460083,
+      "logits/rejected": -1.4007121324539185,
+      "logps/chosen": -237.48316955566406,
+      "logps/rejected": -259.9788818359375,
+      "loss": 0.4143,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.6071898937225342,
+      "rewards/margins": 1.190570592880249,
+      "rewards/rejected": -1.7977604866027832,
+      "step": 1728
+    },
+    {
+      "epoch": 1.9727745705936854,
+      "grad_norm": 61.7108383512953,
+      "learning_rate": 8.750475983472227e-11,
+      "logits/chosen": -1.2676740884780884,
+      "logits/rejected": -1.2757924795150757,
+      "logps/chosen": -190.92140197753906,
+      "logps/rejected": -197.59115600585938,
+      "loss": 0.4226,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.5921288132667542,
+      "rewards/margins": 1.0576057434082031,
+      "rewards/rejected": -1.6497344970703125,
+      "step": 1730
+    },
+    {
+      "epoch": 1.9750552348371464,
+      "grad_norm": 188.64844039216806,
+      "learning_rate": 7.231981413520217e-11,
+      "logits/chosen": -1.263267993927002,
+      "logits/rejected": -1.2889572381973267,
+      "logps/chosen": -157.240478515625,
+      "logps/rejected": -197.0875701904297,
+      "loss": 0.4582,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.24919547140598297,
+      "rewards/margins": 0.7916581630706787,
+      "rewards/rejected": -1.0408536195755005,
+      "step": 1732
+    },
+    {
+      "epoch": 1.9773358990806074,
+      "grad_norm": 65.4504778217422,
+      "learning_rate": 5.858039119077673e-11,
+      "logits/chosen": -1.2087372541427612,
+      "logits/rejected": -1.2781308889389038,
+      "logps/chosen": -176.43057250976562,
+      "logps/rejected": -216.8165740966797,
+      "loss": 0.4563,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.41199278831481934,
+      "rewards/margins": 0.922531008720398,
+      "rewards/rejected": -1.3345237970352173,
+      "step": 1734
+    },
+    {
+      "epoch": 1.979616563324068,
+      "grad_norm": 61.72345732533114,
+      "learning_rate": 4.628668975166228e-11,
+      "logits/chosen": -1.4069479703903198,
+      "logits/rejected": -1.425885558128357,
+      "logps/chosen": -155.35098266601562,
+      "logps/rejected": -172.41546630859375,
+      "loss": 0.3891,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.3574288785457611,
+      "rewards/margins": 1.1354269981384277,
+      "rewards/rejected": -1.4928559064865112,
+      "step": 1736
+    },
+    {
+      "epoch": 1.981897227567529,
+      "grad_norm": 50.29362647543594,
+      "learning_rate": 3.5438887654737346e-11,
+      "logits/chosen": -1.1932240724563599,
+      "logits/rejected": -1.2252675294876099,
+      "logps/chosen": -104.49500274658203,
+      "logps/rejected": -128.2619171142578,
+      "loss": 0.3998,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.0990162342786789,
+      "rewards/margins": 0.8109432458877563,
+      "rewards/rejected": -0.9099595546722412,
+      "step": 1738
+    },
+    {
+      "epoch": 1.9841778918109898,
+      "grad_norm": 63.67796104250234,
+      "learning_rate": 2.603714182093375e-11,
+      "logits/chosen": -1.2760851383209229,
+      "logits/rejected": -1.3577792644500732,
+      "logps/chosen": -172.22714233398438,
+      "logps/rejected": -215.27749633789062,
+      "loss": 0.4565,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.560702919960022,
+      "rewards/margins": 1.0820647478103638,
+      "rewards/rejected": -1.6427676677703857,
+      "step": 1740
+    },
+    {
+      "epoch": 1.9864585560544508,
+      "grad_norm": 58.508228872135426,
+      "learning_rate": 1.808158825297168e-11,
+      "logits/chosen": -1.3208973407745361,
+      "logits/rejected": -1.3905422687530518,
+      "logps/chosen": -147.84127807617188,
+      "logps/rejected": -178.2074737548828,
+      "loss": 0.4758,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.48498719930648804,
+      "rewards/margins": 0.888887882232666,
+      "rewards/rejected": -1.3738751411437988,
+      "step": 1742
+    },
+    {
+      "epoch": 1.9887392202979117,
+      "grad_norm": 58.05411220405763,
+      "learning_rate": 1.1572342033416838e-11,
+      "logits/chosen": -1.3570505380630493,
+      "logits/rejected": -1.3865540027618408,
+      "logps/chosen": -171.88287353515625,
+      "logps/rejected": -210.6277618408203,
+      "loss": 0.4492,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.28449732065200806,
+      "rewards/margins": 1.1635254621505737,
+      "rewards/rejected": -1.448022723197937,
+      "step": 1744
+    },
+    {
+      "epoch": 1.9910198845413727,
+      "grad_norm": 60.292954042523704,
+      "learning_rate": 6.50949732301509e-12,
+      "logits/chosen": -1.2242615222930908,
+      "logits/rejected": -1.3328139781951904,
+      "logps/chosen": -164.70863342285156,
+      "logps/rejected": -213.20814514160156,
+      "loss": 0.3912,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.497256338596344,
+      "rewards/margins": 1.1804652214050293,
+      "rewards/rejected": -1.677721619606018,
+      "step": 1746
+    },
+    {
+      "epoch": 1.9933005487848336,
+      "grad_norm": 68.33201893652561,
+      "learning_rate": 2.893127359282488e-12,
+      "logits/chosen": -1.2822688817977905,
+      "logits/rejected": -1.3241004943847656,
+      "logps/chosen": -207.72021484375,
+      "logps/rejected": -262.7379455566406,
+      "loss": 0.3881,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.34897035360336304,
+      "rewards/margins": 1.5757369995117188,
+      "rewards/rejected": -1.924707293510437,
+      "step": 1748
+    },
+    {
+      "epoch": 1.9955812130282946,
+      "grad_norm": 62.482127853658426,
+      "learning_rate": 7.232844555282725e-13,
+      "logits/chosen": -1.3659021854400635,
+      "logits/rejected": -1.3771145343780518,
+      "logps/chosen": -201.26531982421875,
+      "logps/rejected": -219.556640625,
+      "loss": 0.3859,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.7029599547386169,
+      "rewards/margins": 1.120865821838379,
+      "rewards/rejected": -1.823825716972351,
+      "step": 1750
+    },
+    {
+      "epoch": 1.9978618772717553,
+      "grad_norm": 67.65957613657208,
+      "learning_rate": 0.0,
+      "logits/chosen": -1.171149492263794,
+      "logits/rejected": -1.2374571561813354,
+      "logps/chosen": -129.96214294433594,
+      "logps/rejected": -161.25244140625,
+      "loss": 0.4188,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.34549519419670105,
+      "rewards/margins": 1.0673820972442627,
+      "rewards/rejected": -1.4128773212432861,
+      "step": 1752
     }
   ],
   "logging_steps": 2,
@@ -11511,7 +13433,7 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }