diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,6109 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9999375663357682,
+  "eval_steps": 1000,
+  "global_step": 4004,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.00024973465692701506,
+      "grad_norm": 0.609375,
+      "learning_rate": 1.2468827930174565e-08,
+      "logits/chosen": -0.33114343881607056,
+      "logits/rejected": -0.24089118838310242,
+      "logps/chosen": -44.38773727416992,
+      "logps/rejected": -68.85894775390625,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.0024973465692701507,
+      "grad_norm": 0.609375,
+      "learning_rate": 1.2468827930174566e-07,
+      "logits/chosen": -0.4296959638595581,
+      "logits/rejected": -0.34308701753616333,
+      "logps/chosen": -43.235145568847656,
+      "logps/rejected": -80.90267944335938,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4444444477558136,
+      "rewards/chosen": 0.0005310365813784301,
+      "rewards/margins": 9.072302782442421e-05,
+      "rewards/rejected": 0.0004403134807944298,
+      "step": 10
+    },
+    {
+      "epoch": 0.004994693138540301,
+      "grad_norm": 0.7890625,
+      "learning_rate": 2.493765586034913e-07,
+      "logits/chosen": -0.4125714898109436,
+      "logits/rejected": -0.3169251084327698,
+      "logps/chosen": -42.952693939208984,
+      "logps/rejected": -78.09742736816406,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.000491735409013927,
+      "rewards/margins": 0.0007636584923602641,
+      "rewards/rejected": -0.0012553940759971738,
+      "step": 20
+    },
+    {
+      "epoch": 0.0074920397078104516,
+      "grad_norm": 0.50390625,
+      "learning_rate": 3.7406483790523695e-07,
+      "logits/chosen": -0.4181899130344391,
+      "logits/rejected": -0.3332025110721588,
+      "logps/chosen": -44.16044235229492,
+      "logps/rejected": -71.77767181396484,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.00046690466115251184,
+      "rewards/margins": 0.0001037311740219593,
+      "rewards/rejected": -0.0005706357769668102,
+      "step": 30
+    },
+    {
+      "epoch": 0.009989386277080603,
+      "grad_norm": 0.66796875,
+      "learning_rate": 4.987531172069826e-07,
+      "logits/chosen": -0.43510785698890686,
+      "logits/rejected": -0.34268879890441895,
+      "logps/chosen": -43.815826416015625,
+      "logps/rejected": -80.65787506103516,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.0002724333025980741,
+      "rewards/margins": 0.0016190242022275925,
+      "rewards/rejected": -0.0013465910451486707,
+      "step": 40
+    },
+    {
+      "epoch": 0.012486732846350752,
+      "grad_norm": 0.96484375,
+      "learning_rate": 6.234413965087283e-07,
+      "logits/chosen": -0.4381956160068512,
+      "logits/rejected": -0.32796674966812134,
+      "logps/chosen": -43.25028610229492,
+      "logps/rejected": -77.0926742553711,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0003348322061356157,
+      "rewards/margins": 0.0009277343633584678,
+      "rewards/rejected": -0.0005929021863266826,
+      "step": 50
+    },
+    {
+      "epoch": 0.014984079415620903,
+      "grad_norm": 0.76171875,
+      "learning_rate": 7.481296758104739e-07,
+      "logits/chosen": -0.4058023989200592,
+      "logits/rejected": -0.31743547320365906,
+      "logps/chosen": -43.332618713378906,
+      "logps/rejected": -81.47147369384766,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.0009701108792796731,
+      "rewards/margins": 0.0017434615874662995,
+      "rewards/rejected": -0.0007733507081866264,
+      "step": 60
+    },
+    {
+      "epoch": 0.017481425984891052,
+      "grad_norm": 0.419921875,
+      "learning_rate": 8.728179551122195e-07,
+      "logits/chosen": -0.40699687600135803,
+      "logits/rejected": -0.33324044942855835,
+      "logps/chosen": -42.81806182861328,
+      "logps/rejected": -69.8255844116211,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.0005619878647848964,
+      "rewards/margins": 0.0026690722443163395,
+      "rewards/rejected": -0.0021070842631161213,
+      "step": 70
+    },
+    {
+      "epoch": 0.019978772554161205,
+      "grad_norm": 1.1171875,
+      "learning_rate": 9.975062344139653e-07,
+      "logits/chosen": -0.4060027003288269,
+      "logits/rejected": -0.31257936358451843,
+      "logps/chosen": -43.606048583984375,
+      "logps/rejected": -74.920654296875,
+      "loss": 0.691,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 0.0015056885313242674,
+      "rewards/margins": 0.004372184630483389,
+      "rewards/rejected": -0.0028664960991591215,
+      "step": 80
+    },
+    {
+      "epoch": 0.022476119123431355,
+      "grad_norm": 0.515625,
+      "learning_rate": 1.1221945137157108e-06,
+      "logits/chosen": -0.39966338872909546,
+      "logits/rejected": -0.3299568295478821,
+      "logps/chosen": -42.9066047668457,
+      "logps/rejected": -67.96953582763672,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 0.0013856906443834305,
+      "rewards/margins": 0.00656374916434288,
+      "rewards/rejected": -0.00517805851995945,
+      "step": 90
+    },
+    {
+      "epoch": 0.024973465692701504,
+      "grad_norm": 0.73046875,
+      "learning_rate": 1.2468827930174565e-06,
+      "logits/chosen": -0.4090539515018463,
+      "logits/rejected": -0.31136512756347656,
+      "logps/chosen": -44.48310089111328,
+      "logps/rejected": -84.36518096923828,
+      "loss": 0.688,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 0.0026647746562957764,
+      "rewards/margins": 0.010295169427990913,
+      "rewards/rejected": -0.00763039430603385,
+      "step": 100
+    },
+    {
+      "epoch": 0.027470812261971653,
+      "grad_norm": 0.6484375,
+      "learning_rate": 1.3715710723192023e-06,
+      "logits/chosen": -0.40336164832115173,
+      "logits/rejected": -0.3107188642024994,
+      "logps/chosen": -43.532264709472656,
+      "logps/rejected": -78.85313415527344,
+      "loss": 0.6869,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 0.0021588439121842384,
+      "rewards/margins": 0.012583871372044086,
+      "rewards/rejected": -0.010425028391182423,
+      "step": 110
+    },
+    {
+      "epoch": 0.029968158831241806,
+      "grad_norm": 0.62109375,
+      "learning_rate": 1.4962593516209478e-06,
+      "logits/chosen": -0.44534754753112793,
+      "logits/rejected": -0.3525586724281311,
+      "logps/chosen": -43.2638053894043,
+      "logps/rejected": -73.06291961669922,
+      "loss": 0.6844,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 0.0026553391944617033,
+      "rewards/margins": 0.017566664144396782,
+      "rewards/rejected": -0.014911326579749584,
+      "step": 120
+    },
+    {
+      "epoch": 0.032465505400511956,
+      "grad_norm": 0.734375,
+      "learning_rate": 1.6209476309226935e-06,
+      "logits/chosen": -0.4450170397758484,
+      "logits/rejected": -0.356881707906723,
+      "logps/chosen": -43.020606994628906,
+      "logps/rejected": -74.61629486083984,
+      "loss": 0.6783,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.004791858606040478,
+      "rewards/margins": 0.030012447386980057,
+      "rewards/rejected": -0.025220584124326706,
+      "step": 130
+    },
+    {
+      "epoch": 0.034962851969782105,
+      "grad_norm": 0.52734375,
+      "learning_rate": 1.745635910224439e-06,
+      "logits/chosen": -0.4518131613731384,
+      "logits/rejected": -0.35589107871055603,
+      "logps/chosen": -44.260887145996094,
+      "logps/rejected": -89.10530853271484,
+      "loss": 0.6769,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.0043911924585700035,
+      "rewards/margins": 0.03277025744318962,
+      "rewards/rejected": -0.02837906777858734,
+      "step": 140
+    },
+    {
+      "epoch": 0.037460198539052254,
+      "grad_norm": 0.470703125,
+      "learning_rate": 1.8703241895261848e-06,
+      "logits/chosen": -0.39934635162353516,
+      "logits/rejected": -0.32321810722351074,
+      "logps/chosen": -41.51782989501953,
+      "logps/rejected": -74.04359436035156,
+      "loss": 0.67,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.009082725271582603,
+      "rewards/margins": 0.04699797183275223,
+      "rewards/rejected": -0.037915244698524475,
+      "step": 150
+    },
+    {
+      "epoch": 0.03995754510832241,
+      "grad_norm": 0.703125,
+      "learning_rate": 1.9950124688279305e-06,
+      "logits/chosen": -0.40700763463974,
+      "logits/rejected": -0.33096417784690857,
+      "logps/chosen": -42.31119918823242,
+      "logps/rejected": -82.06526947021484,
+      "loss": 0.6641,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.010724621824920177,
+      "rewards/margins": 0.059142522513866425,
+      "rewards/rejected": -0.04841790720820427,
+      "step": 160
+    },
+    {
+      "epoch": 0.04245489167759256,
+      "grad_norm": 0.443359375,
+      "learning_rate": 2.119700748129676e-06,
+      "logits/chosen": -0.4063618779182434,
+      "logits/rejected": -0.31147629022598267,
+      "logps/chosen": -43.24675750732422,
+      "logps/rejected": -74.64862060546875,
+      "loss": 0.6545,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.009759762324392796,
+      "rewards/margins": 0.07911679148674011,
+      "rewards/rejected": -0.0693570226430893,
+      "step": 170
+    },
+    {
+      "epoch": 0.04495223824686271,
+      "grad_norm": 0.369140625,
+      "learning_rate": 2.2443890274314216e-06,
+      "logits/chosen": -0.3992343842983246,
+      "logits/rejected": -0.30265265703201294,
+      "logps/chosen": -42.563804626464844,
+      "logps/rejected": -89.08007049560547,
+      "loss": 0.6496,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.016897384077310562,
+      "rewards/margins": 0.08930385112762451,
+      "rewards/rejected": -0.07240646332502365,
+      "step": 180
+    },
+    {
+      "epoch": 0.04744958481613286,
+      "grad_norm": 0.388671875,
+      "learning_rate": 2.3690773067331675e-06,
+      "logits/chosen": -0.393463671207428,
+      "logits/rejected": -0.29836633801460266,
+      "logps/chosen": -42.574581146240234,
+      "logps/rejected": -81.52467346191406,
+      "loss": 0.6448,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.01014000829309225,
+      "rewards/margins": 0.09943968802690506,
+      "rewards/rejected": -0.08929967135190964,
+      "step": 190
+    },
+    {
+      "epoch": 0.04994693138540301,
+      "grad_norm": 0.423828125,
+      "learning_rate": 2.493765586034913e-06,
+      "logits/chosen": -0.3705541491508484,
+      "logits/rejected": -0.2776363492012024,
+      "logps/chosen": -43.70671844482422,
+      "logps/rejected": -79.51457214355469,
+      "loss": 0.635,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.011818965896964073,
+      "rewards/margins": 0.12041501700878143,
+      "rewards/rejected": -0.10859604924917221,
+      "step": 200
+    },
+    {
+      "epoch": 0.05244427795467316,
+      "grad_norm": 0.400390625,
+      "learning_rate": 2.6184538653366586e-06,
+      "logits/chosen": -0.3588668704032898,
+      "logits/rejected": -0.27049878239631653,
+      "logps/chosen": -41.42917251586914,
+      "logps/rejected": -80.90901947021484,
+      "loss": 0.6249,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.012408060021698475,
+      "rewards/margins": 0.14213527739048004,
+      "rewards/rejected": -0.12972721457481384,
+      "step": 210
+    },
+    {
+      "epoch": 0.05494162452394331,
+      "grad_norm": 0.376953125,
+      "learning_rate": 2.7431421446384045e-06,
+      "logits/chosen": -0.32904312014579773,
+      "logits/rejected": -0.24068386852741241,
+      "logps/chosen": -42.47250747680664,
+      "logps/rejected": -89.7323226928711,
+      "loss": 0.6272,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.011426225304603577,
+      "rewards/margins": 0.13695809245109558,
+      "rewards/rejected": -0.125531867146492,
+      "step": 220
+    },
+    {
+      "epoch": 0.05743897109321346,
+      "grad_norm": 0.4921875,
+      "learning_rate": 2.86783042394015e-06,
+      "logits/chosen": -0.3509990870952606,
+      "logits/rejected": -0.26932188868522644,
+      "logps/chosen": -41.524696350097656,
+      "logps/rejected": -84.92623901367188,
+      "loss": 0.6128,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.016151348128914833,
+      "rewards/margins": 0.16833417117595673,
+      "rewards/rejected": -0.15218281745910645,
+      "step": 230
+    },
+    {
+      "epoch": 0.05993631766248361,
+      "grad_norm": 0.494140625,
+      "learning_rate": 2.9925187032418956e-06,
+      "logits/chosen": -0.3635261356830597,
+      "logits/rejected": -0.26545146107673645,
+      "logps/chosen": -43.126625061035156,
+      "logps/rejected": -89.85209655761719,
+      "loss": 0.6002,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.004656613804399967,
+      "rewards/margins": 0.19621676206588745,
+      "rewards/rejected": -0.19156016409397125,
+      "step": 240
+    },
+    {
+      "epoch": 0.06243366423175376,
+      "grad_norm": 0.435546875,
+      "learning_rate": 3.117206982543641e-06,
+      "logits/chosen": -0.3245137929916382,
+      "logits/rejected": -0.22121305763721466,
+      "logps/chosen": -42.189552307128906,
+      "logps/rejected": -91.37117767333984,
+      "loss": 0.5891,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.009633781388401985,
+      "rewards/margins": 0.22138457000255585,
+      "rewards/rejected": -0.2117508202791214,
+      "step": 250
+    },
+    {
+      "epoch": 0.06493101080102391,
+      "grad_norm": 0.76171875,
+      "learning_rate": 3.241895261845387e-06,
+      "logits/chosen": -0.3298066258430481,
+      "logits/rejected": -0.2243480682373047,
+      "logps/chosen": -41.658103942871094,
+      "logps/rejected": -97.98243713378906,
+      "loss": 0.5608,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.014573690481483936,
+      "rewards/margins": 0.28632912039756775,
+      "rewards/rejected": -0.2717553973197937,
+      "step": 260
+    },
+    {
+      "epoch": 0.06742835737029407,
+      "grad_norm": 0.84765625,
+      "learning_rate": 3.3665835411471326e-06,
+      "logits/chosen": -0.274959921836853,
+      "logits/rejected": -0.15127086639404297,
+      "logps/chosen": -42.591773986816406,
+      "logps/rejected": -109.750732421875,
+      "loss": 0.5226,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.010403521358966827,
+      "rewards/margins": 0.3793814182281494,
+      "rewards/rejected": -0.36897793412208557,
+      "step": 270
+    },
+    {
+      "epoch": 0.06992570393956421,
+      "grad_norm": 1.265625,
+      "learning_rate": 3.491271820448878e-06,
+      "logits/chosen": -0.26599782705307007,
+      "logits/rejected": -0.14541617035865784,
+      "logps/chosen": -43.68675994873047,
+      "logps/rejected": -131.44851684570312,
+      "loss": 0.4865,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.009956231340765953,
+      "rewards/margins": 0.4785284399986267,
+      "rewards/rejected": -0.4685722291469574,
+      "step": 280
+    },
+    {
+      "epoch": 0.07242305050883437,
+      "grad_norm": 1.8828125,
+      "learning_rate": 3.615960099750624e-06,
+      "logits/chosen": -0.2336564064025879,
+      "logits/rejected": -0.09945651143789291,
+      "logps/chosen": -42.83462142944336,
+      "logps/rejected": -155.51699829101562,
+      "loss": 0.3811,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.010509507730603218,
+      "rewards/margins": 0.8152663111686707,
+      "rewards/rejected": -0.804756760597229,
+      "step": 290
+    },
+    {
+      "epoch": 0.07492039707810451,
+      "grad_norm": 1.484375,
+      "learning_rate": 3.7406483790523696e-06,
+      "logits/chosen": -0.187991201877594,
+      "logits/rejected": -0.015538264997303486,
+      "logps/chosen": -53.440765380859375,
+      "logps/rejected": -225.8020477294922,
+      "loss": 0.2419,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.09434106200933456,
+      "rewards/margins": 1.4534434080123901,
+      "rewards/rejected": -1.5477845668792725,
+      "step": 300
+    },
+    {
+      "epoch": 0.07741774364737466,
+      "grad_norm": 0.94140625,
+      "learning_rate": 3.8653366583541155e-06,
+      "logits/chosen": -0.10128624737262726,
+      "logits/rejected": 0.10228855907917023,
+      "logps/chosen": -66.81230163574219,
+      "logps/rejected": -350.84722900390625,
+      "loss": 0.1818,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.22738003730773926,
+      "rewards/margins": 2.590130567550659,
+      "rewards/rejected": -2.8175110816955566,
+      "step": 310
+    },
+    {
+      "epoch": 0.07991509021664482,
+      "grad_norm": 0.66015625,
+      "learning_rate": 3.990024937655861e-06,
+      "logits/chosen": -0.06341538578271866,
+      "logits/rejected": 0.19574430584907532,
+      "logps/chosen": -63.102256774902344,
+      "logps/rejected": -506.8706970214844,
+      "loss": 0.1113,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.19701920449733734,
+      "rewards/margins": 4.0467329025268555,
+      "rewards/rejected": -4.2437520027160645,
+      "step": 320
+    },
+    {
+      "epoch": 0.08241243678591496,
+      "grad_norm": 0.703125,
+      "learning_rate": 4.114713216957607e-06,
+      "logits/chosen": 0.03162340074777603,
+      "logits/rejected": 0.2972305417060852,
+      "logps/chosen": -55.010414123535156,
+      "logps/rejected": -444.8564453125,
+      "loss": 0.0957,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.1186632290482521,
+      "rewards/margins": 3.6549084186553955,
+      "rewards/rejected": -3.773571729660034,
+      "step": 330
+    },
+    {
+      "epoch": 0.08490978335518512,
+      "grad_norm": 0.48046875,
+      "learning_rate": 4.239401496259352e-06,
+      "logits/chosen": 0.05119480937719345,
+      "logits/rejected": 0.35537463426589966,
+      "logps/chosen": -52.077064514160156,
+      "logps/rejected": -547.7008056640625,
+      "loss": 0.0761,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.08894447982311249,
+      "rewards/margins": 4.584813117980957,
+      "rewards/rejected": -4.673757076263428,
+      "step": 340
+    },
+    {
+      "epoch": 0.08740712992445526,
+      "grad_norm": 0.1796875,
+      "learning_rate": 4.364089775561098e-06,
+      "logits/chosen": 0.17311367392539978,
+      "logits/rejected": 0.49543648958206177,
+      "logps/chosen": -59.5765380859375,
+      "logps/rejected": -548.73876953125,
+      "loss": 0.0769,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.1556529700756073,
+      "rewards/margins": 4.651310443878174,
+      "rewards/rejected": -4.806963920593262,
+      "step": 350
+    },
+    {
+      "epoch": 0.08990447649372542,
+      "grad_norm": 0.216796875,
+      "learning_rate": 4.488778054862843e-06,
+      "logits/chosen": 0.13120940327644348,
+      "logits/rejected": 0.5331201553344727,
+      "logps/chosen": -59.65636444091797,
+      "logps/rejected": -660.1800537109375,
+      "loss": 0.0462,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.15369465947151184,
+      "rewards/margins": 5.723788738250732,
+      "rewards/rejected": -5.877484321594238,
+      "step": 360
+    },
+    {
+      "epoch": 0.09240182306299556,
+      "grad_norm": 0.90625,
+      "learning_rate": 4.6134663341645895e-06,
+      "logits/chosen": 0.25265592336654663,
+      "logits/rejected": 0.6620725989341736,
+      "logps/chosen": -61.8662109375,
+      "logps/rejected": -688.4534912109375,
+      "loss": 0.0476,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.18499073386192322,
+      "rewards/margins": 5.994035720825195,
+      "rewards/rejected": -6.179026126861572,
+      "step": 370
+    },
+    {
+      "epoch": 0.09489916963226572,
+      "grad_norm": 1.0703125,
+      "learning_rate": 4.738154613466335e-06,
+      "logits/chosen": 0.21470895409584045,
+      "logits/rejected": 0.7127342224121094,
+      "logps/chosen": -76.92805480957031,
+      "logps/rejected": -1027.8746337890625,
+      "loss": 0.0318,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3377203345298767,
+      "rewards/margins": 9.173759460449219,
+      "rewards/rejected": -9.511480331420898,
+      "step": 380
+    },
+    {
+      "epoch": 0.09739651620153587,
+      "grad_norm": 0.013671875,
+      "learning_rate": 4.862842892768081e-06,
+      "logits/chosen": 0.2577429413795471,
+      "logits/rejected": 0.7398630380630493,
+      "logps/chosen": -87.82744598388672,
+      "logps/rejected": -828.4269409179688,
+      "loss": 0.0229,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.44494834542274475,
+      "rewards/margins": 7.14224100112915,
+      "rewards/rejected": -7.5871901512146,
+      "step": 390
+    },
+    {
+      "epoch": 0.09989386277080602,
+      "grad_norm": 0.326171875,
+      "learning_rate": 4.987531172069826e-06,
+      "logits/chosen": 0.312338262796402,
+      "logits/rejected": 0.8188611268997192,
+      "logps/chosen": -83.17652130126953,
+      "logps/rejected": -929.5695190429688,
+      "loss": 0.037,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.39534783363342285,
+      "rewards/margins": 8.192750930786133,
+      "rewards/rejected": -8.588098526000977,
+      "step": 400
+    },
+    {
+      "epoch": 0.10239120934007617,
+      "grad_norm": 0.2412109375,
+      "learning_rate": 4.999923022460671e-06,
+      "logits/chosen": 0.2771604657173157,
+      "logits/rejected": 0.875481903553009,
+      "logps/chosen": -71.91412353515625,
+      "logps/rejected": -1142.7008056640625,
+      "loss": 0.0166,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.29563969373703003,
+      "rewards/margins": 10.368195533752441,
+      "rewards/rejected": -10.663835525512695,
+      "step": 410
+    },
+    {
+      "epoch": 0.10488855590934631,
+      "grad_norm": 0.361328125,
+      "learning_rate": 4.999656933348981e-06,
+      "logits/chosen": 0.3595578372478485,
+      "logits/rejected": 0.8383792638778687,
+      "logps/chosen": -77.22844696044922,
+      "logps/rejected": -818.2454833984375,
+      "loss": 0.0347,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3375477194786072,
+      "rewards/margins": 7.169804573059082,
+      "rewards/rejected": -7.50735330581665,
+      "step": 420
+    },
+    {
+      "epoch": 0.10738590247861647,
+      "grad_norm": 0.006805419921875,
+      "learning_rate": 4.99920080255011e-06,
+      "logits/chosen": 0.3072226047515869,
+      "logits/rejected": 0.9365525245666504,
+      "logps/chosen": -69.96758270263672,
+      "logps/rejected": -1107.283447265625,
+      "loss": 0.011,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.24378785490989685,
+      "rewards/margins": 9.968598365783691,
+      "rewards/rejected": -10.212385177612305,
+      "step": 430
+    },
+    {
+      "epoch": 0.10988324904788661,
+      "grad_norm": 0.9375,
+      "learning_rate": 4.998554664742362e-06,
+      "logits/chosen": 0.386096328496933,
+      "logits/rejected": 0.9298421740531921,
+      "logps/chosen": -77.93641662597656,
+      "logps/rejected": -925.2971801757812,
+      "loss": 0.0152,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3427829146385193,
+      "rewards/margins": 8.217004776000977,
+      "rewards/rejected": -8.559788703918457,
+      "step": 440
+    },
+    {
+      "epoch": 0.11238059561715677,
+      "grad_norm": 0.162109375,
+      "learning_rate": 4.997718569049726e-06,
+      "logits/chosen": 0.38062307238578796,
+      "logits/rejected": 0.9510132670402527,
+      "logps/chosen": -62.76348876953125,
+      "logps/rejected": -1039.5853271484375,
+      "loss": 0.0178,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.19411073625087738,
+      "rewards/margins": 9.47815227508545,
+      "rewards/rejected": -9.672263145446777,
+      "step": 450
+    },
+    {
+      "epoch": 0.11487794218642693,
+      "grad_norm": 0.0625,
+      "learning_rate": 4.9966925790381404e-06,
+      "logits/chosen": 0.4757159352302551,
+      "logits/rejected": 1.018425703048706,
+      "logps/chosen": -81.46342468261719,
+      "logps/rejected": -907.7916870117188,
+      "loss": 0.0209,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.38752180337905884,
+      "rewards/margins": 7.994225978851318,
+      "rewards/rejected": -8.381747245788574,
+      "step": 460
+    },
+    {
+      "epoch": 0.11737528875569707,
+      "grad_norm": 0.39453125,
+      "learning_rate": 4.995476772710657e-06,
+      "logits/chosen": 0.40233319997787476,
+      "logits/rejected": 1.0515995025634766,
+      "logps/chosen": -82.72390747070312,
+      "logps/rejected": -1207.831787109375,
+      "loss": 0.0199,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.37652480602264404,
+      "rewards/margins": 10.886858940124512,
+      "rewards/rejected": -11.263383865356445,
+      "step": 470
+    },
+    {
+      "epoch": 0.11987263532496722,
+      "grad_norm": 0.109375,
+      "learning_rate": 4.994071242501516e-06,
+      "logits/chosen": 0.4317776560783386,
+      "logits/rejected": 1.0796253681182861,
+      "logps/chosen": -62.16728591918945,
+      "logps/rejected": -1022.6038208007812,
+      "loss": 0.0134,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.18862931430339813,
+      "rewards/margins": 9.357450485229492,
+      "rewards/rejected": -9.546079635620117,
+      "step": 480
+    },
+    {
+      "epoch": 0.12236998189423737,
+      "grad_norm": 3.3527612686157227e-06,
+      "learning_rate": 4.992476095269112e-06,
+      "logits/chosen": 0.4001534581184387,
+      "logits/rejected": 0.9869491457939148,
+      "logps/chosen": -64.50323486328125,
+      "logps/rejected": -1058.342041015625,
+      "loss": 0.0186,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.21038322150707245,
+      "rewards/margins": 9.569865226745605,
+      "rewards/rejected": -9.780248641967773,
+      "step": 490
+    },
+    {
+      "epoch": 0.12486732846350752,
+      "grad_norm": 0.166015625,
+      "learning_rate": 4.990691452287877e-06,
+      "logits/chosen": 0.513416051864624,
+      "logits/rejected": 1.122924566268921,
+      "logps/chosen": -86.93208312988281,
+      "logps/rejected": -1010.1483154296875,
+      "loss": 0.0175,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.43185362219810486,
+      "rewards/margins": 8.921982765197754,
+      "rewards/rejected": -9.353837966918945,
+      "step": 500
+    },
+    {
+      "epoch": 0.12736467503277768,
+      "grad_norm": 0.1806640625,
+      "learning_rate": 4.988717449239056e-06,
+      "logits/chosen": 0.5288435220718384,
+      "logits/rejected": 1.184326410293579,
+      "logps/chosen": -75.24764251708984,
+      "logps/rejected": -1083.439697265625,
+      "loss": 0.0205,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.31985238194465637,
+      "rewards/margins": 9.819514274597168,
+      "rewards/rejected": -10.139368057250977,
+      "step": 510
+    },
+    {
+      "epoch": 0.12986202160204782,
+      "grad_norm": 0.05126953125,
+      "learning_rate": 4.98655423620039e-06,
+      "logits/chosen": 0.45140591263771057,
+      "logits/rejected": 1.135999083518982,
+      "logps/chosen": -66.84233093261719,
+      "logps/rejected": -1121.6982421875,
+      "loss": 0.0088,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.23821225762367249,
+      "rewards/margins": 10.265534400939941,
+      "rewards/rejected": -10.503746032714844,
+      "step": 520
+    },
+    {
+      "epoch": 0.13235936817131796,
+      "grad_norm": 0.0130615234375,
+      "learning_rate": 4.984201977634711e-06,
+      "logits/chosen": 0.44299745559692383,
+      "logits/rejected": 1.2149170637130737,
+      "logps/chosen": -74.1891860961914,
+      "logps/rejected": -1348.673095703125,
+      "loss": 0.0081,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.30132222175598145,
+      "rewards/margins": 12.441837310791016,
+      "rewards/rejected": -12.74316120147705,
+      "step": 530
+    },
+    {
+      "epoch": 0.13485671474058814,
+      "grad_norm": 0.1982421875,
+      "learning_rate": 4.9816608523774345e-06,
+      "logits/chosen": 0.4906342625617981,
+      "logits/rejected": 1.1866085529327393,
+      "logps/chosen": -61.67924880981445,
+      "logps/rejected": -1052.905029296875,
+      "loss": 0.013,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.1749764382839203,
+      "rewards/margins": 9.563148498535156,
+      "rewards/rejected": -9.73812484741211,
+      "step": 540
+    },
+    {
+      "epoch": 0.13735406130985828,
+      "grad_norm": 0.02099609375,
+      "learning_rate": 4.978931053622964e-06,
+      "logits/chosen": 0.5177958607673645,
+      "logits/rejected": 1.2569612264633179,
+      "logps/chosen": -70.76200866699219,
+      "logps/rejected": -1278.626708984375,
+      "loss": 0.0133,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.27674826979637146,
+      "rewards/margins": 11.792952537536621,
+      "rewards/rejected": -12.069701194763184,
+      "step": 550
+    },
+    {
+      "epoch": 0.13985140787912842,
+      "grad_norm": 0.0003528594970703125,
+      "learning_rate": 4.9760127889100044e-06,
+      "logits/chosen": 0.5248929262161255,
+      "logits/rejected": 1.2501459121704102,
+      "logps/chosen": -81.47541809082031,
+      "logps/rejected": -1154.671630859375,
+      "loss": 0.0119,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3913446366786957,
+      "rewards/margins": 10.44621467590332,
+      "rewards/rejected": -10.837559700012207,
+      "step": 560
+    },
+    {
+      "epoch": 0.1423487544483986,
+      "grad_norm": 0.09423828125,
+      "learning_rate": 4.972906280105781e-06,
+      "logits/chosen": 0.5316249132156372,
+      "logits/rejected": 1.3082139492034912,
+      "logps/chosen": -88.09521484375,
+      "logps/rejected": -1197.23486328125,
+      "loss": 0.0096,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4347292482852936,
+      "rewards/margins": 10.762666702270508,
+      "rewards/rejected": -11.197395324707031,
+      "step": 570
+    },
+    {
+      "epoch": 0.14484610101766873,
+      "grad_norm": 0.0106201171875,
+      "learning_rate": 4.969611763389175e-06,
+      "logits/chosen": 0.5327505469322205,
+      "logits/rejected": 1.3031514883041382,
+      "logps/chosen": -73.19583129882812,
+      "logps/rejected": -1114.89990234375,
+      "loss": 0.0085,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.30240899324417114,
+      "rewards/margins": 10.148442268371582,
+      "rewards/rejected": -10.450851440429688,
+      "step": 580
+    },
+    {
+      "epoch": 0.14734344758693887,
+      "grad_norm": 0.0751953125,
+      "learning_rate": 4.966129489232762e-06,
+      "logits/chosen": 0.47109970450401306,
+      "logits/rejected": 1.3012760877609253,
+      "logps/chosen": -71.85210418701172,
+      "logps/rejected": -1336.1424560546875,
+      "loss": 0.0071,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.277540385723114,
+      "rewards/margins": 12.241964340209961,
+      "rewards/rejected": -12.51950454711914,
+      "step": 590
+    },
+    {
+      "epoch": 0.14984079415620902,
+      "grad_norm": 0.193359375,
+      "learning_rate": 4.962459722383775e-06,
+      "logits/chosen": 0.4269895553588867,
+      "logits/rejected": 1.1828067302703857,
+      "logps/chosen": -71.79056549072266,
+      "logps/rejected": -1337.454345703125,
+      "loss": 0.0111,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.2726261019706726,
+      "rewards/margins": 12.248571395874023,
+      "rewards/rejected": -12.521197319030762,
+      "step": 600
+    },
+    {
+      "epoch": 0.1523381407254792,
+      "grad_norm": 0.09619140625,
+      "learning_rate": 4.958602741843975e-06,
+      "logits/chosen": 0.4595261216163635,
+      "logits/rejected": 1.3015968799591064,
+      "logps/chosen": -78.28643035888672,
+      "logps/rejected": -1252.5814208984375,
+      "loss": 0.0054,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3374534249305725,
+      "rewards/margins": 11.378314018249512,
+      "rewards/rejected": -11.715767860412598,
+      "step": 610
+    },
+    {
+      "epoch": 0.15483548729474933,
+      "grad_norm": 0.048583984375,
+      "learning_rate": 4.954558840848437e-06,
+      "logits/chosen": 0.5825181007385254,
+      "logits/rejected": 1.3565789461135864,
+      "logps/chosen": -76.19590759277344,
+      "logps/rejected": -1119.151611328125,
+      "loss": 0.0109,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3366764783859253,
+      "rewards/margins": 10.178030967712402,
+      "rewards/rejected": -10.514707565307617,
+      "step": 620
+    },
+    {
+      "epoch": 0.15733283386401947,
+      "grad_norm": 0.1572265625,
+      "learning_rate": 4.950328326843258e-06,
+      "logits/chosen": 0.5459114909172058,
+      "logits/rejected": 1.3613156080245972,
+      "logps/chosen": -82.85084533691406,
+      "logps/rejected": -1277.3514404296875,
+      "loss": 0.0104,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3968349099159241,
+      "rewards/margins": 11.52591323852539,
+      "rewards/rejected": -11.922747611999512,
+      "step": 630
+    },
+    {
+      "epoch": 0.15983018043328964,
+      "grad_norm": 0.005096435546875,
+      "learning_rate": 4.945911521462182e-06,
+      "logits/chosen": 0.5720694065093994,
+      "logits/rejected": 1.411368727684021,
+      "logps/chosen": -80.45169067382812,
+      "logps/rejected": -1358.6873779296875,
+      "loss": 0.008,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3696327209472656,
+      "rewards/margins": 12.431825637817383,
+      "rewards/rejected": -12.801457405090332,
+      "step": 640
+    },
+    {
+      "epoch": 0.16232752700255978,
+      "grad_norm": 0.1708984375,
+      "learning_rate": 4.941308760502149e-06,
+      "logits/chosen": 0.5064912438392639,
+      "logits/rejected": 1.2029752731323242,
+      "logps/chosen": -67.90926361083984,
+      "logps/rejected": -1026.6124267578125,
+      "loss": 0.0125,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.2300504744052887,
+      "rewards/margins": 9.079760551452637,
+      "rewards/rejected": -9.30981159210205,
+      "step": 650
+    },
+    {
+      "epoch": 0.16482487357182993,
+      "grad_norm": 0.1787109375,
+      "learning_rate": 4.936520393897762e-06,
+      "logits/chosen": 0.4909030497074127,
+      "logits/rejected": 1.2922523021697998,
+      "logps/chosen": -69.4020004272461,
+      "logps/rejected": -1298.668212890625,
+      "loss": 0.0096,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.25019845366477966,
+      "rewards/margins": 11.917684555053711,
+      "rewards/rejected": -12.167882919311523,
+      "step": 660
+    },
+    {
+      "epoch": 0.16732222014110007,
+      "grad_norm": 0.00469970703125,
+      "learning_rate": 4.931546785694684e-06,
+      "logits/chosen": 0.5053218007087708,
+      "logits/rejected": 1.4669103622436523,
+      "logps/chosen": -86.32283782958984,
+      "logps/rejected": -1483.82763671875,
+      "loss": 0.0045,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.42769041657447815,
+      "rewards/margins": 13.680854797363281,
+      "rewards/rejected": -14.108546257019043,
+      "step": 670
+    },
+    {
+      "epoch": 0.16981956671037024,
+      "grad_norm": 0.0238037109375,
+      "learning_rate": 4.926388314021964e-06,
+      "logits/chosen": 0.6257452368736267,
+      "logits/rejected": 1.5271151065826416,
+      "logps/chosen": -92.75479888916016,
+      "logps/rejected": -1237.61474609375,
+      "loss": 0.0038,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4862573742866516,
+      "rewards/margins": 11.188620567321777,
+      "rewards/rejected": -11.67487907409668,
+      "step": 680
+    },
+    {
+      "epoch": 0.17231691327964038,
+      "grad_norm": 0.000385284423828125,
+      "learning_rate": 4.921045371063283e-06,
+      "logits/chosen": 0.584161102771759,
+      "logits/rejected": 1.478125810623169,
+      "logps/chosen": -89.634033203125,
+      "logps/rejected": -1360.25537109375,
+      "loss": 0.0066,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.46992653608322144,
+      "rewards/margins": 12.42024040222168,
+      "rewards/rejected": -12.890167236328125,
+      "step": 690
+    },
+    {
+      "epoch": 0.17481425984891052,
+      "grad_norm": 0.1318359375,
+      "learning_rate": 4.915518363027142e-06,
+      "logits/chosen": 0.5938104391098022,
+      "logits/rejected": 1.4910205602645874,
+      "logps/chosen": -73.86201477050781,
+      "logps/rejected": -1182.0716552734375,
+      "loss": 0.0059,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3045103847980499,
+      "rewards/margins": 10.839627265930176,
+      "rewards/rejected": -11.144137382507324,
+      "step": 700
+    },
+    {
+      "epoch": 0.1773116064181807,
+      "grad_norm": 0.08740234375,
+      "learning_rate": 4.909807710115977e-06,
+      "logits/chosen": 0.543526828289032,
+      "logits/rejected": 1.467707633972168,
+      "logps/chosen": -81.62191009521484,
+      "logps/rejected": -1380.295166015625,
+      "loss": 0.0066,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.38441941142082214,
+      "rewards/margins": 12.690566062927246,
+      "rewards/rejected": -13.074984550476074,
+      "step": 710
+    },
+    {
+      "epoch": 0.17980895298745084,
+      "grad_norm": 0.025146484375,
+      "learning_rate": 4.903913846494211e-06,
+      "logits/chosen": 0.4768219590187073,
+      "logits/rejected": 1.4783326387405396,
+      "logps/chosen": -79.43184661865234,
+      "logps/rejected": -1673.644287109375,
+      "loss": 0.0023,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3560616374015808,
+      "rewards/margins": 15.518526077270508,
+      "rewards/rejected": -15.87458610534668,
+      "step": 720
+    },
+    {
+      "epoch": 0.18230629955672098,
+      "grad_norm": 0.08154296875,
+      "learning_rate": 4.897837220255251e-06,
+      "logits/chosen": 0.5687042474746704,
+      "logits/rejected": 1.4331896305084229,
+      "logps/chosen": -86.38923645019531,
+      "logps/rejected": -1329.374755859375,
+      "loss": 0.008,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4307560920715332,
+      "rewards/margins": 12.039878845214844,
+      "rewards/rejected": -12.470634460449219,
+      "step": 730
+    },
+    {
+      "epoch": 0.18480364612599112,
+      "grad_norm": 0.0101318359375,
+      "learning_rate": 4.891578293387413e-06,
+      "logits/chosen": 0.604946494102478,
+      "logits/rejected": 1.5590946674346924,
+      "logps/chosen": -80.6954345703125,
+      "logps/rejected": -1429.257080078125,
+      "loss": 0.0056,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.37448957562446594,
+      "rewards/margins": 13.214704513549805,
+      "rewards/rejected": -13.589195251464844,
+      "step": 740
+    },
+    {
+      "epoch": 0.1873009926952613,
+      "grad_norm": 0.07177734375,
+      "learning_rate": 4.885137541738808e-06,
+      "logits/chosen": 0.5679504871368408,
+      "logits/rejected": 1.4625308513641357,
+      "logps/chosen": -74.07333374023438,
+      "logps/rejected": -1173.9857177734375,
+      "loss": 0.0055,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3022187054157257,
+      "rewards/margins": 10.615083694458008,
+      "rewards/rejected": -10.917302131652832,
+      "step": 750
+    },
+    {
+      "epoch": 0.18979833926453143,
+      "grad_norm": 0.000659942626953125,
+      "learning_rate": 4.878515454981153e-06,
+      "logits/chosen": 0.5600544214248657,
+      "logits/rejected": 1.5554611682891846,
+      "logps/chosen": -95.30448913574219,
+      "logps/rejected": -1539.63427734375,
+      "loss": 0.0029,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5141724348068237,
+      "rewards/margins": 14.041679382324219,
+      "rewards/rejected": -14.555851936340332,
+      "step": 760
+    },
+    {
+      "epoch": 0.19229568583380158,
+      "grad_norm": 0.154296875,
+      "learning_rate": 4.8717125365725545e-06,
+      "logits/chosen": 0.6704256534576416,
+      "logits/rejected": 1.511325716972351,
+      "logps/chosen": -78.95833587646484,
+      "logps/rejected": -1096.5010986328125,
+      "loss": 0.0109,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.35167983174324036,
+      "rewards/margins": 9.9315185546875,
+      "rewards/rejected": -10.283197402954102,
+      "step": 770
+    },
+    {
+      "epoch": 0.19479303240307175,
+      "grad_norm": 0.04443359375,
+      "learning_rate": 4.864729303719221e-06,
+      "logits/chosen": 0.49029749631881714,
+      "logits/rejected": 1.4827202558517456,
+      "logps/chosen": -78.9704360961914,
+      "logps/rejected": -1541.809326171875,
+      "loss": 0.0031,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.34706807136535645,
+      "rewards/margins": 14.254959106445312,
+      "rewards/rejected": -14.602025985717773,
+      "step": 780
+    },
+    {
+      "epoch": 0.1972903789723419,
+      "grad_norm": 0.11328125,
+      "learning_rate": 4.857566287336152e-06,
+      "logits/chosen": 0.5910658836364746,
+      "logits/rejected": 1.5483187437057495,
+      "logps/chosen": -99.37945556640625,
+      "logps/rejected": -1432.4755859375,
+      "loss": 0.0086,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5464431047439575,
+      "rewards/margins": 12.95887565612793,
+      "rewards/rejected": -13.505319595336914,
+      "step": 790
+    },
+    {
+      "epoch": 0.19978772554161203,
+      "grad_norm": 0.0011444091796875,
+      "learning_rate": 4.850224032006765e-06,
+      "logits/chosen": 0.6179195642471313,
+      "logits/rejected": 1.5901352167129517,
+      "logps/chosen": -81.73147583007812,
+      "logps/rejected": -1412.5142822265625,
+      "loss": 0.0054,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.38581180572509766,
+      "rewards/margins": 13.030293464660645,
+      "rewards/rejected": -13.416107177734375,
+      "step": 800
+    },
+    {
+      "epoch": 0.20228507211088217,
+      "grad_norm": 0.1357421875,
+      "learning_rate": 4.8427030959414984e-06,
+      "logits/chosen": 0.5971755385398865,
+      "logits/rejected": 1.6486015319824219,
+      "logps/chosen": -74.70821380615234,
+      "logps/rejected": -1532.8193359375,
+      "loss": 0.0063,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3184259831905365,
+      "rewards/margins": 14.300427436828613,
+      "rewards/rejected": -14.618852615356445,
+      "step": 810
+    },
+    {
+      "epoch": 0.20478241868015234,
+      "grad_norm": 0.1689453125,
+      "learning_rate": 4.835004050935369e-06,
+      "logits/chosen": 0.6013739705085754,
+      "logits/rejected": 1.4875710010528564,
+      "logps/chosen": -74.9230728149414,
+      "logps/rejected": -1462.062255859375,
+      "loss": 0.0107,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.31746476888656616,
+      "rewards/margins": 13.55724811553955,
+      "rewards/rejected": -13.874712944030762,
+      "step": 820
+    },
+    {
+      "epoch": 0.2072797652494225,
+      "grad_norm": 0.0279541015625,
+      "learning_rate": 4.8271274823245e-06,
+      "logits/chosen": 0.6184748411178589,
+      "logits/rejected": 1.5838812589645386,
+      "logps/chosen": -74.03543853759766,
+      "logps/rejected": -1436.349853515625,
+      "loss": 0.0048,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.29689908027648926,
+      "rewards/margins": 13.336532592773438,
+      "rewards/rejected": -13.63343334197998,
+      "step": 830
+    },
+    {
+      "epoch": 0.20977711181869263,
+      "grad_norm": 0.0888671875,
+      "learning_rate": 4.8190739889416264e-06,
+      "logits/chosen": 0.6181553602218628,
+      "logits/rejected": 1.6880038976669312,
+      "logps/chosen": -73.8195571899414,
+      "logps/rejected": -1627.819091796875,
+      "loss": 0.003,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3135436177253723,
+      "rewards/margins": 15.254382133483887,
+      "rewards/rejected": -15.567927360534668,
+      "step": 840
+    },
+    {
+      "epoch": 0.2122744583879628,
+      "grad_norm": 0.0703125,
+      "learning_rate": 4.810844183070553e-06,
+      "logits/chosen": 0.5540085434913635,
+      "logits/rejected": 1.6211318969726562,
+      "logps/chosen": -72.0704116821289,
+      "logps/rejected": -1341.570068359375,
+      "loss": 0.0033,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.29530271887779236,
+      "rewards/margins": 12.359551429748535,
+      "rewards/rejected": -12.654852867126465,
+      "step": 850
+    },
+    {
+      "epoch": 0.21477180495723294,
+      "grad_norm": 0.1435546875,
+      "learning_rate": 4.802438690399622e-06,
+      "logits/chosen": 0.600739598274231,
+      "logits/rejected": 1.643431305885315,
+      "logps/chosen": -70.41534423828125,
+      "logps/rejected": -1476.7939453125,
+      "loss": 0.0068,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.26624736189842224,
+      "rewards/margins": 13.719133377075195,
+      "rewards/rejected": -13.985379219055176,
+      "step": 860
+    },
+    {
+      "epoch": 0.21726915152650308,
+      "grad_norm": 0.000701904296875,
+      "learning_rate": 4.793858149974129e-06,
+      "logits/chosen": 0.6058120727539062,
+      "logits/rejected": 1.721599817276001,
+      "logps/chosen": -79.14794921875,
+      "logps/rejected": -1739.1048583984375,
+      "loss": 0.0036,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3524496853351593,
+      "rewards/margins": 16.286325454711914,
+      "rewards/rejected": -16.638776779174805,
+      "step": 870
+    },
+    {
+      "epoch": 0.21976649809577323,
+      "grad_norm": 0.047119140625,
+      "learning_rate": 4.785103214147747e-06,
+      "logits/chosen": 0.6141168475151062,
+      "logits/rejected": 1.7174230813980103,
+      "logps/chosen": -77.48551940917969,
+      "logps/rejected": -1538.275634765625,
+      "loss": 0.004,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.34762924909591675,
+      "rewards/margins": 14.33294677734375,
+      "rewards/rejected": -14.680575370788574,
+      "step": 880
+    },
+    {
+      "epoch": 0.2222638446650434,
+      "grad_norm": 0.000667572021484375,
+      "learning_rate": 4.776174548532926e-06,
+      "logits/chosen": 0.6287072896957397,
+      "logits/rejected": 1.6851770877838135,
+      "logps/chosen": -77.02873229980469,
+      "logps/rejected": -1560.05810546875,
+      "loss": 0.003,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3427741825580597,
+      "rewards/margins": 14.53984260559082,
+      "rewards/rejected": -14.882616996765137,
+      "step": 890
+    },
+    {
+      "epoch": 0.22476119123431354,
+      "grad_norm": 0.07080078125,
+      "learning_rate": 4.767072831950288e-06,
+      "logits/chosen": 0.617357611656189,
+      "logits/rejected": 1.7079490423202515,
+      "logps/chosen": -77.236572265625,
+      "logps/rejected": -1560.383544921875,
+      "loss": 0.002,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3377893567085266,
+      "rewards/margins": 14.535722732543945,
+      "rewards/rejected": -14.873510360717773,
+      "step": 900
+    },
+    {
+      "epoch": 0.22725853780358368,
+      "grad_norm": 0.01129150390625,
+      "learning_rate": 4.7577987563770226e-06,
+      "logits/chosen": 0.6263229250907898,
+      "logits/rejected": 1.674384355545044,
+      "logps/chosen": -81.606201171875,
+      "logps/rejected": -1556.177490234375,
+      "loss": 0.0057,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.37518563866615295,
+      "rewards/margins": 14.350992202758789,
+      "rewards/rejected": -14.726178169250488,
+      "step": 910
+    },
+    {
+      "epoch": 0.22975588437285385,
+      "grad_norm": 0.00042724609375,
+      "learning_rate": 4.748353026894273e-06,
+      "logits/chosen": 0.63312166929245,
+      "logits/rejected": 1.7262632846832275,
+      "logps/chosen": -73.91940307617188,
+      "logps/rejected": -1473.6871337890625,
+      "loss": 0.0037,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.29600244760513306,
+      "rewards/margins": 13.644391059875488,
+      "rewards/rejected": -13.940393447875977,
+      "step": 920
+    },
+    {
+      "epoch": 0.232253230942124,
+      "grad_norm": 0.00836181640625,
+      "learning_rate": 4.738736361633532e-06,
+      "logits/chosen": 0.6512018442153931,
+      "logits/rejected": 1.6386451721191406,
+      "logps/chosen": -75.5640869140625,
+      "logps/rejected": -1397.0894775390625,
+      "loss": 0.0042,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3240968585014343,
+      "rewards/margins": 12.940801620483398,
+      "rewards/rejected": -13.264900207519531,
+      "step": 930
+    },
+    {
+      "epoch": 0.23475057751139414,
+      "grad_norm": 0.00250244140625,
+      "learning_rate": 4.728949491722046e-06,
+      "logits/chosen": 0.6666821837425232,
+      "logits/rejected": 1.6732994318008423,
+      "logps/chosen": -76.89160919189453,
+      "logps/rejected": -1383.048095703125,
+      "loss": 0.0052,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3491043150424957,
+      "rewards/margins": 12.806585311889648,
+      "rewards/rejected": -13.155691146850586,
+      "step": 940
+    },
+    {
+      "epoch": 0.2372479240806643,
+      "grad_norm": 0.07421875,
+      "learning_rate": 4.718993161227231e-06,
+      "logits/chosen": 0.5883976221084595,
+      "logits/rejected": 1.7654300928115845,
+      "logps/chosen": -84.6438217163086,
+      "logps/rejected": -1749.6275634765625,
+      "loss": 0.0031,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3954206109046936,
+      "rewards/margins": 16.34560775756836,
+      "rewards/rejected": -16.74102783203125,
+      "step": 950
+    },
+    {
+      "epoch": 0.23974527064993445,
+      "grad_norm": 0.006500244140625,
+      "learning_rate": 4.708868127100098e-06,
+      "logits/chosen": 0.666793167591095,
+      "logits/rejected": 1.6790577173233032,
+      "logps/chosen": -75.14250183105469,
+      "logps/rejected": -1408.4847412109375,
+      "loss": 0.0034,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3262697458267212,
+      "rewards/margins": 13.068046569824219,
+      "rewards/rejected": -13.394315719604492,
+      "step": 960
+    },
+    {
+      "epoch": 0.2422426172192046,
+      "grad_norm": 0.0001659393310546875,
+      "learning_rate": 4.6985751591177075e-06,
+      "logits/chosen": 0.62273108959198,
+      "logits/rejected": 1.7520809173583984,
+      "logps/chosen": -87.58243560791016,
+      "logps/rejected": -1661.821533203125,
+      "loss": 0.0028,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4351579546928406,
+      "rewards/margins": 15.40393352508545,
+      "rewards/rejected": -15.839093208312988,
+      "step": 970
+    },
+    {
+      "epoch": 0.24473996378847473,
+      "grad_norm": 0.00885009765625,
+      "learning_rate": 4.688115039824648e-06,
+      "logits/chosen": 0.6803555488586426,
+      "logits/rejected": 1.7338272333145142,
+      "logps/chosen": -88.861328125,
+      "logps/rejected": -1561.404541015625,
+      "loss": 0.0032,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4565040171146393,
+      "rewards/margins": 14.439603805541992,
+      "rewards/rejected": -14.896108627319336,
+      "step": 980
+    },
+    {
+      "epoch": 0.2472373103577449,
+      "grad_norm": 5.424022674560547e-06,
+      "learning_rate": 4.677488564473535e-06,
+      "logits/chosen": 0.6470680832862854,
+      "logits/rejected": 1.818428635597229,
+      "logps/chosen": -92.59998321533203,
+      "logps/rejected": -1680.185791015625,
+      "loss": 0.0027,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4968494772911072,
+      "rewards/margins": 15.579751968383789,
+      "rewards/rejected": -16.076602935791016,
+      "step": 990
+    },
+    {
+      "epoch": 0.24973465692701505,
+      "grad_norm": 0.03173828125,
+      "learning_rate": 4.666696540964556e-06,
+      "logits/chosen": 0.7138900756835938,
+      "logits/rejected": 1.7771879434585571,
+      "logps/chosen": -99.16166687011719,
+      "logps/rejected": -1503.458984375,
+      "loss": 0.0033,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.551328182220459,
+      "rewards/margins": 13.802647590637207,
+      "rewards/rejected": -14.353976249694824,
+      "step": 1000
+    },
+    {
+      "epoch": 0.24973465692701505,
+      "eval_logits/chosen": 0.7451997995376587,
+      "eval_logits/rejected": 1.5489047765731812,
+      "eval_logps/chosen": -80.45184326171875,
+      "eval_logps/rejected": -830.6234130859375,
+      "eval_loss": 0.006477854214608669,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.36095961928367615,
+      "eval_rewards/margins": 7.334376335144043,
+      "eval_rewards/rejected": -7.695336818695068,
+      "eval_runtime": 0.619,
+      "eval_samples_per_second": 8.077,
+      "eval_steps_per_second": 8.077,
+      "step": 1000
+    },
+    {
+      "epoch": 0.2522320034962852,
+      "grad_norm": 0.0003871917724609375,
+      "learning_rate": 4.6557397897840454e-06,
+      "logits/chosen": 0.693498969078064,
+      "logits/rejected": 1.753259301185608,
+      "logps/chosen": -102.29890441894531,
+      "logps/rejected": -1543.0269775390625,
+      "loss": 0.005,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.583904504776001,
+      "rewards/margins": 14.051568984985352,
+      "rewards/rejected": -14.635473251342773,
+      "step": 1010
+    },
+    {
+      "epoch": 0.25472935006555536,
+      "grad_norm": 0.0003032684326171875,
+      "learning_rate": 4.644619143942108e-06,
+      "logits/chosen": 0.5707821249961853,
+      "logits/rejected": 1.7276995182037354,
+      "logps/chosen": -88.48922729492188,
+      "logps/rejected": -1698.140625,
+      "loss": 0.0011,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.44621315598487854,
+      "rewards/margins": 15.652883529663086,
+      "rewards/rejected": -16.099096298217773,
+      "step": 1020
+    },
+    {
+      "epoch": 0.2572266966348255,
+      "grad_norm": 0.076171875,
+      "learning_rate": 4.633335448909284e-06,
+      "logits/chosen": 0.658003032207489,
+      "logits/rejected": 1.7504956722259521,
+      "logps/chosen": -89.60042572021484,
+      "logps/rejected": -1624.2664794921875,
+      "loss": 0.0019,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4533475339412689,
+      "rewards/margins": 14.999029159545898,
+      "rewards/rejected": -15.452377319335938,
+      "step": 1030
+    },
+    {
+      "epoch": 0.25972404320409564,
+      "grad_norm": 0.0308837890625,
+      "learning_rate": 4.621889562552272e-06,
+      "logits/chosen": 0.6270695924758911,
+      "logits/rejected": 1.8431812524795532,
+      "logps/chosen": -105.17597961425781,
+      "logps/rejected": -1880.988037109375,
+      "loss": 0.0032,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6098282933235168,
+      "rewards/margins": 17.456111907958984,
+      "rewards/rejected": -18.065940856933594,
+      "step": 1040
+    },
+    {
+      "epoch": 0.2622213897733658,
+      "grad_norm": 3.886222839355469e-05,
+      "learning_rate": 4.610282355068707e-06,
+      "logits/chosen": 0.577286422252655,
+      "logits/rejected": 1.723755121231079,
+      "logps/chosen": -109.06168365478516,
+      "logps/rejected": -1825.4671630859375,
+      "loss": 0.002,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6522443890571594,
+      "rewards/margins": 16.784276962280273,
+      "rewards/rejected": -17.4365234375,
+      "step": 1050
+    },
+    {
+      "epoch": 0.26471873634263593,
+      "grad_norm": 0.078125,
+      "learning_rate": 4.598514708921006e-06,
+      "logits/chosen": 0.6790138483047485,
+      "logits/rejected": 1.8542677164077759,
+      "logps/chosen": -101.43669128417969,
+      "logps/rejected": -1781.3316650390625,
+      "loss": 0.0024,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5860346555709839,
+      "rewards/margins": 16.492624282836914,
+      "rewards/rejected": -17.078659057617188,
+      "step": 1060
+    },
+    {
+      "epoch": 0.26721608291190607,
+      "grad_norm": 0.0002994537353515625,
+      "learning_rate": 4.5865875187692695e-06,
+      "logits/chosen": 0.6998518109321594,
+      "logits/rejected": 1.8515506982803345,
+      "logps/chosen": -90.31967163085938,
+      "logps/rejected": -1559.104736328125,
+      "loss": 0.0043,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.46949252486228943,
+      "rewards/margins": 14.420549392700195,
+      "rewards/rejected": -14.890042304992676,
+      "step": 1070
+    },
+    {
+      "epoch": 0.26971342948117627,
+      "grad_norm": 0.04296875,
+      "learning_rate": 4.57450169140327e-06,
+      "logits/chosen": 0.6541129350662231,
+      "logits/rejected": 1.944573163986206,
+      "logps/chosen": -94.61952209472656,
+      "logps/rejected": -1980.4349365234375,
+      "loss": 0.0018,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5051703453063965,
+      "rewards/margins": 18.555133819580078,
+      "rewards/rejected": -19.060306549072266,
+      "step": 1080
+    },
+    {
+      "epoch": 0.2722107760504464,
+      "grad_norm": 0.0250244140625,
+      "learning_rate": 4.562258145673507e-06,
+      "logits/chosen": 0.624032199382782,
+      "logits/rejected": 1.8918708562850952,
+      "logps/chosen": -105.9957275390625,
+      "logps/rejected": -1974.2584228515625,
+      "loss": 0.0009,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6295837759971619,
+      "rewards/margins": 18.393098831176758,
+      "rewards/rejected": -19.022680282592773,
+      "step": 1090
+    },
+    {
+      "epoch": 0.27470812261971655,
+      "grad_norm": 0.0026092529296875,
+      "learning_rate": 4.549857812421353e-06,
+      "logits/chosen": 0.64922696352005,
+      "logits/rejected": 1.8475501537322998,
+      "logps/chosen": -89.56166076660156,
+      "logps/rejected": -1663.010986328125,
+      "loss": 0.0012,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.46423882246017456,
+      "rewards/margins": 15.462870597839355,
+      "rewards/rejected": -15.92711067199707,
+      "step": 1100
+    },
+    {
+      "epoch": 0.2772054691889867,
+      "grad_norm": 0.0084228515625,
+      "learning_rate": 4.537301634408281e-06,
+      "logits/chosen": 0.6925086975097656,
+      "logits/rejected": 1.7738326787948608,
+      "logps/chosen": -85.5665512084961,
+      "logps/rejected": -1501.200439453125,
+      "loss": 0.0023,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4182400703430176,
+      "rewards/margins": 13.875646591186523,
+      "rewards/rejected": -14.293886184692383,
+      "step": 1110
+    },
+    {
+      "epoch": 0.27970281575825684,
+      "grad_norm": 0.0732421875,
+      "learning_rate": 4.52459056624419e-06,
+      "logits/chosen": 0.7249046564102173,
+      "logits/rejected": 1.795248031616211,
+      "logps/chosen": -102.11865997314453,
+      "logps/rejected": -1670.522216796875,
+      "loss": 0.0035,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5903103351593018,
+      "rewards/margins": 15.393072128295898,
+      "rewards/rejected": -15.983380317687988,
+      "step": 1120
+    },
+    {
+      "epoch": 0.282200162327527,
+      "grad_norm": 0.06494140625,
+      "learning_rate": 4.51172557431483e-06,
+      "logits/chosen": 0.6399365663528442,
+      "logits/rejected": 1.7282390594482422,
+      "logps/chosen": -105.56150817871094,
+      "logps/rejected": -1699.7261962890625,
+      "loss": 0.0022,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6241176724433899,
+      "rewards/margins": 15.525866508483887,
+      "rewards/rejected": -16.14998435974121,
+      "step": 1130
+    },
+    {
+      "epoch": 0.2846975088967972,
+      "grad_norm": 0.054443359375,
+      "learning_rate": 4.49870763670833e-06,
+      "logits/chosen": 0.6207230687141418,
+      "logits/rejected": 1.878230333328247,
+      "logps/chosen": -95.18315124511719,
+      "logps/rejected": -1844.5869140625,
+      "loss": 0.0013,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5091260671615601,
+      "rewards/margins": 17.207239151000977,
+      "rewards/rejected": -17.71636390686035,
+      "step": 1140
+    },
+    {
+      "epoch": 0.2871948554660673,
+      "grad_norm": 0.06591796875,
+      "learning_rate": 4.4855377431408335e-06,
+      "logits/chosen": 0.6628460884094238,
+      "logits/rejected": 1.7432994842529297,
+      "logps/chosen": -109.8287124633789,
+      "logps/rejected": -1642.527099609375,
+      "loss": 0.002,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6535288095474243,
+      "rewards/margins": 14.924878120422363,
+      "rewards/rejected": -15.578405380249023,
+      "step": 1150
+    },
+    {
+      "epoch": 0.28969220203533746,
+      "grad_norm": 0.005096435546875,
+      "learning_rate": 4.472216894881261e-06,
+      "logits/chosen": 0.6672796010971069,
+      "logits/rejected": 1.738856315612793,
+      "logps/chosen": -89.06207275390625,
+      "logps/rejected": -1540.048583984375,
+      "loss": 0.0018,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.46248659491539,
+      "rewards/margins": 14.235052108764648,
+      "rewards/rejected": -14.697538375854492,
+      "step": 1160
+    },
+    {
+      "epoch": 0.2921895486046076,
+      "grad_norm": 0.047607421875,
+      "learning_rate": 4.4587461046751815e-06,
+      "logits/chosen": 0.6774252653121948,
+      "logits/rejected": 1.7696081399917603,
+      "logps/chosen": -81.09745788574219,
+      "logps/rejected": -1620.4942626953125,
+      "loss": 0.003,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.38098251819610596,
+      "rewards/margins": 15.126324653625488,
+      "rewards/rejected": -15.507307052612305,
+      "step": 1170
+    },
+    {
+      "epoch": 0.29468689517387775,
+      "grad_norm": 0.008544921875,
+      "learning_rate": 4.44512639666781e-06,
+      "logits/chosen": 0.6951876878738403,
+      "logits/rejected": 1.7908748388290405,
+      "logps/chosen": -76.070068359375,
+      "logps/rejected": -1503.576416015625,
+      "loss": 0.0032,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.32203441858291626,
+      "rewards/margins": 14.029817581176758,
+      "rewards/rejected": -14.351852416992188,
+      "step": 1180
+    },
+    {
+      "epoch": 0.2971842417431479,
+      "grad_norm": 0.890625,
+      "learning_rate": 4.431358806326158e-06,
+      "logits/chosen": 0.5664582848548889,
+      "logits/rejected": 1.6636062860488892,
+      "logps/chosen": -77.29322814941406,
+      "logps/rejected": -1649.8541259765625,
+      "loss": 0.004,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.32812389731407166,
+      "rewards/margins": 15.302160263061523,
+      "rewards/rejected": -15.630284309387207,
+      "step": 1190
+    },
+    {
+      "epoch": 0.29968158831241803,
+      "grad_norm": 0.765625,
+      "learning_rate": 4.4174443803603e-06,
+      "logits/chosen": 0.7006584405899048,
+      "logits/rejected": 1.8050626516342163,
+      "logps/chosen": -101.17628479003906,
+      "logps/rejected": -1638.1669921875,
+      "loss": 0.004,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5764530301094055,
+      "rewards/margins": 15.088088989257812,
+      "rewards/rejected": -15.664541244506836,
+      "step": 1200
+    },
+    {
+      "epoch": 0.30217893488168823,
+      "grad_norm": 0.045166015625,
+      "learning_rate": 4.4033841766438e-06,
+      "logits/chosen": 0.5828436613082886,
+      "logits/rejected": 1.6060895919799805,
+      "logps/chosen": -79.80711364746094,
+      "logps/rejected": -1499.616943359375,
+      "loss": 0.0038,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.35304346680641174,
+      "rewards/margins": 13.869488716125488,
+      "rewards/rejected": -14.222529411315918,
+      "step": 1210
+    },
+    {
+      "epoch": 0.3046762814509584,
+      "grad_norm": 0.0005035400390625,
+      "learning_rate": 4.389179264133281e-06,
+      "logits/chosen": 0.6191063523292542,
+      "logits/rejected": 1.7791473865509033,
+      "logps/chosen": -74.18888854980469,
+      "logps/rejected": -1617.736572265625,
+      "loss": 0.0013,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.31279683113098145,
+      "rewards/margins": 15.179719924926758,
+      "rewards/rejected": -15.492517471313477,
+      "step": 1220
+    },
+    {
+      "epoch": 0.3071736280202285,
+      "grad_norm": 0.10205078125,
+      "learning_rate": 4.374830722787159e-06,
+      "logits/chosen": 0.5632847547531128,
+      "logits/rejected": 1.709839105606079,
+      "logps/chosen": -71.81803894042969,
+      "logps/rejected": -1728.1201171875,
+      "loss": 0.0035,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.2758982479572296,
+      "rewards/margins": 16.288021087646484,
+      "rewards/rejected": -16.563919067382812,
+      "step": 1230
+    },
+    {
+      "epoch": 0.30967097458949866,
+      "grad_norm": 0.08984375,
+      "learning_rate": 4.360339643483533e-06,
+      "logits/chosen": 0.5613077878952026,
+      "logits/rejected": 1.6441980600357056,
+      "logps/chosen": -71.94982147216797,
+      "logps/rejected": -1693.345947265625,
+      "loss": 0.0031,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.27554553747177124,
+      "rewards/margins": 15.830523490905762,
+      "rewards/rejected": -16.106069564819336,
+      "step": 1240
+    },
+    {
+      "epoch": 0.3121683211587688,
+      "grad_norm": 0.00022411346435546875,
+      "learning_rate": 4.345707127937253e-06,
+      "logits/chosen": 0.5210096836090088,
+      "logits/rejected": 1.81972336769104,
+      "logps/chosen": -72.21741485595703,
+      "logps/rejected": -1985.244873046875,
+      "loss": 0.0011,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.2866365611553192,
+      "rewards/margins": 18.848371505737305,
+      "rewards/rejected": -19.135007858276367,
+      "step": 1250
+    },
+    {
+      "epoch": 0.31466566772803894,
+      "grad_norm": 0.0087890625,
+      "learning_rate": 4.330934288616154e-06,
+      "logits/chosen": 0.6370071172714233,
+      "logits/rejected": 1.816506028175354,
+      "logps/chosen": -77.37321472167969,
+      "logps/rejected": -1700.324951171875,
+      "loss": 0.0013,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.33770519495010376,
+      "rewards/margins": 15.96662712097168,
+      "rewards/rejected": -16.304332733154297,
+      "step": 1260
+    },
+    {
+      "epoch": 0.3171630142973091,
+      "grad_norm": 0.043701171875,
+      "learning_rate": 4.316022248656485e-06,
+      "logits/chosen": 0.5022410154342651,
+      "logits/rejected": 1.5646175146102905,
+      "logps/chosen": -71.90743255615234,
+      "logps/rejected": -1543.3675537109375,
+      "loss": 0.0019,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.2772095799446106,
+      "rewards/margins": 14.190910339355469,
+      "rewards/rejected": -14.468118667602539,
+      "step": 1270
+    },
+    {
+      "epoch": 0.3196603608665793,
+      "grad_norm": 0.01177978515625,
+      "learning_rate": 4.3009721417775166e-06,
+      "logits/chosen": 0.5786353349685669,
+      "logits/rejected": 1.7490533590316772,
+      "logps/chosen": -76.0634536743164,
+      "logps/rejected": -1822.3128662109375,
+      "loss": 0.0014,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.33201277256011963,
+      "rewards/margins": 17.15824317932129,
+      "rewards/rejected": -17.49025535583496,
+      "step": 1280
+    },
+    {
+      "epoch": 0.3221577074358494,
+      "grad_norm": 0.006378173828125,
+      "learning_rate": 4.285785112195346e-06,
+      "logits/chosen": 0.5005044341087341,
+      "logits/rejected": 1.6343857049942017,
+      "logps/chosen": -79.80322265625,
+      "logps/rejected": -1840.0863037109375,
+      "loss": 0.001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.360287070274353,
+      "rewards/margins": 17.2882080078125,
+      "rewards/rejected": -17.648494720458984,
+      "step": 1290
+    },
+    {
+      "epoch": 0.32465505400511957,
+      "grad_norm": 0.0218505859375,
+      "learning_rate": 4.27046231453591e-06,
+      "logits/chosen": 0.5454662442207336,
+      "logits/rejected": 1.7699216604232788,
+      "logps/chosen": -73.63867950439453,
+      "logps/rejected": -1785.0687255859375,
+      "loss": 0.0008,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3033478856086731,
+      "rewards/margins": 16.751049041748047,
+      "rewards/rejected": -17.054393768310547,
+      "step": 1300
+    },
+    {
+      "epoch": 0.3271524005743897,
+      "grad_norm": 3.4458935260772705e-08,
+      "learning_rate": 4.255004913747196e-06,
+      "logits/chosen": 0.5776439905166626,
+      "logits/rejected": 1.7879540920257568,
+      "logps/chosen": -73.9155502319336,
+      "logps/rejected": -1853.062744140625,
+      "loss": 0.0017,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3109303414821625,
+      "rewards/margins": 17.477283477783203,
+      "rewards/rejected": -17.78821563720703,
+      "step": 1310
+    },
+    {
+      "epoch": 0.32964974714365985,
+      "grad_norm": 0.0025787353515625,
+      "learning_rate": 4.2394140850106825e-06,
+      "logits/chosen": 0.5839067697525024,
+      "logits/rejected": 1.7082710266113281,
+      "logps/chosen": -79.85846710205078,
+      "logps/rejected": -1769.608154296875,
+      "loss": 0.0022,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.36045509576797485,
+      "rewards/margins": 16.53636360168457,
+      "rewards/rejected": -16.896818161010742,
+      "step": 1320
+    },
+    {
+      "epoch": 0.33214709371293,
+      "grad_norm": 0.049072265625,
+      "learning_rate": 4.223691013651986e-06,
+      "logits/chosen": 0.4838981032371521,
+      "logits/rejected": 1.5925347805023193,
+      "logps/chosen": -78.32035827636719,
+      "logps/rejected": -1797.3795166015625,
+      "loss": 0.0008,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.33012980222702026,
+      "rewards/margins": 16.690799713134766,
+      "rewards/rejected": -17.02092933654785,
+      "step": 1330
+    },
+    {
+      "epoch": 0.33464444028220014,
+      "grad_norm": 0.07421875,
+      "learning_rate": 4.207836895050748e-06,
+      "logits/chosen": 0.5183486342430115,
+      "logits/rejected": 1.813838243484497,
+      "logps/chosen": -78.78418731689453,
+      "logps/rejected": -2234.604248046875,
+      "loss": 0.0014,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.35720717906951904,
+      "rewards/margins": 21.229446411132812,
+      "rewards/rejected": -21.586654663085938,
+      "step": 1340
+    },
+    {
+      "epoch": 0.33714178685147034,
+      "grad_norm": 0.034912109375,
+      "learning_rate": 4.1918529345497525e-06,
+      "logits/chosen": 0.6919676661491394,
+      "logits/rejected": 1.7151187658309937,
+      "logps/chosen": -73.4660415649414,
+      "logps/rejected": -1405.526123046875,
+      "loss": 0.0024,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.2937282919883728,
+      "rewards/margins": 13.013958930969238,
+      "rewards/rejected": -13.307687759399414,
+      "step": 1350
+    },
+    {
+      "epoch": 0.3396391334207405,
+      "grad_norm": 0.0673828125,
+      "learning_rate": 4.175740347363289e-06,
+      "logits/chosen": 0.58757483959198,
+      "logits/rejected": 1.651389718055725,
+      "logps/chosen": -76.53401947021484,
+      "logps/rejected": -1477.0704345703125,
+      "loss": 0.0025,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3171566426753998,
+      "rewards/margins": 13.620218276977539,
+      "rewards/rejected": -13.93737506866455,
+      "step": 1360
+    },
+    {
+      "epoch": 0.3421364799900106,
+      "grad_norm": 0.00014400482177734375,
+      "learning_rate": 4.159500358484759e-06,
+      "logits/chosen": 0.5347609519958496,
+      "logits/rejected": 1.775714635848999,
+      "logps/chosen": -80.72816467285156,
+      "logps/rejected": -2101.271240234375,
+      "loss": 0.0016,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.37172675132751465,
+      "rewards/margins": 19.874296188354492,
+      "rewards/rejected": -20.246021270751953,
+      "step": 1370
+    },
+    {
+      "epoch": 0.34463382655928076,
+      "grad_norm": 0.08544921875,
+      "learning_rate": 4.143134202593549e-06,
+      "logits/chosen": 0.624781608581543,
+      "logits/rejected": 1.6769899129867554,
+      "logps/chosen": -73.6835708618164,
+      "logps/rejected": -1517.2562255859375,
+      "loss": 0.002,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.28604602813720703,
+      "rewards/margins": 14.03973388671875,
+      "rewards/rejected": -14.325779914855957,
+      "step": 1380
+    },
+    {
+      "epoch": 0.3471311731285509,
+      "grad_norm": 0.000701904296875,
+      "learning_rate": 4.126643123961158e-06,
+      "logits/chosen": 0.5619300007820129,
+      "logits/rejected": 1.733264684677124,
+      "logps/chosen": -85.01399230957031,
+      "logps/rejected": -1977.3433837890625,
+      "loss": 0.0019,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4228588938713074,
+      "rewards/margins": 18.610515594482422,
+      "rewards/rejected": -19.033374786376953,
+      "step": 1390
+    },
+    {
+      "epoch": 0.34962851969782105,
+      "grad_norm": 0.054931640625,
+      "learning_rate": 4.110028376356599e-06,
+      "logits/chosen": 0.6396419405937195,
+      "logits/rejected": 1.709763765335083,
+      "logps/chosen": -78.08328247070312,
+      "logps/rejected": -1361.017822265625,
+      "loss": 0.0018,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.32667404413223267,
+      "rewards/margins": 12.484647750854492,
+      "rewards/rejected": -12.811322212219238,
+      "step": 1400
+    },
+    {
+      "epoch": 0.3521258662670912,
+      "grad_norm": 0.0230712890625,
+      "learning_rate": 4.093291222951079e-06,
+      "logits/chosen": 0.59341961145401,
+      "logits/rejected": 1.863669991493225,
+      "logps/chosen": -88.14995574951172,
+      "logps/rejected": -1872.766845703125,
+      "loss": 0.0006,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4488135874271393,
+      "rewards/margins": 17.484996795654297,
+      "rewards/rejected": -17.93381118774414,
+      "step": 1410
+    },
+    {
+      "epoch": 0.3546232128363614,
+      "grad_norm": 0.058837890625,
+      "learning_rate": 4.076432936221965e-06,
+      "logits/chosen": 0.7002652287483215,
+      "logits/rejected": 1.8483015298843384,
+      "logps/chosen": -82.04905700683594,
+      "logps/rejected": -1564.48388671875,
+      "loss": 0.0025,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.38699063658714294,
+      "rewards/margins": 14.584707260131836,
+      "rewards/rejected": -14.971699714660645,
+      "step": 1420
+    },
+    {
+      "epoch": 0.35712055940563153,
+      "grad_norm": 0.0228271484375,
+      "learning_rate": 4.059454797856039e-06,
+      "logits/chosen": 0.6757210493087769,
+      "logits/rejected": 1.8517526388168335,
+      "logps/chosen": -79.04359436035156,
+      "logps/rejected": -1546.5145263671875,
+      "loss": 0.0025,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.36757320165634155,
+      "rewards/margins": 14.412260055541992,
+      "rewards/rejected": -14.77983570098877,
+      "step": 1430
+    },
+    {
+      "epoch": 0.3596179059749017,
+      "grad_norm": 2.86102294921875e-05,
+      "learning_rate": 4.042358098652057e-06,
+      "logits/chosen": 0.6149075627326965,
+      "logits/rejected": 1.7819246053695679,
+      "logps/chosen": -79.50230407714844,
+      "logps/rejected": -1636.815673828125,
+      "loss": 0.001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.36546218395233154,
+      "rewards/margins": 15.302419662475586,
+      "rewards/rejected": -15.667881965637207,
+      "step": 1440
+    },
+    {
+      "epoch": 0.3621152525441718,
+      "grad_norm": 0.00115203857421875,
+      "learning_rate": 4.025144138422615e-06,
+      "logits/chosen": 0.6270621418952942,
+      "logits/rejected": 1.8290207386016846,
+      "logps/chosen": -94.74217224121094,
+      "logps/rejected": -1927.5875244140625,
+      "loss": 0.0019,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5115745663642883,
+      "rewards/margins": 18.033849716186523,
+      "rewards/rejected": -18.54542350769043,
+      "step": 1450
+    },
+    {
+      "epoch": 0.36461259911344196,
+      "grad_norm": 0.01483154296875,
+      "learning_rate": 4.007814225895321e-06,
+      "logits/chosen": 0.6495813131332397,
+      "logits/rejected": 1.905644178390503,
+      "logps/chosen": -77.37786865234375,
+      "logps/rejected": -1849.2958984375,
+      "loss": 0.0021,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.35049691796302795,
+      "rewards/margins": 17.428863525390625,
+      "rewards/rejected": -17.779361724853516,
+      "step": 1460
+    },
+    {
+      "epoch": 0.3671099456827121,
+      "grad_norm": 0.030029296875,
+      "learning_rate": 3.990369678613303e-06,
+      "logits/chosen": 0.5495260953903198,
+      "logits/rejected": 1.7052236795425415,
+      "logps/chosen": -80.6564712524414,
+      "logps/rejected": -1763.9212646484375,
+      "loss": 0.0008,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3531542718410492,
+      "rewards/margins": 16.24311637878418,
+      "rewards/rejected": -16.596271514892578,
+      "step": 1470
+    },
+    {
+      "epoch": 0.36960729225198224,
+      "grad_norm": 0.005859375,
+      "learning_rate": 3.97281182283504e-06,
+      "logits/chosen": 0.625984251499176,
+      "logits/rejected": 1.8606735467910767,
+      "logps/chosen": -79.91282653808594,
+      "logps/rejected": -2002.6253662109375,
+      "loss": 0.0015,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.37286117672920227,
+      "rewards/margins": 18.913448333740234,
+      "rewards/rejected": -19.28631019592285,
+      "step": 1480
+    },
+    {
+      "epoch": 0.37210463882125244,
+      "grad_norm": 0.0966796875,
+      "learning_rate": 3.955141993433526e-06,
+      "logits/chosen": 0.6155849695205688,
+      "logits/rejected": 1.8059221506118774,
+      "logps/chosen": -83.35047912597656,
+      "logps/rejected": -1705.9954833984375,
+      "loss": 0.0009,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.40526407957077026,
+      "rewards/margins": 15.947067260742188,
+      "rewards/rejected": -16.352331161499023,
+      "step": 1490
+    },
+    {
+      "epoch": 0.3746019853905226,
+      "grad_norm": 0.028076171875,
+      "learning_rate": 3.937361533794784e-06,
+      "logits/chosen": 0.6340751647949219,
+      "logits/rejected": 1.7404279708862305,
+      "logps/chosen": -88.5922622680664,
+      "logps/rejected": -1672.023193359375,
+      "loss": 0.0018,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4479256570339203,
+      "rewards/margins": 15.48670768737793,
+      "rewards/rejected": -15.934633255004883,
+      "step": 1500
+    },
+    {
+      "epoch": 0.3770993319597927,
+      "grad_norm": 0.005706787109375,
+      "learning_rate": 3.919471795715738e-06,
+      "logits/chosen": 0.6204045414924622,
+      "logits/rejected": 1.7921810150146484,
+      "logps/chosen": -76.07710266113281,
+      "logps/rejected": -1581.0501708984375,
+      "loss": 0.0012,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3369273245334625,
+      "rewards/margins": 14.815042495727539,
+      "rewards/rejected": -15.151969909667969,
+      "step": 1510
+    },
+    {
+      "epoch": 0.37959667852906287,
+      "grad_norm": 0.033203125,
+      "learning_rate": 3.901474139301433e-06,
+      "logits/chosen": 0.5973562002182007,
+      "logits/rejected": 1.781730055809021,
+      "logps/chosen": -83.27436065673828,
+      "logps/rejected": -1706.674560546875,
+      "loss": 0.0011,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3964506983757019,
+      "rewards/margins": 15.901094436645508,
+      "rewards/rejected": -16.297544479370117,
+      "step": 1520
+    },
+    {
+      "epoch": 0.382094025098333,
+      "grad_norm": 0.020263671875,
+      "learning_rate": 3.883369932861634e-06,
+      "logits/chosen": 0.66780024766922,
+      "logits/rejected": 1.8160803318023682,
+      "logps/chosen": -88.08844757080078,
+      "logps/rejected": -1613.07373046875,
+      "loss": 0.0006,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.44633254408836365,
+      "rewards/margins": 15.01531982421875,
+      "rewards/rejected": -15.461652755737305,
+      "step": 1530
+    },
+    {
+      "epoch": 0.38459137166760315,
+      "grad_norm": 5.245208740234375e-05,
+      "learning_rate": 3.865160552806796e-06,
+      "logits/chosen": 0.6651610136032104,
+      "logits/rejected": 1.8406972885131836,
+      "logps/chosen": -79.48463439941406,
+      "logps/rejected": -1628.993896484375,
+      "loss": 0.0008,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.36581045389175415,
+      "rewards/margins": 15.247782707214355,
+      "rewards/rejected": -15.613592147827148,
+      "step": 1540
+    },
+    {
+      "epoch": 0.3870887182368733,
+      "grad_norm": 1.7404556274414062e-05,
+      "learning_rate": 3.84684738354342e-06,
+      "logits/chosen": 0.6299249529838562,
+      "logits/rejected": 1.8017246723175049,
+      "logps/chosen": -78.25436401367188,
+      "logps/rejected": -1699.4664306640625,
+      "loss": 0.0021,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.34191855788230896,
+      "rewards/margins": 15.943153381347656,
+      "rewards/rejected": -16.285072326660156,
+      "step": 1550
+    },
+    {
+      "epoch": 0.3895860648061435,
+      "grad_norm": 0.043212890625,
+      "learning_rate": 3.828431817368798e-06,
+      "logits/chosen": 0.6114810705184937,
+      "logits/rejected": 1.7776029109954834,
+      "logps/chosen": -83.11959075927734,
+      "logps/rejected": -1808.9710693359375,
+      "loss": 0.0021,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.39526110887527466,
+      "rewards/margins": 16.893442153930664,
+      "rewards/rejected": -17.2887020111084,
+      "step": 1560
+    },
+    {
+      "epoch": 0.39208341137541364,
+      "grad_norm": 0.0859375,
+      "learning_rate": 3.8099152543651684e-06,
+      "logits/chosen": 0.5259889364242554,
+      "logits/rejected": 1.8491640090942383,
+      "logps/chosen": -76.72280883789062,
+      "logps/rejected": -1907.3863525390625,
+      "loss": 0.0009,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3375067710876465,
+      "rewards/margins": 18.016918182373047,
+      "rewards/rejected": -18.354427337646484,
+      "step": 1570
+    },
+    {
+      "epoch": 0.3945807579446838,
+      "grad_norm": 0.07177734375,
+      "learning_rate": 3.791299102293261e-06,
+      "logits/chosen": 0.5979338884353638,
+      "logits/rejected": 1.8080129623413086,
+      "logps/chosen": -85.99974060058594,
+      "logps/rejected": -1962.4896240234375,
+      "loss": 0.0014,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4262419641017914,
+      "rewards/margins": 18.461589813232422,
+      "rewards/rejected": -18.88783073425293,
+      "step": 1580
+    },
+    {
+      "epoch": 0.3970781045139539,
+      "grad_norm": 0.0078125,
+      "learning_rate": 3.7725847764852774e-06,
+      "logits/chosen": 0.5477781891822815,
+      "logits/rejected": 1.7578785419464111,
+      "logps/chosen": -83.63060760498047,
+      "logps/rejected": -1990.638671875,
+      "loss": 0.0014,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.39986157417297363,
+      "rewards/margins": 18.64605140686035,
+      "rewards/rejected": -19.045909881591797,
+      "step": 1590
+    },
+    {
+      "epoch": 0.39957545108322406,
+      "grad_norm": 0.0067138671875,
+      "learning_rate": 3.7537736997372833e-06,
+      "logits/chosen": 0.5983849167823792,
+      "logits/rejected": 1.6318690776824951,
+      "logps/chosen": -74.38432312011719,
+      "logps/rejected": -1474.78759765625,
+      "loss": 0.0019,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.30779850482940674,
+      "rewards/margins": 13.598607063293457,
+      "rewards/rejected": -13.906405448913574,
+      "step": 1600
+    },
+    {
+      "epoch": 0.4020727976524942,
+      "grad_norm": 0.0035247802734375,
+      "learning_rate": 3.734867302201038e-06,
+      "logits/chosen": 0.620630145072937,
+      "logits/rejected": 1.7149145603179932,
+      "logps/chosen": -75.28178405761719,
+      "logps/rejected": -1552.66650390625,
+      "loss": 0.0012,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.32418856024742126,
+      "rewards/margins": 14.512880325317383,
+      "rewards/rejected": -14.837068557739258,
+      "step": 1610
+    },
+    {
+      "epoch": 0.40457014422176435,
+      "grad_norm": 0.04833984375,
+      "learning_rate": 3.7158670212752666e-06,
+      "logits/chosen": 0.609667181968689,
+      "logits/rejected": 1.8217569589614868,
+      "logps/chosen": -75.36375427246094,
+      "logps/rejected": -1846.893798828125,
+      "loss": 0.0028,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.31798630952835083,
+      "rewards/margins": 17.42896842956543,
+      "rewards/rejected": -17.746957778930664,
+      "step": 1620
+    },
+    {
+      "epoch": 0.40706749079103455,
+      "grad_norm": 0.0034332275390625,
+      "learning_rate": 3.696774301496376e-06,
+      "logits/chosen": 0.6272271871566772,
+      "logits/rejected": 1.8513765335083008,
+      "logps/chosen": -76.99528503417969,
+      "logps/rejected": -1668.758056640625,
+      "loss": 0.0009,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.33525028824806213,
+      "rewards/margins": 15.664929389953613,
+      "rewards/rejected": -16.000181198120117,
+      "step": 1630
+    },
+    {
+      "epoch": 0.4095648373603047,
+      "grad_norm": 0.0020599365234375,
+      "learning_rate": 3.677590594428629e-06,
+      "logits/chosen": 0.6275375485420227,
+      "logits/rejected": 1.746649980545044,
+      "logps/chosen": -82.9039535522461,
+      "logps/rejected": -1647.865478515625,
+      "loss": 0.0023,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.402643620967865,
+      "rewards/margins": 15.365156173706055,
+      "rewards/rejected": -15.767801284790039,
+      "step": 1640
+    },
+    {
+      "epoch": 0.41206218392957483,
+      "grad_norm": 0.0001926422119140625,
+      "learning_rate": 3.658317358553794e-06,
+      "logits/chosen": 0.6051415205001831,
+      "logits/rejected": 1.807227373123169,
+      "logps/chosen": -78.21363830566406,
+      "logps/rejected": -1698.2001953125,
+      "loss": 0.0013,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3519694209098816,
+      "rewards/margins": 15.862031936645508,
+      "rewards/rejected": -16.214000701904297,
+      "step": 1650
+    },
+    {
+      "epoch": 0.414559530498845,
+      "grad_norm": 0.05908203125,
+      "learning_rate": 3.638956059160252e-06,
+      "logits/chosen": 0.659566342830658,
+      "logits/rejected": 1.9395606517791748,
+      "logps/chosen": -79.38732147216797,
+      "logps/rejected": -1887.150390625,
+      "loss": 0.001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.369088351726532,
+      "rewards/margins": 17.81894874572754,
+      "rewards/rejected": -18.188034057617188,
+      "step": 1660
+    },
+    {
+      "epoch": 0.4170568770681151,
+      "grad_norm": 0.0064697265625,
+      "learning_rate": 3.6195081682315972e-06,
+      "logits/chosen": 0.6888834834098816,
+      "logits/rejected": 1.855298638343811,
+      "logps/chosen": -87.92467498779297,
+      "logps/rejected": -1717.685546875,
+      "loss": 0.0006,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.45286068320274353,
+      "rewards/margins": 16.04346466064453,
+      "rewards/rejected": -16.496326446533203,
+      "step": 1670
+    },
+    {
+      "epoch": 0.41955422363738526,
+      "grad_norm": 0.026123046875,
+      "learning_rate": 3.5999751643347342e-06,
+      "logits/chosen": 0.5452974438667297,
+      "logits/rejected": 1.710627794265747,
+      "logps/chosen": -84.69573974609375,
+      "logps/rejected": -1964.759033203125,
+      "loss": 0.0015,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4223009943962097,
+      "rewards/margins": 18.474639892578125,
+      "rewards/rejected": -18.896940231323242,
+      "step": 1680
+    },
+    {
+      "epoch": 0.4220515702066554,
+      "grad_norm": 0.050537109375,
+      "learning_rate": 3.5803585325074536e-06,
+      "logits/chosen": 0.5890778303146362,
+      "logits/rejected": 1.8013776540756226,
+      "logps/chosen": -78.17984008789062,
+      "logps/rejected": -1845.5576171875,
+      "loss": 0.0014,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3491733968257904,
+      "rewards/margins": 17.39130210876465,
+      "rewards/rejected": -17.740474700927734,
+      "step": 1690
+    },
+    {
+      "epoch": 0.4245489167759256,
+      "grad_norm": 0.0849609375,
+      "learning_rate": 3.5606597641455387e-06,
+      "logits/chosen": 0.6665171384811401,
+      "logits/rejected": 1.7867825031280518,
+      "logps/chosen": -82.6786117553711,
+      "logps/rejected": -1745.9921875,
+      "loss": 0.0021,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4073053300380707,
+      "rewards/margins": 16.353519439697266,
+      "rewards/rejected": -16.760822296142578,
+      "step": 1700
+    },
+    {
+      "epoch": 0.42704626334519574,
+      "grad_norm": 0.054443359375,
+      "learning_rate": 3.540880356889376e-06,
+      "logits/chosen": 0.6666916012763977,
+      "logits/rejected": 1.773199439048767,
+      "logps/chosen": -83.08412170410156,
+      "logps/rejected": -1565.4173583984375,
+      "loss": 0.0018,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.39527103304862976,
+      "rewards/margins": 14.483154296875,
+      "rewards/rejected": -14.878425598144531,
+      "step": 1710
+    },
+    {
+      "epoch": 0.4295436099144659,
+      "grad_norm": 0.036865234375,
+      "learning_rate": 3.5210218145100934e-06,
+      "logits/chosen": 0.6796804666519165,
+      "logits/rejected": 1.826575517654419,
+      "logps/chosen": -76.29814147949219,
+      "logps/rejected": -1558.442626953125,
+      "loss": 0.0028,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3252686560153961,
+      "rewards/margins": 14.555532455444336,
+      "rewards/rejected": -14.8808012008667,
+      "step": 1720
+    },
+    {
+      "epoch": 0.432040956483736,
+      "grad_norm": 0.037841796875,
+      "learning_rate": 3.5010856467952335e-06,
+      "logits/chosen": 0.6157870292663574,
+      "logits/rejected": 1.7099930047988892,
+      "logps/chosen": -81.55091094970703,
+      "logps/rejected": -1618.1546630859375,
+      "loss": 0.0021,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3764324188232422,
+      "rewards/margins": 14.980290412902832,
+      "rewards/rejected": -15.356722831726074,
+      "step": 1730
+    },
+    {
+      "epoch": 0.43453830305300617,
+      "grad_norm": 0.1484375,
+      "learning_rate": 3.4810733694339687e-06,
+      "logits/chosen": 0.5888208150863647,
+      "logits/rejected": 1.7958781719207764,
+      "logps/chosen": -84.57051086425781,
+      "logps/rejected": -1871.361083984375,
+      "loss": 0.0017,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4017999768257141,
+      "rewards/margins": 17.520339965820312,
+      "rewards/rejected": -17.922138214111328,
+      "step": 1740
+    },
+    {
+      "epoch": 0.4370356496222763,
+      "grad_norm": 0.032958984375,
+      "learning_rate": 3.4609865039018676e-06,
+      "logits/chosen": 0.682072639465332,
+      "logits/rejected": 1.7670128345489502,
+      "logps/chosen": -83.45240783691406,
+      "logps/rejected": -1766.69140625,
+      "loss": 0.0031,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.40751171112060547,
+      "rewards/margins": 16.532672882080078,
+      "rewards/rejected": -16.940181732177734,
+      "step": 1750
+    },
+    {
+      "epoch": 0.43953299619154645,
+      "grad_norm": 0.024658203125,
+      "learning_rate": 3.4408265773452226e-06,
+      "logits/chosen": 0.6357883214950562,
+      "logits/rejected": 1.7647396326065063,
+      "logps/chosen": -75.90863037109375,
+      "logps/rejected": -1793.2620849609375,
+      "loss": 0.003,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.32462552189826965,
+      "rewards/margins": 16.89077377319336,
+      "rewards/rejected": -17.21540069580078,
+      "step": 1760
+    },
+    {
+      "epoch": 0.44203034276081665,
+      "grad_norm": 0.00022411346435546875,
+      "learning_rate": 3.420595122464942e-06,
+      "logits/chosen": 0.5758832693099976,
+      "logits/rejected": 1.7814972400665283,
+      "logps/chosen": -79.77733612060547,
+      "logps/rejected": -1759.0218505859375,
+      "loss": 0.0016,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.36440029740333557,
+      "rewards/margins": 16.545846939086914,
+      "rewards/rejected": -16.910245895385742,
+      "step": 1770
+    },
+    {
+      "epoch": 0.4445276893300868,
+      "grad_norm": 0.0712890625,
+      "learning_rate": 3.4002936774000284e-06,
+      "logits/chosen": 0.5318555235862732,
+      "logits/rejected": 1.8811362981796265,
+      "logps/chosen": -77.92293548583984,
+      "logps/rejected": -2195.729736328125,
+      "loss": 0.0009,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3443797826766968,
+      "rewards/margins": 20.8763370513916,
+      "rewards/rejected": -21.22071647644043,
+      "step": 1780
+    },
+    {
+      "epoch": 0.44702503589935694,
+      "grad_norm": 0.035888671875,
+      "learning_rate": 3.3799237856106348e-06,
+      "logits/chosen": 0.5407482385635376,
+      "logits/rejected": 1.725608229637146,
+      "logps/chosen": -77.46542358398438,
+      "logps/rejected": -1750.359619140625,
+      "loss": 0.001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.34000295400619507,
+      "rewards/margins": 16.431018829345703,
+      "rewards/rejected": -16.77102279663086,
+      "step": 1790
+    },
+    {
+      "epoch": 0.4495223824686271,
+      "grad_norm": 0.0003185272216796875,
+      "learning_rate": 3.35948699576072e-06,
+      "logits/chosen": 0.5788090825080872,
+      "logits/rejected": 1.8735895156860352,
+      "logps/chosen": -83.07856750488281,
+      "logps/rejected": -2100.754638671875,
+      "loss": 0.0009,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3939378559589386,
+      "rewards/margins": 19.880746841430664,
+      "rewards/rejected": -20.274681091308594,
+      "step": 1800
+    },
+    {
+      "epoch": 0.4520197290378972,
+      "grad_norm": 0.13671875,
+      "learning_rate": 3.3389848616003085e-06,
+      "logits/chosen": 0.5929907560348511,
+      "logits/rejected": 1.6974430084228516,
+      "logps/chosen": -79.56812286376953,
+      "logps/rejected": -1787.6068115234375,
+      "loss": 0.0033,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3681466281414032,
+      "rewards/margins": 16.794055938720703,
+      "rewards/rejected": -17.16220474243164,
+      "step": 1810
+    },
+    {
+      "epoch": 0.45451707560716736,
+      "grad_norm": 0.0390625,
+      "learning_rate": 3.3184189418473674e-06,
+      "logits/chosen": 0.5751794576644897,
+      "logits/rejected": 1.7812267541885376,
+      "logps/chosen": -77.59693908691406,
+      "logps/rejected": -1778.1439208984375,
+      "loss": 0.0014,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3408336043357849,
+      "rewards/margins": 16.739145278930664,
+      "rewards/rejected": -17.079978942871094,
+      "step": 1820
+    },
+    {
+      "epoch": 0.45701442217643756,
+      "grad_norm": 0.001373291015625,
+      "learning_rate": 3.2977908000692925e-06,
+      "logits/chosen": 0.5487096905708313,
+      "logits/rejected": 1.74447500705719,
+      "logps/chosen": -80.25045013427734,
+      "logps/rejected": -1946.0765380859375,
+      "loss": 0.0023,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3701193928718567,
+      "rewards/margins": 18.401947021484375,
+      "rewards/rejected": -18.77206802368164,
+      "step": 1830
+    },
+    {
+      "epoch": 0.4595117687457077,
+      "grad_norm": 0.048095703125,
+      "learning_rate": 3.2771020045640435e-06,
+      "logits/chosen": 0.6444208025932312,
+      "logits/rejected": 1.6972076892852783,
+      "logps/chosen": -78.00311279296875,
+      "logps/rejected": -1579.7557373046875,
+      "loss": 0.0023,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3551621735095978,
+      "rewards/margins": 14.743237495422363,
+      "rewards/rejected": -15.09839916229248,
+      "step": 1840
+    },
+    {
+      "epoch": 0.46200911531497785,
+      "grad_norm": 0.07177734375,
+      "learning_rate": 3.256354128240907e-06,
+      "logits/chosen": 0.6255194544792175,
+      "logits/rejected": 1.7124531269073486,
+      "logps/chosen": -85.12455749511719,
+      "logps/rejected": -1608.01171875,
+      "loss": 0.0024,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.401846319437027,
+      "rewards/margins": 14.84345531463623,
+      "rewards/rejected": -15.245302200317383,
+      "step": 1850
+    },
+    {
+      "epoch": 0.464506461884248,
+      "grad_norm": 1.191438059322536e-10,
+      "learning_rate": 3.235548748500914e-06,
+      "logits/chosen": 0.5620906352996826,
+      "logits/rejected": 1.8212181329727173,
+      "logps/chosen": -78.25764465332031,
+      "logps/rejected": -1836.809326171875,
+      "loss": 0.0014,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3476276397705078,
+      "rewards/margins": 17.330211639404297,
+      "rewards/rejected": -17.677841186523438,
+      "step": 1860
+    },
+    {
+      "epoch": 0.46700380845351813,
+      "grad_norm": 0.038330078125,
+      "learning_rate": 3.214687447116913e-06,
+      "logits/chosen": 0.5774132609367371,
+      "logits/rejected": 1.7261114120483398,
+      "logps/chosen": -76.27984619140625,
+      "logps/rejected": -1707.5220947265625,
+      "loss": 0.002,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3342064321041107,
+      "rewards/margins": 15.912832260131836,
+      "rewards/rejected": -16.247039794921875,
+      "step": 1870
+    },
+    {
+      "epoch": 0.4695011550227883,
+      "grad_norm": 0.002532958984375,
+      "learning_rate": 3.193771810113313e-06,
+      "logits/chosen": 0.5532559752464294,
+      "logits/rejected": 1.8629133701324463,
+      "logps/chosen": -79.43685150146484,
+      "logps/rejected": -2138.56884765625,
+      "loss": 0.001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3567715287208557,
+      "rewards/margins": 20.29428482055664,
+      "rewards/rejected": -20.65105628967285,
+      "step": 1880
+    },
+    {
+      "epoch": 0.4719985015920584,
+      "grad_norm": 0.018310546875,
+      "learning_rate": 3.1728034276455032e-06,
+      "logits/chosen": 0.6407243609428406,
+      "logits/rejected": 1.7773427963256836,
+      "logps/chosen": -75.46717834472656,
+      "logps/rejected": -1624.9876708984375,
+      "loss": 0.0017,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3326609432697296,
+      "rewards/margins": 15.156428337097168,
+      "rewards/rejected": -15.489087104797363,
+      "step": 1890
+    },
+    {
+      "epoch": 0.4744958481613286,
+      "grad_norm": 0.001556396484375,
+      "learning_rate": 3.1517838938789597e-06,
+      "logits/chosen": 0.5151150822639465,
+      "logits/rejected": 1.6990512609481812,
+      "logps/chosen": -79.35011291503906,
+      "logps/rejected": -1993.761962890625,
+      "loss": 0.0018,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3504992127418518,
+      "rewards/margins": 18.6396427154541,
+      "rewards/rejected": -18.990140914916992,
+      "step": 1900
+    },
+    {
+      "epoch": 0.47699319473059876,
+      "grad_norm": 0.036376953125,
+      "learning_rate": 3.130714806868041e-06,
+      "logits/chosen": 0.5437807440757751,
+      "logits/rejected": 1.6498979330062866,
+      "logps/chosen": -77.74958801269531,
+      "logps/rejected": -1746.6015625,
+      "loss": 0.0011,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3441064655780792,
+      "rewards/margins": 16.398990631103516,
+      "rewards/rejected": -16.74309730529785,
+      "step": 1910
+    },
+    {
+      "epoch": 0.4794905412998689,
+      "grad_norm": 0.0269775390625,
+      "learning_rate": 3.1095977684344976e-06,
+      "logits/chosen": 0.6197426319122314,
+      "logits/rejected": 1.865501046180725,
+      "logps/chosen": -83.05316162109375,
+      "logps/rejected": -1912.490234375,
+      "loss": 0.0011,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.40501928329467773,
+      "rewards/margins": 18.013139724731445,
+      "rewards/rejected": -18.41815757751465,
+      "step": 1920
+    },
+    {
+      "epoch": 0.48198788786913904,
+      "grad_norm": 0.004058837890625,
+      "learning_rate": 3.0884343840456874e-06,
+      "logits/chosen": 0.5581328868865967,
+      "logits/rejected": 1.818427324295044,
+      "logps/chosen": -82.58245849609375,
+      "logps/rejected": -2075.47509765625,
+      "loss": 0.0012,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.39019304513931274,
+      "rewards/margins": 19.626462936401367,
+      "rewards/rejected": -20.016658782958984,
+      "step": 1930
+    },
+    {
+      "epoch": 0.4844852344384092,
+      "grad_norm": 5.0067901611328125e-06,
+      "learning_rate": 3.0672262626925174e-06,
+      "logits/chosen": 0.49209919571876526,
+      "logits/rejected": 1.718467354774475,
+      "logps/chosen": -82.48509216308594,
+      "logps/rejected": -1921.3333740234375,
+      "loss": 0.0011,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.35821837186813354,
+      "rewards/margins": 18.051937103271484,
+      "rewards/rejected": -18.410158157348633,
+      "step": 1940
+    },
+    {
+      "epoch": 0.4869825810076793,
+      "grad_norm": 0.0255126953125,
+      "learning_rate": 3.0459750167671147e-06,
+      "logits/chosen": 0.4969088137149811,
+      "logits/rejected": 1.7654139995574951,
+      "logps/chosen": -79.9859390258789,
+      "logps/rejected": -2075.444580078125,
+      "loss": 0.0009,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3472353518009186,
+      "rewards/margins": 19.543991088867188,
+      "rewards/rejected": -19.89122772216797,
+      "step": 1950
+    },
+    {
+      "epoch": 0.48947992757694947,
+      "grad_norm": 0.1552734375,
+      "learning_rate": 3.024682261940247e-06,
+      "logits/chosen": 0.5588921904563904,
+      "logits/rejected": 1.6852495670318604,
+      "logps/chosen": -83.98374938964844,
+      "logps/rejected": -1691.5599365234375,
+      "loss": 0.0013,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.39910078048706055,
+      "rewards/margins": 15.737649917602539,
+      "rewards/rejected": -16.136751174926758,
+      "step": 1960
+    },
+    {
+      "epoch": 0.49197727414621967,
+      "grad_norm": 1.4185905456542969e-05,
+      "learning_rate": 3.0033496170384803e-06,
+      "logits/chosen": 0.6266374588012695,
+      "logits/rejected": 1.8179903030395508,
+      "logps/chosen": -77.69737243652344,
+      "logps/rejected": -1697.776123046875,
+      "loss": 0.0017,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3449680805206299,
+      "rewards/margins": 15.93467903137207,
+      "rewards/rejected": -16.279645919799805,
+      "step": 1970
+    },
+    {
+      "epoch": 0.4944746207154898,
+      "grad_norm": 0.0458984375,
+      "learning_rate": 2.9819787039211068e-06,
+      "logits/chosen": 0.5513324737548828,
+      "logits/rejected": 1.6900783777236938,
+      "logps/chosen": -76.17434692382812,
+      "logps/rejected": -1829.5364990234375,
+      "loss": 0.0011,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.31986111402511597,
+      "rewards/margins": 17.185441970825195,
+      "rewards/rejected": -17.50530242919922,
+      "step": 1980
+    },
+    {
+      "epoch": 0.49697196728475995,
+      "grad_norm": 0.002777099609375,
+      "learning_rate": 2.960571147356845e-06,
+      "logits/chosen": 0.5562096834182739,
+      "logits/rejected": 1.8595008850097656,
+      "logps/chosen": -83.13392639160156,
+      "logps/rejected": -2010.169921875,
+      "loss": 0.0007,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.395429790019989,
+      "rewards/margins": 18.9808292388916,
+      "rewards/rejected": -19.37626075744629,
+      "step": 1990
+    },
+    {
+      "epoch": 0.4994693138540301,
+      "grad_norm": 0.000293731689453125,
+      "learning_rate": 2.9391285749003046e-06,
+      "logits/chosen": 0.5312787294387817,
+      "logits/rejected": 1.7356303930282593,
+      "logps/chosen": -95.15312194824219,
+      "logps/rejected": -2160.408935546875,
+      "loss": 0.0013,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5149081349372864,
+      "rewards/margins": 20.321773529052734,
+      "rewards/rejected": -20.836681365966797,
+      "step": 2000
+    },
+    {
+      "epoch": 0.4994693138540301,
+      "eval_logits/chosen": 0.6455119848251343,
+      "eval_logits/rejected": 1.5546293258666992,
+      "eval_logps/chosen": -82.33647155761719,
+      "eval_logps/rejected": -980.0130615234375,
+      "eval_loss": 0.0030529608484357595,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.37980592250823975,
+      "eval_rewards/margins": 8.809426307678223,
+      "eval_rewards/rejected": -9.189233779907227,
+      "eval_runtime": 0.6247,
+      "eval_samples_per_second": 8.004,
+      "eval_steps_per_second": 8.004,
+      "step": 2000
+    },
+    {
+      "epoch": 0.5019666604233003,
+      "grad_norm": 0.06494140625,
+      "learning_rate": 2.9176526167682543e-06,
+      "logits/chosen": 0.6404844522476196,
+      "logits/rejected": 1.8602796792984009,
+      "logps/chosen": -82.97758483886719,
+      "logps/rejected": -1836.1568603515625,
+      "loss": 0.0024,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4049321115016937,
+      "rewards/margins": 17.25887107849121,
+      "rewards/rejected": -17.663803100585938,
+      "step": 2010
+    },
+    {
+      "epoch": 0.5044640069925704,
+      "grad_norm": 0.0166015625,
+      "learning_rate": 2.8961449057156775e-06,
+      "logits/chosen": 0.5347205400466919,
+      "logits/rejected": 1.700486421585083,
+      "logps/chosen": -84.12736511230469,
+      "logps/rejected": -1874.214111328125,
+      "loss": 0.0007,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3977189064025879,
+      "rewards/margins": 17.567378997802734,
+      "rewards/rejected": -17.965099334716797,
+      "step": 2020
+    },
+    {
+      "epoch": 0.5069613535618406,
+      "grad_norm": 0.0013427734375,
+      "learning_rate": 2.874607076911642e-06,
+      "logits/chosen": 0.5987354516983032,
+      "logits/rejected": 1.7991135120391846,
+      "logps/chosen": -81.83995819091797,
+      "logps/rejected": -1828.4136962890625,
+      "loss": 0.0015,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.38373640179634094,
+      "rewards/margins": 17.181564331054688,
+      "rewards/rejected": -17.5653018951416,
+      "step": 2030
+    },
+    {
+      "epoch": 0.5094587001311107,
+      "grad_norm": 0.000850677490234375,
+      "learning_rate": 2.8530407678149806e-06,
+      "logits/chosen": 0.6027461886405945,
+      "logits/rejected": 1.730103850364685,
+      "logps/chosen": -81.79703521728516,
+      "logps/rejected": -1646.1201171875,
+      "loss": 0.0018,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.37951746582984924,
+      "rewards/margins": 15.314178466796875,
+      "rewards/rejected": -15.693696975708008,
+      "step": 2040
+    },
+    {
+      "epoch": 0.5119560467003809,
+      "grad_norm": 0.00055694580078125,
+      "learning_rate": 2.8314476180498003e-06,
+      "logits/chosen": 0.6401151418685913,
+      "logits/rejected": 1.7924197912216187,
+      "logps/chosen": -85.15666198730469,
+      "logps/rejected": -1746.1314697265625,
+      "loss": 0.0023,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.421166330575943,
+      "rewards/margins": 16.335796356201172,
+      "rewards/rejected": -16.756961822509766,
+      "step": 2050
+    },
+    {
+      "epoch": 0.514453393269651,
+      "grad_norm": 0.032958984375,
+      "learning_rate": 2.8098292692808253e-06,
+      "logits/chosen": 0.6529192328453064,
+      "logits/rejected": 1.7113368511199951,
+      "logps/chosen": -83.26264953613281,
+      "logps/rejected": -1448.375732421875,
+      "loss": 0.0023,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3913651704788208,
+      "rewards/margins": 13.431114196777344,
+      "rewards/rejected": -13.822479248046875,
+      "step": 2060
+    },
+    {
+      "epoch": 0.5169507398389211,
+      "grad_norm": 0.035400390625,
+      "learning_rate": 2.7881873650885904e-06,
+      "logits/chosen": 0.6235641241073608,
+      "logits/rejected": 1.7722196578979492,
+      "logps/chosen": -85.73550415039062,
+      "logps/rejected": -1683.2877197265625,
+      "loss": 0.0015,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.42536306381225586,
+      "rewards/margins": 15.712361335754395,
+      "rewards/rejected": -16.13772201538086,
+      "step": 2070
+    },
+    {
+      "epoch": 0.5194480864081913,
+      "grad_norm": 0.07470703125,
+      "learning_rate": 2.7665235508444772e-06,
+      "logits/chosen": 0.5478901267051697,
+      "logits/rejected": 1.8091357946395874,
+      "logps/chosen": -79.84373474121094,
+      "logps/rejected": -1996.998779296875,
+      "loss": 0.0018,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.368811696767807,
+      "rewards/margins": 18.87316131591797,
+      "rewards/rejected": -19.241975784301758,
+      "step": 2080
+    },
+    {
+      "epoch": 0.5219454329774614,
+      "grad_norm": 0.000518798828125,
+      "learning_rate": 2.7448394735856275e-06,
+      "logits/chosen": 0.5016141533851624,
+      "logits/rejected": 1.7318010330200195,
+      "logps/chosen": -88.0289306640625,
+      "logps/rejected": -2137.91552734375,
+      "loss": 0.0025,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.43785446882247925,
+      "rewards/margins": 20.17669105529785,
+      "rewards/rejected": -20.614543914794922,
+      "step": 2090
+    },
+    {
+      "epoch": 0.5244427795467316,
+      "grad_norm": 0.041748046875,
+      "learning_rate": 2.723136781889722e-06,
+      "logits/chosen": 0.6009372472763062,
+      "logits/rejected": 1.8194977045059204,
+      "logps/chosen": -82.27381896972656,
+      "logps/rejected": -1805.30859375,
+      "loss": 0.0026,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.38621821999549866,
+      "rewards/margins": 16.96898651123047,
+      "rewards/rejected": -17.355205535888672,
+      "step": 2100
+    },
+    {
+      "epoch": 0.5269401261160017,
+      "grad_norm": 0.0242919921875,
+      "learning_rate": 2.7014171257496414e-06,
+      "logits/chosen": 0.5697668790817261,
+      "logits/rejected": 1.7131723165512085,
+      "logps/chosen": -84.2120132446289,
+      "logps/rejected": -1700.6185302734375,
+      "loss": 0.0016,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3910997211933136,
+      "rewards/margins": 15.752738952636719,
+      "rewards/rejected": -16.14383888244629,
+      "step": 2110
+    },
+    {
+      "epoch": 0.5294374726852719,
+      "grad_norm": 0.052978515625,
+      "learning_rate": 2.6796821564480237e-06,
+      "logits/chosen": 0.601753294467926,
+      "logits/rejected": 1.729688048362732,
+      "logps/chosen": -76.5484390258789,
+      "logps/rejected": -1640.7828369140625,
+      "loss": 0.0017,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.32075661420822144,
+      "rewards/margins": 15.291044235229492,
+      "rewards/rejected": -15.611801147460938,
+      "step": 2120
+    },
+    {
+      "epoch": 0.531934819254542,
+      "grad_norm": 0.0012664794921875,
+      "learning_rate": 2.6579335264317253e-06,
+      "logits/chosen": 0.5816048383712769,
+      "logits/rejected": 1.7906360626220703,
+      "logps/chosen": -85.20054626464844,
+      "logps/rejected": -1883.8203125,
+      "loss": 0.0014,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.40961331129074097,
+      "rewards/margins": 17.597332000732422,
+      "rewards/rejected": -18.006946563720703,
+      "step": 2130
+    },
+    {
+      "epoch": 0.5344321658238121,
+      "grad_norm": 0.00101470947265625,
+      "learning_rate": 2.6361728891861843e-06,
+      "logits/chosen": 0.5752017498016357,
+      "logits/rejected": 1.6957403421401978,
+      "logps/chosen": -86.33828735351562,
+      "logps/rejected": -1814.455322265625,
+      "loss": 0.0015,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4084859788417816,
+      "rewards/margins": 16.83043670654297,
+      "rewards/rejected": -17.238922119140625,
+      "step": 2140
+    },
+    {
+      "epoch": 0.5369295123930824,
+      "grad_norm": 0.009521484375,
+      "learning_rate": 2.614401899109716e-06,
+      "logits/chosen": 0.5796340703964233,
+      "logits/rejected": 1.7896589040756226,
+      "logps/chosen": -78.46866607666016,
+      "logps/rejected": -1804.115234375,
+      "loss": 0.0006,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3526671528816223,
+      "rewards/margins": 16.97123908996582,
+      "rewards/rejected": -17.32390785217285,
+      "step": 2150
+    },
+    {
+      "epoch": 0.5394268589623525,
+      "grad_norm": 0.0263671875,
+      "learning_rate": 2.5926222113877282e-06,
+      "logits/chosen": 0.5532792806625366,
+      "logits/rejected": 1.7575023174285889,
+      "logps/chosen": -86.9544906616211,
+      "logps/rejected": -1865.7669677734375,
+      "loss": 0.001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4337650239467621,
+      "rewards/margins": 17.272930145263672,
+      "rewards/rejected": -17.706693649291992,
+      "step": 2160
+    },
+    {
+      "epoch": 0.5419242055316227,
+      "grad_norm": 0.0419921875,
+      "learning_rate": 2.570835481866889e-06,
+      "logits/chosen": 0.6227487921714783,
+      "logits/rejected": 1.7569319009780884,
+      "logps/chosen": -83.56333923339844,
+      "logps/rejected": -1739.373046875,
+      "loss": 0.0014,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4034454822540283,
+      "rewards/margins": 16.28359031677246,
+      "rewards/rejected": -16.687036514282227,
+      "step": 2170
+    },
+    {
+      "epoch": 0.5444215521008928,
+      "grad_norm": 0.0240478515625,
+      "learning_rate": 2.5490433669292337e-06,
+      "logits/chosen": 0.5318483114242554,
+      "logits/rejected": 1.7802917957305908,
+      "logps/chosen": -83.16242218017578,
+      "logps/rejected": -2065.672119140625,
+      "loss": 0.0012,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3906503915786743,
+      "rewards/margins": 19.537628173828125,
+      "rewards/rejected": -19.92827796936035,
+      "step": 2180
+    },
+    {
+      "epoch": 0.546918898670163,
+      "grad_norm": 0.00081634521484375,
+      "learning_rate": 2.527247523366232e-06,
+      "logits/chosen": 0.55711829662323,
+      "logits/rejected": 1.7834659814834595,
+      "logps/chosen": -89.28174591064453,
+      "logps/rejected": -1952.4468994140625,
+      "loss": 0.0006,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.45531249046325684,
+      "rewards/margins": 18.357410430908203,
+      "rewards/rejected": -18.812725067138672,
+      "step": 2190
+    },
+    {
+      "epoch": 0.5494162452394331,
+      "grad_norm": 0.0498046875,
+      "learning_rate": 2.5054496082528336e-06,
+      "logits/chosen": 0.6078628897666931,
+      "logits/rejected": 1.8645546436309814,
+      "logps/chosen": -78.72160339355469,
+      "logps/rejected": -1901.3984375,
+      "loss": 0.0008,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3558036983013153,
+      "rewards/margins": 17.979564666748047,
+      "rewards/rejected": -18.335365295410156,
+      "step": 2200
+    },
+    {
+      "epoch": 0.5519135918087033,
+      "grad_norm": 0.030029296875,
+      "learning_rate": 2.483651278821481e-06,
+      "logits/chosen": 0.6357477903366089,
+      "logits/rejected": 1.8168609142303467,
+      "logps/chosen": -86.10234069824219,
+      "logps/rejected": -1748.644775390625,
+      "loss": 0.001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.43243059515953064,
+      "rewards/margins": 16.351675033569336,
+      "rewards/rejected": -16.78410530090332,
+      "step": 2210
+    },
+    {
+      "epoch": 0.5544109383779734,
+      "grad_norm": 0.01177978515625,
+      "learning_rate": 2.4618541923361166e-06,
+      "logits/chosen": 0.6292850971221924,
+      "logits/rejected": 1.7243268489837646,
+      "logps/chosen": -83.60933685302734,
+      "logps/rejected": -1522.750244140625,
+      "loss": 0.0015,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.39073851704597473,
+      "rewards/margins": 14.038189888000488,
+      "rewards/rejected": -14.428926467895508,
+      "step": 2220
+    },
+    {
+      "epoch": 0.5569082849472435,
+      "grad_norm": 0.00070953369140625,
+      "learning_rate": 2.4400600059661836e-06,
+      "logits/chosen": 0.5282065868377686,
+      "logits/rejected": 1.8849893808364868,
+      "logps/chosen": -86.94523620605469,
+      "logps/rejected": -2129.426025390625,
+      "loss": 0.0015,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.42179185152053833,
+      "rewards/margins": 20.111438751220703,
+      "rewards/rejected": -20.53322982788086,
+      "step": 2230
+    },
+    {
+      "epoch": 0.5594056315165137,
+      "grad_norm": 0.008544921875,
+      "learning_rate": 2.41827037666064e-06,
+      "logits/chosen": 0.6786268353462219,
+      "logits/rejected": 1.8249973058700562,
+      "logps/chosen": -76.79341888427734,
+      "logps/rejected": -1632.072265625,
+      "loss": 0.0015,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.33309558033943176,
+      "rewards/margins": 15.309873580932617,
+      "rewards/rejected": -15.642970085144043,
+      "step": 2240
+    },
+    {
+      "epoch": 0.5619029780857838,
+      "grad_norm": 0.018798828125,
+      "learning_rate": 2.396486961021983e-06,
+      "logits/chosen": 0.617296576499939,
+      "logits/rejected": 1.860708236694336,
+      "logps/chosen": -89.55140686035156,
+      "logps/rejected": -1905.1920166015625,
+      "loss": 0.0022,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4716108441352844,
+      "rewards/margins": 17.87620735168457,
+      "rewards/rejected": -18.347820281982422,
+      "step": 2250
+    },
+    {
+      "epoch": 0.564400324655054,
+      "grad_norm": 0.00726318359375,
+      "learning_rate": 2.3747114151802993e-06,
+      "logits/chosen": 0.6001085638999939,
+      "logits/rejected": 1.8411632776260376,
+      "logps/chosen": -79.05329895019531,
+      "logps/rejected": -1769.6693115234375,
+      "loss": 0.0014,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3584744334220886,
+      "rewards/margins": 16.644119262695312,
+      "rewards/rejected": -17.002593994140625,
+      "step": 2260
+    },
+    {
+      "epoch": 0.5668976712243241,
+      "grad_norm": 0.038330078125,
+      "learning_rate": 2.352945394667363e-06,
+      "logits/chosen": 0.5482415556907654,
+      "logits/rejected": 1.7739299535751343,
+      "logps/chosen": -88.25926971435547,
+      "logps/rejected": -2113.56884765625,
+      "loss": 0.0008,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.43419378995895386,
+      "rewards/margins": 19.882293701171875,
+      "rewards/rejected": -20.31648826599121,
+      "step": 2270
+    },
+    {
+      "epoch": 0.5693950177935944,
+      "grad_norm": 0.06494140625,
+      "learning_rate": 2.3311905542907627e-06,
+      "logits/chosen": 0.6261372566223145,
+      "logits/rejected": 1.787941336631775,
+      "logps/chosen": -80.4935302734375,
+      "logps/rejected": -1684.802001953125,
+      "loss": 0.002,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.378140389919281,
+      "rewards/margins": 15.795267105102539,
+      "rewards/rejected": -16.17340660095215,
+      "step": 2280
+    },
+    {
+      "epoch": 0.5718923643628645,
+      "grad_norm": 0.033935546875,
+      "learning_rate": 2.30944854800809e-06,
+      "logits/chosen": 0.6286464929580688,
+      "logits/rejected": 1.8051517009735107,
+      "logps/chosen": -80.61420440673828,
+      "logps/rejected": -1804.312744140625,
+      "loss": 0.0019,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.37235134840011597,
+      "rewards/margins": 16.959814071655273,
+      "rewards/rejected": -17.332164764404297,
+      "step": 2290
+    },
+    {
+      "epoch": 0.5743897109321346,
+      "grad_norm": 0.0019683837890625,
+      "learning_rate": 2.287721028801204e-06,
+      "logits/chosen": 0.5823894739151001,
+      "logits/rejected": 1.7553184032440186,
+      "logps/chosen": -89.6335678100586,
+      "logps/rejected": -1704.480712890625,
+      "loss": 0.001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.45537322759628296,
+      "rewards/margins": 15.833767890930176,
+      "rewards/rejected": -16.289142608642578,
+      "step": 2300
+    },
+    {
+      "epoch": 0.5768870575014048,
+      "grad_norm": 0.021728515625,
+      "learning_rate": 2.26600964855055e-06,
+      "logits/chosen": 0.6238933205604553,
+      "logits/rejected": 1.7979097366333008,
+      "logps/chosen": -79.57666778564453,
+      "logps/rejected": -1692.2340087890625,
+      "loss": 0.0009,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3568623960018158,
+      "rewards/margins": 15.8856201171875,
+      "rewards/rejected": -16.242483139038086,
+      "step": 2310
+    },
+    {
+      "epoch": 0.5793844040706749,
+      "grad_norm": 0.007476806640625,
+      "learning_rate": 2.244316057909573e-06,
+      "logits/chosen": 0.6190879344940186,
+      "logits/rejected": 1.7797908782958984,
+      "logps/chosen": -86.66450500488281,
+      "logps/rejected": -1799.299560546875,
+      "loss": 0.0022,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4307888448238373,
+      "rewards/margins": 16.876020431518555,
+      "rewards/rejected": -17.30681037902832,
+      "step": 2320
+    },
+    {
+      "epoch": 0.5818817506399451,
+      "grad_norm": 0.005279541015625,
+      "learning_rate": 2.2226419061792282e-06,
+      "logits/chosen": 0.5849915742874146,
+      "logits/rejected": 1.8162180185317993,
+      "logps/chosen": -85.55912017822266,
+      "logps/rejected": -1866.6416015625,
+      "loss": 0.0008,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.42593854665756226,
+      "rewards/margins": 17.541982650756836,
+      "rewards/rejected": -17.96792221069336,
+      "step": 2330
+    },
+    {
+      "epoch": 0.5843790972092152,
+      "grad_norm": 0.002044677734375,
+      "learning_rate": 2.200988841182589e-06,
+      "logits/chosen": 0.6237704157829285,
+      "logits/rejected": 1.8964016437530518,
+      "logps/chosen": -95.87744140625,
+      "logps/rejected": -2077.869140625,
+      "loss": 0.0011,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5327093005180359,
+      "rewards/margins": 19.543682098388672,
+      "rewards/rejected": -20.076391220092773,
+      "step": 2340
+    },
+    {
+      "epoch": 0.5868764437784854,
+      "grad_norm": 0.00469970703125,
+      "learning_rate": 2.179358509139559e-06,
+      "logits/chosen": 0.6188510060310364,
+      "logits/rejected": 1.7551387548446655,
+      "logps/chosen": -82.06452941894531,
+      "logps/rejected": -1564.6396484375,
+      "loss": 0.0014,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.359652578830719,
+      "rewards/margins": 14.480381965637207,
+      "rewards/rejected": -14.840034484863281,
+      "step": 2350
+    },
+    {
+      "epoch": 0.5893737903477555,
+      "grad_norm": 0.05322265625,
+      "learning_rate": 2.1577525545417254e-06,
+      "logits/chosen": 0.642662525177002,
+      "logits/rejected": 1.8487951755523682,
+      "logps/chosen": -85.49614715576172,
+      "logps/rejected": -1861.0296630859375,
+      "loss": 0.0022,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.41231250762939453,
+      "rewards/margins": 17.494285583496094,
+      "rewards/rejected": -17.906597137451172,
+      "step": 2360
+    },
+    {
+      "epoch": 0.5918711369170256,
+      "grad_norm": 0.059326171875,
+      "learning_rate": 2.1361726200273293e-06,
+      "logits/chosen": 0.6013755202293396,
+      "logits/rejected": 1.8614768981933594,
+      "logps/chosen": -82.80476379394531,
+      "logps/rejected": -1880.697021484375,
+      "loss": 0.0013,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.38074877858161926,
+      "rewards/margins": 17.66275978088379,
+      "rewards/rejected": -18.043506622314453,
+      "step": 2370
+    },
+    {
+      "epoch": 0.5943684834862958,
+      "grad_norm": 0.04638671875,
+      "learning_rate": 2.1146203462563773e-06,
+      "logits/chosen": 0.6672108769416809,
+      "logits/rejected": 1.8760160207748413,
+      "logps/chosen": -85.3284912109375,
+      "logps/rejected": -1663.7691650390625,
+      "loss": 0.0014,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.42937812209129333,
+      "rewards/margins": 15.528106689453125,
+      "rewards/rejected": -15.957483291625977,
+      "step": 2380
+    },
+    {
+      "epoch": 0.5968658300555659,
+      "grad_norm": 0.037109375,
+      "learning_rate": 2.0930973717859117e-06,
+      "logits/chosen": 0.5693127512931824,
+      "logits/rejected": 1.8059355020523071,
+      "logps/chosen": -86.84693145751953,
+      "logps/rejected": -1841.8323974609375,
+      "loss": 0.0018,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4406910836696625,
+      "rewards/margins": 17.286680221557617,
+      "rewards/rejected": -17.727371215820312,
+      "step": 2390
+    },
+    {
+      "epoch": 0.5993631766248361,
+      "grad_norm": 4.8160552978515625e-05,
+      "learning_rate": 2.0716053329454337e-06,
+      "logits/chosen": 0.633589506149292,
+      "logits/rejected": 1.8425014019012451,
+      "logps/chosen": -84.12596130371094,
+      "logps/rejected": -1987.7154541015625,
+      "loss": 0.0029,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.40977373719215393,
+      "rewards/margins": 18.74795913696289,
+      "rewards/rejected": -19.157733917236328,
+      "step": 2400
+    },
+    {
+      "epoch": 0.6018605231941062,
+      "grad_norm": 0.020751953125,
+      "learning_rate": 2.0501458637124963e-06,
+      "logits/chosen": 0.6005128026008606,
+      "logits/rejected": 1.9649635553359985,
+      "logps/chosen": -89.92936706542969,
+      "logps/rejected": -2303.25341796875,
+      "loss": 0.0008,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.46929341554641724,
+      "rewards/margins": 21.84885597229004,
+      "rewards/rejected": -22.31814956665039,
+      "step": 2410
+    },
+    {
+      "epoch": 0.6043578697633765,
+      "grad_norm": 0.0059814453125,
+      "learning_rate": 2.0287205955884812e-06,
+      "logits/chosen": 0.5659859776496887,
+      "logits/rejected": 1.7156604528427124,
+      "logps/chosen": -82.71956634521484,
+      "logps/rejected": -1716.356689453125,
+      "loss": 0.0021,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.37312421202659607,
+      "rewards/margins": 15.841397285461426,
+      "rewards/rejected": -16.214521408081055,
+      "step": 2420
+    },
+    {
+      "epoch": 0.6068552163326466,
+      "grad_norm": 0.031494140625,
+      "learning_rate": 2.0073311574745583e-06,
+      "logits/chosen": 0.615592896938324,
+      "logits/rejected": 1.8998111486434937,
+      "logps/chosen": -83.0928726196289,
+      "logps/rejected": -2058.5458984375,
+      "loss": 0.0007,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3958067297935486,
+      "rewards/margins": 19.46796226501465,
+      "rewards/rejected": -19.863767623901367,
+      "step": 2430
+    },
+    {
+      "epoch": 0.6093525629019167,
+      "grad_norm": 0.0108642578125,
+      "learning_rate": 1.9859791755478453e-06,
+      "logits/chosen": 0.612500786781311,
+      "logits/rejected": 1.7525627613067627,
+      "logps/chosen": -79.78617858886719,
+      "logps/rejected": -1609.921142578125,
+      "loss": 0.001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3665863871574402,
+      "rewards/margins": 15.052395820617676,
+      "rewards/rejected": -15.418981552124023,
+      "step": 2440
+    },
+    {
+      "epoch": 0.6118499094711869,
+      "grad_norm": 0.05419921875,
+      "learning_rate": 1.9646662731377737e-06,
+      "logits/chosen": 0.6692113876342773,
+      "logits/rejected": 1.816349983215332,
+      "logps/chosen": -84.40538787841797,
+      "logps/rejected": -1675.842041015625,
+      "loss": 0.002,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.40326443314552307,
+      "rewards/margins": 15.622156143188477,
+      "rewards/rejected": -16.025419235229492,
+      "step": 2450
+    },
+    {
+      "epoch": 0.614347256040457,
+      "grad_norm": 0.00130462646484375,
+      "learning_rate": 1.9433940706026743e-06,
+      "logits/chosen": 0.5840574502944946,
+      "logits/rejected": 1.8281028270721436,
+      "logps/chosen": -86.7694320678711,
+      "logps/rejected": -2069.530517578125,
+      "loss": 0.0016,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4303639531135559,
+      "rewards/margins": 19.5331974029541,
+      "rewards/rejected": -19.963563919067383,
+      "step": 2460
+    },
+    {
+      "epoch": 0.6168446026097272,
+      "grad_norm": 0.00012493133544921875,
+      "learning_rate": 1.9221641852065807e-06,
+      "logits/chosen": 0.6755739450454712,
+      "logits/rejected": 1.845654845237732,
+      "logps/chosen": -88.67765045166016,
+      "logps/rejected": -1714.5185546875,
+      "loss": 0.0021,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.46009987592697144,
+      "rewards/margins": 15.997251510620117,
+      "rewards/rejected": -16.457351684570312,
+      "step": 2470
+    },
+    {
+      "epoch": 0.6193419491789973,
+      "grad_norm": 0.039794921875,
+      "learning_rate": 1.9009782309962805e-06,
+      "logits/chosen": 0.5677890181541443,
+      "logits/rejected": 1.8127906322479248,
+      "logps/chosen": -76.20381164550781,
+      "logps/rejected": -1766.592529296875,
+      "loss": 0.0008,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3160540759563446,
+      "rewards/margins": 16.549949645996094,
+      "rewards/rejected": -16.866003036499023,
+      "step": 2480
+    },
+    {
+      "epoch": 0.6218392957482675,
+      "grad_norm": 0.037841796875,
+      "learning_rate": 1.8798378186785979e-06,
+      "logits/chosen": 0.6165963411331177,
+      "logits/rejected": 1.7844451665878296,
+      "logps/chosen": -80.53484344482422,
+      "logps/rejected": -1799.0015869140625,
+      "loss": 0.0021,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.37462836503982544,
+      "rewards/margins": 16.907718658447266,
+      "rewards/rejected": -17.282346725463867,
+      "step": 2490
+    },
+    {
+      "epoch": 0.6243366423175376,
+      "grad_norm": 0.01007080078125,
+      "learning_rate": 1.8587445554979404e-06,
+      "logits/chosen": 0.6141692399978638,
+      "logits/rejected": 1.8811423778533936,
+      "logps/chosen": -87.24476623535156,
+      "logps/rejected": -2009.0035400390625,
+      "loss": 0.0019,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4570063054561615,
+      "rewards/margins": 18.914798736572266,
+      "rewards/rejected": -19.371807098388672,
+      "step": 2500
+    },
+    {
+      "epoch": 0.6268339888868077,
+      "grad_norm": 0.00011968612670898438,
+      "learning_rate": 1.8377000451141013e-06,
+      "logits/chosen": 0.6391327977180481,
+      "logits/rejected": 1.9281005859375,
+      "logps/chosen": -86.39270782470703,
+      "logps/rejected": -1954.652099609375,
+      "loss": 0.0007,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4386569857597351,
+      "rewards/margins": 18.369953155517578,
+      "rewards/rejected": -18.808609008789062,
+      "step": 2510
+    },
+    {
+      "epoch": 0.6293313354560779,
+      "grad_norm": 0.0277099609375,
+      "learning_rate": 1.8167058874803405e-06,
+      "logits/chosen": 0.5556064248085022,
+      "logits/rejected": 1.718269944190979,
+      "logps/chosen": -86.69864654541016,
+      "logps/rejected": -1939.4332275390625,
+      "loss": 0.0017,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4233566224575043,
+      "rewards/margins": 18.154285430908203,
+      "rewards/rejected": -18.5776424407959,
+      "step": 2520
+    },
+    {
+      "epoch": 0.631828682025348,
+      "grad_norm": 0.0771484375,
+      "learning_rate": 1.7957636787217451e-06,
+      "logits/chosen": 0.5710119009017944,
+      "logits/rejected": 1.7915983200073242,
+      "logps/chosen": -79.21806335449219,
+      "logps/rejected": -1942.8629150390625,
+      "loss": 0.0014,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3619151711463928,
+      "rewards/margins": 18.352802276611328,
+      "rewards/rejected": -18.71471405029297,
+      "step": 2530
+    },
+    {
+      "epoch": 0.6343260285946182,
+      "grad_norm": 0.0155029296875,
+      "learning_rate": 1.7748750110138768e-06,
+      "logits/chosen": 0.5197774171829224,
+      "logits/rejected": 1.737969160079956,
+      "logps/chosen": -88.46708679199219,
+      "logps/rejected": -2104.29296875,
+      "loss": 0.0011,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.42627063393592834,
+      "rewards/margins": 19.79252052307129,
+      "rewards/rejected": -20.21879005432129,
+      "step": 2540
+    },
+    {
+      "epoch": 0.6368233751638883,
+      "grad_norm": 0.021484375,
+      "learning_rate": 1.7540414724617282e-06,
+      "logits/chosen": 0.5759893655776978,
+      "logits/rejected": 1.7303228378295898,
+      "logps/chosen": -76.38877868652344,
+      "logps/rejected": -1828.7796630859375,
+      "loss": 0.0012,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.32031676173210144,
+      "rewards/margins": 17.18129539489746,
+      "rewards/rejected": -17.50161361694336,
+      "step": 2550
+    },
+    {
+      "epoch": 0.6393207217331586,
+      "grad_norm": 0.0196533203125,
+      "learning_rate": 1.7332646469789827e-06,
+      "logits/chosen": 0.6225037574768066,
+      "logits/rejected": 1.779985785484314,
+      "logps/chosen": -85.09439086914062,
+      "logps/rejected": -1516.5433349609375,
+      "loss": 0.0011,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4056459069252014,
+      "rewards/margins": 14.090934753417969,
+      "rewards/rejected": -14.496580123901367,
+      "step": 2560
+    },
+    {
+      "epoch": 0.6418180683024287,
+      "grad_norm": 0.036376953125,
+      "learning_rate": 1.7125461141675881e-06,
+      "logits/chosen": 0.643700122833252,
+      "logits/rejected": 1.8465179204940796,
+      "logps/chosen": -80.20843505859375,
+      "logps/rejected": -1810.455078125,
+      "loss": 0.0015,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3701106905937195,
+      "rewards/margins": 16.961139678955078,
+      "rewards/rejected": -17.331249237060547,
+      "step": 2570
+    },
+    {
+      "epoch": 0.6443154148716989,
+      "grad_norm": 0.0289306640625,
+      "learning_rate": 1.6918874491976744e-06,
+      "logits/chosen": 0.5704053640365601,
+      "logits/rejected": 1.6976230144500732,
+      "logps/chosen": -84.80411529541016,
+      "logps/rejected": -1765.397216796875,
+      "loss": 0.0008,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4006038308143616,
+      "rewards/margins": 16.490291595458984,
+      "rewards/rejected": -16.89089584350586,
+      "step": 2580
+    },
+    {
+      "epoch": 0.646812761440969,
+      "grad_norm": 0.000751495361328125,
+      "learning_rate": 1.6712902226877917e-06,
+      "logits/chosen": 0.6289549469947815,
+      "logits/rejected": 1.850502610206604,
+      "logps/chosen": -88.18513488769531,
+      "logps/rejected": -1988.833984375,
+      "loss": 0.0012,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4497924745082855,
+      "rewards/margins": 18.730804443359375,
+      "rewards/rejected": -19.180593490600586,
+      "step": 2590
+    },
+    {
+      "epoch": 0.6493101080102391,
+      "grad_norm": 0.0012664794921875,
+      "learning_rate": 1.6507560005854977e-06,
+      "logits/chosen": 0.5509642362594604,
+      "logits/rejected": 1.7071406841278076,
+      "logps/chosen": -83.9954833984375,
+      "logps/rejected": -1830.5101318359375,
+      "loss": 0.0014,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3968578577041626,
+      "rewards/margins": 17.073257446289062,
+      "rewards/rejected": -17.470115661621094,
+      "step": 2600
+    },
+    {
+      "epoch": 0.6518074545795093,
+      "grad_norm": 0.008544921875,
+      "learning_rate": 1.6302863440483121e-06,
+      "logits/chosen": 0.5004338026046753,
+      "logits/rejected": 1.7150554656982422,
+      "logps/chosen": -82.43248748779297,
+      "logps/rejected": -1880.423828125,
+      "loss": 0.0014,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.36883944272994995,
+      "rewards/margins": 17.624969482421875,
+      "rewards/rejected": -17.99380874633789,
+      "step": 2610
+    },
+    {
+      "epoch": 0.6543048011487794,
+      "grad_norm": 0.0341796875,
+      "learning_rate": 1.6098828093250203e-06,
+      "logits/chosen": 0.5160735845565796,
+      "logits/rejected": 1.7385032176971436,
+      "logps/chosen": -79.65149688720703,
+      "logps/rejected": -2075.0966796875,
+      "loss": 0.0017,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3529607653617859,
+      "rewards/margins": 19.472675323486328,
+      "rewards/rejected": -19.825634002685547,
+      "step": 2620
+    },
+    {
+      "epoch": 0.6568021477180496,
+      "grad_norm": 0.027099609375,
+      "learning_rate": 1.5895469476373545e-06,
+      "logits/chosen": 0.5671316385269165,
+      "logits/rejected": 1.722346305847168,
+      "logps/chosen": -81.55863189697266,
+      "logps/rejected": -1681.589111328125,
+      "loss": 0.0013,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3755113184452057,
+      "rewards/margins": 15.65942096710205,
+      "rewards/rejected": -16.03493309020996,
+      "step": 2630
+    },
+    {
+      "epoch": 0.6592994942873197,
+      "grad_norm": 0.091796875,
+      "learning_rate": 1.5692803050620642e-06,
+      "logits/chosen": 0.6067586541175842,
+      "logits/rejected": 1.7199970483779907,
+      "logps/chosen": -83.18370056152344,
+      "logps/rejected": -1680.830322265625,
+      "loss": 0.0012,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.38230761885643005,
+      "rewards/margins": 15.612162590026855,
+      "rewards/rejected": -15.994470596313477,
+      "step": 2640
+    },
+    {
+      "epoch": 0.6617968408565899,
+      "grad_norm": 0.007720947265625,
+      "learning_rate": 1.5490844224133717e-06,
+      "logits/chosen": 0.6019744873046875,
+      "logits/rejected": 1.8558555841445923,
+      "logps/chosen": -80.30780792236328,
+      "logps/rejected": -1929.556640625,
+      "loss": 0.0007,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.37255150079727173,
+      "rewards/margins": 18.213796615600586,
+      "rewards/rejected": -18.586347579956055,
+      "step": 2650
+    },
+    {
+      "epoch": 0.66429418742586,
+      "grad_norm": 0.050537109375,
+      "learning_rate": 1.528960835125822e-06,
+      "logits/chosen": 0.6771095991134644,
+      "logits/rejected": 1.8372013568878174,
+      "logps/chosen": -80.95626068115234,
+      "logps/rejected": -1653.2200927734375,
+      "loss": 0.0011,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3823738992214203,
+      "rewards/margins": 15.478759765625,
+      "rewards/rejected": -15.86113452911377,
+      "step": 2660
+    },
+    {
+      "epoch": 0.6667915339951301,
+      "grad_norm": 0.0147705078125,
+      "learning_rate": 1.5089110731375568e-06,
+      "logits/chosen": 0.581728994846344,
+      "logits/rejected": 1.7974720001220703,
+      "logps/chosen": -79.06498718261719,
+      "logps/rejected": -1824.8765869140625,
+      "loss": 0.0009,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.35732579231262207,
+      "rewards/margins": 17.178985595703125,
+      "rewards/rejected": -17.53631019592285,
+      "step": 2670
+    },
+    {
+      "epoch": 0.6692888805644003,
+      "grad_norm": 0.032470703125,
+      "learning_rate": 1.4889366607739925e-06,
+      "logits/chosen": 0.6092284917831421,
+      "logits/rejected": 1.6554501056671143,
+      "logps/chosen": -78.19693756103516,
+      "logps/rejected": -1454.85107421875,
+      "loss": 0.0015,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3566651940345764,
+      "rewards/margins": 13.518289566040039,
+      "rewards/rejected": -13.874954223632812,
+      "step": 2680
+    },
+    {
+      "epoch": 0.6717862271336704,
+      "grad_norm": 0.017578125,
+      "learning_rate": 1.4690391166319307e-06,
+      "logits/chosen": 0.5935393571853638,
+      "logits/rejected": 1.761783242225647,
+      "logps/chosen": -85.10191345214844,
+      "logps/rejected": -1834.072021484375,
+      "loss": 0.0012,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4155469536781311,
+      "rewards/margins": 17.14066505432129,
+      "rewards/rejected": -17.556209564208984,
+      "step": 2690
+    },
+    {
+      "epoch": 0.6742835737029407,
+      "grad_norm": 0.036376953125,
+      "learning_rate": 1.4492199534641055e-06,
+      "logits/chosen": 0.6022766828536987,
+      "logits/rejected": 1.8546326160430908,
+      "logps/chosen": -84.67176055908203,
+      "logps/rejected": -1870.0103759765625,
+      "loss": 0.0013,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.41111135482788086,
+      "rewards/margins": 17.593767166137695,
+      "rewards/rejected": -18.0048770904541,
+      "step": 2700
+    },
+    {
+      "epoch": 0.6767809202722108,
+      "grad_norm": 0.001617431640625,
+      "learning_rate": 1.429480678064174e-06,
+      "logits/chosen": 0.4885890483856201,
+      "logits/rejected": 1.7658706903457642,
+      "logps/chosen": -85.19550323486328,
+      "logps/rejected": -2298.07373046875,
+      "loss": 0.0005,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4054490923881531,
+      "rewards/margins": 21.81135368347168,
+      "rewards/rejected": -22.2168025970459,
+      "step": 2710
+    },
+    {
+      "epoch": 0.679278266841481,
+      "grad_norm": 0.021484375,
+      "learning_rate": 1.4098227911521523e-06,
+      "logits/chosen": 0.6109157204627991,
+      "logits/rejected": 1.8104369640350342,
+      "logps/chosen": -92.25324249267578,
+      "logps/rejected": -1919.9534912109375,
+      "loss": 0.001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4862859845161438,
+      "rewards/margins": 18.005590438842773,
+      "rewards/rejected": -18.49187660217285,
+      "step": 2720
+    },
+    {
+      "epoch": 0.6817756134107511,
+      "grad_norm": 0.0732421875,
+      "learning_rate": 1.3902477872603295e-06,
+      "logits/chosen": 0.6768110990524292,
+      "logits/rejected": 1.7166074514389038,
+      "logps/chosen": -80.58265686035156,
+      "logps/rejected": -1554.531982421875,
+      "loss": 0.0024,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.36700159311294556,
+      "rewards/margins": 14.375741958618164,
+      "rewards/rejected": -14.742744445800781,
+      "step": 2730
+    },
+    {
+      "epoch": 0.6842729599800212,
+      "grad_norm": 0.0478515625,
+      "learning_rate": 1.370757154619638e-06,
+      "logits/chosen": 0.5914765000343323,
+      "logits/rejected": 1.823325753211975,
+      "logps/chosen": -86.78914642333984,
+      "logps/rejected": -1906.5355224609375,
+      "loss": 0.0009,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.42601895332336426,
+      "rewards/margins": 17.833972930908203,
+      "rewards/rejected": -18.259990692138672,
+      "step": 2740
+    },
+    {
+      "epoch": 0.6867703065492914,
+      "grad_norm": 0.005523681640625,
+      "learning_rate": 1.3513523750465049e-06,
+      "logits/chosen": 0.5616365075111389,
+      "logits/rejected": 1.7267478704452515,
+      "logps/chosen": -81.25291442871094,
+      "logps/rejected": -1686.3486328125,
+      "loss": 0.0013,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.36524245142936707,
+      "rewards/margins": 15.707315444946289,
+      "rewards/rejected": -16.07255744934082,
+      "step": 2750
+    },
+    {
+      "epoch": 0.6892676531185615,
+      "grad_norm": 0.00193023681640625,
+      "learning_rate": 1.332034923830199e-06,
+      "logits/chosen": 0.5695074200630188,
+      "logits/rejected": 1.8329308032989502,
+      "logps/chosen": -82.82709503173828,
+      "logps/rejected": -1805.4775390625,
+      "loss": 0.0006,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.39727023243904114,
+      "rewards/margins": 16.94883918762207,
+      "rewards/rejected": -17.346107482910156,
+      "step": 2760
+    },
+    {
+      "epoch": 0.6917649996878317,
+      "grad_norm": 0.040771484375,
+      "learning_rate": 1.31280626962067e-06,
+      "logits/chosen": 0.6029590368270874,
+      "logits/rejected": 1.6939897537231445,
+      "logps/chosen": -86.7250747680664,
+      "logps/rejected": -1587.8193359375,
+      "loss": 0.0013,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.42630377411842346,
+      "rewards/margins": 14.680148124694824,
+      "rewards/rejected": -15.106452941894531,
+      "step": 2770
+    },
+    {
+      "epoch": 0.6942623462571018,
+      "grad_norm": 0.0003566741943359375,
+      "learning_rate": 1.2936678743168813e-06,
+      "logits/chosen": 0.5795254707336426,
+      "logits/rejected": 1.7682584524154663,
+      "logps/chosen": -83.47227478027344,
+      "logps/rejected": -1894.138916015625,
+      "loss": 0.0021,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.39181336760520935,
+      "rewards/margins": 17.82851791381836,
+      "rewards/rejected": -18.22032928466797,
+      "step": 2780
+    },
+    {
+      "epoch": 0.696759692826372,
+      "grad_norm": 0.01373291015625,
+      "learning_rate": 1.2746211929556777e-06,
+      "logits/chosen": 0.5124091506004333,
+      "logits/rejected": 2.0397300720214844,
+      "logps/chosen": -85.71356201171875,
+      "logps/rejected": -2490.38232421875,
+      "loss": 0.0004,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.42116060853004456,
+      "rewards/margins": 23.745744705200195,
+      "rewards/rejected": -24.16690444946289,
+      "step": 2790
+    },
+    {
+      "epoch": 0.6992570393956421,
+      "grad_norm": 1.30385160446167e-08,
+      "learning_rate": 1.2556676736011558e-06,
+      "logits/chosen": 0.6134932637214661,
+      "logits/rejected": 1.816425085067749,
+      "logps/chosen": -85.68560791015625,
+      "logps/rejected": -1998.1363525390625,
+      "loss": 0.0007,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.43286705017089844,
+      "rewards/margins": 18.802690505981445,
+      "rewards/rejected": -19.235559463500977,
+      "step": 2800
+    },
+    {
+      "epoch": 0.7017543859649122,
+      "grad_norm": 0.0089111328125,
+      "learning_rate": 1.2368087572345772e-06,
+      "logits/chosen": 0.6667296886444092,
+      "logits/rejected": 1.7410768270492554,
+      "logps/chosen": -84.29058837890625,
+      "logps/rejected": -1482.312255859375,
+      "loss": 0.0019,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4156969487667084,
+      "rewards/margins": 13.738250732421875,
+      "rewards/rejected": -14.153947830200195,
+      "step": 2810
+    },
+    {
+      "epoch": 0.7042517325341824,
+      "grad_norm": 0.0927734375,
+      "learning_rate": 1.2180458776448067e-06,
+      "logits/chosen": 0.5982272028923035,
+      "logits/rejected": 1.7856439352035522,
+      "logps/chosen": -89.98011016845703,
+      "logps/rejected": -1943.0396728515625,
+      "loss": 0.0022,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4640469551086426,
+      "rewards/margins": 18.235332489013672,
+      "rewards/rejected": -18.699377059936523,
+      "step": 2820
+    },
+    {
+      "epoch": 0.7067490791034525,
+      "grad_norm": 0.02392578125,
+      "learning_rate": 1.1993804613193158e-06,
+      "logits/chosen": 0.6234604120254517,
+      "logits/rejected": 1.765428900718689,
+      "logps/chosen": -87.09599304199219,
+      "logps/rejected": -1579.3837890625,
+      "loss": 0.0012,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.427617609500885,
+      "rewards/margins": 14.572957038879395,
+      "rewards/rejected": -15.000572204589844,
+      "step": 2830
+    },
+    {
+      "epoch": 0.7092464256727228,
+      "grad_norm": 3.7401914596557617e-06,
+      "learning_rate": 1.1808139273357232e-06,
+      "logits/chosen": 0.5544342398643494,
+      "logits/rejected": 1.7727603912353516,
+      "logps/chosen": -83.8676528930664,
+      "logps/rejected": -1906.515380859375,
+      "loss": 0.0018,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3836463987827301,
+      "rewards/margins": 17.823671340942383,
+      "rewards/rejected": -18.207317352294922,
+      "step": 2840
+    },
+    {
+      "epoch": 0.7117437722419929,
+      "grad_norm": 5.5789947509765625e-05,
+      "learning_rate": 1.1623476872539108e-06,
+      "logits/chosen": 0.5153034925460815,
+      "logits/rejected": 1.8462998867034912,
+      "logps/chosen": -94.67253112792969,
+      "logps/rejected": -2197.599609375,
+      "loss": 0.0012,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5202997326850891,
+      "rewards/margins": 20.727157592773438,
+      "rewards/rejected": -21.247455596923828,
+      "step": 2850
+    },
+    {
+      "epoch": 0.7142411188112631,
+      "grad_norm": 0.042724609375,
+      "learning_rate": 1.1439831450087032e-06,
+      "logits/chosen": 0.580392062664032,
+      "logits/rejected": 1.876275658607483,
+      "logps/chosen": -87.93695068359375,
+      "logps/rejected": -2129.860107421875,
+      "loss": 0.002,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4418310225009918,
+      "rewards/margins": 20.121156692504883,
+      "rewards/rejected": -20.56298828125,
+      "step": 2860
+    },
+    {
+      "epoch": 0.7167384653805332,
+      "grad_norm": 0.0458984375,
+      "learning_rate": 1.1257216968031357e-06,
+      "logits/chosen": 0.6597843170166016,
+      "logits/rejected": 1.8998768329620361,
+      "logps/chosen": -80.04630279541016,
+      "logps/rejected": -1752.5791015625,
+      "loss": 0.0013,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.37010782957077026,
+      "rewards/margins": 16.46237564086914,
+      "rewards/rejected": -16.832483291625977,
+      "step": 2870
+    },
+    {
+      "epoch": 0.7192358119498033,
+      "grad_norm": 0.00104522705078125,
+      "learning_rate": 1.1075647310022974e-06,
+      "logits/chosen": 0.634041965007782,
+      "logits/rejected": 1.8106848001480103,
+      "logps/chosen": -78.93439483642578,
+      "logps/rejected": -1525.8900146484375,
+      "loss": 0.0012,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.36263298988342285,
+      "rewards/margins": 14.232080459594727,
+      "rewards/rejected": -14.594714164733887,
+      "step": 2880
+    },
+    {
+      "epoch": 0.7217331585190735,
+      "grad_norm": 0.002899169921875,
+      "learning_rate": 1.0895136280277863e-06,
+      "logits/chosen": 0.5515082478523254,
+      "logits/rejected": 1.7851063013076782,
+      "logps/chosen": -87.03413391113281,
+      "logps/rejected": -2093.734375,
+      "loss": 0.0012,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4326532781124115,
+      "rewards/margins": 19.687620162963867,
+      "rewards/rejected": -20.120275497436523,
+      "step": 2890
+    },
+    {
+      "epoch": 0.7242305050883436,
+      "grad_norm": 0.060546875,
+      "learning_rate": 1.0715697602527542e-06,
+      "logits/chosen": 0.5289216041564941,
+      "logits/rejected": 1.7743902206420898,
+      "logps/chosen": -85.22486114501953,
+      "logps/rejected": -1992.9351806640625,
+      "loss": 0.0012,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4111485481262207,
+      "rewards/margins": 18.597354888916016,
+      "rewards/rejected": -19.00850486755371,
+      "step": 2900
+    },
+    {
+      "epoch": 0.7267278516576138,
+      "grad_norm": 0.04248046875,
+      "learning_rate": 1.0537344918975708e-06,
+      "logits/chosen": 0.654784083366394,
+      "logits/rejected": 1.7333883047103882,
+      "logps/chosen": -85.55310821533203,
+      "logps/rejected": -1545.492919921875,
+      "loss": 0.001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4073718190193176,
+      "rewards/margins": 14.238784790039062,
+      "rewards/rejected": -14.646156311035156,
+      "step": 2910
+    },
+    {
+      "epoch": 0.7292251982268839,
+      "grad_norm": 1.5079975128173828e-05,
+      "learning_rate": 1.036009178926107e-06,
+      "logits/chosen": 0.570530891418457,
+      "logits/rejected": 1.8232314586639404,
+      "logps/chosen": -87.81031799316406,
+      "logps/rejected": -1891.7252197265625,
+      "loss": 0.0012,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.44701051712036133,
+      "rewards/margins": 17.766794204711914,
+      "rewards/rejected": -18.213804244995117,
+      "step": 2920
+    },
+    {
+      "epoch": 0.7317225447961541,
+      "grad_norm": 0.016357421875,
+      "learning_rate": 1.0183951689426438e-06,
+      "logits/chosen": 0.5162047147750854,
+      "logits/rejected": 1.80562424659729,
+      "logps/chosen": -78.40940856933594,
+      "logps/rejected": -2212.948974609375,
+      "loss": 0.0017,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.33928531408309937,
+      "rewards/margins": 21.02674674987793,
+      "rewards/rejected": -21.366031646728516,
+      "step": 2930
+    },
+    {
+      "epoch": 0.7342198913654242,
+      "grad_norm": 0.01409912109375,
+      "learning_rate": 1.0008938010894156e-06,
+      "logits/chosen": 0.5077947974205017,
+      "logits/rejected": 1.8344638347625732,
+      "logps/chosen": -81.39566802978516,
+      "logps/rejected": -2270.706298828125,
+      "loss": 0.0013,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3777496814727783,
+      "rewards/margins": 21.60778045654297,
+      "rewards/rejected": -21.98552894592285,
+      "step": 2940
+    },
+    {
+      "epoch": 0.7367172379346943,
+      "grad_norm": 0.0198974609375,
+      "learning_rate": 9.83506405944804e-07,
+      "logits/chosen": 0.5673650503158569,
+      "logits/rejected": 1.745111107826233,
+      "logps/chosen": -77.10914611816406,
+      "logps/rejected": -1838.6285400390625,
+      "loss": 0.0015,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.34020406007766724,
+      "rewards/margins": 17.217037200927734,
+      "rewards/rejected": -17.55724334716797,
+      "step": 2950
+    },
+    {
+      "epoch": 0.7392145845039645,
+      "grad_norm": 0.00946044921875,
+      "learning_rate": 9.662343054221743e-07,
+      "logits/chosen": 0.5164293050765991,
+      "logits/rejected": 1.726947546005249,
+      "logps/chosen": -88.59376525878906,
+      "logps/rejected": -2064.08642578125,
+      "loss": 0.001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4376547336578369,
+      "rewards/margins": 19.29999351501465,
+      "rewards/rejected": -19.73764991760254,
+      "step": 2960
+    },
+    {
+      "epoch": 0.7417119310732347,
+      "grad_norm": 0.0595703125,
+      "learning_rate": 9.490788126693754e-07,
+      "logits/chosen": 0.6247397661209106,
+      "logits/rejected": 1.8680105209350586,
+      "logps/chosen": -86.50829315185547,
+      "logps/rejected": -1925.3023681640625,
+      "loss": 0.0013,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4314423203468323,
+      "rewards/margins": 17.977245330810547,
+      "rewards/rejected": -18.408687591552734,
+      "step": 2970
+    },
+    {
+      "epoch": 0.7442092776425049,
+      "grad_norm": 0.0030975341796875,
+      "learning_rate": 9.32041231968904e-07,
+      "logits/chosen": 0.582064151763916,
+      "logits/rejected": 1.8307263851165771,
+      "logps/chosen": -87.89469909667969,
+      "logps/rejected": -2049.106689453125,
+      "loss": 0.0009,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4329482614994049,
+      "rewards/margins": 19.329082489013672,
+      "rewards/rejected": -19.762033462524414,
+      "step": 2980
+    },
+    {
+      "epoch": 0.746706624211775,
+      "grad_norm": 0.328125,
+      "learning_rate": 9.151228586387464e-07,
+      "logits/chosen": 0.6141242384910583,
+      "logits/rejected": 1.747831106185913,
+      "logps/chosen": -80.1594467163086,
+      "logps/rejected": -1789.0576171875,
+      "loss": 0.003,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.36252784729003906,
+      "rewards/margins": 16.812475204467773,
+      "rewards/rejected": -17.175004959106445,
+      "step": 2990
+    },
+    {
+      "epoch": 0.7492039707810452,
+      "grad_norm": 0.06494140625,
+      "learning_rate": 8.983249789338941e-07,
+      "logits/chosen": 0.6428495645523071,
+      "logits/rejected": 1.7919700145721436,
+      "logps/chosen": -82.71188354492188,
+      "logps/rejected": -1678.5035400390625,
+      "loss": 0.0019,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3992861807346344,
+      "rewards/margins": 15.69371509552002,
+      "rewards/rejected": -16.092998504638672,
+      "step": 3000
+    },
+    {
+      "epoch": 0.7492039707810452,
+      "eval_logits/chosen": 0.656849205493927,
+      "eval_logits/rejected": 1.5703133344650269,
+      "eval_logps/chosen": -84.02084350585938,
+      "eval_logps/rejected": -995.490234375,
+      "eval_loss": 0.0028192740865051746,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.39664965867996216,
+      "eval_rewards/margins": 8.947355270385742,
+      "eval_rewards/rejected": -9.34400463104248,
+      "eval_runtime": 0.621,
+      "eval_samples_per_second": 8.052,
+      "eval_steps_per_second": 8.052,
+      "step": 3000
+    },
+    {
+      "epoch": 0.7517013173503153,
+      "grad_norm": 0.0205078125,
+      "learning_rate": 8.816488699485593e-07,
+      "logits/chosen": 0.634880006313324,
+      "logits/rejected": 1.8458068370819092,
+      "logps/chosen": -89.79926300048828,
+      "logps/rejected": -1852.7974853515625,
+      "loss": 0.0014,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4646337628364563,
+      "rewards/margins": 17.353429794311523,
+      "rewards/rejected": -17.81806182861328,
+      "step": 3010
+    },
+    {
+      "epoch": 0.7541986639195855,
+      "grad_norm": 4.553794860839844e-05,
+      "learning_rate": 8.650957995190784e-07,
+      "logits/chosen": 0.5151122212409973,
+      "logits/rejected": 1.7481235265731812,
+      "logps/chosen": -79.8306884765625,
+      "logps/rejected": -2110.71923828125,
+      "loss": 0.0008,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3657647669315338,
+      "rewards/margins": 20.01117706298828,
+      "rewards/rejected": -20.37693977355957,
+      "step": 3020
+    },
+    {
+      "epoch": 0.7566960104888556,
+      "grad_norm": 0.045166015625,
+      "learning_rate": 8.486670261275193e-07,
+      "logits/chosen": 0.6202859878540039,
+      "logits/rejected": 1.8134170770645142,
+      "logps/chosen": -84.73997497558594,
+      "logps/rejected": -1783.783203125,
+      "loss": 0.0009,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.42085719108581543,
+      "rewards/margins": 16.739408493041992,
+      "rewards/rejected": -17.160266876220703,
+      "step": 3030
+    },
+    {
+      "epoch": 0.7591933570581257,
+      "grad_norm": 0.0111083984375,
+      "learning_rate": 8.32363798806011e-07,
+      "logits/chosen": 0.5721080303192139,
+      "logits/rejected": 1.739031195640564,
+      "logps/chosen": -85.13414764404297,
+      "logps/rejected": -1891.197509765625,
+      "loss": 0.0016,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.42727169394493103,
+      "rewards/margins": 17.766677856445312,
+      "rewards/rejected": -18.193950653076172,
+      "step": 3040
+    },
+    {
+      "epoch": 0.7616907036273959,
+      "grad_norm": 0.0208740234375,
+      "learning_rate": 8.161873570417742e-07,
+      "logits/chosen": 0.5966504812240601,
+      "logits/rejected": 1.8666013479232788,
+      "logps/chosen": -89.76878356933594,
+      "logps/rejected": -1943.291015625,
+      "loss": 0.0007,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4609198570251465,
+      "rewards/margins": 18.261484146118164,
+      "rewards/rejected": -18.722402572631836,
+      "step": 3050
+    },
+    {
+      "epoch": 0.764188050196666,
+      "grad_norm": 0.0098876953125,
+      "learning_rate": 8.001389306828897e-07,
+      "logits/chosen": 0.4526674151420593,
+      "logits/rejected": 1.7495372295379639,
+      "logps/chosen": -82.58873748779297,
+      "logps/rejected": -2175.00244140625,
+      "loss": 0.0016,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3694322407245636,
+      "rewards/margins": 20.39395523071289,
+      "rewards/rejected": -20.763385772705078,
+      "step": 3060
+    },
+    {
+      "epoch": 0.7666853967659362,
+      "grad_norm": 1.7508864402770996e-06,
+      "learning_rate": 7.842197398447993e-07,
+      "logits/chosen": 0.5828143358230591,
+      "logits/rejected": 1.8392832279205322,
+      "logps/chosen": -79.72120666503906,
+      "logps/rejected": -1959.4478759765625,
+      "loss": 0.0011,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3589129149913788,
+      "rewards/margins": 18.44549560546875,
+      "rewards/rejected": -18.804407119750977,
+      "step": 3070
+    },
+    {
+      "epoch": 0.7691827433352063,
+      "grad_norm": 0.00104522705078125,
+      "learning_rate": 7.684309948175414e-07,
+      "logits/chosen": 0.5747276544570923,
+      "logits/rejected": 1.7614377737045288,
+      "logps/chosen": -83.3620834350586,
+      "logps/rejected": -1877.7086181640625,
+      "loss": 0.0009,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4084179401397705,
+      "rewards/margins": 17.644577026367188,
+      "rewards/rejected": -18.052997589111328,
+      "step": 3080
+    },
+    {
+      "epoch": 0.7716800899044765,
+      "grad_norm": 0.00160980224609375,
+      "learning_rate": 7.527738959737371e-07,
+      "logits/chosen": 0.536163330078125,
+      "logits/rejected": 1.8368165493011475,
+      "logps/chosen": -81.3559799194336,
+      "logps/rejected": -1827.6383056640625,
+      "loss": 0.001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3685862720012665,
+      "rewards/margins": 17.127788543701172,
+      "rewards/rejected": -17.496374130249023,
+      "step": 3090
+    },
+    {
+      "epoch": 0.7741774364737466,
+      "grad_norm": 0.061279296875,
+      "learning_rate": 7.372496336773269e-07,
+      "logits/chosen": 0.6259430050849915,
+      "logits/rejected": 1.7605386972427368,
+      "logps/chosen": -82.03521728515625,
+      "logps/rejected": -1697.513916015625,
+      "loss": 0.0024,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3883189260959625,
+      "rewards/margins": 15.871994018554688,
+      "rewards/rejected": -16.260311126708984,
+      "step": 3100
+    },
+    {
+      "epoch": 0.7766747830430168,
+      "grad_norm": 0.0439453125,
+      "learning_rate": 7.218593881930744e-07,
+      "logits/chosen": 0.6127210259437561,
+      "logits/rejected": 1.7632982730865479,
+      "logps/chosen": -77.52657318115234,
+      "logps/rejected": -1801.5595703125,
+      "loss": 0.0017,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3411308526992798,
+      "rewards/margins": 16.98233985900879,
+      "rewards/rejected": -17.323471069335938,
+      "step": 3110
+    },
+    {
+      "epoch": 0.779172129612287,
+      "grad_norm": 0.017333984375,
+      "learning_rate": 7.066043295968342e-07,
+      "logits/chosen": 0.5858328938484192,
+      "logits/rejected": 1.7214057445526123,
+      "logps/chosen": -82.28968048095703,
+      "logps/rejected": -1686.96875,
+      "loss": 0.0013,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.377600759267807,
+      "rewards/margins": 15.575152397155762,
+      "rewards/rejected": -15.952753067016602,
+      "step": 3120
+    },
+    {
+      "epoch": 0.7816694761815571,
+      "grad_norm": 0.00028228759765625,
+      "learning_rate": 6.914856176865891e-07,
+      "logits/chosen": 0.5658802390098572,
+      "logits/rejected": 1.7670371532440186,
+      "logps/chosen": -78.00981140136719,
+      "logps/rejected": -1716.184814453125,
+      "loss": 0.0012,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.34367048740386963,
+      "rewards/margins": 16.066198348999023,
+      "rewards/rejected": -16.409870147705078,
+      "step": 3130
+    },
+    {
+      "epoch": 0.7841668227508273,
+      "grad_norm": 6.198883056640625e-05,
+      "learning_rate": 6.765044018942804e-07,
+      "logits/chosen": 0.6243360042572021,
+      "logits/rejected": 1.8233163356781006,
+      "logps/chosen": -77.52008819580078,
+      "logps/rejected": -1712.4619140625,
+      "loss": 0.0008,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3422839641571045,
+      "rewards/margins": 16.079269409179688,
+      "rewards/rejected": -16.421554565429688,
+      "step": 3140
+    },
+    {
+      "epoch": 0.7866641693200974,
+      "grad_norm": 0.134765625,
+      "learning_rate": 6.616618211984169e-07,
+      "logits/chosen": 0.617003321647644,
+      "logits/rejected": 1.855446219444275,
+      "logps/chosen": -81.59101104736328,
+      "logps/rejected": -1846.211669921875,
+      "loss": 0.0011,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.37706810235977173,
+      "rewards/margins": 17.38241958618164,
+      "rewards/rejected": -17.75948715209961,
+      "step": 3150
+    },
+    {
+      "epoch": 0.7891615158893676,
+      "grad_norm": 0.04833984375,
+      "learning_rate": 6.469590040374799e-07,
+      "logits/chosen": 0.5514385104179382,
+      "logits/rejected": 1.7037725448608398,
+      "logps/chosen": -92.5042724609375,
+      "logps/rejected": -1909.711181640625,
+      "loss": 0.0013,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4816574156284332,
+      "rewards/margins": 17.822921752929688,
+      "rewards/rejected": -18.304576873779297,
+      "step": 3160
+    },
+    {
+      "epoch": 0.7916588624586377,
+      "grad_norm": 0.03173828125,
+      "learning_rate": 6.32397068224136e-07,
+      "logits/chosen": 0.528624415397644,
+      "logits/rejected": 1.6811710596084595,
+      "logps/chosen": -89.05570983886719,
+      "logps/rejected": -1862.1185302734375,
+      "loss": 0.002,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.44779324531555176,
+      "rewards/margins": 17.432941436767578,
+      "rewards/rejected": -17.880733489990234,
+      "step": 3170
+    },
+    {
+      "epoch": 0.7941562090279078,
+      "grad_norm": 0.04052734375,
+      "learning_rate": 6.17977120860249e-07,
+      "logits/chosen": 0.5938631296157837,
+      "logits/rejected": 1.7992160320281982,
+      "logps/chosen": -80.42100524902344,
+      "logps/rejected": -1828.5582275390625,
+      "loss": 0.0013,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.37270471453666687,
+      "rewards/margins": 17.214147567749023,
+      "rewards/rejected": -17.586851119995117,
+      "step": 3180
+    },
+    {
+      "epoch": 0.796653555597178,
+      "grad_norm": 0.07177734375,
+      "learning_rate": 6.037002582527121e-07,
+      "logits/chosen": 0.6156030893325806,
+      "logits/rejected": 1.7690448760986328,
+      "logps/chosen": -83.84468078613281,
+      "logps/rejected": -1731.0732421875,
+      "loss": 0.002,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4016353189945221,
+      "rewards/margins": 16.079975128173828,
+      "rewards/rejected": -16.481611251831055,
+      "step": 3190
+    },
+    {
+      "epoch": 0.7991509021664481,
+      "grad_norm": 0.03759765625,
+      "learning_rate": 5.895675658300981e-07,
+      "logits/chosen": 0.6333300471305847,
+      "logits/rejected": 1.8136202096939087,
+      "logps/chosen": -79.72032165527344,
+      "logps/rejected": -1559.860595703125,
+      "loss": 0.0009,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.37166255712509155,
+      "rewards/margins": 14.575735092163086,
+      "rewards/rejected": -14.947400093078613,
+      "step": 3200
+    },
+    {
+      "epoch": 0.8016482487357183,
+      "grad_norm": 0.032470703125,
+      "learning_rate": 5.755801180601381e-07,
+      "logits/chosen": 0.5778881907463074,
+      "logits/rejected": 1.754577875137329,
+      "logps/chosen": -85.21940612792969,
+      "logps/rejected": -1787.017333984375,
+      "loss": 0.0008,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.40619927644729614,
+      "rewards/margins": 16.748323440551758,
+      "rewards/rejected": -17.154521942138672,
+      "step": 3210
+    },
+    {
+      "epoch": 0.8041455953049884,
+      "grad_norm": 0.050537109375,
+      "learning_rate": 5.617389783680307e-07,
+      "logits/chosen": 0.5147963762283325,
+      "logits/rejected": 1.858233094215393,
+      "logps/chosen": -85.53825378417969,
+      "logps/rejected": -2189.0078125,
+      "loss": 0.0008,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.41369113326072693,
+      "rewards/margins": 20.725154876708984,
+      "rewards/rejected": -21.13884925842285,
+      "step": 3220
+    },
+    {
+      "epoch": 0.8066429418742586,
+      "grad_norm": 0.035888671875,
+      "learning_rate": 5.48045199055596e-07,
+      "logits/chosen": 0.6537925004959106,
+      "logits/rejected": 1.8616943359375,
+      "logps/chosen": -81.87962341308594,
+      "logps/rejected": -1831.1129150390625,
+      "loss": 0.0024,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3843618333339691,
+      "rewards/margins": 17.22643280029297,
+      "rewards/rejected": -17.610795974731445,
+      "step": 3230
+    },
+    {
+      "epoch": 0.8091402884435287,
+      "grad_norm": 0.02197265625,
+      "learning_rate": 5.344998212212704e-07,
+      "logits/chosen": 0.5282970070838928,
+      "logits/rejected": 1.810681939125061,
+      "logps/chosen": -85.24520111083984,
+      "logps/rejected": -2183.88037109375,
+      "loss": 0.0009,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4112313389778137,
+      "rewards/margins": 20.611225128173828,
+      "rewards/rejected": -21.022457122802734,
+      "step": 3240
+    },
+    {
+      "epoch": 0.811637635012799,
+      "grad_norm": 0.0023956298828125,
+      "learning_rate": 5.211038746809551e-07,
+      "logits/chosen": 0.6539278030395508,
+      "logits/rejected": 1.8353042602539062,
+      "logps/chosen": -83.72335815429688,
+      "logps/rejected": -1798.7239990234375,
+      "loss": 0.0013,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4129239618778229,
+      "rewards/margins": 16.881694793701172,
+      "rewards/rejected": -17.294618606567383,
+      "step": 3250
+    },
+    {
+      "epoch": 0.8141349815820691,
+      "grad_norm": 0.0294189453125,
+      "learning_rate": 5.078583778897216e-07,
+      "logits/chosen": 0.6602455377578735,
+      "logits/rejected": 1.7690246105194092,
+      "logps/chosen": -93.29869079589844,
+      "logps/rejected": -1809.647216796875,
+      "loss": 0.0017,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4940834641456604,
+      "rewards/margins": 16.90252113342285,
+      "rewards/rejected": -17.396602630615234,
+      "step": 3260
+    },
+    {
+      "epoch": 0.8166323281513392,
+      "grad_norm": 0.0830078125,
+      "learning_rate": 4.94764337864384e-07,
+      "logits/chosen": 0.6102297902107239,
+      "logits/rejected": 1.7652736902236938,
+      "logps/chosen": -81.92073059082031,
+      "logps/rejected": -1667.0634765625,
+      "loss": 0.0013,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.38310879468917847,
+      "rewards/margins": 15.526937484741211,
+      "rewards/rejected": -15.910046577453613,
+      "step": 3270
+    },
+    {
+      "epoch": 0.8191296747206094,
+      "grad_norm": 0.04150390625,
+      "learning_rate": 4.818227501069328e-07,
+      "logits/chosen": 0.5220754742622375,
+      "logits/rejected": 1.9412825107574463,
+      "logps/chosen": -81.39790344238281,
+      "logps/rejected": -2294.091064453125,
+      "loss": 0.0003,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3824032247066498,
+      "rewards/margins": 21.823511123657227,
+      "rewards/rejected": -22.205913543701172,
+      "step": 3280
+    },
+    {
+      "epoch": 0.8216270212898795,
+      "grad_norm": 0.025146484375,
+      "learning_rate": 4.690345985288572e-07,
+      "logits/chosen": 0.5984300971031189,
+      "logits/rejected": 1.787674903869629,
+      "logps/chosen": -85.31007385253906,
+      "logps/rejected": -1911.9349365234375,
+      "loss": 0.0016,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.41559547185897827,
+      "rewards/margins": 17.892559051513672,
+      "rewards/rejected": -18.308155059814453,
+      "step": 3290
+    },
+    {
+      "epoch": 0.8241243678591497,
+      "grad_norm": 8.791685104370117e-07,
+      "learning_rate": 4.5640085537633633e-07,
+      "logits/chosen": 0.5342472791671753,
+      "logits/rejected": 1.8117504119873047,
+      "logps/chosen": -77.17405700683594,
+      "logps/rejected": -2120.84912109375,
+      "loss": 0.0005,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.33397001028060913,
+      "rewards/margins": 20.135189056396484,
+      "rewards/rejected": -20.469158172607422,
+      "step": 3300
+    },
+    {
+      "epoch": 0.8266217144284198,
+      "grad_norm": 7.581710815429688e-05,
+      "learning_rate": 4.439224811563211e-07,
+      "logits/chosen": 0.5074091553688049,
+      "logits/rejected": 1.69931960105896,
+      "logps/chosen": -87.99649047851562,
+      "logps/rejected": -1984.584228515625,
+      "loss": 0.0013,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.442207008600235,
+      "rewards/margins": 18.59341049194336,
+      "rewards/rejected": -19.03561782836914,
+      "step": 3310
+    },
+    {
+      "epoch": 0.82911906099769,
+      "grad_norm": 0.00012111663818359375,
+      "learning_rate": 4.316004245635158e-07,
+      "logits/chosen": 0.533842921257019,
+      "logits/rejected": 1.7812063694000244,
+      "logps/chosen": -89.41615295410156,
+      "logps/rejected": -2132.177734375,
+      "loss": 0.0016,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4604712426662445,
+      "rewards/margins": 20.115650177001953,
+      "rewards/rejected": -20.576122283935547,
+      "step": 3320
+    },
+    {
+      "epoch": 0.8316164075669601,
+      "grad_norm": 6.8247318267822266e-06,
+      "learning_rate": 4.194356224082455e-07,
+      "logits/chosen": 0.4998435378074646,
+      "logits/rejected": 1.818884253501892,
+      "logps/chosen": -89.97554779052734,
+      "logps/rejected": -2246.93017578125,
+      "loss": 0.0016,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4607468247413635,
+      "rewards/margins": 21.1903133392334,
+      "rewards/rejected": -21.651060104370117,
+      "step": 3330
+    },
+    {
+      "epoch": 0.8341137541362302,
+      "grad_norm": 0.0008697509765625,
+      "learning_rate": 4.074289995452338e-07,
+      "logits/chosen": 0.663809597492218,
+      "logits/rejected": 1.8902143239974976,
+      "logps/chosen": -79.80634307861328,
+      "logps/rejected": -1863.5374755859375,
+      "loss": 0.0013,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3643765449523926,
+      "rewards/margins": 17.57371711730957,
+      "rewards/rejected": -17.938095092773438,
+      "step": 3340
+    },
+    {
+      "epoch": 0.8366111007055004,
+      "grad_norm": 9.715557098388672e-06,
+      "learning_rate": 3.9558146880329246e-07,
+      "logits/chosen": 0.5806099772453308,
+      "logits/rejected": 1.7180675268173218,
+      "logps/chosen": -88.5436782836914,
+      "logps/rejected": -1847.456298828125,
+      "loss": 0.0022,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.44766122102737427,
+      "rewards/margins": 17.159259796142578,
+      "rewards/rejected": -17.606922149658203,
+      "step": 3350
+    },
+    {
+      "epoch": 0.8391084472747705,
+      "grad_norm": 0.011962890625,
+      "learning_rate": 3.838939309159187e-07,
+      "logits/chosen": 0.6112891435623169,
+      "logits/rejected": 1.7461353540420532,
+      "logps/chosen": -85.6560287475586,
+      "logps/rejected": -1812.320068359375,
+      "loss": 0.0015,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.42908936738967896,
+      "rewards/margins": 16.986312866210938,
+      "rewards/rejected": -17.415403366088867,
+      "step": 3360
+    },
+    {
+      "epoch": 0.8416057938440407,
+      "grad_norm": 0.00148773193359375,
+      "learning_rate": 3.723672744528162e-07,
+      "logits/chosen": 0.5621702671051025,
+      "logits/rejected": 1.8088220357894897,
+      "logps/chosen": -78.2458724975586,
+      "logps/rejected": -1943.078125,
+      "loss": 0.0005,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.34909874200820923,
+      "rewards/margins": 18.354211807250977,
+      "rewards/rejected": -18.703310012817383,
+      "step": 3370
+    },
+    {
+      "epoch": 0.8441031404133108,
+      "grad_norm": 0.00182342529296875,
+      "learning_rate": 3.6100237575233647e-07,
+      "logits/chosen": 0.6288230419158936,
+      "logits/rejected": 1.7829002141952515,
+      "logps/chosen": -84.52963256835938,
+      "logps/rejected": -1607.1246337890625,
+      "loss": 0.001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.41369467973709106,
+      "rewards/margins": 15.001932144165039,
+      "rewards/rejected": -15.415626525878906,
+      "step": 3380
+    },
+    {
+      "epoch": 0.8466004869825811,
+      "grad_norm": 0.0247802734375,
+      "learning_rate": 3.4980009885486054e-07,
+      "logits/chosen": 0.6984633803367615,
+      "logits/rejected": 1.8022708892822266,
+      "logps/chosen": -77.57188415527344,
+      "logps/rejected": -1505.264892578125,
+      "loss": 0.0019,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3510386645793915,
+      "rewards/margins": 14.059832572937012,
+      "rewards/rejected": -14.410870552062988,
+      "step": 3390
+    },
+    {
+      "epoch": 0.8490978335518512,
+      "grad_norm": 0.0277099609375,
+      "learning_rate": 3.3876129543710197e-07,
+      "logits/chosen": 0.5471528768539429,
+      "logits/rejected": 1.768148422241211,
+      "logps/chosen": -87.0343246459961,
+      "logps/rejected": -2079.26708984375,
+      "loss": 0.0008,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.44154462218284607,
+      "rewards/margins": 19.6082706451416,
+      "rewards/rejected": -20.049814224243164,
+      "step": 3400
+    },
+    {
+      "epoch": 0.8515951801211213,
+      "grad_norm": 0.00250244140625,
+      "learning_rate": 3.2788680474735687e-07,
+      "logits/chosen": 0.5990682244300842,
+      "logits/rejected": 1.8559329509735107,
+      "logps/chosen": -80.894287109375,
+      "logps/rejected": -1869.932373046875,
+      "loss": 0.0006,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3692544400691986,
+      "rewards/margins": 17.629741668701172,
+      "rewards/rejected": -17.99899673461914,
+      "step": 3410
+    },
+    {
+      "epoch": 0.8540925266903915,
+      "grad_norm": 0.00060272216796875,
+      "learning_rate": 3.1717745354170214e-07,
+      "logits/chosen": 0.550452470779419,
+      "logits/rejected": 1.9075467586517334,
+      "logps/chosen": -88.12527465820312,
+      "logps/rejected": -2150.641357421875,
+      "loss": 0.0007,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4492851793766022,
+      "rewards/margins": 20.334678649902344,
+      "rewards/rejected": -20.78396224975586,
+      "step": 3420
+    },
+    {
+      "epoch": 0.8565898732596616,
+      "grad_norm": 0.000537872314453125,
+      "learning_rate": 3.0663405602113727e-07,
+      "logits/chosen": 0.5784090757369995,
+      "logits/rejected": 1.8440923690795898,
+      "logps/chosen": -77.76791381835938,
+      "logps/rejected": -1944.1751708984375,
+      "loss": 0.0009,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.34689050912857056,
+      "rewards/margins": 18.372732162475586,
+      "rewards/rejected": -18.719623565673828,
+      "step": 3430
+    },
+    {
+      "epoch": 0.8590872198289318,
+      "grad_norm": 0.01416015625,
+      "learning_rate": 2.9625741376968107e-07,
+      "logits/chosen": 0.5425665378570557,
+      "logits/rejected": 1.7546192407608032,
+      "logps/chosen": -84.84912109375,
+      "logps/rejected": -2032.4622802734375,
+      "loss": 0.0011,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.40073734521865845,
+      "rewards/margins": 18.93886375427246,
+      "rewards/rejected": -19.339599609375,
+      "step": 3440
+    },
+    {
+      "epoch": 0.8615845663982019,
+      "grad_norm": 0.0206298828125,
+      "learning_rate": 2.8604831569343324e-07,
+      "logits/chosen": 0.5840142965316772,
+      "logits/rejected": 1.6774394512176514,
+      "logps/chosen": -87.08283996582031,
+      "logps/rejected": -1656.0628662109375,
+      "loss": 0.0017,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4280741214752197,
+      "rewards/margins": 15.362825393676758,
+      "rewards/rejected": -15.790898323059082,
+      "step": 3450
+    },
+    {
+      "epoch": 0.864081912967472,
+      "grad_norm": 0.00439453125,
+      "learning_rate": 2.760075379605942e-07,
+      "logits/chosen": 0.5762545466423035,
+      "logits/rejected": 1.788022756576538,
+      "logps/chosen": -84.1275634765625,
+      "logps/rejected": -1882.4407958984375,
+      "loss": 0.0006,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.40409308671951294,
+      "rewards/margins": 17.698461532592773,
+      "rewards/rejected": -18.102556228637695,
+      "step": 3460
+    },
+    {
+      "epoch": 0.8665792595367422,
+      "grad_norm": 0.042724609375,
+      "learning_rate": 2.661358439424552e-07,
+      "logits/chosen": 0.6203972697257996,
+      "logits/rejected": 1.7815377712249756,
+      "logps/chosen": -79.66865539550781,
+      "logps/rejected": -1772.3359375,
+      "loss": 0.0019,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.36624962091445923,
+      "rewards/margins": 16.665634155273438,
+      "rewards/rejected": -17.031885147094727,
+      "step": 3470
+    },
+    {
+      "epoch": 0.8690766061060123,
+      "grad_norm": 0.0026397705078125,
+      "learning_rate": 2.564339841553615e-07,
+      "logits/chosen": 0.6519962549209595,
+      "logits/rejected": 1.8045860528945923,
+      "logps/chosen": -84.95247650146484,
+      "logps/rejected": -1720.0745849609375,
+      "loss": 0.0018,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4275601804256439,
+      "rewards/margins": 16.091691970825195,
+      "rewards/rejected": -16.51925277709961,
+      "step": 3480
+    },
+    {
+      "epoch": 0.8715739526752825,
+      "grad_norm": 0.1845703125,
+      "learning_rate": 2.469026962036539e-07,
+      "logits/chosen": 0.5797117352485657,
+      "logits/rejected": 1.6833524703979492,
+      "logps/chosen": -88.14852905273438,
+      "logps/rejected": -1696.3714599609375,
+      "loss": 0.003,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4267314374446869,
+      "rewards/margins": 15.646469116210938,
+      "rewards/rejected": -16.07320213317871,
+      "step": 3490
+    },
+    {
+      "epoch": 0.8740712992445526,
+      "grad_norm": 0.046142578125,
+      "learning_rate": 2.3754270472358786e-07,
+      "logits/chosen": 0.6232503652572632,
+      "logits/rejected": 1.6990222930908203,
+      "logps/chosen": -83.14488983154297,
+      "logps/rejected": -1672.453857421875,
+      "loss": 0.0022,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.39773061871528625,
+      "rewards/margins": 15.525070190429688,
+      "rewards/rejected": -15.922798156738281,
+      "step": 3500
+    },
+    {
+      "epoch": 0.8765686458138228,
+      "grad_norm": 0.07373046875,
+      "learning_rate": 2.283547213282458e-07,
+      "logits/chosen": 0.5654767155647278,
+      "logits/rejected": 1.7425930500030518,
+      "logps/chosen": -85.06395721435547,
+      "logps/rejected": -1774.28125,
+      "loss": 0.0011,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4012150168418884,
+      "rewards/margins": 16.552427291870117,
+      "rewards/rejected": -16.953643798828125,
+      "step": 3510
+    },
+    {
+      "epoch": 0.8790659923830929,
+      "grad_norm": 0.052734375,
+      "learning_rate": 2.1933944455343166e-07,
+      "logits/chosen": 0.5508383512496948,
+      "logits/rejected": 1.7986376285552979,
+      "logps/chosen": -81.19587707519531,
+      "logps/rejected": -2142.4736328125,
+      "loss": 0.0011,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.374092161655426,
+      "rewards/margins": 20.307262420654297,
+      "rewards/rejected": -20.681354522705078,
+      "step": 3520
+    },
+    {
+      "epoch": 0.8815633389523632,
+      "grad_norm": 0.003265380859375,
+      "learning_rate": 2.104975598045647e-07,
+      "logits/chosen": 0.5937038660049438,
+      "logits/rejected": 1.7039823532104492,
+      "logps/chosen": -81.80549621582031,
+      "logps/rejected": -1617.2646484375,
+      "loss": 0.0011,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.38183221220970154,
+      "rewards/margins": 15.109405517578125,
+      "rewards/rejected": -15.491238594055176,
+      "step": 3530
+    },
+    {
+      "epoch": 0.8840606855216333,
+      "grad_norm": 0.032470703125,
+      "learning_rate": 2.018297393045701e-07,
+      "logits/chosen": 0.6291056871414185,
+      "logits/rejected": 1.7945873737335205,
+      "logps/chosen": -81.0144271850586,
+      "logps/rejected": -1751.724609375,
+      "loss": 0.0013,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3717043697834015,
+      "rewards/margins": 16.441532135009766,
+      "rewards/rejected": -16.813236236572266,
+      "step": 3540
+    },
+    {
+      "epoch": 0.8865580320909034,
+      "grad_norm": 0.039306640625,
+      "learning_rate": 1.9333664204277236e-07,
+      "logits/chosen": 0.5141820907592773,
+      "logits/rejected": 1.6924489736557007,
+      "logps/chosen": -83.84037780761719,
+      "logps/rejected": -2023.642822265625,
+      "loss": 0.0015,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3969913423061371,
+      "rewards/margins": 18.992799758911133,
+      "rewards/rejected": -19.38979148864746,
+      "step": 3550
+    },
+    {
+      "epoch": 0.8890553786601736,
+      "grad_norm": 2.4437904357910156e-06,
+      "learning_rate": 1.8501891372479124e-07,
+      "logits/chosen": 0.5262492895126343,
+      "logits/rejected": 1.801138162612915,
+      "logps/chosen": -82.75626373291016,
+      "logps/rejected": -1913.7564697265625,
+      "loss": 0.0009,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.37326088547706604,
+      "rewards/margins": 17.95614242553711,
+      "rewards/rejected": -18.329402923583984,
+      "step": 3560
+    },
+    {
+      "epoch": 0.8915527252294437,
+      "grad_norm": 0.0230712890625,
+      "learning_rate": 1.7687718672345533e-07,
+      "logits/chosen": 0.5257088541984558,
+      "logits/rejected": 1.7338473796844482,
+      "logps/chosen": -84.81585693359375,
+      "logps/rejected": -1994.5963134765625,
+      "loss": 0.0007,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4195898473262787,
+      "rewards/margins": 18.773387908935547,
+      "rewards/rejected": -19.19297981262207,
+      "step": 3570
+    },
+    {
+      "epoch": 0.8940500717987139,
+      "grad_norm": 0.032958984375,
+      "learning_rate": 1.689120800307212e-07,
+      "logits/chosen": 0.43529587984085083,
+      "logits/rejected": 1.6352293491363525,
+      "logps/chosen": -84.8902816772461,
+      "logps/rejected": -2162.602294921875,
+      "loss": 0.0011,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4021669030189514,
+      "rewards/margins": 20.277816772460938,
+      "rewards/rejected": -20.679983139038086,
+      "step": 3580
+    },
+    {
+      "epoch": 0.896547418367984,
+      "grad_norm": 0.09033203125,
+      "learning_rate": 1.6112419921061357e-07,
+      "logits/chosen": 0.6369230151176453,
+      "logits/rejected": 1.848402976989746,
+      "logps/chosen": -89.04129791259766,
+      "logps/rejected": -1788.65234375,
+      "loss": 0.0025,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4632226526737213,
+      "rewards/margins": 16.736385345458984,
+      "rewards/rejected": -17.199607849121094,
+      "step": 3590
+    },
+    {
+      "epoch": 0.8990447649372542,
+      "grad_norm": 0.039306640625,
+      "learning_rate": 1.5351413635318807e-07,
+      "logits/chosen": 0.5430204272270203,
+      "logits/rejected": 1.6954717636108398,
+      "logps/chosen": -80.37650299072266,
+      "logps/rejected": -1739.5556640625,
+      "loss": 0.001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.36847007274627686,
+      "rewards/margins": 16.253259658813477,
+      "rewards/rejected": -16.621726989746094,
+      "step": 3600
+    },
+    {
+      "epoch": 0.9015421115065243,
+      "grad_norm": 0.00982666015625,
+      "learning_rate": 1.460824700295138e-07,
+      "logits/chosen": 0.5976991653442383,
+      "logits/rejected": 1.8408482074737549,
+      "logps/chosen": -83.05894470214844,
+      "logps/rejected": -1974.3404541015625,
+      "loss": 0.0016,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.40586838126182556,
+      "rewards/margins": 18.630273818969727,
+      "rewards/rejected": -19.036144256591797,
+      "step": 3610
+    },
+    {
+      "epoch": 0.9040394580757944,
+      "grad_norm": 0.01531982421875,
+      "learning_rate": 1.3882976524768694e-07,
+      "logits/chosen": 0.6637327075004578,
+      "logits/rejected": 1.75222909450531,
+      "logps/chosen": -82.24571228027344,
+      "logps/rejected": -1613.786865234375,
+      "loss": 0.0023,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.39647185802459717,
+      "rewards/margins": 15.058723449707031,
+      "rewards/rejected": -15.455195426940918,
+      "step": 3620
+    },
+    {
+      "epoch": 0.9065368046450646,
+      "grad_norm": 0.00054931640625,
+      "learning_rate": 1.3175657340987664e-07,
+      "logits/chosen": 0.6287505030632019,
+      "logits/rejected": 1.799709677696228,
+      "logps/chosen": -84.08810424804688,
+      "logps/rejected": -1803.7301025390625,
+      "loss": 0.0014,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4151592254638672,
+      "rewards/margins": 16.90250015258789,
+      "rewards/rejected": -17.317657470703125,
+      "step": 3630
+    },
+    {
+      "epoch": 0.9090341512143347,
+      "grad_norm": 0.0010986328125,
+      "learning_rate": 1.2486343227040122e-07,
+      "logits/chosen": 0.5875022411346436,
+      "logits/rejected": 1.7384836673736572,
+      "logps/chosen": -87.7680435180664,
+      "logps/rejected": -1834.406982421875,
+      "loss": 0.0029,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4344062805175781,
+      "rewards/margins": 17.189672470092773,
+      "rewards/rejected": -17.62407875061035,
+      "step": 3640
+    },
+    {
+      "epoch": 0.9115314977836049,
+      "grad_norm": 0.0419921875,
+      "learning_rate": 1.181508658948452e-07,
+      "logits/chosen": 0.6155994534492493,
+      "logits/rejected": 1.7817541360855103,
+      "logps/chosen": -80.59324645996094,
+      "logps/rejected": -1767.1129150390625,
+      "loss": 0.0012,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3722308278083801,
+      "rewards/margins": 16.59187889099121,
+      "rewards/rejected": -16.964111328125,
+      "step": 3650
+    },
+    {
+      "epoch": 0.9140288443528751,
+      "grad_norm": 0.0458984375,
+      "learning_rate": 1.1161938462021627e-07,
+      "logits/chosen": 0.6269813776016235,
+      "logits/rejected": 1.7340434789657593,
+      "logps/chosen": -86.62757110595703,
+      "logps/rejected": -1731.770751953125,
+      "loss": 0.0028,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4310578405857086,
+      "rewards/margins": 16.17133903503418,
+      "rewards/rejected": -16.602397918701172,
+      "step": 3660
+    },
+    {
+      "epoch": 0.9165261909221453,
+      "grad_norm": 0.1083984375,
+      "learning_rate": 1.0526948501614536e-07,
+      "logits/chosen": 0.5681526064872742,
+      "logits/rejected": 1.8455768823623657,
+      "logps/chosen": -88.42396545410156,
+      "logps/rejected": -1962.5306396484375,
+      "loss": 0.0017,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.44573092460632324,
+      "rewards/margins": 18.391765594482422,
+      "rewards/rejected": -18.837499618530273,
+      "step": 3670
+    },
+    {
+      "epoch": 0.9190235374914154,
+      "grad_norm": 0.0791015625,
+      "learning_rate": 9.910164984713477e-08,
+      "logits/chosen": 0.5716847777366638,
+      "logits/rejected": 1.790804147720337,
+      "logps/chosen": -88.99705505371094,
+      "logps/rejected": -2003.9739990234375,
+      "loss": 0.0015,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.457981675863266,
+      "rewards/margins": 18.8723087310791,
+      "rewards/rejected": -19.330291748046875,
+      "step": 3680
+    },
+    {
+      "epoch": 0.9215208840606856,
+      "grad_norm": 0.0830078125,
+      "learning_rate": 9.311634803585323e-08,
+      "logits/chosen": 0.5493127107620239,
+      "logits/rejected": 1.8056682348251343,
+      "logps/chosen": -80.04996490478516,
+      "logps/rejected": -2063.024169921875,
+      "loss": 0.0024,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3674994707107544,
+      "rewards/margins": 19.52413558959961,
+      "rewards/rejected": -19.891637802124023,
+      "step": 3690
+    },
+    {
+      "epoch": 0.9240182306299557,
+      "grad_norm": 0.000400543212890625,
+      "learning_rate": 8.7314034627487e-08,
+      "logits/chosen": 0.5750405192375183,
+      "logits/rejected": 1.8101009130477905,
+      "logps/chosen": -78.46788024902344,
+      "logps/rejected": -1988.633056640625,
+      "loss": 0.0013,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.35628947615623474,
+      "rewards/margins": 18.821575164794922,
+      "rewards/rejected": -19.1778621673584,
+      "step": 3700
+    },
+    {
+      "epoch": 0.9265155771992258,
+      "grad_norm": 0.03515625,
+      "learning_rate": 8.16951507551439e-08,
+      "logits/chosen": 0.6284887194633484,
+      "logits/rejected": 1.7544790506362915,
+      "logps/chosen": -78.0561752319336,
+      "logps/rejected": -1719.0638427734375,
+      "loss": 0.0012,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3490171432495117,
+      "rewards/margins": 16.02911949157715,
+      "rewards/rejected": -16.378137588500977,
+      "step": 3710
+    },
+    {
+      "epoch": 0.929012923768496,
+      "grad_norm": 6.4849853515625e-05,
+      "learning_rate": 7.626012360631291e-08,
+      "logits/chosen": 0.5767999887466431,
+      "logits/rejected": 1.8027598857879639,
+      "logps/chosen": -87.67066192626953,
+      "logps/rejected": -1751.861328125,
+      "loss": 0.0015,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.44937849044799805,
+      "rewards/margins": 16.375062942504883,
+      "rewards/rejected": -16.82444190979004,
+      "step": 3720
+    },
+    {
+      "epoch": 0.9315102703377661,
+      "grad_norm": 0.0006256103515625,
+      "learning_rate": 7.100936639038936e-08,
+      "logits/chosen": 0.5324774384498596,
+      "logits/rejected": 1.9097219705581665,
+      "logps/chosen": -89.81242370605469,
+      "logps/rejected": -2373.812744140625,
+      "loss": 0.0005,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4651169776916504,
+      "rewards/margins": 22.532739639282227,
+      "rewards/rejected": -22.997854232788086,
+      "step": 3730
+    },
+    {
+      "epoch": 0.9340076169070363,
+      "grad_norm": 1.0356307029724121e-06,
+      "learning_rate": 6.594327830725916e-08,
+      "logits/chosen": 0.5782414674758911,
+      "logits/rejected": 1.906734824180603,
+      "logps/chosen": -80.02381896972656,
+      "logps/rejected": -1915.576416015625,
+      "loss": 0.0006,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3699369728565216,
+      "rewards/margins": 18.08548927307129,
+      "rewards/rejected": -18.455425262451172,
+      "step": 3740
+    },
+    {
+      "epoch": 0.9365049634763064,
+      "grad_norm": 0.039306640625,
+      "learning_rate": 6.106224451694592e-08,
+      "logits/chosen": 0.5905268788337708,
+      "logits/rejected": 1.7933048009872437,
+      "logps/chosen": -77.8465347290039,
+      "logps/rejected": -1931.9468994140625,
+      "loss": 0.0011,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3471309244632721,
+      "rewards/margins": 18.231327056884766,
+      "rewards/rejected": -18.57845687866211,
+      "step": 3750
+    },
+    {
+      "epoch": 0.9390023100455765,
+      "grad_norm": 0.07861328125,
+      "learning_rate": 5.636663611033266e-08,
+      "logits/chosen": 0.6453654170036316,
+      "logits/rejected": 1.9545791149139404,
+      "logps/chosen": -88.19773864746094,
+      "logps/rejected": -2014.8583984375,
+      "loss": 0.0013,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4550137519836426,
+      "rewards/margins": 18.988948822021484,
+      "rewards/rejected": -19.443960189819336,
+      "step": 3760
+    },
+    {
+      "epoch": 0.9414996566148467,
+      "grad_norm": 0.00152587890625,
+      "learning_rate": 5.185681008094579e-08,
+      "logits/chosen": 0.5483246445655823,
+      "logits/rejected": 1.754540205001831,
+      "logps/chosen": -81.78535461425781,
+      "logps/rejected": -1835.6578369140625,
+      "loss": 0.0009,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3718385696411133,
+      "rewards/margins": 17.2070369720459,
+      "rewards/rejected": -17.578876495361328,
+      "step": 3770
+    },
+    {
+      "epoch": 0.9439970031841168,
+      "grad_norm": 0.00086212158203125,
+      "learning_rate": 4.753310929781513e-08,
+      "logits/chosen": 0.6061893701553345,
+      "logits/rejected": 1.7712287902832031,
+      "logps/chosen": -84.43482971191406,
+      "logps/rejected": -1739.1148681640625,
+      "loss": 0.0015,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4173789620399475,
+      "rewards/margins": 16.286354064941406,
+      "rewards/rejected": -16.703731536865234,
+      "step": 3780
+    },
+    {
+      "epoch": 0.946494349753387,
+      "grad_norm": 1.2874603271484375e-05,
+      "learning_rate": 4.3395862479405914e-08,
+      "logits/chosen": 0.5530301928520203,
+      "logits/rejected": 1.790492296218872,
+      "logps/chosen": -98.83070373535156,
+      "logps/rejected": -1927.699462890625,
+      "loss": 0.0005,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5457450747489929,
+      "rewards/margins": 17.921890258789062,
+      "rewards/rejected": -18.467636108398438,
+      "step": 3790
+    },
+    {
+      "epoch": 0.9489916963226572,
+      "grad_norm": 0.0028228759765625,
+      "learning_rate": 3.9445384168628474e-08,
+      "logits/chosen": 0.5836749076843262,
+      "logits/rejected": 1.8246829509735107,
+      "logps/chosen": -80.40269470214844,
+      "logps/rejected": -1708.9019775390625,
+      "loss": 0.0009,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3665417730808258,
+      "rewards/margins": 15.90452766418457,
+      "rewards/rejected": -16.271068572998047,
+      "step": 3800
+    },
+    {
+      "epoch": 0.9514890428919274,
+      "grad_norm": 0.017578125,
+      "learning_rate": 3.5681974708923484e-08,
+      "logits/chosen": 0.6176645159721375,
+      "logits/rejected": 1.7562223672866821,
+      "logps/chosen": -82.15937805175781,
+      "logps/rejected": -1655.8538818359375,
+      "loss": 0.0012,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3880414068698883,
+      "rewards/margins": 15.394001960754395,
+      "rewards/rejected": -15.78204345703125,
+      "step": 3810
+    },
+    {
+      "epoch": 0.9539863894611975,
+      "grad_norm": 0.003021240234375,
+      "learning_rate": 3.210592022142717e-08,
+      "logits/chosen": 0.6430649161338806,
+      "logits/rejected": 1.7840299606323242,
+      "logps/chosen": -88.11245727539062,
+      "logps/rejected": -1835.171630859375,
+      "loss": 0.0009,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4654541611671448,
+      "rewards/margins": 17.16687774658203,
+      "rewards/rejected": -17.6323299407959,
+      "step": 3820
+    },
+    {
+      "epoch": 0.9564837360304677,
+      "grad_norm": 0.0260009765625,
+      "learning_rate": 2.8717492583220095e-08,
+      "logits/chosen": 0.6011831164360046,
+      "logits/rejected": 1.8058007955551147,
+      "logps/chosen": -83.06114959716797,
+      "logps/rejected": -1898.2939453125,
+      "loss": 0.0012,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.39993369579315186,
+      "rewards/margins": 17.886281967163086,
+      "rewards/rejected": -18.28621482849121,
+      "step": 3830
+    },
+    {
+      "epoch": 0.9589810825997378,
+      "grad_norm": 0.109375,
+      "learning_rate": 2.551694940665539e-08,
+      "logits/chosen": 0.600081741809845,
+      "logits/rejected": 1.7849693298339844,
+      "logps/chosen": -82.53587341308594,
+      "logps/rejected": -1787.7008056640625,
+      "loss": 0.0015,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.39744192361831665,
+      "rewards/margins": 16.777345657348633,
+      "rewards/rejected": -17.174787521362305,
+      "step": 3840
+    },
+    {
+      "epoch": 0.9614784291690079,
+      "grad_norm": 5.4836273193359375e-05,
+      "learning_rate": 2.2504534019774092e-08,
+      "logits/chosen": 0.712979257106781,
+      "logits/rejected": 1.8468831777572632,
+      "logps/chosen": -80.92863464355469,
+      "logps/rejected": -1629.561767578125,
+      "loss": 0.0012,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3758103847503662,
+      "rewards/margins": 15.236900329589844,
+      "rewards/rejected": -15.612710952758789,
+      "step": 3850
+    },
+    {
+      "epoch": 0.9639757757382781,
+      "grad_norm": 0.058349609375,
+      "learning_rate": 1.9680475447805826e-08,
+      "logits/chosen": 0.6279615759849548,
+      "logits/rejected": 1.7799345254898071,
+      "logps/chosen": -81.30252838134766,
+      "logps/rejected": -1729.193115234375,
+      "loss": 0.0014,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3813311457633972,
+      "rewards/margins": 16.206878662109375,
+      "rewards/rejected": -16.58820915222168,
+      "step": 3860
+    },
+    {
+      "epoch": 0.9664731223075482,
+      "grad_norm": 0.000812530517578125,
+      "learning_rate": 1.70449883957563e-08,
+      "logits/chosen": 0.5945799350738525,
+      "logits/rejected": 1.7449548244476318,
+      "logps/chosen": -79.51522064208984,
+      "logps/rejected": -1772.429931640625,
+      "loss": 0.0011,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3488074243068695,
+      "rewards/margins": 16.572925567626953,
+      "rewards/rejected": -16.921733856201172,
+      "step": 3870
+    },
+    {
+      "epoch": 0.9689704688768184,
+      "grad_norm": 0.03662109375,
+      "learning_rate": 1.4598273232083182e-08,
+      "logits/chosen": 0.5940654873847961,
+      "logits/rejected": 1.737255334854126,
+      "logps/chosen": -89.48096466064453,
+      "logps/rejected": -1763.216064453125,
+      "loss": 0.0026,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4539431631565094,
+      "rewards/margins": 16.4827880859375,
+      "rewards/rejected": -16.93673324584961,
+      "step": 3880
+    },
+    {
+      "epoch": 0.9714678154460885,
+      "grad_norm": 7.677078247070312e-05,
+      "learning_rate": 1.2340515973464917e-08,
+      "logits/chosen": 0.5371723771095276,
+      "logits/rejected": 1.6967008113861084,
+      "logps/chosen": -89.69468688964844,
+      "logps/rejected": -1855.769287109375,
+      "loss": 0.0005,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4602131247520447,
+      "rewards/margins": 17.378524780273438,
+      "rewards/rejected": -17.8387393951416,
+      "step": 3890
+    },
+    {
+      "epoch": 0.9739651620153587,
+      "grad_norm": 0.01055908203125,
+      "learning_rate": 1.0271888270655118e-08,
+      "logits/chosen": 0.5918472409248352,
+      "logits/rejected": 1.6669203042984009,
+      "logps/chosen": -84.76191711425781,
+      "logps/rejected": -1762.7152099609375,
+      "loss": 0.0017,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.40269866585731506,
+      "rewards/margins": 16.396747589111328,
+      "rewards/rejected": -16.79944610595703,
+      "step": 3900
+    },
+    {
+      "epoch": 0.9764625085846288,
+      "grad_norm": 0.005584716796875,
+      "learning_rate": 8.392547395435769e-09,
+      "logits/chosen": 0.6482867002487183,
+      "logits/rejected": 1.7531925439834595,
+      "logps/chosen": -84.86217498779297,
+      "logps/rejected": -1594.6502685546875,
+      "loss": 0.0014,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4176376461982727,
+      "rewards/margins": 14.827906608581543,
+      "rewards/rejected": -15.245546340942383,
+      "step": 3910
+    },
+    {
+      "epoch": 0.9789598551538989,
+      "grad_norm": 1.0251998901367188e-05,
+      "learning_rate": 6.702636228657911e-09,
+      "logits/chosen": 0.6012560129165649,
+      "logits/rejected": 1.7790956497192383,
+      "logps/chosen": -85.35179138183594,
+      "logps/rejected": -1768.960693359375,
+      "loss": 0.001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.41833052039146423,
+      "rewards/margins": 16.56346321105957,
+      "rewards/rejected": -16.981792449951172,
+      "step": 3920
+    },
+    {
+      "epoch": 0.9814572017231691,
+      "grad_norm": 0.049560546875,
+      "learning_rate": 5.2022832493800465e-09,
+      "logits/chosen": 0.575610339641571,
+      "logits/rejected": 1.6641845703125,
+      "logps/chosen": -87.38862609863281,
+      "logps/rejected": -1609.995849609375,
+      "loss": 0.001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.43747106194496155,
+      "rewards/margins": 15.003010749816895,
+      "rewards/rejected": -15.440483093261719,
+      "step": 3930
+    },
+    {
+      "epoch": 0.9839545482924393,
+      "grad_norm": 0.00927734375,
+      "learning_rate": 3.891602525100124e-09,
+      "logits/chosen": 0.5365520119667053,
+      "logits/rejected": 1.7841014862060547,
+      "logps/chosen": -77.71090698242188,
+      "logps/rejected": -1843.400634765625,
+      "loss": 0.0006,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.33150431513786316,
+      "rewards/margins": 17.311891555786133,
+      "rewards/rejected": -17.643396377563477,
+      "step": 3940
+    },
+    {
+      "epoch": 0.9864518948617095,
+      "grad_norm": 0.000896453857421875,
+      "learning_rate": 2.7706937030827495e-09,
+      "logits/chosen": 0.6269220113754272,
+      "logits/rejected": 1.821447730064392,
+      "logps/chosen": -79.47364807128906,
+      "logps/rejected": -1583.856689453125,
+      "loss": 0.0012,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.35821038484573364,
+      "rewards/margins": 14.733774185180664,
+      "rewards/rejected": -15.091984748840332,
+      "step": 3950
+    },
+    {
+      "epoch": 0.9889492414309796,
+      "grad_norm": 0.021728515625,
+      "learning_rate": 1.839642002783859e-09,
+      "logits/chosen": 0.7017726302146912,
+      "logits/rejected": 1.8069097995758057,
+      "logps/chosen": -79.8754653930664,
+      "logps/rejected": -1646.0501708984375,
+      "loss": 0.002,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3667237162590027,
+      "rewards/margins": 15.337608337402344,
+      "rewards/rejected": -15.704330444335938,
+      "step": 3960
+    },
+    {
+      "epoch": 0.9914465880002498,
+      "grad_norm": 0.021728515625,
+      "learning_rate": 1.0985182093714574e-09,
+      "logits/chosen": 0.6416125297546387,
+      "logits/rejected": 1.760498046875,
+      "logps/chosen": -85.49261474609375,
+      "logps/rejected": -1687.677978515625,
+      "loss": 0.0006,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.42397341132164,
+      "rewards/margins": 15.762815475463867,
+      "rewards/rejected": -16.18678855895996,
+      "step": 3970
+    },
+    {
+      "epoch": 0.9939439345695199,
+      "grad_norm": 0.003143310546875,
+      "learning_rate": 5.473786683440896e-10,
+      "logits/chosen": 0.5962403416633606,
+      "logits/rejected": 1.8247146606445312,
+      "logps/chosen": -83.6251220703125,
+      "logps/rejected": -2010.3323974609375,
+      "loss": 0.0019,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4074554443359375,
+      "rewards/margins": 18.965452194213867,
+      "rewards/rejected": -19.372909545898438,
+      "step": 3980
+    },
+    {
+      "epoch": 0.99644128113879,
+      "grad_norm": 0.049560546875,
+      "learning_rate": 1.862652812467669e-10,
+      "logits/chosen": 0.5162760615348816,
+      "logits/rejected": 1.6673294305801392,
+      "logps/chosen": -83.96633911132812,
+      "logps/rejected": -1847.5804443359375,
+      "loss": 0.0008,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.39398157596588135,
+      "rewards/margins": 17.087627410888672,
+      "rewards/rejected": -17.481609344482422,
+      "step": 3990
+    },
+    {
+      "epoch": 0.9989386277080602,
+      "grad_norm": 2.9802322387695312e-05,
+      "learning_rate": 1.5205502486292932e-11,
+      "logits/chosen": 0.582720935344696,
+      "logits/rejected": 1.8548141717910767,
+      "logps/chosen": -80.02593994140625,
+      "logps/rejected": -1968.4027099609375,
+      "loss": 0.0011,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.35573673248291016,
+      "rewards/margins": 18.611202239990234,
+      "rewards/rejected": -18.96693992614746,
+      "step": 4000
+    },
+    {
+      "epoch": 0.9989386277080602,
+      "eval_logits/chosen": 0.6540641784667969,
+      "eval_logits/rejected": 1.569779634475708,
+      "eval_logps/chosen": -84.51407623291016,
+      "eval_logps/rejected": -994.2071533203125,
+      "eval_loss": 0.00282670627348125,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.40158194303512573,
+      "eval_rewards/margins": 8.929591178894043,
+      "eval_rewards/rejected": -9.33117389678955,
+      "eval_runtime": 0.6151,
+      "eval_samples_per_second": 8.128,
+      "eval_steps_per_second": 8.128,
+      "step": 4000
+    },
+    {
+      "epoch": 0.9999375663357682,
+      "step": 4004,
+      "total_flos": 0.0,
+      "train_loss": 0.05122942747121405,
+      "train_runtime": 6577.5594,
+      "train_samples_per_second": 2.435,
+      "train_steps_per_second": 0.609
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 4004,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000,
+  "total_flos": 0.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}