Model save

README.md

@@ -18,15 +18,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.0007
-- Rewards/chosen: -2.2108
-- Rewards/rejected: -84.9647
+- Loss: 0.0010
+- Rewards/chosen: -2.2051
+- Rewards/rejected: -74.7846
 - Rewards/accuracies: 1.0
-- Rewards/margins: 82.7539
-- Logps/rejected: -9137.2617
-- Logps/chosen: -588.0638
-- Logits/rejected: 4.7555
-- Logits/chosen: -0.0442
+- Rewards/margins: 72.5795
+- Logps/rejected: -8119.2456
+- Logps/chosen: -587.4921
+- Logits/rejected: 2.7746
+- Logits/chosen: -0.3615
 
 ## Model description
 
@@ -63,13 +63,13 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.0158        | 0.1420 | 50   | 0.0104          | -1.4246        | -52.2276         | 0.9960             | 50.8030         | -5863.5493     | -509.4384    | -2.3895         | -2.8735       |
-| 0.0111        | 0.2841 | 100  | 0.0024          | -2.2596        | -93.3357         | 1.0                | 91.0762         | -9974.3623     | -592.9393    | 0.1786          | -2.9450       |
-| 0.0039        | 0.4261 | 150  | 0.0016          | -2.3351        | -100.4879        | 1.0                | 98.1528         | -10689.5820    | -600.4949    | 2.3549          | -1.8831       |
-| 0.0022        | 0.5682 | 200  | 0.0012          | -2.2027        | -86.1756         | 1.0                | 83.9729         | -9258.3438     | -587.2476    | 2.5108          | -1.6726       |
-| 0.0022        | 0.7102 | 250  | 0.0008          | -2.2903        | -83.1896         | 1.0                | 80.8993         | -8959.7471     | -596.0095    | 3.7585          | -1.0150       |
-| 0.001         | 0.8523 | 300  | 0.0007          | -2.1936        | -83.9541         | 1.0                | 81.7606         | -9036.2012     | -586.3376    | 4.7089          | -0.1221       |
-| 0.008         | 0.9943 | 350  | 0.0007          | -2.2108        | -84.9647         | 1.0                | 82.7539         | -9137.2617     | -588.0638    | 4.7555          | -0.0442       |
+| 0.0135        | 0.1420 | 50   | 0.0099          | -1.3212        | -47.4111         | 0.9960             | 46.0899         | -5381.9028     | -499.1044    | -2.4386         | -2.9291       |
+| 0.0109        | 0.2841 | 100  | 0.0025          | -2.5970        | -80.9884         | 1.0                | 78.3913         | -8739.6260     | -626.6862    | -0.0340         | -2.5428       |
+| 0.0017        | 0.4261 | 150  | 0.0011          | -1.9943        | -77.1591         | 1.0                | 75.1648         | -8356.6973     | -566.4090    | 2.8304          | -1.6476       |
+| 0.002         | 0.5682 | 200  | 0.0008          | -2.1292        | -82.8472         | 1.0                | 80.7180         | -8925.5107     | -579.9021    | 2.8840          | -1.3436       |
+| 0.0018        | 0.7102 | 250  | 0.0009          | -2.1417        | -72.3491         | 1.0                | 70.2074         | -7875.6992     | -581.1540    | 2.5447          | -1.1532       |
+| 0.0013        | 0.8523 | 300  | 0.0010          | -2.2050        | -73.9724         | 1.0                | 71.7674         | -8038.0322     | -587.4813    | 2.7313          | -0.4348       |
+| 0.0058        | 0.9943 | 350  | 0.0010          | -2.2051        | -74.7846         | 1.0                | 72.5795         | -8119.2456     | -587.4921    | 2.7746          | -0.3615       |
 
 
 ### Framework versions

all_results.json

@@ -1,22 +1,22 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -0.5261995196342468,
-    "eval_logits/rejected": 2.7145843505859375,
-    "eval_logps/chosen": -577.6981201171875,
-    "eval_logps/rejected": -7961.19677734375,
-    "eval_loss": 0.000621323473751545,
+    "eval_logits/chosen": -0.043021149933338165,
+    "eval_logits/rejected": 4.756043434143066,
+    "eval_logps/chosen": -587.93701171875,
+    "eval_logps/rejected": -9136.2529296875,
+    "eval_loss": 0.0007141114911064506,
     "eval_rewards/accuracies": 1.0,
-    "eval_rewards/chosen": -2.1071643829345703,
-    "eval_rewards/margins": 71.09691619873047,
-    "eval_rewards/rejected": -73.20408630371094,
-    "eval_runtime": 196.4219,
+    "eval_rewards/chosen": -2.2095537185668945,
+    "eval_rewards/margins": 82.74508666992188,
+    "eval_rewards/rejected": -84.95464324951172,
+    "eval_runtime": 192.6276,
     "eval_samples": 3905,
-    "eval_samples_per_second": 19.881,
-    "eval_steps_per_second": 0.316,
+    "eval_samples_per_second": 20.272,
+    "eval_steps_per_second": 0.322,
     "total_flos": 0.0,
-    "train_loss": 0.0941242430602539,
-    "train_runtime": 10036.5915,
+    "train_loss": 0.08915322791869006,
+    "train_runtime": 10109.537,
     "train_samples": 45000,
-    "train_samples_per_second": 4.484,
+    "train_samples_per_second": 4.451,
     "train_steps_per_second": 0.035
 }

eval_results.json

@@ -1,16 +1,16 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -0.5261995196342468,
-    "eval_logits/rejected": 2.7145843505859375,
-    "eval_logps/chosen": -577.6981201171875,
-    "eval_logps/rejected": -7961.19677734375,
-    "eval_loss": 0.000621323473751545,
+    "eval_logits/chosen": -0.043021149933338165,
+    "eval_logits/rejected": 4.756043434143066,
+    "eval_logps/chosen": -587.93701171875,
+    "eval_logps/rejected": -9136.2529296875,
+    "eval_loss": 0.0007141114911064506,
     "eval_rewards/accuracies": 1.0,
-    "eval_rewards/chosen": -2.1071643829345703,
-    "eval_rewards/margins": 71.09691619873047,
-    "eval_rewards/rejected": -73.20408630371094,
-    "eval_runtime": 196.4219,
+    "eval_rewards/chosen": -2.2095537185668945,
+    "eval_rewards/margins": 82.74508666992188,
+    "eval_rewards/rejected": -84.95464324951172,
+    "eval_runtime": 192.6276,
     "eval_samples": 3905,
-    "eval_samples_per_second": 19.881,
-    "eval_steps_per_second": 0.316
+    "eval_samples_per_second": 20.272,
+    "eval_steps_per_second": 0.322
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f45f4617ade43cbd1c152d467a79efb24b1005ab4dc580a4984340acf70f4d9a
+oid sha256:ea062bb686c221314613acef65146caf3a37869e4cc120273b6a23c064b4f51e
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:66198156d42f0ed0b2654c2fc5301e9c15d14f00a11804852a1f4d8ae40c6f8d
+oid sha256:8589631baa5d782c5f19f5553cede0e0dc4c913ec78556038707b221c4223125
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aab1ddfcd9ff288d3d156b06d15d150fceb54220ae64fab783bbcb76386844e8
+oid sha256:cf137591cd2b521550761718081f24b02337f26e652ec7f14ddaeb325d2334c7
 size 4540516344

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.0,
     "total_flos": 0.0,
-    "train_loss": 0.0941242430602539,
-    "train_runtime": 10036.5915,
+    "train_loss": 0.08915322791869006,
+    "train_runtime": 10109.537,
     "train_samples": 45000,
-    "train_samples_per_second": 4.484,
+    "train_samples_per_second": 4.451,
     "train_steps_per_second": 0.035
 }

trainer_state.json

@@ -10,638 +10,638 @@
   "log_history": [
     {
       "epoch": 0.028409090909090908,
-      "grad_norm": 63.70768739794084,
+      "grad_norm": 77.31810180796927,
       "learning_rate": 1.3888888888888888e-07,
-      "logits/chosen": -2.859254837036133,
-      "logits/rejected": -2.642847776412964,
-      "logps/chosen": -390.5564880371094,
-      "logps/rejected": -607.8642578125,
-      "loss": 1.3793,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.0020330864936113358,
-      "rewards/margins": 0.013599475845694542,
-      "rewards/rejected": -0.01156639028340578,
+      "logits/chosen": -2.8591513633728027,
+      "logits/rejected": -2.6428322792053223,
+      "logps/chosen": -390.54095458984375,
+      "logps/rejected": -607.8343505859375,
+      "loss": 1.3778,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.002188617829233408,
+      "rewards/margins": 0.013455559499561787,
+      "rewards/rejected": -0.011266940273344517,
       "step": 10
     },
     {
       "epoch": 0.056818181818181816,
-      "grad_norm": 37.53559671175899,
+      "grad_norm": 38.318260170576636,
       "learning_rate": 2.7777777777777776e-07,
-      "logits/chosen": -2.842411518096924,
-      "logits/rejected": -2.694638967514038,
-      "logps/chosen": -328.69854736328125,
-      "logps/rejected": -775.2159423828125,
-      "loss": 1.1119,
-      "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": 0.08446629345417023,
-      "rewards/margins": 0.6622289419174194,
-      "rewards/rejected": -0.5777625441551208,
+      "logits/chosen": -2.842257261276245,
+      "logits/rejected": -2.6946189403533936,
+      "logps/chosen": -328.62799072265625,
+      "logps/rejected": -775.0516357421875,
+      "loss": 1.0648,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.08517134189605713,
+      "rewards/margins": 0.6612905859947205,
+      "rewards/rejected": -0.5761191844940186,
       "step": 20
     },
     {
       "epoch": 0.08522727272727272,
-      "grad_norm": 11.233762764198163,
+      "grad_norm": 10.337672358301827,
       "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -2.8567159175872803,
-      "logits/rejected": -2.6723504066467285,
-      "logps/chosen": -296.57916259765625,
-      "logps/rejected": -1129.5751953125,
-      "loss": 0.4749,
+      "logits/chosen": -2.856661319732666,
+      "logits/rejected": -2.6734800338745117,
+      "logps/chosen": -295.8897399902344,
+      "logps/rejected": -1121.2652587890625,
+      "loss": 0.4072,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.35702842473983765,
-      "rewards/margins": 4.873236179351807,
-      "rewards/rejected": -4.516207218170166,
+      "rewards/chosen": 0.36392277479171753,
+      "rewards/margins": 4.797031402587891,
+      "rewards/rejected": -4.433108329772949,
       "step": 30
     },
     {
       "epoch": 0.11363636363636363,
-      "grad_norm": 3.6495758175953084,
+      "grad_norm": 3.166840866383729,
       "learning_rate": 4.998023493068254e-07,
-      "logits/chosen": -2.814579725265503,
-      "logits/rejected": -2.6064682006835938,
-      "logps/chosen": -323.31414794921875,
-      "logps/rejected": -2251.45361328125,
-      "loss": 0.1291,
+      "logits/chosen": -2.8248767852783203,
+      "logits/rejected": -2.627469539642334,
+      "logps/chosen": -318.3464050292969,
+      "logps/rejected": -2185.42578125,
+      "loss": 0.1013,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.23441331088542938,
-      "rewards/margins": 15.669992446899414,
-      "rewards/rejected": -15.435577392578125,
+      "rewards/chosen": 0.28409063816070557,
+      "rewards/margins": 15.0593900680542,
+      "rewards/rejected": -14.775299072265625,
       "step": 40
     },
     {
       "epoch": 0.14204545454545456,
-      "grad_norm": 0.3660230434321398,
+      "grad_norm": 1.0472115502346335,
       "learning_rate": 4.975823666181255e-07,
-      "logits/chosen": -2.7671985626220703,
-      "logits/rejected": -2.447579860687256,
-      "logps/chosen": -483.5135803222656,
-      "logps/rejected": -4943.55126953125,
-      "loss": 0.0158,
+      "logits/chosen": -2.8039865493774414,
+      "logits/rejected": -2.5030531883239746,
+      "logps/chosen": -466.212890625,
+      "logps/rejected": -4635.23095703125,
+      "loss": 0.0135,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.2087831497192383,
-      "rewards/margins": 41.68732452392578,
-      "rewards/rejected": -42.8961067199707,
+      "rewards/chosen": -1.0357766151428223,
+      "rewards/margins": 38.77712631225586,
+      "rewards/rejected": -39.81290054321289,
       "step": 50
     },
     {
       "epoch": 0.14204545454545456,
-      "eval_logits/chosen": -2.873523473739624,
-      "eval_logits/rejected": -2.389453411102295,
-      "eval_logps/chosen": -509.4383850097656,
-      "eval_logps/rejected": -5863.54931640625,
-      "eval_loss": 0.010370529256761074,
+      "eval_logits/chosen": -2.9291088581085205,
+      "eval_logits/rejected": -2.438577175140381,
+      "eval_logps/chosen": -499.10443115234375,
+      "eval_logps/rejected": -5381.90283203125,
+      "eval_loss": 0.00992405042052269,
       "eval_rewards/accuracies": 0.9959677457809448,
-      "eval_rewards/chosen": -1.4245665073394775,
-      "eval_rewards/margins": 50.803043365478516,
-      "eval_rewards/rejected": -52.22761154174805,
-      "eval_runtime": 195.8611,
-      "eval_samples_per_second": 19.938,
-      "eval_steps_per_second": 0.317,
+      "eval_rewards/chosen": -1.321226954460144,
+      "eval_rewards/margins": 46.089908599853516,
+      "eval_rewards/rejected": -47.41114044189453,
+      "eval_runtime": 194.7089,
+      "eval_samples_per_second": 20.056,
+      "eval_steps_per_second": 0.318,
       "step": 50
     },
     {
       "epoch": 0.17045454545454544,
-      "grad_norm": 0.9752123540317181,
+      "grad_norm": 0.9673005612652019,
       "learning_rate": 4.929173350101024e-07,
-      "logits/chosen": -2.865919351577759,
-      "logits/rejected": -2.1330180168151855,
-      "logps/chosen": -531.2206420898438,
-      "logps/rejected": -6913.0869140625,
-      "loss": 0.0245,
+      "logits/chosen": -2.9523978233337402,
+      "logits/rejected": -2.3117947578430176,
+      "logps/chosen": -503.43365478515625,
+      "logps/rejected": -5926.720703125,
+      "loss": 0.0114,
       "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": -1.7132558822631836,
-      "rewards/margins": 60.98906707763672,
-      "rewards/rejected": -62.70232009887695,
+      "rewards/chosen": -1.4353859424591064,
+      "rewards/margins": 51.40327835083008,
+      "rewards/rejected": -52.83866500854492,
       "step": 60
     },
     {
       "epoch": 0.19886363636363635,
-      "grad_norm": 0.478800645918279,
+      "grad_norm": 0.3988175169368107,
       "learning_rate": 4.858533249305336e-07,
-      "logits/chosen": -2.895341396331787,
-      "logits/rejected": -1.7176485061645508,
-      "logps/chosen": -551.4154052734375,
-      "logps/rejected": -7227.5439453125,
-      "loss": 0.0068,
+      "logits/chosen": -2.878004550933838,
+      "logits/rejected": -1.9669125080108643,
+      "logps/chosen": -545.6673583984375,
+      "logps/rejected": -6498.20263671875,
+      "loss": 0.0065,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.7225472927093506,
-      "rewards/margins": 64.42552185058594,
-      "rewards/rejected": -66.14808654785156,
+      "rewards/chosen": -1.6650670766830444,
+      "rewards/margins": 57.189605712890625,
+      "rewards/rejected": -58.85467529296875,
       "step": 70
     },
     {
       "epoch": 0.22727272727272727,
-      "grad_norm": 10.076839489234553,
+      "grad_norm": 13.315517602902379,
       "learning_rate": 4.764600984163808e-07,
-      "logits/chosen": -2.921323537826538,
-      "logits/rejected": -1.122173547744751,
-      "logps/chosen": -585.6353149414062,
-      "logps/rejected": -8391.6171875,
-      "loss": 0.0053,
+      "logits/chosen": -2.780555248260498,
+      "logits/rejected": -1.0796103477478027,
+      "logps/chosen": -584.189697265625,
+      "logps/rejected": -7447.61572265625,
+      "loss": 0.0056,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.7931442260742188,
-      "rewards/margins": 75.38569641113281,
-      "rewards/rejected": -77.17884826660156,
+      "rewards/chosen": -1.7786884307861328,
+      "rewards/margins": 65.96015167236328,
+      "rewards/rejected": -67.73884582519531,
       "step": 80
     },
     {
       "epoch": 0.2556818181818182,
-      "grad_norm": 0.06316207964609313,
+      "grad_norm": 0.060351047390217884,
       "learning_rate": 4.6483042014491527e-07,
-      "logits/chosen": -2.8797919750213623,
-      "logits/rejected": -1.447209119796753,
-      "logps/chosen": -596.3788452148438,
-      "logps/rejected": -8167.92724609375,
-      "loss": 0.0137,
+      "logits/chosen": -2.8976826667785645,
+      "logits/rejected": -1.835821509361267,
+      "logps/chosen": -552.1006469726562,
+      "logps/rejected": -6640.58984375,
+      "loss": 0.0148,
       "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": -2.3426554203033447,
-      "rewards/margins": 72.96277618408203,
-      "rewards/rejected": -75.305419921875,
+      "rewards/chosen": -1.8998725414276123,
+      "rewards/margins": 58.132171630859375,
+      "rewards/rejected": -60.03204345703125,
       "step": 90
     },
     {
       "epoch": 0.2840909090909091,
-      "grad_norm": 1.4636731976721238,
+      "grad_norm": 1.627678131457394,
       "learning_rate": 4.510791413176912e-07,
-      "logits/chosen": -2.9729702472686768,
-      "logits/rejected": -0.3022725284099579,
-      "logps/chosen": -607.1732177734375,
-      "logps/rejected": -10554.962890625,
-      "loss": 0.0111,
+      "logits/chosen": -2.7810730934143066,
+      "logits/rejected": -0.7030321359634399,
+      "logps/chosen": -615.0423583984375,
+      "logps/rejected": -8701.7958984375,
+      "loss": 0.0109,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.3770313262939453,
-      "rewards/margins": 96.11477661132812,
-      "rewards/rejected": -98.49180603027344,
+      "rewards/chosen": -2.455723285675049,
+      "rewards/margins": 77.5044174194336,
+      "rewards/rejected": -79.96015167236328,
       "step": 100
     },
     {
       "epoch": 0.2840909090909091,
-      "eval_logits/chosen": -2.945009231567383,
-      "eval_logits/rejected": 0.17861562967300415,
-      "eval_logps/chosen": -592.9392700195312,
-      "eval_logps/rejected": -9974.3623046875,
-      "eval_loss": 0.0023908319417387247,
+      "eval_logits/chosen": -2.5427753925323486,
+      "eval_logits/rejected": -0.03400120511651039,
+      "eval_logps/chosen": -626.6861572265625,
+      "eval_logps/rejected": -8739.6259765625,
+      "eval_loss": 0.0025400689337402582,
       "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -2.259575366973877,
-      "eval_rewards/margins": 91.07615661621094,
-      "eval_rewards/rejected": -93.33573150634766,
-      "eval_runtime": 193.9785,
-      "eval_samples_per_second": 20.131,
-      "eval_steps_per_second": 0.32,
+      "eval_rewards/chosen": -2.5970447063446045,
+      "eval_rewards/margins": 78.39134216308594,
+      "eval_rewards/rejected": -80.9883804321289,
+      "eval_runtime": 194.2924,
+      "eval_samples_per_second": 20.099,
+      "eval_steps_per_second": 0.319,
       "step": 100
     },
     {
       "epoch": 0.3125,
-      "grad_norm": 0.06640741403344876,
+      "grad_norm": 0.04122796046055686,
       "learning_rate": 4.353420654246546e-07,
-      "logits/chosen": -2.9682483673095703,
-      "logits/rejected": -0.10797711461782455,
-      "logps/chosen": -584.2022094726562,
-      "logps/rejected": -9037.5048828125,
-      "loss": 0.0062,
+      "logits/chosen": -2.4864227771759033,
+      "logits/rejected": -0.10299022495746613,
+      "logps/chosen": -605.4061279296875,
+      "logps/rejected": -7677.7255859375,
+      "loss": 0.0083,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.087810754776001,
-      "rewards/margins": 82.29454803466797,
-      "rewards/rejected": -84.3823471069336,
+      "rewards/chosen": -2.29984974861145,
+      "rewards/margins": 68.48472595214844,
+      "rewards/rejected": -70.78457641601562,
       "step": 110
     },
     {
       "epoch": 0.3409090909090909,
-      "grad_norm": 0.11961136514165396,
+      "grad_norm": 0.09070738570484652,
       "learning_rate": 4.177746070897592e-07,
-      "logits/chosen": -2.854703187942505,
-      "logits/rejected": 0.2248195856809616,
-      "logps/chosen": -577.8111572265625,
-      "logps/rejected": -8910.1435546875,
-      "loss": 0.0301,
+      "logits/chosen": -2.3363277912139893,
+      "logits/rejected": 0.5452026724815369,
+      "logps/chosen": -594.1629638671875,
+      "logps/rejected": -7349.20068359375,
+      "loss": 0.0063,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.9629430770874023,
-      "rewards/margins": 80.76622009277344,
-      "rewards/rejected": -82.72917175292969,
+      "rewards/chosen": -2.1264612674713135,
+      "rewards/margins": 64.9932632446289,
+      "rewards/rejected": -67.1197280883789,
       "step": 120
     },
     {
       "epoch": 0.3693181818181818,
-      "grad_norm": 0.44055468356671706,
+      "grad_norm": 0.4807602533089145,
       "learning_rate": 3.9855025724292763e-07,
-      "logits/chosen": -2.659201145172119,
-      "logits/rejected": 1.2345632314682007,
-      "logps/chosen": -610.489990234375,
-      "logps/rejected": -9278.4609375,
-      "loss": 0.0023,
+      "logits/chosen": -2.243814468383789,
+      "logits/rejected": 1.6017320156097412,
+      "logps/chosen": -609.6871337890625,
+      "logps/rejected": -7587.91552734375,
+      "loss": 0.0019,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.4003846645355225,
-      "rewards/margins": 83.97884368896484,
-      "rewards/rejected": -86.37923431396484,
+      "rewards/chosen": -2.392354965209961,
+      "rewards/margins": 67.08141326904297,
+      "rewards/rejected": -69.47377014160156,
       "step": 130
     },
     {
       "epoch": 0.3977272727272727,
-      "grad_norm": 2.2725494852752717,
+      "grad_norm": 1.856213293772278,
       "learning_rate": 3.7785886977585555e-07,
-      "logits/chosen": -2.1955535411834717,
-      "logits/rejected": 1.7099689245224,
-      "logps/chosen": -569.4786376953125,
-      "logps/rejected": -11541.3544921875,
-      "loss": 0.003,
+      "logits/chosen": -1.8541723489761353,
+      "logits/rejected": 2.44315505027771,
+      "logps/chosen": -582.9074096679688,
+      "logps/rejected": -9157.189453125,
+      "loss": 0.0035,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.034423589706421,
-      "rewards/margins": 106.959228515625,
-      "rewards/rejected": -108.99365234375,
+      "rewards/chosen": -2.168710231781006,
+      "rewards/margins": 82.98329162597656,
+      "rewards/rejected": -85.1520004272461,
       "step": 140
     },
     {
       "epoch": 0.42613636363636365,
-      "grad_norm": 13.991084938762679,
+      "grad_norm": 1.4955330936107978,
       "learning_rate": 3.5590478660213206e-07,
-      "logits/chosen": -1.8308794498443604,
-      "logits/rejected": 2.1075541973114014,
-      "logps/chosen": -663.3027954101562,
-      "logps/rejected": -11290.779296875,
-      "loss": 0.0039,
-      "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": -2.852395534515381,
-      "rewards/margins": 102.93562316894531,
-      "rewards/rejected": -105.78802490234375,
+      "logits/chosen": -1.5623462200164795,
+      "logits/rejected": 2.7078518867492676,
+      "logps/chosen": -628.0105590820312,
+      "logps/rejected": -8812.60546875,
+      "loss": 0.0017,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.499473810195923,
+      "rewards/margins": 78.50679779052734,
+      "rewards/rejected": -81.00627136230469,
       "step": 150
     },
     {
       "epoch": 0.42613636363636365,
-      "eval_logits/chosen": -1.883123755455017,
-      "eval_logits/rejected": 2.354902982711792,
-      "eval_logps/chosen": -600.4949340820312,
-      "eval_logps/rejected": -10689.58203125,
-      "eval_loss": 0.0015989234670996666,
+      "eval_logits/chosen": -1.6475961208343506,
+      "eval_logits/rejected": 2.8304052352905273,
+      "eval_logps/chosen": -566.4089965820312,
+      "eval_logps/rejected": -8356.697265625,
+      "eval_loss": 0.0010887953685596585,
       "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -2.3351328372955322,
-      "eval_rewards/margins": 98.15279388427734,
-      "eval_rewards/rejected": -100.48792266845703,
-      "eval_runtime": 194.1289,
-      "eval_samples_per_second": 20.116,
+      "eval_rewards/chosen": -1.9942734241485596,
+      "eval_rewards/margins": 75.1648178100586,
+      "eval_rewards/rejected": -77.15908813476562,
+      "eval_runtime": 194.5753,
+      "eval_samples_per_second": 20.069,
       "eval_steps_per_second": 0.319,
       "step": 150
     },
     {
       "epoch": 0.45454545454545453,
-      "grad_norm": 15.634259463426035,
+      "grad_norm": 8.762005226870338,
       "learning_rate": 3.3290481963801696e-07,
-      "logits/chosen": -2.0796897411346436,
-      "logits/rejected": 2.3097519874572754,
-      "logps/chosen": -567.3259887695312,
-      "logps/rejected": -9335.7529296875,
-      "loss": 0.0018,
+      "logits/chosen": -2.1097240447998047,
+      "logits/rejected": 2.4051706790924072,
+      "logps/chosen": -540.6964111328125,
+      "logps/rejected": -7950.2998046875,
+      "loss": 0.001,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.9718801975250244,
-      "rewards/margins": 85.22074890136719,
-      "rewards/rejected": -87.19263458251953,
+      "rewards/chosen": -1.705583930015564,
+      "rewards/margins": 71.63252258300781,
+      "rewards/rejected": -73.33810424804688,
       "step": 160
     },
     {
       "epoch": 0.48295454545454547,
-      "grad_norm": 0.04468151503862641,
+      "grad_norm": 0.030626366308689413,
       "learning_rate": 3.0908610963322626e-07,
-      "logits/chosen": -2.098179340362549,
-      "logits/rejected": 1.9576606750488281,
-      "logps/chosen": -608.5695190429688,
-      "logps/rejected": -9132.7138671875,
-      "loss": 0.0059,
+      "logits/chosen": -2.4956324100494385,
+      "logits/rejected": 1.2054059505462646,
+      "logps/chosen": -597.5648803710938,
+      "logps/rejected": -8411.56640625,
+      "loss": 0.006,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.2095131874084473,
-      "rewards/margins": 81.83451843261719,
-      "rewards/rejected": -84.04402160644531,
+      "rewards/chosen": -2.099465847015381,
+      "rewards/margins": 74.73308563232422,
+      "rewards/rejected": -76.83256530761719,
       "step": 170
     },
     {
       "epoch": 0.5113636363636364,
-      "grad_norm": 0.046664236017310436,
+      "grad_norm": 0.014685643153554214,
       "learning_rate": 2.846838829972671e-07,
-      "logits/chosen": -2.053903102874756,
-      "logits/rejected": 2.1386537551879883,
-      "logps/chosen": -602.9306030273438,
-      "logps/rejected": -8629.5322265625,
-      "loss": 0.0205,
+      "logits/chosen": -2.5342214107513428,
+      "logits/rejected": 1.157962441444397,
+      "logps/chosen": -590.8978271484375,
+      "logps/rejected": -7980.2666015625,
+      "loss": 0.0116,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.4202630519866943,
-      "rewards/margins": 77.7452163696289,
-      "rewards/rejected": -80.16548156738281,
+      "rewards/chosen": -2.2999348640441895,
+      "rewards/margins": 71.37288665771484,
+      "rewards/rejected": -73.67282104492188,
       "step": 180
     },
     {
       "epoch": 0.5397727272727273,
-      "grad_norm": 0.2531340520912063,
+      "grad_norm": 0.1745056386100847,
       "learning_rate": 2.5993912877423147e-07,
-      "logits/chosen": -1.8579347133636475,
-      "logits/rejected": 1.7126737833023071,
-      "logps/chosen": -551.639404296875,
-      "logps/rejected": -9325.890625,
-      "loss": 0.0012,
+      "logits/chosen": -2.2881453037261963,
+      "logits/rejected": 1.435770869255066,
+      "logps/chosen": -543.35205078125,
+      "logps/rejected": -8919.642578125,
+      "loss": 0.0011,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.060236692428589,
-      "rewards/margins": 84.2825698852539,
-      "rewards/rejected": -86.34281158447266,
+      "rewards/chosen": -1.9773623943328857,
+      "rewards/margins": 80.30296325683594,
+      "rewards/rejected": -82.28031921386719,
       "step": 190
     },
     {
       "epoch": 0.5681818181818182,
-      "grad_norm": 1.2128193169362667,
+      "grad_norm": 0.7368778028750317,
       "learning_rate": 2.3509621870754504e-07,
-      "logits/chosen": -1.699134111404419,
-      "logits/rejected": 2.7847816944122314,
-      "logps/chosen": -572.9049072265625,
-      "logps/rejected": -8628.181640625,
-      "loss": 0.0022,
+      "logits/chosen": -1.6877973079681396,
+      "logits/rejected": 3.008781671524048,
+      "logps/chosen": -565.8089599609375,
+      "logps/rejected": -8412.712890625,
+      "loss": 0.002,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.0777506828308105,
-      "rewards/margins": 77.8956527709961,
-      "rewards/rejected": -79.97340393066406,
+      "rewards/chosen": -2.0067920684814453,
+      "rewards/margins": 75.81190490722656,
+      "rewards/rejected": -77.8187026977539,
       "step": 200
     },
     {
       "epoch": 0.5681818181818182,
-      "eval_logits/chosen": -1.6725599765777588,
-      "eval_logits/rejected": 2.5107614994049072,
-      "eval_logps/chosen": -587.24755859375,
-      "eval_logps/rejected": -9258.34375,
-      "eval_loss": 0.0011689095990732312,
+      "eval_logits/chosen": -1.3436212539672852,
+      "eval_logits/rejected": 2.8839986324310303,
+      "eval_logps/chosen": -579.902099609375,
+      "eval_logps/rejected": -8925.5107421875,
+      "eval_loss": 0.000815804407466203,
       "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -2.202658176422119,
-      "eval_rewards/margins": 83.97289276123047,
-      "eval_rewards/rejected": -86.1755599975586,
-      "eval_runtime": 193.9485,
-      "eval_samples_per_second": 20.134,
-      "eval_steps_per_second": 0.32,
+      "eval_rewards/chosen": -2.1292035579681396,
+      "eval_rewards/margins": 80.71800994873047,
+      "eval_rewards/rejected": -82.84722137451172,
+      "eval_runtime": 194.6381,
+      "eval_samples_per_second": 20.063,
+      "eval_steps_per_second": 0.319,
       "step": 200
     },
     {
       "epoch": 0.5965909090909091,
-      "grad_norm": 1.5143241412178876,
+      "grad_norm": 0.9148796537082613,
       "learning_rate": 2.1040049389819624e-07,
-      "logits/chosen": -1.7269471883773804,
-      "logits/rejected": 2.757033586502075,
-      "logps/chosen": -597.5856323242188,
-      "logps/rejected": -8867.4111328125,
-      "loss": 0.0051,
+      "logits/chosen": -1.2771052122116089,
+      "logits/rejected": 3.0268094539642334,
+      "logps/chosen": -591.0138549804688,
+      "logps/rejected": -8609.7705078125,
+      "loss": 0.0067,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.0694241523742676,
-      "rewards/margins": 80.28562927246094,
-      "rewards/rejected": -82.35506439208984,
+      "rewards/chosen": -2.0037057399749756,
+      "rewards/margins": 77.77495574951172,
+      "rewards/rejected": -79.77867126464844,
       "step": 210
     },
     {
       "epoch": 0.625,
-      "grad_norm": 3.3477814516315405,
+      "grad_norm": 4.610253822471946,
       "learning_rate": 1.8609584188988133e-07,
-      "logits/chosen": -1.5605463981628418,
-      "logits/rejected": 2.3806936740875244,
-      "logps/chosen": -627.6144409179688,
-      "logps/rejected": -7452.0048828125,
-      "loss": 0.0066,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.6301381587982178,
-      "rewards/margins": 65.39354705810547,
-      "rewards/rejected": -68.02368927001953,
+      "logits/chosen": -0.9021345376968384,
+      "logits/rejected": 2.644864559173584,
+      "logps/chosen": -637.8895263671875,
+      "logps/rejected": -7473.07666015625,
+      "loss": 0.016,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": -2.732889175415039,
+      "rewards/margins": 65.50151824951172,
+      "rewards/rejected": -68.2343978881836,
       "step": 220
     },
     {
       "epoch": 0.6534090909090909,
-      "grad_norm": 86.10043361080564,
+      "grad_norm": 80.2393605437428,
       "learning_rate": 1.624222881090439e-07,
-      "logits/chosen": -1.8570022583007812,
-      "logits/rejected": 2.0753836631774902,
-      "logps/chosen": -650.1534423828125,
-      "logps/rejected": -8688.783203125,
-      "loss": 0.0216,
+      "logits/chosen": -1.08247971534729,
+      "logits/rejected": 2.224297046661377,
+      "logps/chosen": -652.9135131835938,
+      "logps/rejected": -8397.490234375,
+      "loss": 0.0294,
       "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": -2.5548887252807617,
-      "rewards/margins": 77.47712707519531,
-      "rewards/rejected": -80.03202056884766,
+      "rewards/chosen": -2.5824902057647705,
+      "rewards/margins": 74.53660583496094,
+      "rewards/rejected": -77.11909484863281,
       "step": 230
     },
     {
       "epoch": 0.6818181818181818,
-      "grad_norm": 0.6247509460136759,
+      "grad_norm": 0.967663894443262,
       "learning_rate": 1.3961362544602212e-07,
-      "logits/chosen": -1.4957786798477173,
-      "logits/rejected": 2.78035306930542,
-      "logps/chosen": -612.0357666015625,
-      "logps/rejected": -7844.0234375,
-      "loss": 0.0027,
+      "logits/chosen": -1.0073425769805908,
+      "logits/rejected": 2.469111442565918,
+      "logps/chosen": -597.539794921875,
+      "logps/rejected": -7158.27490234375,
+      "loss": 0.0022,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.360233783721924,
-      "rewards/margins": 70.1392822265625,
-      "rewards/rejected": -72.49951171875,
+      "rewards/chosen": -2.2152743339538574,
+      "rewards/margins": 63.4267463684082,
+      "rewards/rejected": -65.64202117919922,
       "step": 240
     },
     {
       "epoch": 0.7102272727272727,
-      "grad_norm": 0.8355271945684738,
+      "grad_norm": 0.5670683312609751,
       "learning_rate": 1.1789510538684522e-07,
-      "logits/chosen": -1.3467962741851807,
-      "logits/rejected": 3.0541701316833496,
-      "logps/chosen": -582.3834838867188,
-      "logps/rejected": -10293.1650390625,
-      "loss": 0.0022,
+      "logits/chosen": -1.1133817434310913,
+      "logits/rejected": 2.4663312435150146,
+      "logps/chosen": -563.9259643554688,
+      "logps/rejected": -9071.0546875,
+      "loss": 0.0018,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.3034989833831787,
-      "rewards/margins": 93.33232116699219,
-      "rewards/rejected": -95.63582611083984,
+      "rewards/chosen": -2.1189236640930176,
+      "rewards/margins": 81.29580688476562,
+      "rewards/rejected": -83.41471862792969,
       "step": 250
     },
     {
       "epoch": 0.7102272727272727,
-      "eval_logits/chosen": -1.0149785280227661,
-      "eval_logits/rejected": 3.758545160293579,
-      "eval_logps/chosen": -596.009521484375,
-      "eval_logps/rejected": -8959.7470703125,
-      "eval_loss": 0.0008426356362178922,
+      "eval_logits/chosen": -1.153204321861267,
+      "eval_logits/rejected": 2.5446887016296387,
+      "eval_logps/chosen": -581.1539916992188,
+      "eval_logps/rejected": -7875.69921875,
+      "eval_loss": 0.0009098726441152394,
       "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -2.290278196334839,
-      "eval_rewards/margins": 80.89930725097656,
-      "eval_rewards/rejected": -83.18959045410156,
-      "eval_runtime": 193.2341,
-      "eval_samples_per_second": 20.209,
-      "eval_steps_per_second": 0.321,
+      "eval_rewards/chosen": -2.1417224407196045,
+      "eval_rewards/margins": 70.20738220214844,
+      "eval_rewards/rejected": -72.34910583496094,
+      "eval_runtime": 194.9128,
+      "eval_samples_per_second": 20.035,
+      "eval_steps_per_second": 0.318,
       "step": 250
     },
     {
       "epoch": 0.7386363636363636,
-      "grad_norm": 0.3552207920354382,
+      "grad_norm": 1.0164128695484267,
       "learning_rate": 9.748121349736891e-08,
-      "logits/chosen": -0.6737051010131836,
-      "logits/rejected": 3.9108054637908936,
-      "logps/chosen": -637.5172119140625,
-      "logps/rejected": -9320.220703125,
-      "loss": 0.0011,
+      "logits/chosen": -1.0999664068222046,
+      "logits/rejected": 2.403217315673828,
+      "logps/chosen": -622.03173828125,
+      "logps/rejected": -8243.462890625,
+      "loss": 0.0014,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.5368902683258057,
-      "rewards/margins": 83.88404083251953,
-      "rewards/rejected": -86.4209213256836,
+      "rewards/chosen": -2.3820364475250244,
+      "rewards/margins": 73.27131652832031,
+      "rewards/rejected": -75.65335845947266,
       "step": 260
     },
     {
       "epoch": 0.7670454545454546,
-      "grad_norm": 0.09771710883654211,
+      "grad_norm": 0.08769848802929946,
       "learning_rate": 7.857355122839673e-08,
-      "logits/chosen": -0.7171241044998169,
-      "logits/rejected": 3.9581894874572754,
-      "logps/chosen": -603.0701293945312,
-      "logps/rejected": -8722.890625,
-      "loss": 0.0013,
+      "logits/chosen": -1.1708943843841553,
+      "logits/rejected": 2.2550511360168457,
+      "logps/chosen": -594.3570556640625,
+      "logps/rejected": -7704.54541015625,
+      "loss": 0.0016,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.222205638885498,
-      "rewards/margins": 78.31710052490234,
-      "rewards/rejected": -80.539306640625,
+      "rewards/chosen": -2.135075092315674,
+      "rewards/margins": 68.22078704833984,
+      "rewards/rejected": -70.3558578491211,
       "step": 270
     },
     {
       "epoch": 0.7954545454545454,
-      "grad_norm": 0.20312109356571723,
+      "grad_norm": 0.21922033699532475,
       "learning_rate": 6.135884496044244e-08,
-      "logits/chosen": -0.48570650815963745,
-      "logits/rejected": 4.524535655975342,
-      "logps/chosen": -585.2769775390625,
-      "logps/rejected": -8691.55859375,
-      "loss": 0.0015,
+      "logits/chosen": -0.8851491212844849,
+      "logits/rejected": 2.587214708328247,
+      "logps/chosen": -582.6546630859375,
+      "logps/rejected": -7667.8701171875,
+      "loss": 0.0019,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.0610411167144775,
-      "rewards/margins": 78.40565490722656,
-      "rewards/rejected": -80.46669006347656,
+      "rewards/chosen": -2.034817934036255,
+      "rewards/margins": 68.19500732421875,
+      "rewards/rejected": -70.22981262207031,
       "step": 280
     },
     {
       "epoch": 0.8238636363636364,
-      "grad_norm": 0.0562319233114269,
+      "grad_norm": 0.1479090204803531,
       "learning_rate": 4.600710195020982e-08,
-      "logits/chosen": -0.22825559973716736,
-      "logits/rejected": 4.695797443389893,
-      "logps/chosen": -607.1776123046875,
-      "logps/rejected": -8615.1923828125,
-      "loss": 0.0052,
+      "logits/chosen": -0.6345096826553345,
+      "logits/rejected": 2.7067768573760986,
+      "logps/chosen": -606.1185913085938,
+      "logps/rejected": -7632.70068359375,
+      "loss": 0.006,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.1803441047668457,
-      "rewards/margins": 77.55899810791016,
-      "rewards/rejected": -79.73933410644531,
+      "rewards/chosen": -2.1697540283203125,
+      "rewards/margins": 67.74466705322266,
+      "rewards/rejected": -69.91442108154297,
       "step": 290
     },
     {
       "epoch": 0.8522727272727273,
-      "grad_norm": 1.3409016161535439,
+      "grad_norm": 1.431809898038739,
       "learning_rate": 3.2669931390104374e-08,
-      "logits/chosen": -0.32437822222709656,
-      "logits/rejected": 4.543366432189941,
-      "logps/chosen": -559.5152587890625,
-      "logps/rejected": -9425.8134765625,
-      "loss": 0.001,
+      "logits/chosen": -0.6104838252067566,
+      "logits/rejected": 2.62327241897583,
+      "logps/chosen": -560.2278442382812,
+      "logps/rejected": -8326.173828125,
+      "loss": 0.0013,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.032606601715088,
-      "rewards/margins": 86.01726531982422,
-      "rewards/rejected": -88.04988098144531,
+      "rewards/chosen": -2.0397324562072754,
+      "rewards/margins": 75.01374816894531,
+      "rewards/rejected": -77.05347442626953,
       "step": 300
     },
     {
       "epoch": 0.8522727272727273,
-      "eval_logits/chosen": -0.12213863432407379,
-      "eval_logits/rejected": 4.708926677703857,
-      "eval_logps/chosen": -586.3375854492188,
-      "eval_logps/rejected": -9036.201171875,
-      "eval_loss": 0.0007051606080494821,
+      "eval_logits/chosen": -0.43478134274482727,
+      "eval_logits/rejected": 2.7312941551208496,
+      "eval_logps/chosen": -587.4812622070312,
+      "eval_logps/rejected": -8038.0322265625,
+      "eval_loss": 0.0010067835683003068,
       "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -2.1935579776763916,
-      "eval_rewards/margins": 81.76056671142578,
-      "eval_rewards/rejected": -83.95413208007812,
-      "eval_runtime": 194.0614,
-      "eval_samples_per_second": 20.122,
-      "eval_steps_per_second": 0.319,
+      "eval_rewards/chosen": -2.2049953937530518,
+      "eval_rewards/margins": 71.7674331665039,
+      "eval_rewards/rejected": -73.9724349975586,
+      "eval_runtime": 193.3426,
+      "eval_samples_per_second": 20.197,
+      "eval_steps_per_second": 0.321,
       "step": 300
     },
     {
       "epoch": 0.8806818181818182,
-      "grad_norm": 0.1502360317259883,
+      "grad_norm": 0.1560528226318461,
       "learning_rate": 2.147904716149135e-08,
-      "logits/chosen": -0.03174494951963425,
-      "logits/rejected": 4.578262805938721,
-      "logps/chosen": -568.2017822265625,
-      "logps/rejected": -9189.7119140625,
-      "loss": 0.0037,
+      "logits/chosen": -0.37615886330604553,
+      "logits/rejected": 2.69954252243042,
+      "logps/chosen": -567.6695556640625,
+      "logps/rejected": -8216.876953125,
+      "loss": 0.0046,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.061352014541626,
-      "rewards/margins": 82.83074188232422,
-      "rewards/rejected": -84.89209747314453,
+      "rewards/chosen": -2.0560302734375,
+      "rewards/margins": 73.10771942138672,
+      "rewards/rejected": -75.16374206542969,
       "step": 310
     },
     {
       "epoch": 0.9090909090909091,
-      "grad_norm": 0.12997178207019558,
+      "grad_norm": 0.024456003641593862,
       "learning_rate": 1.254496706805433e-08,
-      "logits/chosen": -0.2779918313026428,
-      "logits/rejected": 4.609086036682129,
-      "logps/chosen": -596.7515869140625,
-      "logps/rejected": -9248.6181640625,
-      "loss": 0.0005,
+      "logits/chosen": -0.5904131531715393,
+      "logits/rejected": 2.5954599380493164,
+      "logps/chosen": -595.1561279296875,
+      "logps/rejected": -8192.4501953125,
+      "loss": 0.0006,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.140784978866577,
-      "rewards/margins": 83.82556915283203,
-      "rewards/rejected": -85.96635437011719,
+      "rewards/chosen": -2.124830722808838,
+      "rewards/margins": 73.27985382080078,
+      "rewards/rejected": -75.40467834472656,
       "step": 320
     },
     {
       "epoch": 0.9375,
-      "grad_norm": 0.20243054667676932,
+      "grad_norm": 0.24568296389199945,
       "learning_rate": 5.955921395237318e-09,
-      "logits/chosen": -0.13022509217262268,
-      "logits/rejected": 4.544893741607666,
-      "logps/chosen": -555.28271484375,
-      "logps/rejected": -9316.8115234375,
-      "loss": 0.0006,
+      "logits/chosen": -0.48061466217041016,
+      "logits/rejected": 2.652150869369507,
+      "logps/chosen": -554.1439208984375,
+      "logps/rejected": -8231.5478515625,
+      "loss": 0.0007,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.0001461505889893,
-      "rewards/margins": 84.69043731689453,
-      "rewards/rejected": -86.69058990478516,
+      "rewards/chosen": -1.9887583255767822,
+      "rewards/margins": 73.84918975830078,
+      "rewards/rejected": -75.83795166015625,
       "step": 330
     },
     {
       "epoch": 0.9659090909090909,
-      "grad_norm": 0.25686140623112713,
+      "grad_norm": 0.22303732669521928,
       "learning_rate": 1.7769815745066474e-09,
-      "logits/chosen": -0.500297486782074,
-      "logits/rejected": 4.298487186431885,
-      "logps/chosen": -568.9089965820312,
-      "logps/rejected": -8478.435546875,
-      "loss": 0.0026,
+      "logits/chosen": -0.7278428673744202,
+      "logits/rejected": 2.4799537658691406,
+      "logps/chosen": -569.5009765625,
+      "logps/rejected": -7504.6611328125,
+      "loss": 0.0013,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.9992825984954834,
-      "rewards/margins": 76.69742584228516,
-      "rewards/rejected": -78.69671630859375,
+      "rewards/chosen": -2.005201816558838,
+      "rewards/margins": 66.95375061035156,
+      "rewards/rejected": -68.9589614868164,
       "step": 340
     },
     {
       "epoch": 0.9943181818181818,
-      "grad_norm": 2.1488990981394114,
+      "grad_norm": 1.783067144450784,
       "learning_rate": 4.9417557483610875e-11,
-      "logits/chosen": -0.16308510303497314,
-      "logits/rejected": 4.496396064758301,
-      "logps/chosen": -605.3216552734375,
-      "logps/rejected": -9169.884765625,
-      "loss": 0.008,
+      "logits/chosen": -0.47829318046569824,
+      "logits/rejected": 2.6286473274230957,
+      "logps/chosen": -601.5467529296875,
+      "logps/rejected": -8092.9609375,
+      "loss": 0.0058,
       "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": -2.310539484024048,
-      "rewards/margins": 82.91272735595703,
-      "rewards/rejected": -85.2232666015625,
+      "rewards/chosen": -2.272789716720581,
+      "rewards/margins": 72.18123626708984,
+      "rewards/rejected": -74.45402526855469,
       "step": 350
     },
     {
       "epoch": 0.9943181818181818,
-      "eval_logits/chosen": -0.04419805109500885,
-      "eval_logits/rejected": 4.755471229553223,
-      "eval_logps/chosen": -588.0638427734375,
-      "eval_logps/rejected": -9137.26171875,
-      "eval_loss": 0.0007233908982016146,
+      "eval_logits/chosen": -0.3615128695964813,
+      "eval_logits/rejected": 2.7746219635009766,
+      "eval_logps/chosen": -587.4920654296875,
+      "eval_logps/rejected": -8119.24560546875,
+      "eval_loss": 0.0010170801542699337,
       "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -2.2108209133148193,
-      "eval_rewards/margins": 82.75391387939453,
-      "eval_rewards/rejected": -84.96473693847656,
-      "eval_runtime": 194.5802,
-      "eval_samples_per_second": 20.069,
+      "eval_rewards/chosen": -2.205103874206543,
+      "eval_rewards/margins": 72.57946014404297,
+      "eval_rewards/rejected": -74.78457641601562,
+      "eval_runtime": 194.3785,
+      "eval_samples_per_second": 20.09,
       "eval_steps_per_second": 0.319,
       "step": 350
     },
@@ -649,9 +649,9 @@
       "epoch": 1.0,
       "step": 352,
       "total_flos": 0.0,
-      "train_loss": 0.0941242430602539,
-      "train_runtime": 10036.5915,
-      "train_samples_per_second": 4.484,
+      "train_loss": 0.08915322791869006,
+      "train_runtime": 10109.537,
+      "train_samples_per_second": 4.451,
       "train_steps_per_second": 0.035
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7478770c4681452fb8a226c8859be376547be9d21f5c658595e7281ecf7c0650
+oid sha256:1d4409e813389e25fd4bcad04907a450b0a34c4d2dbaa6266b5991ef8c160b92
 size 7608