Model save

README.md

@@ -18,15 +18,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.0006
-- Rewards/chosen: -2.1063
-- Rewards/rejected: -73.2023
+- Loss: 0.0007
+- Rewards/chosen: -2.2108
+- Rewards/rejected: -84.9647
 - Rewards/accuracies: 1.0
-- Rewards/margins: 71.0960
-- Logps/rejected: -7961.0239
-- Logps/chosen: -577.6113
-- Logits/rejected: 2.7152
-- Logits/chosen: -0.5258
+- Rewards/margins: 82.7539
+- Logps/rejected: -9137.2617
+- Logps/chosen: -588.0638
+- Logits/rejected: 4.7555
+- Logits/chosen: -0.0442
 
 ## Model description
 
@@ -63,13 +63,13 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.011         | 0.1420 | 50   | 0.0091          | -0.9203        | -43.3611         | 0.9960             | 42.4408         | -4976.9014     | -459.0154    | -2.5410         | -2.9653       |
-| 0.0029        | 0.2841 | 100  | 0.0016          | -1.9179        | -61.7743         | 1.0                | 59.8564         | -6818.2158     | -558.7726    | 0.0358          | -2.5257       |
-| 0.0023        | 0.4261 | 150  | 0.0012          | -1.9106        | -69.0581         | 1.0                | 67.1475         | -7546.6006     | -558.0397    | 1.8614          | -2.4309       |
-| 0.0037        | 0.5682 | 200  | 0.0009          | -1.9357        | -71.7717         | 1.0                | 69.8360         | -7817.9595     | -560.5506    | 2.4905          | -1.0992       |
-| 0.0015        | 0.7102 | 250  | 0.0009          | -2.1266        | -66.6026         | 1.0                | 64.4760         | -7301.0474     | -579.6391    | 2.6537          | -0.9365       |
-| 0.0017        | 0.8523 | 300  | 0.0007          | -2.0868        | -72.2794         | 1.0                | 70.1926         | -7868.7246     | -575.6600    | 2.6898          | -0.6080       |
-| 0.0036        | 0.9943 | 350  | 0.0006          | -2.1063        | -73.2023         | 1.0                | 71.0960         | -7961.0239     | -577.6113    | 2.7152          | -0.5258       |
+| 0.0158        | 0.1420 | 50   | 0.0104          | -1.4246        | -52.2276         | 0.9960             | 50.8030         | -5863.5493     | -509.4384    | -2.3895         | -2.8735       |
+| 0.0111        | 0.2841 | 100  | 0.0024          | -2.2596        | -93.3357         | 1.0                | 91.0762         | -9974.3623     | -592.9393    | 0.1786          | -2.9450       |
+| 0.0039        | 0.4261 | 150  | 0.0016          | -2.3351        | -100.4879        | 1.0                | 98.1528         | -10689.5820    | -600.4949    | 2.3549          | -1.8831       |
+| 0.0022        | 0.5682 | 200  | 0.0012          | -2.2027        | -86.1756         | 1.0                | 83.9729         | -9258.3438     | -587.2476    | 2.5108          | -1.6726       |
+| 0.0022        | 0.7102 | 250  | 0.0008          | -2.2903        | -83.1896         | 1.0                | 80.8993         | -8959.7471     | -596.0095    | 3.7585          | -1.0150       |
+| 0.001         | 0.8523 | 300  | 0.0007          | -2.1936        | -83.9541         | 1.0                | 81.7606         | -9036.2012     | -586.3376    | 4.7089          | -0.1221       |
+| 0.008         | 0.9943 | 350  | 0.0007          | -2.2108        | -84.9647         | 1.0                | 82.7539         | -9137.2617     | -588.0638    | 4.7555          | -0.0442       |
 
 
 ### Framework versions

all_results.json

@@ -1,22 +1,22 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -0.21594780683517456,
-    "eval_logits/rejected": 2.576901435852051,
-    "eval_logps/chosen": -609.2966918945312,
-    "eval_logps/rejected": -7672.10595703125,
-    "eval_loss": 7.102515519363806e-05,
+    "eval_logits/chosen": -0.5261995196342468,
+    "eval_logits/rejected": 2.7145843505859375,
+    "eval_logps/chosen": -577.6981201171875,
+    "eval_logps/rejected": -7961.19677734375,
+    "eval_loss": 0.000621323473751545,
     "eval_rewards/accuracies": 1.0,
-    "eval_rewards/chosen": -2.423149824142456,
-    "eval_rewards/margins": 67.89002990722656,
-    "eval_rewards/rejected": -70.31317901611328,
-    "eval_runtime": 191.0845,
+    "eval_rewards/chosen": -2.1071643829345703,
+    "eval_rewards/margins": 71.09691619873047,
+    "eval_rewards/rejected": -73.20408630371094,
+    "eval_runtime": 196.4219,
     "eval_samples": 3905,
-    "eval_samples_per_second": 20.436,
-    "eval_steps_per_second": 0.324,
+    "eval_samples_per_second": 19.881,
+    "eval_steps_per_second": 0.316,
     "total_flos": 0.0,
-    "train_loss": 0.08502711300074721,
-    "train_runtime": 9968.7204,
+    "train_loss": 0.0941242430602539,
+    "train_runtime": 10036.5915,
     "train_samples": 45000,
-    "train_samples_per_second": 4.514,
+    "train_samples_per_second": 4.484,
     "train_steps_per_second": 0.035
 }

eval_results.json

@@ -1,16 +1,16 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -0.21594780683517456,
-    "eval_logits/rejected": 2.576901435852051,
-    "eval_logps/chosen": -609.2966918945312,
-    "eval_logps/rejected": -7672.10595703125,
-    "eval_loss": 7.102515519363806e-05,
+    "eval_logits/chosen": -0.5261995196342468,
+    "eval_logits/rejected": 2.7145843505859375,
+    "eval_logps/chosen": -577.6981201171875,
+    "eval_logps/rejected": -7961.19677734375,
+    "eval_loss": 0.000621323473751545,
     "eval_rewards/accuracies": 1.0,
-    "eval_rewards/chosen": -2.423149824142456,
-    "eval_rewards/margins": 67.89002990722656,
-    "eval_rewards/rejected": -70.31317901611328,
-    "eval_runtime": 191.0845,
+    "eval_rewards/chosen": -2.1071643829345703,
+    "eval_rewards/margins": 71.09691619873047,
+    "eval_rewards/rejected": -73.20408630371094,
+    "eval_runtime": 196.4219,
     "eval_samples": 3905,
-    "eval_samples_per_second": 20.436,
-    "eval_steps_per_second": 0.324
+    "eval_samples_per_second": 19.881,
+    "eval_steps_per_second": 0.316
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:19a4d988d4c8ea906c6a4a6a87b07fe045301edf48e28839de9be0c286ebc349
+oid sha256:f45f4617ade43cbd1c152d467a79efb24b1005ab4dc580a4984340acf70f4d9a
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:221c5b2974706b5866fe55da9a903f9aa856e047c987867494ba6d94c0e033db
+oid sha256:66198156d42f0ed0b2654c2fc5301e9c15d14f00a11804852a1f4d8ae40c6f8d
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ffcf7851352d6a651c1aa0629b34d9e52a84887ba228a345c457ef2d14f17122
+oid sha256:aab1ddfcd9ff288d3d156b06d15d150fceb54220ae64fab783bbcb76386844e8
 size 4540516344

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.0,
     "total_flos": 0.0,
-    "train_loss": 0.08502711300074721,
-    "train_runtime": 9968.7204,
+    "train_loss": 0.0941242430602539,
+    "train_runtime": 10036.5915,
     "train_samples": 45000,
-    "train_samples_per_second": 4.514,
+    "train_samples_per_second": 4.484,
     "train_steps_per_second": 0.035
 }

trainer_state.json

@@ -10,648 +10,648 @@
   "log_history": [
     {
       "epoch": 0.028409090909090908,
-      "grad_norm": 90.01349453953418,
+      "grad_norm": 63.70768739794084,
       "learning_rate": 1.3888888888888888e-07,
-      "logits/chosen": -2.8592569828033447,
-      "logits/rejected": -2.642957925796509,
-      "logps/chosen": -390.56573486328125,
-      "logps/rejected": -607.8802490234375,
-      "loss": 1.3762,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.001940618036314845,
-      "rewards/margins": 0.013666175305843353,
-      "rewards/rejected": -0.011725558899343014,
+      "logits/chosen": -2.859254837036133,
+      "logits/rejected": -2.642847776412964,
+      "logps/chosen": -390.5564880371094,
+      "logps/rejected": -607.8642578125,
+      "loss": 1.3793,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.0020330864936113358,
+      "rewards/margins": 0.013599475845694542,
+      "rewards/rejected": -0.01156639028340578,
       "step": 10
     },
     {
       "epoch": 0.056818181818181816,
-      "grad_norm": 38.16702665689489,
+      "grad_norm": 37.53559671175899,
       "learning_rate": 2.7777777777777776e-07,
-      "logits/chosen": -2.842255115509033,
-      "logits/rejected": -2.694774627685547,
-      "logps/chosen": -328.5555419921875,
-      "logps/rejected": -774.9117431640625,
-      "loss": 1.0262,
+      "logits/chosen": -2.842411518096924,
+      "logits/rejected": -2.694638967514038,
+      "logps/chosen": -328.69854736328125,
+      "logps/rejected": -775.2159423828125,
+      "loss": 1.1119,
       "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": 0.08589581400156021,
-      "rewards/margins": 0.660616934299469,
-      "rewards/rejected": -0.5747210383415222,
+      "rewards/chosen": 0.08446629345417023,
+      "rewards/margins": 0.6622289419174194,
+      "rewards/rejected": -0.5777625441551208,
       "step": 20
     },
     {
       "epoch": 0.08522727272727272,
-      "grad_norm": 9.474354712422207,
+      "grad_norm": 11.233762764198163,
       "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -2.857499837875366,
-      "logits/rejected": -2.675124406814575,
-      "logps/chosen": -295.38397216796875,
-      "logps/rejected": -1112.8095703125,
-      "loss": 0.3585,
+      "logits/chosen": -2.8567159175872803,
+      "logits/rejected": -2.6723504066467285,
+      "logps/chosen": -296.57916259765625,
+      "logps/rejected": -1129.5751953125,
+      "loss": 0.4749,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.36898040771484375,
-      "rewards/margins": 4.717532157897949,
-      "rewards/rejected": -4.3485517501831055,
+      "rewards/chosen": 0.35702842473983765,
+      "rewards/margins": 4.873236179351807,
+      "rewards/rejected": -4.516207218170166,
       "step": 30
     },
     {
       "epoch": 0.11363636363636363,
-      "grad_norm": 2.844664039016391,
+      "grad_norm": 3.6495758175953084,
       "learning_rate": 4.998023493068254e-07,
-      "logits/chosen": -2.8356635570526123,
-      "logits/rejected": -2.645285129547119,
-      "logps/chosen": -314.98876953125,
-      "logps/rejected": -2130.987060546875,
-      "loss": 0.083,
+      "logits/chosen": -2.814579725265503,
+      "logits/rejected": -2.6064682006835938,
+      "logps/chosen": -323.31414794921875,
+      "logps/rejected": -2251.45361328125,
+      "loss": 0.1291,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.3176669478416443,
-      "rewards/margins": 14.54858112335205,
-      "rewards/rejected": -14.230911254882812,
+      "rewards/chosen": 0.23441331088542938,
+      "rewards/margins": 15.669992446899414,
+      "rewards/rejected": -15.435577392578125,
       "step": 40
     },
     {
       "epoch": 0.14204545454545456,
-      "grad_norm": 0.18581946577481065,
+      "grad_norm": 0.3660230434321398,
       "learning_rate": 4.975823666181255e-07,
-      "logits/chosen": -2.8282175064086914,
-      "logits/rejected": -2.554896831512451,
-      "logps/chosen": -441.171875,
-      "logps/rejected": -4335.5703125,
-      "loss": 0.011,
+      "logits/chosen": -2.7671985626220703,
+      "logits/rejected": -2.447579860687256,
+      "logps/chosen": -483.5135803222656,
+      "logps/rejected": -4943.55126953125,
+      "loss": 0.0158,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.785366415977478,
-      "rewards/margins": 36.03093338012695,
-      "rewards/rejected": -36.8162956237793,
+      "rewards/chosen": -1.2087831497192383,
+      "rewards/margins": 41.68732452392578,
+      "rewards/rejected": -42.8961067199707,
       "step": 50
     },
     {
       "epoch": 0.14204545454545456,
-      "eval_logits/chosen": -2.965341806411743,
-      "eval_logits/rejected": -2.5409843921661377,
-      "eval_logps/chosen": -459.0153503417969,
-      "eval_logps/rejected": -4976.9013671875,
-      "eval_loss": 0.009123104624450207,
+      "eval_logits/chosen": -2.873523473739624,
+      "eval_logits/rejected": -2.389453411102295,
+      "eval_logps/chosen": -509.4383850097656,
+      "eval_logps/rejected": -5863.54931640625,
+      "eval_loss": 0.010370529256761074,
       "eval_rewards/accuracies": 0.9959677457809448,
-      "eval_rewards/chosen": -0.9203360676765442,
-      "eval_rewards/margins": 42.44078063964844,
-      "eval_rewards/rejected": -43.361122131347656,
-      "eval_runtime": 195.4208,
-      "eval_samples_per_second": 19.983,
+      "eval_rewards/chosen": -1.4245665073394775,
+      "eval_rewards/margins": 50.803043365478516,
+      "eval_rewards/rejected": -52.22761154174805,
+      "eval_runtime": 195.8611,
+      "eval_samples_per_second": 19.938,
       "eval_steps_per_second": 0.317,
       "step": 50
     },
     {
       "epoch": 0.17045454545454544,
-      "grad_norm": 1.8496563592278583,
+      "grad_norm": 0.9752123540317181,
       "learning_rate": 4.929173350101024e-07,
-      "logits/chosen": -3.018855571746826,
-      "logits/rejected": -2.489605188369751,
-      "logps/chosen": -478.2201232910156,
-      "logps/rejected": -5343.02783203125,
-      "loss": 0.0066,
+      "logits/chosen": -2.865919351577759,
+      "logits/rejected": -2.1330180168151855,
+      "logps/chosen": -531.2206420898438,
+      "logps/rejected": -6913.0869140625,
+      "loss": 0.0245,
       "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": -1.1832506656646729,
-      "rewards/margins": 45.8184814453125,
-      "rewards/rejected": -47.00173568725586,
+      "rewards/chosen": -1.7132558822631836,
+      "rewards/margins": 60.98906707763672,
+      "rewards/rejected": -62.70232009887695,
       "step": 60
     },
     {
       "epoch": 0.19886363636363635,
-      "grad_norm": 0.42838497413063575,
+      "grad_norm": 0.478800645918279,
       "learning_rate": 4.858533249305336e-07,
-      "logits/chosen": -2.8961918354034424,
-      "logits/rejected": -1.8984692096710205,
-      "logps/chosen": -537.8681640625,
-      "logps/rejected": -5729.29150390625,
-      "loss": 0.0061,
+      "logits/chosen": -2.895341396331787,
+      "logits/rejected": -1.7176485061645508,
+      "logps/chosen": -551.4154052734375,
+      "logps/rejected": -7227.5439453125,
+      "loss": 0.0068,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.587074875831604,
-      "rewards/margins": 49.57848358154297,
-      "rewards/rejected": -51.165557861328125,
+      "rewards/chosen": -1.7225472927093506,
+      "rewards/margins": 64.42552185058594,
+      "rewards/rejected": -66.14808654785156,
       "step": 70
     },
     {
       "epoch": 0.22727272727272727,
-      "grad_norm": 23.183269598091908,
+      "grad_norm": 10.076839489234553,
       "learning_rate": 4.764600984163808e-07,
-      "logits/chosen": -3.015650510787964,
-      "logits/rejected": -1.4822012186050415,
-      "logps/chosen": -574.1959838867188,
-      "logps/rejected": -7196.5283203125,
-      "loss": 0.007,
+      "logits/chosen": -2.921323537826538,
+      "logits/rejected": -1.122173547744751,
+      "logps/chosen": -585.6353149414062,
+      "logps/rejected": -8391.6171875,
+      "loss": 0.0053,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.6787515878677368,
-      "rewards/margins": 63.5491943359375,
-      "rewards/rejected": -65.22795104980469,
+      "rewards/chosen": -1.7931442260742188,
+      "rewards/margins": 75.38569641113281,
+      "rewards/rejected": -77.17884826660156,
       "step": 80
     },
     {
       "epoch": 0.2556818181818182,
-      "grad_norm": 0.2590854602849931,
+      "grad_norm": 0.06316207964609313,
       "learning_rate": 4.6483042014491527e-07,
-      "logits/chosen": -2.880042314529419,
-      "logits/rejected": -1.109546422958374,
-      "logps/chosen": -574.42529296875,
-      "logps/rejected": -6483.9189453125,
-      "loss": 0.0194,
+      "logits/chosen": -2.8797919750213623,
+      "logits/rejected": -1.447209119796753,
+      "logps/chosen": -596.3788452148438,
+      "logps/rejected": -8167.92724609375,
+      "loss": 0.0137,
       "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": -2.123119831085205,
-      "rewards/margins": 56.34221267700195,
-      "rewards/rejected": -58.4653434753418,
+      "rewards/chosen": -2.3426554203033447,
+      "rewards/margins": 72.96277618408203,
+      "rewards/rejected": -75.305419921875,
       "step": 90
     },
     {
       "epoch": 0.2840909090909091,
-      "grad_norm": 0.2975727339596996,
+      "grad_norm": 1.4636731976721238,
       "learning_rate": 4.510791413176912e-07,
-      "logits/chosen": -2.7033655643463135,
-      "logits/rejected": -0.46150803565979004,
-      "logps/chosen": -574.6309814453125,
-      "logps/rejected": -7050.77099609375,
-      "loss": 0.0029,
+      "logits/chosen": -2.9729702472686768,
+      "logits/rejected": -0.3022725284099579,
+      "logps/chosen": -607.1732177734375,
+      "logps/rejected": -10554.962890625,
+      "loss": 0.0111,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.0516088008880615,
-      "rewards/margins": 61.398292541503906,
-      "rewards/rejected": -63.44989776611328,
+      "rewards/chosen": -2.3770313262939453,
+      "rewards/margins": 96.11477661132812,
+      "rewards/rejected": -98.49180603027344,
       "step": 100
     },
     {
       "epoch": 0.2840909090909091,
-      "eval_logits/chosen": -2.5256500244140625,
-      "eval_logits/rejected": 0.035765551030635834,
-      "eval_logps/chosen": -558.7725830078125,
-      "eval_logps/rejected": -6818.2158203125,
-      "eval_loss": 0.0016336780972778797,
+      "eval_logits/chosen": -2.945009231567383,
+      "eval_logits/rejected": 0.17861562967300415,
+      "eval_logps/chosen": -592.9392700195312,
+      "eval_logps/rejected": -9974.3623046875,
+      "eval_loss": 0.0023908319417387247,
       "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -1.9179083108901978,
-      "eval_rewards/margins": 59.856361389160156,
-      "eval_rewards/rejected": -61.77427673339844,
-      "eval_runtime": 192.2507,
-      "eval_samples_per_second": 20.312,
-      "eval_steps_per_second": 0.322,
+      "eval_rewards/chosen": -2.259575366973877,
+      "eval_rewards/margins": 91.07615661621094,
+      "eval_rewards/rejected": -93.33573150634766,
+      "eval_runtime": 193.9785,
+      "eval_samples_per_second": 20.131,
+      "eval_steps_per_second": 0.32,
       "step": 100
     },
     {
       "epoch": 0.3125,
-      "grad_norm": 0.04984017585029643,
+      "grad_norm": 0.06640741403344876,
       "learning_rate": 4.353420654246546e-07,
-      "logits/chosen": -2.2970995903015137,
-      "logits/rejected": 0.2104659080505371,
-      "logps/chosen": -568.5081787109375,
-      "logps/rejected": -6664.22412109375,
-      "loss": 0.0043,
+      "logits/chosen": -2.9682483673095703,
+      "logits/rejected": -0.10797711461782455,
+      "logps/chosen": -584.2022094726562,
+      "logps/rejected": -9037.5048828125,
+      "loss": 0.0062,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.930869460105896,
-      "rewards/margins": 58.71869659423828,
-      "rewards/rejected": -60.649559020996094,
+      "rewards/chosen": -2.087810754776001,
+      "rewards/margins": 82.29454803466797,
+      "rewards/rejected": -84.3823471069336,
       "step": 110
     },
     {
       "epoch": 0.3409090909090909,
-      "grad_norm": 0.03797940797804543,
+      "grad_norm": 0.11961136514165396,
       "learning_rate": 4.177746070897592e-07,
-      "logits/chosen": -1.7653045654296875,
-      "logits/rejected": 1.1280748844146729,
-      "logps/chosen": -600.4093017578125,
-      "logps/rejected": -7361.03662109375,
-      "loss": 0.0084,
+      "logits/chosen": -2.854703187942505,
+      "logits/rejected": 0.2248195856809616,
+      "logps/chosen": -577.8111572265625,
+      "logps/rejected": -8910.1435546875,
+      "loss": 0.0301,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.188924789428711,
-      "rewards/margins": 65.04915618896484,
-      "rewards/rejected": -67.23808288574219,
+      "rewards/chosen": -1.9629430770874023,
+      "rewards/margins": 80.76622009277344,
+      "rewards/rejected": -82.72917175292969,
       "step": 120
     },
     {
       "epoch": 0.3693181818181818,
-      "grad_norm": 0.47736957873502217,
+      "grad_norm": 0.44055468356671706,
       "learning_rate": 3.9855025724292763e-07,
-      "logits/chosen": -2.10666823387146,
-      "logits/rejected": 1.7087266445159912,
-      "logps/chosen": -594.9124755859375,
-      "logps/rejected": -7779.0634765625,
-      "loss": 0.003,
+      "logits/chosen": -2.659201145172119,
+      "logits/rejected": 1.2345632314682007,
+      "logps/chosen": -610.489990234375,
+      "logps/rejected": -9278.4609375,
+      "loss": 0.0023,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.2446084022521973,
-      "rewards/margins": 69.14064025878906,
-      "rewards/rejected": -71.38525390625,
+      "rewards/chosen": -2.4003846645355225,
+      "rewards/margins": 83.97884368896484,
+      "rewards/rejected": -86.37923431396484,
       "step": 130
     },
     {
       "epoch": 0.3977272727272727,
-      "grad_norm": 1.020605288874105,
+      "grad_norm": 2.2725494852752717,
       "learning_rate": 3.7785886977585555e-07,
-      "logits/chosen": -2.2910099029541016,
-      "logits/rejected": 1.9655479192733765,
-      "logps/chosen": -569.71533203125,
-      "logps/rejected": -8695.2900390625,
-      "loss": 0.0045,
+      "logits/chosen": -2.1955535411834717,
+      "logits/rejected": 1.7099689245224,
+      "logps/chosen": -569.4786376953125,
+      "logps/rejected": -11541.3544921875,
+      "loss": 0.003,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.0367894172668457,
-      "rewards/margins": 78.4962158203125,
-      "rewards/rejected": -80.53299713134766,
+      "rewards/chosen": -2.034423589706421,
+      "rewards/margins": 106.959228515625,
+      "rewards/rejected": -108.99365234375,
       "step": 140
     },
     {
       "epoch": 0.42613636363636365,
-      "grad_norm": 0.5904857854916888,
+      "grad_norm": 13.991084938762679,
       "learning_rate": 3.5590478660213206e-07,
-      "logits/chosen": -2.2953405380249023,
-      "logits/rejected": 1.9216325283050537,
-      "logps/chosen": -595.8374633789062,
-      "logps/rejected": -8074.73974609375,
-      "loss": 0.0023,
+      "logits/chosen": -1.8308794498443604,
+      "logits/rejected": 2.1075541973114014,
+      "logps/chosen": -663.3027954101562,
+      "logps/rejected": -11290.779296875,
+      "loss": 0.0039,
       "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": -2.1777424812316895,
-      "rewards/margins": 71.44985961914062,
-      "rewards/rejected": -73.62760925292969,
+      "rewards/chosen": -2.852395534515381,
+      "rewards/margins": 102.93562316894531,
+      "rewards/rejected": -105.78802490234375,
       "step": 150
     },
     {
       "epoch": 0.42613636363636365,
-      "eval_logits/chosen": -2.4309346675872803,
-      "eval_logits/rejected": 1.8614341020584106,
-      "eval_logps/chosen": -558.0396728515625,
-      "eval_logps/rejected": -7546.6005859375,
-      "eval_loss": 0.001163232373073697,
+      "eval_logits/chosen": -1.883123755455017,
+      "eval_logits/rejected": 2.354902982711792,
+      "eval_logps/chosen": -600.4949340820312,
+      "eval_logps/rejected": -10689.58203125,
+      "eval_loss": 0.0015989234670996666,
       "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -1.9105795621871948,
-      "eval_rewards/margins": 67.14754486083984,
-      "eval_rewards/rejected": -69.05812072753906,
-      "eval_runtime": 193.8607,
-      "eval_samples_per_second": 20.143,
-      "eval_steps_per_second": 0.32,
+      "eval_rewards/chosen": -2.3351328372955322,
+      "eval_rewards/margins": 98.15279388427734,
+      "eval_rewards/rejected": -100.48792266845703,
+      "eval_runtime": 194.1289,
+      "eval_samples_per_second": 20.116,
+      "eval_steps_per_second": 0.319,
       "step": 150
     },
     {
       "epoch": 0.45454545454545453,
-      "grad_norm": 5.391869442805235,
+      "grad_norm": 15.634259463426035,
       "learning_rate": 3.3290481963801696e-07,
-      "logits/chosen": -2.435987949371338,
-      "logits/rejected": 2.048819065093994,
-      "logps/chosen": -532.9867553710938,
-      "logps/rejected": -7208.0419921875,
-      "loss": 0.0007,
+      "logits/chosen": -2.0796897411346436,
+      "logits/rejected": 2.3097519874572754,
+      "logps/chosen": -567.3259887695312,
+      "logps/rejected": -9335.7529296875,
+      "loss": 0.0018,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.628488540649414,
-      "rewards/margins": 64.28703308105469,
-      "rewards/rejected": -65.91552734375,
+      "rewards/chosen": -1.9718801975250244,
+      "rewards/margins": 85.22074890136719,
+      "rewards/rejected": -87.19263458251953,
       "step": 160
     },
     {
       "epoch": 0.48295454545454547,
-      "grad_norm": 0.031936510635288025,
+      "grad_norm": 0.04468151503862641,
       "learning_rate": 3.0908610963322626e-07,
-      "logits/chosen": -2.173886775970459,
-      "logits/rejected": 2.0431816577911377,
-      "logps/chosen": -577.22314453125,
-      "logps/rejected": -7725.6796875,
-      "loss": 0.0074,
+      "logits/chosen": -2.098179340362549,
+      "logits/rejected": 1.9576606750488281,
+      "logps/chosen": -608.5695190429688,
+      "logps/rejected": -9132.7138671875,
+      "loss": 0.0059,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.896048903465271,
-      "rewards/margins": 68.07765197753906,
-      "rewards/rejected": -69.97369384765625,
+      "rewards/chosen": -2.2095131874084473,
+      "rewards/margins": 81.83451843261719,
+      "rewards/rejected": -84.04402160644531,
       "step": 170
     },
     {
       "epoch": 0.5113636363636364,
-      "grad_norm": 0.018094668159337083,
+      "grad_norm": 0.046664236017310436,
       "learning_rate": 2.846838829972671e-07,
-      "logits/chosen": -1.9897123575210571,
-      "logits/rejected": 2.31811785697937,
-      "logps/chosen": -567.0270385742188,
-      "logps/rejected": -7191.4697265625,
-      "loss": 0.0021,
+      "logits/chosen": -2.053903102874756,
+      "logits/rejected": 2.1386537551879883,
+      "logps/chosen": -602.9306030273438,
+      "logps/rejected": -8629.5322265625,
+      "loss": 0.0205,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.061227798461914,
-      "rewards/margins": 63.72364044189453,
-      "rewards/rejected": -65.78486633300781,
+      "rewards/chosen": -2.4202630519866943,
+      "rewards/margins": 77.7452163696289,
+      "rewards/rejected": -80.16548156738281,
       "step": 180
     },
     {
       "epoch": 0.5397727272727273,
-      "grad_norm": 0.3615128328570267,
+      "grad_norm": 0.2531340520912063,
       "learning_rate": 2.5993912877423147e-07,
-      "logits/chosen": -1.5569745302200317,
-      "logits/rejected": 1.8910541534423828,
-      "logps/chosen": -519.6112670898438,
-      "logps/rejected": -8060.75390625,
-      "loss": 0.0013,
+      "logits/chosen": -1.8579347133636475,
+      "logits/rejected": 1.7126737833023071,
+      "logps/chosen": -551.639404296875,
+      "logps/rejected": -9325.890625,
+      "loss": 0.0012,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.7399555444717407,
-      "rewards/margins": 71.95148468017578,
-      "rewards/rejected": -73.69144439697266,
+      "rewards/chosen": -2.060236692428589,
+      "rewards/margins": 84.2825698852539,
+      "rewards/rejected": -86.34281158447266,
       "step": 190
     },
     {
       "epoch": 0.5681818181818182,
-      "grad_norm": 1.460001191569848,
+      "grad_norm": 1.2128193169362667,
       "learning_rate": 2.3509621870754504e-07,
-      "logits/chosen": -1.1832640171051025,
-      "logits/rejected": 2.8146424293518066,
-      "logps/chosen": -540.25,
-      "logps/rejected": -7328.6376953125,
-      "loss": 0.0037,
+      "logits/chosen": -1.699134111404419,
+      "logits/rejected": 2.7847816944122314,
+      "logps/chosen": -572.9049072265625,
+      "logps/rejected": -8628.181640625,
+      "loss": 0.0022,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.751201868057251,
-      "rewards/margins": 65.22674560546875,
-      "rewards/rejected": -66.97795104980469,
+      "rewards/chosen": -2.0777506828308105,
+      "rewards/margins": 77.8956527709961,
+      "rewards/rejected": -79.97340393066406,
       "step": 200
     },
     {
       "epoch": 0.5681818181818182,
-      "eval_logits/chosen": -1.0991721153259277,
-      "eval_logits/rejected": 2.4905478954315186,
-      "eval_logps/chosen": -560.5505981445312,
-      "eval_logps/rejected": -7817.95947265625,
-      "eval_loss": 0.0009270149166695774,
+      "eval_logits/chosen": -1.6725599765777588,
+      "eval_logits/rejected": 2.5107614994049072,
+      "eval_logps/chosen": -587.24755859375,
+      "eval_logps/rejected": -9258.34375,
+      "eval_loss": 0.0011689095990732312,
       "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -1.9356889724731445,
-      "eval_rewards/margins": 69.83601379394531,
-      "eval_rewards/rejected": -71.7717056274414,
-      "eval_runtime": 195.666,
-      "eval_samples_per_second": 19.957,
-      "eval_steps_per_second": 0.317,
+      "eval_rewards/chosen": -2.202658176422119,
+      "eval_rewards/margins": 83.97289276123047,
+      "eval_rewards/rejected": -86.1755599975586,
+      "eval_runtime": 193.9485,
+      "eval_samples_per_second": 20.134,
+      "eval_steps_per_second": 0.32,
       "step": 200
     },
     {
       "epoch": 0.5965909090909091,
-      "grad_norm": 0.6173080616307016,
+      "grad_norm": 1.5143241412178876,
       "learning_rate": 2.1040049389819624e-07,
-      "logits/chosen": -1.0591375827789307,
-      "logits/rejected": 2.6843762397766113,
-      "logps/chosen": -585.5335083007812,
-      "logps/rejected": -7571.32666015625,
-      "loss": 0.0065,
+      "logits/chosen": -1.7269471883773804,
+      "logits/rejected": 2.757033586502075,
+      "logps/chosen": -597.5856323242188,
+      "logps/rejected": -8867.4111328125,
+      "loss": 0.0051,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.9489023685455322,
-      "rewards/margins": 67.44532775878906,
-      "rewards/rejected": -69.39422607421875,
+      "rewards/chosen": -2.0694241523742676,
+      "rewards/margins": 80.28562927246094,
+      "rewards/rejected": -82.35506439208984,
       "step": 210
     },
     {
       "epoch": 0.625,
-      "grad_norm": 3.3870532652701733,
+      "grad_norm": 3.3477814516315405,
       "learning_rate": 1.8609584188988133e-07,
-      "logits/chosen": -0.8726997375488281,
-      "logits/rejected": 2.3560879230499268,
-      "logps/chosen": -635.2283935546875,
-      "logps/rejected": -6522.76708984375,
-      "loss": 0.0054,
+      "logits/chosen": -1.5605463981628418,
+      "logits/rejected": 2.3806936740875244,
+      "logps/chosen": -627.6144409179688,
+      "logps/rejected": -7452.0048828125,
+      "loss": 0.0066,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.706278085708618,
-      "rewards/margins": 56.02503204345703,
-      "rewards/rejected": -58.73130416870117,
+      "rewards/chosen": -2.6301381587982178,
+      "rewards/margins": 65.39354705810547,
+      "rewards/rejected": -68.02368927001953,
       "step": 220
     },
     {
       "epoch": 0.6534090909090909,
-      "grad_norm": 84.74544834753664,
+      "grad_norm": 86.10043361080564,
       "learning_rate": 1.624222881090439e-07,
-      "logits/chosen": -1.068495512008667,
-      "logits/rejected": 2.1958107948303223,
-      "logps/chosen": -655.9327392578125,
-      "logps/rejected": -7632.9365234375,
-      "loss": 0.0224,
+      "logits/chosen": -1.8570022583007812,
+      "logits/rejected": 2.0753836631774902,
+      "logps/chosen": -650.1534423828125,
+      "logps/rejected": -8688.783203125,
+      "loss": 0.0216,
       "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": -2.6126813888549805,
-      "rewards/margins": 66.86088562011719,
-      "rewards/rejected": -69.47355651855469,
+      "rewards/chosen": -2.5548887252807617,
+      "rewards/margins": 77.47712707519531,
+      "rewards/rejected": -80.03202056884766,
       "step": 230
     },
     {
       "epoch": 0.6818181818181818,
-      "grad_norm": 0.655326329468716,
+      "grad_norm": 0.6247509460136759,
       "learning_rate": 1.3961362544602212e-07,
-      "logits/chosen": -0.8508334159851074,
-      "logits/rejected": 2.6375107765197754,
-      "logps/chosen": -604.7547607421875,
-      "logps/rejected": -6603.0595703125,
-      "loss": 0.0022,
+      "logits/chosen": -1.4957786798477173,
+      "logits/rejected": 2.78035306930542,
+      "logps/chosen": -612.0357666015625,
+      "logps/rejected": -7844.0234375,
+      "loss": 0.0027,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.287423849105835,
-      "rewards/margins": 57.80244827270508,
-      "rewards/rejected": -60.08986282348633,
+      "rewards/chosen": -2.360233783721924,
+      "rewards/margins": 70.1392822265625,
+      "rewards/rejected": -72.49951171875,
       "step": 240
     },
     {
       "epoch": 0.7102272727272727,
-      "grad_norm": 0.8566182395838629,
+      "grad_norm": 0.8355271945684738,
       "learning_rate": 1.1789510538684522e-07,
-      "logits/chosen": -0.9583337903022766,
-      "logits/rejected": 2.481625556945801,
-      "logps/chosen": -564.140869140625,
-      "logps/rejected": -8373.404296875,
-      "loss": 0.0015,
+      "logits/chosen": -1.3467962741851807,
+      "logits/rejected": 3.0541701316833496,
+      "logps/chosen": -582.3834838867188,
+      "logps/rejected": -10293.1650390625,
+      "loss": 0.0022,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.12107253074646,
-      "rewards/margins": 74.31715393066406,
-      "rewards/rejected": -76.43821716308594,
+      "rewards/chosen": -2.3034989833831787,
+      "rewards/margins": 93.33232116699219,
+      "rewards/rejected": -95.63582611083984,
       "step": 250
     },
     {
       "epoch": 0.7102272727272727,
-      "eval_logits/chosen": -0.9364973902702332,
-      "eval_logits/rejected": 2.6537041664123535,
-      "eval_logps/chosen": -579.6390991210938,
-      "eval_logps/rejected": -7301.04736328125,
-      "eval_loss": 0.0008862165850587189,
+      "eval_logits/chosen": -1.0149785280227661,
+      "eval_logits/rejected": 3.758545160293579,
+      "eval_logps/chosen": -596.009521484375,
+      "eval_logps/rejected": -8959.7470703125,
+      "eval_loss": 0.0008426356362178922,
       "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -2.126574754714966,
-      "eval_rewards/margins": 64.47601318359375,
-      "eval_rewards/rejected": -66.60258483886719,
-      "eval_runtime": 193.2705,
-      "eval_samples_per_second": 20.205,
+      "eval_rewards/chosen": -2.290278196334839,
+      "eval_rewards/margins": 80.89930725097656,
+      "eval_rewards/rejected": -83.18959045410156,
+      "eval_runtime": 193.2341,
+      "eval_samples_per_second": 20.209,
       "eval_steps_per_second": 0.321,
       "step": 250
     },
     {
       "epoch": 0.7386363636363636,
-      "grad_norm": 0.3669759262567033,
+      "grad_norm": 0.3552207920354382,
       "learning_rate": 9.748121349736891e-08,
-      "logits/chosen": -0.815521240234375,
-      "logits/rejected": 2.5474696159362793,
-      "logps/chosen": -614.1104736328125,
-      "logps/rejected": -7755.7451171875,
-      "loss": 0.0009,
+      "logits/chosen": -0.6737051010131836,
+      "logits/rejected": 3.9108054637908936,
+      "logps/chosen": -637.5172119140625,
+      "logps/rejected": -9320.220703125,
+      "loss": 0.0011,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.302823305130005,
-      "rewards/margins": 68.47334289550781,
-      "rewards/rejected": -70.77616882324219,
+      "rewards/chosen": -2.5368902683258057,
+      "rewards/margins": 83.88404083251953,
+      "rewards/rejected": -86.4209213256836,
       "step": 260
     },
     {
       "epoch": 0.7670454545454546,
-      "grad_norm": 0.14376906522497312,
+      "grad_norm": 0.09771710883654211,
       "learning_rate": 7.857355122839673e-08,
-      "logits/chosen": -0.8737711906433105,
-      "logits/rejected": 2.419881582260132,
-      "logps/chosen": -592.2354125976562,
-      "logps/rejected": -7475.37109375,
-      "loss": 0.0009,
+      "logits/chosen": -0.7171241044998169,
+      "logits/rejected": 3.9581894874572754,
+      "logps/chosen": -603.0701293945312,
+      "logps/rejected": -8722.890625,
+      "loss": 0.0013,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.113858461380005,
-      "rewards/margins": 65.95025634765625,
-      "rewards/rejected": -68.0641098022461,
+      "rewards/chosen": -2.222205638885498,
+      "rewards/margins": 78.31710052490234,
+      "rewards/rejected": -80.539306640625,
       "step": 270
     },
     {
       "epoch": 0.7954545454545454,
-      "grad_norm": 0.17275320368872452,
+      "grad_norm": 0.20312109356571723,
       "learning_rate": 6.135884496044244e-08,
-      "logits/chosen": -0.8250367045402527,
-      "logits/rejected": 2.680527687072754,
-      "logps/chosen": -574.1306762695312,
-      "logps/rejected": -7469.40087890625,
-      "loss": 0.0022,
+      "logits/chosen": -0.48570650815963745,
+      "logits/rejected": 4.524535655975342,
+      "logps/chosen": -585.2769775390625,
+      "logps/rejected": -8691.55859375,
+      "loss": 0.0015,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.9495769739151,
-      "rewards/margins": 66.2955322265625,
-      "rewards/rejected": -68.24510955810547,
+      "rewards/chosen": -2.0610411167144775,
+      "rewards/margins": 78.40565490722656,
+      "rewards/rejected": -80.46669006347656,
       "step": 280
     },
     {
       "epoch": 0.8238636363636364,
-      "grad_norm": 0.036939229346128824,
+      "grad_norm": 0.0562319233114269,
       "learning_rate": 4.600710195020982e-08,
-      "logits/chosen": -0.6854007244110107,
-      "logits/rejected": 2.711479902267456,
-      "logps/chosen": -595.4219970703125,
-      "logps/rejected": -7571.99853515625,
-      "loss": 0.0054,
+      "logits/chosen": -0.22825559973716736,
+      "logits/rejected": 4.695797443389893,
+      "logps/chosen": -607.1776123046875,
+      "logps/rejected": -8615.1923828125,
+      "loss": 0.0052,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.0627870559692383,
-      "rewards/margins": 67.24461364746094,
-      "rewards/rejected": -69.30741119384766,
+      "rewards/chosen": -2.1803441047668457,
+      "rewards/margins": 77.55899810791016,
+      "rewards/rejected": -79.73933410644531,
       "step": 290
     },
     {
       "epoch": 0.8522727272727273,
-      "grad_norm": 1.0005598289334474,
+      "grad_norm": 1.3409016161535439,
       "learning_rate": 3.2669931390104374e-08,
-      "logits/chosen": -0.7712670564651489,
-      "logits/rejected": 2.537261724472046,
-      "logps/chosen": -546.4788818359375,
-      "logps/rejected": -8221.009765625,
-      "loss": 0.0017,
+      "logits/chosen": -0.32437822222709656,
+      "logits/rejected": 4.543366432189941,
+      "logps/chosen": -559.5152587890625,
+      "logps/rejected": -9425.8134765625,
+      "loss": 0.001,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.9022438526153564,
-      "rewards/margins": 74.09959411621094,
-      "rewards/rejected": -76.00184631347656,
+      "rewards/chosen": -2.032606601715088,
+      "rewards/margins": 86.01726531982422,
+      "rewards/rejected": -88.04988098144531,
       "step": 300
     },
     {
       "epoch": 0.8522727272727273,
-      "eval_logits/chosen": -0.6079820394515991,
-      "eval_logits/rejected": 2.6897590160369873,
-      "eval_logps/chosen": -575.6599731445312,
-      "eval_logps/rejected": -7868.724609375,
-      "eval_loss": 0.0006528676021844149,
+      "eval_logits/chosen": -0.12213863432407379,
+      "eval_logits/rejected": 4.708926677703857,
+      "eval_logps/chosen": -586.3375854492188,
+      "eval_logps/rejected": -9036.201171875,
+      "eval_loss": 0.0007051606080494821,
       "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -2.086782693862915,
-      "eval_rewards/margins": 70.19257354736328,
-      "eval_rewards/rejected": -72.27935791015625,
-      "eval_runtime": 193.3061,
-      "eval_samples_per_second": 20.201,
-      "eval_steps_per_second": 0.321,
+      "eval_rewards/chosen": -2.1935579776763916,
+      "eval_rewards/margins": 81.76056671142578,
+      "eval_rewards/rejected": -83.95413208007812,
+      "eval_runtime": 194.0614,
+      "eval_samples_per_second": 20.122,
+      "eval_steps_per_second": 0.319,
       "step": 300
     },
     {
       "epoch": 0.8806818181818182,
-      "grad_norm": 0.0660494270632579,
+      "grad_norm": 0.1502360317259883,
       "learning_rate": 2.147904716149135e-08,
-      "logits/chosen": -0.5389717817306519,
-      "logits/rejected": 2.602092981338501,
-      "logps/chosen": -555.8554077148438,
-      "logps/rejected": -7988.98974609375,
-      "loss": 0.0032,
+      "logits/chosen": -0.03174494951963425,
+      "logits/rejected": 4.578262805938721,
+      "logps/chosen": -568.2017822265625,
+      "logps/rejected": -9189.7119140625,
+      "loss": 0.0037,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.9378883838653564,
-      "rewards/margins": 70.9469985961914,
-      "rewards/rejected": -72.8848876953125,
+      "rewards/chosen": -2.061352014541626,
+      "rewards/margins": 82.83074188232422,
+      "rewards/rejected": -84.89209747314453,
       "step": 310
     },
     {
       "epoch": 0.9090909090909091,
-      "grad_norm": 0.0488754281750263,
+      "grad_norm": 0.12997178207019558,
       "learning_rate": 1.254496706805433e-08,
-      "logits/chosen": -0.7335542440414429,
-      "logits/rejected": 2.5863664150238037,
-      "logps/chosen": -590.0091552734375,
-      "logps/rejected": -8039.3125,
-      "loss": 0.0007,
+      "logits/chosen": -0.2779918313026428,
+      "logits/rejected": 4.609086036682129,
+      "logps/chosen": -596.7515869140625,
+      "logps/rejected": -9248.6181640625,
+      "loss": 0.0005,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.0733609199523926,
-      "rewards/margins": 71.79994201660156,
-      "rewards/rejected": -73.87330627441406,
+      "rewards/chosen": -2.140784978866577,
+      "rewards/margins": 83.82556915283203,
+      "rewards/rejected": -85.96635437011719,
       "step": 320
     },
     {
       "epoch": 0.9375,
-      "grad_norm": 0.2519430705585286,
+      "grad_norm": 0.20243054667676932,
       "learning_rate": 5.955921395237318e-09,
-      "logits/chosen": -0.6468337178230286,
-      "logits/rejected": 2.546152353286743,
-      "logps/chosen": -548.1500244140625,
-      "logps/rejected": -8072.88037109375,
+      "logits/chosen": -0.13022509217262268,
+      "logits/rejected": 4.544893741607666,
+      "logps/chosen": -555.28271484375,
+      "logps/rejected": -9316.8115234375,
       "loss": 0.0006,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.928818941116333,
-      "rewards/margins": 72.32246398925781,
-      "rewards/rejected": -74.25126647949219,
+      "rewards/chosen": -2.0001461505889893,
+      "rewards/margins": 84.69043731689453,
+      "rewards/rejected": -86.69058990478516,
       "step": 330
     },
     {
       "epoch": 0.9659090909090909,
-      "grad_norm": 0.8725515919272617,
+      "grad_norm": 0.25686140623112713,
       "learning_rate": 1.7769815745066474e-09,
-      "logits/chosen": -0.8968937993049622,
-      "logits/rejected": 2.3741860389709473,
-      "logps/chosen": -561.4879760742188,
-      "logps/rejected": -7435.28125,
-      "loss": 0.001,
+      "logits/chosen": -0.500297486782074,
+      "logits/rejected": 4.298487186431885,
+      "logps/chosen": -568.9089965820312,
+      "logps/rejected": -8478.435546875,
+      "loss": 0.0026,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.9250719547271729,
-      "rewards/margins": 66.34008026123047,
-      "rewards/rejected": -68.26514434814453,
+      "rewards/chosen": -1.9992825984954834,
+      "rewards/margins": 76.69742584228516,
+      "rewards/rejected": -78.69671630859375,
       "step": 340
     },
     {
       "epoch": 0.9943181818181818,
-      "grad_norm": 3.1933280173345713,
+      "grad_norm": 2.1488990981394114,
       "learning_rate": 4.9417557483610875e-11,
-      "logits/chosen": -0.6227324604988098,
-      "logits/rejected": 2.5037405490875244,
-      "logps/chosen": -589.6453247070312,
-      "logps/rejected": -7948.50927734375,
-      "loss": 0.0036,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.153775215148926,
-      "rewards/margins": 70.8557357788086,
-      "rewards/rejected": -73.00951385498047,
+      "logits/chosen": -0.16308510303497314,
+      "logits/rejected": 4.496396064758301,
+      "logps/chosen": -605.3216552734375,
+      "logps/rejected": -9169.884765625,
+      "loss": 0.008,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": -2.310539484024048,
+      "rewards/margins": 82.91272735595703,
+      "rewards/rejected": -85.2232666015625,
       "step": 350
     },
     {
       "epoch": 0.9943181818181818,
-      "eval_logits/chosen": -0.5257502198219299,
-      "eval_logits/rejected": 2.7151966094970703,
-      "eval_logps/chosen": -577.6112670898438,
-      "eval_logps/rejected": -7961.02392578125,
-      "eval_loss": 0.000620901002548635,
+      "eval_logits/chosen": -0.04419805109500885,
+      "eval_logits/rejected": 4.755471229553223,
+      "eval_logps/chosen": -588.0638427734375,
+      "eval_logps/rejected": -9137.26171875,
+      "eval_loss": 0.0007233908982016146,
       "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -2.106295585632324,
-      "eval_rewards/margins": 71.09603881835938,
-      "eval_rewards/rejected": -73.20234680175781,
-      "eval_runtime": 193.4431,
-      "eval_samples_per_second": 20.187,
-      "eval_steps_per_second": 0.321,
+      "eval_rewards/chosen": -2.2108209133148193,
+      "eval_rewards/margins": 82.75391387939453,
+      "eval_rewards/rejected": -84.96473693847656,
+      "eval_runtime": 194.5802,
+      "eval_samples_per_second": 20.069,
+      "eval_steps_per_second": 0.319,
       "step": 350
     },
     {
       "epoch": 1.0,
       "step": 352,
       "total_flos": 0.0,
-      "train_loss": 0.08502711300074721,
-      "train_runtime": 9968.7204,
-      "train_samples_per_second": 4.514,
+      "train_loss": 0.0941242430602539,
+      "train_runtime": 10036.5915,
+      "train_samples_per_second": 4.484,
       "train_steps_per_second": 0.035
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2f90a87fb6fd9aa61f5033db9c384a612bcf2c0d3c1b8b9a6be2f8e6177929fc
+oid sha256:7478770c4681452fb8a226c8859be376547be9d21f5c658595e7281ecf7c0650
 size 7608