Model save

README.md

@@ -4,6 +4,7 @@ base_model: alignment-handbook/zephyr-7b-sft-full
 tags:
 - trl
 - dpo
+- alignment-handbook
 - generated_from_trainer
 model-index:
 - name: zephyr-7b-dpo-full-magpi-reward-scale-1
@@ -17,15 +18,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.0001
-- Rewards/chosen: -2.4218
-- Rewards/rejected: -70.3140
+- Loss: 0.0006
+- Rewards/chosen: -2.1063
+- Rewards/rejected: -73.2023
 - Rewards/accuracies: 1.0
-- Rewards/margins: 67.8922
-- Logps/rejected: -7672.1890
-- Logps/chosen: -609.1660
-- Logits/rejected: 2.5753
-- Logits/chosen: -0.2188
+- Rewards/margins: 71.0960
+- Logps/rejected: -7961.0239
+- Logps/chosen: -577.6113
+- Logits/rejected: 2.7152
+- Logits/chosen: -0.5258
 
 ## Model description
 
@@ -62,13 +63,13 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.0055        | 0.1420 | 50   | 0.0031          | -0.8850        | -42.4340         | 0.9960             | 41.5490         | -4884.1895     | -455.4866    | -2.5280         | -2.9176       |
-| 0.0014        | 0.2841 | 100  | 0.0003          | -1.7467        | -57.2050         | 1.0                | 55.4583         | -6361.2881     | -541.6496    | -0.1436         | -2.5818       |
-| 0.001         | 0.4261 | 150  | 0.0002          | -2.4059        | -66.6788         | 1.0                | 64.2728         | -7308.6660     | -607.5745    | 1.9422          | -1.9097       |
-| 0.0018        | 0.5682 | 200  | 0.0002          | -2.1797        | -67.6111         | 1.0                | 65.4314         | -7401.8965     | -584.9550    | 2.3189          | -1.0395       |
-| 0.0009        | 0.7102 | 250  | 0.0001          | -2.4169        | -67.5787         | 1.0                | 65.1618         | -7398.6553     | -608.6732    | 2.5585          | -0.3354       |
-| 0.0009        | 0.8523 | 300  | 0.0001          | -2.4125        | -70.2443         | 1.0                | 67.8319         | -7665.2217     | -608.2272    | 2.5751          | -0.2549       |
-| 0.0024        | 0.9943 | 350  | 0.0001          | -2.4218        | -70.3140         | 1.0                | 67.8922         | -7672.1890     | -609.1660    | 2.5753          | -0.2188       |
+| 0.011         | 0.1420 | 50   | 0.0091          | -0.9203        | -43.3611         | 0.9960             | 42.4408         | -4976.9014     | -459.0154    | -2.5410         | -2.9653       |
+| 0.0029        | 0.2841 | 100  | 0.0016          | -1.9179        | -61.7743         | 1.0                | 59.8564         | -6818.2158     | -558.7726    | 0.0358          | -2.5257       |
+| 0.0023        | 0.4261 | 150  | 0.0012          | -1.9106        | -69.0581         | 1.0                | 67.1475         | -7546.6006     | -558.0397    | 1.8614          | -2.4309       |
+| 0.0037        | 0.5682 | 200  | 0.0009          | -1.9357        | -71.7717         | 1.0                | 69.8360         | -7817.9595     | -560.5506    | 2.4905          | -1.0992       |
+| 0.0015        | 0.7102 | 250  | 0.0009          | -2.1266        | -66.6026         | 1.0                | 64.4760         | -7301.0474     | -579.6391    | 2.6537          | -0.9365       |
+| 0.0017        | 0.8523 | 300  | 0.0007          | -2.0868        | -72.2794         | 1.0                | 70.1926         | -7868.7246     | -575.6600    | 2.6898          | -0.6080       |
+| 0.0036        | 0.9943 | 350  | 0.0006          | -2.1063        | -73.2023         | 1.0                | 71.0960         | -7961.0239     | -577.6113    | 2.7152          | -0.5258       |
 
 
 ### Framework versions

all_results.json

@@ -1,9 +1,22 @@
 {
     "epoch": 1.0,
+    "eval_logits/chosen": -0.21594780683517456,
+    "eval_logits/rejected": 2.576901435852051,
+    "eval_logps/chosen": -609.2966918945312,
+    "eval_logps/rejected": -7672.10595703125,
+    "eval_loss": 7.102515519363806e-05,
+    "eval_rewards/accuracies": 1.0,
+    "eval_rewards/chosen": -2.423149824142456,
+    "eval_rewards/margins": 67.89002990722656,
+    "eval_rewards/rejected": -70.31317901611328,
+    "eval_runtime": 191.0845,
+    "eval_samples": 3905,
+    "eval_samples_per_second": 20.436,
+    "eval_steps_per_second": 0.324,
     "total_flos": 0.0,
-    "train_loss": 0.042729352997742935,
-    "train_runtime": 10052.8601,
+    "train_loss": 0.08502711300074721,
+    "train_runtime": 9968.7204,
     "train_samples": 45000,
-    "train_samples_per_second": 4.476,
+    "train_samples_per_second": 4.514,
     "train_steps_per_second": 0.035
 }

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": -0.21594780683517456,
+    "eval_logits/rejected": 2.576901435852051,
+    "eval_logps/chosen": -609.2966918945312,
+    "eval_logps/rejected": -7672.10595703125,
+    "eval_loss": 7.102515519363806e-05,
+    "eval_rewards/accuracies": 1.0,
+    "eval_rewards/chosen": -2.423149824142456,
+    "eval_rewards/margins": 67.89002990722656,
+    "eval_rewards/rejected": -70.31317901611328,
+    "eval_runtime": 191.0845,
+    "eval_samples": 3905,
+    "eval_samples_per_second": 20.436,
+    "eval_steps_per_second": 0.324
+}

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9dcaf4dc0c0b909976aa056b81e0a2b319bd4cc0fbe64b4ad6e3bff5b5e9e4a8
+oid sha256:19a4d988d4c8ea906c6a4a6a87b07fe045301edf48e28839de9be0c286ebc349
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c30155721a38abfe4d3230f2893721359f0c129fc60a0d96725ac39198147de8
+oid sha256:221c5b2974706b5866fe55da9a903f9aa856e047c987867494ba6d94c0e033db
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8a1a9bbcf8a17b2d6d5785f3df9d8faf5023eb273d7fae9702a0c719f089c4ac
+oid sha256:ffcf7851352d6a651c1aa0629b34d9e52a84887ba228a345c457ef2d14f17122
 size 4540516344

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.0,
     "total_flos": 0.0,
-    "train_loss": 0.042729352997742935,
-    "train_runtime": 10052.8601,
+    "train_loss": 0.08502711300074721,
+    "train_runtime": 9968.7204,
     "train_samples": 45000,
-    "train_samples_per_second": 4.476,
+    "train_samples_per_second": 4.514,
     "train_steps_per_second": 0.035
 }

trainer_state.json

@@ -10,648 +10,648 @@
   "log_history": [
     {
       "epoch": 0.028409090909090908,
-      "grad_norm": 45.06143403082133,
+      "grad_norm": 90.01349453953418,
       "learning_rate": 1.3888888888888888e-07,
-      "logits/chosen": -2.859044075012207,
-      "logits/rejected": -2.64284086227417,
-      "logps/chosen": -390.5040588378906,
-      "logps/rejected": -607.8244018554688,
-      "loss": 0.6882,
+      "logits/chosen": -2.8592569828033447,
+      "logits/rejected": -2.642957925796509,
+      "logps/chosen": -390.56573486328125,
+      "logps/rejected": -607.8802490234375,
+      "loss": 1.3762,
       "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.002557548927143216,
-      "rewards/margins": 0.013725275173783302,
-      "rewards/rejected": -0.01116772647947073,
+      "rewards/chosen": 0.001940618036314845,
+      "rewards/margins": 0.013666175305843353,
+      "rewards/rejected": -0.011725558899343014,
       "step": 10
     },
     {
       "epoch": 0.056818181818181816,
-      "grad_norm": 19.067293358665005,
+      "grad_norm": 38.16702665689489,
       "learning_rate": 2.7777777777777776e-07,
-      "logits/chosen": -2.8423688411712646,
-      "logits/rejected": -2.6946520805358887,
-      "logps/chosen": -328.6228332519531,
-      "logps/rejected": -774.904541015625,
-      "loss": 0.5132,
-      "rewards/accuracies": 0.981249988079071,
-      "rewards/chosen": 0.08522307872772217,
-      "rewards/margins": 0.6598724722862244,
-      "rewards/rejected": -0.574649453163147,
+      "logits/chosen": -2.842255115509033,
+      "logits/rejected": -2.694774627685547,
+      "logps/chosen": -328.5555419921875,
+      "logps/rejected": -774.9117431640625,
+      "loss": 1.0262,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": 0.08589581400156021,
+      "rewards/margins": 0.660616934299469,
+      "rewards/rejected": -0.5747210383415222,
       "step": 20
     },
     {
       "epoch": 0.08522727272727272,
-      "grad_norm": 4.736974625657813,
+      "grad_norm": 9.474354712422207,
       "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -2.8571083545684814,
-      "logits/rejected": -2.6747167110443115,
-      "logps/chosen": -295.3691711425781,
-      "logps/rejected": -1112.830078125,
-      "loss": 0.1792,
+      "logits/chosen": -2.857499837875366,
+      "logits/rejected": -2.675124406814575,
+      "logps/chosen": -295.38397216796875,
+      "logps/rejected": -1112.8095703125,
+      "loss": 0.3585,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.36912840604782104,
-      "rewards/margins": 4.717885494232178,
-      "rewards/rejected": -4.348757266998291,
+      "rewards/chosen": 0.36898040771484375,
+      "rewards/margins": 4.717532157897949,
+      "rewards/rejected": -4.3485517501831055,
       "step": 30
     },
     {
       "epoch": 0.11363636363636363,
-      "grad_norm": 1.4117830566891867,
+      "grad_norm": 2.844664039016391,
       "learning_rate": 4.998023493068254e-07,
-      "logits/chosen": -2.834780216217041,
-      "logits/rejected": -2.644721508026123,
-      "logps/chosen": -314.90582275390625,
-      "logps/rejected": -2130.47900390625,
-      "loss": 0.0415,
+      "logits/chosen": -2.8356635570526123,
+      "logits/rejected": -2.645285129547119,
+      "logps/chosen": -314.98876953125,
+      "logps/rejected": -2130.987060546875,
+      "loss": 0.083,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.31849661469459534,
-      "rewards/margins": 14.544331550598145,
-      "rewards/rejected": -14.225835800170898,
+      "rewards/chosen": 0.3176669478416443,
+      "rewards/margins": 14.54858112335205,
+      "rewards/rejected": -14.230911254882812,
       "step": 40
     },
     {
       "epoch": 0.14204545454545456,
-      "grad_norm": 0.09320830768011595,
+      "grad_norm": 0.18581946577481065,
       "learning_rate": 4.975823666181255e-07,
-      "logits/chosen": -2.814044713973999,
-      "logits/rejected": -2.548596143722534,
-      "logps/chosen": -437.7023010253906,
-      "logps/rejected": -4283.9775390625,
-      "loss": 0.0055,
+      "logits/chosen": -2.8282175064086914,
+      "logits/rejected": -2.554896831512451,
+      "logps/chosen": -441.171875,
+      "logps/rejected": -4335.5703125,
+      "loss": 0.011,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -0.7506707906723022,
-      "rewards/margins": 35.5496940612793,
-      "rewards/rejected": -36.30036926269531,
+      "rewards/chosen": -0.785366415977478,
+      "rewards/margins": 36.03093338012695,
+      "rewards/rejected": -36.8162956237793,
       "step": 50
     },
     {
       "epoch": 0.14204545454545456,
-      "eval_logits/chosen": -2.9176247119903564,
-      "eval_logits/rejected": -2.528038263320923,
-      "eval_logps/chosen": -455.486572265625,
-      "eval_logps/rejected": -4884.189453125,
-      "eval_loss": 0.0030528793577104807,
+      "eval_logits/chosen": -2.965341806411743,
+      "eval_logits/rejected": -2.5409843921661377,
+      "eval_logps/chosen": -459.0153503417969,
+      "eval_logps/rejected": -4976.9013671875,
+      "eval_loss": 0.009123104624450207,
       "eval_rewards/accuracies": 0.9959677457809448,
-      "eval_rewards/chosen": -0.8850486874580383,
-      "eval_rewards/margins": 41.54896545410156,
-      "eval_rewards/rejected": -42.43401336669922,
-      "eval_runtime": 193.4773,
-      "eval_samples_per_second": 20.183,
-      "eval_steps_per_second": 0.32,
+      "eval_rewards/chosen": -0.9203360676765442,
+      "eval_rewards/margins": 42.44078063964844,
+      "eval_rewards/rejected": -43.361122131347656,
+      "eval_runtime": 195.4208,
+      "eval_samples_per_second": 19.983,
+      "eval_steps_per_second": 0.317,
       "step": 50
     },
     {
       "epoch": 0.17045454545454544,
-      "grad_norm": 1.0625936242535339,
+      "grad_norm": 1.8496563592278583,
       "learning_rate": 4.929173350101024e-07,
-      "logits/chosen": -2.990762710571289,
-      "logits/rejected": -2.484398603439331,
-      "logps/chosen": -463.26251220703125,
-      "logps/rejected": -5167.1591796875,
-      "loss": 0.0033,
+      "logits/chosen": -3.018855571746826,
+      "logits/rejected": -2.489605188369751,
+      "logps/chosen": -478.2201232910156,
+      "logps/rejected": -5343.02783203125,
+      "loss": 0.0066,
       "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": -1.033675193786621,
-      "rewards/margins": 44.20937728881836,
-      "rewards/rejected": -45.2430534362793,
+      "rewards/chosen": -1.1832506656646729,
+      "rewards/margins": 45.8184814453125,
+      "rewards/rejected": -47.00173568725586,
       "step": 60
     },
     {
       "epoch": 0.19886363636363635,
-      "grad_norm": 0.16389340246703632,
+      "grad_norm": 0.42838497413063575,
       "learning_rate": 4.858533249305336e-07,
-      "logits/chosen": -2.814481735229492,
-      "logits/rejected": -1.7779146432876587,
-      "logps/chosen": -507.0504455566406,
-      "logps/rejected": -5547.2275390625,
-      "loss": 0.0031,
+      "logits/chosen": -2.8961918354034424,
+      "logits/rejected": -1.8984692096710205,
+      "logps/chosen": -537.8681640625,
+      "logps/rejected": -5729.29150390625,
+      "loss": 0.0061,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.2788976430892944,
-      "rewards/margins": 48.066017150878906,
-      "rewards/rejected": -49.34491729736328,
+      "rewards/chosen": -1.587074875831604,
+      "rewards/margins": 49.57848358154297,
+      "rewards/rejected": -51.165557861328125,
       "step": 70
     },
     {
       "epoch": 0.22727272727272727,
-      "grad_norm": 15.588141585226081,
+      "grad_norm": 23.183269598091908,
       "learning_rate": 4.764600984163808e-07,
-      "logits/chosen": -3.000598669052124,
-      "logits/rejected": -1.848249077796936,
-      "logps/chosen": -532.5420532226562,
-      "logps/rejected": -6873.24853515625,
-      "loss": 0.0039,
+      "logits/chosen": -3.015650510787964,
+      "logits/rejected": -1.4822012186050415,
+      "logps/chosen": -574.1959838867188,
+      "logps/rejected": -7196.5283203125,
+      "loss": 0.007,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.2622119188308716,
-      "rewards/margins": 60.732948303222656,
-      "rewards/rejected": -61.99515914916992,
+      "rewards/chosen": -1.6787515878677368,
+      "rewards/margins": 63.5491943359375,
+      "rewards/rejected": -65.22795104980469,
       "step": 80
     },
     {
       "epoch": 0.2556818181818182,
-      "grad_norm": 0.02323120317433541,
+      "grad_norm": 0.2590854602849931,
       "learning_rate": 4.6483042014491527e-07,
-      "logits/chosen": -3.130722761154175,
-      "logits/rejected": -2.385204792022705,
-      "logps/chosen": -520.1354370117188,
-      "logps/rejected": -6028.5537109375,
-      "loss": 0.0138,
+      "logits/chosen": -2.880042314529419,
+      "logits/rejected": -1.109546422958374,
+      "logps/chosen": -574.42529296875,
+      "logps/rejected": -6483.9189453125,
+      "loss": 0.0194,
       "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": -1.5802205801010132,
-      "rewards/margins": 52.33146286010742,
-      "rewards/rejected": -53.91168212890625,
+      "rewards/chosen": -2.123119831085205,
+      "rewards/margins": 56.34221267700195,
+      "rewards/rejected": -58.4653434753418,
       "step": 90
     },
     {
       "epoch": 0.2840909090909091,
-      "grad_norm": 0.16442080948160046,
+      "grad_norm": 0.2975727339596996,
       "learning_rate": 4.510791413176912e-07,
-      "logits/chosen": -2.8645615577697754,
-      "logits/rejected": -1.0432738065719604,
-      "logps/chosen": -532.5404052734375,
-      "logps/rejected": -6479.7802734375,
-      "loss": 0.0014,
+      "logits/chosen": -2.7033655643463135,
+      "logits/rejected": -0.46150803565979004,
+      "logps/chosen": -574.6309814453125,
+      "logps/rejected": -7050.77099609375,
+      "loss": 0.0029,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.6307029724121094,
-      "rewards/margins": 56.10929489135742,
-      "rewards/rejected": -57.73999786376953,
+      "rewards/chosen": -2.0516088008880615,
+      "rewards/margins": 61.398292541503906,
+      "rewards/rejected": -63.44989776611328,
       "step": 100
     },
     {
       "epoch": 0.2840909090909091,
-      "eval_logits/chosen": -2.5817885398864746,
-      "eval_logits/rejected": -0.1435869336128235,
-      "eval_logps/chosen": -541.6495971679688,
-      "eval_logps/rejected": -6361.2880859375,
-      "eval_loss": 0.0003222826635465026,
+      "eval_logits/chosen": -2.5256500244140625,
+      "eval_logits/rejected": 0.035765551030635834,
+      "eval_logps/chosen": -558.7725830078125,
+      "eval_logps/rejected": -6818.2158203125,
+      "eval_loss": 0.0016336780972778797,
       "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -1.7466782331466675,
-      "eval_rewards/margins": 55.458316802978516,
-      "eval_rewards/rejected": -57.20499801635742,
-      "eval_runtime": 190.7671,
-      "eval_samples_per_second": 20.47,
-      "eval_steps_per_second": 0.325,
+      "eval_rewards/chosen": -1.9179083108901978,
+      "eval_rewards/margins": 59.856361389160156,
+      "eval_rewards/rejected": -61.77427673339844,
+      "eval_runtime": 192.2507,
+      "eval_samples_per_second": 20.312,
+      "eval_steps_per_second": 0.322,
       "step": 100
     },
     {
       "epoch": 0.3125,
-      "grad_norm": 0.03669115199922144,
+      "grad_norm": 0.04984017585029643,
       "learning_rate": 4.353420654246546e-07,
-      "logits/chosen": -2.299072742462158,
-      "logits/rejected": 0.060857243835926056,
-      "logps/chosen": -552.1079711914062,
-      "logps/rejected": -6287.04736328125,
-      "loss": 0.0024,
+      "logits/chosen": -2.2970995903015137,
+      "logits/rejected": 0.2104659080505371,
+      "logps/chosen": -568.5081787109375,
+      "logps/rejected": -6664.22412109375,
+      "loss": 0.0043,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.766867995262146,
-      "rewards/margins": 55.110923767089844,
-      "rewards/rejected": -56.87778854370117,
+      "rewards/chosen": -1.930869460105896,
+      "rewards/margins": 58.71869659423828,
+      "rewards/rejected": -60.649559020996094,
       "step": 110
     },
     {
       "epoch": 0.3409090909090909,
-      "grad_norm": 0.03951441158135731,
+      "grad_norm": 0.03797940797804543,
       "learning_rate": 4.177746070897592e-07,
-      "logits/chosen": -2.019587278366089,
-      "logits/rejected": 0.7790449261665344,
-      "logps/chosen": -561.4166870117188,
-      "logps/rejected": -7041.1259765625,
-      "loss": 0.0033,
+      "logits/chosen": -1.7653045654296875,
+      "logits/rejected": 1.1280748844146729,
+      "logps/chosen": -600.4093017578125,
+      "logps/rejected": -7361.03662109375,
+      "loss": 0.0084,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.798998236656189,
-      "rewards/margins": 62.23997116088867,
-      "rewards/rejected": -64.03898620605469,
+      "rewards/chosen": -2.188924789428711,
+      "rewards/margins": 65.04915618896484,
+      "rewards/rejected": -67.23808288574219,
       "step": 120
     },
     {
       "epoch": 0.3693181818181818,
-      "grad_norm": 0.3141553784150294,
+      "grad_norm": 0.47736957873502217,
       "learning_rate": 3.9855025724292763e-07,
-      "logits/chosen": -2.1000161170959473,
-      "logits/rejected": 1.278324842453003,
-      "logps/chosen": -563.4060668945312,
-      "logps/rejected": -7105.3544921875,
-      "loss": 0.0027,
-      "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": -1.929544448852539,
-      "rewards/margins": 62.7186164855957,
-      "rewards/rejected": -64.64815521240234,
+      "logits/chosen": -2.10666823387146,
+      "logits/rejected": 1.7087266445159912,
+      "logps/chosen": -594.9124755859375,
+      "logps/rejected": -7779.0634765625,
+      "loss": 0.003,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.2446084022521973,
+      "rewards/margins": 69.14064025878906,
+      "rewards/rejected": -71.38525390625,
       "step": 130
     },
     {
       "epoch": 0.3977272727272727,
-      "grad_norm": 0.5821229491227392,
+      "grad_norm": 1.020605288874105,
       "learning_rate": 3.7785886977585555e-07,
-      "logits/chosen": -2.022021532058716,
-      "logits/rejected": 1.353015422821045,
-      "logps/chosen": -552.7689208984375,
-      "logps/rejected": -7177.63427734375,
-      "loss": 0.0017,
+      "logits/chosen": -2.2910099029541016,
+      "logits/rejected": 1.9655479192733765,
+      "logps/chosen": -569.71533203125,
+      "logps/rejected": -8695.2900390625,
+      "loss": 0.0045,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -1.867326021194458,
-      "rewards/margins": 63.489112854003906,
-      "rewards/rejected": -65.35643768310547,
+      "rewards/chosen": -2.0367894172668457,
+      "rewards/margins": 78.4962158203125,
+      "rewards/rejected": -80.53299713134766,
       "step": 140
     },
     {
       "epoch": 0.42613636363636365,
-      "grad_norm": 0.7534850925649158,
+      "grad_norm": 0.5904857854916888,
       "learning_rate": 3.5590478660213206e-07,
-      "logits/chosen": -1.8572208881378174,
-      "logits/rejected": 1.6534769535064697,
-      "logps/chosen": -621.11181640625,
-      "logps/rejected": -7095.47802734375,
-      "loss": 0.001,
+      "logits/chosen": -2.2953405380249023,
+      "logits/rejected": 1.9216325283050537,
+      "logps/chosen": -595.8374633789062,
+      "logps/rejected": -8074.73974609375,
+      "loss": 0.0023,
       "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": -2.4304866790771484,
-      "rewards/margins": 61.404510498046875,
-      "rewards/rejected": -63.834999084472656,
+      "rewards/chosen": -2.1777424812316895,
+      "rewards/margins": 71.44985961914062,
+      "rewards/rejected": -73.62760925292969,
       "step": 150
     },
     {
       "epoch": 0.42613636363636365,
-      "eval_logits/chosen": -1.9097061157226562,
-      "eval_logits/rejected": 1.942201852798462,
-      "eval_logps/chosen": -607.574462890625,
-      "eval_logps/rejected": -7308.666015625,
-      "eval_loss": 0.0001972884201677516,
+      "eval_logits/chosen": -2.4309346675872803,
+      "eval_logits/rejected": 1.8614341020584106,
+      "eval_logps/chosen": -558.0396728515625,
+      "eval_logps/rejected": -7546.6005859375,
+      "eval_loss": 0.001163232373073697,
       "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -2.4059276580810547,
-      "eval_rewards/margins": 64.27284240722656,
-      "eval_rewards/rejected": -66.67876434326172,
-      "eval_runtime": 193.1671,
-      "eval_samples_per_second": 20.216,
-      "eval_steps_per_second": 0.321,
+      "eval_rewards/chosen": -1.9105795621871948,
+      "eval_rewards/margins": 67.14754486083984,
+      "eval_rewards/rejected": -69.05812072753906,
+      "eval_runtime": 193.8607,
+      "eval_samples_per_second": 20.143,
+      "eval_steps_per_second": 0.32,
       "step": 150
     },
     {
       "epoch": 0.45454545454545453,
-      "grad_norm": 0.5761003352865638,
+      "grad_norm": 5.391869442805235,
       "learning_rate": 3.3290481963801696e-07,
-      "logits/chosen": -2.1488444805145264,
-      "logits/rejected": 1.8684966564178467,
-      "logps/chosen": -605.7991943359375,
-      "logps/rejected": -7330.78759765625,
-      "loss": 0.0002,
+      "logits/chosen": -2.435987949371338,
+      "logits/rejected": 2.048819065093994,
+      "logps/chosen": -532.9867553710938,
+      "logps/rejected": -7208.0419921875,
+      "loss": 0.0007,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.356612205505371,
-      "rewards/margins": 64.78636169433594,
-      "rewards/rejected": -67.14297485351562,
+      "rewards/chosen": -1.628488540649414,
+      "rewards/margins": 64.28703308105469,
+      "rewards/rejected": -65.91552734375,
       "step": 160
     },
     {
       "epoch": 0.48295454545454547,
-      "grad_norm": 0.0136866819292242,
+      "grad_norm": 0.031936510635288025,
       "learning_rate": 3.0908610963322626e-07,
-      "logits/chosen": -2.361186981201172,
-      "logits/rejected": 1.371411919593811,
-      "logps/chosen": -656.9031982421875,
-      "logps/rejected": -7962.04443359375,
-      "loss": 0.0059,
+      "logits/chosen": -2.173886775970459,
+      "logits/rejected": 2.0431816577911377,
+      "logps/chosen": -577.22314453125,
+      "logps/rejected": -7725.6796875,
+      "loss": 0.0074,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.6928484439849854,
-      "rewards/margins": 69.64448547363281,
-      "rewards/rejected": -72.33734130859375,
+      "rewards/chosen": -1.896048903465271,
+      "rewards/margins": 68.07765197753906,
+      "rewards/rejected": -69.97369384765625,
       "step": 170
     },
     {
       "epoch": 0.5113636363636364,
-      "grad_norm": 0.00510201676789354,
+      "grad_norm": 0.018094668159337083,
       "learning_rate": 2.846838829972671e-07,
-      "logits/chosen": -2.2552402019500732,
-      "logits/rejected": 1.9691696166992188,
-      "logps/chosen": -620.7752685546875,
-      "logps/rejected": -7178.3388671875,
+      "logits/chosen": -1.9897123575210571,
+      "logits/rejected": 2.31811785697937,
+      "logps/chosen": -567.0270385742188,
+      "logps/rejected": -7191.4697265625,
       "loss": 0.0021,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.5987088680267334,
-      "rewards/margins": 63.054847717285156,
-      "rewards/rejected": -65.65354919433594,
+      "rewards/chosen": -2.061227798461914,
+      "rewards/margins": 63.72364044189453,
+      "rewards/rejected": -65.78486633300781,
       "step": 180
     },
     {
       "epoch": 0.5397727272727273,
-      "grad_norm": 0.2102394829040594,
+      "grad_norm": 0.3615128328570267,
       "learning_rate": 2.5993912877423147e-07,
-      "logits/chosen": -1.9847707748413086,
-      "logits/rejected": 1.6557881832122803,
-      "logps/chosen": -555.5148315429688,
-      "logps/rejected": -7729.9189453125,
-      "loss": 0.0007,
+      "logits/chosen": -1.5569745302200317,
+      "logits/rejected": 1.8910541534423828,
+      "logps/chosen": -519.6112670898438,
+      "logps/rejected": -8060.75390625,
+      "loss": 0.0013,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.0989906787872314,
-      "rewards/margins": 68.28409576416016,
-      "rewards/rejected": -70.38308715820312,
+      "rewards/chosen": -1.7399555444717407,
+      "rewards/margins": 71.95148468017578,
+      "rewards/rejected": -73.69144439697266,
       "step": 190
     },
     {
       "epoch": 0.5681818181818182,
-      "grad_norm": 0.62932736058334,
+      "grad_norm": 1.460001191569848,
       "learning_rate": 2.3509621870754504e-07,
-      "logits/chosen": -1.2217730283737183,
-      "logits/rejected": 2.607821226119995,
-      "logps/chosen": -571.2921142578125,
-      "logps/rejected": -6882.54833984375,
-      "loss": 0.0018,
+      "logits/chosen": -1.1832640171051025,
+      "logits/rejected": 2.8146424293518066,
+      "logps/chosen": -540.25,
+      "logps/rejected": -7328.6376953125,
+      "loss": 0.0037,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.0616230964660645,
-      "rewards/margins": 60.45543670654297,
-      "rewards/rejected": -62.517051696777344,
+      "rewards/chosen": -1.751201868057251,
+      "rewards/margins": 65.22674560546875,
+      "rewards/rejected": -66.97795104980469,
       "step": 200
     },
     {
       "epoch": 0.5681818181818182,
-      "eval_logits/chosen": -1.0394700765609741,
-      "eval_logits/rejected": 2.318869113922119,
-      "eval_logps/chosen": -584.9549560546875,
-      "eval_logps/rejected": -7401.896484375,
-      "eval_loss": 0.0001572135806782171,
+      "eval_logits/chosen": -1.0991721153259277,
+      "eval_logits/rejected": 2.4905478954315186,
+      "eval_logps/chosen": -560.5505981445312,
+      "eval_logps/rejected": -7817.95947265625,
+      "eval_loss": 0.0009270149166695774,
       "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -2.179731845855713,
-      "eval_rewards/margins": 65.43135070800781,
-      "eval_rewards/rejected": -67.61107635498047,
-      "eval_runtime": 192.1225,
-      "eval_samples_per_second": 20.326,
-      "eval_steps_per_second": 0.323,
+      "eval_rewards/chosen": -1.9356889724731445,
+      "eval_rewards/margins": 69.83601379394531,
+      "eval_rewards/rejected": -71.7717056274414,
+      "eval_runtime": 195.666,
+      "eval_samples_per_second": 19.957,
+      "eval_steps_per_second": 0.317,
       "step": 200
     },
     {
       "epoch": 0.5965909090909091,
-      "grad_norm": 0.5427913238664086,
+      "grad_norm": 0.6173080616307016,
       "learning_rate": 2.1040049389819624e-07,
-      "logits/chosen": -1.0268203020095825,
-      "logits/rejected": 2.479326009750366,
-      "logps/chosen": -601.0950927734375,
-      "logps/rejected": -7280.90771484375,
-      "loss": 0.0028,
+      "logits/chosen": -1.0591375827789307,
+      "logits/rejected": 2.6843762397766113,
+      "logps/chosen": -585.5335083007812,
+      "logps/rejected": -7571.32666015625,
+      "loss": 0.0065,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.104518413543701,
-      "rewards/margins": 64.38551330566406,
-      "rewards/rejected": -66.49003601074219,
+      "rewards/chosen": -1.9489023685455322,
+      "rewards/margins": 67.44532775878906,
+      "rewards/rejected": -69.39422607421875,
       "step": 210
     },
     {
       "epoch": 0.625,
-      "grad_norm": 2.0347197022348307,
+      "grad_norm": 3.3870532652701733,
       "learning_rate": 1.8609584188988133e-07,
-      "logits/chosen": -0.5109957456588745,
-      "logits/rejected": 2.219853401184082,
-      "logps/chosen": -642.4463500976562,
-      "logps/rejected": -6641.6826171875,
-      "loss": 0.003,
+      "logits/chosen": -0.8726997375488281,
+      "logits/rejected": 2.3560879230499268,
+      "logps/chosen": -635.2283935546875,
+      "logps/rejected": -6522.76708984375,
+      "loss": 0.0054,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.7784571647644043,
-      "rewards/margins": 57.14200973510742,
-      "rewards/rejected": -59.92046356201172,
+      "rewards/chosen": -2.706278085708618,
+      "rewards/margins": 56.02503204345703,
+      "rewards/rejected": -58.73130416870117,
       "step": 220
     },
     {
       "epoch": 0.6534090909090909,
-      "grad_norm": 34.246565820645614,
+      "grad_norm": 84.74544834753664,
       "learning_rate": 1.624222881090439e-07,
-      "logits/chosen": -0.5791199207305908,
-      "logits/rejected": 2.0261969566345215,
-      "logps/chosen": -674.46044921875,
-      "logps/rejected": -7846.1572265625,
-      "loss": 0.0099,
+      "logits/chosen": -1.068495512008667,
+      "logits/rejected": 2.1958107948303223,
+      "logps/chosen": -655.9327392578125,
+      "logps/rejected": -7632.9365234375,
+      "loss": 0.0224,
       "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": -2.7979581356048584,
-      "rewards/margins": 68.80779266357422,
-      "rewards/rejected": -71.60575103759766,
+      "rewards/chosen": -2.6126813888549805,
+      "rewards/margins": 66.86088562011719,
+      "rewards/rejected": -69.47355651855469,
       "step": 230
     },
     {
       "epoch": 0.6818181818181818,
-      "grad_norm": 0.31954869640074296,
+      "grad_norm": 0.655326329468716,
       "learning_rate": 1.3961362544602212e-07,
-      "logits/chosen": -0.41727203130722046,
-      "logits/rejected": 2.4299378395080566,
-      "logps/chosen": -630.17919921875,
-      "logps/rejected": -6758.9619140625,
-      "loss": 0.001,
+      "logits/chosen": -0.8508334159851074,
+      "logits/rejected": 2.6375107765197754,
+      "logps/chosen": -604.7547607421875,
+      "logps/rejected": -6603.0595703125,
+      "loss": 0.0022,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.541667938232422,
-      "rewards/margins": 59.10721969604492,
-      "rewards/rejected": -61.648887634277344,
+      "rewards/chosen": -2.287423849105835,
+      "rewards/margins": 57.80244827270508,
+      "rewards/rejected": -60.08986282348633,
       "step": 240
     },
     {
       "epoch": 0.7102272727272727,
-      "grad_norm": 0.2930942520900322,
+      "grad_norm": 0.8566182395838629,
       "learning_rate": 1.1789510538684522e-07,
-      "logits/chosen": -0.4175181984901428,
-      "logits/rejected": 2.409147024154663,
-      "logps/chosen": -591.2493896484375,
-      "logps/rejected": -8521.580078125,
-      "loss": 0.0009,
+      "logits/chosen": -0.9583337903022766,
+      "logits/rejected": 2.481625556945801,
+      "logps/chosen": -564.140869140625,
+      "logps/rejected": -8373.404296875,
+      "loss": 0.0015,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.392158031463623,
-      "rewards/margins": 75.52781677246094,
-      "rewards/rejected": -77.91997528076172,
+      "rewards/chosen": -2.12107253074646,
+      "rewards/margins": 74.31715393066406,
+      "rewards/rejected": -76.43821716308594,
       "step": 250
     },
     {
       "epoch": 0.7102272727272727,
-      "eval_logits/chosen": -0.3353612720966339,
-      "eval_logits/rejected": 2.558471918106079,
-      "eval_logps/chosen": -608.6731567382812,
-      "eval_logps/rejected": -7398.6552734375,
-      "eval_loss": 0.0001336697314400226,
+      "eval_logits/chosen": -0.9364973902702332,
+      "eval_logits/rejected": 2.6537041664123535,
+      "eval_logps/chosen": -579.6390991210938,
+      "eval_logps/rejected": -7301.04736328125,
+      "eval_loss": 0.0008862165850587189,
       "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -2.4169137477874756,
-      "eval_rewards/margins": 65.16175079345703,
-      "eval_rewards/rejected": -67.57865905761719,
-      "eval_runtime": 191.4599,
-      "eval_samples_per_second": 20.396,
-      "eval_steps_per_second": 0.324,
+      "eval_rewards/chosen": -2.126574754714966,
+      "eval_rewards/margins": 64.47601318359375,
+      "eval_rewards/rejected": -66.60258483886719,
+      "eval_runtime": 193.2705,
+      "eval_samples_per_second": 20.205,
+      "eval_steps_per_second": 0.321,
       "step": 250
     },
     {
       "epoch": 0.7386363636363636,
-      "grad_norm": 0.18266288675107914,
+      "grad_norm": 0.3669759262567033,
       "learning_rate": 9.748121349736891e-08,
-      "logits/chosen": -0.2991776764392853,
-      "logits/rejected": 2.4152047634124756,
-      "logps/chosen": -644.2991333007812,
-      "logps/rejected": -7754.68359375,
-      "loss": 0.0006,
+      "logits/chosen": -0.815521240234375,
+      "logits/rejected": 2.5474696159362793,
+      "logps/chosen": -614.1104736328125,
+      "logps/rejected": -7755.7451171875,
+      "loss": 0.0009,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.604710578918457,
-      "rewards/margins": 68.16084289550781,
-      "rewards/rejected": -70.76555633544922,
+      "rewards/chosen": -2.302823305130005,
+      "rewards/margins": 68.47334289550781,
+      "rewards/rejected": -70.77616882324219,
       "step": 260
     },
     {
       "epoch": 0.7670454545454546,
-      "grad_norm": 0.02843345452840696,
+      "grad_norm": 0.14376906522497312,
       "learning_rate": 7.857355122839673e-08,
-      "logits/chosen": -0.5736527442932129,
-      "logits/rejected": 2.2506680488586426,
-      "logps/chosen": -621.7249145507812,
-      "logps/rejected": -7290.41015625,
-      "loss": 0.0005,
+      "logits/chosen": -0.8737711906433105,
+      "logits/rejected": 2.419881582260132,
+      "logps/chosen": -592.2354125976562,
+      "logps/rejected": -7475.37109375,
+      "loss": 0.0009,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.408754348754883,
-      "rewards/margins": 63.805747985839844,
-      "rewards/rejected": -66.21451568603516,
+      "rewards/chosen": -2.113858461380005,
+      "rewards/margins": 65.95025634765625,
+      "rewards/rejected": -68.0641098022461,
       "step": 270
     },
     {
       "epoch": 0.7954545454545454,
-      "grad_norm": 0.06722410834807425,
+      "grad_norm": 0.17275320368872452,
       "learning_rate": 6.135884496044244e-08,
-      "logits/chosen": -0.5607927441596985,
-      "logits/rejected": 2.5057239532470703,
-      "logps/chosen": -604.9693603515625,
-      "logps/rejected": -7248.1767578125,
-      "loss": 0.001,
+      "logits/chosen": -0.8250367045402527,
+      "logits/rejected": 2.680527687072754,
+      "logps/chosen": -574.1306762695312,
+      "logps/rejected": -7469.40087890625,
+      "loss": 0.0022,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.257964611053467,
-      "rewards/margins": 63.7749137878418,
-      "rewards/rejected": -66.03287506103516,
+      "rewards/chosen": -1.9495769739151,
+      "rewards/margins": 66.2955322265625,
+      "rewards/rejected": -68.24510955810547,
       "step": 280
     },
     {
       "epoch": 0.8238636363636364,
-      "grad_norm": 0.007877917419381247,
+      "grad_norm": 0.036939229346128824,
       "learning_rate": 4.600710195020982e-08,
-      "logits/chosen": -0.44481128454208374,
-      "logits/rejected": 2.559147357940674,
-      "logps/chosen": -628.8250732421875,
-      "logps/rejected": -7324.6533203125,
-      "loss": 0.0027,
+      "logits/chosen": -0.6854007244110107,
+      "logits/rejected": 2.711479902267456,
+      "logps/chosen": -595.4219970703125,
+      "logps/rejected": -7571.99853515625,
+      "loss": 0.0054,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.3968186378479004,
-      "rewards/margins": 64.43711853027344,
-      "rewards/rejected": -66.83393859863281,
+      "rewards/chosen": -2.0627870559692383,
+      "rewards/margins": 67.24461364746094,
+      "rewards/rejected": -69.30741119384766,
       "step": 290
     },
     {
       "epoch": 0.8522727272727273,
-      "grad_norm": 0.5905211140332576,
+      "grad_norm": 1.0005598289334474,
       "learning_rate": 3.2669931390104374e-08,
-      "logits/chosen": -0.4180712103843689,
-      "logits/rejected": 2.4567205905914307,
-      "logps/chosen": -577.669921875,
-      "logps/rejected": -7993.05859375,
-      "loss": 0.0009,
+      "logits/chosen": -0.7712670564651489,
+      "logits/rejected": 2.537261724472046,
+      "logps/chosen": -546.4788818359375,
+      "logps/rejected": -8221.009765625,
+      "loss": 0.0017,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.214153289794922,
-      "rewards/margins": 71.5081787109375,
-      "rewards/rejected": -73.72233581542969,
+      "rewards/chosen": -1.9022438526153564,
+      "rewards/margins": 74.09959411621094,
+      "rewards/rejected": -76.00184631347656,
       "step": 300
     },
     {
       "epoch": 0.8522727272727273,
-      "eval_logits/chosen": -0.2549493908882141,
-      "eval_logits/rejected": 2.575054407119751,
-      "eval_logps/chosen": -608.2271728515625,
-      "eval_logps/rejected": -7665.2216796875,
-      "eval_loss": 7.371166429948062e-05,
+      "eval_logits/chosen": -0.6079820394515991,
+      "eval_logits/rejected": 2.6897590160369873,
+      "eval_logps/chosen": -575.6599731445312,
+      "eval_logps/rejected": -7868.724609375,
+      "eval_loss": 0.0006528676021844149,
       "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -2.412454843521118,
-      "eval_rewards/margins": 67.83187866210938,
-      "eval_rewards/rejected": -70.24433898925781,
-      "eval_runtime": 192.5021,
-      "eval_samples_per_second": 20.285,
-      "eval_steps_per_second": 0.322,
+      "eval_rewards/chosen": -2.086782693862915,
+      "eval_rewards/margins": 70.19257354736328,
+      "eval_rewards/rejected": -72.27935791015625,
+      "eval_runtime": 193.3061,
+      "eval_samples_per_second": 20.201,
+      "eval_steps_per_second": 0.321,
       "step": 300
     },
     {
       "epoch": 0.8806818181818182,
-      "grad_norm": 0.04237285499239183,
+      "grad_norm": 0.0660494270632579,
       "learning_rate": 2.147904716149135e-08,
-      "logits/chosen": -0.1846945583820343,
-      "logits/rejected": 2.5058579444885254,
-      "logps/chosen": -589.5569458007812,
-      "logps/rejected": -7777.6474609375,
-      "loss": 0.0018,
+      "logits/chosen": -0.5389717817306519,
+      "logits/rejected": 2.602092981338501,
+      "logps/chosen": -555.8554077148438,
+      "logps/rejected": -7988.98974609375,
+      "loss": 0.0032,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.2749037742614746,
-      "rewards/margins": 68.49655151367188,
-      "rewards/rejected": -70.77145385742188,
+      "rewards/chosen": -1.9378883838653564,
+      "rewards/margins": 70.9469985961914,
+      "rewards/rejected": -72.8848876953125,
       "step": 310
     },
     {
       "epoch": 0.9090909090909091,
-      "grad_norm": 0.016783305893164634,
+      "grad_norm": 0.0488754281750263,
       "learning_rate": 1.254496706805433e-08,
-      "logits/chosen": -0.39586615562438965,
-      "logits/rejected": 2.4627411365509033,
-      "logps/chosen": -622.9832763671875,
-      "logps/rejected": -7786.82421875,
-      "loss": 0.0003,
+      "logits/chosen": -0.7335542440414429,
+      "logits/rejected": 2.5863664150238037,
+      "logps/chosen": -590.0091552734375,
+      "logps/rejected": -8039.3125,
+      "loss": 0.0007,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.403102159500122,
-      "rewards/margins": 68.9453125,
-      "rewards/rejected": -71.3484115600586,
+      "rewards/chosen": -2.0733609199523926,
+      "rewards/margins": 71.79994201660156,
+      "rewards/rejected": -73.87330627441406,
       "step": 320
     },
     {
       "epoch": 0.9375,
-      "grad_norm": 0.149850718402827,
+      "grad_norm": 0.2519430705585286,
       "learning_rate": 5.955921395237318e-09,
-      "logits/chosen": -0.37074390053749084,
-      "logits/rejected": 2.418095588684082,
-      "logps/chosen": -577.0670776367188,
-      "logps/rejected": -7757.859375,
-      "loss": 0.0004,
+      "logits/chosen": -0.6468337178230286,
+      "logits/rejected": 2.546152353286743,
+      "logps/chosen": -548.1500244140625,
+      "logps/rejected": -8072.88037109375,
+      "loss": 0.0006,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.2179901599884033,
-      "rewards/margins": 68.88307189941406,
-      "rewards/rejected": -71.10105895996094,
+      "rewards/chosen": -1.928818941116333,
+      "rewards/margins": 72.32246398925781,
+      "rewards/rejected": -74.25126647949219,
       "step": 330
     },
     {
       "epoch": 0.9659090909090909,
-      "grad_norm": 0.24861872079475503,
+      "grad_norm": 0.8725515919272617,
       "learning_rate": 1.7769815745066474e-09,
-      "logits/chosen": -0.6141588687896729,
-      "logits/rejected": 2.2593562602996826,
-      "logps/chosen": -593.7769775390625,
-      "logps/rejected": -7168.8701171875,
-      "loss": 0.0006,
+      "logits/chosen": -0.8968937993049622,
+      "logits/rejected": 2.3741860389709473,
+      "logps/chosen": -561.4879760742188,
+      "logps/rejected": -7435.28125,
+      "loss": 0.001,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": -2.2479617595672607,
-      "rewards/margins": 63.35308074951172,
-      "rewards/rejected": -65.60104370117188,
+      "rewards/chosen": -1.9250719547271729,
+      "rewards/margins": 66.34008026123047,
+      "rewards/rejected": -68.26514434814453,
       "step": 340
     },
     {
       "epoch": 0.9943181818181818,
-      "grad_norm": 0.890402381245667,
+      "grad_norm": 3.1933280173345713,
       "learning_rate": 4.9417557483610875e-11,
-      "logits/chosen": -0.3333211839199066,
-      "logits/rejected": 2.404022455215454,
-      "logps/chosen": -621.5592041015625,
-      "logps/rejected": -7657.5458984375,
-      "loss": 0.0024,
-      "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": -2.472914457321167,
-      "rewards/margins": 67.62696075439453,
-      "rewards/rejected": -70.0998764038086,
+      "logits/chosen": -0.6227324604988098,
+      "logits/rejected": 2.5037405490875244,
+      "logps/chosen": -589.6453247070312,
+      "logps/rejected": -7948.50927734375,
+      "loss": 0.0036,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.153775215148926,
+      "rewards/margins": 70.8557357788086,
+      "rewards/rejected": -73.00951385498047,
       "step": 350
     },
     {
       "epoch": 0.9943181818181818,
-      "eval_logits/chosen": -0.21884487569332123,
-      "eval_logits/rejected": 2.5753109455108643,
-      "eval_logps/chosen": -609.166015625,
-      "eval_logps/rejected": -7672.18896484375,
-      "eval_loss": 7.062770600896329e-05,
+      "eval_logits/chosen": -0.5257502198219299,
+      "eval_logits/rejected": 2.7151966094970703,
+      "eval_logps/chosen": -577.6112670898438,
+      "eval_logps/rejected": -7961.02392578125,
+      "eval_loss": 0.000620901002548635,
       "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -2.4218428134918213,
-      "eval_rewards/margins": 67.89216613769531,
-      "eval_rewards/rejected": -70.31401062011719,
-      "eval_runtime": 191.9742,
-      "eval_samples_per_second": 20.341,
-      "eval_steps_per_second": 0.323,
+      "eval_rewards/chosen": -2.106295585632324,
+      "eval_rewards/margins": 71.09603881835938,
+      "eval_rewards/rejected": -73.20234680175781,
+      "eval_runtime": 193.4431,
+      "eval_samples_per_second": 20.187,
+      "eval_steps_per_second": 0.321,
       "step": 350
     },
     {
       "epoch": 1.0,
       "step": 352,
       "total_flos": 0.0,
-      "train_loss": 0.042729352997742935,
-      "train_runtime": 10052.8601,
-      "train_samples_per_second": 4.476,
+      "train_loss": 0.08502711300074721,
+      "train_runtime": 9968.7204,
+      "train_samples_per_second": 4.514,
       "train_steps_per_second": 0.035
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b3e28f271f9978004e67023838acf232e8d98af3ef0ec8816fbc2113bcb47a32
-size 7480
+oid sha256:2f90a87fb6fd9aa61f5033db9c384a612bcf2c0d3c1b8b9a6be2f8e6177929fc
+size 7608