Model save

README.md

@@ -2,16 +2,9 @@
 license: apache-2.0
 base_model: hZzy/qwen2.5-0.5b-sft-news-IFT
 tags:
-- alignment-handbook
-- ndcg
 - trl
 - expo
 - generated_from_trainer
-- trl
-- expo
-- generated_from_trainer
-datasets:
-- hZzy/train_pairwise
 model-index:
 - name: qwen2.5-0.5b-expo-L2EXPO-ES-0.1
   results: []
@@ -20,21 +13,21 @@ model-index:
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 should probably proofread and complete it, then remove this comment. -->
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/zhiyuzha-university-of-florida/huggingface/runs/z6ixm6bo)
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/zhiyuzha-university-of-florida/huggingface/runs/a4jed762)
 # qwen2.5-0.5b-expo-L2EXPO-ES-0.1
 
-This model is a fine-tuned version of [hZzy/qwen2.5-0.5b-sft-news-IFT](https://huggingface.co/hZzy/qwen2.5-0.5b-sft-news-IFT) on the hZzy/train_pairwise dataset.
+This model is a fine-tuned version of [hZzy/qwen2.5-0.5b-sft-news-IFT](https://huggingface.co/hZzy/qwen2.5-0.5b-sft-news-IFT) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.4217
-- Logps: -89.1060
-- Logits: -1.3837
-- Objective: 0.4142
-- Dpo Loss: 0.6791
-- Regularize: 0.4142
-- Ranking Simple: 0.5347
+- Loss: 0.6428
+- Logps: -79.5818
+- Logits: -0.6068
+- Objective: 0.6266
+- Dpo Loss: 0.7211
+- Regularize: 0.6266
+- Ranking Simple: 0.5316
 - Ranking Idealized: 0.6030
 - Ranking Idealized Expo: 0.5223
-- Wo Beta: 15.9847
+- Wo Beta: 14.3406
 
 ## Model description
 
@@ -53,7 +46,7 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 1e-06
+- learning_rate: 5e-06
 - train_batch_size: 4
 - eval_batch_size: 4
 - seed: 42
@@ -69,34 +62,34 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch  | Step | Dpo Loss | Logits  | Logps    | Validation Loss | Objective | Ranking Idealized | Ranking Idealized Expo | Ranking Simple | Regularize | Wo Beta |
-|:-------------:|:------:|:----:|:--------:|:-------:|:--------:|:---------------:|:---------:|:-----------------:|:----------------------:|:--------------:|:----------:|:-------:|
-| 0.4117        | 0.1417 | 50   | 0.6893   | -1.4691 | -90.8535 | 0.4102          | 0.4090    | 0.6030            | 0.5223                 | 0.5248         | 0.4090     | 16.3208 |
-| 0.3871        | 0.2834 | 100  | 0.6833   | -1.5346 | -91.2757 | 0.4049          | 0.4029    | 0.6030            | 0.5223                 | 0.5316         | 0.4029     | 16.2699 |
-| 0.3451        | 0.4251 | 150  | 0.6789   | -1.4902 | -91.1637 | 0.4013          | 0.3996    | 0.6030            | 0.5223                 | 0.5347         | 0.3996     | 16.5907 |
-| 0.3166        | 0.5668 | 200  | 0.6811   | -1.4523 | -93.2695 | 0.4148          | 0.4132    | 0.6030            | 0.5223                 | 0.5316         | 0.4132     | 16.3512 |
-| 0.2939        | 0.7085 | 250  | 0.6790   | -1.5465 | -90.5537 | 0.4131          | 0.4077    | 0.6030            | 0.5223                 | 0.5342         | 0.4077     | 16.4807 |
-| 0.2655        | 0.8503 | 300  | 0.6806   | -1.4553 | -91.3521 | 0.4126          | 0.4082    | 0.6030            | 0.5223                 | 0.5311         | 0.4082     | 16.4429 |
-| 0.2513        | 0.9920 | 350  | 0.6782   | -1.4532 | -91.2408 | 0.4110          | 0.4044    | 0.6030            | 0.5223                 | 0.5352         | 0.4044     | 16.3768 |
-| 0.2206        | 1.1337 | 400  | 0.4128   | -87.3470| -1.4764  | 0.4049          | 0.6769    | 0.4049            | 0.5336                 | 0.6030         | 0.5223     | 16.2024 |
-| 0.2077        | 1.2754 | 450  | 0.4144   | -89.8793| -1.4177  | 0.4106          | 0.6788    | 0.4106            | 0.5331                 | 0.6030         | 0.5223     | 16.1977 |
-| 0.1943        | 1.4171 | 500  | 0.4169   | -87.6699| -1.4544  | 0.4092          | 0.6782    | 0.4092            | 0.5352                 | 0.6030         | 0.5223     | 16.0510 |
-| 0.1879        | 1.5588 | 550  | 0.4173   | -89.0111| -1.4268  | 0.4102          | 0.6787    | 0.4102            | 0.5347                 | 0.6030         | 0.5223     | 16.0707 |
-| 0.1768        | 1.7005 | 600  | 0.4190   | -87.0605| -1.4411  | 0.4116          | 0.6796    | 0.4116            | 0.5352                 | 0.6030         | 0.5223     | 16.0697 |
-| 0.1736        | 1.8422 | 650  | 0.4219   | -90.0508| -1.4601  | 0.4144          | 0.6802    | 0.4144            | 0.5347                 | 0.6030         | 0.5223     | 16.1057 |
-| 0.1598        | 1.9839 | 700  | 0.4217   | -90.5630| -1.4110  | 0.4148          | 0.6799    | 0.4148            | 0.5362                 | 0.6030         | 0.5223     | 16.0493 |
-| 0.1454        | 2.1256 | 750  | 0.4215   | -89.5433| -1.3859  | 0.4151          | 0.6797    | 0.4151            | 0.5316                 | 0.6030         | 0.5223     | 16.0459 |
-| 0.1333        | 2.2674 | 800  | 0.4217   | -89.1060| -1.3837  | 0.4142          | 0.6791    | 0.4142            | 0.5347                 | 0.6030         | 0.5223     | 15.9847 |
-| 0.1287        | 2.4091 | 850  | 0.4241   | -88.6145| -1.3856  | 0.4153          | 0.6795    | 0.4153            | 0.5357                 | 0.6030         | 0.5223     | 15.9979 |
-| 0.12          | 2.5508 | 900  | 0.4207   | -88.6663| -1.3921  | 0.4129          | 0.6795    | 0.4129            | 0.5331                 | 0.6030         | 0.5223     | 16.0698 |
-| 0.1148        | 2.6925 | 950  | 0.4215   | -88.2854| -1.3690  | 0.4149          | 0.6792    | 0.4149            | 0.5336                 | 0.6030         | 0.5223     | 16.0513 |
-| 0.1068        | 2.8342 | 1000 | 0.4229   | -89.1782| -1.3724  | 0.4168          | 0.6809    | 0.4168            | 0.5321                 | 0.6030         | 0.5223     | 16.0722 |
-| 0.0991        | 2.9759 | 1050 | 0.4210   | -88.9607| -1.3982  | 0.4141          | 0.6792    | 0.4141            | 0.5336                 | 0.6030         | 0.5223     | 16.0444 |
+| Training Loss | Epoch  | Step | Validation Loss | Logps    | Logits  | Objective | Dpo Loss | Regularize | Ranking Simple | Ranking Idealized | Ranking Idealized Expo | Wo Beta |
+|:-------------:|:------:|:----:|:---------------:|:--------:|:-------:|:---------:|:--------:|:----------:|:--------------:|:-----------------:|:----------------------:|:-------:|
+| 0.4017        | 0.1417 | 50   | 0.4165          | -93.1726 | -1.5024 | 0.4149    | 0.6868   | 0.4149     | 0.5259         | 0.6030            | 0.5223                 | 16.4267 |
+| 0.3777        | 0.2834 | 100  | 0.4360          | -92.8653 | -1.4775 | 0.4269    | 0.6818   | 0.4269     | 0.5316         | 0.6030            | 0.5223                 | 16.2439 |
+| 0.4057        | 0.4251 | 150  | 0.4911          | -84.1774 | -1.2946 | 0.4805    | 0.6897   | 0.4805     | 0.5383         | 0.6030            | 0.5223                 | 15.6306 |
+| 0.4475        | 0.5668 | 200  | 0.5660          | -89.7342 | -0.9897 | 0.5515    | 0.7103   | 0.5515     | 0.5316         | 0.6030            | 0.5223                 | 15.1280 |
+| 0.455         | 0.7085 | 250  | 0.5978          | -78.1917 | -1.0033 | 0.5822    | 0.7171   | 0.5822     | 0.5311         | 0.6030            | 0.5223                 | 14.6763 |
+| 0.4337        | 0.8503 | 300  | 0.5993          | -78.8918 | -0.6761 | 0.5779    | 0.7105   | 0.5779     | 0.5300         | 0.6030            | 0.5223                 | 14.9196 |
+| 0.4039        | 0.9920 | 350  | 0.5978          | -75.1520 | -0.7968 | 0.5765    | 0.7078   | 0.5765     | 0.5290         | 0.6030            | 0.5223                 | 14.6531 |
+| 0.3729        | 1.1337 | 400  | 0.6180          | -75.1433 | -0.5569 | 0.6000    | 0.7153   | 0.6000     | 0.5228         | 0.6030            | 0.5223                 | 14.6471 |
+| 0.3454        | 1.2754 | 450  | 0.6316          | -76.2289 | -0.6214 | 0.6131    | 0.7165   | 0.6131     | 0.5336         | 0.6030            | 0.5223                 | 14.5034 |
+| 0.3226        | 1.4171 | 500  | 0.6255          | -77.6040 | -0.5608 | 0.6084    | 0.7204   | 0.6084     | 0.5285         | 0.6030            | 0.5223                 | 14.4998 |
+| 0.3133        | 1.5588 | 550  | 0.6282          | -78.6291 | -0.6736 | 0.6138    | 0.7139   | 0.6138     | 0.5336         | 0.6030            | 0.5223                 | 14.4069 |
+| 0.2944        | 1.7005 | 600  | 0.6321          | -78.9179 | -0.5620 | 0.6139    | 0.7175   | 0.6139     | 0.5357         | 0.6030            | 0.5223                 | 14.6142 |
+| 0.2915        | 1.8422 | 650  | 0.6321          | -77.4437 | -0.7021 | 0.6157    | 0.7138   | 0.6157     | 0.5367         | 0.6030            | 0.5223                 | 14.3858 |
+| 0.2675        | 1.9839 | 700  | 0.6386          | -79.3600 | -0.5612 | 0.6233    | 0.7185   | 0.6233     | 0.5290         | 0.6030            | 0.5223                 | 14.3171 |
+| 0.2415        | 2.1256 | 750  | 0.6405          | -80.0990 | -0.6174 | 0.6263    | 0.7177   | 0.6263     | 0.5347         | 0.6030            | 0.5223                 | 14.4302 |
+| 0.2263        | 2.2674 | 800  | 0.6458          | -79.3784 | -0.5665 | 0.6297    | 0.7206   | 0.6297     | 0.5347         | 0.6030            | 0.5223                 | 14.3163 |
+| 0.2148        | 2.4091 | 850  | 0.6436          | -79.0806 | -0.5793 | 0.6276    | 0.7192   | 0.6276     | 0.5362         | 0.6030            | 0.5223                 | 14.4263 |
+| 0.1993        | 2.5508 | 900  | 0.6454          | -80.3815 | -0.5621 | 0.6302    | 0.7217   | 0.6302     | 0.5342         | 0.6030            | 0.5223                 | 14.4491 |
+| 0.1887        | 2.6925 | 950  | 0.6443          | -79.1446 | -0.6216 | 0.6274    | 0.7204   | 0.6274     | 0.5336         | 0.6030            | 0.5223                 | 14.3186 |
+| 0.1764        | 2.8342 | 1000 | 0.6399          | -79.7721 | -0.6087 | 0.6246    | 0.7200   | 0.6246     | 0.5336         | 0.6030            | 0.5223                 | 14.4502 |
+| 0.163         | 2.9759 | 1050 | 0.6428          | -79.5818 | -0.6068 | 0.6266    | 0.7211   | 0.6266     | 0.5316         | 0.6030            | 0.5223                 | 14.3406 |
 
 
 ### Framework versions
 
 - Transformers 4.42.0
 - Pytorch 2.3.0+cu121
-- Datasets 2.19.1
+- Datasets 3.2.0
 - Tokenizers 0.19.1

all_results.json

@@ -1,23 +1,9 @@
 {
     "epoch": 2.975909305621162,
-    "eval_dpo_loss": 0.6790949106216431,
-    "eval_logits": -1.383711576461792,
-    "eval_logps": -89.10604858398438,
-    "eval_loss": 0.4217349588871002,
-    "eval_objective": 0.4141845107078552,
-    "eval_ranking_idealized": 0.6030020713806152,
-    "eval_ranking_idealized_expo": 0.5222567319869995,
-    "eval_ranking_simple": 0.534679114818573,
-    "eval_regularize": 0.4141845107078552,
-    "eval_runtime": 467.1821,
-    "eval_samples": 5790,
-    "eval_samples_per_second": 12.393,
-    "eval_steps_per_second": 1.034,
-    "eval_wo_beta": 15.984663963317871,
     "total_flos": 0.0,
-    "train_loss": 0.10327800432840983,
-    "train_runtime": 29791.2438,
+    "train_loss": 0.3115594020343962,
+    "train_runtime": 28931.7373,
     "train_samples": 50802,
-    "train_samples_per_second": 8.526,
-    "train_steps_per_second": 0.059
+    "train_samples_per_second": 8.78,
+    "train_steps_per_second": 0.061
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e4daa952865ca064ad4313f6918c8e382449d734e68ba33a1affdf79c5ed511e
+oid sha256:c71576ee2010dd8563a4e11771c0e95e7de75516cc6b9df519854a974f5b7987
 size 1975192208

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 2.975909305621162,
     "total_flos": 0.0,
-    "train_loss": 0.10327800432840983,
-    "train_runtime": 29791.2438,
+    "train_loss": 0.3115594020343962,
+    "train_runtime": 28931.7373,
     "train_samples": 50802,
-    "train_samples_per_second": 8.526,
-    "train_steps_per_second": 0.059
+    "train_samples_per_second": 8.78,
+    "train_steps_per_second": 0.061
 }

trainer_state.json

@@ -1,5 +1,5 @@
 {
-  "best_metric": 15.984663963317871,
+  "best_metric": 14.316285133361816,
   "best_model_checkpoint": "./qwen2.5-0.5b/qwen2.5-0.5b-expo-L2EXPO-ES-0.1/checkpoint-800",
   "epoch": 2.975909305621162,
   "eval_steps": 50,
@@ -11,8 +11,8 @@
     {
       "dpo_loss": 0.6931471824645996,
       "epoch": 0.002834199338686821,
-      "grad_norm": 36.88506026977242,
-      "learning_rate": 5.681818181818181e-09,
+      "grad_norm": 36.88507599678088,
+      "learning_rate": 2.840909090909091e-08,
       "logits": -1.359458565711975,
       "logps": -84.69721221923828,
       "loss": 0.3913,
@@ -25,706 +25,706 @@
       "wo_beta": 14.830931663513184
     },
     {
-      "dpo_loss": 0.689619779586792,
+      "dpo_loss": 0.6800611615180969,
       "epoch": 0.14170996693434104,
-      "grad_norm": 35.61374868059241,
-      "learning_rate": 2.840909090909091e-07,
-      "logits": -1.4646235704421997,
-      "logps": -83.92658996582031,
-      "loss": 0.4117,
-      "objective": 0.4160037934780121,
+      "grad_norm": 34.65452784204521,
+      "learning_rate": 1.4204545454545458e-06,
+      "logits": -1.481619119644165,
+      "logps": -83.80532836914062,
+      "loss": 0.4017,
+      "objective": 0.4062296152114868,
       "ranking_idealized": 0.608418345451355,
       "ranking_idealized_expo": 0.5229591727256775,
-      "ranking_simple": 0.5255101919174194,
-      "regularize": 0.4160037934780121,
+      "ranking_simple": 0.5250850319862366,
+      "regularize": 0.4062296152114868,
       "step": 50,
-      "wo_beta": 15.691085815429688
+      "wo_beta": 15.713354110717773
     },
     {
       "epoch": 0.14170996693434104,
-      "eval_dpo_loss": 0.6893064975738525,
-      "eval_logits": -1.4691106081008911,
-      "eval_logps": -90.8535385131836,
-      "eval_loss": 0.4102482497692108,
-      "eval_objective": 0.4089997410774231,
+      "eval_dpo_loss": 0.6868039965629578,
+      "eval_logits": -1.5024017095565796,
+      "eval_logps": -93.17259979248047,
+      "eval_loss": 0.4164615273475647,
+      "eval_objective": 0.41487643122673035,
       "eval_ranking_idealized": 0.6030020713806152,
       "eval_ranking_idealized_expo": 0.5222567319869995,
-      "eval_ranking_simple": 0.5248447060585022,
-      "eval_regularize": 0.4089997410774231,
-      "eval_runtime": 309.1952,
-      "eval_samples_per_second": 18.726,
-      "eval_steps_per_second": 1.562,
-      "eval_wo_beta": 16.32078742980957,
+      "eval_ranking_simple": 0.5258799195289612,
+      "eval_regularize": 0.41487643122673035,
+      "eval_runtime": 308.9601,
+      "eval_samples_per_second": 18.74,
+      "eval_steps_per_second": 1.563,
+      "eval_wo_beta": 16.426729202270508,
       "step": 50
     },
     {
-      "dpo_loss": 0.666653573513031,
+      "dpo_loss": 0.6331456303596497,
       "epoch": 0.2834199338686821,
-      "grad_norm": 39.001717769013226,
-      "learning_rate": 5.681818181818182e-07,
-      "logits": -1.5049233436584473,
-      "logps": -84.32721710205078,
-      "loss": 0.3871,
-      "objective": 0.3919191360473633,
+      "grad_norm": 29.48988078429707,
+      "learning_rate": 2.8409090909090916e-06,
+      "logits": -1.514231562614441,
+      "logps": -84.27212524414062,
+      "loss": 0.3777,
+      "objective": 0.38673925399780273,
       "ranking_idealized": 0.6016666889190674,
       "ranking_idealized_expo": 0.5141666531562805,
-      "ranking_simple": 0.5274999737739563,
-      "regularize": 0.3919191360473633,
+      "ranking_simple": 0.5520833134651184,
+      "regularize": 0.38673925399780273,
       "step": 100,
-      "wo_beta": 15.436232566833496
+      "wo_beta": 15.562942504882812
     },
     {
       "epoch": 0.2834199338686821,
-      "eval_dpo_loss": 0.6833388805389404,
-      "eval_logits": -1.5346405506134033,
-      "eval_logps": -91.27565002441406,
-      "eval_loss": 0.4049375057220459,
-      "eval_objective": 0.40290772914886475,
+      "eval_dpo_loss": 0.6818161606788635,
+      "eval_logits": -1.4774748086929321,
+      "eval_logps": -92.86526489257812,
+      "eval_loss": 0.4359625577926636,
+      "eval_objective": 0.4269382953643799,
       "eval_ranking_idealized": 0.6030020713806152,
       "eval_ranking_idealized_expo": 0.5222567319869995,
       "eval_ranking_simple": 0.531573474407196,
-      "eval_regularize": 0.40290772914886475,
-      "eval_runtime": 308.7302,
-      "eval_samples_per_second": 18.754,
-      "eval_steps_per_second": 1.564,
-      "eval_wo_beta": 16.269933700561523,
+      "eval_regularize": 0.4269382953643799,
+      "eval_runtime": 308.6402,
+      "eval_samples_per_second": 18.76,
+      "eval_steps_per_second": 1.565,
+      "eval_wo_beta": 16.243934631347656,
       "step": 100
     },
     {
-      "dpo_loss": 0.6460586190223694,
+      "dpo_loss": 0.6157870292663574,
       "epoch": 0.42512990080302315,
-      "grad_norm": 33.54647406338888,
-      "learning_rate": 8.522727272727273e-07,
-      "logits": -1.5278972387313843,
-      "logps": -86.29918670654297,
-      "loss": 0.3451,
-      "objective": 0.3537640869617462,
+      "grad_norm": 23.64962148842917,
+      "learning_rate": 4.2613636363636365e-06,
+      "logits": -1.363812804222107,
+      "logps": -83.03893280029297,
+      "loss": 0.4057,
+      "objective": 0.4109911620616913,
       "ranking_idealized": 0.6066666841506958,
       "ranking_idealized_expo": 0.5287500023841858,
-      "ranking_simple": 0.5441666841506958,
-      "regularize": 0.3537640869617462,
+      "ranking_simple": 0.5679166913032532,
+      "regularize": 0.4109911620616913,
       "step": 150,
-      "wo_beta": 15.559906005859375
+      "wo_beta": 15.67545223236084
     },
     {
       "epoch": 0.42512990080302315,
-      "eval_dpo_loss": 0.6788700819015503,
-      "eval_logits": -1.4902477264404297,
-      "eval_logps": -91.16365051269531,
-      "eval_loss": 0.40134957432746887,
-      "eval_objective": 0.39963677525520325,
+      "eval_dpo_loss": 0.6897013783454895,
+      "eval_logits": -1.2946008443832397,
+      "eval_logps": -84.17744445800781,
+      "eval_loss": 0.49110475182533264,
+      "eval_objective": 0.48045814037323,
       "eval_ranking_idealized": 0.6030020713806152,
       "eval_ranking_idealized_expo": 0.5222567319869995,
-      "eval_ranking_simple": 0.534679114818573,
-      "eval_regularize": 0.39963677525520325,
-      "eval_runtime": 313.2125,
-      "eval_samples_per_second": 18.486,
-      "eval_steps_per_second": 1.542,
-      "eval_wo_beta": 16.59067726135254,
+      "eval_ranking_simple": 0.5383023023605347,
+      "eval_regularize": 0.48045814037323,
+      "eval_runtime": 312.8899,
+      "eval_samples_per_second": 18.505,
+      "eval_steps_per_second": 1.544,
+      "eval_wo_beta": 15.630563735961914,
       "step": 150
     },
     {
-      "dpo_loss": 0.616335928440094,
+      "dpo_loss": 0.6037490963935852,
       "epoch": 0.5668398677373642,
-      "grad_norm": 40.07384713581775,
-      "learning_rate": 9.99433669591504e-07,
-      "logits": -1.5112419128417969,
-      "logps": -85.97952270507812,
-      "loss": 0.3166,
-      "objective": 0.31608816981315613,
+      "grad_norm": 20.792422526564724,
+      "learning_rate": 4.997168347957521e-06,
+      "logits": -1.1515488624572754,
+      "logps": -78.54210662841797,
+      "loss": 0.4475,
+      "objective": 0.4344017505645752,
       "ranking_idealized": 0.5924999713897705,
       "ranking_idealized_expo": 0.5166666507720947,
-      "ranking_simple": 0.5454166531562805,
-      "regularize": 0.31608816981315613,
+      "ranking_simple": 0.5537499785423279,
+      "regularize": 0.4344017505645752,
       "step": 200,
-      "wo_beta": 15.945281982421875
+      "wo_beta": 15.705690383911133
     },
     {
       "epoch": 0.5668398677373642,
-      "eval_dpo_loss": 0.681080162525177,
-      "eval_logits": -1.4522831439971924,
-      "eval_logps": -93.26953887939453,
-      "eval_loss": 0.4148460626602173,
-      "eval_objective": 0.41316577792167664,
+      "eval_dpo_loss": 0.7102847695350647,
+      "eval_logits": -0.9896814823150635,
+      "eval_logps": -89.7341537475586,
+      "eval_loss": 0.5660186409950256,
+      "eval_objective": 0.5515478253364563,
       "eval_ranking_idealized": 0.6030020713806152,
       "eval_ranking_idealized_expo": 0.5222567319869995,
       "eval_ranking_simple": 0.531573474407196,
-      "eval_regularize": 0.41316577792167664,
-      "eval_runtime": 308.9819,
-      "eval_samples_per_second": 18.739,
-      "eval_steps_per_second": 1.563,
-      "eval_wo_beta": 16.35118865966797,
+      "eval_regularize": 0.5515478253364563,
+      "eval_runtime": 307.7447,
+      "eval_samples_per_second": 18.814,
+      "eval_steps_per_second": 1.569,
+      "eval_wo_beta": 15.128002166748047,
       "step": 200
     },
     {
-      "dpo_loss": 0.5952258110046387,
+      "dpo_loss": 0.6019502878189087,
       "epoch": 0.7085498346717053,
-      "grad_norm": 34.54234310666894,
-      "learning_rate": 9.94624571028813e-07,
-      "logits": -1.5106024742126465,
-      "logps": -84.41282653808594,
-      "loss": 0.2939,
-      "objective": 0.2922481894493103,
+      "grad_norm": 19.149849902460005,
+      "learning_rate": 4.973122855144066e-06,
+      "logits": -0.9062835574150085,
+      "logps": -77.80750274658203,
+      "loss": 0.455,
+      "objective": 0.4496636390686035,
       "ranking_idealized": 0.5991666913032532,
       "ranking_idealized_expo": 0.5170833468437195,
-      "ranking_simple": 0.5612499713897705,
-      "regularize": 0.2922481894493103,
+      "ranking_simple": 0.5724999904632568,
+      "regularize": 0.4496636390686035,
       "step": 250,
-      "wo_beta": 16.29503631591797
+      "wo_beta": 16.00295639038086
     },
     {
       "epoch": 0.7085498346717053,
-      "eval_dpo_loss": 0.679045557975769,
-      "eval_logits": -1.5465130805969238,
-      "eval_logps": -90.55366516113281,
-      "eval_loss": 0.4130644202232361,
-      "eval_objective": 0.4076613783836365,
+      "eval_dpo_loss": 0.7171492576599121,
+      "eval_logits": -1.0032674074172974,
+      "eval_logps": -78.19169616699219,
+      "eval_loss": 0.5978298783302307,
+      "eval_objective": 0.5822399258613586,
       "eval_ranking_idealized": 0.6030020713806152,
       "eval_ranking_idealized_expo": 0.5222567319869995,
-      "eval_ranking_simple": 0.5341615080833435,
-      "eval_regularize": 0.4076613783836365,
-      "eval_runtime": 308.7233,
-      "eval_samples_per_second": 18.755,
-      "eval_steps_per_second": 1.565,
-      "eval_wo_beta": 16.480682373046875,
+      "eval_ranking_simple": 0.5310559272766113,
+      "eval_regularize": 0.5822399258613586,
+      "eval_runtime": 309.2921,
+      "eval_samples_per_second": 18.72,
+      "eval_steps_per_second": 1.562,
+      "eval_wo_beta": 14.676263809204102,
       "step": 250
     },
     {
-      "dpo_loss": 0.586772620677948,
+      "dpo_loss": 0.5993608236312866,
       "epoch": 0.8502598016060463,
-      "grad_norm": 34.42071615370769,
-      "learning_rate": 9.849553282839024e-07,
-      "logits": -1.5501227378845215,
-      "logps": -85.52662658691406,
-      "loss": 0.2655,
-      "objective": 0.27248048782348633,
+      "grad_norm": 17.89888160897824,
+      "learning_rate": 4.924776641419513e-06,
+      "logits": -0.8504629731178284,
+      "logps": -74.18943786621094,
+      "loss": 0.4337,
+      "objective": 0.42672449350357056,
       "ranking_idealized": 0.5799999833106995,
       "ranking_idealized_expo": 0.4970833361148834,
-      "ranking_simple": 0.5445833206176758,
-      "regularize": 0.27248048782348633,
+      "ranking_simple": 0.5529166460037231,
+      "regularize": 0.42672449350357056,
       "step": 300,
-      "wo_beta": 15.878409385681152
+      "wo_beta": 15.642317771911621
     },
     {
       "epoch": 0.8502598016060463,
-      "eval_dpo_loss": 0.6806344985961914,
-      "eval_logits": -1.4552550315856934,
-      "eval_logps": -91.35205078125,
-      "eval_loss": 0.4126306176185608,
-      "eval_objective": 0.4082447588443756,
+      "eval_dpo_loss": 0.710507333278656,
+      "eval_logits": -0.6760910153388977,
+      "eval_logps": -78.8918228149414,
+      "eval_loss": 0.5993344783782959,
+      "eval_objective": 0.5779486894607544,
       "eval_ranking_idealized": 0.6030020713806152,
       "eval_ranking_idealized_expo": 0.5222567319869995,
-      "eval_ranking_simple": 0.5310559272766113,
-      "eval_regularize": 0.4082447588443756,
-      "eval_runtime": 467.8777,
-      "eval_samples_per_second": 12.375,
-      "eval_steps_per_second": 1.032,
-      "eval_wo_beta": 16.44291877746582,
+      "eval_ranking_simple": 0.5300207138061523,
+      "eval_regularize": 0.5779486894607544,
+      "eval_runtime": 308.8688,
+      "eval_samples_per_second": 18.746,
+      "eval_steps_per_second": 1.564,
+      "eval_wo_beta": 14.919622421264648,
       "step": 300
     },
     {
-      "dpo_loss": 0.5711151361465454,
+      "dpo_loss": 0.5866905450820923,
       "epoch": 0.9919697685403873,
-      "grad_norm": 31.544141880018735,
-      "learning_rate": 9.705209506155634e-07,
-      "logits": -1.468552589416504,
-      "logps": -85.4828872680664,
-      "loss": 0.2513,
-      "objective": 0.24532364308834076,
+      "grad_norm": 18.680005245459032,
+      "learning_rate": 4.8526047530778175e-06,
+      "logits": -0.6557392477989197,
+      "logps": -72.1249008178711,
+      "loss": 0.4039,
+      "objective": 0.3963530361652374,
       "ranking_idealized": 0.60916668176651,
       "ranking_idealized_expo": 0.5270833373069763,
-      "ranking_simple": 0.5920833349227905,
-      "regularize": 0.24532364308834076,
+      "ranking_simple": 0.5874999761581421,
+      "regularize": 0.3963530361652374,
       "step": 350,
-      "wo_beta": 15.914597511291504
+      "wo_beta": 15.802534103393555
     },
     {
       "epoch": 0.9919697685403873,
-      "eval_dpo_loss": 0.6781877279281616,
-      "eval_logits": -1.4532182216644287,
-      "eval_logps": -91.24078369140625,
-      "eval_loss": 0.41097965836524963,
-      "eval_objective": 0.4043864607810974,
+      "eval_dpo_loss": 0.707767128944397,
+      "eval_logits": -0.7968087792396545,
+      "eval_logps": -75.1519775390625,
+      "eval_loss": 0.5977659821510315,
+      "eval_objective": 0.5765314102172852,
       "eval_ranking_idealized": 0.6030020713806152,
       "eval_ranking_idealized_expo": 0.5222567319869995,
-      "eval_ranking_simple": 0.5351966619491577,
-      "eval_regularize": 0.4043864607810974,
-      "eval_runtime": 471.3464,
-      "eval_samples_per_second": 12.284,
-      "eval_steps_per_second": 1.025,
-      "eval_wo_beta": 16.376806259155273,
+      "eval_ranking_simple": 0.5289855003356934,
+      "eval_regularize": 0.5765314102172852,
+      "eval_runtime": 309.2048,
+      "eval_samples_per_second": 18.725,
+      "eval_steps_per_second": 1.562,
+      "eval_wo_beta": 14.653112411499023,
       "step": 350
     },
     {
-      "dpo_loss": 0.5529460906982422,
+      "dpo_loss": 0.571822464466095,
       "epoch": 1.1336797354747283,
-      "grad_norm": 30.9851919334132,
-      "learning_rate": 9.514632691433106e-07,
-      "logits": -1.4837555885314941,
-      "logps": -83.03319549560547,
-      "loss": 0.2206,
-      "objective": 0.2209722399711609,
+      "grad_norm": 17.758722724655755,
+      "learning_rate": 4.757316345716554e-06,
+      "logits": -0.6229808926582336,
+      "logps": -71.7779541015625,
+      "loss": 0.3729,
+      "objective": 0.38071343302726746,
       "ranking_idealized": 0.6087499856948853,
       "ranking_idealized_expo": 0.5337499976158142,
-      "ranking_simple": 0.5941666960716248,
-      "regularize": 0.2209722399711609,
+      "ranking_simple": 0.596666693687439,
+      "regularize": 0.38071343302726746,
       "step": 400,
-      "wo_beta": 16.069154739379883
+      "wo_beta": 15.872475624084473
     },
     {
       "epoch": 1.1336797354747283,
-      "eval_dpo_loss": 0.6769281625747681,
-      "eval_logits": -1.4763766527175903,
-      "eval_logps": -87.34698486328125,
-      "eval_loss": 0.41283363103866577,
-      "eval_objective": 0.4048711359500885,
+      "eval_dpo_loss": 0.7153333425521851,
+      "eval_logits": -0.5569362044334412,
+      "eval_logps": -75.14326477050781,
+      "eval_loss": 0.6180254817008972,
+      "eval_objective": 0.6000439524650574,
       "eval_ranking_idealized": 0.6030020713806152,
       "eval_ranking_idealized_expo": 0.5222567319869995,
-      "eval_ranking_simple": 0.533643901348114,
-      "eval_regularize": 0.4048711359500885,
-      "eval_runtime": 481.9587,
-      "eval_samples_per_second": 12.013,
-      "eval_steps_per_second": 1.002,
-      "eval_wo_beta": 16.20241355895996,
+      "eval_ranking_simple": 0.522774338722229,
+      "eval_regularize": 0.6000439524650574,
+      "eval_runtime": 308.681,
+      "eval_samples_per_second": 18.757,
+      "eval_steps_per_second": 1.565,
+      "eval_wo_beta": 14.647075653076172,
       "step": 400
     },
     {
-      "dpo_loss": 0.5509263277053833,
+      "dpo_loss": 0.5686503052711487,
       "epoch": 1.2753897024090695,
-      "grad_norm": 33.28599031549051,
-      "learning_rate": 9.279695432253708e-07,
-      "logits": -1.4624412059783936,
-      "logps": -82.20590209960938,
-      "loss": 0.2077,
-      "objective": 0.2058807760477066,
+      "grad_norm": 17.208572847816768,
+      "learning_rate": 4.639847716126855e-06,
+      "logits": -0.558698296546936,
+      "logps": -71.48841094970703,
+      "loss": 0.3454,
+      "objective": 0.34283894300460815,
       "ranking_idealized": 0.5975000262260437,
       "ranking_idealized_expo": 0.5199999809265137,
-      "ranking_simple": 0.5841666460037231,
-      "regularize": 0.2058807760477066,
+      "ranking_simple": 0.59375,
+      "regularize": 0.34283894300460815,
       "step": 450,
-      "wo_beta": 16.36168098449707
+      "wo_beta": 16.339805603027344
     },
     {
       "epoch": 1.2753897024090695,
-      "eval_dpo_loss": 0.6788477897644043,
-      "eval_logits": -1.417741060256958,
-      "eval_logps": -89.87925720214844,
-      "eval_loss": 0.41441377997398376,
-      "eval_objective": 0.4105576276779175,
+      "eval_dpo_loss": 0.716464638710022,
+      "eval_logits": -0.6214241981506348,
+      "eval_logps": -76.22888946533203,
+      "eval_loss": 0.6315773725509644,
+      "eval_objective": 0.6131163239479065,
       "eval_ranking_idealized": 0.6030020713806152,
       "eval_ranking_idealized_expo": 0.5222567319869995,
-      "eval_ranking_simple": 0.5331262946128845,
-      "eval_regularize": 0.4105576276779175,
-      "eval_runtime": 479.889,
-      "eval_samples_per_second": 12.065,
-      "eval_steps_per_second": 1.006,
-      "eval_wo_beta": 16.197683334350586,
+      "eval_ranking_simple": 0.533643901348114,
+      "eval_regularize": 0.6131163239479065,
+      "eval_runtime": 309.544,
+      "eval_samples_per_second": 18.705,
+      "eval_steps_per_second": 1.56,
+      "eval_wo_beta": 14.503443717956543,
       "step": 450
     },
     {
-      "dpo_loss": 0.5497527718544006,
+      "dpo_loss": 0.5579800605773926,
       "epoch": 1.4170996693434104,
-      "grad_norm": 28.875129958259482,
-      "learning_rate": 9.002706204621802e-07,
-      "logits": -1.4527512788772583,
-      "logps": -82.58905029296875,
-      "loss": 0.1943,
-      "objective": 0.19459716975688934,
+      "grad_norm": 17.247428143304518,
+      "learning_rate": 4.501353102310901e-06,
+      "logits": -0.5506080985069275,
+      "logps": -72.05484008789062,
+      "loss": 0.3226,
+      "objective": 0.321167916059494,
       "ranking_idealized": 0.57833331823349,
       "ranking_idealized_expo": 0.4983333349227905,
-      "ranking_simple": 0.5625,
-      "regularize": 0.19459716975688934,
+      "ranking_simple": 0.5724999904632568,
+      "regularize": 0.321167916059494,
       "step": 500,
-      "wo_beta": 15.90869426727295
+      "wo_beta": 15.88575553894043
     },
     {
       "epoch": 1.4170996693434104,
-      "eval_dpo_loss": 0.6781710982322693,
-      "eval_logits": -1.454372763633728,
-      "eval_logps": -87.66993713378906,
-      "eval_loss": 0.4168953001499176,
-      "eval_objective": 0.40915447473526,
+      "eval_dpo_loss": 0.7203696370124817,
+      "eval_logits": -0.5608097910881042,
+      "eval_logps": -77.60398864746094,
+      "eval_loss": 0.6255373358726501,
+      "eval_objective": 0.6084341406822205,
       "eval_ranking_idealized": 0.6030020713806152,
       "eval_ranking_idealized_expo": 0.5222567319869995,
-      "eval_ranking_simple": 0.5351966619491577,
-      "eval_regularize": 0.40915447473526,
-      "eval_runtime": 488.9325,
-      "eval_samples_per_second": 11.842,
-      "eval_steps_per_second": 0.988,
-      "eval_wo_beta": 16.050954818725586,
+      "eval_ranking_simple": 0.5284678936004639,
+      "eval_regularize": 0.6084341406822205,
+      "eval_runtime": 308.7073,
+      "eval_samples_per_second": 18.756,
+      "eval_steps_per_second": 1.565,
+      "eval_wo_beta": 14.499795913696289,
       "step": 500
     },
     {
-      "dpo_loss": 0.5426554083824158,
+      "dpo_loss": 0.5586966872215271,
       "epoch": 1.5588096362777515,
-      "grad_norm": 30.318446040470374,
-      "learning_rate": 8.68638668405062e-07,
-      "logits": -1.4643546342849731,
-      "logps": -82.28606414794922,
-      "loss": 0.1879,
-      "objective": 0.1849183589220047,
+      "grad_norm": 16.23813175362919,
+      "learning_rate": 4.34319334202531e-06,
+      "logits": -0.5342339873313904,
+      "logps": -74.90955352783203,
+      "loss": 0.3133,
+      "objective": 0.31819403171539307,
       "ranking_idealized": 0.5945833325386047,
       "ranking_idealized_expo": 0.5116666555404663,
       "ranking_simple": 0.5854166746139526,
-      "regularize": 0.1849183589220047,
+      "regularize": 0.31819403171539307,
       "step": 550,
-      "wo_beta": 15.804949760437012
+      "wo_beta": 15.694311141967773
     },
     {
       "epoch": 1.5588096362777515,
-      "eval_dpo_loss": 0.6786677241325378,
-      "eval_logits": -1.4267942905426025,
-      "eval_logps": -89.01109313964844,
-      "eval_loss": 0.4173260033130646,
-      "eval_objective": 0.41023388504981995,
+      "eval_dpo_loss": 0.7138590216636658,
+      "eval_logits": -0.6736307144165039,
+      "eval_logps": -78.62907409667969,
+      "eval_loss": 0.6281688809394836,
+      "eval_objective": 0.613820493221283,
       "eval_ranking_idealized": 0.6030020713806152,
       "eval_ranking_idealized_expo": 0.5222567319869995,
-      "eval_ranking_simple": 0.534679114818573,
-      "eval_regularize": 0.41023388504981995,
-      "eval_runtime": 497.8976,
-      "eval_samples_per_second": 11.629,
-      "eval_steps_per_second": 0.97,
-      "eval_wo_beta": 16.070680618286133,
+      "eval_ranking_simple": 0.533643901348114,
+      "eval_regularize": 0.613820493221283,
+      "eval_runtime": 309.2677,
+      "eval_samples_per_second": 18.722,
+      "eval_steps_per_second": 1.562,
+      "eval_wo_beta": 14.406906127929688,
       "step": 550
     },
     {
-      "dpo_loss": 0.5401077270507812,
+      "dpo_loss": 0.5538429021835327,
       "epoch": 1.7005196032120926,
-      "grad_norm": 28.092335905186818,
-      "learning_rate": 8.333845002581458e-07,
-      "logits": -1.4184807538986206,
-      "logps": -81.81605529785156,
-      "loss": 0.1768,
-      "objective": 0.1826876848936081,
+      "grad_norm": 16.284668340767176,
+      "learning_rate": 4.16692250129073e-06,
+      "logits": -0.5140345692634583,
+      "logps": -74.2342758178711,
+      "loss": 0.2944,
+      "objective": 0.29470422863960266,
       "ranking_idealized": 0.6004166603088379,
       "ranking_idealized_expo": 0.51583331823349,
-      "ranking_simple": 0.5870833396911621,
-      "regularize": 0.1826876848936081,
+      "ranking_simple": 0.5895833373069763,
+      "regularize": 0.29470422863960266,
       "step": 600,
-      "wo_beta": 15.617234230041504
+      "wo_beta": 15.488865852355957
     },
     {
       "epoch": 1.7005196032120926,
-      "eval_dpo_loss": 0.6796310544013977,
-      "eval_logits": -1.4410563707351685,
-      "eval_logps": -87.06050872802734,
-      "eval_loss": 0.4189995527267456,
-      "eval_objective": 0.411603182554245,
+      "eval_dpo_loss": 0.717461884021759,
+      "eval_logits": -0.5620033740997314,
+      "eval_logps": -78.91792297363281,
+      "eval_loss": 0.6321352124214172,
+      "eval_objective": 0.6138916015625,
       "eval_ranking_idealized": 0.6030020713806152,
       "eval_ranking_idealized_expo": 0.5222567319869995,
-      "eval_ranking_simple": 0.5351966619491577,
-      "eval_regularize": 0.411603182554245,
-      "eval_runtime": 490.295,
-      "eval_samples_per_second": 11.809,
-      "eval_steps_per_second": 0.985,
-      "eval_wo_beta": 16.06967544555664,
+      "eval_ranking_simple": 0.5357142686843872,
+      "eval_regularize": 0.6138916015625,
+      "eval_runtime": 308.6792,
+      "eval_samples_per_second": 18.757,
+      "eval_steps_per_second": 1.565,
+      "eval_wo_beta": 14.614200592041016,
       "step": 600
     },
     {
-      "dpo_loss": 0.5417632460594177,
+      "dpo_loss": 0.5554340481758118,
       "epoch": 1.8422295701464337,
-      "grad_norm": 30.564010457091317,
-      "learning_rate": 7.948545208509811e-07,
-      "logits": -1.4855570793151855,
-      "logps": -82.33342742919922,
-      "loss": 0.1736,
-      "objective": 0.16996659338474274,
+      "grad_norm": 15.91791899711329,
+      "learning_rate": 3.974272604254906e-06,
+      "logits": -0.5311375260353088,
+      "logps": -74.39502716064453,
+      "loss": 0.2915,
+      "objective": 0.28872814774513245,
       "ranking_idealized": 0.6058333516120911,
       "ranking_idealized_expo": 0.5295833349227905,
-      "ranking_simple": 0.5916666388511658,
-      "regularize": 0.16996659338474274,
+      "ranking_simple": 0.5991666913032532,
+      "regularize": 0.28872814774513245,
       "step": 650,
-      "wo_beta": 16.523035049438477
+      "wo_beta": 16.528623580932617
     },
     {
       "epoch": 1.8422295701464337,
-      "eval_dpo_loss": 0.680221676826477,
-      "eval_logits": -1.4600505828857422,
-      "eval_logps": -90.0507583618164,
-      "eval_loss": 0.4218791425228119,
-      "eval_objective": 0.4143694341182709,
+      "eval_dpo_loss": 0.7137619256973267,
+      "eval_logits": -0.702060341835022,
+      "eval_logps": -77.44371795654297,
+      "eval_loss": 0.6321162581443787,
+      "eval_objective": 0.6157041788101196,
       "eval_ranking_idealized": 0.6030020713806152,
       "eval_ranking_idealized_expo": 0.5222567319869995,
-      "eval_ranking_simple": 0.534679114818573,
-      "eval_regularize": 0.4143694341182709,
-      "eval_runtime": 465.9154,
-      "eval_samples_per_second": 12.427,
-      "eval_steps_per_second": 1.037,
-      "eval_wo_beta": 16.10569953918457,
+      "eval_ranking_simple": 0.5367494821548462,
+      "eval_regularize": 0.6157041788101196,
+      "eval_runtime": 309.0389,
+      "eval_samples_per_second": 18.736,
+      "eval_steps_per_second": 1.563,
+      "eval_wo_beta": 14.385796546936035,
       "step": 650
     },
     {
-      "dpo_loss": 0.5461080074310303,
+      "dpo_loss": 0.5520748496055603,
       "epoch": 1.9839395370807746,
-      "grad_norm": 29.849998371301844,
-      "learning_rate": 7.534273228904915e-07,
-      "logits": -1.4225118160247803,
-      "logps": -84.2075424194336,
-      "loss": 0.1598,
-      "objective": 0.16184575855731964,
+      "grad_norm": 15.323602142583136,
+      "learning_rate": 3.767136614452458e-06,
+      "logits": -0.5264750719070435,
+      "logps": -75.2638931274414,
+      "loss": 0.2675,
+      "objective": 0.2728944420814514,
       "ranking_idealized": 0.5954166650772095,
       "ranking_idealized_expo": 0.5129166841506958,
-      "ranking_simple": 0.5870833396911621,
-      "regularize": 0.16184575855731964,
+      "ranking_simple": 0.5883333086967468,
+      "regularize": 0.2728944420814514,
       "step": 700,
-      "wo_beta": 15.974614143371582
+      "wo_beta": 15.999488830566406
     },
     {
       "epoch": 1.9839395370807746,
-      "eval_dpo_loss": 0.6798617243766785,
-      "eval_logits": -1.411017656326294,
-      "eval_logps": -90.56302642822266,
-      "eval_loss": 0.4216688573360443,
-      "eval_objective": 0.41477063298225403,
+      "eval_dpo_loss": 0.7185091972351074,
+      "eval_logits": -0.561150312423706,
+      "eval_logps": -79.35997009277344,
+      "eval_loss": 0.6386255621910095,
+      "eval_objective": 0.6233482956886292,
       "eval_ranking_idealized": 0.6030020713806152,
       "eval_ranking_idealized_expo": 0.5222567319869995,
-      "eval_ranking_simple": 0.5362318754196167,
-      "eval_regularize": 0.41477063298225403,
-      "eval_runtime": 468.8368,
-      "eval_samples_per_second": 12.35,
-      "eval_steps_per_second": 1.03,
-      "eval_wo_beta": 16.049283981323242,
+      "eval_ranking_simple": 0.5289855003356934,
+      "eval_regularize": 0.6233482956886292,
+      "eval_runtime": 309.5574,
+      "eval_samples_per_second": 18.704,
+      "eval_steps_per_second": 1.56,
+      "eval_wo_beta": 14.317137718200684,
       "step": 700
     },
     {
-      "dpo_loss": 0.5335285663604736,
+      "dpo_loss": 0.5413146615028381,
       "epoch": 2.1256495040151155,
-      "grad_norm": 27.349376686984996,
-      "learning_rate": 7.095099669372443e-07,
-      "logits": -1.4041997194290161,
-      "logps": -83.5035400390625,
-      "loss": 0.1454,
-      "objective": 0.14657247066497803,
+      "grad_norm": 14.916489496350042,
+      "learning_rate": 3.547549834686222e-06,
+      "logits": -0.5209631323814392,
+      "logps": -75.42438507080078,
+      "loss": 0.2415,
+      "objective": 0.23713654279708862,
       "ranking_idealized": 0.6066666841506958,
       "ranking_idealized_expo": 0.5133333206176758,
-      "ranking_simple": 0.596666693687439,
-      "regularize": 0.14657247066497803,
+      "ranking_simple": 0.597083330154419,
+      "regularize": 0.23713654279708862,
       "step": 750,
-      "wo_beta": 16.065366744995117
+      "wo_beta": 15.934895515441895
     },
     {
       "epoch": 2.1256495040151155,
-      "eval_dpo_loss": 0.6797157526016235,
-      "eval_logits": -1.3859150409698486,
-      "eval_logps": -89.54334259033203,
-      "eval_loss": 0.4214833676815033,
-      "eval_objective": 0.41514915227890015,
+      "eval_dpo_loss": 0.7177355885505676,
+      "eval_logits": -0.6173678636550903,
+      "eval_logps": -80.0989761352539,
+      "eval_loss": 0.6405187249183655,
+      "eval_objective": 0.6263132095336914,
       "eval_ranking_idealized": 0.6030020713806152,
       "eval_ranking_idealized_expo": 0.5222567319869995,
-      "eval_ranking_simple": 0.531573474407196,
-      "eval_regularize": 0.41514915227890015,
-      "eval_runtime": 480.0022,
-      "eval_samples_per_second": 12.062,
-      "eval_steps_per_second": 1.006,
-      "eval_wo_beta": 16.04585075378418,
+      "eval_ranking_simple": 0.534679114818573,
+      "eval_regularize": 0.6263132095336914,
+      "eval_runtime": 309.3835,
+      "eval_samples_per_second": 18.715,
+      "eval_steps_per_second": 1.561,
+      "eval_wo_beta": 14.430180549621582,
       "step": 750
     },
     {
-      "dpo_loss": 0.5314897894859314,
+      "dpo_loss": 0.5419275760650635,
       "epoch": 2.2673594709494567,
-      "grad_norm": 28.42084450535516,
-      "learning_rate": 6.635339816587108e-07,
-      "logits": -1.3851786851882935,
-      "logps": -83.51881408691406,
-      "loss": 0.1333,
-      "objective": 0.13326440751552582,
+      "grad_norm": 15.335797694469315,
+      "learning_rate": 3.3176699082935546e-06,
+      "logits": -0.5584273338317871,
+      "logps": -76.04695129394531,
+      "loss": 0.2263,
+      "objective": 0.223616361618042,
       "ranking_idealized": 0.5941666960716248,
       "ranking_idealized_expo": 0.5129166841506958,
-      "ranking_simple": 0.5874999761581421,
-      "regularize": 0.13326440751552582,
+      "ranking_simple": 0.5866666436195374,
+      "regularize": 0.223616361618042,
       "step": 800,
-      "wo_beta": 15.551568984985352
+      "wo_beta": 15.526464462280273
     },
     {
       "epoch": 2.2673594709494567,
-      "eval_dpo_loss": 0.6790949106216431,
-      "eval_logits": -1.383711576461792,
-      "eval_logps": -89.10604858398438,
-      "eval_loss": 0.4217349588871002,
-      "eval_objective": 0.4141845107078552,
+      "eval_dpo_loss": 0.7205542922019958,
+      "eval_logits": -0.5665243268013,
+      "eval_logps": -79.37840270996094,
+      "eval_loss": 0.6457626223564148,
+      "eval_objective": 0.6297247409820557,
       "eval_ranking_idealized": 0.6030020713806152,
       "eval_ranking_idealized_expo": 0.5222567319869995,
       "eval_ranking_simple": 0.534679114818573,
-      "eval_regularize": 0.4141845107078552,
-      "eval_runtime": 479.8627,
-      "eval_samples_per_second": 12.066,
-      "eval_steps_per_second": 1.007,
-      "eval_wo_beta": 15.984663963317871,
+      "eval_regularize": 0.6297247409820557,
+      "eval_runtime": 308.5045,
+      "eval_samples_per_second": 18.768,
+      "eval_steps_per_second": 1.566,
+      "eval_wo_beta": 14.316285133361816,
       "step": 800
     },
     {
-      "dpo_loss": 0.5344111919403076,
+      "dpo_loss": 0.539085865020752,
       "epoch": 2.409069437883798,
-      "grad_norm": 27.376346461788764,
-      "learning_rate": 6.159511236607315e-07,
-      "logits": -1.4081922769546509,
-      "logps": -83.36618041992188,
-      "loss": 0.1287,
-      "objective": 0.12692613899707794,
+      "grad_norm": 15.300324456571563,
+      "learning_rate": 3.0797556183036582e-06,
+      "logits": -0.5494623780250549,
+      "logps": -75.91812133789062,
+      "loss": 0.2148,
+      "objective": 0.2133045643568039,
       "ranking_idealized": 0.5979166626930237,
       "ranking_idealized_expo": 0.5166666507720947,
-      "ranking_simple": 0.5933333039283752,
-      "regularize": 0.12692613899707794,
+      "ranking_simple": 0.5879166722297668,
+      "regularize": 0.2133045643568039,
       "step": 850,
-      "wo_beta": 15.760154724121094
+      "wo_beta": 15.76329231262207
     },
     {
       "epoch": 2.409069437883798,
-      "eval_dpo_loss": 0.6794790625572205,
-      "eval_logits": -1.3856061697006226,
-      "eval_logps": -88.614501953125,
-      "eval_loss": 0.4240662753582001,
-      "eval_objective": 0.4153030812740326,
+      "eval_dpo_loss": 0.7192490100860596,
+      "eval_logits": -0.5793017148971558,
+      "eval_logps": -79.08055114746094,
+      "eval_loss": 0.6435712575912476,
+      "eval_objective": 0.6275891661643982,
       "eval_ranking_idealized": 0.6030020713806152,
       "eval_ranking_idealized_expo": 0.5222567319869995,
-      "eval_ranking_simple": 0.5357142686843872,
-      "eval_regularize": 0.4153030812740326,
-      "eval_runtime": 481.1385,
-      "eval_samples_per_second": 12.034,
-      "eval_steps_per_second": 1.004,
-      "eval_wo_beta": 15.997852325439453,
+      "eval_ranking_simple": 0.5362318754196167,
+      "eval_regularize": 0.6275891661643982,
+      "eval_runtime": 312.2924,
+      "eval_samples_per_second": 18.54,
+      "eval_steps_per_second": 1.547,
+      "eval_wo_beta": 14.426342010498047,
       "step": 850
     },
     {
-      "dpo_loss": 0.5347414016723633,
+      "dpo_loss": 0.5431251525878906,
       "epoch": 2.550779404818139,
-      "grad_norm": 29.1093651939844,
-      "learning_rate": 5.67228938560766e-07,
-      "logits": -1.3715587854385376,
-      "logps": -82.14668273925781,
-      "loss": 0.12,
-      "objective": 0.12150037288665771,
+      "grad_norm": 15.744993696603856,
+      "learning_rate": 2.8361446928038298e-06,
+      "logits": -0.5151351690292358,
+      "logps": -75.99987030029297,
+      "loss": 0.1993,
+      "objective": 0.1999633014202118,
       "ranking_idealized": 0.5887500047683716,
       "ranking_idealized_expo": 0.518750011920929,
-      "ranking_simple": 0.5891666412353516,
-      "regularize": 0.12150037288665771,
+      "ranking_simple": 0.5899999737739563,
+      "regularize": 0.1999633014202118,
       "step": 900,
-      "wo_beta": 16.280019760131836
+      "wo_beta": 16.303361892700195
     },
     {
       "epoch": 2.550779404818139,
-      "eval_dpo_loss": 0.6795074343681335,
-      "eval_logits": -1.3921464681625366,
-      "eval_logps": -88.66633605957031,
-      "eval_loss": 0.4207339584827423,
-      "eval_objective": 0.4128892719745636,
+      "eval_dpo_loss": 0.7217252850532532,
+      "eval_logits": -0.5620540976524353,
+      "eval_logps": -80.38152313232422,
+      "eval_loss": 0.6453951001167297,
+      "eval_objective": 0.6301912665367126,
       "eval_ranking_idealized": 0.6030020713806152,
       "eval_ranking_idealized_expo": 0.5222567319869995,
-      "eval_ranking_simple": 0.5331262946128845,
-      "eval_regularize": 0.4128892719745636,
-      "eval_runtime": 493.2688,
-      "eval_samples_per_second": 11.738,
-      "eval_steps_per_second": 0.979,
-      "eval_wo_beta": 16.06983184814453,
+      "eval_ranking_simple": 0.5341615080833435,
+      "eval_regularize": 0.6301912665367126,
+      "eval_runtime": 308.7307,
+      "eval_samples_per_second": 18.754,
+      "eval_steps_per_second": 1.564,
+      "eval_wo_beta": 14.44913387298584,
       "step": 900
     },
     {
-      "dpo_loss": 0.5318446159362793,
+      "dpo_loss": 0.5340785980224609,
       "epoch": 2.69248937175248,
-      "grad_norm": 28.62178309135148,
-      "learning_rate": 5.178461669194903e-07,
-      "logits": -1.3740975856781006,
-      "logps": -82.18826293945312,
-      "loss": 0.1148,
-      "objective": 0.11303483694791794,
+      "grad_norm": 15.888338408049977,
+      "learning_rate": 2.5892308345974517e-06,
+      "logits": -0.5341619253158569,
+      "logps": -76.2621841430664,
+      "loss": 0.1887,
+      "objective": 0.1899857223033905,
       "ranking_idealized": 0.5975000262260437,
       "ranking_idealized_expo": 0.5087500214576721,
-      "ranking_simple": 0.5929166674613953,
-      "regularize": 0.11303483694791794,
+      "ranking_simple": 0.5933333039283752,
+      "regularize": 0.1899857223033905,
       "step": 950,
-      "wo_beta": 16.24332618713379
+      "wo_beta": 16.227678298950195
     },
     {
       "epoch": 2.69248937175248,
-      "eval_dpo_loss": 0.6791985034942627,
-      "eval_logits": -1.3689535856246948,
-      "eval_logps": -88.28536224365234,
-      "eval_loss": 0.421545147895813,
-      "eval_objective": 0.41494688391685486,
+      "eval_dpo_loss": 0.7204239964485168,
+      "eval_logits": -0.6216442584991455,
+      "eval_logps": -79.14459228515625,
+      "eval_loss": 0.6443176865577698,
+      "eval_objective": 0.6274449825286865,
       "eval_ranking_idealized": 0.6030020713806152,
       "eval_ranking_idealized_expo": 0.5222567319869995,
       "eval_ranking_simple": 0.533643901348114,
-      "eval_regularize": 0.41494688391685486,
-      "eval_runtime": 492.3545,
-      "eval_samples_per_second": 11.76,
-      "eval_steps_per_second": 0.981,
-      "eval_wo_beta": 16.05128288269043,
+      "eval_regularize": 0.6274449825286865,
+      "eval_runtime": 309.1927,
+      "eval_samples_per_second": 18.726,
+      "eval_steps_per_second": 1.562,
+      "eval_wo_beta": 14.318567276000977,
       "step": 950
     },
     {
-      "dpo_loss": 0.5294139385223389,
+      "dpo_loss": 0.5356190204620361,
       "epoch": 2.8341993386868207,
-      "grad_norm": 27.974854830973783,
-      "learning_rate": 4.682880401717177e-07,
-      "logits": -1.3566088676452637,
-      "logps": -81.84488677978516,
-      "loss": 0.1068,
-      "objective": 0.109143927693367,
+      "grad_norm": 14.415639648177313,
+      "learning_rate": 2.341440200858589e-06,
+      "logits": -0.5420577526092529,
+      "logps": -74.890869140625,
+      "loss": 0.1764,
+      "objective": 0.17968998849391937,
       "ranking_idealized": 0.6020833253860474,
       "ranking_idealized_expo": 0.5104166865348816,
-      "ranking_simple": 0.5975000262260437,
-      "regularize": 0.109143927693367,
+      "ranking_simple": 0.590416669845581,
+      "regularize": 0.17968998849391937,
       "step": 1000,
-      "wo_beta": 15.627644538879395
+      "wo_beta": 15.519268035888672
     },
     {
       "epoch": 2.8341993386868207,
-      "eval_dpo_loss": 0.6808813214302063,
-      "eval_logits": -1.3724371194839478,
-      "eval_logps": -89.17817687988281,
-      "eval_loss": 0.42285358905792236,
-      "eval_objective": 0.41681256890296936,
+      "eval_dpo_loss": 0.7200449705123901,
+      "eval_logits": -0.6086606383323669,
+      "eval_logps": -79.77206420898438,
+      "eval_loss": 0.639886736869812,
+      "eval_objective": 0.6246000528335571,
       "eval_ranking_idealized": 0.6030020713806152,
       "eval_ranking_idealized_expo": 0.5222567319869995,
-      "eval_ranking_simple": 0.5320910811424255,
-      "eval_regularize": 0.41681256890296936,
-      "eval_runtime": 490.5576,
-      "eval_samples_per_second": 11.803,
-      "eval_steps_per_second": 0.985,
-      "eval_wo_beta": 16.072154998779297,
+      "eval_ranking_simple": 0.533643901348114,
+      "eval_regularize": 0.6246000528335571,
+      "eval_runtime": 309.9286,
+      "eval_samples_per_second": 18.682,
+      "eval_steps_per_second": 1.558,
+      "eval_wo_beta": 14.450177192687988,
       "step": 1000
     },
     {
-      "dpo_loss": 0.5339093208312988,
+      "dpo_loss": 0.5345789194107056,
       "epoch": 2.975909305621162,
-      "grad_norm": 27.555604149776222,
-      "learning_rate": 4.1904151277847305e-07,
-      "logits": -1.3810933828353882,
-      "logps": -82.09051513671875,
-      "loss": 0.0991,
-      "objective": 0.09846866875886917,
+      "grad_norm": 14.911688792125858,
+      "learning_rate": 2.0952075638923656e-06,
+      "logits": -0.5775164365768433,
+      "logps": -75.94026184082031,
+      "loss": 0.163,
+      "objective": 0.162851020693779,
       "ranking_idealized": 0.6075000166893005,
       "ranking_idealized_expo": 0.5179166793823242,
-      "ranking_simple": 0.5975000262260437,
-      "regularize": 0.09846866875886917,
+      "ranking_simple": 0.6041666865348816,
+      "regularize": 0.162851020693779,
       "step": 1050,
-      "wo_beta": 16.260292053222656
+      "wo_beta": 16.27153205871582
     },
     {
       "epoch": 2.975909305621162,
-      "eval_dpo_loss": 0.6791900992393494,
-      "eval_logits": -1.398189663887024,
-      "eval_logps": -88.96070098876953,
-      "eval_loss": 0.421024352312088,
-      "eval_objective": 0.4140646755695343,
+      "eval_dpo_loss": 0.721105694770813,
+      "eval_logits": -0.6067584156990051,
+      "eval_logps": -79.58184814453125,
+      "eval_loss": 0.6428102850914001,
+      "eval_objective": 0.6266354322433472,
       "eval_ranking_idealized": 0.6030020713806152,
       "eval_ranking_idealized_expo": 0.5222567319869995,
-      "eval_ranking_simple": 0.533643901348114,
-      "eval_regularize": 0.4140646755695343,
-      "eval_runtime": 468.6466,
-      "eval_samples_per_second": 12.355,
-      "eval_steps_per_second": 1.031,
-      "eval_wo_beta": 16.04439353942871,
+      "eval_ranking_simple": 0.531573474407196,
+      "eval_regularize": 0.6266354322433472,
+      "eval_runtime": 308.0819,
+      "eval_samples_per_second": 18.794,
+      "eval_steps_per_second": 1.568,
+      "eval_wo_beta": 14.340644836425781,
       "step": 1050
     },
     {
       "epoch": 2.975909305621162,
       "step": 1050,
       "total_flos": 0.0,
-      "train_loss": 0.10327800432840983,
-      "train_runtime": 29791.2438,
-      "train_samples_per_second": 8.526,
-      "train_steps_per_second": 0.059
+      "train_loss": 0.3115594020343962,
+      "train_runtime": 28931.7373,
+      "train_samples_per_second": 8.78,
+      "train_steps_per_second": 0.061
     }
   ],
   "logging_steps": 50,