Model save

README.md

@@ -2,16 +2,9 @@
 license: apache-2.0
 base_model: hZzy/qwen2.5-0.5b-sft-news-IFT
 tags:
-- alignment-handbook
-- ndcg
 - trl
 - expo
 - generated_from_trainer
-- trl
-- expo
-- generated_from_trainer
-datasets:
-- hZzy/train_pairwise
 model-index:
 - name: qwen2.5-0.5b-expo-DPO-ES-1
   results: []
@@ -20,21 +13,21 @@ model-index:
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 should probably proofread and complete it, then remove this comment. -->
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/zhiyuzha-university-of-florida/huggingface/runs/qhqu3qtk)
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/zhiyuzha-university-of-florida/huggingface/runs/zba5f93y)
 # qwen2.5-0.5b-expo-DPO-ES-1
 
-This model is a fine-tuned version of [hZzy/qwen2.5-0.5b-sft-news-IFT](https://huggingface.co/hZzy/qwen2.5-0.5b-sft-news-IFT) on the hZzy/train_pairwise dataset.
+This model is a fine-tuned version of [hZzy/qwen2.5-0.5b-sft-news-IFT](https://huggingface.co/hZzy/qwen2.5-0.5b-sft-news-IFT) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 2.3243
-- Logps: -83.2882
-- Logits: -0.6651
-- Objective: 2.2471
-- Dpo Loss: 2.2471
-- Regularize: 2.2471
-- Ranking Simple: 0.5378
+- Loss: 2.3304
+- Logps: -83.6421
+- Logits: -0.7129
+- Objective: 2.2523
+- Dpo Loss: 2.2523
+- Regularize: 2.2523
+- Ranking Simple: 0.5336
 - Ranking Idealized: 0.5295
 - Ranking Idealized Expo: 0.5212
-- Wo Beta: 6.6815
+- Wo Beta: 6.8942
 
 ## Model description
 
@@ -71,25 +64,25 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Logps    | Logits  | Objective | Dpo Loss | Regularize | Ranking Simple | Ranking Idealized | Ranking Idealized Expo | Wo Beta |
 |:-------------:|:------:|:----:|:---------------:|:--------:|:-------:|:---------:|:--------:|:----------:|:--------------:|:-----------------:|:----------------------:|:-------:|
-| 0.7017        | 0.1417 | 50   | 0.8470          | -93.0243 | -1.4582 | 0.8570    | 0.8570   | 0.8570     | 0.5238         | 0.5295            | 0.5212                 | 7.8507  |
-| 0.8112        | 0.2834 | 100  | 1.0529          | -86.6835 | -1.4382 | 1.0273    | 1.0273   | 1.0273     | 0.5285         | 0.5295            | 0.5212                 | 7.4982  |
-| 1.0895        | 0.4251 | 150  | 1.4497          | -84.4337 | -1.2965 | 1.4010    | 1.4010   | 1.4010     | 0.5321         | 0.5295            | 0.5212                 | 7.2692  |
-| 1.2363        | 0.5668 | 200  | 1.7035          | -77.7201 | -1.2956 | 1.6116    | 1.6116   | 1.6116     | 0.5321         | 0.5295            | 0.5212                 | 7.2264  |
-| 1.3152        | 0.7085 | 250  | 1.9222          | -92.7241 | -1.2565 | 1.8319    | 1.8319   | 1.8319     | 0.5311         | 0.5295            | 0.5212                 | 7.1856  |
-| 1.1899        | 0.8503 | 300  | 2.0298          | -90.9373 | -0.9785 | 1.9588    | 1.9588   | 1.9588     | 0.5367         | 0.5295            | 0.5212                 | 6.9336  |
-| 1.1443        | 0.9920 | 350  | 2.1654          | -82.1414 | -1.0214 | 2.0541    | 2.0541   | 2.0541     | 0.5435         | 0.5295            | 0.5212                 | 7.0024  |
-| 0.725         | 1.1337 | 400  | 2.2884          | -84.2526 | -0.7535 | 2.2360    | 2.2360   | 2.2360     | 0.5336         | 0.5295            | 0.5212                 | 7.1525  |
-| 0.7629        | 1.2754 | 450  | 2.1606          | -80.4165 | -0.8866 | 2.0671    | 2.0671   | 2.0671     | 0.5321         | 0.5295            | 0.5212                 | 6.7949  |
-| 0.8044        | 1.4171 | 500  | 2.2094          | -82.3927 | -0.7503 | 2.0981    | 2.0981   | 2.0981     | 0.5347         | 0.5295            | 0.5212                 | 6.8050  |
-| 0.7105        | 1.5588 | 550  | 2.1697          | -84.9780 | -0.6734 | 2.0733    | 2.0733   | 2.0733     | 0.5321         | 0.5295            | 0.5212                 | 6.8722  |
-| 0.6925        | 1.7005 | 600  | 2.1957          | -81.5342 | -0.7411 | 2.0558    | 2.0558   | 2.0558     | 0.5357         | 0.5295            | 0.5212                 | 6.7186  |
-| 0.6883        | 1.8422 | 650  | 2.2080          | -82.7303 | -0.6908 | 2.1330    | 2.1330   | 2.1330     | 0.5383         | 0.5295            | 0.5212                 | 6.8081  |
-| 0.6486        | 1.9839 | 700  | 2.3243          | -83.2882 | -0.6651 | 2.2471    | 2.2471   | 2.2471     | 0.5378         | 0.5295            | 0.5212                 | 6.6815  |
-| 0.3793        | 2.1256 | 750  | 2.2675          | -84.2296 | -0.7879 | 2.1825    | 2.1825   | 2.1825     | 0.5409         | 0.5295            | 0.5212                 | 6.8794  |
-| 0.3314        | 2.2674 | 800  | 2.2106          | -84.3675 | -0.6651 | 2.1041    | 2.1041   | 2.1041     | 0.5414         | 0.5295            | 0.5212                 | 6.7463  |
-| 0.3301        | 2.4091 | 850  | 2.2964          | -84.8913 | -0.6177 | 2.2221    | 2.2221   | 2.2221     | 0.5388         | 0.5295            | 0.5212                 | 6.8020  |
-| 0.3509        | 2.5508 | 900  | 2.2796          | -84.3833 | -0.6097 | 2.2099    | 2.2099   | 2.2099     | 0.5393         | 0.5295            | 0.5212                 | 6.7934  |
-| 0.321         | 2.6925 | 950  | 2.3403          | -83.2967 | -0.7158 | 2.2649    | 2.2649   | 2.2649     | 0.5331         | 0.5295            | 0.5212                 | 6.8864  |
+| 0.7017        | 0.1417 | 50   | 0.8470          | -93.0245 | -1.4582 | 0.8570    | 0.8570   | 0.8570     | 0.5238         | 0.5295            | 0.5212                 | 7.8506  |
+| 0.8112        | 0.2834 | 100  | 1.0529          | -86.6804 | -1.4383 | 1.0274    | 1.0274   | 1.0274     | 0.5285         | 0.5295            | 0.5212                 | 7.4982  |
+| 1.0895        | 0.4251 | 150  | 1.4498          | -84.4335 | -1.2965 | 1.4010    | 1.4010   | 1.4010     | 0.5321         | 0.5295            | 0.5212                 | 7.2692  |
+| 1.2362        | 0.5668 | 200  | 1.7035          | -77.7194 | -1.2955 | 1.6116    | 1.6116   | 1.6116     | 0.5321         | 0.5295            | 0.5212                 | 7.2264  |
+| 1.3151        | 0.7085 | 250  | 1.9222          | -92.7224 | -1.2565 | 1.8319    | 1.8319   | 1.8319     | 0.5311         | 0.5295            | 0.5212                 | 7.1855  |
+| 1.1899        | 0.8503 | 300  | 2.0297          | -90.9351 | -0.9786 | 1.9587    | 1.9587   | 1.9587     | 0.5367         | 0.5295            | 0.5212                 | 6.9337  |
+| 1.1441        | 0.9920 | 350  | 2.1653          | -82.1291 | -1.0211 | 2.0545    | 2.0545   | 2.0545     | 0.5424         | 0.5295            | 0.5212                 | 7.0017  |
+| 0.725         | 1.1337 | 400  | 2.2886          | -84.3458 | -0.7529 | 2.2360    | 2.2360   | 2.2360     | 0.5331         | 0.5295            | 0.5212                 | 7.1541  |
+| 0.7626        | 1.2754 | 450  | 2.1595          | -80.5955 | -0.8863 | 2.0657    | 2.0657   | 2.0657     | 0.5326         | 0.5295            | 0.5212                 | 6.7939  |
+| 0.8048        | 1.4171 | 500  | 2.2134          | -82.3489 | -0.7432 | 2.0975    | 2.0975   | 2.0975     | 0.5342         | 0.5295            | 0.5212                 | 6.7984  |
+| 0.7106        | 1.5588 | 550  | 2.1705          | -85.0673 | -0.6665 | 2.0696    | 2.0696   | 2.0696     | 0.5321         | 0.5295            | 0.5212                 | 6.8614  |
+| 0.6934        | 1.7005 | 600  | 2.2127          | -81.6773 | -0.7358 | 2.0693    | 2.0693   | 2.0693     | 0.5362         | 0.5295            | 0.5212                 | 6.7265  |
+| 0.6885        | 1.8422 | 650  | 2.2198          | -82.8870 | -0.6787 | 2.1432    | 2.1432   | 2.1432     | 0.5362         | 0.5295            | 0.5212                 | 6.8202  |
+| 0.6477        | 1.9839 | 700  | 2.3420          | -83.4105 | -0.6597 | 2.2592    | 2.2592   | 2.2592     | 0.5404         | 0.5295            | 0.5212                 | 6.6836  |
+| 0.3785        | 2.1256 | 750  | 2.2919          | -84.0369 | -0.7841 | 2.2005    | 2.2005   | 2.2005     | 0.5435         | 0.5295            | 0.5212                 | 6.8514  |
+| 0.3316        | 2.2674 | 800  | 2.2220          | -84.2990 | -0.6767 | 2.1123    | 2.1123   | 2.1123     | 0.5409         | 0.5295            | 0.5212                 | 6.7663  |
+| 0.3283        | 2.4091 | 850  | 2.3020          | -85.0834 | -0.6538 | 2.2212    | 2.2212   | 2.2212     | 0.5409         | 0.5295            | 0.5212                 | 6.7773  |
+| 0.3516        | 2.5508 | 900  | 2.2723          | -84.7564 | -0.6225 | 2.1911    | 2.1911   | 2.1911     | 0.5362         | 0.5295            | 0.5212                 | 6.8162  |
+| 0.3245        | 2.6925 | 950  | 2.3304          | -83.6421 | -0.7129 | 2.2523    | 2.2523   | 2.2523     | 0.5336         | 0.5295            | 0.5212                 | 6.8942  |
 
 
 ### Framework versions

all_results.json

@@ -1,23 +1,9 @@
 {
     "epoch": 2.69248937175248,
-    "eval_dpo_loss": 2.2471413612365723,
-    "eval_logits": -0.6650525331497192,
-    "eval_logps": -83.28822326660156,
-    "eval_loss": 2.3242855072021484,
-    "eval_objective": 2.2471413612365723,
-    "eval_ranking_idealized": 0.5295031070709229,
-    "eval_ranking_idealized_expo": 0.5212215185165405,
-    "eval_ranking_simple": 0.5377846956253052,
-    "eval_regularize": 2.2471413612365723,
-    "eval_runtime": 314.6348,
-    "eval_samples": 5790,
-    "eval_samples_per_second": 18.402,
-    "eval_steps_per_second": 1.535,
-    "eval_wo_beta": 6.681451797485352,
     "total_flos": 0.0,
-    "train_loss": 0.7490891541932758,
-    "train_runtime": 28216.8085,
+    "train_loss": 0.7491915085441188,
+    "train_runtime": 26152.6935,
     "train_samples": 50802,
-    "train_samples_per_second": 9.002,
-    "train_steps_per_second": 0.062
+    "train_samples_per_second": 9.713,
+    "train_steps_per_second": 0.067
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b913a79a3c077b7cf00b62c745ff7dfec07093ccbf7a7f2c62c910b54f5fadb8
+oid sha256:92477df1ba269cbc6b373f8a1073187c1821843696650c5aab98bba461f3df44
 size 1975192208

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 2.69248937175248,
     "total_flos": 0.0,
-    "train_loss": 0.7490891541932758,
-    "train_runtime": 28216.8085,
+    "train_loss": 0.7491915085441188,
+    "train_runtime": 26152.6935,
     "train_samples": 50802,
-    "train_samples_per_second": 9.002,
-    "train_steps_per_second": 0.062
+    "train_samples_per_second": 9.713,
+    "train_steps_per_second": 0.067
 }

trainer_state.json

@@ -1,5 +1,5 @@
 {
-  "best_metric": 6.681451797485352,
+  "best_metric": 6.68360710144043,
   "best_model_checkpoint": "./qwen2.5-0.5b/qwen2.5-0.5b-expo-DPO-ES-1/checkpoint-700",
   "epoch": 2.69248937175248,
   "eval_steps": 50,
@@ -11,7 +11,7 @@
     {
       "dpo_loss": 0.6931471824645996,
       "epoch": 0.002834199338686821,
-      "grad_norm": 184.4249864582654,
+      "grad_norm": 184.42504161174296,
       "learning_rate": 2.840909090909091e-08,
       "logits": -1.359458565711975,
       "logps": -84.69721221923828,
@@ -25,640 +25,640 @@
       "wo_beta": 5.271125316619873
     },
     {
-      "dpo_loss": 0.7105385065078735,
+      "dpo_loss": 0.7105370759963989,
       "epoch": 0.14170996693434104,
-      "grad_norm": 184.86918480832395,
+      "grad_norm": 184.8690438155141,
       "learning_rate": 1.4204545454545458e-06,
-      "logits": -1.4640510082244873,
-      "logps": -84.16496276855469,
+      "logits": -1.464050531387329,
+      "logps": -84.1650161743164,
       "loss": 0.7017,
-      "objective": 0.7105385065078735,
+      "objective": 0.7105370759963989,
       "ranking_idealized": 0.5289115905761719,
       "ranking_idealized_expo": 0.5221088528633118,
       "ranking_simple": 0.5225340127944946,
-      "regularize": 0.7105385065078735,
+      "regularize": 0.7105370759963989,
       "step": 50,
-      "wo_beta": 7.0693583488464355
+      "wo_beta": 7.069356918334961
     },
     {
       "epoch": 0.14170996693434104,
-      "eval_dpo_loss": 0.8569971919059753,
-      "eval_logits": -1.4581880569458008,
-      "eval_logps": -93.02434539794922,
-      "eval_loss": 0.8469977974891663,
-      "eval_objective": 0.8569971919059753,
+      "eval_dpo_loss": 0.856997013092041,
+      "eval_logits": -1.458188533782959,
+      "eval_logps": -93.02446746826172,
+      "eval_loss": 0.8469988107681274,
+      "eval_objective": 0.856997013092041,
       "eval_ranking_idealized": 0.5295031070709229,
       "eval_ranking_idealized_expo": 0.5212215185165405,
       "eval_ranking_simple": 0.523809552192688,
-      "eval_regularize": 0.8569971919059753,
-      "eval_runtime": 308.8499,
-      "eval_samples_per_second": 18.747,
-      "eval_steps_per_second": 1.564,
-      "eval_wo_beta": 7.850653171539307,
+      "eval_regularize": 0.856997013092041,
+      "eval_runtime": 312.0194,
+      "eval_samples_per_second": 18.557,
+      "eval_steps_per_second": 1.548,
+      "eval_wo_beta": 7.850645065307617,
       "step": 50
     },
     {
-      "dpo_loss": 0.804880678653717,
+      "dpo_loss": 0.8048794865608215,
       "epoch": 0.2834199338686821,
-      "grad_norm": 165.0032665851806,
+      "grad_norm": 165.00318532005144,
       "learning_rate": 2.8409090909090916e-06,
-      "logits": -1.454908847808838,
-      "logps": -84.21910095214844,
+      "logits": -1.4549062252044678,
+      "logps": -84.21845245361328,
       "loss": 0.8112,
-      "objective": 0.804880678653717,
+      "objective": 0.8048794865608215,
       "ranking_idealized": 0.5241666436195374,
       "ranking_idealized_expo": 0.5137500166893005,
       "ranking_simple": 0.543749988079071,
-      "regularize": 0.804880678653717,
+      "regularize": 0.8048794865608215,
       "step": 100,
       "wo_beta": 6.3764801025390625
     },
     {
       "epoch": 0.2834199338686821,
-      "eval_dpo_loss": 1.0273478031158447,
-      "eval_logits": -1.4382472038269043,
-      "eval_logps": -86.68347930908203,
-      "eval_loss": 1.0528956651687622,
-      "eval_objective": 1.0273478031158447,
+      "eval_dpo_loss": 1.0273518562316895,
+      "eval_logits": -1.4382514953613281,
+      "eval_logps": -86.68037414550781,
+      "eval_loss": 1.0529025793075562,
+      "eval_objective": 1.0273518562316895,
       "eval_ranking_idealized": 0.5295031070709229,
       "eval_ranking_idealized_expo": 0.5212215185165405,
       "eval_ranking_simple": 0.5284678936004639,
-      "eval_regularize": 1.0273478031158447,
-      "eval_runtime": 325.8896,
-      "eval_samples_per_second": 17.767,
-      "eval_steps_per_second": 1.482,
-      "eval_wo_beta": 7.498155117034912,
+      "eval_regularize": 1.0273518562316895,
+      "eval_runtime": 308.916,
+      "eval_samples_per_second": 18.743,
+      "eval_steps_per_second": 1.564,
+      "eval_wo_beta": 7.498167991638184,
       "step": 100
     },
     {
-      "dpo_loss": 1.0717346668243408,
+      "dpo_loss": 1.0717324018478394,
       "epoch": 0.42512990080302315,
-      "grad_norm": 132.12956442526567,
+      "grad_norm": 132.13237608472397,
       "learning_rate": 4.2613636363636365e-06,
-      "logits": -1.3764687776565552,
-      "logps": -75.39175415039062,
+      "logits": -1.3764770030975342,
+      "logps": -75.389892578125,
       "loss": 1.0895,
-      "objective": 1.0717346668243408,
+      "objective": 1.0717324018478394,
       "ranking_idealized": 0.5333333611488342,
       "ranking_idealized_expo": 0.527916669845581,
       "ranking_simple": 0.5645833611488342,
-      "regularize": 1.0717346668243408,
+      "regularize": 1.0717324018478394,
       "step": 150,
-      "wo_beta": 6.200186252593994
+      "wo_beta": 6.200186729431152
     },
     {
       "epoch": 0.42512990080302315,
-      "eval_dpo_loss": 1.4009853601455688,
-      "eval_logits": -1.2964659929275513,
-      "eval_logps": -84.43374633789062,
-      "eval_loss": 1.4497475624084473,
-      "eval_objective": 1.4009853601455688,
+      "eval_dpo_loss": 1.4009839296340942,
+      "eval_logits": -1.2964812517166138,
+      "eval_logps": -84.43347930908203,
+      "eval_loss": 1.4497517347335815,
+      "eval_objective": 1.4009839296340942,
       "eval_ranking_idealized": 0.5295031070709229,
       "eval_ranking_idealized_expo": 0.5212215185165405,
       "eval_ranking_simple": 0.5320910811424255,
-      "eval_regularize": 1.4009853601455688,
-      "eval_runtime": 308.2116,
-      "eval_samples_per_second": 18.786,
-      "eval_steps_per_second": 1.567,
-      "eval_wo_beta": 7.269238471984863,
+      "eval_regularize": 1.4009839296340942,
+      "eval_runtime": 308.861,
+      "eval_samples_per_second": 18.746,
+      "eval_steps_per_second": 1.564,
+      "eval_wo_beta": 7.269246578216553,
       "step": 150
     },
     {
-      "dpo_loss": 1.2378712892532349,
+      "dpo_loss": 1.2378751039505005,
       "epoch": 0.5668398677373642,
-      "grad_norm": 120.89571295715444,
+      "grad_norm": 120.88819473175299,
       "learning_rate": 4.997168347957521e-06,
-      "logits": -1.3378713130950928,
-      "logps": -75.85318756103516,
-      "loss": 1.2363,
-      "objective": 1.2378712892532349,
+      "logits": -1.3378355503082275,
+      "logps": -75.85352325439453,
+      "loss": 1.2362,
+      "objective": 1.2378751039505005,
       "ranking_idealized": 0.5204166769981384,
       "ranking_idealized_expo": 0.51541668176651,
       "ranking_simple": 0.5699999928474426,
-      "regularize": 1.2378712892532349,
+      "regularize": 1.2378751039505005,
       "step": 200,
-      "wo_beta": 6.13140869140625
+      "wo_beta": 6.1314239501953125
     },
     {
       "epoch": 0.5668398677373642,
-      "eval_dpo_loss": 1.6115626096725464,
-      "eval_logits": -1.295562505722046,
-      "eval_logps": -77.72007751464844,
-      "eval_loss": 1.7034852504730225,
-      "eval_objective": 1.6115626096725464,
+      "eval_dpo_loss": 1.611555576324463,
+      "eval_logits": -1.2954967021942139,
+      "eval_logps": -77.71936798095703,
+      "eval_loss": 1.7034811973571777,
+      "eval_objective": 1.611555576324463,
       "eval_ranking_idealized": 0.5295031070709229,
       "eval_ranking_idealized_expo": 0.5212215185165405,
       "eval_ranking_simple": 0.5320910811424255,
-      "eval_regularize": 1.6115626096725464,
-      "eval_runtime": 334.6734,
-      "eval_samples_per_second": 17.3,
-      "eval_steps_per_second": 1.443,
-      "eval_wo_beta": 7.226394176483154,
+      "eval_regularize": 1.611555576324463,
+      "eval_runtime": 309.5132,
+      "eval_samples_per_second": 18.707,
+      "eval_steps_per_second": 1.561,
+      "eval_wo_beta": 7.226413726806641,
       "step": 200
     },
     {
-      "dpo_loss": 1.2734622955322266,
+      "dpo_loss": 1.2734363079071045,
       "epoch": 0.7085498346717053,
-      "grad_norm": 126.97038898052826,
+      "grad_norm": 126.97448798013814,
       "learning_rate": 4.973122855144066e-06,
-      "logits": -1.1745208501815796,
-      "logps": -80.53473663330078,
-      "loss": 1.3152,
-      "objective": 1.2734622955322266,
+      "logits": -1.1745147705078125,
+      "logps": -80.53407287597656,
+      "loss": 1.3151,
+      "objective": 1.2734363079071045,
       "ranking_idealized": 0.5249999761581421,
       "ranking_idealized_expo": 0.5162500143051147,
       "ranking_simple": 0.5929166674613953,
-      "regularize": 1.2734622955322266,
+      "regularize": 1.2734363079071045,
       "step": 250,
-      "wo_beta": 6.077574253082275
+      "wo_beta": 6.07757568359375
     },
     {
       "epoch": 0.7085498346717053,
-      "eval_dpo_loss": 1.8319419622421265,
-      "eval_logits": -1.2564637660980225,
-      "eval_logps": -92.72406005859375,
-      "eval_loss": 1.9222145080566406,
-      "eval_objective": 1.8319419622421265,
+      "eval_dpo_loss": 1.831916093826294,
+      "eval_logits": -1.2565279006958008,
+      "eval_logps": -92.72237396240234,
+      "eval_loss": 1.9222025871276855,
+      "eval_objective": 1.831916093826294,
       "eval_ranking_idealized": 0.5295031070709229,
       "eval_ranking_idealized_expo": 0.5212215185165405,
       "eval_ranking_simple": 0.5310559272766113,
-      "eval_regularize": 1.8319419622421265,
-      "eval_runtime": 308.15,
-      "eval_samples_per_second": 18.79,
-      "eval_steps_per_second": 1.567,
-      "eval_wo_beta": 7.185611248016357,
+      "eval_regularize": 1.831916093826294,
+      "eval_runtime": 308.4533,
+      "eval_samples_per_second": 18.771,
+      "eval_steps_per_second": 1.566,
+      "eval_wo_beta": 7.185470104217529,
       "step": 250
     },
     {
-      "dpo_loss": 1.1058255434036255,
+      "dpo_loss": 1.1058416366577148,
       "epoch": 0.8502598016060463,
-      "grad_norm": 93.62858320092595,
+      "grad_norm": 93.62402691602496,
       "learning_rate": 4.924776641419513e-06,
-      "logits": -1.0275437831878662,
-      "logps": -86.21664428710938,
+      "logits": -1.027516484260559,
+      "logps": -86.21515655517578,
       "loss": 1.1899,
-      "objective": 1.1058255434036255,
+      "objective": 1.1058416366577148,
       "ranking_idealized": 0.5062500238418579,
       "ranking_idealized_expo": 0.4950000047683716,
       "ranking_simple": 0.5833333134651184,
-      "regularize": 1.1058255434036255,
+      "regularize": 1.1058416366577148,
       "step": 300,
-      "wo_beta": 5.932236194610596
+      "wo_beta": 5.9321818351745605
     },
     {
       "epoch": 0.8502598016060463,
-      "eval_dpo_loss": 1.958790898323059,
-      "eval_logits": -0.9785082340240479,
-      "eval_logps": -90.93729400634766,
-      "eval_loss": 2.0297622680664062,
-      "eval_objective": 1.958790898323059,
+      "eval_dpo_loss": 1.9587332010269165,
+      "eval_logits": -0.9785559177398682,
+      "eval_logps": -90.93506622314453,
+      "eval_loss": 2.0297279357910156,
+      "eval_objective": 1.9587332010269165,
       "eval_ranking_idealized": 0.5295031070709229,
       "eval_ranking_idealized_expo": 0.5212215185165405,
       "eval_ranking_simple": 0.5367494821548462,
-      "eval_regularize": 1.958790898323059,
-      "eval_runtime": 320.477,
-      "eval_samples_per_second": 18.067,
-      "eval_steps_per_second": 1.507,
-      "eval_wo_beta": 6.933594703674316,
+      "eval_regularize": 1.9587332010269165,
+      "eval_runtime": 308.5179,
+      "eval_samples_per_second": 18.767,
+      "eval_steps_per_second": 1.566,
+      "eval_wo_beta": 6.933654308319092,
       "step": 300
     },
     {
-      "dpo_loss": 1.104241967201233,
+      "dpo_loss": 1.1039565801620483,
       "epoch": 0.9919697685403873,
-      "grad_norm": 81.69582378962005,
+      "grad_norm": 81.61244111449733,
       "learning_rate": 4.8526047530778175e-06,
-      "logits": -0.8600361943244934,
-      "logps": -83.0993423461914,
-      "loss": 1.1443,
-      "objective": 1.104241967201233,
+      "logits": -0.8597685694694519,
+      "logps": -83.09705352783203,
+      "loss": 1.1441,
+      "objective": 1.1039565801620483,
       "ranking_idealized": 0.5354166626930237,
       "ranking_idealized_expo": 0.5254166722297668,
       "ranking_simple": 0.6141666769981384,
-      "regularize": 1.104241967201233,
+      "regularize": 1.1039565801620483,
       "step": 350,
-      "wo_beta": 5.234530925750732
+      "wo_beta": 5.234945297241211
     },
     {
       "epoch": 0.9919697685403873,
-      "eval_dpo_loss": 2.0541160106658936,
-      "eval_logits": -1.0213720798492432,
-      "eval_logps": -82.14143371582031,
-      "eval_loss": 2.165400743484497,
-      "eval_objective": 2.0541160106658936,
+      "eval_dpo_loss": 2.054466485977173,
+      "eval_logits": -1.0211108922958374,
+      "eval_logps": -82.12905883789062,
+      "eval_loss": 2.1653144359588623,
+      "eval_objective": 2.054466485977173,
       "eval_ranking_idealized": 0.5295031070709229,
       "eval_ranking_idealized_expo": 0.5212215185165405,
-      "eval_ranking_simple": 0.54347825050354,
-      "eval_regularize": 2.0541160106658936,
-      "eval_runtime": 352.7203,
-      "eval_samples_per_second": 16.415,
-      "eval_steps_per_second": 1.369,
-      "eval_wo_beta": 7.002377033233643,
+      "eval_ranking_simple": 0.542443037033081,
+      "eval_regularize": 2.054466485977173,
+      "eval_runtime": 308.5833,
+      "eval_samples_per_second": 18.763,
+      "eval_steps_per_second": 1.565,
+      "eval_wo_beta": 7.001650810241699,
       "step": 350
     },
     {
-      "dpo_loss": 0.7012434005737305,
+      "dpo_loss": 0.7012167572975159,
       "epoch": 1.1336797354747283,
-      "grad_norm": 72.02321161366883,
+      "grad_norm": 72.06842141832568,
       "learning_rate": 4.757316345716554e-06,
-      "logits": -0.8228326439857483,
-      "logps": -77.82636260986328,
+      "logits": -0.8224175572395325,
+      "logps": -77.92687225341797,
       "loss": 0.725,
-      "objective": 0.7012434005737305,
+      "objective": 0.7012167572975159,
       "ranking_idealized": 0.5412499904632568,
       "ranking_idealized_expo": 0.5320833325386047,
-      "ranking_simple": 0.6487500071525574,
-      "regularize": 0.7012434005737305,
+      "ranking_simple": 0.6483333110809326,
+      "regularize": 0.7012167572975159,
       "step": 400,
-      "wo_beta": 4.493627548217773
+      "wo_beta": 4.492737770080566
     },
     {
       "epoch": 1.1336797354747283,
-      "eval_dpo_loss": 2.2360119819641113,
-      "eval_logits": -0.7534947991371155,
-      "eval_logps": -84.25261688232422,
-      "eval_loss": 2.288405179977417,
-      "eval_objective": 2.2360119819641113,
+      "eval_dpo_loss": 2.2359888553619385,
+      "eval_logits": -0.7528972029685974,
+      "eval_logps": -84.34579467773438,
+      "eval_loss": 2.288560152053833,
+      "eval_objective": 2.2359888553619385,
       "eval_ranking_idealized": 0.5295031070709229,
       "eval_ranking_idealized_expo": 0.5212215185165405,
-      "eval_ranking_simple": 0.533643901348114,
-      "eval_regularize": 2.2360119819641113,
-      "eval_runtime": 343.9192,
-      "eval_samples_per_second": 16.835,
-      "eval_steps_per_second": 1.404,
-      "eval_wo_beta": 7.152450084686279,
+      "eval_ranking_simple": 0.5331262946128845,
+      "eval_regularize": 2.2359888553619385,
+      "eval_runtime": 308.7822,
+      "eval_samples_per_second": 18.751,
+      "eval_steps_per_second": 1.564,
+      "eval_wo_beta": 7.154054164886475,
       "step": 400
     },
     {
-      "dpo_loss": 0.787284255027771,
+      "dpo_loss": 0.78798508644104,
       "epoch": 1.2753897024090695,
-      "grad_norm": 67.24645157140542,
+      "grad_norm": 67.40926541067125,
       "learning_rate": 4.639847716126855e-06,
-      "logits": -0.7304993867874146,
-      "logps": -79.94586181640625,
-      "loss": 0.7629,
-      "objective": 0.787284255027771,
+      "logits": -0.7312514185905457,
+      "logps": -80.1772232055664,
+      "loss": 0.7626,
+      "objective": 0.78798508644104,
       "ranking_idealized": 0.5245833396911621,
       "ranking_idealized_expo": 0.5191666483879089,
       "ranking_simple": 0.6483333110809326,
-      "regularize": 0.787284255027771,
+      "regularize": 0.78798508644104,
       "step": 450,
-      "wo_beta": 5.019043922424316
+      "wo_beta": 5.0193963050842285
     },
     {
       "epoch": 1.2753897024090695,
-      "eval_dpo_loss": 2.0671327114105225,
-      "eval_logits": -0.8865557909011841,
-      "eval_logps": -80.41649627685547,
-      "eval_loss": 2.160627841949463,
-      "eval_objective": 2.0671327114105225,
+      "eval_dpo_loss": 2.0657246112823486,
+      "eval_logits": -0.8863243460655212,
+      "eval_logps": -80.59545135498047,
+      "eval_loss": 2.1594510078430176,
+      "eval_objective": 2.0657246112823486,
       "eval_ranking_idealized": 0.5295031070709229,
       "eval_ranking_idealized_expo": 0.5212215185165405,
-      "eval_ranking_simple": 0.5320910811424255,
-      "eval_regularize": 2.0671327114105225,
-      "eval_runtime": 361.4999,
-      "eval_samples_per_second": 16.017,
-      "eval_steps_per_second": 1.336,
-      "eval_wo_beta": 6.794886589050293,
+      "eval_ranking_simple": 0.532608687877655,
+      "eval_regularize": 2.0657246112823486,
+      "eval_runtime": 314.8953,
+      "eval_samples_per_second": 18.387,
+      "eval_steps_per_second": 1.534,
+      "eval_wo_beta": 6.7939372062683105,
       "step": 450
     },
     {
-      "dpo_loss": 0.7948352098464966,
+      "dpo_loss": 0.7955495119094849,
       "epoch": 1.4170996693434104,
-      "grad_norm": 80.7228445203749,
+      "grad_norm": 80.21112193307313,
       "learning_rate": 4.501353102310901e-06,
-      "logits": -0.7204355001449585,
-      "logps": -76.1670150756836,
-      "loss": 0.8044,
-      "objective": 0.7948352098464966,
+      "logits": -0.717734694480896,
+      "logps": -76.27667999267578,
+      "loss": 0.8048,
+      "objective": 0.7955495119094849,
       "ranking_idealized": 0.5054166913032532,
       "ranking_idealized_expo": 0.4970833361148834,
-      "ranking_simple": 0.6416666507720947,
-      "regularize": 0.7948352098464966,
+      "ranking_simple": 0.6424999833106995,
+      "regularize": 0.7955495119094849,
       "step": 500,
-      "wo_beta": 4.868573188781738
+      "wo_beta": 4.868187427520752
     },
     {
       "epoch": 1.4170996693434104,
-      "eval_dpo_loss": 2.0981180667877197,
-      "eval_logits": -0.7502567172050476,
-      "eval_logps": -82.39266967773438,
-      "eval_loss": 2.2094054222106934,
-      "eval_objective": 2.0981180667877197,
+      "eval_dpo_loss": 2.0975162982940674,
+      "eval_logits": -0.7432325482368469,
+      "eval_logps": -82.348876953125,
+      "eval_loss": 2.2134060859680176,
+      "eval_objective": 2.0975162982940674,
       "eval_ranking_idealized": 0.5295031070709229,
       "eval_ranking_idealized_expo": 0.5212215185165405,
-      "eval_ranking_simple": 0.534679114818573,
-      "eval_regularize": 2.0981180667877197,
-      "eval_runtime": 308.1869,
-      "eval_samples_per_second": 18.787,
-      "eval_steps_per_second": 1.567,
-      "eval_wo_beta": 6.8049798011779785,
+      "eval_ranking_simple": 0.5341615080833435,
+      "eval_regularize": 2.0975162982940674,
+      "eval_runtime": 308.5741,
+      "eval_samples_per_second": 18.764,
+      "eval_steps_per_second": 1.565,
+      "eval_wo_beta": 6.79838752746582,
       "step": 500
     },
     {
-      "dpo_loss": 0.7342749834060669,
+      "dpo_loss": 0.733691394329071,
       "epoch": 1.5588096362777515,
-      "grad_norm": 72.59655396244632,
+      "grad_norm": 73.44558837421432,
       "learning_rate": 4.34319334202531e-06,
-      "logits": -0.6162157654762268,
-      "logps": -79.39734649658203,
-      "loss": 0.7105,
-      "objective": 0.7342749834060669,
+      "logits": -0.6085548996925354,
+      "logps": -79.48100280761719,
+      "loss": 0.7106,
+      "objective": 0.733691394329071,
       "ranking_idealized": 0.5199999809265137,
       "ranking_idealized_expo": 0.5108333230018616,
-      "ranking_simple": 0.6587499976158142,
-      "regularize": 0.7342749834060669,
+      "ranking_simple": 0.6579166650772095,
+      "regularize": 0.733691394329071,
       "step": 550,
-      "wo_beta": 4.5973711013793945
+      "wo_beta": 4.594529151916504
     },
     {
       "epoch": 1.5588096362777515,
-      "eval_dpo_loss": 2.073348045349121,
-      "eval_logits": -0.6734257340431213,
-      "eval_logps": -84.9780044555664,
-      "eval_loss": 2.16965651512146,
-      "eval_objective": 2.073348045349121,
+      "eval_dpo_loss": 2.069565534591675,
+      "eval_logits": -0.6665427088737488,
+      "eval_logps": -85.06731414794922,
+      "eval_loss": 2.1704676151275635,
+      "eval_objective": 2.069565534591675,
       "eval_ranking_idealized": 0.5295031070709229,
       "eval_ranking_idealized_expo": 0.5212215185165405,
       "eval_ranking_simple": 0.5320910811424255,
-      "eval_regularize": 2.073348045349121,
-      "eval_runtime": 374.3868,
-      "eval_samples_per_second": 15.465,
-      "eval_steps_per_second": 1.29,
-      "eval_wo_beta": 6.8721699714660645,
+      "eval_regularize": 2.069565534591675,
+      "eval_runtime": 308.5,
+      "eval_samples_per_second": 18.768,
+      "eval_steps_per_second": 1.566,
+      "eval_wo_beta": 6.861428260803223,
       "step": 550
     },
     {
-      "dpo_loss": 0.6709804534912109,
+      "dpo_loss": 0.6737102270126343,
       "epoch": 1.7005196032120926,
-      "grad_norm": 58.045592434636404,
+      "grad_norm": 58.923373544475695,
       "learning_rate": 4.16692250129073e-06,
-      "logits": -0.5785077810287476,
-      "logps": -79.41590118408203,
-      "loss": 0.6925,
-      "objective": 0.6709804534912109,
+      "logits": -0.5712339282035828,
+      "logps": -79.50873565673828,
+      "loss": 0.6934,
+      "objective": 0.6737102270126343,
       "ranking_idealized": 0.5220833420753479,
       "ranking_idealized_expo": 0.5149999856948853,
-      "ranking_simple": 0.6520833373069763,
-      "regularize": 0.6709804534912109,
+      "ranking_simple": 0.6537500023841858,
+      "regularize": 0.6737102270126343,
       "step": 600,
-      "wo_beta": 4.63525390625
+      "wo_beta": 4.632404804229736
     },
     {
       "epoch": 1.7005196032120926,
-      "eval_dpo_loss": 2.055753707885742,
-      "eval_logits": -0.7411422729492188,
-      "eval_logps": -81.53422546386719,
-      "eval_loss": 2.195674419403076,
-      "eval_objective": 2.055753707885742,
+      "eval_dpo_loss": 2.0693418979644775,
+      "eval_logits": -0.7358241081237793,
+      "eval_logps": -81.67726135253906,
+      "eval_loss": 2.2126805782318115,
+      "eval_objective": 2.0693418979644775,
       "eval_ranking_idealized": 0.5295031070709229,
       "eval_ranking_idealized_expo": 0.5212215185165405,
-      "eval_ranking_simple": 0.5357142686843872,
-      "eval_regularize": 2.055753707885742,
-      "eval_runtime": 308.1508,
-      "eval_samples_per_second": 18.79,
-      "eval_steps_per_second": 1.567,
-      "eval_wo_beta": 6.718571662902832,
+      "eval_ranking_simple": 0.5362318754196167,
+      "eval_regularize": 2.0693418979644775,
+      "eval_runtime": 308.5887,
+      "eval_samples_per_second": 18.763,
+      "eval_steps_per_second": 1.565,
+      "eval_wo_beta": 6.726527690887451,
       "step": 600
     },
     {
-      "dpo_loss": 0.6708642244338989,
+      "dpo_loss": 0.6718880534172058,
       "epoch": 1.8422295701464337,
-      "grad_norm": 71.07668633244941,
+      "grad_norm": 71.22185869553977,
       "learning_rate": 3.974272604254906e-06,
-      "logits": -0.6567726135253906,
-      "logps": -78.93568420410156,
-      "loss": 0.6883,
-      "objective": 0.6708642244338989,
+      "logits": -0.6514729261398315,
+      "logps": -79.11984252929688,
+      "loss": 0.6885,
+      "objective": 0.6718880534172058,
       "ranking_idealized": 0.5408333539962769,
       "ranking_idealized_expo": 0.527916669845581,
       "ranking_simple": 0.6583333611488342,
-      "regularize": 0.6708642244338989,
+      "regularize": 0.6718880534172058,
       "step": 650,
-      "wo_beta": 4.770144939422607
+      "wo_beta": 4.779858112335205
     },
     {
       "epoch": 1.8422295701464337,
-      "eval_dpo_loss": 2.1330411434173584,
-      "eval_logits": -0.6908087134361267,
-      "eval_logps": -82.73030853271484,
-      "eval_loss": 2.208021640777588,
-      "eval_objective": 2.1330411434173584,
+      "eval_dpo_loss": 2.1432294845581055,
+      "eval_logits": -0.6786962747573853,
+      "eval_logps": -82.88699340820312,
+      "eval_loss": 2.2198147773742676,
+      "eval_objective": 2.1432294845581055,
       "eval_ranking_idealized": 0.5295031070709229,
       "eval_ranking_idealized_expo": 0.5212215185165405,
-      "eval_ranking_simple": 0.5383023023605347,
-      "eval_regularize": 2.1330411434173584,
-      "eval_runtime": 351.6869,
-      "eval_samples_per_second": 16.464,
-      "eval_steps_per_second": 1.373,
-      "eval_wo_beta": 6.808131694793701,
+      "eval_ranking_simple": 0.5362318754196167,
+      "eval_regularize": 2.1432294845581055,
+      "eval_runtime": 308.8348,
+      "eval_samples_per_second": 18.748,
+      "eval_steps_per_second": 1.564,
+      "eval_wo_beta": 6.820231914520264,
       "step": 650
     },
     {
-      "dpo_loss": 0.6912000179290771,
+      "dpo_loss": 0.6824830770492554,
       "epoch": 1.9839395370807746,
-      "grad_norm": 70.15640375480324,
+      "grad_norm": 69.2974675476568,
       "learning_rate": 3.767136614452458e-06,
-      "logits": -0.5081126093864441,
-      "logps": -78.9532699584961,
-      "loss": 0.6486,
-      "objective": 0.6912000179290771,
+      "logits": -0.5030444860458374,
+      "logps": -78.93128967285156,
+      "loss": 0.6477,
+      "objective": 0.6824830770492554,
       "ranking_idealized": 0.5170833468437195,
       "ranking_idealized_expo": 0.5112500190734863,
-      "ranking_simple": 0.6445833444595337,
-      "regularize": 0.6912000179290771,
+      "ranking_simple": 0.6433333158493042,
+      "regularize": 0.6824830770492554,
       "step": 700,
-      "wo_beta": 4.81070613861084
+      "wo_beta": 4.807421684265137
     },
     {
       "epoch": 1.9839395370807746,
-      "eval_dpo_loss": 2.2471413612365723,
-      "eval_logits": -0.6650525331497192,
-      "eval_logps": -83.28822326660156,
-      "eval_loss": 2.3242855072021484,
-      "eval_objective": 2.2471413612365723,
+      "eval_dpo_loss": 2.2591824531555176,
+      "eval_logits": -0.659694254398346,
+      "eval_logps": -83.41045379638672,
+      "eval_loss": 2.3420462608337402,
+      "eval_objective": 2.2591824531555176,
       "eval_ranking_idealized": 0.5295031070709229,
       "eval_ranking_idealized_expo": 0.5212215185165405,
-      "eval_ranking_simple": 0.5377846956253052,
-      "eval_regularize": 2.2471413612365723,
-      "eval_runtime": 308.3677,
-      "eval_samples_per_second": 18.776,
-      "eval_steps_per_second": 1.566,
-      "eval_wo_beta": 6.681451797485352,
+      "eval_ranking_simple": 0.5403726696968079,
+      "eval_regularize": 2.2591824531555176,
+      "eval_runtime": 308.6048,
+      "eval_samples_per_second": 18.762,
+      "eval_steps_per_second": 1.565,
+      "eval_wo_beta": 6.68360710144043,
       "step": 700
     },
     {
-      "dpo_loss": 0.3827283978462219,
+      "dpo_loss": 0.38009390234947205,
       "epoch": 2.1256495040151155,
-      "grad_norm": 45.41487893439101,
+      "grad_norm": 45.305368238738886,
       "learning_rate": 3.547549834686222e-06,
-      "logits": -0.5765664577484131,
-      "logps": -79.59234619140625,
-      "loss": 0.3793,
-      "objective": 0.3827283978462219,
+      "logits": -0.5769148468971252,
+      "logps": -79.54972076416016,
+      "loss": 0.3785,
+      "objective": 0.38009390234947205,
       "ranking_idealized": 0.5216666460037231,
       "ranking_idealized_expo": 0.5112500190734863,
-      "ranking_simple": 0.6779166460037231,
-      "regularize": 0.3827283978462219,
+      "ranking_simple": 0.6787499785423279,
+      "regularize": 0.38009390234947205,
       "step": 750,
-      "wo_beta": 4.318643569946289
+      "wo_beta": 4.319232940673828
     },
     {
       "epoch": 2.1256495040151155,
-      "eval_dpo_loss": 2.182473659515381,
-      "eval_logits": -0.7878842353820801,
-      "eval_logps": -84.22957611083984,
-      "eval_loss": 2.2674732208251953,
-      "eval_objective": 2.182473659515381,
+      "eval_dpo_loss": 2.2005364894866943,
+      "eval_logits": -0.7841140627861023,
+      "eval_logps": -84.03689575195312,
+      "eval_loss": 2.2919445037841797,
+      "eval_objective": 2.2005364894866943,
       "eval_ranking_idealized": 0.5295031070709229,
       "eval_ranking_idealized_expo": 0.5212215185165405,
-      "eval_ranking_simple": 0.5408902764320374,
-      "eval_regularize": 2.182473659515381,
-      "eval_runtime": 325.9016,
-      "eval_samples_per_second": 17.766,
-      "eval_steps_per_second": 1.482,
-      "eval_wo_beta": 6.879360675811768,
+      "eval_ranking_simple": 0.54347825050354,
+      "eval_regularize": 2.2005364894866943,
+      "eval_runtime": 311.4559,
+      "eval_samples_per_second": 18.59,
+      "eval_steps_per_second": 1.551,
+      "eval_wo_beta": 6.8514323234558105,
       "step": 750
     },
     {
-      "dpo_loss": 0.3352104723453522,
+      "dpo_loss": 0.33919599652290344,
       "epoch": 2.2673594709494567,
-      "grad_norm": 40.69824942472133,
+      "grad_norm": 40.098481963513144,
       "learning_rate": 3.3176699082935546e-06,
-      "logits": -0.6111759543418884,
-      "logps": -80.46817779541016,
-      "loss": 0.3314,
-      "objective": 0.3352104723453522,
+      "logits": -0.6165332198143005,
+      "logps": -80.33509826660156,
+      "loss": 0.3316,
+      "objective": 0.33919599652290344,
       "ranking_idealized": 0.5195833444595337,
       "ranking_idealized_expo": 0.512499988079071,
-      "ranking_simple": 0.6787499785423279,
-      "regularize": 0.3352104723453522,
+      "ranking_simple": 0.6795833110809326,
+      "regularize": 0.33919599652290344,
       "step": 800,
-      "wo_beta": 3.9888339042663574
+      "wo_beta": 3.9735937118530273
     },
     {
       "epoch": 2.2673594709494567,
-      "eval_dpo_loss": 2.104069471359253,
-      "eval_logits": -0.665122926235199,
-      "eval_logps": -84.3675308227539,
-      "eval_loss": 2.2106120586395264,
-      "eval_objective": 2.104069471359253,
+      "eval_dpo_loss": 2.1123316287994385,
+      "eval_logits": -0.6766893267631531,
+      "eval_logps": -84.29901123046875,
+      "eval_loss": 2.2220041751861572,
+      "eval_objective": 2.1123316287994385,
       "eval_ranking_idealized": 0.5295031070709229,
       "eval_ranking_idealized_expo": 0.5212215185165405,
-      "eval_ranking_simple": 0.5414078831672668,
-      "eval_regularize": 2.104069471359253,
-      "eval_runtime": 341.6693,
-      "eval_samples_per_second": 16.946,
-      "eval_steps_per_second": 1.414,
-      "eval_wo_beta": 6.746298313140869,
+      "eval_ranking_simple": 0.5408902764320374,
+      "eval_regularize": 2.1123316287994385,
+      "eval_runtime": 309.0664,
+      "eval_samples_per_second": 18.734,
+      "eval_steps_per_second": 1.563,
+      "eval_wo_beta": 6.766322612762451,
       "step": 800
     },
     {
-      "dpo_loss": 0.3527311086654663,
+      "dpo_loss": 0.34950682520866394,
       "epoch": 2.409069437883798,
-      "grad_norm": 34.29885501684047,
+      "grad_norm": 35.45933765377846,
       "learning_rate": 3.0797556183036582e-06,
-      "logits": -0.582695722579956,
-      "logps": -80.7869873046875,
-      "loss": 0.3301,
-      "objective": 0.3527311086654663,
+      "logits": -0.6031672954559326,
+      "logps": -80.82646942138672,
+      "loss": 0.3283,
+      "objective": 0.34950682520866394,
       "ranking_idealized": 0.5254166722297668,
       "ranking_idealized_expo": 0.5141666531562805,
-      "ranking_simple": 0.6770833134651184,
-      "regularize": 0.3527311086654663,
+      "ranking_simple": 0.6779166460037231,
+      "regularize": 0.34950682520866394,
       "step": 850,
-      "wo_beta": 4.281794548034668
+      "wo_beta": 4.280696392059326
     },
     {
       "epoch": 2.409069437883798,
-      "eval_dpo_loss": 2.222111940383911,
-      "eval_logits": -0.6176895499229431,
-      "eval_logps": -84.89134979248047,
-      "eval_loss": 2.296431303024292,
-      "eval_objective": 2.222111940383911,
+      "eval_dpo_loss": 2.221233606338501,
+      "eval_logits": -0.6537899374961853,
+      "eval_logps": -85.08342742919922,
+      "eval_loss": 2.3019895553588867,
+      "eval_objective": 2.221233606338501,
       "eval_ranking_idealized": 0.5295031070709229,
       "eval_ranking_idealized_expo": 0.5212215185165405,
-      "eval_ranking_simple": 0.5388198494911194,
-      "eval_regularize": 2.222111940383911,
-      "eval_runtime": 308.4106,
-      "eval_samples_per_second": 18.774,
-      "eval_steps_per_second": 1.566,
-      "eval_wo_beta": 6.801980495452881,
+      "eval_ranking_simple": 0.5408902764320374,
+      "eval_regularize": 2.221233606338501,
+      "eval_runtime": 308.3167,
+      "eval_samples_per_second": 18.779,
+      "eval_steps_per_second": 1.567,
+      "eval_wo_beta": 6.777287006378174,
       "step": 850
     },
     {
-      "dpo_loss": 0.3560827672481537,
+      "dpo_loss": 0.35370326042175293,
       "epoch": 2.550779404818139,
-      "grad_norm": 59.77914010101811,
+      "grad_norm": 63.704496862923826,
       "learning_rate": 2.8361446928038298e-06,
-      "logits": -0.5723668336868286,
-      "logps": -79.74351501464844,
-      "loss": 0.3509,
-      "objective": 0.3560827672481537,
+      "logits": -0.600394606590271,
+      "logps": -79.94818115234375,
+      "loss": 0.3516,
+      "objective": 0.35370326042175293,
       "ranking_idealized": 0.5249999761581421,
       "ranking_idealized_expo": 0.5183333158493042,
-      "ranking_simple": 0.675000011920929,
-      "regularize": 0.3560827672481537,
+      "ranking_simple": 0.6729166507720947,
+      "regularize": 0.35370326042175293,
       "step": 900,
-      "wo_beta": 4.2809739112854
+      "wo_beta": 4.2681732177734375
     },
     {
       "epoch": 2.550779404818139,
-      "eval_dpo_loss": 2.2098634243011475,
-      "eval_logits": -0.6096944808959961,
-      "eval_logps": -84.38330841064453,
-      "eval_loss": 2.2795627117156982,
-      "eval_objective": 2.2098634243011475,
+      "eval_dpo_loss": 2.1911425590515137,
+      "eval_logits": -0.6225402355194092,
+      "eval_logps": -84.7564468383789,
+      "eval_loss": 2.272311210632324,
+      "eval_objective": 2.1911425590515137,
       "eval_ranking_idealized": 0.5295031070709229,
       "eval_ranking_idealized_expo": 0.5212215185165405,
-      "eval_ranking_simple": 0.5393374562263489,
-      "eval_regularize": 2.2098634243011475,
-      "eval_runtime": 342.057,
-      "eval_samples_per_second": 16.927,
-      "eval_steps_per_second": 1.412,
-      "eval_wo_beta": 6.793368339538574,
+      "eval_ranking_simple": 0.5362318754196167,
+      "eval_regularize": 2.1911425590515137,
+      "eval_runtime": 308.2356,
+      "eval_samples_per_second": 18.784,
+      "eval_steps_per_second": 1.567,
+      "eval_wo_beta": 6.816178798675537,
       "step": 900
     },
     {
-      "dpo_loss": 0.3693172037601471,
+      "dpo_loss": 0.38168859481811523,
       "epoch": 2.69248937175248,
-      "grad_norm": 43.046650555153526,
+      "grad_norm": 42.960212326170364,
       "learning_rate": 2.5892308345974517e-06,
-      "logits": -0.5750654339790344,
-      "logps": -79.79653930664062,
-      "loss": 0.321,
-      "objective": 0.3693172037601471,
+      "logits": -0.5876100659370422,
+      "logps": -80.23540496826172,
+      "loss": 0.3245,
+      "objective": 0.38168859481811523,
       "ranking_idealized": 0.5162500143051147,
       "ranking_idealized_expo": 0.5058333277702332,
-      "ranking_simple": 0.6879166960716248,
-      "regularize": 0.3693172037601471,
+      "ranking_simple": 0.6866666674613953,
+      "regularize": 0.38168859481811523,
       "step": 950,
-      "wo_beta": 4.201274871826172
+      "wo_beta": 4.207254409790039
     },
     {
       "epoch": 2.69248937175248,
-      "eval_dpo_loss": 2.264862298965454,
-      "eval_logits": -0.7158052921295166,
-      "eval_logps": -83.29668426513672,
-      "eval_loss": 2.3403165340423584,
-      "eval_objective": 2.264862298965454,
+      "eval_dpo_loss": 2.2523224353790283,
+      "eval_logits": -0.7128884196281433,
+      "eval_logps": -83.6421127319336,
+      "eval_loss": 2.3303723335266113,
+      "eval_objective": 2.2523224353790283,
       "eval_ranking_idealized": 0.5295031070709229,
       "eval_ranking_idealized_expo": 0.5212215185165405,
-      "eval_ranking_simple": 0.5331262946128845,
-      "eval_regularize": 2.264862298965454,
-      "eval_runtime": 315.8555,
-      "eval_samples_per_second": 18.331,
-      "eval_steps_per_second": 1.529,
-      "eval_wo_beta": 6.886437892913818,
+      "eval_ranking_simple": 0.533643901348114,
+      "eval_regularize": 2.2523224353790283,
+      "eval_runtime": 308.3878,
+      "eval_samples_per_second": 18.775,
+      "eval_steps_per_second": 1.566,
+      "eval_wo_beta": 6.89421272277832,
       "step": 950
     },
     {
       "epoch": 2.69248937175248,
       "step": 950,
       "total_flos": 0.0,
-      "train_loss": 0.7490891541932758,
-      "train_runtime": 28216.8085,
-      "train_samples_per_second": 9.002,
-      "train_steps_per_second": 0.062
+      "train_loss": 0.7491915085441188,
+      "train_runtime": 26152.6935,
+      "train_samples_per_second": 9.713,
+      "train_steps_per_second": 0.067
     }
   ],
   "logging_steps": 50,