Model save

README.md

@@ -2,16 +2,9 @@
 license: apache-2.0
 base_model: hZzy/qwen2.5-0.5b-sft-news-IFT
 tags:
-- alignment-handbook
-- ndcg
 - trl
 - expo
 - generated_from_trainer
-- trl
-- expo
-- generated_from_trainer
-datasets:
-- hZzy/train_pairwise
 model-index:
 - name: qwen2.5-0.5b-expo-DPO-ES-0.1
   results: []
@@ -20,21 +13,21 @@ model-index:
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 should probably proofread and complete it, then remove this comment. -->
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/zhiyuzha-university-of-florida/huggingface/runs/w0nbtpl2)
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/zhiyuzha-university-of-florida/huggingface/runs/hy7b2tcq)
 # qwen2.5-0.5b-expo-DPO-ES-0.1
 
-This model is a fine-tuned version of [hZzy/qwen2.5-0.5b-sft-news-IFT](https://huggingface.co/hZzy/qwen2.5-0.5b-sft-news-IFT) on the hZzy/train_pairwise dataset.
+This model is a fine-tuned version of [hZzy/qwen2.5-0.5b-sft-news-IFT](https://huggingface.co/hZzy/qwen2.5-0.5b-sft-news-IFT) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6923
-- Logps: -90.9491
-- Logits: -2.1209
-- Objective: 0.6894
-- Dpo Loss: 0.6894
-- Regularize: 0.6894
-- Ranking Simple: 0.5564
+- Loss: 0.7044
+- Logps: -104.2469
+- Logits: -1.8870
+- Objective: 0.7077
+- Dpo Loss: 0.7077
+- Regularize: 0.7077
+- Ranking Simple: 0.5652
 - Ranking Idealized: 0.6030
 - Ranking Idealized Expo: 0.5223
-- Wo Beta: 7.4232
+- Wo Beta: 8.1947
 
 ## Model description
 
@@ -71,22 +64,22 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Logps     | Logits  | Objective | Dpo Loss | Regularize | Ranking Simple | Ranking Idealized | Ranking Idealized Expo | Wo Beta |
 |:-------------:|:------:|:----:|:---------------:|:---------:|:-------:|:---------:|:--------:|:----------:|:--------------:|:-----------------:|:----------------------:|:-------:|
-| 0.6785        | 0.1417 | 50   | 0.6814          | -90.8721  | -1.6022 | 0.6843    | 0.6843   | 0.6843     | 0.5259         | 0.6030            | 0.5223                 | 7.8749  |
-| 0.618         | 0.2834 | 100  | 0.6733          | -98.8900  | -1.7799 | 0.6766    | 0.6766   | 0.6766     | 0.5399         | 0.6030            | 0.5223                 | 7.7840  |
-| 0.5667        | 0.4251 | 150  | 0.6867          | -99.1217  | -1.8072 | 0.6829    | 0.6829   | 0.6829     | 0.5409         | 0.6030            | 0.5223                 | 7.8537  |
-| 0.5214        | 0.5668 | 200  | 0.6902          | -99.5153  | -1.8895 | 0.6905    | 0.6905   | 0.6905     | 0.5445         | 0.6030            | 0.5223                 | 7.7013  |
-| 0.4922        | 0.7085 | 250  | 0.6976          | -82.8384  | -1.9887 | 0.6914    | 0.6914   | 0.6914     | 0.5481         | 0.6030            | 0.5223                 | 7.8784  |
-| 0.4535        | 0.8503 | 300  | 0.6923          | -90.9491  | -2.1209 | 0.6894    | 0.6894   | 0.6894     | 0.5564         | 0.6030            | 0.5223                 | 7.4232  |
-| 0.4228        | 0.9920 | 350  | 0.7064          | -87.7231  | -1.9803 | 0.6968    | 0.6968   | 0.6968     | 0.5538         | 0.6030            | 0.5223                 | 8.0253  |
-| 0.2845        | 1.1337 | 400  | 0.7305          | -101.3180 | -2.0805 | 0.7269    | 0.7269   | 0.7269     | 0.5430         | 0.6030            | 0.5223                 | 8.6164  |
-| 0.2989        | 1.2754 | 450  | 0.7005          | -93.1955  | -1.8646 | 0.6974    | 0.6974   | 0.6974     | 0.5606         | 0.6030            | 0.5223                 | 8.2386  |
-| 0.3065        | 1.4171 | 500  | 0.7179          | -97.0137  | -1.9983 | 0.7147    | 0.7147   | 0.7147     | 0.5549         | 0.6030            | 0.5223                 | 8.2760  |
-| 0.2885        | 1.5588 | 550  | 0.7091          | -107.9610 | -1.9041 | 0.7134    | 0.7134   | 0.7134     | 0.5616         | 0.6030            | 0.5223                 | 8.1968  |
+| 0.6785        | 0.1417 | 50   | 0.6813          | -90.8716  | -1.6022 | 0.6843    | 0.6843   | 0.6843     | 0.5259         | 0.6030            | 0.5223                 | 7.8749  |
+| 0.618         | 0.2834 | 100  | 0.6733          | -98.8899  | -1.7799 | 0.6766    | 0.6766   | 0.6766     | 0.5399         | 0.6030            | 0.5223                 | 7.7840  |
+| 0.5667        | 0.4251 | 150  | 0.6866          | -99.1230  | -1.8072 | 0.6829    | 0.6829   | 0.6829     | 0.5409         | 0.6030            | 0.5223                 | 7.8533  |
+| 0.5214        | 0.5668 | 200  | 0.6901          | -99.5388  | -1.8894 | 0.6904    | 0.6904   | 0.6904     | 0.5445         | 0.6030            | 0.5223                 | 7.6995  |
+| 0.4922        | 0.7085 | 250  | 0.6976          | -82.7973  | -1.9880 | 0.6916    | 0.6916   | 0.6916     | 0.5476         | 0.6030            | 0.5223                 | 7.8790  |
+| 0.4535        | 0.8503 | 300  | 0.6921          | -91.0309  | -2.1203 | 0.6893    | 0.6893   | 0.6893     | 0.5559         | 0.6030            | 0.5223                 | 7.4262  |
+| 0.423         | 0.9920 | 350  | 0.7057          | -88.1615  | -1.9880 | 0.6959    | 0.6959   | 0.6959     | 0.5549         | 0.6030            | 0.5223                 | 7.9979  |
+| 0.2847        | 1.1337 | 400  | 0.7315          | -101.6926 | -2.0862 | 0.7281    | 0.7281   | 0.7281     | 0.5424         | 0.6030            | 0.5223                 | 8.6326  |
+| 0.2991        | 1.2754 | 450  | 0.7008          | -92.7942  | -1.8470 | 0.6980    | 0.6980   | 0.6980     | 0.5621         | 0.6030            | 0.5223                 | 8.2584  |
+| 0.3065        | 1.4171 | 500  | 0.7180          | -96.6747  | -2.0065 | 0.7147    | 0.7147   | 0.7147     | 0.5554         | 0.6030            | 0.5223                 | 8.2522  |
+| 0.2895        | 1.5588 | 550  | 0.7044          | -104.2469 | -1.8870 | 0.7077    | 0.7077   | 0.7077     | 0.5652         | 0.6030            | 0.5223                 | 8.1947  |
 
 
 ### Framework versions
 
 - Transformers 4.42.0
 - Pytorch 2.3.0+cu121
-- Datasets 2.19.1
+- Datasets 3.2.0
 - Tokenizers 0.19.1

all_results.json

@@ -1,23 +1,9 @@
 {
     "epoch": 1.5588096362777515,
-    "eval_dpo_loss": 0.6893911957740784,
-    "eval_logits": -2.1208713054656982,
-    "eval_logps": -90.9490966796875,
-    "eval_loss": 0.6922824382781982,
-    "eval_objective": 0.6893911957740784,
-    "eval_ranking_idealized": 0.6030020713806152,
-    "eval_ranking_idealized_expo": 0.5222567319869995,
-    "eval_ranking_simple": 0.556418240070343,
-    "eval_regularize": 0.6893911957740784,
-    "eval_runtime": 314.8886,
-    "eval_samples": 5790,
-    "eval_samples_per_second": 18.387,
-    "eval_steps_per_second": 1.534,
-    "eval_wo_beta": 7.4231791496276855,
     "total_flos": 0.0,
-    "train_loss": 0.4483427975394509,
-    "train_runtime": 15087.4424,
+    "train_loss": 0.4484944924441251,
+    "train_runtime": 15200.6621,
     "train_samples": 50802,
-    "train_samples_per_second": 16.836,
-    "train_steps_per_second": 0.117
+    "train_samples_per_second": 16.71,
+    "train_steps_per_second": 0.116
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e42889b51298a05b37f983b9f4e2044f618425ed9654eee76d9c8ca3131def1c
+oid sha256:654597aa1c17b0ef958f9135269de2f1d79597cdfbc53eea3e4411458602cc1b
 size 1975192208

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.5588096362777515,
     "total_flos": 0.0,
-    "train_loss": 0.4483427975394509,
-    "train_runtime": 15087.4424,
+    "train_loss": 0.4484944924441251,
+    "train_runtime": 15200.6621,
     "train_samples": 50802,
-    "train_samples_per_second": 16.836,
-    "train_steps_per_second": 0.117
+    "train_samples_per_second": 16.71,
+    "train_steps_per_second": 0.116
 }

trainer_state.json

@@ -1,5 +1,5 @@
 {
-  "best_metric": 7.4231791496276855,
+  "best_metric": 7.426205635070801,
   "best_model_checkpoint": "./qwen2.5-0.5b/qwen2.5-0.5b-expo-DPO-ES-0.1/checkpoint-300",
   "epoch": 1.5588096362777515,
   "eval_steps": 50,
@@ -11,7 +11,7 @@
     {
       "dpo_loss": 0.6931471824645996,
       "epoch": 0.002834199338686821,
-      "grad_norm": 18.44253347826331,
+      "grad_norm": 18.442536934850562,
       "learning_rate": 2.840909090909091e-08,
       "logits": -1.359458565711975,
       "logps": -84.69721221923828,
@@ -27,10 +27,10 @@
     {
       "dpo_loss": 0.6822353601455688,
       "epoch": 0.14170996693434104,
-      "grad_norm": 18.641365531241362,
+      "grad_norm": 18.641661833444882,
       "learning_rate": 1.4204545454545458e-06,
-      "logits": -1.508646011352539,
-      "logps": -82.822021484375,
+      "logits": -1.5086464881896973,
+      "logps": -82.8218765258789,
       "loss": 0.6785,
       "objective": 0.6822353601455688,
       "ranking_idealized": 0.608418345451355,
@@ -38,363 +38,363 @@
       "ranking_simple": 0.5267857313156128,
       "regularize": 0.6822353601455688,
       "step": 50,
-      "wo_beta": 7.122643947601318
+      "wo_beta": 7.122647285461426
     },
     {
       "epoch": 0.14170996693434104,
-      "eval_dpo_loss": 0.6842507719993591,
-      "eval_logits": -1.6022367477416992,
-      "eval_logps": -90.87205505371094,
-      "eval_loss": 0.6813501715660095,
-      "eval_objective": 0.6842507719993591,
+      "eval_dpo_loss": 0.6842505931854248,
+      "eval_logits": -1.6022353172302246,
+      "eval_logps": -90.87158203125,
+      "eval_loss": 0.6813499927520752,
+      "eval_objective": 0.6842505931854248,
       "eval_ranking_idealized": 0.6030020713806152,
       "eval_ranking_idealized_expo": 0.5222567319869995,
       "eval_ranking_simple": 0.5258799195289612,
-      "eval_regularize": 0.6842507719993591,
-      "eval_runtime": 308.2905,
-      "eval_samples_per_second": 18.781,
-      "eval_steps_per_second": 1.567,
-      "eval_wo_beta": 7.874889850616455,
+      "eval_regularize": 0.6842505931854248,
+      "eval_runtime": 309.4614,
+      "eval_samples_per_second": 18.71,
+      "eval_steps_per_second": 1.561,
+      "eval_wo_beta": 7.874892711639404,
       "step": 50
     },
     {
-      "dpo_loss": 0.61030513048172,
+      "dpo_loss": 0.6103044152259827,
       "epoch": 0.2834199338686821,
-      "grad_norm": 19.29211957833529,
+      "grad_norm": 19.292131977363915,
       "learning_rate": 2.8409090909090916e-06,
-      "logits": -1.6967747211456299,
-      "logps": -88.03068542480469,
+      "logits": -1.696779727935791,
+      "logps": -88.03015899658203,
       "loss": 0.618,
-      "objective": 0.61030513048172,
+      "objective": 0.6103044152259827,
       "ranking_idealized": 0.6016666889190674,
       "ranking_idealized_expo": 0.5141666531562805,
       "ranking_simple": 0.5729166865348816,
-      "regularize": 0.61030513048172,
+      "regularize": 0.6103044152259827,
       "step": 100,
-      "wo_beta": 6.357723236083984
+      "wo_beta": 6.357714653015137
     },
     {
       "epoch": 0.2834199338686821,
-      "eval_dpo_loss": 0.6765881776809692,
-      "eval_logits": -1.7799152135849,
-      "eval_logps": -98.88995361328125,
-      "eval_loss": 0.6732921004295349,
-      "eval_objective": 0.6765881776809692,
+      "eval_dpo_loss": 0.6765866279602051,
+      "eval_logits": -1.7799253463745117,
+      "eval_logps": -98.88992309570312,
+      "eval_loss": 0.6732903122901917,
+      "eval_objective": 0.6765866279602051,
       "eval_ranking_idealized": 0.6030020713806152,
       "eval_ranking_idealized_expo": 0.5222567319869995,
       "eval_ranking_simple": 0.5398550629615784,
-      "eval_regularize": 0.6765881776809692,
-      "eval_runtime": 307.7771,
-      "eval_samples_per_second": 18.812,
-      "eval_steps_per_second": 1.569,
-      "eval_wo_beta": 7.7840118408203125,
+      "eval_regularize": 0.6765866279602051,
+      "eval_runtime": 308.813,
+      "eval_samples_per_second": 18.749,
+      "eval_steps_per_second": 1.564,
+      "eval_wo_beta": 7.784023284912109,
       "step": 100
     },
     {
-      "dpo_loss": 0.5696364045143127,
+      "dpo_loss": 0.5696373581886292,
       "epoch": 0.42512990080302315,
-      "grad_norm": 18.142654654279198,
+      "grad_norm": 18.14389066821128,
       "learning_rate": 4.2613636363636365e-06,
-      "logits": -1.7447518110275269,
-      "logps": -90.71894073486328,
+      "logits": -1.744734287261963,
+      "logps": -90.7203140258789,
       "loss": 0.5667,
-      "objective": 0.5696364045143127,
+      "objective": 0.5696373581886292,
       "ranking_idealized": 0.6066666841506958,
       "ranking_idealized_expo": 0.5287500023841858,
       "ranking_simple": 0.6016666889190674,
-      "regularize": 0.5696364045143127,
+      "regularize": 0.5696373581886292,
       "step": 150,
-      "wo_beta": 5.971243858337402
+      "wo_beta": 5.971276760101318
     },
     {
       "epoch": 0.42512990080302315,
-      "eval_dpo_loss": 0.6829443573951721,
-      "eval_logits": -1.807220697402954,
-      "eval_logps": -99.12174987792969,
-      "eval_loss": 0.686660647392273,
-      "eval_objective": 0.6829443573951721,
+      "eval_dpo_loss": 0.6829108595848083,
+      "eval_logits": -1.8072086572647095,
+      "eval_logps": -99.12300109863281,
+      "eval_loss": 0.6866306066513062,
+      "eval_objective": 0.6829108595848083,
       "eval_ranking_idealized": 0.6030020713806152,
       "eval_ranking_idealized_expo": 0.5222567319869995,
       "eval_ranking_simple": 0.5408902764320374,
-      "eval_regularize": 0.6829443573951721,
-      "eval_runtime": 308.0058,
-      "eval_samples_per_second": 18.798,
-      "eval_steps_per_second": 1.568,
-      "eval_wo_beta": 7.853672981262207,
+      "eval_regularize": 0.6829108595848083,
+      "eval_runtime": 312.2547,
+      "eval_samples_per_second": 18.543,
+      "eval_steps_per_second": 1.547,
+      "eval_wo_beta": 7.8532514572143555,
       "step": 150
     },
     {
-      "dpo_loss": 0.5278546214103699,
+      "dpo_loss": 0.5278292298316956,
       "epoch": 0.5668398677373642,
-      "grad_norm": 15.355172339669656,
+      "grad_norm": 15.350860718764396,
       "learning_rate": 4.997168347957521e-06,
-      "logits": -1.9084746837615967,
-      "logps": -92.50672149658203,
+      "logits": -1.908250093460083,
+      "logps": -92.51087951660156,
       "loss": 0.5214,
-      "objective": 0.5278546214103699,
+      "objective": 0.5278292298316956,
       "ranking_idealized": 0.5924999713897705,
       "ranking_idealized_expo": 0.5166666507720947,
       "ranking_simple": 0.6362500190734863,
-      "regularize": 0.5278546214103699,
+      "regularize": 0.5278292298316956,
       "step": 200,
-      "wo_beta": 5.543394565582275
+      "wo_beta": 5.543264389038086
     },
     {
       "epoch": 0.5668398677373642,
-      "eval_dpo_loss": 0.69049072265625,
-      "eval_logits": -1.8895256519317627,
-      "eval_logps": -99.51531982421875,
-      "eval_loss": 0.6901801824569702,
-      "eval_objective": 0.69049072265625,
+      "eval_dpo_loss": 0.6904094815254211,
+      "eval_logits": -1.8893996477127075,
+      "eval_logps": -99.53878784179688,
+      "eval_loss": 0.6900797486305237,
+      "eval_objective": 0.6904094815254211,
       "eval_ranking_idealized": 0.6030020713806152,
       "eval_ranking_idealized_expo": 0.5222567319869995,
       "eval_ranking_simple": 0.544513463973999,
-      "eval_regularize": 0.69049072265625,
-      "eval_runtime": 308.0169,
-      "eval_samples_per_second": 18.798,
-      "eval_steps_per_second": 1.568,
-      "eval_wo_beta": 7.701313495635986,
+      "eval_regularize": 0.6904094815254211,
+      "eval_runtime": 315.3971,
+      "eval_samples_per_second": 18.358,
+      "eval_steps_per_second": 1.531,
+      "eval_wo_beta": 7.699478626251221,
       "step": 200
     },
     {
-      "dpo_loss": 0.4935472011566162,
+      "dpo_loss": 0.49355897307395935,
       "epoch": 0.7085498346717053,
-      "grad_norm": 14.324676993767012,
+      "grad_norm": 14.313286499637714,
       "learning_rate": 4.973122855144066e-06,
-      "logits": -1.9081355333328247,
-      "logps": -88.3556137084961,
+      "logits": -1.9076462984085083,
+      "logps": -88.3504867553711,
       "loss": 0.4922,
-      "objective": 0.4935472011566162,
+      "objective": 0.49355897307395935,
       "ranking_idealized": 0.5991666913032532,
       "ranking_idealized_expo": 0.5170833468437195,
       "ranking_simple": 0.6608333587646484,
-      "regularize": 0.4935472011566162,
+      "regularize": 0.49355897307395935,
       "step": 250,
-      "wo_beta": 5.325418472290039
+      "wo_beta": 5.325013637542725
     },
     {
       "epoch": 0.7085498346717053,
-      "eval_dpo_loss": 0.6914450526237488,
-      "eval_logits": -1.9887231588363647,
-      "eval_logps": -82.8383560180664,
-      "eval_loss": 0.6975522041320801,
-      "eval_objective": 0.6914450526237488,
+      "eval_dpo_loss": 0.6915506720542908,
+      "eval_logits": -1.9879554510116577,
+      "eval_logps": -82.79730224609375,
+      "eval_loss": 0.6976116895675659,
+      "eval_objective": 0.6915506720542908,
       "eval_ranking_idealized": 0.6030020713806152,
       "eval_ranking_idealized_expo": 0.5222567319869995,
-      "eval_ranking_simple": 0.5481366515159607,
-      "eval_regularize": 0.6914450526237488,
-      "eval_runtime": 307.641,
-      "eval_samples_per_second": 18.821,
-      "eval_steps_per_second": 1.57,
-      "eval_wo_beta": 7.878448486328125,
+      "eval_ranking_simple": 0.5476190447807312,
+      "eval_regularize": 0.6915506720542908,
+      "eval_runtime": 308.7255,
+      "eval_samples_per_second": 18.755,
+      "eval_steps_per_second": 1.564,
+      "eval_wo_beta": 7.87898588180542,
       "step": 250
     },
     {
-      "dpo_loss": 0.4521400034427643,
+      "dpo_loss": 0.4521573483943939,
       "epoch": 0.8502598016060463,
-      "grad_norm": 13.428528764338076,
+      "grad_norm": 13.44014162581437,
       "learning_rate": 4.924776641419513e-06,
-      "logits": -2.0954272747039795,
-      "logps": -81.39044952392578,
+      "logits": -2.09318470954895,
+      "logps": -81.41643524169922,
       "loss": 0.4535,
-      "objective": 0.4521400034427643,
+      "objective": 0.4521573483943939,
       "ranking_idealized": 0.5799999833106995,
       "ranking_idealized_expo": 0.4970833361148834,
       "ranking_simple": 0.6837499737739563,
-      "regularize": 0.4521400034427643,
+      "regularize": 0.4521573483943939,
       "step": 300,
-      "wo_beta": 5.026640892028809
+      "wo_beta": 5.026339054107666
     },
     {
       "epoch": 0.8502598016060463,
-      "eval_dpo_loss": 0.6893911957740784,
-      "eval_logits": -2.1208713054656982,
-      "eval_logps": -90.9490966796875,
-      "eval_loss": 0.6922824382781982,
-      "eval_objective": 0.6893911957740784,
+      "eval_dpo_loss": 0.6892624497413635,
+      "eval_logits": -2.120311975479126,
+      "eval_logps": -91.03094482421875,
+      "eval_loss": 0.6920701265335083,
+      "eval_objective": 0.6892624497413635,
       "eval_ranking_idealized": 0.6030020713806152,
       "eval_ranking_idealized_expo": 0.5222567319869995,
-      "eval_ranking_simple": 0.556418240070343,
-      "eval_regularize": 0.6893911957740784,
-      "eval_runtime": 307.9716,
-      "eval_samples_per_second": 18.8,
-      "eval_steps_per_second": 1.568,
-      "eval_wo_beta": 7.4231791496276855,
+      "eval_ranking_simple": 0.5559006333351135,
+      "eval_regularize": 0.6892624497413635,
+      "eval_runtime": 308.4714,
+      "eval_samples_per_second": 18.77,
+      "eval_steps_per_second": 1.566,
+      "eval_wo_beta": 7.426205635070801,
       "step": 300
     },
     {
-      "dpo_loss": 0.40847164392471313,
+      "dpo_loss": 0.40864402055740356,
       "epoch": 0.9919697685403873,
-      "grad_norm": 15.041732370189118,
+      "grad_norm": 15.102301026818012,
       "learning_rate": 4.8526047530778175e-06,
-      "logits": -1.9279303550720215,
-      "logps": -84.23763275146484,
-      "loss": 0.4228,
-      "objective": 0.40847164392471313,
+      "logits": -1.9232014417648315,
+      "logps": -84.39765930175781,
+      "loss": 0.423,
+      "objective": 0.40864402055740356,
       "ranking_idealized": 0.60916668176651,
       "ranking_idealized_expo": 0.5270833373069763,
-      "ranking_simple": 0.7191666960716248,
-      "regularize": 0.40847164392471313,
+      "ranking_simple": 0.7174999713897705,
+      "regularize": 0.40864402055740356,
       "step": 350,
-      "wo_beta": 3.9990389347076416
+      "wo_beta": 3.9979019165039062
     },
     {
       "epoch": 0.9919697685403873,
-      "eval_dpo_loss": 0.6968410611152649,
-      "eval_logits": -1.9802873134613037,
-      "eval_logps": -87.72307586669922,
-      "eval_loss": 0.7063526511192322,
-      "eval_objective": 0.6968410611152649,
+      "eval_dpo_loss": 0.6959461569786072,
+      "eval_logits": -1.987973690032959,
+      "eval_logps": -88.1614990234375,
+      "eval_loss": 0.7056758403778076,
+      "eval_objective": 0.6959461569786072,
       "eval_ranking_idealized": 0.6030020713806152,
       "eval_ranking_idealized_expo": 0.5222567319869995,
-      "eval_ranking_simple": 0.5538302063941956,
-      "eval_regularize": 0.6968410611152649,
-      "eval_runtime": 307.977,
-      "eval_samples_per_second": 18.8,
-      "eval_steps_per_second": 1.568,
-      "eval_wo_beta": 8.025344848632812,
+      "eval_ranking_simple": 0.5548654198646545,
+      "eval_regularize": 0.6959461569786072,
+      "eval_runtime": 309.9548,
+      "eval_samples_per_second": 18.68,
+      "eval_steps_per_second": 1.558,
+      "eval_wo_beta": 7.997907638549805,
       "step": 350
     },
     {
-      "dpo_loss": 0.28143200278282166,
+      "dpo_loss": 0.28186026215553284,
       "epoch": 1.1336797354747283,
-      "grad_norm": 12.112116162894052,
+      "grad_norm": 12.122355016988976,
       "learning_rate": 4.757316345716554e-06,
-      "logits": -2.0127880573272705,
-      "logps": -91.47932434082031,
-      "loss": 0.2845,
-      "objective": 0.28143200278282166,
+      "logits": -2.0241637229919434,
+      "logps": -91.86678314208984,
+      "loss": 0.2847,
+      "objective": 0.28186026215553284,
       "ranking_idealized": 0.6087499856948853,
       "ranking_idealized_expo": 0.5337499976158142,
-      "ranking_simple": 0.8141666650772095,
-      "regularize": 0.28143200278282166,
+      "ranking_simple": 0.8145833611488342,
+      "regularize": 0.28186026215553284,
       "step": 400,
-      "wo_beta": 2.649115562438965
+      "wo_beta": 2.646965980529785
     },
     {
       "epoch": 1.1336797354747283,
-      "eval_dpo_loss": 0.7269378900527954,
-      "eval_logits": -2.080526828765869,
-      "eval_logps": -101.31802368164062,
-      "eval_loss": 0.7304782867431641,
-      "eval_objective": 0.7269378900527954,
+      "eval_dpo_loss": 0.7281294465065002,
+      "eval_logits": -2.0862255096435547,
+      "eval_logps": -101.69258117675781,
+      "eval_loss": 0.7314654588699341,
+      "eval_objective": 0.7281294465065002,
       "eval_ranking_idealized": 0.6030020713806152,
       "eval_ranking_idealized_expo": 0.5222567319869995,
-      "eval_ranking_simple": 0.5429606437683105,
-      "eval_regularize": 0.7269378900527954,
-      "eval_runtime": 308.0503,
-      "eval_samples_per_second": 18.796,
-      "eval_steps_per_second": 1.568,
-      "eval_wo_beta": 8.616350173950195,
+      "eval_ranking_simple": 0.542443037033081,
+      "eval_regularize": 0.7281294465065002,
+      "eval_runtime": 308.7063,
+      "eval_samples_per_second": 18.756,
+      "eval_steps_per_second": 1.565,
+      "eval_wo_beta": 8.632596969604492,
       "step": 400
     },
     {
-      "dpo_loss": 0.3042532503604889,
+      "dpo_loss": 0.30417078733444214,
       "epoch": 1.2753897024090695,
-      "grad_norm": 12.171625264502326,
+      "grad_norm": 12.255120909837279,
       "learning_rate": 4.639847716126855e-06,
-      "logits": -1.9084649085998535,
-      "logps": -92.91566467285156,
-      "loss": 0.2989,
-      "objective": 0.3042532503604889,
+      "logits": -1.9048844575881958,
+      "logps": -92.96432495117188,
+      "loss": 0.2991,
+      "objective": 0.30417078733444214,
       "ranking_idealized": 0.5975000262260437,
       "ranking_idealized_expo": 0.5199999809265137,
-      "ranking_simple": 0.8075000047683716,
-      "regularize": 0.3042532503604889,
+      "ranking_simple": 0.8087499737739563,
+      "regularize": 0.30417078733444214,
       "step": 450,
-      "wo_beta": 3.341869592666626
+      "wo_beta": 3.335141658782959
     },
     {
       "epoch": 1.2753897024090695,
-      "eval_dpo_loss": 0.6973706483840942,
-      "eval_logits": -1.8646337985992432,
-      "eval_logps": -93.1955337524414,
-      "eval_loss": 0.7005103826522827,
-      "eval_objective": 0.6973706483840942,
+      "eval_dpo_loss": 0.6979546546936035,
+      "eval_logits": -1.8470289707183838,
+      "eval_logps": -92.79419708251953,
+      "eval_loss": 0.7008146047592163,
+      "eval_objective": 0.6979546546936035,
       "eval_ranking_idealized": 0.6030020713806152,
       "eval_ranking_idealized_expo": 0.5222567319869995,
-      "eval_ranking_simple": 0.5605590343475342,
-      "eval_regularize": 0.6973706483840942,
-      "eval_runtime": 308.4071,
-      "eval_samples_per_second": 18.774,
-      "eval_steps_per_second": 1.566,
-      "eval_wo_beta": 8.238639831542969,
+      "eval_ranking_simple": 0.5621117949485779,
+      "eval_regularize": 0.6979546546936035,
+      "eval_runtime": 308.7789,
+      "eval_samples_per_second": 18.751,
+      "eval_steps_per_second": 1.564,
+      "eval_wo_beta": 8.258440017700195,
       "step": 450
     },
     {
-      "dpo_loss": 0.29817140102386475,
+      "dpo_loss": 0.29814377427101135,
       "epoch": 1.4170996693434104,
-      "grad_norm": 15.895661459470155,
+      "grad_norm": 15.647218082922008,
       "learning_rate": 4.501353102310901e-06,
-      "logits": -1.9325114488601685,
-      "logps": -91.53684997558594,
+      "logits": -1.9365119934082031,
+      "logps": -90.89854431152344,
       "loss": 0.3065,
-      "objective": 0.29817140102386475,
+      "objective": 0.29814377427101135,
       "ranking_idealized": 0.57833331823349,
       "ranking_idealized_expo": 0.4983333349227905,
-      "ranking_simple": 0.7975000143051147,
-      "regularize": 0.29817140102386475,
+      "ranking_simple": 0.7991666793823242,
+      "regularize": 0.29814377427101135,
       "step": 500,
-      "wo_beta": 3.1243510246276855
+      "wo_beta": 3.127906560897827
     },
     {
       "epoch": 1.4170996693434104,
-      "eval_dpo_loss": 0.7146824598312378,
-      "eval_logits": -1.9982556104660034,
-      "eval_logps": -97.01371002197266,
-      "eval_loss": 0.717850923538208,
-      "eval_objective": 0.7146824598312378,
+      "eval_dpo_loss": 0.7147387266159058,
+      "eval_logits": -2.006500005722046,
+      "eval_logps": -96.67472076416016,
+      "eval_loss": 0.7179672122001648,
+      "eval_objective": 0.7147387266159058,
       "eval_ranking_idealized": 0.6030020713806152,
       "eval_ranking_idealized_expo": 0.5222567319869995,
-      "eval_ranking_simple": 0.5548654198646545,
-      "eval_regularize": 0.7146824598312378,
-      "eval_runtime": 308.8392,
-      "eval_samples_per_second": 18.748,
-      "eval_steps_per_second": 1.564,
-      "eval_wo_beta": 8.27602767944336,
+      "eval_ranking_simple": 0.555383026599884,
+      "eval_regularize": 0.7147387266159058,
+      "eval_runtime": 309.3319,
+      "eval_samples_per_second": 18.718,
+      "eval_steps_per_second": 1.561,
+      "eval_wo_beta": 8.252218246459961,
       "step": 500
     },
     {
-      "dpo_loss": 0.2845906615257263,
+      "dpo_loss": 0.2861484885215759,
       "epoch": 1.5588096362777515,
-      "grad_norm": 10.66218965165015,
+      "grad_norm": 10.885590323378537,
       "learning_rate": 4.34319334202531e-06,
-      "logits": -1.9555292129516602,
-      "logps": -97.59473419189453,
-      "loss": 0.2885,
-      "objective": 0.2845906615257263,
+      "logits": -1.9361701011657715,
+      "logps": -95.84064483642578,
+      "loss": 0.2895,
+      "objective": 0.2861484885215759,
       "ranking_idealized": 0.5945833325386047,
       "ranking_idealized_expo": 0.5116666555404663,
-      "ranking_simple": 0.8041666746139526,
-      "regularize": 0.2845906615257263,
+      "ranking_simple": 0.8066666722297668,
+      "regularize": 0.2861484885215759,
       "step": 550,
-      "wo_beta": 2.6915340423583984
+      "wo_beta": 2.692445993423462
     },
     {
       "epoch": 1.5588096362777515,
-      "eval_dpo_loss": 0.7133627533912659,
-      "eval_logits": -1.9041162729263306,
-      "eval_logps": -107.96095275878906,
-      "eval_loss": 0.7091230750083923,
-      "eval_objective": 0.7133627533912659,
+      "eval_dpo_loss": 0.7076632976531982,
+      "eval_logits": -1.887025237083435,
+      "eval_logps": -104.24694061279297,
+      "eval_loss": 0.7044315338134766,
+      "eval_objective": 0.7076632976531982,
       "eval_ranking_idealized": 0.6030020713806152,
       "eval_ranking_idealized_expo": 0.5222567319869995,
-      "eval_ranking_simple": 0.5615941882133484,
-      "eval_regularize": 0.7133627533912659,
-      "eval_runtime": 308.0507,
-      "eval_samples_per_second": 18.796,
-      "eval_steps_per_second": 1.568,
-      "eval_wo_beta": 8.196797370910645,
+      "eval_ranking_simple": 0.5652173757553101,
+      "eval_regularize": 0.7076632976531982,
+      "eval_runtime": 309.8676,
+      "eval_samples_per_second": 18.685,
+      "eval_steps_per_second": 1.559,
+      "eval_wo_beta": 8.194681167602539,
       "step": 550
     },
     {
       "epoch": 1.5588096362777515,
       "step": 550,
       "total_flos": 0.0,
-      "train_loss": 0.4483427975394509,
-      "train_runtime": 15087.4424,
-      "train_samples_per_second": 16.836,
-      "train_steps_per_second": 0.117
+      "train_loss": 0.4484944924441251,
+      "train_runtime": 15200.6621,
+      "train_samples_per_second": 16.71,
+      "train_steps_per_second": 0.116
     }
   ],
   "logging_steps": 50,