Model save

README.md

@@ -0,0 +1,95 @@
+---
+license: apache-2.0
+base_model: hZzy/qwen2.5-0.5b-sft-news-IFT
+tags:
+- trl
+- expo
+- generated_from_trainer
+model-index:
+- name: qwen2.5-0.5b-expo-L2EXPO-ES-0.1
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/zhiyuzha-university-of-florida/huggingface/runs/z6ixm6bo)
+# qwen2.5-0.5b-expo-L2EXPO-ES-0.1
+
+This model is a fine-tuned version of [hZzy/qwen2.5-0.5b-sft-news-IFT](https://huggingface.co/hZzy/qwen2.5-0.5b-sft-news-IFT) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.4210
+- Logps: -88.9607
+- Logits: -1.3982
+- Objective: 0.4141
+- Dpo Loss: 0.6792
+- Regularize: 0.4141
+- Ranking Simple: 0.5336
+- Ranking Idealized: 0.6030
+- Ranking Idealized Expo: 0.5223
+- Wo Beta: 16.0444
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 1e-06
+- train_batch_size: 4
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 3
+- gradient_accumulation_steps: 12
+- total_train_batch_size: 144
+- total_eval_batch_size: 12
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 5
+
+### Training results
+
+| Training Loss | Epoch  | Step | Dpo Loss | Logits  | Logps    | Validation Loss | Objective | Ranking Idealized | Ranking Idealized Expo | Ranking Simple | Regularize | Wo Beta |
+|:-------------:|:------:|:----:|:--------:|:-------:|:--------:|:---------------:|:---------:|:-----------------:|:----------------------:|:--------------:|:----------:|:-------:|
+| 0.4117        | 0.1417 | 50   | 0.6893   | -1.4691 | -90.8535 | 0.4102          | 0.4090    | 0.6030            | 0.5223                 | 0.5248         | 0.4090     | 16.3208 |
+| 0.3871        | 0.2834 | 100  | 0.6833   | -1.5346 | -91.2757 | 0.4049          | 0.4029    | 0.6030            | 0.5223                 | 0.5316         | 0.4029     | 16.2699 |
+| 0.3451        | 0.4251 | 150  | 0.6789   | -1.4902 | -91.1637 | 0.4013          | 0.3996    | 0.6030            | 0.5223                 | 0.5347         | 0.3996     | 16.5907 |
+| 0.3166        | 0.5668 | 200  | 0.6811   | -1.4523 | -93.2695 | 0.4148          | 0.4132    | 0.6030            | 0.5223                 | 0.5316         | 0.4132     | 16.3512 |
+| 0.2939        | 0.7085 | 250  | 0.6790   | -1.5465 | -90.5537 | 0.4131          | 0.4077    | 0.6030            | 0.5223                 | 0.5342         | 0.4077     | 16.4807 |
+| 0.2655        | 0.8503 | 300  | 0.6806   | -1.4553 | -91.3521 | 0.4126          | 0.4082    | 0.6030            | 0.5223                 | 0.5311         | 0.4082     | 16.4429 |
+| 0.2513        | 0.9920 | 350  | 0.6782   | -1.4532 | -91.2408 | 0.4110          | 0.4044    | 0.6030            | 0.5223                 | 0.5352         | 0.4044     | 16.3768 |
+| 0.2206        | 1.1337 | 400  | 0.4128   | -87.3470| -1.4764  | 0.4049          | 0.6769    | 0.4049            | 0.5336                 | 0.6030         | 0.5223     | 16.2024 |
+| 0.2077        | 1.2754 | 450  | 0.4144   | -89.8793| -1.4177  | 0.4106          | 0.6788    | 0.4106            | 0.5331                 | 0.6030         | 0.5223     | 16.1977 |
+| 0.1943        | 1.4171 | 500  | 0.4169   | -87.6699| -1.4544  | 0.4092          | 0.6782    | 0.4092            | 0.5352                 | 0.6030         | 0.5223     | 16.0510 |
+| 0.1879        | 1.5588 | 550  | 0.4173   | -89.0111| -1.4268  | 0.4102          | 0.6787    | 0.4102            | 0.5347                 | 0.6030         | 0.5223     | 16.0707 |
+| 0.1768        | 1.7005 | 600  | 0.4190   | -87.0605| -1.4411  | 0.4116          | 0.6796    | 0.4116            | 0.5352                 | 0.6030         | 0.5223     | 16.0697 |
+| 0.1736        | 1.8422 | 650  | 0.4219   | -90.0508| -1.4601  | 0.4144          | 0.6802    | 0.4144            | 0.5347                 | 0.6030         | 0.5223     | 16.1057 |
+| 0.1598        | 1.9839 | 700  | 0.4217   | -90.5630| -1.4110  | 0.4148          | 0.6799    | 0.4148            | 0.5362                 | 0.6030         | 0.5223     | 16.0493 |
+| 0.1454        | 2.1256 | 750  | 0.4215   | -89.5433| -1.3859  | 0.4151          | 0.6797    | 0.4151            | 0.5316                 | 0.6030         | 0.5223     | 16.0459 |
+| 0.1333        | 2.2674 | 800  | 0.4217   | -89.1060| -1.3837  | 0.4142          | 0.6791    | 0.4142            | 0.5347                 | 0.6030         | 0.5223     | 15.9847 |
+| 0.1287        | 2.4091 | 850  | 0.4241   | -88.6145| -1.3856  | 0.4153          | 0.6795    | 0.4153            | 0.5357                 | 0.6030         | 0.5223     | 15.9979 |
+| 0.12          | 2.5508 | 900  | 0.4207   | -88.6663| -1.3921  | 0.4129          | 0.6795    | 0.4129            | 0.5331                 | 0.6030         | 0.5223     | 16.0698 |
+| 0.1148        | 2.6925 | 950  | 0.4215   | -88.2854| -1.3690  | 0.4149          | 0.6792    | 0.4149            | 0.5336                 | 0.6030         | 0.5223     | 16.0513 |
+| 0.1068        | 2.8342 | 1000 | 0.4229   | -89.1782| -1.3724  | 0.4168          | 0.6809    | 0.4168            | 0.5321                 | 0.6030         | 0.5223     | 16.0722 |
+| 0.0991        | 2.9759 | 1050 | 0.4210   | -88.9607| -1.3982  | 0.4141          | 0.6792    | 0.4141            | 0.5336                 | 0.6030         | 0.5223     | 16.0444 |
+
+
+### Framework versions
+
+- Transformers 4.42.0
+- Pytorch 2.3.0+cu121
+- Datasets 2.19.1
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 2.975909305621162,
+    "total_flos": 0.0,
+    "train_loss": 0.10327800432840983,
+    "train_runtime": 29791.2438,
+    "train_samples": 50802,
+    "train_samples_per_second": 8.526,
+    "train_steps_per_second": 0.059
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "bos_token_id": 151644,
+  "eos_token_id": 151645,
+  "max_new_tokens": 2048,
+  "pad_token_id": 151645,
+  "transformers_version": "4.42.0"
+}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7717a671d1b0a5fe4d74ea7e66c7e4bc6782cc89fcc66bfd303c165e10d410e5
+oid sha256:c9468ed3cf0bffef935dd97ce64fe9d76e42af26dc6b0beb0866f65a511f979d
 size 1975192208

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 2.975909305621162,
+    "total_flos": 0.0,
+    "train_loss": 0.10327800432840983,
+    "train_runtime": 29791.2438,
+    "train_samples": 50802,
+    "train_samples_per_second": 8.526,
+    "train_steps_per_second": 0.059
+}

trainer_state.json

@@ -0,0 +1,760 @@
+{
+  "best_metric": 15.984663963317871,
+  "best_model_checkpoint": "./qwen2.5-0.5b/qwen2.5-0.5b-expo-L2EXPO-ES-0.1/checkpoint-800",
+  "epoch": 2.975909305621162,
+  "eval_steps": 50,
+  "global_step": 1050,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "dpo_loss": 0.6931471824645996,
+      "epoch": 0.002834199338686821,
+      "grad_norm": 36.88506026977242,
+      "learning_rate": 5.681818181818181e-09,
+      "logits": -1.359458565711975,
+      "logps": -84.69721221923828,
+      "loss": 0.3913,
+      "objective": 0.3618059456348419,
+      "ranking_idealized": 0.6458333134651184,
+      "ranking_idealized_expo": 0.5833333134651184,
+      "ranking_simple": 0.5833333134651184,
+      "regularize": 0.3618059456348419,
+      "step": 1,
+      "wo_beta": 14.830931663513184
+    },
+    {
+      "dpo_loss": 0.689619779586792,
+      "epoch": 0.14170996693434104,
+      "grad_norm": 35.61374868059241,
+      "learning_rate": 2.840909090909091e-07,
+      "logits": -1.4646235704421997,
+      "logps": -83.92658996582031,
+      "loss": 0.4117,
+      "objective": 0.4160037934780121,
+      "ranking_idealized": 0.608418345451355,
+      "ranking_idealized_expo": 0.5229591727256775,
+      "ranking_simple": 0.5255101919174194,
+      "regularize": 0.4160037934780121,
+      "step": 50,
+      "wo_beta": 15.691085815429688
+    },
+    {
+      "epoch": 0.14170996693434104,
+      "eval_dpo_loss": 0.6893064975738525,
+      "eval_logits": -1.4691106081008911,
+      "eval_logps": -90.8535385131836,
+      "eval_loss": 0.4102482497692108,
+      "eval_objective": 0.4089997410774231,
+      "eval_ranking_idealized": 0.6030020713806152,
+      "eval_ranking_idealized_expo": 0.5222567319869995,
+      "eval_ranking_simple": 0.5248447060585022,
+      "eval_regularize": 0.4089997410774231,
+      "eval_runtime": 309.1952,
+      "eval_samples_per_second": 18.726,
+      "eval_steps_per_second": 1.562,
+      "eval_wo_beta": 16.32078742980957,
+      "step": 50
+    },
+    {
+      "dpo_loss": 0.666653573513031,
+      "epoch": 0.2834199338686821,
+      "grad_norm": 39.001717769013226,
+      "learning_rate": 5.681818181818182e-07,
+      "logits": -1.5049233436584473,
+      "logps": -84.32721710205078,
+      "loss": 0.3871,
+      "objective": 0.3919191360473633,
+      "ranking_idealized": 0.6016666889190674,
+      "ranking_idealized_expo": 0.5141666531562805,
+      "ranking_simple": 0.5274999737739563,
+      "regularize": 0.3919191360473633,
+      "step": 100,
+      "wo_beta": 15.436232566833496
+    },
+    {
+      "epoch": 0.2834199338686821,
+      "eval_dpo_loss": 0.6833388805389404,
+      "eval_logits": -1.5346405506134033,
+      "eval_logps": -91.27565002441406,
+      "eval_loss": 0.4049375057220459,
+      "eval_objective": 0.40290772914886475,
+      "eval_ranking_idealized": 0.6030020713806152,
+      "eval_ranking_idealized_expo": 0.5222567319869995,
+      "eval_ranking_simple": 0.531573474407196,
+      "eval_regularize": 0.40290772914886475,
+      "eval_runtime": 308.7302,
+      "eval_samples_per_second": 18.754,
+      "eval_steps_per_second": 1.564,
+      "eval_wo_beta": 16.269933700561523,
+      "step": 100
+    },
+    {
+      "dpo_loss": 0.6460586190223694,
+      "epoch": 0.42512990080302315,
+      "grad_norm": 33.54647406338888,
+      "learning_rate": 8.522727272727273e-07,
+      "logits": -1.5278972387313843,
+      "logps": -86.29918670654297,
+      "loss": 0.3451,
+      "objective": 0.3537640869617462,
+      "ranking_idealized": 0.6066666841506958,
+      "ranking_idealized_expo": 0.5287500023841858,
+      "ranking_simple": 0.5441666841506958,
+      "regularize": 0.3537640869617462,
+      "step": 150,
+      "wo_beta": 15.559906005859375
+    },
+    {
+      "epoch": 0.42512990080302315,
+      "eval_dpo_loss": 0.6788700819015503,
+      "eval_logits": -1.4902477264404297,
+      "eval_logps": -91.16365051269531,
+      "eval_loss": 0.40134957432746887,
+      "eval_objective": 0.39963677525520325,
+      "eval_ranking_idealized": 0.6030020713806152,
+      "eval_ranking_idealized_expo": 0.5222567319869995,
+      "eval_ranking_simple": 0.534679114818573,
+      "eval_regularize": 0.39963677525520325,
+      "eval_runtime": 313.2125,
+      "eval_samples_per_second": 18.486,
+      "eval_steps_per_second": 1.542,
+      "eval_wo_beta": 16.59067726135254,
+      "step": 150
+    },
+    {
+      "dpo_loss": 0.616335928440094,
+      "epoch": 0.5668398677373642,
+      "grad_norm": 40.07384713581775,
+      "learning_rate": 9.99433669591504e-07,
+      "logits": -1.5112419128417969,
+      "logps": -85.97952270507812,
+      "loss": 0.3166,
+      "objective": 0.31608816981315613,
+      "ranking_idealized": 0.5924999713897705,
+      "ranking_idealized_expo": 0.5166666507720947,
+      "ranking_simple": 0.5454166531562805,
+      "regularize": 0.31608816981315613,
+      "step": 200,
+      "wo_beta": 15.945281982421875
+    },
+    {
+      "epoch": 0.5668398677373642,
+      "eval_dpo_loss": 0.681080162525177,
+      "eval_logits": -1.4522831439971924,
+      "eval_logps": -93.26953887939453,
+      "eval_loss": 0.4148460626602173,
+      "eval_objective": 0.41316577792167664,
+      "eval_ranking_idealized": 0.6030020713806152,
+      "eval_ranking_idealized_expo": 0.5222567319869995,
+      "eval_ranking_simple": 0.531573474407196,
+      "eval_regularize": 0.41316577792167664,
+      "eval_runtime": 308.9819,
+      "eval_samples_per_second": 18.739,
+      "eval_steps_per_second": 1.563,
+      "eval_wo_beta": 16.35118865966797,
+      "step": 200
+    },
+    {
+      "dpo_loss": 0.5952258110046387,
+      "epoch": 0.7085498346717053,
+      "grad_norm": 34.54234310666894,
+      "learning_rate": 9.94624571028813e-07,
+      "logits": -1.5106024742126465,
+      "logps": -84.41282653808594,
+      "loss": 0.2939,
+      "objective": 0.2922481894493103,
+      "ranking_idealized": 0.5991666913032532,
+      "ranking_idealized_expo": 0.5170833468437195,
+      "ranking_simple": 0.5612499713897705,
+      "regularize": 0.2922481894493103,
+      "step": 250,
+      "wo_beta": 16.29503631591797
+    },
+    {
+      "epoch": 0.7085498346717053,
+      "eval_dpo_loss": 0.679045557975769,
+      "eval_logits": -1.5465130805969238,
+      "eval_logps": -90.55366516113281,
+      "eval_loss": 0.4130644202232361,
+      "eval_objective": 0.4076613783836365,
+      "eval_ranking_idealized": 0.6030020713806152,
+      "eval_ranking_idealized_expo": 0.5222567319869995,
+      "eval_ranking_simple": 0.5341615080833435,
+      "eval_regularize": 0.4076613783836365,
+      "eval_runtime": 308.7233,
+      "eval_samples_per_second": 18.755,
+      "eval_steps_per_second": 1.565,
+      "eval_wo_beta": 16.480682373046875,
+      "step": 250
+    },
+    {
+      "dpo_loss": 0.586772620677948,
+      "epoch": 0.8502598016060463,
+      "grad_norm": 34.42071615370769,
+      "learning_rate": 9.849553282839024e-07,
+      "logits": -1.5501227378845215,
+      "logps": -85.52662658691406,
+      "loss": 0.2655,
+      "objective": 0.27248048782348633,
+      "ranking_idealized": 0.5799999833106995,
+      "ranking_idealized_expo": 0.4970833361148834,
+      "ranking_simple": 0.5445833206176758,
+      "regularize": 0.27248048782348633,
+      "step": 300,
+      "wo_beta": 15.878409385681152
+    },
+    {
+      "epoch": 0.8502598016060463,
+      "eval_dpo_loss": 0.6806344985961914,
+      "eval_logits": -1.4552550315856934,
+      "eval_logps": -91.35205078125,
+      "eval_loss": 0.4126306176185608,
+      "eval_objective": 0.4082447588443756,
+      "eval_ranking_idealized": 0.6030020713806152,
+      "eval_ranking_idealized_expo": 0.5222567319869995,
+      "eval_ranking_simple": 0.5310559272766113,
+      "eval_regularize": 0.4082447588443756,
+      "eval_runtime": 467.8777,
+      "eval_samples_per_second": 12.375,
+      "eval_steps_per_second": 1.032,
+      "eval_wo_beta": 16.44291877746582,
+      "step": 300
+    },
+    {
+      "dpo_loss": 0.5711151361465454,
+      "epoch": 0.9919697685403873,
+      "grad_norm": 31.544141880018735,
+      "learning_rate": 9.705209506155634e-07,
+      "logits": -1.468552589416504,
+      "logps": -85.4828872680664,
+      "loss": 0.2513,
+      "objective": 0.24532364308834076,
+      "ranking_idealized": 0.60916668176651,
+      "ranking_idealized_expo": 0.5270833373069763,
+      "ranking_simple": 0.5920833349227905,
+      "regularize": 0.24532364308834076,
+      "step": 350,
+      "wo_beta": 15.914597511291504
+    },
+    {
+      "epoch": 0.9919697685403873,
+      "eval_dpo_loss": 0.6781877279281616,
+      "eval_logits": -1.4532182216644287,
+      "eval_logps": -91.24078369140625,
+      "eval_loss": 0.41097965836524963,
+      "eval_objective": 0.4043864607810974,
+      "eval_ranking_idealized": 0.6030020713806152,
+      "eval_ranking_idealized_expo": 0.5222567319869995,
+      "eval_ranking_simple": 0.5351966619491577,
+      "eval_regularize": 0.4043864607810974,
+      "eval_runtime": 471.3464,
+      "eval_samples_per_second": 12.284,
+      "eval_steps_per_second": 1.025,
+      "eval_wo_beta": 16.376806259155273,
+      "step": 350
+    },
+    {
+      "dpo_loss": 0.5529460906982422,
+      "epoch": 1.1336797354747283,
+      "grad_norm": 30.9851919334132,
+      "learning_rate": 9.514632691433106e-07,
+      "logits": -1.4837555885314941,
+      "logps": -83.03319549560547,
+      "loss": 0.2206,
+      "objective": 0.2209722399711609,
+      "ranking_idealized": 0.6087499856948853,
+      "ranking_idealized_expo": 0.5337499976158142,
+      "ranking_simple": 0.5941666960716248,
+      "regularize": 0.2209722399711609,
+      "step": 400,
+      "wo_beta": 16.069154739379883
+    },
+    {
+      "epoch": 1.1336797354747283,
+      "eval_dpo_loss": 0.6769281625747681,
+      "eval_logits": -1.4763766527175903,
+      "eval_logps": -87.34698486328125,
+      "eval_loss": 0.41283363103866577,
+      "eval_objective": 0.4048711359500885,
+      "eval_ranking_idealized": 0.6030020713806152,
+      "eval_ranking_idealized_expo": 0.5222567319869995,
+      "eval_ranking_simple": 0.533643901348114,
+      "eval_regularize": 0.4048711359500885,
+      "eval_runtime": 481.9587,
+      "eval_samples_per_second": 12.013,
+      "eval_steps_per_second": 1.002,
+      "eval_wo_beta": 16.20241355895996,
+      "step": 400
+    },
+    {
+      "dpo_loss": 0.5509263277053833,
+      "epoch": 1.2753897024090695,
+      "grad_norm": 33.28599031549051,
+      "learning_rate": 9.279695432253708e-07,
+      "logits": -1.4624412059783936,
+      "logps": -82.20590209960938,
+      "loss": 0.2077,
+      "objective": 0.2058807760477066,
+      "ranking_idealized": 0.5975000262260437,
+      "ranking_idealized_expo": 0.5199999809265137,
+      "ranking_simple": 0.5841666460037231,
+      "regularize": 0.2058807760477066,
+      "step": 450,
+      "wo_beta": 16.36168098449707
+    },
+    {
+      "epoch": 1.2753897024090695,
+      "eval_dpo_loss": 0.6788477897644043,
+      "eval_logits": -1.417741060256958,
+      "eval_logps": -89.87925720214844,
+      "eval_loss": 0.41441377997398376,
+      "eval_objective": 0.4105576276779175,
+      "eval_ranking_idealized": 0.6030020713806152,
+      "eval_ranking_idealized_expo": 0.5222567319869995,
+      "eval_ranking_simple": 0.5331262946128845,
+      "eval_regularize": 0.4105576276779175,
+      "eval_runtime": 479.889,
+      "eval_samples_per_second": 12.065,
+      "eval_steps_per_second": 1.006,
+      "eval_wo_beta": 16.197683334350586,
+      "step": 450
+    },
+    {
+      "dpo_loss": 0.5497527718544006,
+      "epoch": 1.4170996693434104,
+      "grad_norm": 28.875129958259482,
+      "learning_rate": 9.002706204621802e-07,
+      "logits": -1.4527512788772583,
+      "logps": -82.58905029296875,
+      "loss": 0.1943,
+      "objective": 0.19459716975688934,
+      "ranking_idealized": 0.57833331823349,
+      "ranking_idealized_expo": 0.4983333349227905,
+      "ranking_simple": 0.5625,
+      "regularize": 0.19459716975688934,
+      "step": 500,
+      "wo_beta": 15.90869426727295
+    },
+    {
+      "epoch": 1.4170996693434104,
+      "eval_dpo_loss": 0.6781710982322693,
+      "eval_logits": -1.454372763633728,
+      "eval_logps": -87.66993713378906,
+      "eval_loss": 0.4168953001499176,
+      "eval_objective": 0.40915447473526,
+      "eval_ranking_idealized": 0.6030020713806152,
+      "eval_ranking_idealized_expo": 0.5222567319869995,
+      "eval_ranking_simple": 0.5351966619491577,
+      "eval_regularize": 0.40915447473526,
+      "eval_runtime": 488.9325,
+      "eval_samples_per_second": 11.842,
+      "eval_steps_per_second": 0.988,
+      "eval_wo_beta": 16.050954818725586,
+      "step": 500
+    },
+    {
+      "dpo_loss": 0.5426554083824158,
+      "epoch": 1.5588096362777515,
+      "grad_norm": 30.318446040470374,
+      "learning_rate": 8.68638668405062e-07,
+      "logits": -1.4643546342849731,
+      "logps": -82.28606414794922,
+      "loss": 0.1879,
+      "objective": 0.1849183589220047,
+      "ranking_idealized": 0.5945833325386047,
+      "ranking_idealized_expo": 0.5116666555404663,
+      "ranking_simple": 0.5854166746139526,
+      "regularize": 0.1849183589220047,
+      "step": 550,
+      "wo_beta": 15.804949760437012
+    },
+    {
+      "epoch": 1.5588096362777515,
+      "eval_dpo_loss": 0.6786677241325378,
+      "eval_logits": -1.4267942905426025,
+      "eval_logps": -89.01109313964844,
+      "eval_loss": 0.4173260033130646,
+      "eval_objective": 0.41023388504981995,
+      "eval_ranking_idealized": 0.6030020713806152,
+      "eval_ranking_idealized_expo": 0.5222567319869995,
+      "eval_ranking_simple": 0.534679114818573,
+      "eval_regularize": 0.41023388504981995,
+      "eval_runtime": 497.8976,
+      "eval_samples_per_second": 11.629,
+      "eval_steps_per_second": 0.97,
+      "eval_wo_beta": 16.070680618286133,
+      "step": 550
+    },
+    {
+      "dpo_loss": 0.5401077270507812,
+      "epoch": 1.7005196032120926,
+      "grad_norm": 28.092335905186818,
+      "learning_rate": 8.333845002581458e-07,
+      "logits": -1.4184807538986206,
+      "logps": -81.81605529785156,
+      "loss": 0.1768,
+      "objective": 0.1826876848936081,
+      "ranking_idealized": 0.6004166603088379,
+      "ranking_idealized_expo": 0.51583331823349,
+      "ranking_simple": 0.5870833396911621,
+      "regularize": 0.1826876848936081,
+      "step": 600,
+      "wo_beta": 15.617234230041504
+    },
+    {
+      "epoch": 1.7005196032120926,
+      "eval_dpo_loss": 0.6796310544013977,
+      "eval_logits": -1.4410563707351685,
+      "eval_logps": -87.06050872802734,
+      "eval_loss": 0.4189995527267456,
+      "eval_objective": 0.411603182554245,
+      "eval_ranking_idealized": 0.6030020713806152,
+      "eval_ranking_idealized_expo": 0.5222567319869995,
+      "eval_ranking_simple": 0.5351966619491577,
+      "eval_regularize": 0.411603182554245,
+      "eval_runtime": 490.295,
+      "eval_samples_per_second": 11.809,
+      "eval_steps_per_second": 0.985,
+      "eval_wo_beta": 16.06967544555664,
+      "step": 600
+    },
+    {
+      "dpo_loss": 0.5417632460594177,
+      "epoch": 1.8422295701464337,
+      "grad_norm": 30.564010457091317,
+      "learning_rate": 7.948545208509811e-07,
+      "logits": -1.4855570793151855,
+      "logps": -82.33342742919922,
+      "loss": 0.1736,
+      "objective": 0.16996659338474274,
+      "ranking_idealized": 0.6058333516120911,
+      "ranking_idealized_expo": 0.5295833349227905,
+      "ranking_simple": 0.5916666388511658,
+      "regularize": 0.16996659338474274,
+      "step": 650,
+      "wo_beta": 16.523035049438477
+    },
+    {
+      "epoch": 1.8422295701464337,
+      "eval_dpo_loss": 0.680221676826477,
+      "eval_logits": -1.4600505828857422,
+      "eval_logps": -90.0507583618164,
+      "eval_loss": 0.4218791425228119,
+      "eval_objective": 0.4143694341182709,
+      "eval_ranking_idealized": 0.6030020713806152,
+      "eval_ranking_idealized_expo": 0.5222567319869995,
+      "eval_ranking_simple": 0.534679114818573,
+      "eval_regularize": 0.4143694341182709,
+      "eval_runtime": 465.9154,
+      "eval_samples_per_second": 12.427,
+      "eval_steps_per_second": 1.037,
+      "eval_wo_beta": 16.10569953918457,
+      "step": 650
+    },
+    {
+      "dpo_loss": 0.5461080074310303,
+      "epoch": 1.9839395370807746,
+      "grad_norm": 29.849998371301844,
+      "learning_rate": 7.534273228904915e-07,
+      "logits": -1.4225118160247803,
+      "logps": -84.2075424194336,
+      "loss": 0.1598,
+      "objective": 0.16184575855731964,
+      "ranking_idealized": 0.5954166650772095,
+      "ranking_idealized_expo": 0.5129166841506958,
+      "ranking_simple": 0.5870833396911621,
+      "regularize": 0.16184575855731964,
+      "step": 700,
+      "wo_beta": 15.974614143371582
+    },
+    {
+      "epoch": 1.9839395370807746,
+      "eval_dpo_loss": 0.6798617243766785,
+      "eval_logits": -1.411017656326294,
+      "eval_logps": -90.56302642822266,
+      "eval_loss": 0.4216688573360443,
+      "eval_objective": 0.41477063298225403,
+      "eval_ranking_idealized": 0.6030020713806152,
+      "eval_ranking_idealized_expo": 0.5222567319869995,
+      "eval_ranking_simple": 0.5362318754196167,
+      "eval_regularize": 0.41477063298225403,
+      "eval_runtime": 468.8368,
+      "eval_samples_per_second": 12.35,
+      "eval_steps_per_second": 1.03,
+      "eval_wo_beta": 16.049283981323242,
+      "step": 700
+    },
+    {
+      "dpo_loss": 0.5335285663604736,
+      "epoch": 2.1256495040151155,
+      "grad_norm": 27.349376686984996,
+      "learning_rate": 7.095099669372443e-07,
+      "logits": -1.4041997194290161,
+      "logps": -83.5035400390625,
+      "loss": 0.1454,
+      "objective": 0.14657247066497803,
+      "ranking_idealized": 0.6066666841506958,
+      "ranking_idealized_expo": 0.5133333206176758,
+      "ranking_simple": 0.596666693687439,
+      "regularize": 0.14657247066497803,
+      "step": 750,
+      "wo_beta": 16.065366744995117
+    },
+    {
+      "epoch": 2.1256495040151155,
+      "eval_dpo_loss": 0.6797157526016235,
+      "eval_logits": -1.3859150409698486,
+      "eval_logps": -89.54334259033203,
+      "eval_loss": 0.4214833676815033,
+      "eval_objective": 0.41514915227890015,
+      "eval_ranking_idealized": 0.6030020713806152,
+      "eval_ranking_idealized_expo": 0.5222567319869995,
+      "eval_ranking_simple": 0.531573474407196,
+      "eval_regularize": 0.41514915227890015,
+      "eval_runtime": 480.0022,
+      "eval_samples_per_second": 12.062,
+      "eval_steps_per_second": 1.006,
+      "eval_wo_beta": 16.04585075378418,
+      "step": 750
+    },
+    {
+      "dpo_loss": 0.5314897894859314,
+      "epoch": 2.2673594709494567,
+      "grad_norm": 28.42084450535516,
+      "learning_rate": 6.635339816587108e-07,
+      "logits": -1.3851786851882935,
+      "logps": -83.51881408691406,
+      "loss": 0.1333,
+      "objective": 0.13326440751552582,
+      "ranking_idealized": 0.5941666960716248,
+      "ranking_idealized_expo": 0.5129166841506958,
+      "ranking_simple": 0.5874999761581421,
+      "regularize": 0.13326440751552582,
+      "step": 800,
+      "wo_beta": 15.551568984985352
+    },
+    {
+      "epoch": 2.2673594709494567,
+      "eval_dpo_loss": 0.6790949106216431,
+      "eval_logits": -1.383711576461792,
+      "eval_logps": -89.10604858398438,
+      "eval_loss": 0.4217349588871002,
+      "eval_objective": 0.4141845107078552,
+      "eval_ranking_idealized": 0.6030020713806152,
+      "eval_ranking_idealized_expo": 0.5222567319869995,
+      "eval_ranking_simple": 0.534679114818573,
+      "eval_regularize": 0.4141845107078552,
+      "eval_runtime": 479.8627,
+      "eval_samples_per_second": 12.066,
+      "eval_steps_per_second": 1.007,
+      "eval_wo_beta": 15.984663963317871,
+      "step": 800
+    },
+    {
+      "dpo_loss": 0.5344111919403076,
+      "epoch": 2.409069437883798,
+      "grad_norm": 27.376346461788764,
+      "learning_rate": 6.159511236607315e-07,
+      "logits": -1.4081922769546509,
+      "logps": -83.36618041992188,
+      "loss": 0.1287,
+      "objective": 0.12692613899707794,
+      "ranking_idealized": 0.5979166626930237,
+      "ranking_idealized_expo": 0.5166666507720947,
+      "ranking_simple": 0.5933333039283752,
+      "regularize": 0.12692613899707794,
+      "step": 850,
+      "wo_beta": 15.760154724121094
+    },
+    {
+      "epoch": 2.409069437883798,
+      "eval_dpo_loss": 0.6794790625572205,
+      "eval_logits": -1.3856061697006226,
+      "eval_logps": -88.614501953125,
+      "eval_loss": 0.4240662753582001,
+      "eval_objective": 0.4153030812740326,
+      "eval_ranking_idealized": 0.6030020713806152,
+      "eval_ranking_idealized_expo": 0.5222567319869995,
+      "eval_ranking_simple": 0.5357142686843872,
+      "eval_regularize": 0.4153030812740326,
+      "eval_runtime": 481.1385,
+      "eval_samples_per_second": 12.034,
+      "eval_steps_per_second": 1.004,
+      "eval_wo_beta": 15.997852325439453,
+      "step": 850
+    },
+    {
+      "dpo_loss": 0.5347414016723633,
+      "epoch": 2.550779404818139,
+      "grad_norm": 29.1093651939844,
+      "learning_rate": 5.67228938560766e-07,
+      "logits": -1.3715587854385376,
+      "logps": -82.14668273925781,
+      "loss": 0.12,
+      "objective": 0.12150037288665771,
+      "ranking_idealized": 0.5887500047683716,
+      "ranking_idealized_expo": 0.518750011920929,
+      "ranking_simple": 0.5891666412353516,
+      "regularize": 0.12150037288665771,
+      "step": 900,
+      "wo_beta": 16.280019760131836
+    },
+    {
+      "epoch": 2.550779404818139,
+      "eval_dpo_loss": 0.6795074343681335,
+      "eval_logits": -1.3921464681625366,
+      "eval_logps": -88.66633605957031,
+      "eval_loss": 0.4207339584827423,
+      "eval_objective": 0.4128892719745636,
+      "eval_ranking_idealized": 0.6030020713806152,
+      "eval_ranking_idealized_expo": 0.5222567319869995,
+      "eval_ranking_simple": 0.5331262946128845,
+      "eval_regularize": 0.4128892719745636,
+      "eval_runtime": 493.2688,
+      "eval_samples_per_second": 11.738,
+      "eval_steps_per_second": 0.979,
+      "eval_wo_beta": 16.06983184814453,
+      "step": 900
+    },
+    {
+      "dpo_loss": 0.5318446159362793,
+      "epoch": 2.69248937175248,
+      "grad_norm": 28.62178309135148,
+      "learning_rate": 5.178461669194903e-07,
+      "logits": -1.3740975856781006,
+      "logps": -82.18826293945312,
+      "loss": 0.1148,
+      "objective": 0.11303483694791794,
+      "ranking_idealized": 0.5975000262260437,
+      "ranking_idealized_expo": 0.5087500214576721,
+      "ranking_simple": 0.5929166674613953,
+      "regularize": 0.11303483694791794,
+      "step": 950,
+      "wo_beta": 16.24332618713379
+    },
+    {
+      "epoch": 2.69248937175248,
+      "eval_dpo_loss": 0.6791985034942627,
+      "eval_logits": -1.3689535856246948,
+      "eval_logps": -88.28536224365234,
+      "eval_loss": 0.421545147895813,
+      "eval_objective": 0.41494688391685486,
+      "eval_ranking_idealized": 0.6030020713806152,
+      "eval_ranking_idealized_expo": 0.5222567319869995,
+      "eval_ranking_simple": 0.533643901348114,
+      "eval_regularize": 0.41494688391685486,
+      "eval_runtime": 492.3545,
+      "eval_samples_per_second": 11.76,
+      "eval_steps_per_second": 0.981,
+      "eval_wo_beta": 16.05128288269043,
+      "step": 950
+    },
+    {
+      "dpo_loss": 0.5294139385223389,
+      "epoch": 2.8341993386868207,
+      "grad_norm": 27.974854830973783,
+      "learning_rate": 4.682880401717177e-07,
+      "logits": -1.3566088676452637,
+      "logps": -81.84488677978516,
+      "loss": 0.1068,
+      "objective": 0.109143927693367,
+      "ranking_idealized": 0.6020833253860474,
+      "ranking_idealized_expo": 0.5104166865348816,
+      "ranking_simple": 0.5975000262260437,
+      "regularize": 0.109143927693367,
+      "step": 1000,
+      "wo_beta": 15.627644538879395
+    },
+    {
+      "epoch": 2.8341993386868207,
+      "eval_dpo_loss": 0.6808813214302063,
+      "eval_logits": -1.3724371194839478,
+      "eval_logps": -89.17817687988281,
+      "eval_loss": 0.42285358905792236,
+      "eval_objective": 0.41681256890296936,
+      "eval_ranking_idealized": 0.6030020713806152,
+      "eval_ranking_idealized_expo": 0.5222567319869995,
+      "eval_ranking_simple": 0.5320910811424255,
+      "eval_regularize": 0.41681256890296936,
+      "eval_runtime": 490.5576,
+      "eval_samples_per_second": 11.803,
+      "eval_steps_per_second": 0.985,
+      "eval_wo_beta": 16.072154998779297,
+      "step": 1000
+    },
+    {
+      "dpo_loss": 0.5339093208312988,
+      "epoch": 2.975909305621162,
+      "grad_norm": 27.555604149776222,
+      "learning_rate": 4.1904151277847305e-07,
+      "logits": -1.3810933828353882,
+      "logps": -82.09051513671875,
+      "loss": 0.0991,
+      "objective": 0.09846866875886917,
+      "ranking_idealized": 0.6075000166893005,
+      "ranking_idealized_expo": 0.5179166793823242,
+      "ranking_simple": 0.5975000262260437,
+      "regularize": 0.09846866875886917,
+      "step": 1050,
+      "wo_beta": 16.260292053222656
+    },
+    {
+      "epoch": 2.975909305621162,
+      "eval_dpo_loss": 0.6791900992393494,
+      "eval_logits": -1.398189663887024,
+      "eval_logps": -88.96070098876953,
+      "eval_loss": 0.421024352312088,
+      "eval_objective": 0.4140646755695343,
+      "eval_ranking_idealized": 0.6030020713806152,
+      "eval_ranking_idealized_expo": 0.5222567319869995,
+      "eval_ranking_simple": 0.533643901348114,
+      "eval_regularize": 0.4140646755695343,
+      "eval_runtime": 468.6466,
+      "eval_samples_per_second": 12.355,
+      "eval_steps_per_second": 1.031,
+      "eval_wo_beta": 16.04439353942871,
+      "step": 1050
+    },
+    {
+      "epoch": 2.975909305621162,
+      "step": 1050,
+      "total_flos": 0.0,
+      "train_loss": 0.10327800432840983,
+      "train_runtime": 29791.2438,
+      "train_samples_per_second": 8.526,
+      "train_steps_per_second": 0.059
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 1760,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 50,
+  "stateful_callbacks": {
+    "EarlyStoppingCallback": {
+      "args": {
+        "early_stopping_patience": 5,
+        "early_stopping_threshold": 0.0
+      },
+      "attributes": {
+        "early_stopping_patience_counter": 0
+      }
+    },
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}