Model save

README.md

@@ -2,16 +2,11 @@
 license: apache-2.0
 base_model: hZzy/qwen2.5-0.5b-sft-news-IFT
 tags:
-- alignment-handbook
-- ndcg
-- trl
-- expo
-- generated_from_trainer
 - trl
 - expo
+- alignment-handbook
+- ndcg
 - generated_from_trainer
-datasets:
-- hZzy/train_pairwise
 model-index:
 - name: qwen2.5-0.5b-expo-DPO-ES-0.01
   results: []
@@ -20,21 +15,21 @@ model-index:
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 should probably proofread and complete it, then remove this comment. -->
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/zhiyuzha-university-of-florida/huggingface/runs/aeakfz58)
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/zhiyuzha-university-of-florida/huggingface/runs/r1jti62c)
 # qwen2.5-0.5b-expo-DPO-ES-0.01
 
-This model is a fine-tuned version of [hZzy/qwen2.5-0.5b-sft-news-IFT](https://huggingface.co/hZzy/qwen2.5-0.5b-sft-news-IFT) on the hZzy/train_pairwise dataset.
+This model is a fine-tuned version of [hZzy/qwen2.5-0.5b-sft-news-IFT](https://huggingface.co/hZzy/qwen2.5-0.5b-sft-news-IFT) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6894
-- Logps: -99.3821
-- Logits: -1.7611
-- Objective: 0.6902
-- Dpo Loss: 0.6902
-- Regularize: 0.6902
-- Ranking Simple: 0.5305
+- Loss: 0.6901
+- Logps: -349.5348
+- Logits: -5.2492
+- Objective: 0.6972
+- Dpo Loss: 0.6972
+- Regularize: 0.6972
+- Ranking Simple: 0.5719
 - Ranking Idealized: 0.8732
 - Ranking Idealized Expo: 0.5321
-- Wo Beta: 9.3575
+- Wo Beta: 31.1765
 
 ## Model description
 
@@ -69,14 +64,19 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch  | Step | Validation Loss | Logps     | Logits  | Objective | Dpo Loss | Regularize | Ranking Simple | Ranking Idealized | Ranking Idealized Expo | Wo Beta |
-|:-------------:|:------:|:----:|:---------------:|:---------:|:-------:|:---------:|:--------:|:----------:|:--------------:|:-----------------:|:----------------------:|:-------:|
-| 0.6907        | 0.1417 | 50   | 0.6894          | -99.3821  | -1.7611 | 0.6902    | 0.6902   | 0.6902     | 0.5305         | 0.8732            | 0.5321                 | 9.3575  |
-| 0.6701        | 0.2834 | 100  | 0.6837          | -153.5716 | -1.8467 | 0.6896    | 0.6896   | 0.6896     | 0.5518         | 0.8732            | 0.5321                 | 14.1741 |
-| 0.637         | 0.4251 | 150  | 0.6723          | -198.9269 | -2.4614 | 0.6765    | 0.6765   | 0.6765     | 0.5823         | 0.8732            | 0.5321                 | 17.2098 |
-| 0.5833        | 0.5668 | 200  | 0.6729          | -256.0312 | -3.3478 | 0.6780    | 0.6780   | 0.6780     | 0.5797         | 0.8732            | 0.5321                 | 21.7109 |
-| 0.5439        | 0.7085 | 250  | 0.6781          | -257.3546 | -3.6269 | 0.6858    | 0.6858   | 0.6858     | 0.5683         | 0.8732            | 0.5321                 | 22.9139 |
-| 0.5077        | 0.8503 | 300  | 0.6640          | -319.3935 | -4.6100 | 0.6685    | 0.6685   | 0.6685     | 0.5828         | 0.8732            | 0.5321                 | 23.6506 |
+| Training Loss | Epoch  | Step | Dpo Loss | Logits  | Logps     | Validation Loss | Objective | Ranking Idealized | Ranking Idealized Expo | Ranking Simple | Regularize | Wo Beta |
+|:-------------:|:------:|:----:|:--------:|:-------:|:---------:|:---------------:|:---------:|:-----------------:|:----------------------:|:--------------:|:----------:|:-------:|
+| 0.6907        | 0.1417 | 50   | 0.6902   | -1.7611 | -99.3821  | 0.6894          | 0.6902    | 0.8732            | 0.5321                 | 0.5305         | 0.6902     | 9.3575  |
+| 0.6701        | 0.2834 | 100  | 0.6896   | -1.8467 | -153.5716 | 0.6837          | 0.6896    | 0.8732            | 0.5321                 | 0.5518         | 0.6896     | 14.1741 |
+| 0.637         | 0.4251 | 150  | 0.6765   | -2.4614 | -198.9269 | 0.6723          | 0.6765    | 0.8732            | 0.5321                 | 0.5823         | 0.6765     | 17.2098 |
+| 0.5833        | 0.5668 | 200  | 0.6780   | -3.3478 | -256.0312 | 0.6729          | 0.6780    | 0.8732            | 0.5321                 | 0.5797         | 0.6780     | 21.7109 |
+| 0.5439        | 0.7085 | 250  | 0.6858   | -3.6269 | -257.3546 | 0.6781          | 0.6858    | 0.8732            | 0.5321                 | 0.5683         | 0.6858     | 22.9139 |
+| 0.5077        | 0.8503 | 300  | 0.6685   | -4.6100 | -319.3935 | 0.6640          | 0.6685    | 0.8732            | 0.5321                 | 0.5828         | 0.6685     | 23.6506 |
+| 0.4786        | 0.9920 | 350  | 0.6867   | -368.7014| -4.9192   | 0.6897          | 0.6897    | 0.6897            | 0.5751                 | 0.8732         | 0.5321     | 27.7113 |
+| 0.3619        | 1.1337 | 400  | 0.6961   | -392.7018| -5.5801   | 0.6990          | 0.6990    | 0.6990            | 0.5849                 | 0.8732         | 0.5321     | 32.0730 |
+| 0.3679        | 1.2754 | 450  | 0.6843   | -349.2029| -5.2450   | 0.6953          | 0.6953    | 0.6953            | 0.5885                 | 0.8732         | 0.5321     | 31.3199 |
+| 0.3662        | 1.4171 | 500  | 0.6858   | -350.7137| -5.8233   | 0.6903          | 0.6903    | 0.6903            | 0.5890                 | 0.8732         | 0.5321     | 30.2726 |
+| 0.3485        | 1.5588 | 550  | 0.6901   | -349.5348| -5.2492   | 0.6972          | 0.6972    | 0.6972            | 0.5719                 | 0.8732         | 0.5321     | 31.1765 |
 
 
 ### Framework versions

all_results.json

@@ -1,5 +1,5 @@
 {
-    "epoch": 0.8502598016060463,
+    "epoch": 1.5588096362777515,
     "eval_dpo_loss": 0.6902037858963013,
     "eval_logits": -1.7611440420150757,
     "eval_logps": -99.38214111328125,
@@ -15,9 +15,9 @@
     "eval_steps_per_second": 1.567,
     "eval_wo_beta": 9.357504844665527,
     "total_flos": 0.0,
-    "train_loss": 0.6054576412836711,
-    "train_runtime": 8287.5068,
+    "train_loss": 0.17483963706276634,
+    "train_runtime": 6877.8936,
     "train_samples": 50802,
-    "train_samples_per_second": 30.65,
-    "train_steps_per_second": 0.212
+    "train_samples_per_second": 36.931,
+    "train_steps_per_second": 0.256
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0bef5fbd1a333a0938f0a0e465dcbd17cd3370807d8870ed7c029141e83f0648
+oid sha256:a6fbadd5f72d783e4eef10b0ec932f1d5e13f20e83ebacd844cdbf5fb249cb55
 size 1975192208

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 0.8502598016060463,
+    "epoch": 1.5588096362777515,
     "total_flos": 0.0,
-    "train_loss": 0.6054576412836711,
-    "train_runtime": 8287.5068,
+    "train_loss": 0.17483963706276634,
+    "train_runtime": 6877.8936,
     "train_samples": 50802,
-    "train_samples_per_second": 30.65,
-    "train_steps_per_second": 0.212
+    "train_samples_per_second": 36.931,
+    "train_steps_per_second": 0.256
 }

trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 9.357504844665527,
   "best_model_checkpoint": "./qwen2.5-0.5b/qwen2.5-0.5b-expo-DPO-ES-0.01/checkpoint-50",
-  "epoch": 0.8502598016060463,
+  "epoch": 1.5588096362777515,
   "eval_steps": 50,
-  "global_step": 300,
+  "global_step": 550,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -223,13 +223,178 @@
       "step": 300
     },
     {
-      "epoch": 0.8502598016060463,
-      "step": 300,
+      "dpo_loss": 0.4693358838558197,
+      "epoch": 0.9919697685403873,
+      "grad_norm": 17.283303885786346,
+      "learning_rate": 4.8526047530778175e-06,
+      "logits": -4.669376373291016,
+      "logps": -331.3940124511719,
+      "loss": 0.4786,
+      "objective": 0.4693358838558197,
+      "ranking_idealized": 0.8804166913032532,
+      "ranking_idealized_expo": 0.5387499928474426,
+      "ranking_simple": 0.7683333158493042,
+      "regularize": 0.4693358838558197,
+      "step": 350,
+      "wo_beta": 12.894118309020996
+    },
+    {
+      "epoch": 0.9919697685403873,
+      "eval_dpo_loss": 0.6896602511405945,
+      "eval_logits": -4.919209957122803,
+      "eval_logps": -368.701416015625,
+      "eval_loss": 0.6866692304611206,
+      "eval_objective": 0.6896602511405945,
+      "eval_ranking_idealized": 0.8731883764266968,
+      "eval_ranking_idealized_expo": 0.5320910811424255,
+      "eval_ranking_simple": 0.5750517845153809,
+      "eval_regularize": 0.6896602511405945,
+      "eval_runtime": 308.8788,
+      "eval_samples_per_second": 18.745,
+      "eval_steps_per_second": 1.564,
+      "eval_wo_beta": 27.71128273010254,
+      "step": 350
+    },
+    {
+      "dpo_loss": 0.3561079800128937,
+      "epoch": 1.1336797354747283,
+      "grad_norm": 15.512579363366164,
+      "learning_rate": 4.757316345716554e-06,
+      "logits": -4.89206075668335,
+      "logps": -377.2569580078125,
+      "loss": 0.3619,
+      "objective": 0.3561079800128937,
+      "ranking_idealized": 0.8895833492279053,
+      "ranking_idealized_expo": 0.5450000166893005,
+      "ranking_simple": 0.8650000095367432,
+      "regularize": 0.3561079800128937,
+      "step": 400,
+      "wo_beta": 8.194127082824707
+    },
+    {
+      "epoch": 1.1336797354747283,
+      "eval_dpo_loss": 0.6989732384681702,
+      "eval_logits": -5.580094814300537,
+      "eval_logps": -392.70184326171875,
+      "eval_loss": 0.6960746049880981,
+      "eval_objective": 0.6989732384681702,
+      "eval_ranking_idealized": 0.8731883764266968,
+      "eval_ranking_idealized_expo": 0.5320910811424255,
+      "eval_ranking_simple": 0.5848861336708069,
+      "eval_regularize": 0.6989732384681702,
+      "eval_runtime": 308.969,
+      "eval_samples_per_second": 18.74,
+      "eval_steps_per_second": 1.563,
+      "eval_wo_beta": 32.072998046875,
+      "step": 400
+    },
+    {
+      "dpo_loss": 0.3608836829662323,
+      "epoch": 1.2753897024090695,
+      "grad_norm": 16.08489095978575,
+      "learning_rate": 4.639847716126855e-06,
+      "logits": -5.059500217437744,
+      "logps": -364.7832336425781,
+      "loss": 0.3679,
+      "objective": 0.3608836829662323,
+      "ranking_idealized": 0.8845833539962769,
+      "ranking_idealized_expo": 0.5266666412353516,
+      "ranking_simple": 0.8412500023841858,
+      "regularize": 0.3608836829662323,
+      "step": 450,
+      "wo_beta": 9.603814125061035
+    },
+    {
+      "epoch": 1.2753897024090695,
+      "eval_dpo_loss": 0.6952692866325378,
+      "eval_logits": -5.245004177093506,
+      "eval_logps": -349.20294189453125,
+      "eval_loss": 0.6843227744102478,
+      "eval_objective": 0.6952692866325378,
+      "eval_ranking_idealized": 0.8731883764266968,
+      "eval_ranking_idealized_expo": 0.5320910811424255,
+      "eval_ranking_simple": 0.5885093212127686,
+      "eval_regularize": 0.6952692866325378,
+      "eval_runtime": 308.243,
+      "eval_samples_per_second": 18.784,
+      "eval_steps_per_second": 1.567,
+      "eval_wo_beta": 31.319866180419922,
+      "step": 450
+    },
+    {
+      "dpo_loss": 0.3647218942642212,
+      "epoch": 1.4170996693434104,
+      "grad_norm": 16.426230236098732,
+      "learning_rate": 4.501353102310901e-06,
+      "logits": -5.431642055511475,
+      "logps": -371.9469299316406,
+      "loss": 0.3662,
+      "objective": 0.3647218942642212,
+      "ranking_idealized": 0.877916693687439,
+      "ranking_idealized_expo": 0.5095833539962769,
+      "ranking_simple": 0.8420833349227905,
+      "regularize": 0.3647218942642212,
+      "step": 500,
+      "wo_beta": 10.912171363830566
+    },
+    {
+      "epoch": 1.4170996693434104,
+      "eval_dpo_loss": 0.6903234720230103,
+      "eval_logits": -5.823331832885742,
+      "eval_logps": -350.71368408203125,
+      "eval_loss": 0.6858127117156982,
+      "eval_objective": 0.6903234720230103,
+      "eval_ranking_idealized": 0.8731883764266968,
+      "eval_ranking_idealized_expo": 0.5320910811424255,
+      "eval_ranking_simple": 0.589026927947998,
+      "eval_regularize": 0.6903234720230103,
+      "eval_runtime": 308.1061,
+      "eval_samples_per_second": 18.792,
+      "eval_steps_per_second": 1.568,
+      "eval_wo_beta": 30.27263641357422,
+      "step": 500
+    },
+    {
+      "dpo_loss": 0.3470539152622223,
+      "epoch": 1.5588096362777515,
+      "grad_norm": 13.971929990602174,
+      "learning_rate": 4.34319334202531e-06,
+      "logits": -5.66475248336792,
+      "logps": -394.5787048339844,
+      "loss": 0.3485,
+      "objective": 0.3470539152622223,
+      "ranking_idealized": 0.8870833516120911,
+      "ranking_idealized_expo": 0.5220833420753479,
+      "ranking_simple": 0.8383333086967468,
+      "regularize": 0.3470539152622223,
+      "step": 550,
+      "wo_beta": 10.396123886108398
+    },
+    {
+      "epoch": 1.5588096362777515,
+      "eval_dpo_loss": 0.6972023844718933,
+      "eval_logits": -5.249249458312988,
+      "eval_logps": -349.5347900390625,
+      "eval_loss": 0.6901025772094727,
+      "eval_objective": 0.6972023844718933,
+      "eval_ranking_idealized": 0.8731883764266968,
+      "eval_ranking_idealized_expo": 0.5320910811424255,
+      "eval_ranking_simple": 0.5719461441040039,
+      "eval_regularize": 0.6972023844718933,
+      "eval_runtime": 309.5507,
+      "eval_samples_per_second": 18.705,
+      "eval_steps_per_second": 1.56,
+      "eval_wo_beta": 31.17648696899414,
+      "step": 550
+    },
+    {
+      "epoch": 1.5588096362777515,
+      "step": 550,
       "total_flos": 0.0,
-      "train_loss": 0.6054576412836711,
-      "train_runtime": 8287.5068,
-      "train_samples_per_second": 30.65,
-      "train_steps_per_second": 0.212
+      "train_loss": 0.17483963706276634,
+      "train_runtime": 6877.8936,
+      "train_samples_per_second": 36.931,
+      "train_steps_per_second": 0.256
     }
   ],
   "logging_steps": 50,