Model save

README.md

@@ -0,0 +1,80 @@
+---
+license: apache-2.0
+base_model: hZzy/qwen2.5-0.5b-sft-news-IFT
+tags:
+- trl
+- expo
+- generated_from_trainer
+model-index:
+- name: qwen2.5-0.5b-expo-DPO-ES-0.01
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/zhiyuzha-university-of-florida/huggingface/runs/aeakfz58)
+# qwen2.5-0.5b-expo-DPO-ES-0.01
+
+This model is a fine-tuned version of [hZzy/qwen2.5-0.5b-sft-news-IFT](https://huggingface.co/hZzy/qwen2.5-0.5b-sft-news-IFT) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.6640
+- Logps: -319.3935
+- Logits: -4.6100
+- Objective: 0.6685
+- Dpo Loss: 0.6685
+- Regularize: 0.6685
+- Ranking Simple: 0.5828
+- Ranking Idealized: 0.8732
+- Ranking Idealized Expo: 0.5321
+- Wo Beta: 23.6506
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-06
+- train_batch_size: 4
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 3
+- gradient_accumulation_steps: 12
+- total_train_batch_size: 144
+- total_eval_batch_size: 12
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 5
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss | Logps     | Logits  | Objective | Dpo Loss | Regularize | Ranking Simple | Ranking Idealized | Ranking Idealized Expo | Wo Beta |
+|:-------------:|:------:|:----:|:---------------:|:---------:|:-------:|:---------:|:--------:|:----------:|:--------------:|:-----------------:|:----------------------:|:-------:|
+| 0.6907        | 0.1417 | 50   | 0.6894          | -99.3821  | -1.7611 | 0.6902    | 0.6902   | 0.6902     | 0.5305         | 0.8732            | 0.5321                 | 9.3575  |
+| 0.6701        | 0.2834 | 100  | 0.6837          | -153.5716 | -1.8467 | 0.6896    | 0.6896   | 0.6896     | 0.5518         | 0.8732            | 0.5321                 | 14.1741 |
+| 0.637         | 0.4251 | 150  | 0.6723          | -198.9269 | -2.4614 | 0.6765    | 0.6765   | 0.6765     | 0.5823         | 0.8732            | 0.5321                 | 17.2098 |
+| 0.5833        | 0.5668 | 200  | 0.6729          | -256.0312 | -3.3478 | 0.6780    | 0.6780   | 0.6780     | 0.5797         | 0.8732            | 0.5321                 | 21.7109 |
+| 0.5439        | 0.7085 | 250  | 0.6781          | -257.3546 | -3.6269 | 0.6858    | 0.6858   | 0.6858     | 0.5683         | 0.8732            | 0.5321                 | 22.9139 |
+| 0.5077        | 0.8503 | 300  | 0.6640          | -319.3935 | -4.6100 | 0.6685    | 0.6685   | 0.6685     | 0.5828         | 0.8732            | 0.5321                 | 23.6506 |
+
+
+### Framework versions
+
+- Transformers 4.42.0
+- Pytorch 2.3.0+cu121
+- Datasets 2.19.1
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.8502598016060463,
+    "total_flos": 0.0,
+    "train_loss": 0.6054576412836711,
+    "train_runtime": 8287.5068,
+    "train_samples": 50802,
+    "train_samples_per_second": 30.65,
+    "train_steps_per_second": 0.212
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "bos_token_id": 151644,
+  "eos_token_id": 151645,
+  "max_new_tokens": 2048,
+  "pad_token_id": 151645,
+  "transformers_version": "4.42.0"
+}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:520c19027f98c4a2feafb45c7d81bb1e84cbf06c71d7d8530e453a702a49d37a
+oid sha256:a6fbadd5f72d783e4eef10b0ec932f1d5e13f20e83ebacd844cdbf5fb249cb55
 size 1975192208

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.8502598016060463,
+    "total_flos": 0.0,
+    "train_loss": 0.6054576412836711,
+    "train_runtime": 8287.5068,
+    "train_samples": 50802,
+    "train_samples_per_second": 30.65,
+    "train_steps_per_second": 0.212
+}

trainer_state.json

@@ -0,0 +1,265 @@
+{
+  "best_metric": 9.357504844665527,
+  "best_model_checkpoint": "./qwen2.5-0.5b/qwen2.5-0.5b-expo-DPO-ES-0.01/checkpoint-50",
+  "epoch": 0.8502598016060463,
+  "eval_steps": 50,
+  "global_step": 300,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "dpo_loss": 0.6931471824645996,
+      "epoch": 0.002834199338686821,
+      "grad_norm": 1.8442555137313956,
+      "learning_rate": 2.840909090909091e-08,
+      "logits": -1.359458565711975,
+      "logps": -84.69721221923828,
+      "loss": 0.6931,
+      "objective": 0.6931471824645996,
+      "ranking_idealized": 0.875,
+      "ranking_idealized_expo": 0.5833333134651184,
+      "ranking_simple": 0.5833333134651184,
+      "regularize": 0.6931471824645996,
+      "step": 1,
+      "wo_beta": 5.271125316619873
+    },
+    {
+      "dpo_loss": 0.6914740800857544,
+      "epoch": 0.14170996693434104,
+      "grad_norm": 2.883110438441685,
+      "learning_rate": 1.4204545454545458e-06,
+      "logits": -1.5272094011306763,
+      "logps": -83.2472152709961,
+      "loss": 0.6907,
+      "objective": 0.6914740800857544,
+      "ranking_idealized": 0.8945578336715698,
+      "ranking_idealized_expo": 0.5348639488220215,
+      "ranking_simple": 0.5293367505073547,
+      "regularize": 0.6914740800857544,
+      "step": 50,
+      "wo_beta": 7.299497604370117
+    },
+    {
+      "epoch": 0.14170996693434104,
+      "eval_dpo_loss": 0.6902037858963013,
+      "eval_logits": -1.7611440420150757,
+      "eval_logps": -99.38214111328125,
+      "eval_loss": 0.6893583536148071,
+      "eval_objective": 0.6902037858963013,
+      "eval_ranking_idealized": 0.8731883764266968,
+      "eval_ranking_idealized_expo": 0.5320910811424255,
+      "eval_ranking_simple": 0.5305383205413818,
+      "eval_regularize": 0.6902037858963013,
+      "eval_runtime": 314.0138,
+      "eval_samples_per_second": 18.439,
+      "eval_steps_per_second": 1.538,
+      "eval_wo_beta": 9.357504844665527,
+      "step": 50
+    },
+    {
+      "dpo_loss": 0.6664106249809265,
+      "epoch": 0.2834199338686821,
+      "grad_norm": 7.452572178923395,
+      "learning_rate": 2.8409090909090916e-06,
+      "logits": -1.6917132139205933,
+      "logps": -118.66804504394531,
+      "loss": 0.6701,
+      "objective": 0.6664106249809265,
+      "ranking_idealized": 0.8958333134651184,
+      "ranking_idealized_expo": 0.5270833373069763,
+      "ranking_simple": 0.5866666436195374,
+      "regularize": 0.6664106249809265,
+      "step": 100,
+      "wo_beta": 9.330028533935547
+    },
+    {
+      "epoch": 0.2834199338686821,
+      "eval_dpo_loss": 0.6895959377288818,
+      "eval_logits": -1.8466589450836182,
+      "eval_logps": -153.5715789794922,
+      "eval_loss": 0.6837136149406433,
+      "eval_objective": 0.6895959377288818,
+      "eval_ranking_idealized": 0.8731883764266968,
+      "eval_ranking_idealized_expo": 0.5320910811424255,
+      "eval_ranking_simple": 0.5517598390579224,
+      "eval_regularize": 0.6895959377288818,
+      "eval_runtime": 308.2112,
+      "eval_samples_per_second": 18.786,
+      "eval_steps_per_second": 1.567,
+      "eval_wo_beta": 14.174129486083984,
+      "step": 100
+    },
+    {
+      "dpo_loss": 0.6343104243278503,
+      "epoch": 0.42512990080302315,
+      "grad_norm": 13.296765037353357,
+      "learning_rate": 4.2613636363636365e-06,
+      "logits": -1.825290322303772,
+      "logps": -173.7165985107422,
+      "loss": 0.637,
+      "objective": 0.6343104243278503,
+      "ranking_idealized": 0.8862500190734863,
+      "ranking_idealized_expo": 0.5358333587646484,
+      "ranking_simple": 0.6316666603088379,
+      "regularize": 0.6343104243278503,
+      "step": 150,
+      "wo_beta": 12.891901016235352
+    },
+    {
+      "epoch": 0.42512990080302315,
+      "eval_dpo_loss": 0.6764841079711914,
+      "eval_logits": -2.461402416229248,
+      "eval_logps": -198.92694091796875,
+      "eval_loss": 0.6722760796546936,
+      "eval_objective": 0.6764841079711914,
+      "eval_ranking_idealized": 0.8731883764266968,
+      "eval_ranking_idealized_expo": 0.5320910811424255,
+      "eval_ranking_simple": 0.5822981595993042,
+      "eval_regularize": 0.6764841079711914,
+      "eval_runtime": 308.7795,
+      "eval_samples_per_second": 18.751,
+      "eval_steps_per_second": 1.564,
+      "eval_wo_beta": 17.20980453491211,
+      "step": 150
+    },
+    {
+      "dpo_loss": 0.5824262499809265,
+      "epoch": 0.5668398677373642,
+      "grad_norm": 14.182378256685057,
+      "learning_rate": 4.997168347957521e-06,
+      "logits": -2.928657054901123,
+      "logps": -216.66006469726562,
+      "loss": 0.5833,
+      "objective": 0.5824262499809265,
+      "ranking_idealized": 0.8883333206176758,
+      "ranking_idealized_expo": 0.527916669845581,
+      "ranking_simple": 0.6833333373069763,
+      "regularize": 0.5824262499809265,
+      "step": 200,
+      "wo_beta": 13.758678436279297
+    },
+    {
+      "epoch": 0.5668398677373642,
+      "eval_dpo_loss": 0.6780264377593994,
+      "eval_logits": -3.3478028774261475,
+      "eval_logps": -256.0312194824219,
+      "eval_loss": 0.6728978157043457,
+      "eval_objective": 0.6780264377593994,
+      "eval_ranking_idealized": 0.8731883764266968,
+      "eval_ranking_idealized_expo": 0.5320910811424255,
+      "eval_ranking_simple": 0.5797101259231567,
+      "eval_regularize": 0.6780264377593994,
+      "eval_runtime": 312.7328,
+      "eval_samples_per_second": 18.514,
+      "eval_steps_per_second": 1.544,
+      "eval_wo_beta": 21.710927963256836,
+      "step": 200
+    },
+    {
+      "dpo_loss": 0.541719377040863,
+      "epoch": 0.7085498346717053,
+      "grad_norm": 16.105604013075503,
+      "learning_rate": 4.973122855144066e-06,
+      "logits": -3.4248011112213135,
+      "logps": -254.68023681640625,
+      "loss": 0.5439,
+      "objective": 0.541719377040863,
+      "ranking_idealized": 0.8787500262260437,
+      "ranking_idealized_expo": 0.5266666412353516,
+      "ranking_simple": 0.7054166793823242,
+      "regularize": 0.541719377040863,
+      "step": 250,
+      "wo_beta": 15.057023048400879
+    },
+    {
+      "epoch": 0.7085498346717053,
+      "eval_dpo_loss": 0.6857688426971436,
+      "eval_logits": -3.626901865005493,
+      "eval_logps": -257.3546447753906,
+      "eval_loss": 0.6781259775161743,
+      "eval_objective": 0.6857688426971436,
+      "eval_ranking_idealized": 0.8731883764266968,
+      "eval_ranking_idealized_expo": 0.5320910811424255,
+      "eval_ranking_simple": 0.5683229565620422,
+      "eval_regularize": 0.6857688426971436,
+      "eval_runtime": 308.1728,
+      "eval_samples_per_second": 18.788,
+      "eval_steps_per_second": 1.567,
+      "eval_wo_beta": 22.9139461517334,
+      "step": 250
+    },
+    {
+      "dpo_loss": 0.49844247102737427,
+      "epoch": 0.8502598016060463,
+      "grad_norm": 14.80120970807248,
+      "learning_rate": 4.924776641419513e-06,
+      "logits": -4.0557355880737305,
+      "logps": -292.7536315917969,
+      "loss": 0.5077,
+      "objective": 0.49844247102737427,
+      "ranking_idealized": 0.8733333349227905,
+      "ranking_idealized_expo": 0.5104166865348816,
+      "ranking_simple": 0.747083306312561,
+      "regularize": 0.49844247102737427,
+      "step": 300,
+      "wo_beta": 13.577784538269043
+    },
+    {
+      "epoch": 0.8502598016060463,
+      "eval_dpo_loss": 0.6684697866439819,
+      "eval_logits": -4.609988689422607,
+      "eval_logps": -319.3935241699219,
+      "eval_loss": 0.663960874080658,
+      "eval_objective": 0.6684697866439819,
+      "eval_ranking_idealized": 0.8731883764266968,
+      "eval_ranking_idealized_expo": 0.5320910811424255,
+      "eval_ranking_simple": 0.5828157067298889,
+      "eval_regularize": 0.6684697866439819,
+      "eval_runtime": 309.749,
+      "eval_samples_per_second": 18.693,
+      "eval_steps_per_second": 1.559,
+      "eval_wo_beta": 23.650588989257812,
+      "step": 300
+    },
+    {
+      "epoch": 0.8502598016060463,
+      "step": 300,
+      "total_flos": 0.0,
+      "train_loss": 0.6054576412836711,
+      "train_runtime": 8287.5068,
+      "train_samples_per_second": 30.65,
+      "train_steps_per_second": 0.212
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 1760,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 50,
+  "stateful_callbacks": {
+    "EarlyStoppingCallback": {
+      "args": {
+        "early_stopping_patience": 5,
+        "early_stopping_threshold": 0.0
+      },
+      "attributes": {
+        "early_stopping_patience_counter": 0
+      }
+    },
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}