Model save

README.md

@@ -0,0 +1,81 @@
+---
+license: apache-2.0
+base_model: hZzy/qwen2.5-0.5b-sft-news-IFT
+tags:
+- trl
+- expo
+- generated_from_trainer
+model-index:
+- name: qwen2.5-0.5b-expo-DPO-ES2-0.1
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/zhiyuzha-university-of-florida/huggingface/runs/vy4xlg1g)
+# qwen2.5-0.5b-expo-DPO-ES2-0.1
+
+This model is a fine-tuned version of [hZzy/qwen2.5-0.5b-sft-news-IFT](https://huggingface.co/hZzy/qwen2.5-0.5b-sft-news-IFT) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.6897
+- Logps: -102.1435
+- Logits: -1.9358
+- Objective: 0.6916
+- Dpo Loss: 0.6916
+- Regularize: 0.6916
+- Ranking Simple: 0.5419
+- Ranking Idealized: 0.6030
+- Ranking Idealized Expo: 0.5223
+- Wo Beta: 8.3961
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 1e-06
+- train_batch_size: 4
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 3
+- gradient_accumulation_steps: 12
+- total_train_batch_size: 144
+- total_eval_batch_size: 12
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 5
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss | Logps     | Logits  | Objective | Dpo Loss | Regularize | Ranking Simple | Ranking Idealized | Ranking Idealized Expo | Wo Beta |
+|:-------------:|:------:|:----:|:---------------:|:---------:|:-------:|:---------:|:--------:|:----------:|:--------------:|:-----------------:|:----------------------:|:-------:|
+| 0.689         | 0.1417 | 50   | 0.6875          | -90.0815  | -1.4869 | 0.6892    | 0.6892   | 0.6892     | 0.5259         | 0.6030            | 0.5223                 | 7.8857  |
+| 0.6673        | 0.2834 | 100  | 0.6808          | -90.9674  | -1.6164 | 0.6836    | 0.6836   | 0.6836     | 0.5331         | 0.6030            | 0.5223                 | 7.8643  |
+| 0.6376        | 0.4251 | 150  | 0.6785          | -94.6386  | -1.6873 | 0.6833    | 0.6833   | 0.6833     | 0.5342         | 0.6030            | 0.5223                 | 8.1745  |
+| 0.5955        | 0.5668 | 200  | 0.6808          | -100.2786 | -1.8583 | 0.6818    | 0.6818   | 0.6818     | 0.5342         | 0.6030            | 0.5223                 | 7.9037  |
+| 0.5623        | 0.7085 | 250  | 0.6757          | -97.3034  | -1.9407 | 0.6757    | 0.6757   | 0.6757     | 0.5362         | 0.6030            | 0.5223                 | 7.9161  |
+| 0.5255        | 0.8503 | 300  | 0.7037          | -102.4820 | -2.0313 | 0.7119    | 0.7119   | 0.7119     | 0.5352         | 0.6030            | 0.5223                 | 8.7956  |
+| 0.4939        | 0.9920 | 350  | 0.6897          | -102.1435 | -1.9358 | 0.6916    | 0.6916   | 0.6916     | 0.5419         | 0.6030            | 0.5223                 | 8.3961  |
+
+
+### Framework versions
+
+- Transformers 4.42.0
+- Pytorch 2.3.0+cu121
+- Datasets 2.19.1
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9919697685403873,
+    "total_flos": 0.0,
+    "train_loss": 0.5958910301753453,
+    "train_runtime": 9674.8125,
+    "train_samples": 50802,
+    "train_samples_per_second": 26.255,
+    "train_steps_per_second": 0.182
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "bos_token_id": 151644,
+  "eos_token_id": 151645,
+  "max_new_tokens": 2048,
+  "pad_token_id": 151645,
+  "transformers_version": "4.42.0"
+}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4bb2b9d3c397a29b6bdd3a6278e23b513b8ecf28c9b29562dee47dc8290907f7
+oid sha256:ac600b94196565ee85509c7575016178d5c3e3197230633e6a5523519a22c68f
 size 1975192208

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9919697685403873,
+    "total_flos": 0.0,
+    "train_loss": 0.5958910301753453,
+    "train_runtime": 9674.8125,
+    "train_samples": 50802,
+    "train_samples_per_second": 26.255,
+    "train_steps_per_second": 0.182
+}

trainer_state.json

@@ -0,0 +1,298 @@
+{
+  "best_metric": 7.86433744430542,
+  "best_model_checkpoint": "./qwen2.5-0.5b/qwen2.5-0.5b-expo-DPO-ES2-0.1/checkpoint-100",
+  "epoch": 0.9919697685403873,
+  "eval_steps": 50,
+  "global_step": 350,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "dpo_loss": 0.6931471824645996,
+      "epoch": 0.002834199338686821,
+      "grad_norm": 18.442528136913708,
+      "learning_rate": 5.681818181818181e-09,
+      "logits": -1.359458565711975,
+      "logps": -84.69721221923828,
+      "loss": 0.6931,
+      "objective": 0.6931471824645996,
+      "ranking_idealized": 0.6458333134651184,
+      "ranking_idealized_expo": 0.5833333134651184,
+      "ranking_simple": 0.5833333134651184,
+      "regularize": 0.6931471824645996,
+      "step": 1,
+      "wo_beta": 5.271125316619873
+    },
+    {
+      "dpo_loss": 0.6897825002670288,
+      "epoch": 0.14170996693434104,
+      "grad_norm": 17.687331232419115,
+      "learning_rate": 2.840909090909091e-07,
+      "logits": -1.467690110206604,
+      "logps": -83.78176879882812,
+      "loss": 0.689,
+      "objective": 0.6897825002670288,
+      "ranking_idealized": 0.608418345451355,
+      "ranking_idealized_expo": 0.5229591727256775,
+      "ranking_simple": 0.5259353518486023,
+      "regularize": 0.6897825002670288,
+      "step": 50,
+      "wo_beta": 7.163306713104248
+    },
+    {
+      "epoch": 0.14170996693434104,
+      "eval_dpo_loss": 0.6892488598823547,
+      "eval_logits": -1.486922264099121,
+      "eval_logps": -90.08146667480469,
+      "eval_loss": 0.6874589920043945,
+      "eval_objective": 0.6892488598823547,
+      "eval_ranking_idealized": 0.6030020713806152,
+      "eval_ranking_idealized_expo": 0.5222567319869995,
+      "eval_ranking_simple": 0.5258799195289612,
+      "eval_regularize": 0.6892488598823547,
+      "eval_runtime": 308.1362,
+      "eval_samples_per_second": 18.79,
+      "eval_steps_per_second": 1.567,
+      "eval_wo_beta": 7.8856682777404785,
+      "step": 50
+    },
+    {
+      "dpo_loss": 0.6650223731994629,
+      "epoch": 0.2834199338686821,
+      "grad_norm": 19.38239421391755,
+      "learning_rate": 5.681818181818182e-07,
+      "logits": -1.5395574569702148,
+      "logps": -83.69255828857422,
+      "loss": 0.6673,
+      "objective": 0.6650223731994629,
+      "ranking_idealized": 0.6016666889190674,
+      "ranking_idealized_expo": 0.5141666531562805,
+      "ranking_simple": 0.527916669845581,
+      "regularize": 0.6650223731994629,
+      "step": 100,
+      "wo_beta": 6.79261589050293
+    },
+    {
+      "epoch": 0.2834199338686821,
+      "eval_dpo_loss": 0.6835533380508423,
+      "eval_logits": -1.6164112091064453,
+      "eval_logps": -90.96741485595703,
+      "eval_loss": 0.6808017492294312,
+      "eval_objective": 0.6835533380508423,
+      "eval_ranking_idealized": 0.6030020713806152,
+      "eval_ranking_idealized_expo": 0.5222567319869995,
+      "eval_ranking_simple": 0.5331262946128845,
+      "eval_regularize": 0.6835533380508423,
+      "eval_runtime": 307.7224,
+      "eval_samples_per_second": 18.816,
+      "eval_steps_per_second": 1.57,
+      "eval_wo_beta": 7.86433744430542,
+      "step": 100
+    },
+    {
+      "dpo_loss": 0.6406400799751282,
+      "epoch": 0.42512990080302315,
+      "grad_norm": 18.95367759941799,
+      "learning_rate": 8.522727272727273e-07,
+      "logits": -1.6548144817352295,
+      "logps": -86.31576538085938,
+      "loss": 0.6376,
+      "objective": 0.6406400799751282,
+      "ranking_idealized": 0.6066666841506958,
+      "ranking_idealized_expo": 0.5287500023841858,
+      "ranking_simple": 0.5554166436195374,
+      "regularize": 0.6406400799751282,
+      "step": 150,
+      "wo_beta": 6.808902740478516
+    },
+    {
+      "epoch": 0.42512990080302315,
+      "eval_dpo_loss": 0.683335542678833,
+      "eval_logits": -1.6873152256011963,
+      "eval_logps": -94.63862609863281,
+      "eval_loss": 0.6785484552383423,
+      "eval_objective": 0.683335542678833,
+      "eval_ranking_idealized": 0.6030020713806152,
+      "eval_ranking_idealized_expo": 0.5222567319869995,
+      "eval_ranking_simple": 0.5341615080833435,
+      "eval_regularize": 0.683335542678833,
+      "eval_runtime": 308.25,
+      "eval_samples_per_second": 18.783,
+      "eval_steps_per_second": 1.567,
+      "eval_wo_beta": 8.17447566986084,
+      "step": 150
+    },
+    {
+      "dpo_loss": 0.595325231552124,
+      "epoch": 0.5668398677373642,
+      "grad_norm": 18.26661888886481,
+      "learning_rate": 9.99433669591504e-07,
+      "logits": -1.7552146911621094,
+      "logps": -90.12830352783203,
+      "loss": 0.5955,
+      "objective": 0.595325231552124,
+      "ranking_idealized": 0.5924999713897705,
+      "ranking_idealized_expo": 0.5166666507720947,
+      "ranking_simple": 0.5649999976158142,
+      "regularize": 0.595325231552124,
+      "step": 200,
+      "wo_beta": 6.764244556427002
+    },
+    {
+      "epoch": 0.5668398677373642,
+      "eval_dpo_loss": 0.6818291544914246,
+      "eval_logits": -1.8582650423049927,
+      "eval_logps": -100.27864837646484,
+      "eval_loss": 0.6808217167854309,
+      "eval_objective": 0.6818291544914246,
+      "eval_ranking_idealized": 0.6030020713806152,
+      "eval_ranking_idealized_expo": 0.5222567319869995,
+      "eval_ranking_simple": 0.5341615080833435,
+      "eval_regularize": 0.6818291544914246,
+      "eval_runtime": 307.6797,
+      "eval_samples_per_second": 18.818,
+      "eval_steps_per_second": 1.57,
+      "eval_wo_beta": 7.903720855712891,
+      "step": 200
+    },
+    {
+      "dpo_loss": 0.5614926218986511,
+      "epoch": 0.7085498346717053,
+      "grad_norm": 17.975386844775777,
+      "learning_rate": 9.94624571028813e-07,
+      "logits": -1.847279667854309,
+      "logps": -92.22396850585938,
+      "loss": 0.5623,
+      "objective": 0.5614926218986511,
+      "ranking_idealized": 0.5991666913032532,
+      "ranking_idealized_expo": 0.5170833468437195,
+      "ranking_simple": 0.5924999713897705,
+      "regularize": 0.5614926218986511,
+      "step": 250,
+      "wo_beta": 6.590356349945068
+    },
+    {
+      "epoch": 0.7085498346717053,
+      "eval_dpo_loss": 0.675693154335022,
+      "eval_logits": -1.9406613111495972,
+      "eval_logps": -97.30335998535156,
+      "eval_loss": 0.6757029294967651,
+      "eval_objective": 0.675693154335022,
+      "eval_ranking_idealized": 0.6030020713806152,
+      "eval_ranking_idealized_expo": 0.5222567319869995,
+      "eval_ranking_simple": 0.5362318754196167,
+      "eval_regularize": 0.675693154335022,
+      "eval_runtime": 307.504,
+      "eval_samples_per_second": 18.829,
+      "eval_steps_per_second": 1.571,
+      "eval_wo_beta": 7.916144847869873,
+      "step": 250
+    },
+    {
+      "dpo_loss": 0.5231651067733765,
+      "epoch": 0.8502598016060463,
+      "grad_norm": 21.98858332084632,
+      "learning_rate": 9.849553282839024e-07,
+      "logits": -1.9162583351135254,
+      "logps": -92.107177734375,
+      "loss": 0.5255,
+      "objective": 0.5231651067733765,
+      "ranking_idealized": 0.5799999833106995,
+      "ranking_idealized_expo": 0.4970833361148834,
+      "ranking_simple": 0.5849999785423279,
+      "regularize": 0.5231651067733765,
+      "step": 300,
+      "wo_beta": 6.37797212600708
+    },
+    {
+      "epoch": 0.8502598016060463,
+      "eval_dpo_loss": 0.7118534445762634,
+      "eval_logits": -2.031287431716919,
+      "eval_logps": -102.48200988769531,
+      "eval_loss": 0.7037488222122192,
+      "eval_objective": 0.7118534445762634,
+      "eval_ranking_idealized": 0.6030020713806152,
+      "eval_ranking_idealized_expo": 0.5222567319869995,
+      "eval_ranking_simple": 0.5351966619491577,
+      "eval_regularize": 0.7118534445762634,
+      "eval_runtime": 307.7923,
+      "eval_samples_per_second": 18.811,
+      "eval_steps_per_second": 1.569,
+      "eval_wo_beta": 8.7955904006958,
+      "step": 300
+    },
+    {
+      "dpo_loss": 0.4828701913356781,
+      "epoch": 0.9919697685403873,
+      "grad_norm": 21.148105751270688,
+      "learning_rate": 9.705209506155634e-07,
+      "logits": -1.863725185394287,
+      "logps": -97.61394500732422,
+      "loss": 0.4939,
+      "objective": 0.4828701913356781,
+      "ranking_idealized": 0.60916668176651,
+      "ranking_idealized_expo": 0.5270833373069763,
+      "ranking_simple": 0.6449999809265137,
+      "regularize": 0.4828701913356781,
+      "step": 350,
+      "wo_beta": 5.530836582183838
+    },
+    {
+      "epoch": 0.9919697685403873,
+      "eval_dpo_loss": 0.6916147470474243,
+      "eval_logits": -1.9357593059539795,
+      "eval_logps": -102.14349365234375,
+      "eval_loss": 0.6897423267364502,
+      "eval_objective": 0.6916147470474243,
+      "eval_ranking_idealized": 0.6030020713806152,
+      "eval_ranking_idealized_expo": 0.5222567319869995,
+      "eval_ranking_simple": 0.5419254899024963,
+      "eval_regularize": 0.6916147470474243,
+      "eval_runtime": 309.7171,
+      "eval_samples_per_second": 18.694,
+      "eval_steps_per_second": 1.559,
+      "eval_wo_beta": 8.39614486694336,
+      "step": 350
+    },
+    {
+      "epoch": 0.9919697685403873,
+      "step": 350,
+      "total_flos": 0.0,
+      "train_loss": 0.5958910301753453,
+      "train_runtime": 9674.8125,
+      "train_samples_per_second": 26.255,
+      "train_steps_per_second": 0.182
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 1760,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 50,
+  "stateful_callbacks": {
+    "EarlyStoppingCallback": {
+      "args": {
+        "early_stopping_patience": 5,
+        "early_stopping_threshold": 0.0
+      },
+      "attributes": {
+        "early_stopping_patience_counter": 0
+      }
+    },
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}