Model save

README.md

@@ -0,0 +1,75 @@
+---
+license: apache-2.0
+library_name: peft
+tags:
+- trl
+- dpo
+- generated_from_trainer
+base_model: alignment-handbook/zephyr-7b-sft-full
+model-index:
+- name: nash_simple_online_iter_3
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# nash_simple_online_iter_3
+
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.6670
+- Rewards/chosen: -0.0073
+- Rewards/rejected: -0.0639
+- Rewards/accuracies: 0.6260
+- Rewards/margins: 0.0566
+- Logps/rejected: -276.2926
+- Logps/chosen: -295.2946
+- Logits/rejected: -2.5081
+- Logits/chosen: -2.6012
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-06
+- train_batch_size: 2
+- eval_batch_size: 2
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 16
+- total_train_batch_size: 128
+- total_eval_batch_size: 8
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.6816        | 0.61  | 100  | 0.6670          | -0.0073        | -0.0639          | 0.6260             | 0.0566          | -276.2926      | -295.2946    | -2.5081         | -2.6012       |
+
+
+### Framework versions
+
+- PEFT 0.7.1
+- Transformers 4.36.2
+- Pytorch 2.3.0+cu121
+- Datasets 2.14.6
+- Tokenizers 0.15.2

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.6834371660694931,
+    "train_runtime": 6903.7645,
+    "train_samples": 21135,
+    "train_samples_per_second": 3.061,
+    "train_steps_per_second": 0.024
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.6834371660694931,
+    "train_runtime": 6903.7645,
+    "train_samples": 21135,
+    "train_samples_per_second": 3.061,
+    "train_steps_per_second": 0.024
+}

trainer_state.json

@@ -0,0 +1,284 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9992429977289932,
+  "eval_steps": 100,
+  "global_step": 165,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01,
+      "learning_rate": 2.9411764705882356e-07,
+      "logits/chosen": -2.7495079040527344,
+      "logits/rejected": -2.7064833641052246,
+      "logps/chosen": -164.81121826171875,
+      "logps/rejected": -156.92835998535156,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.9411764705882355e-06,
+      "logits/chosen": -2.7017762660980225,
+      "logits/rejected": -2.6830601692199707,
+      "logps/chosen": -139.487548828125,
+      "logps/rejected": -143.1040802001953,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.3958333432674408,
+      "rewards/chosen": 0.004722592420876026,
+      "rewards/margins": -0.0002141917502740398,
+      "rewards/rejected": 0.004936783574521542,
+      "step": 10
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.994932636402032e-06,
+      "logits/chosen": -2.735475540161133,
+      "logits/rejected": -2.686282157897949,
+      "logps/chosen": -130.50253295898438,
+      "logps/rejected": -130.809326171875,
+      "loss": 0.692,
+      "rewards/accuracies": 0.49687498807907104,
+      "rewards/chosen": 0.08099536597728729,
+      "rewards/margins": 0.001809996203519404,
+      "rewards/rejected": 0.0791853740811348,
+      "step": 20
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.905416503522124e-06,
+      "logits/chosen": -2.7524123191833496,
+      "logits/rejected": -2.7140748500823975,
+      "logps/chosen": -123.51778411865234,
+      "logps/rejected": -126.41548919677734,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": 0.14591369032859802,
+      "rewards/margins": 0.008296089246869087,
+      "rewards/rejected": 0.13761760294437408,
+      "step": 30
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.707922373336524e-06,
+      "logits/chosen": -2.6890153884887695,
+      "logits/rejected": -2.6654117107391357,
+      "logps/chosen": -132.10757446289062,
+      "logps/rejected": -136.89508056640625,
+      "loss": 0.6877,
+      "rewards/accuracies": 0.5531250238418579,
+      "rewards/chosen": 0.09638135135173798,
+      "rewards/margins": 0.012408262118697166,
+      "rewards/rejected": 0.08397307246923447,
+      "step": 40
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.411315662967732e-06,
+      "logits/chosen": -2.6580519676208496,
+      "logits/rejected": -2.691502809524536,
+      "logps/chosen": -122.78013610839844,
+      "logps/rejected": -132.18321228027344,
+      "loss": 0.6865,
+      "rewards/accuracies": 0.515625,
+      "rewards/chosen": 0.1647396981716156,
+      "rewards/margins": 0.004126954823732376,
+      "rewards/rejected": 0.16061276197433472,
+      "step": 50
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.028910905897229e-06,
+      "logits/chosen": -2.655439615249634,
+      "logits/rejected": -2.6334452629089355,
+      "logps/chosen": -124.12255859375,
+      "logps/rejected": -132.15664672851562,
+      "loss": 0.6837,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.14119209349155426,
+      "rewards/margins": 0.026216480880975723,
+      "rewards/rejected": 0.11497560888528824,
+      "step": 60
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.577874068920446e-06,
+      "logits/chosen": -2.6378352642059326,
+      "logits/rejected": -2.601242780685425,
+      "logps/chosen": -144.02236938476562,
+      "logps/rejected": -147.04486083984375,
+      "loss": 0.6856,
+      "rewards/accuracies": 0.590624988079071,
+      "rewards/chosen": -0.013076819479465485,
+      "rewards/margins": 0.030333761125802994,
+      "rewards/rejected": -0.04341058060526848,
+      "step": 70
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.0784519801008546e-06,
+      "logits/chosen": -2.613574504852295,
+      "logits/rejected": -2.547489643096924,
+      "logps/chosen": -128.6609649658203,
+      "logps/rejected": -128.91639709472656,
+      "loss": 0.6795,
+      "rewards/accuracies": 0.5531250238418579,
+      "rewards/chosen": 0.1631721556186676,
+      "rewards/margins": 0.03735139220952988,
+      "rewards/rejected": 0.12582075595855713,
+      "step": 80
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.553063458334059e-06,
+      "logits/chosen": -2.6456406116485596,
+      "logits/rejected": -2.6266322135925293,
+      "logps/chosen": -124.13682556152344,
+      "logps/rejected": -135.78231811523438,
+      "loss": 0.6817,
+      "rewards/accuracies": 0.5718749761581421,
+      "rewards/chosen": 0.15800026059150696,
+      "rewards/margins": 0.03171014413237572,
+      "rewards/rejected": 0.12629011273384094,
+      "step": 90
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.025292943281429e-06,
+      "logits/chosen": -2.659860134124756,
+      "logits/rejected": -2.62748122215271,
+      "logps/chosen": -129.3069305419922,
+      "logps/rejected": -131.54519653320312,
+      "loss": 0.6816,
+      "rewards/accuracies": 0.590624988079071,
+      "rewards/chosen": 0.12287310510873795,
+      "rewards/margins": 0.0398445650935173,
+      "rewards/rejected": 0.08302854001522064,
+      "step": 100
+    },
+    {
+      "epoch": 0.61,
+      "eval_logits/chosen": -2.6012067794799805,
+      "eval_logits/rejected": -2.5080597400665283,
+      "eval_logps/chosen": -295.2945556640625,
+      "eval_logps/rejected": -276.2926330566406,
+      "eval_loss": 0.6669759750366211,
+      "eval_rewards/accuracies": 0.6259999871253967,
+      "eval_rewards/chosen": -0.0072901868261396885,
+      "eval_rewards/margins": 0.05662847310304642,
+      "eval_rewards/rejected": -0.06391866505146027,
+      "eval_runtime": 383.8552,
+      "eval_samples_per_second": 5.21,
+      "eval_steps_per_second": 0.651,
+      "step": 100
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.5188318011445907e-06,
+      "logits/chosen": -2.6604905128479004,
+      "logits/rejected": -2.6317429542541504,
+      "logps/chosen": -140.4075927734375,
+      "logps/rejected": -148.9583282470703,
+      "loss": 0.674,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": 0.03068632446229458,
+      "rewards/margins": 0.04105384275317192,
+      "rewards/rejected": -0.01036751363426447,
+      "step": 110
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0564148305586296e-06,
+      "logits/chosen": -2.6386797428131104,
+      "logits/rejected": -2.65537428855896,
+      "logps/chosen": -133.14720153808594,
+      "logps/rejected": -141.28182983398438,
+      "loss": 0.6802,
+      "rewards/accuracies": 0.578125,
+      "rewards/chosen": 0.07068979740142822,
+      "rewards/margins": 0.035736724734306335,
+      "rewards/rejected": 0.034953076392412186,
+      "step": 120
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.587997083462197e-07,
+      "logits/chosen": -2.6442322731018066,
+      "logits/rejected": -2.62335467338562,
+      "logps/chosen": -145.12142944335938,
+      "logps/rejected": -146.34092712402344,
+      "loss": 0.6763,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": 0.05981719493865967,
+      "rewards/margins": 0.03406576067209244,
+      "rewards/rejected": 0.02575143240392208,
+      "step": 130
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.438351873250492e-07,
+      "logits/chosen": -2.591649293899536,
+      "logits/rejected": -2.575448513031006,
+      "logps/chosen": -136.5642852783203,
+      "logps/rejected": -135.15036010742188,
+      "loss": 0.6799,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.02418154664337635,
+      "rewards/margins": 0.014547958970069885,
+      "rewards/rejected": 0.009633589535951614,
+      "step": 140
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.2565987432367032e-07,
+      "logits/chosen": -2.628605842590332,
+      "logits/rejected": -2.6137516498565674,
+      "logps/chosen": -131.29632568359375,
+      "logps/rejected": -132.41908264160156,
+      "loss": 0.681,
+      "rewards/accuracies": 0.590624988079071,
+      "rewards/chosen": 0.05179730802774429,
+      "rewards/margins": 0.021983899176120758,
+      "rewards/rejected": 0.029813403263688087,
+      "step": 150
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4067554877743861e-08,
+      "logits/chosen": -2.5783798694610596,
+      "logits/rejected": -2.5735974311828613,
+      "logps/chosen": -126.33128356933594,
+      "logps/rejected": -133.07003784179688,
+      "loss": 0.6789,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.06782820075750351,
+      "rewards/margins": 0.04517916589975357,
+      "rewards/rejected": 0.022649036720395088,
+      "step": 160
+    },
+    {
+      "epoch": 1.0,
+      "step": 165,
+      "total_flos": 0.0,
+      "train_loss": 0.6834371660694931,
+      "train_runtime": 6903.7645,
+      "train_samples_per_second": 3.061,
+      "train_steps_per_second": 0.024
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 165,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}