Model save

README.md

@@ -0,0 +1,75 @@
+---
+license: apache-2.0
+library_name: peft
+tags:
+- trl
+- dpo
+- generated_from_trainer
+base_model: alignment-handbook/zephyr-7b-sft-full
+model-index:
+- name: nash_dpo_doff_real_no_golden_iter_2
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# nash_dpo_doff_real_no_golden_iter_2
+
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.6731
+- Rewards/chosen: -0.0620
+- Rewards/rejected: -0.1035
+- Rewards/accuracies: 0.5940
+- Rewards/margins: 0.0416
+- Logps/rejected: -280.7516
+- Logps/chosen: -298.0970
+- Logits/rejected: -2.5621
+- Logits/chosen: -2.6577
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-06
+- train_batch_size: 2
+- eval_batch_size: 2
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 16
+- total_train_batch_size: 128
+- total_eval_batch_size: 8
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.685         | 0.66  | 100  | 0.6731          | -0.0620        | -0.1035          | 0.5940             | 0.0416          | -280.7516      | -298.0970    | -2.5621         | -2.6577       |
+
+
+### Framework versions
+
+- PEFT 0.7.1
+- Transformers 4.36.2
+- Pytorch 2.1.2+cu121
+- Datasets 2.14.6
+- Tokenizers 0.15.2

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.686891817493944,
+    "train_runtime": 6659.5732,
+    "train_samples": 19400,
+    "train_samples_per_second": 2.913,
+    "train_steps_per_second": 0.023
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.686891817493944,
+    "train_runtime": 6659.5732,
+    "train_samples": 19400,
+    "train_samples_per_second": 2.913,
+    "train_steps_per_second": 0.023
+}

trainer_state.json

@@ -0,0 +1,270 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9962886597938144,
+  "eval_steps": 100,
+  "global_step": 151,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01,
+      "learning_rate": 3.125e-07,
+      "logits/chosen": -2.7441580295562744,
+      "logits/rejected": -2.7849507331848145,
+      "logps/chosen": -179.16078186035156,
+      "logps/rejected": -219.60662841796875,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.125e-06,
+      "logits/chosen": -2.698986768722534,
+      "logits/rejected": -2.668747901916504,
+      "logps/chosen": -186.90733337402344,
+      "logps/rejected": -196.28076171875,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4826388955116272,
+      "rewards/chosen": -0.004796468652784824,
+      "rewards/margins": 0.0007755096885375679,
+      "rewards/rejected": -0.0055719781666994095,
+      "step": 10
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.989176976624511e-06,
+      "logits/chosen": -2.70469331741333,
+      "logits/rejected": -2.713209390640259,
+      "logps/chosen": -193.64369201660156,
+      "logps/rejected": -209.58154296875,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.565625011920929,
+      "rewards/chosen": -0.018005412071943283,
+      "rewards/margins": 0.0009977234294638038,
+      "rewards/rejected": -0.019003134220838547,
+      "step": 20
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.868491606285823e-06,
+      "logits/chosen": -2.7084383964538574,
+      "logits/rejected": -2.6812875270843506,
+      "logps/chosen": -199.87318420410156,
+      "logps/rejected": -204.8383026123047,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.006463692523539066,
+      "rewards/margins": 0.002128707943484187,
+      "rewards/rejected": 0.0043349843472242355,
+      "step": 30
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.620120240391065e-06,
+      "logits/chosen": -2.7429275512695312,
+      "logits/rejected": -2.739238739013672,
+      "logps/chosen": -213.66110229492188,
+      "logps/rejected": -217.1997833251953,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.559374988079071,
+      "rewards/chosen": -0.019007813185453415,
+      "rewards/margins": 0.005641926545649767,
+      "rewards/rejected": -0.024649741128087044,
+      "step": 40
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.257452643564155e-06,
+      "logits/chosen": -2.698044538497925,
+      "logits/rejected": -2.675020217895508,
+      "logps/chosen": -198.57852172851562,
+      "logps/rejected": -207.1144256591797,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.534375011920929,
+      "rewards/chosen": 0.023732703179121017,
+      "rewards/margins": 0.009518267586827278,
+      "rewards/rejected": 0.01421443559229374,
+      "step": 50
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.8000403198230385e-06,
+      "logits/chosen": -2.733064889907837,
+      "logits/rejected": -2.707578659057617,
+      "logps/chosen": -199.5453338623047,
+      "logps/rejected": -211.5819549560547,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.5093749761581421,
+      "rewards/chosen": -0.09771570563316345,
+      "rewards/margins": 0.007795141544193029,
+      "rewards/rejected": -0.10551085323095322,
+      "step": 60
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.272542485937369e-06,
+      "logits/chosen": -2.6484038829803467,
+      "logits/rejected": -2.620896816253662,
+      "logps/chosen": -205.17855834960938,
+      "logps/rejected": -207.8897247314453,
+      "loss": 0.6872,
+      "rewards/accuracies": 0.565625011920929,
+      "rewards/chosen": -0.02447286620736122,
+      "rewards/margins": 0.014284002594649792,
+      "rewards/rejected": -0.03875686600804329,
+      "step": 70
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.703396686669646e-06,
+      "logits/chosen": -2.6457228660583496,
+      "logits/rejected": -2.627340316772461,
+      "logps/chosen": -194.91592407226562,
+      "logps/rejected": -201.43222045898438,
+      "loss": 0.6851,
+      "rewards/accuracies": 0.546875,
+      "rewards/chosen": -0.0020533394999802113,
+      "rewards/margins": 0.020867574959993362,
+      "rewards/rejected": -0.022920912131667137,
+      "step": 80
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1232857193762923e-06,
+      "logits/chosen": -2.6845784187316895,
+      "logits/rejected": -2.6753616333007812,
+      "logps/chosen": -200.05587768554688,
+      "logps/rejected": -214.01171875,
+      "loss": 0.6846,
+      "rewards/accuracies": 0.528124988079071,
+      "rewards/chosen": -0.07258043438196182,
+      "rewards/margins": 0.012966667301952839,
+      "rewards/rejected": -0.08554709702730179,
+      "step": 90
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.56348351646022e-06,
+      "logits/chosen": -2.6686387062072754,
+      "logits/rejected": -2.6655027866363525,
+      "logps/chosen": -218.01806640625,
+      "logps/rejected": -232.34365844726562,
+      "loss": 0.685,
+      "rewards/accuracies": 0.534375011920929,
+      "rewards/chosen": -0.03596334531903267,
+      "rewards/margins": 0.025272289291024208,
+      "rewards/rejected": -0.06123562902212143,
+      "step": 100
+    },
+    {
+      "epoch": 0.66,
+      "eval_logits/chosen": -2.657667398452759,
+      "eval_logits/rejected": -2.562147855758667,
+      "eval_logps/chosen": -298.0969543457031,
+      "eval_logps/rejected": -280.75164794921875,
+      "eval_loss": 0.6731066107749939,
+      "eval_rewards/accuracies": 0.593999981880188,
+      "eval_rewards/chosen": -0.06198841705918312,
+      "eval_rewards/margins": 0.04155047610402107,
+      "eval_rewards/rejected": -0.1035388857126236,
+      "eval_runtime": 395.0763,
+      "eval_samples_per_second": 5.062,
+      "eval_steps_per_second": 0.633,
+      "step": 100
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0541691595800338e-06,
+      "logits/chosen": -2.6971821784973145,
+      "logits/rejected": -2.69277024269104,
+      "logps/chosen": -206.67330932617188,
+      "logps/rejected": -217.5113525390625,
+      "loss": 0.685,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.017587538808584213,
+      "rewards/margins": 0.005972611717879772,
+      "rewards/rejected": -0.02356014773249626,
+      "step": 110
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.227999175462521e-07,
+      "logits/chosen": -2.645538806915283,
+      "logits/rejected": -2.6399052143096924,
+      "logps/chosen": -208.75247192382812,
+      "logps/rejected": -216.8068389892578,
+      "loss": 0.6839,
+      "rewards/accuracies": 0.590624988079071,
+      "rewards/chosen": -0.050704460591077805,
+      "rewards/margins": 0.024100570008158684,
+      "rewards/rejected": -0.07480503618717194,
+      "step": 120
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.9263101785268253e-07,
+      "logits/chosen": -2.683271646499634,
+      "logits/rejected": -2.6696271896362305,
+      "logps/chosen": -195.18931579589844,
+      "logps/rejected": -207.58615112304688,
+      "loss": 0.6817,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.07591215521097183,
+      "rewards/margins": 0.022446561604738235,
+      "rewards/rejected": -0.09835871309041977,
+      "step": 130
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.146195134284052e-08,
+      "logits/chosen": -2.6115384101867676,
+      "logits/rejected": -2.6409411430358887,
+      "logps/chosen": -203.01991271972656,
+      "logps/rejected": -218.3817138671875,
+      "loss": 0.6815,
+      "rewards/accuracies": 0.5218750238418579,
+      "rewards/chosen": -0.08360076695680618,
+      "rewards/margins": 0.017637768760323524,
+      "rewards/rejected": -0.10123852640390396,
+      "step": 140
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.768970513457151e-10,
+      "logits/chosen": -2.642993450164795,
+      "logits/rejected": -2.6141514778137207,
+      "logps/chosen": -193.86155700683594,
+      "logps/rejected": -207.05917358398438,
+      "loss": 0.6827,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.060324084013700485,
+      "rewards/margins": 0.04426536709070206,
+      "rewards/rejected": -0.10458944737911224,
+      "step": 150
+    },
+    {
+      "epoch": 1.0,
+      "step": 151,
+      "total_flos": 0.0,
+      "train_loss": 0.686891817493944,
+      "train_runtime": 6659.5732,
+      "train_samples_per_second": 2.913,
+      "train_steps_per_second": 0.023
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 151,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}