Model save

README.md

@@ -0,0 +1,75 @@
+---
+license: apache-2.0
+library_name: peft
+tags:
+- trl
+- dpo
+- generated_from_trainer
+base_model: alignment-handbook/zephyr-7b-sft-full
+model-index:
+- name: nash_dpo_rank4_on_vanilla_iter_3
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# nash_dpo_rank4_on_vanilla_iter_3
+
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.6884
+- Rewards/chosen: -0.1839
+- Rewards/rejected: -0.2354
+- Rewards/accuracies: 0.5440
+- Rewards/margins: 0.0514
+- Logps/rejected: -311.2962
+- Logps/chosen: -324.9433
+- Logits/rejected: -2.0325
+- Logits/chosen: -2.1221
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-06
+- train_batch_size: 2
+- eval_batch_size: 2
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 16
+- total_train_batch_size: 128
+- total_eval_batch_size: 8
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.5749        | 0.61  | 100  | 0.6884          | -0.1839        | -0.2354          | 0.5440             | 0.0514          | -311.2962      | -324.9433    | -2.0325         | -2.1221       |
+
+
+### Framework versions
+
+- PEFT 0.7.1
+- Transformers 4.36.2
+- Pytorch 2.3.0+cu121
+- Datasets 2.14.6
+- Tokenizers 0.15.2

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.5940336465835572,
+    "train_runtime": 8165.5058,
+    "train_samples": 21135,
+    "train_samples_per_second": 2.588,
+    "train_steps_per_second": 0.02
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.5940336465835572,
+    "train_runtime": 8165.5058,
+    "train_samples": 21135,
+    "train_samples_per_second": 2.588,
+    "train_steps_per_second": 0.02
+}

trainer_state.json

@@ -0,0 +1,284 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9992429977289932,
+  "eval_steps": 100,
+  "global_step": 165,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01,
+      "learning_rate": 2.9411764705882356e-07,
+      "logits/chosen": -2.4571897983551025,
+      "logits/rejected": -2.364729166030884,
+      "logps/chosen": -393.0217590332031,
+      "logps/rejected": -392.042236328125,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.9411764705882355e-06,
+      "logits/chosen": -2.4598512649536133,
+      "logits/rejected": -2.5237348079681396,
+      "logps/chosen": -319.2628479003906,
+      "logps/rejected": -363.0235290527344,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.4513888955116272,
+      "rewards/chosen": -0.005169111769646406,
+      "rewards/margins": 0.0021531255915760994,
+      "rewards/rejected": -0.007322237826883793,
+      "step": 10
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.994932636402032e-06,
+      "logits/chosen": -2.4682602882385254,
+      "logits/rejected": -2.506861925125122,
+      "logps/chosen": -310.16632080078125,
+      "logps/rejected": -378.22735595703125,
+      "loss": 0.6703,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 0.03243725374341011,
+      "rewards/margins": 0.057667654007673264,
+      "rewards/rejected": -0.02523040398955345,
+      "step": 20
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.905416503522124e-06,
+      "logits/chosen": -2.458878993988037,
+      "logits/rejected": -2.5221245288848877,
+      "logps/chosen": -329.98956298828125,
+      "logps/rejected": -392.0107421875,
+      "loss": 0.6549,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.022809982299804688,
+      "rewards/margins": 0.09913644939661026,
+      "rewards/rejected": -0.12194643169641495,
+      "step": 30
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.707922373336524e-06,
+      "logits/chosen": -2.3397133350372314,
+      "logits/rejected": -2.440927505493164,
+      "logps/chosen": -318.717041015625,
+      "logps/rejected": -400.6865539550781,
+      "loss": 0.6365,
+      "rewards/accuracies": 0.690625011920929,
+      "rewards/chosen": -0.11206980794668198,
+      "rewards/margins": 0.15860508382320404,
+      "rewards/rejected": -0.2706748843193054,
+      "step": 40
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.411315662967732e-06,
+      "logits/chosen": -2.3495843410491943,
+      "logits/rejected": -2.3737149238586426,
+      "logps/chosen": -344.6326599121094,
+      "logps/rejected": -397.68157958984375,
+      "loss": 0.627,
+      "rewards/accuracies": 0.6656249761581421,
+      "rewards/chosen": -0.17710785567760468,
+      "rewards/margins": 0.15342721343040466,
+      "rewards/rejected": -0.33053505420684814,
+      "step": 50
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.028910905897229e-06,
+      "logits/chosen": -2.30446720123291,
+      "logits/rejected": -2.3595499992370605,
+      "logps/chosen": -353.98602294921875,
+      "logps/rejected": -411.59515380859375,
+      "loss": 0.6064,
+      "rewards/accuracies": 0.715624988079071,
+      "rewards/chosen": -0.18608322739601135,
+      "rewards/margins": 0.23026804625988007,
+      "rewards/rejected": -0.4163512587547302,
+      "step": 60
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.577874068920446e-06,
+      "logits/chosen": -2.239515781402588,
+      "logits/rejected": -2.3043527603149414,
+      "logps/chosen": -348.39581298828125,
+      "logps/rejected": -414.6133728027344,
+      "loss": 0.5915,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": -0.22679059207439423,
+      "rewards/margins": 0.23025627434253693,
+      "rewards/rejected": -0.45704689621925354,
+      "step": 70
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.0784519801008546e-06,
+      "logits/chosen": -2.16733980178833,
+      "logits/rejected": -2.2091336250305176,
+      "logps/chosen": -339.76129150390625,
+      "logps/rejected": -419.670654296875,
+      "loss": 0.595,
+      "rewards/accuracies": 0.715624988079071,
+      "rewards/chosen": -0.2677019238471985,
+      "rewards/margins": 0.26828354597091675,
+      "rewards/rejected": -0.53598552942276,
+      "step": 80
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.553063458334059e-06,
+      "logits/chosen": -2.177290439605713,
+      "logits/rejected": -2.2133171558380127,
+      "logps/chosen": -337.5870666503906,
+      "logps/rejected": -410.07464599609375,
+      "loss": 0.5779,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": -0.21148499846458435,
+      "rewards/margins": 0.3073815107345581,
+      "rewards/rejected": -0.5188664793968201,
+      "step": 90
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.025292943281429e-06,
+      "logits/chosen": -2.137645721435547,
+      "logits/rejected": -2.2139902114868164,
+      "logps/chosen": -362.3526611328125,
+      "logps/rejected": -451.25543212890625,
+      "loss": 0.5749,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": -0.38997265696525574,
+      "rewards/margins": 0.3522176146507263,
+      "rewards/rejected": -0.7421903610229492,
+      "step": 100
+    },
+    {
+      "epoch": 0.61,
+      "eval_logits/chosen": -2.1221249103546143,
+      "eval_logits/rejected": -2.0325379371643066,
+      "eval_logps/chosen": -324.94329833984375,
+      "eval_logps/rejected": -311.2961730957031,
+      "eval_loss": 0.688376247882843,
+      "eval_rewards/accuracies": 0.5440000295639038,
+      "eval_rewards/chosen": -0.18393579125404358,
+      "eval_rewards/margins": 0.05143474042415619,
+      "eval_rewards/rejected": -0.23537051677703857,
+      "eval_runtime": 384.2989,
+      "eval_samples_per_second": 5.204,
+      "eval_steps_per_second": 0.651,
+      "step": 100
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.5188318011445907e-06,
+      "logits/chosen": -2.0991530418395996,
+      "logits/rejected": -2.1866981983184814,
+      "logps/chosen": -343.84478759765625,
+      "logps/rejected": -443.0243225097656,
+      "loss": 0.5695,
+      "rewards/accuracies": 0.753125011920929,
+      "rewards/chosen": -0.24366803467273712,
+      "rewards/margins": 0.37467828392982483,
+      "rewards/rejected": -0.6183463335037231,
+      "step": 110
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0564148305586296e-06,
+      "logits/chosen": -2.12058424949646,
+      "logits/rejected": -2.183859348297119,
+      "logps/chosen": -358.75238037109375,
+      "logps/rejected": -454.56982421875,
+      "loss": 0.5503,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": -0.3431812822818756,
+      "rewards/margins": 0.4134922921657562,
+      "rewards/rejected": -0.7566735148429871,
+      "step": 120
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.587997083462197e-07,
+      "logits/chosen": -2.1187376976013184,
+      "logits/rejected": -2.1680569648742676,
+      "logps/chosen": -377.47918701171875,
+      "logps/rejected": -470.26470947265625,
+      "loss": 0.5556,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.4671626091003418,
+      "rewards/margins": 0.353724867105484,
+      "rewards/rejected": -0.8208874464035034,
+      "step": 130
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.438351873250492e-07,
+      "logits/chosen": -2.0515809059143066,
+      "logits/rejected": -2.1326162815093994,
+      "logps/chosen": -359.46771240234375,
+      "logps/rejected": -465.7957458496094,
+      "loss": 0.5475,
+      "rewards/accuracies": 0.7906249761581421,
+      "rewards/chosen": -0.36117881536483765,
+      "rewards/margins": 0.44570302963256836,
+      "rewards/rejected": -0.8068817853927612,
+      "step": 140
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.2565987432367032e-07,
+      "logits/chosen": -2.070026159286499,
+      "logits/rejected": -2.1689717769622803,
+      "logps/chosen": -349.19305419921875,
+      "logps/rejected": -467.4447326660156,
+      "loss": 0.541,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.32100656628608704,
+      "rewards/margins": 0.48250946402549744,
+      "rewards/rejected": -0.8035160303115845,
+      "step": 150
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4067554877743861e-08,
+      "logits/chosen": -2.0285654067993164,
+      "logits/rejected": -2.1381633281707764,
+      "logps/chosen": -329.7977294921875,
+      "logps/rejected": -446.18780517578125,
+      "loss": 0.5574,
+      "rewards/accuracies": 0.778124988079071,
+      "rewards/chosen": -0.29442131519317627,
+      "rewards/margins": 0.48129719495773315,
+      "rewards/rejected": -0.7757185697555542,
+      "step": 160
+    },
+    {
+      "epoch": 1.0,
+      "step": 165,
+      "total_flos": 0.0,
+      "train_loss": 0.5940336465835572,
+      "train_runtime": 8165.5058,
+      "train_samples_per_second": 2.588,
+      "train_steps_per_second": 0.02
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 165,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}