Model save

README.md

@@ -0,0 +1,74 @@
+---
+license: apache-2.0
+library_name: peft
+tags:
+- trl
+- dpo
+- generated_from_trainer
+base_model: mistralai/Mistral-7B-v0.1
+model-index:
+- name: zephyr-7b-gpo-update3-i1
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# zephyr-7b-gpo-update3-i1
+
+This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.0574
+- Rewards/chosen: -0.0003
+- Rewards/rejected: 0.0038
+- Rewards/accuracies: 0.3765
+- Rewards/margins: -0.0041
+- Logps/rejected: -254.1840
+- Logps/chosen: -266.7596
+- Logits/rejected: -1.8151
+- Logits/chosen: -1.9709
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-06
+- train_batch_size: 2
+- eval_batch_size: 2
+- seed: 42
+- distributed_type: multi-GPU
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 4
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.0013        | 0.4   | 100  | 0.0537          | 0.0            | 0.0              | 0.0                | 0.0             | -254.9398      | -266.6976    | -1.8067         | -1.9618       |
+| 0.0013        | 0.8   | 200  | 0.0575          | -0.0013        | 0.0029           | 0.3800             | -0.0041         | -254.3691      | -266.9557    | -1.8139         | -1.9695       |
+
+
+### Framework versions
+
+- PEFT 0.7.1
+- Transformers 4.36.2
+- Pytorch 2.1.2+cu121
+- Datasets 2.14.6
+- Tokenizers 0.15.2

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ab63b8a7f3809338c650ccfd5464a84ac38fcd15e4aefbf270f009e2e8301b2c
+oid sha256:d9d2f25b332c4650a67eac7e3ca2b1bf46300c95604dee259941128783de35e6
 size 671150064

all_results.json

@@ -0,0 +1,21 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": -1.9709315299987793,
+    "eval_logits/rejected": -1.8150750398635864,
+    "eval_logps/chosen": -266.7596130371094,
+    "eval_logps/rejected": -254.18397521972656,
+    "eval_loss": 0.057394202798604965,
+    "eval_rewards/accuracies": 0.3765000104904175,
+    "eval_rewards/chosen": -0.0003101456386502832,
+    "eval_rewards/margins": -0.004089393652975559,
+    "eval_rewards/rejected": 0.003779248334467411,
+    "eval_runtime": 709.1028,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 2.82,
+    "eval_steps_per_second": 1.41,
+    "train_loss": 0.0016641309279948472,
+    "train_runtime": 2302.1859,
+    "train_samples": 61135,
+    "train_samples_per_second": 0.434,
+    "train_steps_per_second": 0.109
+}

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": -1.9709315299987793,
+    "eval_logits/rejected": -1.8150750398635864,
+    "eval_logps/chosen": -266.7596130371094,
+    "eval_logps/rejected": -254.18397521972656,
+    "eval_loss": 0.057394202798604965,
+    "eval_rewards/accuracies": 0.3765000104904175,
+    "eval_rewards/chosen": -0.0003101456386502832,
+    "eval_rewards/margins": -0.004089393652975559,
+    "eval_rewards/rejected": 0.003779248334467411,
+    "eval_runtime": 709.1028,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 2.82,
+    "eval_steps_per_second": 1.41
+}

runs/Apr08_23-22-40_gpu4-119-5/events.out.tfevents.1712582624.gpu4-119-5.42432.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8cb34f378552a24a218061d0d4164fcdab48d7e751f857f22b3d490d56cbb57a
-size 19225
+oid sha256:a702b666d34efb4625a58f041788f819432b68e6a49c86b3acb93aa2d4f7bb4c
+size 22749

runs/Apr08_23-22-40_gpu4-119-5/events.out.tfevents.1712585635.gpu4-119-5.42432.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:210bf1be03dda53fd4018949430cf6ddd5ab27ca2d6abb0cf8aae3e313086cb1
+size 828

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.0016641309279948472,
+    "train_runtime": 2302.1859,
+    "train_samples": 61135,
+    "train_samples_per_second": 0.434,
+    "train_steps_per_second": 0.109
+}

trainer_state.json

@@ -0,0 +1,426 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 100,
+  "global_step": 250,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.0000000000000002e-07,
+      "logits/chosen": -1.8503975868225098,
+      "logits/rejected": -1.8503975868225098,
+      "logps/chosen": 0.0,
+      "logps/rejected": 0.0,
+      "loss": 0.0011,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.0000000000000003e-06,
+      "logits/chosen": -1.8588156700134277,
+      "logits/rejected": -1.8588156700134277,
+      "logps/chosen": 0.0,
+      "logps/rejected": 0.0,
+      "loss": 0.0019,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 10
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.000000000000001e-06,
+      "logits/chosen": -1.970517873764038,
+      "logits/rejected": -1.970517873764038,
+      "logps/chosen": 0.0,
+      "logps/rejected": 0.0,
+      "loss": 0.0018,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 20
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.993910125649561e-06,
+      "logits/chosen": -1.9209930896759033,
+      "logits/rejected": -1.9209930896759033,
+      "logps/chosen": 0.0,
+      "logps/rejected": 0.0,
+      "loss": 0.0013,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 30
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.9453690018345144e-06,
+      "logits/chosen": -1.883547067642212,
+      "logits/rejected": -1.883547067642212,
+      "logps/chosen": 0.0,
+      "logps/rejected": 0.0,
+      "loss": 0.0019,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 40
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.849231551964771e-06,
+      "logits/chosen": -1.9128715991973877,
+      "logits/rejected": -1.9128715991973877,
+      "logps/chosen": 0.0,
+      "logps/rejected": 0.0,
+      "loss": 0.0013,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 50
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.707368982147318e-06,
+      "logits/chosen": -2.0107295513153076,
+      "logits/rejected": -2.0107295513153076,
+      "logps/chosen": 0.0,
+      "logps/rejected": 0.0,
+      "loss": 0.0019,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 60
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.522542485937369e-06,
+      "logits/chosen": -1.9920228719711304,
+      "logits/rejected": -1.9920228719711304,
+      "logps/chosen": 0.0,
+      "logps/rejected": 0.0,
+      "loss": 0.0014,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 70
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.2983495008466285e-06,
+      "logits/chosen": -1.8801155090332031,
+      "logits/rejected": -1.8801155090332031,
+      "logps/chosen": 0.0,
+      "logps/rejected": 0.0,
+      "loss": 0.0024,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 80
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.039153688314146e-06,
+      "logits/chosen": -2.050198793411255,
+      "logits/rejected": -2.050198793411255,
+      "logps/chosen": 0.0,
+      "logps/rejected": 0.0,
+      "loss": 0.0021,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 90
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7500000000000005e-06,
+      "logits/chosen": -1.8852717876434326,
+      "logits/rejected": -1.8852717876434326,
+      "logps/chosen": 0.0,
+      "logps/rejected": 0.0,
+      "loss": 0.0013,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 100
+    },
+    {
+      "epoch": 0.4,
+      "eval_logits/chosen": -1.9617642164230347,
+      "eval_logits/rejected": -1.8066532611846924,
+      "eval_logps/chosen": -266.6976013183594,
+      "eval_logps/rejected": -254.9398193359375,
+      "eval_loss": 0.053734518587589264,
+      "eval_rewards/accuracies": 0.0,
+      "eval_rewards/chosen": 0.0,
+      "eval_rewards/margins": 0.0,
+      "eval_rewards/rejected": 0.0,
+      "eval_runtime": 702.6753,
+      "eval_samples_per_second": 2.846,
+      "eval_steps_per_second": 1.423,
+      "step": 100
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.436516483539781e-06,
+      "logits/chosen": -1.731688141822815,
+      "logits/rejected": -1.731688141822815,
+      "logps/chosen": 0.0,
+      "logps/rejected": 0.0,
+      "loss": 0.0012,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 110
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.1048047389991693e-06,
+      "logits/chosen": -1.8530235290527344,
+      "logits/rejected": -1.8530235290527344,
+      "logps/chosen": 0.0,
+      "logps/rejected": 0.0,
+      "loss": 0.0013,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 120
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.761321158169134e-06,
+      "logits/chosen": -2.0225424766540527,
+      "logits/rejected": -2.0225424766540527,
+      "logps/chosen": 0.0,
+      "logps/rejected": 0.0,
+      "loss": 0.0016,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 130
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.4127512582437486e-06,
+      "logits/chosen": -1.8995482921600342,
+      "logits/rejected": -1.8995482921600342,
+      "logps/chosen": 0.0,
+      "logps/rejected": 0.0,
+      "loss": 0.0014,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 140
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0658795558326745e-06,
+      "logits/chosen": -1.86004638671875,
+      "logits/rejected": -1.8391777276992798,
+      "logps/chosen": -4.896004676818848,
+      "logps/rejected": -1.6084611415863037,
+      "loss": 0.0016,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 150
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7274575140626318e-06,
+      "logits/chosen": -2.013669490814209,
+      "logits/rejected": -2.013669490814209,
+      "logps/chosen": 0.0,
+      "logps/rejected": 0.0,
+      "loss": 0.0019,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 160
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4040721330273063e-06,
+      "logits/chosen": -1.8206443786621094,
+      "logits/rejected": -1.8206443786621094,
+      "logps/chosen": 0.0,
+      "logps/rejected": 0.0,
+      "loss": 0.0019,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 170
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.1020177413231334e-06,
+      "logits/chosen": -1.9178645610809326,
+      "logits/rejected": -1.8847808837890625,
+      "logps/chosen": -10.10865306854248,
+      "logps/rejected": -3.267775297164917,
+      "loss": 0.0015,
+      "rewards/accuracies": 0.02500000037252903,
+      "rewards/chosen": -0.0017203291645273566,
+      "rewards/margins": 0.0016426773509010673,
+      "rewards/rejected": -0.003363006515428424,
+      "step": 180
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.271734841028553e-07,
+      "logits/chosen": -1.8469321727752686,
+      "logits/rejected": -1.8469321727752686,
+      "logps/chosen": 0.0,
+      "logps/rejected": 0.0,
+      "loss": 0.0021,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 190
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.848888922025553e-07,
+      "logits/chosen": -1.9015337228775024,
+      "logits/rejected": -1.9031871557235718,
+      "logps/chosen": -5.545676231384277,
+      "logps/rejected": -6.33315372467041,
+      "loss": 0.0013,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0002652378170751035,
+      "rewards/margins": -0.0003098316374234855,
+      "rewards/rejected": 0.000575069454498589,
+      "step": 200
+    },
+    {
+      "epoch": 0.8,
+      "eval_logits/chosen": -1.969506859779358,
+      "eval_logits/rejected": -1.813860297203064,
+      "eval_logps/chosen": -266.95574951171875,
+      "eval_logps/rejected": -254.3690948486328,
+      "eval_loss": 0.057529229670763016,
+      "eval_rewards/accuracies": 0.3799999952316284,
+      "eval_rewards/chosen": -0.001290707616135478,
+      "eval_rewards/margins": -0.004144246224313974,
+      "eval_rewards/rejected": 0.00285353884100914,
+      "eval_runtime": 704.8428,
+      "eval_samples_per_second": 2.838,
+      "eval_steps_per_second": 1.419,
+      "step": 200
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.798797596089351e-07,
+      "logits/chosen": -1.8618186712265015,
+      "logits/rejected": -1.8618186712265015,
+      "logps/chosen": 0.0,
+      "logps/rejected": 0.0,
+      "loss": 0.0012,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 210
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.1613635589349756e-07,
+      "logits/chosen": -2.0101757049560547,
+      "logits/rejected": -2.0101757049560547,
+      "logps/chosen": 0.0,
+      "logps/rejected": 0.0,
+      "loss": 0.0018,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 220
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 9.684576015420277e-08,
+      "logits/chosen": -1.938612699508667,
+      "logits/rejected": -1.9380161762237549,
+      "logps/chosen": -1.706319808959961,
+      "logps/rejected": -1.771810531616211,
+      "loss": 0.0022,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": -0.0014353947481140494,
+      "rewards/margins": -0.0004587741568684578,
+      "rewards/rejected": -0.0009766205912455916,
+      "step": 230
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.4329828146074096e-08,
+      "logits/chosen": -2.1376256942749023,
+      "logits/rejected": -2.1376256942749023,
+      "logps/chosen": 0.0,
+      "logps/rejected": 0.0,
+      "loss": 0.0016,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 240
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 0.0,
+      "logits/chosen": -1.9790098667144775,
+      "logits/rejected": -1.9790098667144775,
+      "logps/chosen": 0.0,
+      "logps/rejected": 0.0,
+      "loss": 0.0018,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 250
+    },
+    {
+      "epoch": 1.0,
+      "step": 250,
+      "total_flos": 0.0,
+      "train_loss": 0.0016641309279948472,
+      "train_runtime": 2302.1859,
+      "train_samples_per_second": 0.434,
+      "train_steps_per_second": 0.109
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 250,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}