Model save

README.md

@@ -0,0 +1,62 @@
+---
+license: llama3
+library_name: peft
+tags:
+- trl
+- dpo
+- generated_from_trainer
+base_model: Jackie999/llama3-sudo-5epochs-tofu_full_sft
+model-index:
+- name: llama3-sudo-dpo-instruct-5epochs-forget10-lora
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# llama3-sudo-dpo-instruct-5epochs-forget10-lora
+
+This model is a fine-tuned version of [Jackie999/llama3-sudo-5epochs-tofu_full_sft](https://huggingface.co/Jackie999/llama3-sudo-5epochs-tofu_full_sft) on an unknown dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-06
+- train_batch_size: 2
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 32
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 5
+
+### Training results
+
+
+
+### Framework versions
+
+- PEFT 0.12.0
+- Transformers 4.44.0
+- Pytorch 2.1.2
+- Datasets 3.0.0
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 5.0,
+    "total_flos": 0.0,
+    "train_loss": 0.40015009021759035,
+    "train_runtime": 1492.5212,
+    "train_samples": 800,
+    "train_samples_per_second": 2.68,
+    "train_steps_per_second": 0.084
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 5.0,
+    "total_flos": 0.0,
+    "train_loss": 0.40015009021759035,
+    "train_runtime": 1492.5212,
+    "train_samples": 800,
+    "train_samples_per_second": 2.68,
+    "train_steps_per_second": 0.084
+}

trainer_state.json

@@ -0,0 +1,237 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 5.0,
+  "eval_steps": 1000,
+  "global_step": 125,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.04,
+      "grad_norm": 0.2611100745548823,
+      "learning_rate": 3.846153846153847e-07,
+      "logits/chosen": -1.7501684427261353,
+      "logits/rejected": -1.5946102142333984,
+      "logps/chosen": -84.6928939819336,
+      "logps/rejected": -66.21642303466797,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 0.22439873610113092,
+      "learning_rate": 3.846153846153847e-06,
+      "logits/chosen": -1.7436565160751343,
+      "logits/rejected": -1.7198147773742676,
+      "logps/chosen": -75.07618713378906,
+      "logps/rejected": -78.49008178710938,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.6944444179534912,
+      "rewards/chosen": 0.0014147770125418901,
+      "rewards/margins": 0.001200773986056447,
+      "rewards/rejected": 0.00021400292462203652,
+      "step": 10
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 0.23238100037617523,
+      "learning_rate": 4.9519632010080765e-06,
+      "logits/chosen": -1.748544454574585,
+      "logits/rejected": -1.6868740320205688,
+      "logps/chosen": -76.10453796386719,
+      "logps/rejected": -74.7530288696289,
+      "loss": 0.6845,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.013784198090434074,
+      "rewards/margins": 0.017130162566900253,
+      "rewards/rejected": -0.00334596517495811,
+      "step": 20
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 0.2815020587025619,
+      "learning_rate": 4.721114089947181e-06,
+      "logits/chosen": -1.689690351486206,
+      "logits/rejected": -1.7480777502059937,
+      "logps/chosen": -69.04388427734375,
+      "logps/rejected": -83.00643920898438,
+      "loss": 0.6604,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.04013435170054436,
+      "rewards/margins": 0.0667264312505722,
+      "rewards/rejected": -0.02659207209944725,
+      "step": 30
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 0.5012169942018447,
+      "learning_rate": 4.316650805085068e-06,
+      "logits/chosen": -1.6957839727401733,
+      "logits/rejected": -1.7012145519256592,
+      "logps/chosen": -66.49241638183594,
+      "logps/rejected": -90.0240249633789,
+      "loss": 0.6028,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.0827840119600296,
+      "rewards/margins": 0.19734814763069153,
+      "rewards/rejected": -0.11456414312124252,
+      "step": 40
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 0.5738595020225901,
+      "learning_rate": 3.770188363116324e-06,
+      "logits/chosen": -1.6569753885269165,
+      "logits/rejected": -1.5992107391357422,
+      "logps/chosen": -59.36053466796875,
+      "logps/rejected": -114.33049011230469,
+      "loss": 0.4751,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.16371119022369385,
+      "rewards/margins": 0.5395737886428833,
+      "rewards/rejected": -0.37586259841918945,
+      "step": 50
+    },
+    {
+      "epoch": 2.4,
+      "grad_norm": 0.3468932614622257,
+      "learning_rate": 3.1244411954180677e-06,
+      "logits/chosen": -1.6431405544281006,
+      "logits/rejected": -1.4894822835922241,
+      "logps/chosen": -53.25394821166992,
+      "logps/rejected": -131.90545654296875,
+      "loss": 0.3531,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2888104021549225,
+      "rewards/margins": 0.8971719741821289,
+      "rewards/rejected": -0.6083616018295288,
+      "step": 60
+    },
+    {
+      "epoch": 2.8,
+      "grad_norm": 0.2456344580019736,
+      "learning_rate": 2.429884359310328e-06,
+      "logits/chosen": -1.4915359020233154,
+      "logits/rejected": -1.4094572067260742,
+      "logps/chosen": -45.98115539550781,
+      "logps/rejected": -179.42784118652344,
+      "loss": 0.2706,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3235810399055481,
+      "rewards/margins": 1.3730812072753906,
+      "rewards/rejected": -1.0495002269744873,
+      "step": 70
+    },
+    {
+      "epoch": 3.2,
+      "grad_norm": 0.21353252313421264,
+      "learning_rate": 1.7408081372259633e-06,
+      "logits/chosen": -1.4443209171295166,
+      "logits/rejected": -1.3889185190200806,
+      "logps/chosen": -41.034461975097656,
+      "logps/rejected": -219.5689697265625,
+      "loss": 0.2546,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3277435898780823,
+      "rewards/margins": 1.7629969120025635,
+      "rewards/rejected": -1.435253381729126,
+      "step": 80
+    },
+    {
+      "epoch": 3.6,
+      "grad_norm": 0.147745405217318,
+      "learning_rate": 1.1110744174509952e-06,
+      "logits/chosen": -1.5578348636627197,
+      "logits/rejected": -1.5076854228973389,
+      "logps/chosen": -40.28083801269531,
+      "logps/rejected": -228.35556030273438,
+      "loss": 0.2351,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3738650679588318,
+      "rewards/margins": 1.9237592220306396,
+      "rewards/rejected": -1.5498943328857422,
+      "step": 90
+    },
+    {
+      "epoch": 4.0,
+      "grad_norm": 0.14178606869515006,
+      "learning_rate": 5.899065604459814e-07,
+      "logits/chosen": -1.4096633195877075,
+      "logits/rejected": -1.4153110980987549,
+      "logps/chosen": -38.80024719238281,
+      "logps/rejected": -268.9517822265625,
+      "loss": 0.2152,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3712707459926605,
+      "rewards/margins": 2.266303300857544,
+      "rewards/rejected": -1.895032525062561,
+      "step": 100
+    },
+    {
+      "epoch": 4.4,
+      "grad_norm": 0.15997409765623294,
+      "learning_rate": 2.1804183734670277e-07,
+      "logits/chosen": -1.4095523357391357,
+      "logits/rejected": -1.4386852979660034,
+      "logps/chosen": -37.86528778076172,
+      "logps/rejected": -281.3670654296875,
+      "loss": 0.2041,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.34532371163368225,
+      "rewards/margins": 2.3692967891693115,
+      "rewards/rejected": -2.023972988128662,
+      "step": 110
+    },
+    {
+      "epoch": 4.8,
+      "grad_norm": 0.12276969074858735,
+      "learning_rate": 2.454718665888589e-08,
+      "logits/chosen": -1.541815161705017,
+      "logits/rejected": -1.4467297792434692,
+      "logps/chosen": -39.434669494628906,
+      "logps/rejected": -230.53213500976562,
+      "loss": 0.2316,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4158347249031067,
+      "rewards/margins": 2.008779287338257,
+      "rewards/rejected": -1.592944622039795,
+      "step": 120
+    },
+    {
+      "epoch": 5.0,
+      "step": 125,
+      "total_flos": 0.0,
+      "train_loss": 0.40015009021759035,
+      "train_runtime": 1492.5212,
+      "train_samples_per_second": 2.68,
+      "train_steps_per_second": 0.084
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 125,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}