Model save

README.md

@@ -0,0 +1,68 @@
+---
+base_model: cognitivecomputations/dolphin-2.9.4-llama3.1-8b
+library_name: transformers
+model_name: datagen_round_0_rpo
+tags:
+- generated_from_trainer
+- trl
+- dpo
+licence: license
+---
+
+# Model Card for datagen_round_0_rpo
+
+This model is a fine-tuned version of [cognitivecomputations/dolphin-2.9.4-llama3.1-8b](https://huggingface.co/cognitivecomputations/dolphin-2.9.4-llama3.1-8b).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="ydeng9/datagen_round_0_rpo", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/yihedeng9/huggingface/runs/az7oyxn2)
+
+This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290).
+
+### Framework versions
+
+- TRL: 0.12.0
+- Transformers: 4.46.2
+- Pytorch: 2.4.0
+- Datasets: 3.0.0
+- Tokenizers: 0.20.3
+
+## Citations
+
+Cite DPO as:
+
+```bibtex
+@inproceedings{rafailov2023direct,
+    title        = {{Direct Preference Optimization: Your Language Model is Secretly a Reward Model}},
+    author       = {Rafael Rafailov and Archit Sharma and Eric Mitchell and Christopher D. Manning and Stefano Ermon and Chelsea Finn},
+    year         = 2023,
+    booktitle    = {Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023},
+    url          = {http://papers.nips.cc/paper_files/paper/2023/hash/a85b405ed65c6477a4fe8302b5e06ce7-Abstract-Conference.html},
+    editor       = {Alice Oh and Tristan Naumann and Amir Globerson and Kate Saenko and Moritz Hardt and Sergey Levine},
+}
+```
+
+Cite TRL as:
+    
+```bibtex
+@misc{vonwerra2022trl,
+	title        = {{TRL: Transformer Reinforcement Learning}},
+	author       = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallouédec},
+	year         = 2020,
+	journal      = {GitHub repository},
+	publisher    = {GitHub},
+	howpublished = {\url{https://github.com/huggingface/trl}}
+}
+```

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9990141307919816,
+    "total_flos": 0.0,
+    "train_loss": 1.0731394717567846,
+    "train_runtime": 3128.4011,
+    "train_samples": 24339,
+    "train_samples_per_second": 7.78,
+    "train_steps_per_second": 0.121
+}

generation_config.json

@@ -0,0 +1,14 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009,
+    128256
+  ],
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.46.2"
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9990141307919816,
+    "total_flos": 0.0,
+    "train_loss": 1.0731394717567846,
+    "train_runtime": 3128.4011,
+    "train_samples": 24339,
+    "train_samples_per_second": 7.78,
+    "train_steps_per_second": 0.121
+}

trainer_state.json

@@ -0,0 +1,717 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9990141307919816,
+  "eval_steps": 100,
+  "global_step": 380,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0026289845547157412,
+      "grad_norm": 430.5451731957118,
+      "learning_rate": 1.3157894736842104e-08,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -1.21875,
+      "logps/chosen": -189.0,
+      "logps/rejected": -255.0,
+      "loss": 2.0842,
+      "nll_loss": 2.96875,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.02628984554715741,
+      "grad_norm": 380.94348914412785,
+      "learning_rate": 1.3157894736842104e-07,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -1.25,
+      "logps/chosen": -209.0,
+      "logps/rejected": -338.0,
+      "loss": 2.0892,
+      "nll_loss": 2.953125,
+      "rewards/accuracies": 0.2916666567325592,
+      "rewards/chosen": 0.00933837890625,
+      "rewards/margins": -0.00396728515625,
+      "rewards/rejected": 0.0133056640625,
+      "step": 10
+    },
+    {
+      "epoch": 0.05257969109431482,
+      "grad_norm": 192.52773167988067,
+      "learning_rate": 2.631578947368421e-07,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -1.203125,
+      "logps/chosen": -206.0,
+      "logps/rejected": -288.0,
+      "loss": 2.0101,
+      "nll_loss": 2.546875,
+      "rewards/accuracies": 0.33125001192092896,
+      "rewards/chosen": 0.3125,
+      "rewards/margins": -0.1083984375,
+      "rewards/rejected": 0.419921875,
+      "step": 20
+    },
+    {
+      "epoch": 0.07886953664147223,
+      "grad_norm": 29.353212473973965,
+      "learning_rate": 3.9473684210526315e-07,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -1.2578125,
+      "logps/chosen": -134.0,
+      "logps/rejected": -182.0,
+      "loss": 1.7046,
+      "nll_loss": 1.625,
+      "rewards/accuracies": 0.33125001192092896,
+      "rewards/chosen": 1.1171875,
+      "rewards/margins": -0.353515625,
+      "rewards/rejected": 1.4765625,
+      "step": 30
+    },
+    {
+      "epoch": 0.10515938218862964,
+      "grad_norm": 13.110128400909407,
+      "learning_rate": 4.999578104083306e-07,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -1.3359375,
+      "logps/chosen": -87.0,
+      "logps/rejected": -123.5,
+      "loss": 1.4689,
+      "nll_loss": 1.015625,
+      "rewards/accuracies": 0.4312500059604645,
+      "rewards/chosen": 1.4921875,
+      "rewards/margins": -0.296875,
+      "rewards/rejected": 1.7890625,
+      "step": 40
+    },
+    {
+      "epoch": 0.13144922773578704,
+      "grad_norm": 12.991424819775641,
+      "learning_rate": 4.984826693294873e-07,
+      "logits/chosen": -1.203125,
+      "logits/rejected": -1.2265625,
+      "logps/chosen": -69.0,
+      "logps/rejected": -121.0,
+      "loss": 1.4155,
+      "nll_loss": 0.87890625,
+      "rewards/accuracies": 0.4124999940395355,
+      "rewards/chosen": 1.5703125,
+      "rewards/margins": -0.3984375,
+      "rewards/rejected": 1.96875,
+      "step": 50
+    },
+    {
+      "epoch": 0.15773907328294445,
+      "grad_norm": 48.0340809183726,
+      "learning_rate": 4.949122667718934e-07,
+      "logits/chosen": -1.125,
+      "logits/rejected": -1.1640625,
+      "logps/chosen": -83.0,
+      "logps/rejected": -189.0,
+      "loss": 1.3282,
+      "nll_loss": 1.0546875,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 1.3828125,
+      "rewards/margins": -0.09326171875,
+      "rewards/rejected": 1.4765625,
+      "step": 60
+    },
+    {
+      "epoch": 0.18402891883010186,
+      "grad_norm": 18.079466671824335,
+      "learning_rate": 4.892767091689785e-07,
+      "logits/chosen": -1.0234375,
+      "logits/rejected": -1.1015625,
+      "logps/chosen": -93.0,
+      "logps/rejected": -253.0,
+      "loss": 1.0846,
+      "nll_loss": 1.0,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 1.5390625,
+      "rewards/margins": 0.76171875,
+      "rewards/rejected": 0.7734375,
+      "step": 70
+    },
+    {
+      "epoch": 0.21031876437725927,
+      "grad_norm": 22.52030988227434,
+      "learning_rate": 4.816235168037004e-07,
+      "logits/chosen": -1.0078125,
+      "logits/rejected": -1.1015625,
+      "logps/chosen": -89.5,
+      "logps/rejected": -324.0,
+      "loss": 1.0849,
+      "nll_loss": 0.92578125,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": 1.5546875,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": -0.1484375,
+      "step": 80
+    },
+    {
+      "epoch": 0.2366086099244167,
+      "grad_norm": 23.44831636466729,
+      "learning_rate": 4.720172231068844e-07,
+      "logits/chosen": -1.0390625,
+      "logits/rejected": -1.1171875,
+      "logps/chosen": -88.0,
+      "logps/rejected": -318.0,
+      "loss": 1.0445,
+      "nll_loss": 0.99609375,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 1.6015625,
+      "rewards/margins": 1.6015625,
+      "rewards/rejected": -0.002410888671875,
+      "step": 90
+    },
+    {
+      "epoch": 0.2628984554715741,
+      "grad_norm": 15.553732102036754,
+      "learning_rate": 4.605388304968914e-07,
+      "logits/chosen": -1.0625,
+      "logits/rejected": -1.1796875,
+      "logps/chosen": -85.0,
+      "logps/rejected": -492.0,
+      "loss": 1.0258,
+      "nll_loss": 1.0625,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 1.4765625,
+      "rewards/margins": 3.203125,
+      "rewards/rejected": -1.734375,
+      "step": 100
+    },
+    {
+      "epoch": 0.2628984554715741,
+      "eval_logits/chosen": -1.1171875,
+      "eval_logits/rejected": -1.2265625,
+      "eval_logps/chosen": -84.0,
+      "eval_logps/rejected": -420.0,
+      "eval_loss": 1.0434179306030273,
+      "eval_nll_loss": 1.203125,
+      "eval_rewards/accuracies": 0.7200000286102295,
+      "eval_rewards/chosen": 1.3828125,
+      "eval_rewards/margins": 2.375,
+      "eval_rewards/rejected": -0.99609375,
+      "eval_runtime": 9.2898,
+      "eval_samples_per_second": 21.529,
+      "eval_steps_per_second": 2.691,
+      "step": 100
+    },
+    {
+      "epoch": 0.2891883010187315,
+      "grad_norm": 13.573028085768115,
+      "learning_rate": 4.472851273490984e-07,
+      "logits/chosen": -1.046875,
+      "logits/rejected": -1.1171875,
+      "logps/chosen": -82.5,
+      "logps/rejected": -246.0,
+      "loss": 1.0418,
+      "nll_loss": 0.921875,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 1.6796875,
+      "rewards/margins": 1.1640625,
+      "rewards/rejected": 0.515625,
+      "step": 110
+    },
+    {
+      "epoch": 0.3154781465658889,
+      "grad_norm": 24.07297014456965,
+      "learning_rate": 4.323678718546552e-07,
+      "logits/chosen": -1.0625,
+      "logits/rejected": -1.125,
+      "logps/chosen": -79.0,
+      "logps/rejected": -380.0,
+      "loss": 0.9516,
+      "nll_loss": 0.921875,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 1.625,
+      "rewards/margins": 2.203125,
+      "rewards/rejected": -0.5859375,
+      "step": 120
+    },
+    {
+      "epoch": 0.3417679921130463,
+      "grad_norm": 13.788627232860541,
+      "learning_rate": 4.159128496504053e-07,
+      "logits/chosen": -1.0234375,
+      "logits/rejected": -1.0859375,
+      "logps/chosen": -82.5,
+      "logps/rejected": -402.0,
+      "loss": 0.9785,
+      "nll_loss": 0.96484375,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 1.5703125,
+      "rewards/margins": 2.515625,
+      "rewards/rejected": -0.9453125,
+      "step": 130
+    },
+    {
+      "epoch": 0.3680578376602037,
+      "grad_norm": 56.02055058840433,
+      "learning_rate": 3.9805881316624503e-07,
+      "logits/chosen": -0.98046875,
+      "logits/rejected": -1.0703125,
+      "logps/chosen": -94.0,
+      "logps/rejected": -364.0,
+      "loss": 0.9971,
+      "nll_loss": 1.015625,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 1.5546875,
+      "rewards/margins": 2.078125,
+      "rewards/rejected": -0.515625,
+      "step": 140
+    },
+    {
+      "epoch": 0.39434768320736113,
+      "grad_norm": 13.013669525516791,
+      "learning_rate": 3.78956311633581e-07,
+      "logits/chosen": -0.984375,
+      "logits/rejected": -1.1015625,
+      "logps/chosen": -80.5,
+      "logps/rejected": -478.0,
+      "loss": 0.9852,
+      "nll_loss": 0.93359375,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 1.609375,
+      "rewards/margins": 2.84375,
+      "rewards/rejected": -1.2421875,
+      "step": 150
+    },
+    {
+      "epoch": 0.42063752875451854,
+      "grad_norm": 40.15490393114984,
+      "learning_rate": 3.587664216205183e-07,
+      "logits/chosen": -1.0078125,
+      "logits/rejected": -1.078125,
+      "logps/chosen": -66.0,
+      "logps/rejected": -388.0,
+      "loss": 0.9339,
+      "nll_loss": 0.80859375,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 1.5859375,
+      "rewards/margins": 2.46875,
+      "rewards/rejected": -0.87890625,
+      "step": 160
+    },
+    {
+      "epoch": 0.44692737430167595,
+      "grad_norm": 17.30322447545255,
+      "learning_rate": 3.376593887981886e-07,
+      "logits/chosen": -1.015625,
+      "logits/rejected": -1.0859375,
+      "logps/chosen": -93.5,
+      "logps/rejected": -464.0,
+      "loss": 0.9452,
+      "nll_loss": 0.94921875,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 1.6015625,
+      "rewards/margins": 3.0,
+      "rewards/rejected": -1.40625,
+      "step": 170
+    },
+    {
+      "epoch": 0.4732172198488334,
+      "grad_norm": 27.318196890495926,
+      "learning_rate": 3.1581319239114976e-07,
+      "logits/chosen": -0.984375,
+      "logits/rejected": -1.0390625,
+      "logps/chosen": -85.0,
+      "logps/rejected": -416.0,
+      "loss": 0.9577,
+      "nll_loss": 0.94140625,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 1.6953125,
+      "rewards/margins": 2.578125,
+      "rewards/rejected": -0.88671875,
+      "step": 180
+    },
+    {
+      "epoch": 0.4995070653959908,
+      "grad_norm": 19.71975113316725,
+      "learning_rate": 2.934120444167326e-07,
+      "logits/chosen": -0.9609375,
+      "logits/rejected": -1.0390625,
+      "logps/chosen": -71.5,
+      "logps/rejected": -456.0,
+      "loss": 0.9496,
+      "nll_loss": 0.8203125,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 1.6484375,
+      "rewards/margins": 3.125,
+      "rewards/rejected": -1.484375,
+      "step": 190
+    },
+    {
+      "epoch": 0.5257969109431482,
+      "grad_norm": 14.910685294534735,
+      "learning_rate": 2.706448363680831e-07,
+      "logits/chosen": -0.98046875,
+      "logits/rejected": -1.046875,
+      "logps/chosen": -90.0,
+      "logps/rejected": -402.0,
+      "loss": 0.9286,
+      "nll_loss": 0.9140625,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": 1.7421875,
+      "rewards/margins": 2.78125,
+      "rewards/rejected": -1.0390625,
+      "step": 200
+    },
+    {
+      "epoch": 0.5257969109431482,
+      "eval_logits/chosen": -1.046875,
+      "eval_logits/rejected": -1.140625,
+      "eval_logps/chosen": -83.0,
+      "eval_logps/rejected": -648.0,
+      "eval_loss": 0.9766796827316284,
+      "eval_nll_loss": 1.1640625,
+      "eval_rewards/accuracies": 0.7200000286102295,
+      "eval_rewards/chosen": 1.390625,
+      "eval_rewards/margins": 4.6875,
+      "eval_rewards/rejected": -3.28125,
+      "eval_runtime": 8.9047,
+      "eval_samples_per_second": 22.46,
+      "eval_steps_per_second": 2.808,
+      "step": 200
+    },
+    {
+      "epoch": 0.5520867564903056,
+      "grad_norm": 12.064748996822903,
+      "learning_rate": 2.477035464388184e-07,
+      "logits/chosen": -0.96875,
+      "logits/rejected": -1.078125,
+      "logps/chosen": -84.0,
+      "logps/rejected": -636.0,
+      "loss": 0.9051,
+      "nll_loss": 0.85546875,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": 1.7734375,
+      "rewards/margins": 4.59375,
+      "rewards/rejected": -2.8125,
+      "step": 210
+    },
+    {
+      "epoch": 0.578376602037463,
+      "grad_norm": 123.10630313220432,
+      "learning_rate": 2.2478162071993296e-07,
+      "logits/chosen": -0.98828125,
+      "logits/rejected": -1.1171875,
+      "logps/chosen": -61.75,
+      "logps/rejected": -764.0,
+      "loss": 0.8841,
+      "nll_loss": 0.85546875,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 1.484375,
+      "rewards/margins": 6.0625,
+      "rewards/rejected": -4.5625,
+      "step": 220
+    },
+    {
+      "epoch": 0.6046664475846204,
+      "grad_norm": 26.672092273760438,
+      "learning_rate": 2.0207234201906545e-07,
+      "logits/chosen": -0.97265625,
+      "logits/rejected": -1.09375,
+      "logps/chosen": -67.0,
+      "logps/rejected": -592.0,
+      "loss": 0.9282,
+      "nll_loss": 0.8203125,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 1.5,
+      "rewards/margins": 4.46875,
+      "rewards/rejected": -2.953125,
+      "step": 230
+    },
+    {
+      "epoch": 0.6309562931317778,
+      "grad_norm": 61.7684539616632,
+      "learning_rate": 1.7976720005660767e-07,
+      "logits/chosen": -0.96875,
+      "logits/rejected": -1.0859375,
+      "logps/chosen": -80.0,
+      "logps/rejected": -632.0,
+      "loss": 0.9369,
+      "nll_loss": 0.828125,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": 1.6015625,
+      "rewards/margins": 4.5625,
+      "rewards/rejected": -2.96875,
+      "step": 240
+    },
+    {
+      "epoch": 0.6572461386789352,
+      "grad_norm": 11.575062050232066,
+      "learning_rate": 1.5805427678152674e-07,
+      "logits/chosen": -0.96484375,
+      "logits/rejected": -1.078125,
+      "logps/chosen": -74.0,
+      "logps/rejected": -576.0,
+      "loss": 0.9185,
+      "nll_loss": 0.90234375,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": 1.6484375,
+      "rewards/margins": 4.21875,
+      "rewards/rejected": -2.578125,
+      "step": 250
+    },
+    {
+      "epoch": 0.6835359842260926,
+      "grad_norm": 18.395086537639692,
+      "learning_rate": 1.371166604222777e-07,
+      "logits/chosen": -0.9296875,
+      "logits/rejected": -1.046875,
+      "logps/chosen": -72.5,
+      "logps/rejected": -510.0,
+      "loss": 0.9474,
+      "nll_loss": 0.8359375,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 1.5625,
+      "rewards/margins": 3.65625,
+      "rewards/rejected": -2.09375,
+      "step": 260
+    },
+    {
+      "epoch": 0.70982582977325,
+      "grad_norm": 66.67533840049661,
+      "learning_rate": 1.1713090164588606e-07,
+      "logits/chosen": -0.9609375,
+      "logits/rejected": -1.046875,
+      "logps/chosen": -85.5,
+      "logps/rejected": -568.0,
+      "loss": 0.9435,
+      "nll_loss": 0.93359375,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": 1.6484375,
+      "rewards/margins": 3.984375,
+      "rewards/rejected": -2.328125,
+      "step": 270
+    },
+    {
+      "epoch": 0.7361156753204074,
+      "grad_norm": 13.452197354246106,
+      "learning_rate": 9.826552484321085e-08,
+      "logits/chosen": -0.92578125,
+      "logits/rejected": -1.0,
+      "logps/chosen": -71.0,
+      "logps/rejected": -380.0,
+      "loss": 0.9612,
+      "nll_loss": 0.83203125,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 1.6640625,
+      "rewards/margins": 2.46875,
+      "rewards/rejected": -0.8046875,
+      "step": 280
+    },
+    {
+      "epoch": 0.7624055208675649,
+      "grad_norm": 14.445709675738982,
+      "learning_rate": 8.067960709356478e-08,
+      "logits/chosen": -0.9296875,
+      "logits/rejected": -1.0,
+      "logps/chosen": -78.5,
+      "logps/rejected": -382.0,
+      "loss": 0.9563,
+      "nll_loss": 0.890625,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": 1.6484375,
+      "rewards/margins": 2.46875,
+      "rewards/rejected": -0.82421875,
+      "step": 290
+    },
+    {
+      "epoch": 0.7886953664147223,
+      "grad_norm": 12.432822027887278,
+      "learning_rate": 6.452143679117964e-08,
+      "logits/chosen": -0.921875,
+      "logits/rejected": -1.0,
+      "logps/chosen": -74.0,
+      "logps/rejected": -414.0,
+      "loss": 0.9449,
+      "nll_loss": 0.796875,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": 1.609375,
+      "rewards/margins": 2.46875,
+      "rewards/rejected": -0.85546875,
+      "step": 300
+    },
+    {
+      "epoch": 0.7886953664147223,
+      "eval_logits/chosen": -0.99609375,
+      "eval_logits/rejected": -1.078125,
+      "eval_logps/chosen": -76.5,
+      "eval_logps/rejected": -572.0,
+      "eval_loss": 0.9423046708106995,
+      "eval_nll_loss": 1.09375,
+      "eval_rewards/accuracies": 0.7599999904632568,
+      "eval_rewards/chosen": 1.453125,
+      "eval_rewards/margins": 3.984375,
+      "eval_rewards/rejected": -2.515625,
+      "eval_runtime": 9.1636,
+      "eval_samples_per_second": 21.825,
+      "eval_steps_per_second": 2.728,
+      "step": 300
+    },
+    {
+      "epoch": 0.8149852119618797,
+      "grad_norm": 13.49396528945543,
+      "learning_rate": 4.992726324427901e-08,
+      "logits/chosen": -0.92578125,
+      "logits/rejected": -1.046875,
+      "logps/chosen": -72.5,
+      "logps/rejected": -612.0,
+      "loss": 0.9326,
+      "nll_loss": 0.859375,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 1.7265625,
+      "rewards/margins": 4.25,
+      "rewards/rejected": -2.53125,
+      "step": 310
+    },
+    {
+      "epoch": 0.8412750575090371,
+      "grad_norm": 85.18507320569196,
+      "learning_rate": 3.702014779041826e-08,
+      "logits/chosen": -0.94140625,
+      "logits/rejected": -1.046875,
+      "logps/chosen": -86.5,
+      "logps/rejected": -644.0,
+      "loss": 0.9377,
+      "nll_loss": 0.9609375,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": 1.6640625,
+      "rewards/margins": 4.6875,
+      "rewards/rejected": -3.046875,
+      "step": 320
+    },
+    {
+      "epoch": 0.8675649030561945,
+      "grad_norm": 9.69506288075215,
+      "learning_rate": 2.5908926115744994e-08,
+      "logits/chosen": -0.921875,
+      "logits/rejected": -1.0078125,
+      "logps/chosen": -80.5,
+      "logps/rejected": -428.0,
+      "loss": 0.9131,
+      "nll_loss": 0.90234375,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 1.7109375,
+      "rewards/margins": 2.78125,
+      "rewards/rejected": -1.0703125,
+      "step": 330
+    },
+    {
+      "epoch": 0.8938547486033519,
+      "grad_norm": 17.413302922847254,
+      "learning_rate": 1.6687290528135722e-08,
+      "logits/chosen": -0.9453125,
+      "logits/rejected": -1.0390625,
+      "logps/chosen": -90.5,
+      "logps/rejected": -540.0,
+      "loss": 0.9705,
+      "nll_loss": 0.921875,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 1.6953125,
+      "rewards/margins": 3.828125,
+      "rewards/rejected": -2.140625,
+      "step": 340
+    },
+    {
+      "epoch": 0.9201445941505094,
+      "grad_norm": 25.357904792782886,
+      "learning_rate": 9.432999922687396e-09,
+      "logits/chosen": -0.93359375,
+      "logits/rejected": -1.015625,
+      "logps/chosen": -77.0,
+      "logps/rejected": -536.0,
+      "loss": 0.9608,
+      "nll_loss": 0.859375,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 1.625,
+      "rewards/margins": 3.890625,
+      "rewards/rejected": -2.265625,
+      "step": 350
+    },
+    {
+      "epoch": 0.9464344396976668,
+      "grad_norm": 17.381689281866176,
+      "learning_rate": 4.207224101311246e-09,
+      "logits/chosen": -0.93359375,
+      "logits/rejected": -1.015625,
+      "logps/chosen": -100.0,
+      "logps/rejected": -580.0,
+      "loss": 0.895,
+      "nll_loss": 0.9296875,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 1.53125,
+      "rewards/margins": 4.0625,
+      "rewards/rejected": -2.53125,
+      "step": 360
+    },
+    {
+      "epoch": 0.9727242852448242,
+      "grad_norm": 62.410423904563885,
+      "learning_rate": 1.0540279752731252e-09,
+      "logits/chosen": -0.95703125,
+      "logits/rejected": -1.03125,
+      "logps/chosen": -79.0,
+      "logps/rejected": -524.0,
+      "loss": 0.9448,
+      "nll_loss": 0.92578125,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": 1.6171875,
+      "rewards/margins": 3.578125,
+      "rewards/rejected": -1.9609375,
+      "step": 370
+    },
+    {
+      "epoch": 0.9990141307919816,
+      "grad_norm": 14.434777424546557,
+      "learning_rate": 0.0,
+      "logits/chosen": -0.93359375,
+      "logits/rejected": -1.03125,
+      "logps/chosen": -93.0,
+      "logps/rejected": -624.0,
+      "loss": 0.974,
+      "nll_loss": 0.953125,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 1.6484375,
+      "rewards/margins": 4.3125,
+      "rewards/rejected": -2.671875,
+      "step": 380
+    },
+    {
+      "epoch": 0.9990141307919816,
+      "step": 380,
+      "total_flos": 0.0,
+      "train_loss": 1.0731394717567846,
+      "train_runtime": 3128.4011,
+      "train_samples_per_second": 7.78,
+      "train_steps_per_second": 0.121
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 380,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}