Model save

README.md

@@ -0,0 +1,81 @@
+---
+license: apache-2.0
+base_model: alignment-handbook/zephyr-7b-sft-full
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: zephyr-7b-dpo-full
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# zephyr-7b-dpo-full
+
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.4893
+- Rewards/chosen: -1.9379
+- Rewards/rejected: -3.0213
+- Rewards/accuracies: 0.7718
+- Rewards/margins: 1.0835
+- Logps/rejected: -563.9073
+- Logps/chosen: -477.8896
+- Logits/rejected: 0.6827
+- Logits/chosen: -0.4606
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 64
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.6338        | 0.1   | 100  | 0.6333          | -0.4184        | -0.6017          | 0.6865             | 0.1833          | -321.9407      | -325.9421    | -2.4857         | -2.5392       |
+| 0.5643        | 0.21  | 200  | 0.5547          | -1.1977        | -1.8547          | 0.7480             | 0.6570          | -447.2422      | -403.8748    | 0.1190          | -0.4672       |
+| 0.5066        | 0.31  | 300  | 0.5214          | -0.9561        | -1.7858          | 0.7778             | 0.8297          | -440.3582      | -379.7161    | -0.7390         | -1.4155       |
+| 0.4941        | 0.42  | 400  | 0.5082          | -1.2581        | -2.1325          | 0.7599             | 0.8744          | -475.0238      | -409.9142    | 0.1688          | -0.7662       |
+| 0.506         | 0.52  | 500  | 0.5090          | -1.1067        | -2.0712          | 0.7639             | 0.9645          | -468.8966      | -394.7739    | 1.3983          | 0.0857        |
+| 0.4893        | 0.63  | 600  | 0.4953          | -1.4696        | -2.4963          | 0.7579             | 1.0267          | -511.4048      | -431.0652    | 0.9613          | -0.4181       |
+| 0.4558        | 0.73  | 700  | 0.4937          | -1.8124        | -2.8894          | 0.7698             | 1.0770          | -550.7128      | -465.3409    | 0.6946          | -0.4445       |
+| 0.4781        | 0.84  | 800  | 0.4898          | -1.9968        | -3.0983          | 0.7698             | 1.1015          | -571.6086      | -483.7863    | 0.7311          | -0.4503       |
+| 0.495         | 0.94  | 900  | 0.4894          | -1.9365        | -3.0176          | 0.7698             | 1.0812          | -563.5378      | -477.7505    | 0.6757          | -0.4642       |
+
+
+### Framework versions
+
+- Transformers 4.36.2
+- Pytorch 2.1.2+cu118
+- Datasets 2.14.6
+- Tokenizers 0.15.0

all_results.json

@@ -0,0 +1,21 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": -0.4606437087059021,
+    "eval_logits/rejected": 0.6826901435852051,
+    "eval_logps/chosen": -477.88958740234375,
+    "eval_logps/rejected": -563.9073486328125,
+    "eval_loss": 0.48932957649230957,
+    "eval_rewards/accuracies": 0.77182537317276,
+    "eval_rewards/chosen": -1.9378634691238403,
+    "eval_rewards/margins": 1.0834674835205078,
+    "eval_rewards/rejected": -3.0213310718536377,
+    "eval_runtime": 279.233,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 7.162,
+    "eval_steps_per_second": 0.226,
+    "train_loss": 0.5274335609056563,
+    "train_runtime": 22383.3635,
+    "train_samples": 61135,
+    "train_samples_per_second": 2.731,
+    "train_steps_per_second": 0.043
+}

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": -0.4606437087059021,
+    "eval_logits/rejected": 0.6826901435852051,
+    "eval_logps/chosen": -477.88958740234375,
+    "eval_logps/rejected": -563.9073486328125,
+    "eval_loss": 0.48932957649230957,
+    "eval_rewards/accuracies": 0.77182537317276,
+    "eval_rewards/chosen": -1.9378634691238403,
+    "eval_rewards/margins": 1.0834674835205078,
+    "eval_rewards/rejected": -3.0213310718536377,
+    "eval_runtime": 279.233,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 7.162,
+    "eval_steps_per_second": 0.226
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.36.2"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d396669aecb89be161259b09d255abbb9173ac204f7b4d1f336b2b724753966f
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:293e46fc28924d5258cc69ce175999f4fc4b8b6a406d4b30ecd45dabb999073b
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a15dfe4081e25656dcacc88ecef82d9756c61c2245dfa875e37ebf77b3d1294
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

runs/Jan12_11-39-02_main1/events.out.tfevents.1705059886.main1.9143.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:add430ad40a428f682c7db431ab804dd0664b086d8ad0e5c0222f3f09480a482
-size 68726
+oid sha256:60ec38f7ce3691c8b95ff48e8a2c30d96e6ad2675688376a3e12de66871f20c5
+size 72250

runs/Jan12_11-39-02_main1/events.out.tfevents.1705082549.main1.9143.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c3097dfbeb1b99fbf99dc573f017edae70ced0d93cb40080aac127bfe287566
+size 828

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.5274335609056563,
+    "train_runtime": 22383.3635,
+    "train_samples": 61135,
+    "train_samples_per_second": 2.731,
+    "train_steps_per_second": 0.043
+}

trainer_state.json

@@ -0,0 +1,1518 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9994767137624281,
+  "eval_steps": 100,
+  "global_step": 955,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.208333333333333e-09,
+      "logits/chosen": -2.721794605255127,
+      "logits/rejected": -2.6868748664855957,
+      "logps/chosen": -229.75424194335938,
+      "logps/rejected": -214.74229431152344,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.208333333333333e-08,
+      "logits/chosen": -2.6621577739715576,
+      "logits/rejected": -2.5832412242889404,
+      "logps/chosen": -295.84637451171875,
+      "logps/rejected": -250.62530517578125,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4305555522441864,
+      "rewards/chosen": -3.170721174683422e-05,
+      "rewards/margins": 0.00016743924061302096,
+      "rewards/rejected": -0.00019914642325602472,
+      "step": 10
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.0416666666666667e-07,
+      "logits/chosen": -2.5875017642974854,
+      "logits/rejected": -2.5754282474517822,
+      "logps/chosen": -271.33502197265625,
+      "logps/rejected": -246.907470703125,
+      "loss": 0.693,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.0004090077127330005,
+      "rewards/margins": -0.00012708954454865307,
+      "rewards/rejected": -0.0002819181536324322,
+      "step": 20
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.5624999999999999e-07,
+      "logits/chosen": -2.6605145931243896,
+      "logits/rejected": -2.5901198387145996,
+      "logps/chosen": -278.30572509765625,
+      "logps/rejected": -254.51986694335938,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": 0.001268151798285544,
+      "rewards/margins": 0.001779665588401258,
+      "rewards/rejected": -0.0005115137319080532,
+      "step": 30
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.0833333333333333e-07,
+      "logits/chosen": -2.631272792816162,
+      "logits/rejected": -2.5783164501190186,
+      "logps/chosen": -273.85552978515625,
+      "logps/rejected": -237.7501220703125,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 0.003829254535958171,
+      "rewards/margins": 0.006563174072653055,
+      "rewards/rejected": -0.002733920468017459,
+      "step": 40
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.604166666666667e-07,
+      "logits/chosen": -2.634207248687744,
+      "logits/rejected": -2.5876431465148926,
+      "logps/chosen": -296.025146484375,
+      "logps/rejected": -274.70440673828125,
+      "loss": 0.6865,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": 0.009410797618329525,
+      "rewards/margins": 0.016297370195388794,
+      "rewards/rejected": -0.006886570248752832,
+      "step": 50
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": -2.598982334136963,
+      "logits/rejected": -2.5420570373535156,
+      "logps/chosen": -285.24249267578125,
+      "logps/rejected": -274.27044677734375,
+      "loss": 0.6802,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": 0.025370482355356216,
+      "rewards/margins": 0.02863493002951145,
+      "rewards/rejected": -0.0032644483726471663,
+      "step": 60
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.645833333333333e-07,
+      "logits/chosen": -2.6306140422821045,
+      "logits/rejected": -2.5784096717834473,
+      "logps/chosen": -310.9300537109375,
+      "logps/rejected": -290.3727111816406,
+      "loss": 0.6673,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.03530525788664818,
+      "rewards/margins": 0.053881775587797165,
+      "rewards/rejected": -0.018576517701148987,
+      "step": 70
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.1666666666666667e-07,
+      "logits/chosen": -2.4795422554016113,
+      "logits/rejected": -2.400930881500244,
+      "logps/chosen": -304.25933837890625,
+      "logps/rejected": -281.20159912109375,
+      "loss": 0.644,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.03674982860684395,
+      "rewards/margins": 0.11624778807163239,
+      "rewards/rejected": -0.15299761295318604,
+      "step": 80
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.6874999999999996e-07,
+      "logits/chosen": -2.5025558471679688,
+      "logits/rejected": -2.459231376647949,
+      "logps/chosen": -291.841064453125,
+      "logps/rejected": -283.4093322753906,
+      "loss": 0.6346,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 0.000455733563285321,
+      "rewards/margins": 0.17087821662425995,
+      "rewards/rejected": -0.1704224795103073,
+      "step": 90
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.999732492681437e-07,
+      "logits/chosen": -2.4846038818359375,
+      "logits/rejected": -2.455655336380005,
+      "logps/chosen": -335.92242431640625,
+      "logps/rejected": -340.43475341796875,
+      "loss": 0.6338,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.30497807264328003,
+      "rewards/margins": 0.1776745617389679,
+      "rewards/rejected": -0.4826526641845703,
+      "step": 100
+    },
+    {
+      "epoch": 0.1,
+      "eval_logits/chosen": -2.5391652584075928,
+      "eval_logits/rejected": -2.4857265949249268,
+      "eval_logps/chosen": -325.942138671875,
+      "eval_logps/rejected": -321.94073486328125,
+      "eval_loss": 0.6332610249519348,
+      "eval_rewards/accuracies": 0.6865079402923584,
+      "eval_rewards/chosen": -0.4183884263038635,
+      "eval_rewards/margins": 0.1832766830921173,
+      "eval_rewards/rejected": -0.6016651391983032,
+      "eval_runtime": 280.8039,
+      "eval_samples_per_second": 7.122,
+      "eval_steps_per_second": 0.224,
+      "step": 100
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.996723692767926e-07,
+      "logits/chosen": -2.335371255874634,
+      "logits/rejected": -2.2089285850524902,
+      "logps/chosen": -307.1370544433594,
+      "logps/rejected": -291.34033203125,
+      "loss": 0.618,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.46787112951278687,
+      "rewards/margins": 0.27501681447029114,
+      "rewards/rejected": -0.7428879737854004,
+      "step": 110
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.990375746213598e-07,
+      "logits/chosen": -1.776285171508789,
+      "logits/rejected": -1.5675570964813232,
+      "logps/chosen": -336.9173889160156,
+      "logps/rejected": -321.92755126953125,
+      "loss": 0.6025,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.31301265954971313,
+      "rewards/margins": 0.37262943387031555,
+      "rewards/rejected": -0.6856420636177063,
+      "step": 120
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.980697142834314e-07,
+      "logits/chosen": -1.6279542446136475,
+      "logits/rejected": -1.286072015762329,
+      "logps/chosen": -341.5347900390625,
+      "logps/rejected": -326.0841369628906,
+      "loss": 0.5902,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.21236875653266907,
+      "rewards/margins": 0.35374048352241516,
+      "rewards/rejected": -0.566109299659729,
+      "step": 130
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.967700826904229e-07,
+      "logits/chosen": -1.0164812803268433,
+      "logits/rejected": -0.7005228996276855,
+      "logps/chosen": -264.51629638671875,
+      "logps/rejected": -297.2378234863281,
+      "loss": 0.5579,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.24701330065727234,
+      "rewards/margins": 0.4705452024936676,
+      "rewards/rejected": -0.7175585031509399,
+      "step": 140
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.951404179843962e-07,
+      "logits/chosen": -0.418118953704834,
+      "logits/rejected": -0.1309668868780136,
+      "logps/chosen": -362.30755615234375,
+      "logps/rejected": -408.3031311035156,
+      "loss": 0.6149,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.8524826765060425,
+      "rewards/margins": 0.3839341700077057,
+      "rewards/rejected": -1.2364168167114258,
+      "step": 150
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.931828996974498e-07,
+      "logits/chosen": -1.7524774074554443,
+      "logits/rejected": -1.3019535541534424,
+      "logps/chosen": -368.46270751953125,
+      "logps/rejected": -398.49981689453125,
+      "loss": 0.5535,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.7256969809532166,
+      "rewards/margins": 0.35368454456329346,
+      "rewards/rejected": -1.0793814659118652,
+      "step": 160
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.909001458367866e-07,
+      "logits/chosen": -1.5669044256210327,
+      "logits/rejected": -1.0334669351577759,
+      "logps/chosen": -347.9214782714844,
+      "logps/rejected": -377.63446044921875,
+      "loss": 0.5472,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.7482506632804871,
+      "rewards/margins": 0.5563966631889343,
+      "rewards/rejected": -1.3046473264694214,
+      "step": 170
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.882952093833627e-07,
+      "logits/chosen": -0.7614485025405884,
+      "logits/rejected": 0.021318623796105385,
+      "logps/chosen": -343.43798828125,
+      "logps/rejected": -394.9954528808594,
+      "loss": 0.5239,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.9424430131912231,
+      "rewards/margins": 0.6877145767211914,
+      "rewards/rejected": -1.630157470703125,
+      "step": 180
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.853715742087946e-07,
+      "logits/chosen": -1.3107954263687134,
+      "logits/rejected": -0.5814211368560791,
+      "logps/chosen": -347.8511047363281,
+      "logps/rejected": -398.18890380859375,
+      "loss": 0.5298,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.6900913715362549,
+      "rewards/margins": 0.7306533455848694,
+      "rewards/rejected": -1.4207446575164795,
+      "step": 190
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.821331504159906e-07,
+      "logits/chosen": -0.4228581488132477,
+      "logits/rejected": 0.10827291011810303,
+      "logps/chosen": -387.1797180175781,
+      "logps/rejected": -437.99200439453125,
+      "loss": 0.5643,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.005066990852356,
+      "rewards/margins": 0.6199973821640015,
+      "rewards/rejected": -1.6250642538070679,
+      "step": 200
+    },
+    {
+      "epoch": 0.21,
+      "eval_logits/chosen": -0.4671556353569031,
+      "eval_logits/rejected": 0.11897020787000656,
+      "eval_logps/chosen": -403.8748474121094,
+      "eval_logps/rejected": -447.24224853515625,
+      "eval_loss": 0.5547109246253967,
+      "eval_rewards/accuracies": 0.7480158805847168,
+      "eval_rewards/chosen": -1.1977157592773438,
+      "eval_rewards/margins": 0.6569646000862122,
+      "eval_rewards/rejected": -1.8546805381774902,
+      "eval_runtime": 279.4406,
+      "eval_samples_per_second": 7.157,
+      "eval_steps_per_second": 0.225,
+      "step": 200
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.785842691097342e-07,
+      "logits/chosen": -0.3859473466873169,
+      "logits/rejected": 0.4077689051628113,
+      "logps/chosen": -418.86883544921875,
+      "logps/rejected": -427.7865295410156,
+      "loss": 0.5491,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.1351739168167114,
+      "rewards/margins": 0.6033421754837036,
+      "rewards/rejected": -1.738515853881836,
+      "step": 210
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.7472967660421603e-07,
+      "logits/chosen": -1.136400580406189,
+      "logits/rejected": -0.6079188585281372,
+      "logps/chosen": -363.44915771484375,
+      "logps/rejected": -404.9462585449219,
+      "loss": 0.55,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.7393938302993774,
+      "rewards/margins": 0.5468804836273193,
+      "rewards/rejected": -1.2862741947174072,
+      "step": 220
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.705745280752585e-07,
+      "logits/chosen": -0.7287824153900146,
+      "logits/rejected": 0.3289051949977875,
+      "logps/chosen": -380.1632080078125,
+      "logps/rejected": -415.7613220214844,
+      "loss": 0.5434,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.8638499975204468,
+      "rewards/margins": 0.8072270154953003,
+      "rewards/rejected": -1.671076774597168,
+      "step": 230
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.6612438066572555e-07,
+      "logits/chosen": -1.221251130104065,
+      "logits/rejected": -0.22343508899211884,
+      "logps/chosen": -349.05084228515625,
+      "logps/rejected": -355.9934997558594,
+      "loss": 0.5308,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.7182748913764954,
+      "rewards/margins": 0.5810624957084656,
+      "rewards/rejected": -1.2993375062942505,
+      "step": 240
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.6138518605333664e-07,
+      "logits/chosen": -0.4631493091583252,
+      "logits/rejected": 0.10230980068445206,
+      "logps/chosen": -364.1564025878906,
+      "logps/rejected": -439.31463623046875,
+      "loss": 0.5407,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.9659140706062317,
+      "rewards/margins": 0.6127485036849976,
+      "rewards/rejected": -1.5786627531051636,
+      "step": 250
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.5636328249082514e-07,
+      "logits/chosen": -0.5042780637741089,
+      "logits/rejected": 0.3967960774898529,
+      "logps/chosen": -377.323974609375,
+      "logps/rejected": -418.09661865234375,
+      "loss": 0.5323,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.9326316714286804,
+      "rewards/margins": 0.6699790358543396,
+      "rewards/rejected": -1.6026105880737305,
+      "step": 260
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.510653863290871e-07,
+      "logits/chosen": -0.23900368809700012,
+      "logits/rejected": 1.1343748569488525,
+      "logps/chosen": -397.4126892089844,
+      "logps/rejected": -434.02203369140625,
+      "loss": 0.5028,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.9560322761535645,
+      "rewards/margins": 0.8822371363639832,
+      "rewards/rejected": -1.8382694721221924,
+      "step": 270
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.4549858303465737e-07,
+      "logits/chosen": -0.15336796641349792,
+      "logits/rejected": 0.8832536935806274,
+      "logps/chosen": -387.30743408203125,
+      "logps/rejected": -461.30084228515625,
+      "loss": 0.5195,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.9608807563781738,
+      "rewards/margins": 0.8189412355422974,
+      "rewards/rejected": -1.779821753501892,
+      "step": 280
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.396703177135261e-07,
+      "logits/chosen": -1.3611189126968384,
+      "logits/rejected": -0.8717398643493652,
+      "logps/chosen": -381.35321044921875,
+      "logps/rejected": -414.11279296875,
+      "loss": 0.527,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.9636392593383789,
+      "rewards/margins": 0.6538799405097961,
+      "rewards/rejected": -1.6175191402435303,
+      "step": 290
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.335883851539693e-07,
+      "logits/chosen": -1.2623040676116943,
+      "logits/rejected": -0.05791844055056572,
+      "logps/chosen": -380.7442626953125,
+      "logps/rejected": -440.4820251464844,
+      "loss": 0.5066,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.9736257791519165,
+      "rewards/margins": 0.9214972257614136,
+      "rewards/rejected": -1.8951228857040405,
+      "step": 300
+    },
+    {
+      "epoch": 0.31,
+      "eval_logits/chosen": -1.4155113697052002,
+      "eval_logits/rejected": -0.7390369772911072,
+      "eval_logps/chosen": -379.7160949707031,
+      "eval_logps/rejected": -440.3581848144531,
+      "eval_loss": 0.5213961005210876,
+      "eval_rewards/accuracies": 0.7777777910232544,
+      "eval_rewards/chosen": -0.9561280608177185,
+      "eval_rewards/margins": 0.8297119140625,
+      "eval_rewards/rejected": -1.7858400344848633,
+      "eval_runtime": 277.1592,
+      "eval_samples_per_second": 7.216,
+      "eval_steps_per_second": 0.227,
+      "step": 300
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.272609194017105e-07,
+      "logits/chosen": -1.365898847579956,
+      "logits/rejected": -0.6500005722045898,
+      "logps/chosen": -368.51959228515625,
+      "logps/rejected": -451.77130126953125,
+      "loss": 0.4823,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.8627009391784668,
+      "rewards/margins": 0.9636882543563843,
+      "rewards/rejected": -1.8263890743255615,
+      "step": 310
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.2069638288135547e-07,
+      "logits/chosen": -1.1361403465270996,
+      "logits/rejected": -0.4094177186489105,
+      "logps/chosen": -378.14337158203125,
+      "logps/rejected": -442.03179931640625,
+      "loss": 0.5231,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.9646435976028442,
+      "rewards/margins": 0.7620329856872559,
+      "rewards/rejected": -1.7266767024993896,
+      "step": 320
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.139035550786494e-07,
+      "logits/chosen": -1.151816487312317,
+      "logits/rejected": -0.6147797703742981,
+      "logps/chosen": -371.8530578613281,
+      "logps/rejected": -387.37786865234375,
+      "loss": 0.557,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.9230759739875793,
+      "rewards/margins": 0.6179936528205872,
+      "rewards/rejected": -1.541069507598877,
+      "step": 330
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.0689152079869306e-07,
+      "logits/chosen": -1.3167518377304077,
+      "logits/rejected": -0.7884875535964966,
+      "logps/chosen": -342.40643310546875,
+      "logps/rejected": -372.789794921875,
+      "loss": 0.5621,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.966681957244873,
+      "rewards/margins": 0.5185996890068054,
+      "rewards/rejected": -1.4852817058563232,
+      "step": 340
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.99669658015821e-07,
+      "logits/chosen": -0.2449747771024704,
+      "logits/rejected": 0.10235898196697235,
+      "logps/chosen": -384.5868225097656,
+      "logps/rejected": -475.15606689453125,
+      "loss": 0.5192,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.1246081590652466,
+      "rewards/margins": 0.8372184038162231,
+      "rewards/rejected": -1.9618265628814697,
+      "step": 350
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.92247625331392e-07,
+      "logits/chosen": -0.19500190019607544,
+      "logits/rejected": 0.6360437273979187,
+      "logps/chosen": -383.85845947265625,
+      "logps/rejected": -425.1438903808594,
+      "loss": 0.5123,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.011595606803894,
+      "rewards/margins": 0.8127263188362122,
+      "rewards/rejected": -1.8243221044540405,
+      "step": 360
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.846353490562664e-07,
+      "logits/chosen": -0.6418613791465759,
+      "logits/rejected": 0.004262035712599754,
+      "logps/chosen": -332.0767517089844,
+      "logps/rejected": -436.4134826660156,
+      "loss": 0.4874,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.919625461101532,
+      "rewards/margins": 0.9047123789787292,
+      "rewards/rejected": -1.8243379592895508,
+      "step": 370
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.768430099352445e-07,
+      "logits/chosen": -1.2811148166656494,
+      "logits/rejected": -0.3615255355834961,
+      "logps/chosen": -408.3636474609375,
+      "logps/rejected": -467.22601318359375,
+      "loss": 0.4952,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.1174873113632202,
+      "rewards/margins": 0.9306632280349731,
+      "rewards/rejected": -2.0481505393981934,
+      "step": 380
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6888102953122304e-07,
+      "logits/chosen": -0.6116107702255249,
+      "logits/rejected": 0.6004475355148315,
+      "logps/chosen": -408.0143737792969,
+      "logps/rejected": -468.11602783203125,
+      "loss": 0.5044,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.3232171535491943,
+      "rewards/margins": 1.0072014331817627,
+      "rewards/rejected": -2.330418348312378,
+      "step": 390
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.607600562872785e-07,
+      "logits/chosen": -0.746349036693573,
+      "logits/rejected": 0.21193762123584747,
+      "logps/chosen": -457.70703125,
+      "logps/rejected": -493.6814880371094,
+      "loss": 0.4941,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.5325865745544434,
+      "rewards/margins": 0.7493036985397339,
+      "rewards/rejected": -2.281890392303467,
+      "step": 400
+    },
+    {
+      "epoch": 0.42,
+      "eval_logits/chosen": -0.766248881816864,
+      "eval_logits/rejected": 0.168849915266037,
+      "eval_logps/chosen": -409.9141540527344,
+      "eval_logps/rejected": -475.0238037109375,
+      "eval_loss": 0.5082493424415588,
+      "eval_rewards/accuracies": 0.7599206566810608,
+      "eval_rewards/chosen": -1.2581086158752441,
+      "eval_rewards/margins": 0.8743875622749329,
+      "eval_rewards/rejected": -2.1324961185455322,
+      "eval_runtime": 277.9817,
+      "eval_samples_per_second": 7.195,
+      "eval_steps_per_second": 0.227,
+      "step": 400
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5249095128531856e-07,
+      "logits/chosen": -1.081780195236206,
+      "logits/rejected": -0.14936906099319458,
+      "logps/chosen": -433.837890625,
+      "logps/rejected": -488.9054260253906,
+      "loss": 0.5236,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.1999945640563965,
+      "rewards/margins": 0.7789229154586792,
+      "rewards/rejected": -1.9789173603057861,
+      "step": 410
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4408477372034736e-07,
+      "logits/chosen": -0.9525474309921265,
+      "logits/rejected": -0.22369150817394257,
+      "logps/chosen": -385.2050476074219,
+      "logps/rejected": -414.0570373535156,
+      "loss": 0.5265,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.2109800577163696,
+      "rewards/margins": 0.6298609972000122,
+      "rewards/rejected": -1.8408409357070923,
+      "step": 420
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3555276610977276e-07,
+      "logits/chosen": -1.1188991069793701,
+      "logits/rejected": -0.6098810434341431,
+      "logps/chosen": -366.67327880859375,
+      "logps/rejected": -413.8002014160156,
+      "loss": 0.5316,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.0450365543365479,
+      "rewards/margins": 0.6988474726676941,
+      "rewards/rejected": -1.7438838481903076,
+      "step": 430
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.269063392575352e-07,
+      "logits/chosen": -0.8952062726020813,
+      "logits/rejected": -0.29640626907348633,
+      "logps/chosen": -369.5331115722656,
+      "logps/rejected": -437.5091857910156,
+      "loss": 0.5019,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.1057814359664917,
+      "rewards/margins": 0.7806274890899658,
+      "rewards/rejected": -1.886409044265747,
+      "step": 440
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.1815705699316964e-07,
+      "logits/chosen": -0.5310872793197632,
+      "logits/rejected": 0.35491126775741577,
+      "logps/chosen": -383.77337646484375,
+      "logps/rejected": -466.27252197265625,
+      "loss": 0.5111,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.1957778930664062,
+      "rewards/margins": 1.0301498174667358,
+      "rewards/rejected": -2.2259278297424316,
+      "step": 450
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.0931662070620794e-07,
+      "logits/chosen": -0.14580872654914856,
+      "logits/rejected": 1.0018532276153564,
+      "logps/chosen": -406.70404052734375,
+      "logps/rejected": -492.42333984375,
+      "loss": 0.5107,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.4167234897613525,
+      "rewards/margins": 0.9401258230209351,
+      "rewards/rejected": -2.3568496704101562,
+      "step": 460
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.003968536966078e-07,
+      "logits/chosen": -0.4022013545036316,
+      "logits/rejected": 0.9017314910888672,
+      "logps/chosen": -438.36492919921875,
+      "logps/rejected": -496.6222229003906,
+      "loss": 0.506,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.4015130996704102,
+      "rewards/margins": 0.9599510431289673,
+      "rewards/rejected": -2.361464023590088,
+      "step": 470
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.9140968536213693e-07,
+      "logits/chosen": -0.5105709433555603,
+      "logits/rejected": 0.3333401083946228,
+      "logps/chosen": -348.86138916015625,
+      "logps/rejected": -420.00909423828125,
+      "loss": 0.5345,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.1617661714553833,
+      "rewards/margins": 0.749540388584137,
+      "rewards/rejected": -1.911306619644165,
+      "step": 480
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.823671352438608e-07,
+      "logits/chosen": -0.5324286818504333,
+      "logits/rejected": 0.6832448244094849,
+      "logps/chosen": -373.8572692871094,
+      "logps/rejected": -425.37188720703125,
+      "loss": 0.5055,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.9815096855163574,
+      "rewards/margins": 0.7788732051849365,
+      "rewards/rejected": -1.760382890701294,
+      "step": 490
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.73281296951072e-07,
+      "logits/chosen": 0.2629791498184204,
+      "logits/rejected": 1.6376205682754517,
+      "logps/chosen": -394.38934326171875,
+      "logps/rejected": -471.0135803222656,
+      "loss": 0.506,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.2273036241531372,
+      "rewards/margins": 0.9877279996871948,
+      "rewards/rejected": -2.215031623840332,
+      "step": 500
+    },
+    {
+      "epoch": 0.52,
+      "eval_logits/chosen": 0.08570393919944763,
+      "eval_logits/rejected": 1.3982725143432617,
+      "eval_logps/chosen": -394.77392578125,
+      "eval_logps/rejected": -468.8965759277344,
+      "eval_loss": 0.5090134739875793,
+      "eval_rewards/accuracies": 0.7638888955116272,
+      "eval_rewards/chosen": -1.1067068576812744,
+      "eval_rewards/margins": 0.9645172357559204,
+      "eval_rewards/rejected": -2.0712242126464844,
+      "eval_runtime": 278.9033,
+      "eval_samples_per_second": 7.171,
+      "eval_steps_per_second": 0.226,
+      "step": 500
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.641643219871597e-07,
+      "logits/chosen": 0.26743632555007935,
+      "logits/rejected": 1.9989964962005615,
+      "logps/chosen": -418.77044677734375,
+      "logps/rejected": -468.5576171875,
+      "loss": 0.4874,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.1739251613616943,
+      "rewards/margins": 0.9960535168647766,
+      "rewards/rejected": -2.1699788570404053,
+      "step": 510
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.550284034980507e-07,
+      "logits/chosen": 0.5876134634017944,
+      "logits/rejected": 1.631603479385376,
+      "logps/chosen": -406.91351318359375,
+      "logps/rejected": -484.844482421875,
+      "loss": 0.5042,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.4775892496109009,
+      "rewards/margins": 0.853603184223175,
+      "rewards/rejected": -2.3311924934387207,
+      "step": 520
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.4588575996495794e-07,
+      "logits/chosen": 0.36718782782554626,
+      "logits/rejected": 1.523099660873413,
+      "logps/chosen": -433.95208740234375,
+      "logps/rejected": -500.431396484375,
+      "loss": 0.4987,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.4977753162384033,
+      "rewards/margins": 0.9368091821670532,
+      "rewards/rejected": -2.434584379196167,
+      "step": 530
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.367486188632446e-07,
+      "logits/chosen": -0.19433559477329254,
+      "logits/rejected": 1.0568033456802368,
+      "logps/chosen": -429.4280700683594,
+      "logps/rejected": -540.71337890625,
+      "loss": 0.5031,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.3522653579711914,
+      "rewards/margins": 1.0526258945465088,
+      "rewards/rejected": -2.4048912525177,
+      "step": 540
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.276292003092593e-07,
+      "logits/chosen": -0.5677329897880554,
+      "logits/rejected": 0.8163228034973145,
+      "logps/chosen": -383.25537109375,
+      "logps/rejected": -460.65643310546875,
+      "loss": 0.519,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.1353278160095215,
+      "rewards/margins": 0.9918322563171387,
+      "rewards/rejected": -2.127159833908081,
+      "step": 550
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.185397007170141e-07,
+      "logits/chosen": -0.5527084469795227,
+      "logits/rejected": 0.5254577398300171,
+      "logps/chosen": -373.94189453125,
+      "logps/rejected": -429.65283203125,
+      "loss": 0.5028,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.071791410446167,
+      "rewards/margins": 0.8905409574508667,
+      "rewards/rejected": -1.962332010269165,
+      "step": 560
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.094922764865619e-07,
+      "logits/chosen": -0.08112873882055283,
+      "logits/rejected": 1.072463035583496,
+      "logps/chosen": -403.42254638671875,
+      "logps/rejected": -467.67449951171875,
+      "loss": 0.5061,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.2727501392364502,
+      "rewards/margins": 0.8145562410354614,
+      "rewards/rejected": -2.087306499481201,
+      "step": 570
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.0049902774588797e-07,
+      "logits/chosen": 0.4094156324863434,
+      "logits/rejected": 1.793280005455017,
+      "logps/chosen": -428.2859802246094,
+      "logps/rejected": -489.5137634277344,
+      "loss": 0.5007,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.5437203645706177,
+      "rewards/margins": 0.9729151725769043,
+      "rewards/rejected": -2.5166354179382324,
+      "step": 580
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.9157198216806238e-07,
+      "logits/chosen": -0.1933516561985016,
+      "logits/rejected": 0.8015575408935547,
+      "logps/chosen": -403.41021728515625,
+      "logps/rejected": -485.6842346191406,
+      "loss": 0.496,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.3032294511795044,
+      "rewards/margins": 0.7608039379119873,
+      "rewards/rejected": -2.0640335083007812,
+      "step": 590
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8272307888529274e-07,
+      "logits/chosen": -0.43585944175720215,
+      "logits/rejected": 0.9283556938171387,
+      "logps/chosen": -461.6996154785156,
+      "logps/rejected": -543.3409423828125,
+      "loss": 0.4893,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.3578338623046875,
+      "rewards/margins": 0.9542685747146606,
+      "rewards/rejected": -2.3121023178100586,
+      "step": 600
+    },
+    {
+      "epoch": 0.63,
+      "eval_logits/chosen": -0.41807228326797485,
+      "eval_logits/rejected": 0.961281418800354,
+      "eval_logps/chosen": -431.0652160644531,
+      "eval_logps/rejected": -511.40478515625,
+      "eval_loss": 0.49531620740890503,
+      "eval_rewards/accuracies": 0.7579365372657776,
+      "eval_rewards/chosen": -1.469619870185852,
+      "eval_rewards/margins": 1.0266858339309692,
+      "eval_rewards/rejected": -2.4963057041168213,
+      "eval_runtime": 279.0041,
+      "eval_samples_per_second": 7.168,
+      "eval_steps_per_second": 0.226,
+      "step": 600
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7396415252139288e-07,
+      "logits/chosen": -0.44998255372047424,
+      "logits/rejected": 0.763337254524231,
+      "logps/chosen": -419.0155334472656,
+      "logps/rejected": -462.7840270996094,
+      "loss": 0.4842,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.4401321411132812,
+      "rewards/margins": 0.9432790875434875,
+      "rewards/rejected": -2.383411407470703,
+      "step": 610
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6530691736402316e-07,
+      "logits/chosen": -0.37868770956993103,
+      "logits/rejected": 1.2032816410064697,
+      "logps/chosen": -442.1136169433594,
+      "logps/rejected": -499.60504150390625,
+      "loss": 0.5035,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.7007478475570679,
+      "rewards/margins": 0.9936334490776062,
+      "rewards/rejected": -2.6943812370300293,
+      "step": 620
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5676295169786864e-07,
+      "logits/chosen": -0.10520428419113159,
+      "logits/rejected": 1.3110377788543701,
+      "logps/chosen": -421.3946838378906,
+      "logps/rejected": -537.7481689453125,
+      "loss": 0.4811,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.5981671810150146,
+      "rewards/margins": 1.225342869758606,
+      "rewards/rejected": -2.823509931564331,
+      "step": 630
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.483436823197092e-07,
+      "logits/chosen": -0.4932429790496826,
+      "logits/rejected": 0.5939461588859558,
+      "logps/chosen": -437.61572265625,
+      "logps/rejected": -519.6408081054688,
+      "loss": 0.496,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.5301989316940308,
+      "rewards/margins": 1.0939643383026123,
+      "rewards/rejected": -2.6241631507873535,
+      "step": 640
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4006036925609243e-07,
+      "logits/chosen": -0.7950553894042969,
+      "logits/rejected": 0.526989221572876,
+      "logps/chosen": -480.47161865234375,
+      "logps/rejected": -521.6663208007812,
+      "loss": 0.5013,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.7415778636932373,
+      "rewards/margins": 0.8471624255180359,
+      "rewards/rejected": -2.588740587234497,
+      "step": 650
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.319240907040458e-07,
+      "logits/chosen": -0.6145257353782654,
+      "logits/rejected": 0.26546335220336914,
+      "logps/chosen": -462.09527587890625,
+      "logps/rejected": -522.7366943359375,
+      "loss": 0.503,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.6060371398925781,
+      "rewards/margins": 0.9507848024368286,
+      "rewards/rejected": -2.556821823120117,
+      "step": 660
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.239457282149695e-07,
+      "logits/chosen": -0.6238754391670227,
+      "logits/rejected": 0.2821710407733917,
+      "logps/chosen": -436.31097412109375,
+      "logps/rejected": -520.7191772460938,
+      "loss": 0.4934,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.5321730375289917,
+      "rewards/margins": 0.9870231747627258,
+      "rewards/rejected": -2.5191962718963623,
+      "step": 670
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1613595214152711e-07,
+      "logits/chosen": -0.5683806538581848,
+      "logits/rejected": 0.05843483284115791,
+      "logps/chosen": -399.19952392578125,
+      "logps/rejected": -444.88604736328125,
+      "loss": 0.5183,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.3775449991226196,
+      "rewards/margins": 0.6896665692329407,
+      "rewards/rejected": -2.067211627960205,
+      "step": 680
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.0850520736699362e-07,
+      "logits/chosen": -0.8241173028945923,
+      "logits/rejected": 0.25952425599098206,
+      "logps/chosen": -385.21063232421875,
+      "logps/rejected": -466.60809326171875,
+      "loss": 0.4836,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.237121343612671,
+      "rewards/margins": 1.0103009939193726,
+      "rewards/rejected": -2.247422456741333,
+      "step": 690
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0106369933615042e-07,
+      "logits/chosen": -0.5171055793762207,
+      "logits/rejected": 0.7539600133895874,
+      "logps/chosen": -430.15069580078125,
+      "logps/rejected": -530.7283935546875,
+      "loss": 0.4558,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.7596960067749023,
+      "rewards/margins": 1.09982168674469,
+      "rewards/rejected": -2.8595175743103027,
+      "step": 700
+    },
+    {
+      "epoch": 0.73,
+      "eval_logits/chosen": -0.44453164935112,
+      "eval_logits/rejected": 0.6945505738258362,
+      "eval_logps/chosen": -465.3408508300781,
+      "eval_logps/rejected": -550.7127685546875,
+      "eval_loss": 0.49365857243537903,
+      "eval_rewards/accuracies": 0.7698412537574768,
+      "eval_rewards/chosen": -1.8123756647109985,
+      "eval_rewards/margins": 1.0770103931427002,
+      "eval_rewards/rejected": -2.889385938644409,
+      "eval_runtime": 278.4646,
+      "eval_samples_per_second": 7.182,
+      "eval_steps_per_second": 0.226,
+      "step": 700
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.382138040640714e-08,
+      "logits/chosen": -0.43636417388916016,
+      "logits/rejected": 0.8631266355514526,
+      "logps/chosen": -496.98138427734375,
+      "logps/rejected": -531.872314453125,
+      "loss": 0.4688,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.92108154296875,
+      "rewards/margins": 0.9923149943351746,
+      "rewards/rejected": -2.9133965969085693,
+      "step": 710
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.678793653740632e-08,
+      "logits/chosen": -0.10378441959619522,
+      "logits/rejected": 1.2110047340393066,
+      "logps/chosen": -494.89990234375,
+      "logps/rejected": -584.9482421875,
+      "loss": 0.4768,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -2.034623146057129,
+      "rewards/margins": 1.1489760875701904,
+      "rewards/rejected": -3.1835992336273193,
+      "step": 720
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 7.997277433690983e-08,
+      "logits/chosen": -0.11848801374435425,
+      "logits/rejected": 1.194339632987976,
+      "logps/chosen": -507.22760009765625,
+      "logps/rejected": -562.0817260742188,
+      "loss": 0.4949,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.0333375930786133,
+      "rewards/margins": 1.0677392482757568,
+      "rewards/rejected": -3.101076602935791,
+      "step": 730
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.338500848029602e-08,
+      "logits/chosen": 0.04791594296693802,
+      "logits/rejected": 1.755448579788208,
+      "logps/chosen": -520.539794921875,
+      "logps/rejected": -599.8433837890625,
+      "loss": 0.4849,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -2.0619003772735596,
+      "rewards/margins": 1.345897912979126,
+      "rewards/rejected": -3.4077982902526855,
+      "step": 740
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.70334495204884e-08,
+      "logits/chosen": -0.3863813579082489,
+      "logits/rejected": 0.8495087623596191,
+      "logps/chosen": -478.74835205078125,
+      "logps/rejected": -582.2133178710938,
+      "loss": 0.4878,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.953743577003479,
+      "rewards/margins": 1.0831085443496704,
+      "rewards/rejected": -3.0368518829345703,
+      "step": 750
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 6.092659210462231e-08,
+      "logits/chosen": -0.1735175997018814,
+      "logits/rejected": 0.8358144760131836,
+      "logps/chosen": -487.7015075683594,
+      "logps/rejected": -554.705810546875,
+      "loss": 0.4576,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -2.09541392326355,
+      "rewards/margins": 0.945289134979248,
+      "rewards/rejected": -3.040703058242798,
+      "step": 760
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.507260361320737e-08,
+      "logits/chosen": -0.21100804209709167,
+      "logits/rejected": 0.5948140621185303,
+      "logps/chosen": -511.5699157714844,
+      "logps/rejected": -609.0560302734375,
+      "loss": 0.5073,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.175572156906128,
+      "rewards/margins": 0.9202457666397095,
+      "rewards/rejected": -3.095817804336548,
+      "step": 770
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.947931323697982e-08,
+      "logits/chosen": -0.36330968141555786,
+      "logits/rejected": 0.8800643682479858,
+      "logps/chosen": -465.767333984375,
+      "logps/rejected": -530.8380737304688,
+      "loss": 0.5019,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -2.095672130584717,
+      "rewards/margins": 0.8483352661132812,
+      "rewards/rejected": -2.944007158279419,
+      "step": 780
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.415420150605398e-08,
+      "logits/chosen": -0.35998591780662537,
+      "logits/rejected": 0.594068706035614,
+      "logps/chosen": -519.5208740234375,
+      "logps/rejected": -622.3743896484375,
+      "loss": 0.4857,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.191953182220459,
+      "rewards/margins": 1.1867144107818604,
+      "rewards/rejected": -3.3786678314208984,
+      "step": 790
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.9104390285376374e-08,
+      "logits/chosen": -0.4554923474788666,
+      "logits/rejected": 0.7812598943710327,
+      "logps/chosen": -540.5608520507812,
+      "logps/rejected": -596.4019775390625,
+      "loss": 0.4781,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.9934720993041992,
+      "rewards/margins": 1.1518696546554565,
+      "rewards/rejected": -3.1453421115875244,
+      "step": 800
+    },
+    {
+      "epoch": 0.84,
+      "eval_logits/chosen": -0.45034220814704895,
+      "eval_logits/rejected": 0.7310919761657715,
+      "eval_logps/chosen": -483.78631591796875,
+      "eval_logps/rejected": -571.6085815429688,
+      "eval_loss": 0.4898362457752228,
+      "eval_rewards/accuracies": 0.7698412537574768,
+      "eval_rewards/chosen": -1.9968301057815552,
+      "eval_rewards/margins": 1.1015136241912842,
+      "eval_rewards/rejected": -3.09834361076355,
+      "eval_runtime": 275.7757,
+      "eval_samples_per_second": 7.252,
+      "eval_steps_per_second": 0.228,
+      "step": 800
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.433663324986208e-08,
+      "logits/chosen": -0.3436442017555237,
+      "logits/rejected": 1.0277094841003418,
+      "logps/chosen": -494.46343994140625,
+      "logps/rejected": -540.2312622070312,
+      "loss": 0.4921,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.1496834754943848,
+      "rewards/margins": 0.8621308207511902,
+      "rewards/rejected": -3.0118138790130615,
+      "step": 810
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.9857306851953897e-08,
+      "logits/chosen": -0.10989487171173096,
+      "logits/rejected": 0.818514347076416,
+      "logps/chosen": -449.99346923828125,
+      "logps/rejected": -534.0044555664062,
+      "loss": 0.5095,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.9733314514160156,
+      "rewards/margins": 1.014764428138733,
+      "rewards/rejected": -2.988095760345459,
+      "step": 820
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.567240179368185e-08,
+      "logits/chosen": -0.16947147250175476,
+      "logits/rejected": 0.5960814952850342,
+      "logps/chosen": -450.3282165527344,
+      "logps/rejected": -539.936767578125,
+      "loss": 0.4804,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.0146961212158203,
+      "rewards/margins": 0.9625784754753113,
+      "rewards/rejected": -2.9772744178771973,
+      "step": 830
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.1787515014630357e-08,
+      "logits/chosen": -0.3402346968650818,
+      "logits/rejected": 0.5662250518798828,
+      "logps/chosen": -507.84747314453125,
+      "logps/rejected": -563.3038940429688,
+      "loss": 0.5113,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.9471012353897095,
+      "rewards/margins": 0.8456957936286926,
+      "rewards/rejected": -2.792797327041626,
+      "step": 840
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.820784220652766e-08,
+      "logits/chosen": -0.5917896032333374,
+      "logits/rejected": 0.3662889003753662,
+      "logps/chosen": -483.69134521484375,
+      "logps/rejected": -513.99658203125,
+      "loss": 0.494,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.8341186046600342,
+      "rewards/margins": 0.8557528257369995,
+      "rewards/rejected": -2.689871311187744,
+      "step": 850
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.4938170864468636e-08,
+      "logits/chosen": -0.317844957113266,
+      "logits/rejected": 0.6835850477218628,
+      "logps/chosen": -478.897705078125,
+      "logps/rejected": -561.5941772460938,
+      "loss": 0.4905,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -2.0050323009490967,
+      "rewards/margins": 1.1022390127182007,
+      "rewards/rejected": -3.107271432876587,
+      "step": 860
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.1982873884064465e-08,
+      "logits/chosen": -0.48837724328041077,
+      "logits/rejected": 0.6178911924362183,
+      "logps/chosen": -414.78631591796875,
+      "logps/rejected": -539.1580200195312,
+      "loss": 0.4814,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -1.678101897239685,
+      "rewards/margins": 1.251063346862793,
+      "rewards/rejected": -2.9291653633117676,
+      "step": 870
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 9.345903713082304e-09,
+      "logits/chosen": -0.3579421043395996,
+      "logits/rejected": 0.8588876724243164,
+      "logps/chosen": -478.02728271484375,
+      "logps/rejected": -552.3193359375,
+      "loss": 0.4827,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -2.0388994216918945,
+      "rewards/margins": 0.9511061906814575,
+      "rewards/rejected": -2.9900057315826416,
+      "step": 880
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.030787065396865e-09,
+      "logits/chosen": -0.30088967084884644,
+      "logits/rejected": 0.4045659601688385,
+      "logps/chosen": -469.99920654296875,
+      "logps/rejected": -559.8849487304688,
+      "loss": 0.509,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.034852981567383,
+      "rewards/margins": 0.909083366394043,
+      "rewards/rejected": -2.943936586380005,
+      "step": 890
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.04062020432286e-09,
+      "logits/chosen": -0.4078744351863861,
+      "logits/rejected": 0.6653724908828735,
+      "logps/chosen": -496.38848876953125,
+      "logps/rejected": -572.7337646484375,
+      "loss": 0.495,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.979608178138733,
+      "rewards/margins": 0.9050843119621277,
+      "rewards/rejected": -2.8846921920776367,
+      "step": 900
+    },
+    {
+      "epoch": 0.94,
+      "eval_logits/chosen": -0.46421733498573303,
+      "eval_logits/rejected": 0.6757184267044067,
+      "eval_logps/chosen": -477.7505187988281,
+      "eval_logps/rejected": -563.537841796875,
+      "eval_loss": 0.48935839533805847,
+      "eval_rewards/accuracies": 0.7698412537574768,
+      "eval_rewards/chosen": -1.9364722967147827,
+      "eval_rewards/margins": 1.0811636447906494,
+      "eval_rewards/rejected": -3.0176358222961426,
+      "eval_runtime": 278.8433,
+      "eval_samples_per_second": 7.172,
+      "eval_steps_per_second": 0.226,
+      "step": 900
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.3780648016376866e-09,
+      "logits/chosen": -0.34233179688453674,
+      "logits/rejected": 0.7931967973709106,
+      "logps/chosen": -453.32513427734375,
+      "logps/rejected": -545.7423095703125,
+      "loss": 0.4928,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.057847023010254,
+      "rewards/margins": 0.984235942363739,
+      "rewards/rejected": -3.0420830249786377,
+      "step": 910
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.0453443778310766e-09,
+      "logits/chosen": -0.42071524262428284,
+      "logits/rejected": 0.7710467576980591,
+      "logps/chosen": -491.179931640625,
+      "logps/rejected": -565.1658935546875,
+      "loss": 0.4639,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.9547284841537476,
+      "rewards/margins": 1.1154911518096924,
+      "rewards/rejected": -3.0702195167541504,
+      "step": 920
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.0442413283435758e-09,
+      "logits/chosen": -0.39107799530029297,
+      "logits/rejected": 0.5957974195480347,
+      "logps/chosen": -469.1175231933594,
+      "logps/rejected": -551.2074584960938,
+      "loss": 0.4664,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -1.9638084173202515,
+      "rewards/margins": 1.1078821420669556,
+      "rewards/rejected": -3.071690797805786,
+      "step": 930
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 3.760945397705828e-10,
+      "logits/chosen": -0.2911260724067688,
+      "logits/rejected": 0.8524629473686218,
+      "logps/chosen": -528.3253784179688,
+      "logps/rejected": -595.6929931640625,
+      "loss": 0.4725,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.9858026504516602,
+      "rewards/margins": 1.0581741333007812,
+      "rewards/rejected": -3.0439765453338623,
+      "step": 940
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.17975992204056e-11,
+      "logits/chosen": -0.4992304742336273,
+      "logits/rejected": 0.760162353515625,
+      "logps/chosen": -487.5731506347656,
+      "logps/rejected": -578.0675048828125,
+      "loss": 0.5097,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.031038761138916,
+      "rewards/margins": 1.11741042137146,
+      "rewards/rejected": -3.148449182510376,
+      "step": 950
+    },
+    {
+      "epoch": 1.0,
+      "step": 955,
+      "total_flos": 0.0,
+      "train_loss": 0.5274335609056563,
+      "train_runtime": 22383.3635,
+      "train_samples_per_second": 2.731,
+      "train_steps_per_second": 0.043
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 955,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}