Model save

README.md

@@ -0,0 +1,76 @@
+---
+license: apache-2.0
+base_model: alignment-handbook/zephyr-7b-sft-full
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: zephyr-7b-dpo-full
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# zephyr-7b-dpo-full
+
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.5168
+- Rewards/chosen: 0.1142
+- Rewards/rejected: -0.9329
+- Rewards/accuracies: 0.7695
+- Rewards/margins: 1.0471
+- Logps/rejected: -272.7659
+- Logps/chosen: -262.3270
+- Logits/rejected: -2.5421
+- Logits/chosen: -2.5184
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 128
+- total_eval_batch_size: 64
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.5155        | 0.21  | 100  | 0.5466          | 0.2422         | -0.6585          | 0.7656             | 0.9007          | -270.0222      | -261.0468    | -2.6271         | -2.6117       |
+| 0.4988        | 0.42  | 200  | 0.5294          | 0.0326         | -0.9993          | 0.7617             | 1.0320          | -273.4304      | -263.1426    | -2.5977         | -2.5791       |
+| 0.5354        | 0.63  | 300  | 0.5235          | 0.5272         | -0.4961          | 0.7656             | 1.0232          | -268.3980      | -258.1974    | -2.5449         | -2.5231       |
+| 0.5043        | 0.84  | 400  | 0.5181          | 0.1494         | -0.9114          | 0.7734             | 1.0608          | -272.5513      | -261.9752    | -2.5355         | -2.5122       |
+
+
+### Framework versions
+
+- Transformers 4.36.2
+- Pytorch 2.1.2
+- Datasets 2.14.6
+- Tokenizers 0.15.2

all_results.json

@@ -0,0 +1,21 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": -2.5183939933776855,
+    "eval_logits/rejected": -2.542128086090088,
+    "eval_logps/chosen": -262.3270263671875,
+    "eval_logps/rejected": -272.76593017578125,
+    "eval_loss": 0.5167638063430786,
+    "eval_rewards/accuracies": 0.76953125,
+    "eval_rewards/chosen": 0.11419209837913513,
+    "eval_rewards/margins": 1.047075629234314,
+    "eval_rewards/rejected": -0.932883620262146,
+    "eval_runtime": 110.44,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 18.109,
+    "eval_steps_per_second": 0.29,
+    "train_loss": 0.5342033233602675,
+    "train_runtime": 8549.6251,
+    "train_samples": 61135,
+    "train_samples_per_second": 7.151,
+    "train_steps_per_second": 0.056
+}

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": -2.5183939933776855,
+    "eval_logits/rejected": -2.542128086090088,
+    "eval_logps/chosen": -262.3270263671875,
+    "eval_logps/rejected": -272.76593017578125,
+    "eval_loss": 0.5167638063430786,
+    "eval_rewards/accuracies": 0.76953125,
+    "eval_rewards/chosen": 0.11419209837913513,
+    "eval_rewards/margins": 1.047075629234314,
+    "eval_rewards/rejected": -0.932883620262146,
+    "eval_runtime": 110.44,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 18.109,
+    "eval_steps_per_second": 0.29
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.36.2"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb781c5a1590b1b2cff3f9ae568198877a4545ed55fe160ed4826ae81c933ab8
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ec574e01e07b0d368cc7b0b4495c16999b1353f7da36e1c608480aea8d5989d
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dcd449ad535ecf5d109e1e308f8930738c139f57eb492b202edb552d8990a6d9
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

runs/May04_09-52-11_53d93485dab94692b1c5bb9b9bc47a1d000000/events.out.tfevents.1714816689.53d93485dab94692b1c5bb9b9bc47a1d000000.180.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3afe07ea9e46d8fb3286b3016e0b725094250be8c6228cf4d8fea9e09455424d
-size 33373
+oid sha256:e30abd7be5757adc9dc777ddbeef814e22c9bcdb5de5ca81263c974571ad1b3f
+size 38165

runs/May04_09-52-11_53d93485dab94692b1c5bb9b9bc47a1d000000/events.out.tfevents.1714825349.53d93485dab94692b1c5bb9b9bc47a1d000000.180.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1108930bd1a0dc770a00816cbd61772e15771a3db71f5708cd8cc73cb7aacca3
+size 828

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.5342033233602675,
+    "train_runtime": 8549.6251,
+    "train_samples": 61135,
+    "train_samples_per_second": 7.151,
+    "train_steps_per_second": 0.056
+}

trainer_state.json

@@ -0,0 +1,766 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 100,
+  "global_step": 478,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.0416666666666666e-08,
+      "logits/chosen": -2.7303905487060547,
+      "logits/rejected": -2.491323471069336,
+      "logps/chosen": -330.3969421386719,
+      "logps/rejected": -275.2320251464844,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.0416666666666667e-07,
+      "logits/chosen": -2.641596794128418,
+      "logits/rejected": -2.6669137477874756,
+      "logps/chosen": -247.8772735595703,
+      "logps/rejected": -215.13897705078125,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.4930555522441864,
+      "rewards/chosen": 0.003492807038128376,
+      "rewards/margins": 0.0050589111633598804,
+      "rewards/rejected": -0.0015661040088161826,
+      "step": 10
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.0833333333333333e-07,
+      "logits/chosen": -2.6507956981658936,
+      "logits/rejected": -2.6236915588378906,
+      "logps/chosen": -273.2765197753906,
+      "logps/rejected": -251.28515625,
+      "loss": 0.6864,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.009784101508557796,
+      "rewards/margins": 0.016010593622922897,
+      "rewards/rejected": -0.006226491183042526,
+      "step": 20
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": -2.710038661956787,
+      "logits/rejected": -2.690446615219116,
+      "logps/chosen": -284.8275451660156,
+      "logps/rejected": -277.4762878417969,
+      "loss": 0.6591,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.04264455288648605,
+      "rewards/margins": 0.07662136852741241,
+      "rewards/rejected": -0.03397681191563606,
+      "step": 30
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.1666666666666667e-07,
+      "logits/chosen": -2.6249327659606934,
+      "logits/rejected": -2.5905442237854004,
+      "logps/chosen": -292.10052490234375,
+      "logps/rejected": -273.6890869140625,
+      "loss": 0.5967,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.2472788393497467,
+      "rewards/margins": 0.2821725308895111,
+      "rewards/rejected": -0.034893669188022614,
+      "step": 40
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.999733114418725e-07,
+      "logits/chosen": -2.568021297454834,
+      "logits/rejected": -2.5584471225738525,
+      "logps/chosen": -310.86553955078125,
+      "logps/rejected": -303.51788330078125,
+      "loss": 0.5946,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.30402347445487976,
+      "rewards/margins": 0.43341511487960815,
+      "rewards/rejected": -0.12939168512821198,
+      "step": 50
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.990398100856366e-07,
+      "logits/chosen": -2.523190975189209,
+      "logits/rejected": -2.5076286792755127,
+      "logps/chosen": -256.69189453125,
+      "logps/rejected": -236.0826416015625,
+      "loss": 0.5986,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.15096399188041687,
+      "rewards/margins": 0.5717790722846985,
+      "rewards/rejected": -0.4208151400089264,
+      "step": 60
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.967775735898179e-07,
+      "logits/chosen": -2.566596269607544,
+      "logits/rejected": -2.539604663848877,
+      "logps/chosen": -277.34051513671875,
+      "logps/rejected": -253.0051727294922,
+      "loss": 0.555,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.4086539149284363,
+      "rewards/margins": 0.6797145009040833,
+      "rewards/rejected": -0.2710605561733246,
+      "step": 70
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.931986719649298e-07,
+      "logits/chosen": -2.5624489784240723,
+      "logits/rejected": -2.5468833446502686,
+      "logps/chosen": -264.40777587890625,
+      "logps/rejected": -277.04302978515625,
+      "loss": 0.5779,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": 0.5318883657455444,
+      "rewards/margins": 0.6019696593284607,
+      "rewards/rejected": -0.07008129358291626,
+      "step": 80
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.883222001996351e-07,
+      "logits/chosen": -2.5509326457977295,
+      "logits/rejected": -2.51672101020813,
+      "logps/chosen": -268.6958923339844,
+      "logps/rejected": -259.604248046875,
+      "loss": 0.5422,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.08079380542039871,
+      "rewards/margins": 0.6855199933052063,
+      "rewards/rejected": -0.7663138508796692,
+      "step": 90
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.821741763807186e-07,
+      "logits/chosen": -2.588852882385254,
+      "logits/rejected": -2.5498127937316895,
+      "logps/chosen": -285.1400146484375,
+      "logps/rejected": -271.84210205078125,
+      "loss": 0.5155,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.07230035215616226,
+      "rewards/margins": 0.7331563234329224,
+      "rewards/rejected": -0.8054566383361816,
+      "step": 100
+    },
+    {
+      "epoch": 0.21,
+      "eval_logits/chosen": -2.611691951751709,
+      "eval_logits/rejected": -2.6270973682403564,
+      "eval_logps/chosen": -261.0467529296875,
+      "eval_logps/rejected": -270.02215576171875,
+      "eval_loss": 0.5465933680534363,
+      "eval_rewards/accuracies": 0.765625,
+      "eval_rewards/chosen": 0.2422177791595459,
+      "eval_rewards/margins": 0.9007211327552795,
+      "eval_rewards/rejected": -0.6585034728050232,
+      "eval_runtime": 110.7473,
+      "eval_samples_per_second": 18.059,
+      "eval_steps_per_second": 0.289,
+      "step": 100
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.747874028753375e-07,
+      "logits/chosen": -2.6185667514801025,
+      "logits/rejected": -2.616579294204712,
+      "logps/chosen": -314.2326354980469,
+      "logps/rejected": -275.5158386230469,
+      "loss": 0.5319,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": 0.5200657844543457,
+      "rewards/margins": 0.9619105458259583,
+      "rewards/rejected": -0.4418447017669678,
+      "step": 110
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.662012913161997e-07,
+      "logits/chosen": -2.536588430404663,
+      "logits/rejected": -2.513469696044922,
+      "logps/chosen": -277.8913879394531,
+      "logps/rejected": -236.05810546875,
+      "loss": 0.553,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.32471024990081787,
+      "rewards/margins": 0.877071738243103,
+      "rewards/rejected": -0.5523615479469299,
+      "step": 120
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.5646165232345103e-07,
+      "logits/chosen": -2.5779452323913574,
+      "logits/rejected": -2.5439867973327637,
+      "logps/chosen": -259.2991638183594,
+      "logps/rejected": -295.8136901855469,
+      "loss": 0.5512,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 0.41181427240371704,
+      "rewards/margins": 0.8539644479751587,
+      "rewards/rejected": -0.442150354385376,
+      "step": 130
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.456204510851956e-07,
+      "logits/chosen": -2.6304543018341064,
+      "logits/rejected": -2.6672914028167725,
+      "logps/chosen": -291.71868896484375,
+      "logps/rejected": -261.11785888671875,
+      "loss": 0.5089,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.29633384943008423,
+      "rewards/margins": 0.8693134188652039,
+      "rewards/rejected": -0.5729796290397644,
+      "step": 140
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.337355301007335e-07,
+      "logits/chosen": -2.5667216777801514,
+      "logits/rejected": -2.551162004470825,
+      "logps/chosen": -287.31976318359375,
+      "logps/rejected": -248.73007202148438,
+      "loss": 0.5386,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": 0.08558840304613113,
+      "rewards/margins": 1.1002191305160522,
+      "rewards/rejected": -1.014630675315857,
+      "step": 150
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.2087030056579986e-07,
+      "logits/chosen": -2.5007309913635254,
+      "logits/rejected": -2.500986337661743,
+      "logps/chosen": -285.1891174316406,
+      "logps/rejected": -280.5631408691406,
+      "loss": 0.4998,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.1950608193874359,
+      "rewards/margins": 1.127140998840332,
+      "rewards/rejected": -0.932080090045929,
+      "step": 160
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.070934040463998e-07,
+      "logits/chosen": -2.535205841064453,
+      "logits/rejected": -2.4786734580993652,
+      "logps/chosen": -243.1121368408203,
+      "logps/rejected": -233.311767578125,
+      "loss": 0.5795,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.4860494136810303,
+      "rewards/margins": 0.6808937191963196,
+      "rewards/rejected": -0.19484423100948334,
+      "step": 170
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.9247834624635404e-07,
+      "logits/chosen": -2.559175968170166,
+      "logits/rejected": -2.6240577697753906,
+      "logps/chosen": -290.93310546875,
+      "logps/rejected": -285.2959289550781,
+      "loss": 0.5659,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.6741276979446411,
+      "rewards/margins": 0.9162510633468628,
+      "rewards/rejected": -0.2421233355998993,
+      "step": 180
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7710310482256523e-07,
+      "logits/chosen": -2.6226439476013184,
+      "logits/rejected": -2.5742576122283936,
+      "logps/chosen": -283.1707458496094,
+      "logps/rejected": -281.86883544921875,
+      "loss": 0.4922,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.17462602257728577,
+      "rewards/margins": 1.022231101989746,
+      "rewards/rejected": -0.8476050496101379,
+      "step": 190
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.610497133404795e-07,
+      "logits/chosen": -2.507990837097168,
+      "logits/rejected": -2.497056484222412,
+      "logps/chosen": -285.31103515625,
+      "logps/rejected": -257.9646911621094,
+      "loss": 0.4988,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.23659539222717285,
+      "rewards/margins": 0.9270391464233398,
+      "rewards/rejected": -1.1636345386505127,
+      "step": 200
+    },
+    {
+      "epoch": 0.42,
+      "eval_logits/chosen": -2.5791051387786865,
+      "eval_logits/rejected": -2.597656726837158,
+      "eval_logps/chosen": -263.1426086425781,
+      "eval_logps/rejected": -273.430419921875,
+      "eval_loss": 0.5293660759925842,
+      "eval_rewards/accuracies": 0.76171875,
+      "eval_rewards/chosen": 0.03263232856988907,
+      "eval_rewards/margins": 1.0319640636444092,
+      "eval_rewards/rejected": -0.9993317127227783,
+      "eval_runtime": 110.4395,
+      "eval_samples_per_second": 18.109,
+      "eval_steps_per_second": 0.29,
+      "step": 200
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4440382358952115e-07,
+      "logits/chosen": -2.5910706520080566,
+      "logits/rejected": -2.4992268085479736,
+      "logps/chosen": -277.0013122558594,
+      "logps/rejected": -247.256591796875,
+      "loss": 0.5301,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.11889532953500748,
+      "rewards/margins": 0.8859857320785522,
+      "rewards/rejected": -0.7670904397964478,
+      "step": 210
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.272542485937368e-07,
+      "logits/chosen": -2.515289783477783,
+      "logits/rejected": -2.5274503231048584,
+      "logps/chosen": -272.37982177734375,
+      "logps/rejected": -248.88864135742188,
+      "loss": 0.5328,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.5251632928848267,
+      "rewards/margins": 0.9271014332771301,
+      "rewards/rejected": -0.40193814039230347,
+      "step": 220
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.096924887558854e-07,
+      "logits/chosen": -2.510225772857666,
+      "logits/rejected": -2.4751222133636475,
+      "logps/chosen": -273.2637023925781,
+      "logps/rejected": -302.44781494140625,
+      "loss": 0.5334,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.19920429587364197,
+      "rewards/margins": 1.1725027561187744,
+      "rewards/rejected": -0.9732984304428101,
+      "step": 230
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.9181224366319943e-07,
+      "logits/chosen": -2.539271116256714,
+      "logits/rejected": -2.4599385261535645,
+      "logps/chosen": -256.0094909667969,
+      "logps/rejected": -257.43890380859375,
+      "loss": 0.5244,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.3548157811164856,
+      "rewards/margins": 1.0358607769012451,
+      "rewards/rejected": -1.3906766176223755,
+      "step": 240
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7370891215954565e-07,
+      "logits/chosen": -2.5517690181732178,
+      "logits/rejected": -2.5313477516174316,
+      "logps/chosen": -268.627197265625,
+      "logps/rejected": -268.250244140625,
+      "loss": 0.5021,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.05188677832484245,
+      "rewards/margins": 0.9482690691947937,
+      "rewards/rejected": -1.0001558065414429,
+      "step": 250
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.55479083351317e-07,
+      "logits/chosen": -2.5209710597991943,
+      "logits/rejected": -2.5104966163635254,
+      "logps/chosen": -281.55816650390625,
+      "logps/rejected": -252.4012451171875,
+      "loss": 0.5137,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.04571589455008507,
+      "rewards/margins": 0.9159566760063171,
+      "rewards/rejected": -0.9616724848747253,
+      "step": 260
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.3722002126275822e-07,
+      "logits/chosen": -2.527885913848877,
+      "logits/rejected": -2.5149693489074707,
+      "logps/chosen": -274.4147033691406,
+      "logps/rejected": -274.0594177246094,
+      "loss": 0.5049,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.08336488157510757,
+      "rewards/margins": 1.141265630722046,
+      "rewards/rejected": -1.0579006671905518,
+      "step": 270
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.19029145890313e-07,
+      "logits/chosen": -2.5481579303741455,
+      "logits/rejected": -2.549409866333008,
+      "logps/chosen": -274.05084228515625,
+      "logps/rejected": -248.07510375976562,
+      "loss": 0.5354,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.2714424133300781,
+      "rewards/margins": 0.9555227160453796,
+      "rewards/rejected": -0.6840803027153015,
+      "step": 280
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.0100351342479216e-07,
+      "logits/chosen": -2.5490007400512695,
+      "logits/rejected": -2.5670957565307617,
+      "logps/chosen": -285.09149169921875,
+      "logps/rejected": -280.4388122558594,
+      "loss": 0.5433,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": 0.27259546518325806,
+      "rewards/margins": 0.915685772895813,
+      "rewards/rejected": -0.6430902481079102,
+      "step": 290
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8323929841460178e-07,
+      "logits/chosen": -2.4967257976531982,
+      "logits/rejected": -2.456782579421997,
+      "logps/chosen": -298.3119812011719,
+      "logps/rejected": -319.1778564453125,
+      "loss": 0.5354,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.21918602287769318,
+      "rewards/margins": 0.772106945514679,
+      "rewards/rejected": -0.552920937538147,
+      "step": 300
+    },
+    {
+      "epoch": 0.63,
+      "eval_logits/chosen": -2.52313232421875,
+      "eval_logits/rejected": -2.5448904037475586,
+      "eval_logps/chosen": -258.1974182128906,
+      "eval_logps/rejected": -268.3979797363281,
+      "eval_loss": 0.5234915018081665,
+      "eval_rewards/accuracies": 0.765625,
+      "eval_rewards/chosen": 0.5271521210670471,
+      "eval_rewards/margins": 1.0232384204864502,
+      "eval_rewards/rejected": -0.49608638882637024,
+      "eval_runtime": 110.4859,
+      "eval_samples_per_second": 18.102,
+      "eval_steps_per_second": 0.29,
+      "step": 300
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6583128063291573e-07,
+      "logits/chosen": -2.5454556941986084,
+      "logits/rejected": -2.4632349014282227,
+      "logps/chosen": -271.7674255371094,
+      "logps/rejected": -225.97720336914062,
+      "loss": 0.5092,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.27959296107292175,
+      "rewards/margins": 0.8714268803596497,
+      "rewards/rejected": -0.5918339490890503,
+      "step": 310
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.488723393865766e-07,
+      "logits/chosen": -2.4652187824249268,
+      "logits/rejected": -2.5084636211395264,
+      "logps/chosen": -245.5274658203125,
+      "logps/rejected": -247.57510375976562,
+      "loss": 0.4848,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.3343454599380493,
+      "rewards/margins": 1.2461203336715698,
+      "rewards/rejected": -0.9117751121520996,
+      "step": 320
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3245295796480788e-07,
+      "logits/chosen": -2.5566658973693848,
+      "logits/rejected": -2.5458455085754395,
+      "logps/chosen": -284.0519714355469,
+      "logps/rejected": -261.8670349121094,
+      "loss": 0.5339,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.23870639503002167,
+      "rewards/margins": 0.971710205078125,
+      "rewards/rejected": -0.7330037951469421,
+      "step": 330
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1666074087171627e-07,
+      "logits/chosen": -2.4353904724121094,
+      "logits/rejected": -2.4397363662719727,
+      "logps/chosen": -260.65948486328125,
+      "logps/rejected": -255.44589233398438,
+      "loss": 0.5303,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": 0.24421295523643494,
+      "rewards/margins": 0.8219479322433472,
+      "rewards/rejected": -0.5777350664138794,
+      "step": 340
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0157994641835734e-07,
+      "logits/chosen": -2.4673986434936523,
+      "logits/rejected": -2.4053406715393066,
+      "logps/chosen": -248.89453125,
+      "logps/rejected": -241.627685546875,
+      "loss": 0.4901,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.24581214785575867,
+      "rewards/margins": 1.065041184425354,
+      "rewards/rejected": -0.8192291259765625,
+      "step": 350
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.729103716819111e-08,
+      "logits/chosen": -2.510063886642456,
+      "logits/rejected": -2.494906187057495,
+      "logps/chosen": -317.6705322265625,
+      "logps/rejected": -272.2886047363281,
+      "loss": 0.4807,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": 0.12424838542938232,
+      "rewards/margins": 1.1248137950897217,
+      "rewards/rejected": -1.0005652904510498,
+      "step": 360
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.387025063449081e-08,
+      "logits/chosen": -2.5114152431488037,
+      "logits/rejected": -2.487001657485962,
+      "logps/chosen": -310.0940856933594,
+      "logps/rejected": -267.06549072265625,
+      "loss": 0.505,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.1273534893989563,
+      "rewards/margins": 1.071047067642212,
+      "rewards/rejected": -0.9436934590339661,
+      "step": 370
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.138919252022435e-08,
+      "logits/chosen": -2.462189197540283,
+      "logits/rejected": -2.4567205905914307,
+      "logps/chosen": -280.74456787109375,
+      "logps/rejected": -289.3302307128906,
+      "loss": 0.4948,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.03562026470899582,
+      "rewards/margins": 0.998038113117218,
+      "rewards/rejected": -1.0336583852767944,
+      "step": 380
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.991445467064689e-08,
+      "logits/chosen": -2.474853992462158,
+      "logits/rejected": -2.440526247024536,
+      "logps/chosen": -273.2267150878906,
+      "logps/rejected": -282.3059387207031,
+      "loss": 0.5195,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.009925110265612602,
+      "rewards/margins": 0.8709251284599304,
+      "rewards/rejected": -0.8808503150939941,
+      "step": 390
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.9507259776993954e-08,
+      "logits/chosen": -2.439959764480591,
+      "logits/rejected": -2.4382309913635254,
+      "logps/chosen": -324.0129089355469,
+      "logps/rejected": -306.1255798339844,
+      "loss": 0.5043,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": 0.014118212275207043,
+      "rewards/margins": 1.1355412006378174,
+      "rewards/rejected": -1.1214230060577393,
+      "step": 400
+    },
+    {
+      "epoch": 0.84,
+      "eval_logits/chosen": -2.512162685394287,
+      "eval_logits/rejected": -2.535545587539673,
+      "eval_logps/chosen": -261.9752197265625,
+      "eval_logps/rejected": -272.55133056640625,
+      "eval_loss": 0.5180773735046387,
+      "eval_rewards/accuracies": 0.7734375,
+      "eval_rewards/chosen": 0.14937430620193481,
+      "eval_rewards/margins": 1.0607978105545044,
+      "eval_rewards/rejected": -0.9114235043525696,
+      "eval_runtime": 110.4584,
+      "eval_samples_per_second": 18.106,
+      "eval_steps_per_second": 0.29,
+      "step": 400
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 3.022313472693447e-08,
+      "logits/chosen": -2.4538586139678955,
+      "logits/rejected": -2.430116653442383,
+      "logps/chosen": -259.824462890625,
+      "logps/rejected": -259.74822998046875,
+      "loss": 0.5273,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.08546312898397446,
+      "rewards/margins": 1.1317038536071777,
+      "rewards/rejected": -1.0462408065795898,
+      "step": 410
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.2111614344599684e-08,
+      "logits/chosen": -2.4714725017547607,
+      "logits/rejected": -2.4253392219543457,
+      "logps/chosen": -275.32476806640625,
+      "logps/rejected": -297.7317810058594,
+      "loss": 0.5154,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": 0.21276769042015076,
+      "rewards/margins": 1.185510277748108,
+      "rewards/rejected": -0.9727425575256348,
+      "step": 420
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.521597710086439e-08,
+      "logits/chosen": -2.528935194015503,
+      "logits/rejected": -2.5169296264648438,
+      "logps/chosen": -295.73089599609375,
+      "logps/rejected": -264.444091796875,
+      "loss": 0.5192,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.1730698049068451,
+      "rewards/margins": 1.1036224365234375,
+      "rewards/rejected": -0.93055260181427,
+      "step": 430
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 9.57301420397924e-09,
+      "logits/chosen": -2.5310583114624023,
+      "logits/rejected": -2.525994300842285,
+      "logps/chosen": -258.9687805175781,
+      "logps/rejected": -266.4095764160156,
+      "loss": 0.4924,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.0379396490752697,
+      "rewards/margins": 0.9913676381111145,
+      "rewards/rejected": -0.9534279108047485,
+      "step": 440
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.212833302556258e-09,
+      "logits/chosen": -2.423041343688965,
+      "logits/rejected": -2.3861000537872314,
+      "logps/chosen": -245.48684692382812,
+      "logps/rejected": -242.0968017578125,
+      "loss": 0.5088,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": 0.1688850075006485,
+      "rewards/margins": 1.0535351037979126,
+      "rewards/rejected": -0.8846500515937805,
+      "step": 450
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.158697848236607e-09,
+      "logits/chosen": -2.437401294708252,
+      "logits/rejected": -2.418217420578003,
+      "logps/chosen": -244.7450714111328,
+      "logps/rejected": -240.0804443359375,
+      "loss": 0.4846,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.012026339769363403,
+      "rewards/margins": 1.0095479488372803,
+      "rewards/rejected": -1.0215742588043213,
+      "step": 460
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 4.269029751107489e-10,
+      "logits/chosen": -2.5386886596679688,
+      "logits/rejected": -2.498887062072754,
+      "logps/chosen": -298.3239440917969,
+      "logps/rejected": -276.06463623046875,
+      "loss": 0.4882,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.09313023090362549,
+      "rewards/margins": 1.2526581287384033,
+      "rewards/rejected": -1.1595278978347778,
+      "step": 470
+    },
+    {
+      "epoch": 1.0,
+      "step": 478,
+      "total_flos": 0.0,
+      "train_loss": 0.5342033233602675,
+      "train_runtime": 8549.6251,
+      "train_samples_per_second": 7.151,
+      "train_steps_per_second": 0.056
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 478,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}