Model save

README.md

@@ -0,0 +1,76 @@
+---
+license: other
+base_model: lewtun/gemma-7b-sft-full-ultrachat-v0
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: gemma-7b-dpo-full-ultrafeedback-beta-0.01
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# gemma-7b-dpo-full-ultrafeedback-beta-0.01
+
+This model is a fine-tuned version of [lewtun/gemma-7b-sft-full-ultrachat-v0](https://huggingface.co/lewtun/gemma-7b-sft-full-ultrachat-v0) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.4718
+- Rewards/chosen: -0.8508
+- Rewards/rejected: -2.1538
+- Rewards/accuracies: 0.7817
+- Rewards/margins: 1.3030
+- Logps/rejected: -1100.8470
+- Logps/chosen: -990.8950
+- Logits/rejected: 89.1600
+- Logits/chosen: 104.0108
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 2
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 128
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.552         | 0.21  | 100  | 0.5756          | -2.8657        | -3.5901          | 0.7460             | 0.7243          | -1244.4771     | -1192.3933   | 82.3244         | 96.5612       |
+| 0.501         | 0.42  | 200  | 0.4914          | -1.6427        | -2.6660          | 0.7817             | 1.0233          | -1152.0745     | -1070.0895   | 91.1202         | 105.1467      |
+| 0.4893        | 0.63  | 300  | 0.4810          | -1.6604        | -2.8398          | 0.7619             | 1.1794          | -1169.4480     | -1071.8550   | 87.4237         | 101.9799      |
+| 0.4759        | 0.84  | 400  | 0.4718          | -0.8508        | -2.1538          | 0.7817             | 1.3030          | -1100.8470     | -990.8950    | 89.1600         | 104.0108      |
+
+
+### Framework versions
+
+- Transformers 4.39.0.dev0
+- Pytorch 2.1.2+cu121
+- Datasets 2.14.6
+- Tokenizers 0.15.1

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.5152078939433867,
+    "train_runtime": 5321.4428,
+    "train_samples": 61135,
+    "train_samples_per_second": 11.488,
+    "train_steps_per_second": 0.09
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "eos_token_id": 1,
+  "pad_token_id": 0,
+  "transformers_version": "4.39.0.dev0"
+}

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34d37d3e20e6926aee530b2cab69fbb64f6a22df46d727a472fd44b710f424e8
+size 4995496656

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17111c7a14501043c274618bb8ec08e66ddf9447e102b75cb2e3ebf07dffad6f
+size 4982953168

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d3bbbffa1f9a6cd81d6b5b7273d1d26d399317f65517cbf5c74bd5f1d7e99c1
+size 4982953200

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91e5b1fb7457fbf470a4844162ff99e5397754c5576b938a46e11c613151cfbc
+size 2113988336

model.safetensors.index.json

@@ -0,0 +1,261 @@
+{
+  "metadata": {
+    "total_size": 17075361792
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

runs/Feb29_22-08-46_ip-26-0-161-178/events.out.tfevents.1709244785.ip-26-0-161-178.1167714.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:32292ebbde3c8f16459199568711098c4770b7cc9a9d4b9d373006d42f21b24c
-size 35892
+oid sha256:b5278b627d72d88fd1f2e5d65580a25bb7b3aa9b7fb3793b1bfc38c30b7afa80
+size 41062

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.5152078939433867,
+    "train_runtime": 5321.4428,
+    "train_samples": 61135,
+    "train_samples_per_second": 11.488,
+    "train_steps_per_second": 0.09
+}

trainer_state.json

@@ -0,0 +1,814 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.998691442030882,
+  "eval_steps": 100,
+  "global_step": 477,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "grad_norm": 19.889135020542987,
+      "learning_rate": 1.0416666666666666e-08,
+      "logits/chosen": 173.92840576171875,
+      "logits/rejected": 133.92694091796875,
+      "logps/chosen": -944.73779296875,
+      "logps/rejected": -877.1348876953125,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 18.648812633756638,
+      "learning_rate": 1.0416666666666667e-07,
+      "logits/chosen": 158.15008544921875,
+      "logits/rejected": 151.00912475585938,
+      "logps/chosen": -866.2621459960938,
+      "logps/rejected": -858.798583984375,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.4444444477558136,
+      "rewards/chosen": 0.0011810450814664364,
+      "rewards/margins": -9.146943193627521e-05,
+      "rewards/rejected": 0.0012725150445476174,
+      "step": 10
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 132.24935780648104,
+      "learning_rate": 2.0833333333333333e-07,
+      "logits/chosen": 162.58755493164062,
+      "logits/rejected": 141.87896728515625,
+      "logps/chosen": -871.9114990234375,
+      "logps/rejected": -856.8277587890625,
+      "loss": 0.69,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.02589397504925728,
+      "rewards/margins": 0.012839567847549915,
+      "rewards/rejected": 0.013054406270384789,
+      "step": 20
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 29.299152561983977,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": 148.5764617919922,
+      "logits/rejected": 148.60226440429688,
+      "logps/chosen": -834.642578125,
+      "logps/rejected": -821.4388427734375,
+      "loss": 0.6792,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.09783056378364563,
+      "rewards/margins": 0.019630378112196922,
+      "rewards/rejected": 0.07820017635822296,
+      "step": 30
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 20.662529763504946,
+      "learning_rate": 4.1666666666666667e-07,
+      "logits/chosen": 149.3659210205078,
+      "logits/rejected": 131.38018798828125,
+      "logps/chosen": -907.97509765625,
+      "logps/rejected": -912.3134765625,
+      "loss": 0.6489,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": 0.009358148090541363,
+      "rewards/margins": 0.11060555279254913,
+      "rewards/rejected": -0.1012474074959755,
+      "step": 40
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 37.45560682117618,
+      "learning_rate": 4.999731868769026e-07,
+      "logits/chosen": 141.44911193847656,
+      "logits/rejected": 129.70591735839844,
+      "logps/chosen": -925.8624877929688,
+      "logps/rejected": -942.8903198242188,
+      "loss": 0.6263,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.42348790168762207,
+      "rewards/margins": 0.23968985676765442,
+      "rewards/rejected": -0.6631777286529541,
+      "step": 50
+    },
+    {
+      "epoch": 0.13,
+      "grad_norm": 35.27997957962436,
+      "learning_rate": 4.990353313429303e-07,
+      "logits/chosen": 104.57405090332031,
+      "logits/rejected": 102.98997497558594,
+      "logps/chosen": -994.7630615234375,
+      "logps/rejected": -1038.8465576171875,
+      "loss": 0.6063,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.1830905675888062,
+      "rewards/margins": 0.3269960284233093,
+      "rewards/rejected": -1.5100867748260498,
+      "step": 60
+    },
+    {
+      "epoch": 0.15,
+      "grad_norm": 49.67365299996332,
+      "learning_rate": 4.967625656594781e-07,
+      "logits/chosen": 119.69749450683594,
+      "logits/rejected": 109.80323791503906,
+      "logps/chosen": -1019.2648315429688,
+      "logps/rejected": -1040.623779296875,
+      "loss": 0.5565,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.3080466985702515,
+      "rewards/margins": 0.5215330123901367,
+      "rewards/rejected": -1.8295797109603882,
+      "step": 70
+    },
+    {
+      "epoch": 0.17,
+      "grad_norm": 37.66437513199864,
+      "learning_rate": 4.93167072587771e-07,
+      "logits/chosen": 120.96675109863281,
+      "logits/rejected": 99.65242767333984,
+      "logps/chosen": -1079.8636474609375,
+      "logps/rejected": -1099.0491943359375,
+      "loss": 0.5567,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.4773412942886353,
+      "rewards/margins": 0.6472301483154297,
+      "rewards/rejected": -2.1245713233947754,
+      "step": 80
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 40.6528309450039,
+      "learning_rate": 4.882681251368548e-07,
+      "logits/chosen": 102.38163757324219,
+      "logits/rejected": 100.23868560791016,
+      "logps/chosen": -1027.5,
+      "logps/rejected": -1059.419921875,
+      "loss": 0.5699,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.292600393295288,
+      "rewards/margins": 0.5152118802070618,
+      "rewards/rejected": -1.8078124523162842,
+      "step": 90
+    },
+    {
+      "epoch": 0.21,
+      "grad_norm": 38.60858209436158,
+      "learning_rate": 4.820919832540181e-07,
+      "logits/chosen": 118.79133605957031,
+      "logits/rejected": 104.7783432006836,
+      "logps/chosen": -1127.0018310546875,
+      "logps/rejected": -1152.863037109375,
+      "loss": 0.552,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.3822240829467773,
+      "rewards/margins": 0.6033691167831421,
+      "rewards/rejected": -2.985593318939209,
+      "step": 100
+    },
+    {
+      "epoch": 0.21,
+      "eval_logits/chosen": 96.5611801147461,
+      "eval_logits/rejected": 82.32440185546875,
+      "eval_logps/chosen": -1192.393310546875,
+      "eval_logps/rejected": -1244.47705078125,
+      "eval_loss": 0.5756222009658813,
+      "eval_rewards/accuracies": 0.7460317611694336,
+      "eval_rewards/chosen": -2.8657495975494385,
+      "eval_rewards/margins": 0.7243059873580933,
+      "eval_rewards/rejected": -3.590055465698242,
+      "eval_runtime": 98.8707,
+      "eval_samples_per_second": 20.228,
+      "eval_steps_per_second": 0.637,
+      "step": 100
+    },
+    {
+      "epoch": 0.23,
+      "grad_norm": 34.77575362187783,
+      "learning_rate": 4.7467175306295647e-07,
+      "logits/chosen": 113.10517883300781,
+      "logits/rejected": 108.6837158203125,
+      "logps/chosen": -1058.0145263671875,
+      "logps/rejected": -1144.630859375,
+      "loss": 0.5379,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.9207611083984375,
+      "rewards/margins": 0.8025757074356079,
+      "rewards/rejected": -2.723336696624756,
+      "step": 110
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 37.99281988815398,
+      "learning_rate": 4.6604720940421207e-07,
+      "logits/chosen": 127.37345123291016,
+      "logits/rejected": 113.18028259277344,
+      "logps/chosen": -1034.3157958984375,
+      "logps/rejected": -1087.8922119140625,
+      "loss": 0.561,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.195056676864624,
+      "rewards/margins": 0.6624664068222046,
+      "rewards/rejected": -1.857522964477539,
+      "step": 120
+    },
+    {
+      "epoch": 0.27,
+      "grad_norm": 30.358803498156306,
+      "learning_rate": 4.5626458262912735e-07,
+      "logits/chosen": 117.21207427978516,
+      "logits/rejected": 122.45979309082031,
+      "logps/chosen": -1013.4964599609375,
+      "logps/rejected": -1076.5015869140625,
+      "loss": 0.5326,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.4635175466537476,
+      "rewards/margins": 0.6195371150970459,
+      "rewards/rejected": -2.083055019378662,
+      "step": 130
+    },
+    {
+      "epoch": 0.29,
+      "grad_norm": 33.33546006946111,
+      "learning_rate": 4.453763107901675e-07,
+      "logits/chosen": 120.96626281738281,
+      "logits/rejected": 99.00543212890625,
+      "logps/chosen": -1037.393798828125,
+      "logps/rejected": -1113.715576171875,
+      "loss": 0.5061,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.6467615365982056,
+      "rewards/margins": 0.959250807762146,
+      "rewards/rejected": -2.6060123443603516,
+      "step": 140
+    },
+    {
+      "epoch": 0.31,
+      "grad_norm": 36.216638034956645,
+      "learning_rate": 4.3344075855595097e-07,
+      "logits/chosen": 107.6335678100586,
+      "logits/rejected": 96.20887756347656,
+      "logps/chosen": -982.7149658203125,
+      "logps/rejected": -1042.314697265625,
+      "loss": 0.5158,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.9869440197944641,
+      "rewards/margins": 0.917373538017273,
+      "rewards/rejected": -1.9043172597885132,
+      "step": 150
+    },
+    {
+      "epoch": 0.33,
+      "grad_norm": 41.67256305682121,
+      "learning_rate": 4.2052190435769554e-07,
+      "logits/chosen": 113.42124938964844,
+      "logits/rejected": 102.60457611083984,
+      "logps/chosen": -1003.9383544921875,
+      "logps/rejected": -1058.088134765625,
+      "loss": 0.5019,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.3599907159805298,
+      "rewards/margins": 0.8325563669204712,
+      "rewards/rejected": -2.192547082901001,
+      "step": 160
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 45.33240260875551,
+      "learning_rate": 4.0668899744407567e-07,
+      "logits/chosen": 111.82502746582031,
+      "logits/rejected": 104.21772766113281,
+      "logps/chosen": -1027.629638671875,
+      "logps/rejected": -1113.077880859375,
+      "loss": 0.5128,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.418601632118225,
+      "rewards/margins": 0.8126633763313293,
+      "rewards/rejected": -2.231265068054199,
+      "step": 170
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 39.41765859124985,
+      "learning_rate": 3.920161866827889e-07,
+      "logits/chosen": 119.5909423828125,
+      "logits/rejected": 101.44880676269531,
+      "logps/chosen": -926.8933715820312,
+      "logps/rejected": -969.1472778320312,
+      "loss": 0.5088,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.4561997354030609,
+      "rewards/margins": 0.7785950899124146,
+      "rewards/rejected": -1.2347948551177979,
+      "step": 180
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 43.60352127706551,
+      "learning_rate": 3.765821230985757e-07,
+      "logits/chosen": 129.9263916015625,
+      "logits/rejected": 123.3591079711914,
+      "logps/chosen": -996.8531494140625,
+      "logps/rejected": -1059.179443359375,
+      "loss": 0.5073,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.3905432224273682,
+      "rewards/margins": 0.9462689161300659,
+      "rewards/rejected": -2.3368120193481445,
+      "step": 190
+    },
+    {
+      "epoch": 0.42,
+      "grad_norm": 41.35808766754113,
+      "learning_rate": 3.604695382782159e-07,
+      "logits/chosen": 124.6067886352539,
+      "logits/rejected": 118.35124206542969,
+      "logps/chosen": -1090.52783203125,
+      "logps/rejected": -1147.1068115234375,
+      "loss": 0.501,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.105625629425049,
+      "rewards/margins": 0.8636106252670288,
+      "rewards/rejected": -2.969236373901367,
+      "step": 200
+    },
+    {
+      "epoch": 0.42,
+      "eval_logits/chosen": 105.14665222167969,
+      "eval_logits/rejected": 91.12017059326172,
+      "eval_logps/chosen": -1070.0894775390625,
+      "eval_logps/rejected": -1152.074462890625,
+      "eval_loss": 0.49135974049568176,
+      "eval_rewards/accuracies": 0.7817460298538208,
+      "eval_rewards/chosen": -1.6427117586135864,
+      "eval_rewards/margins": 1.0233176946640015,
+      "eval_rewards/rejected": -2.666029453277588,
+      "eval_runtime": 98.2325,
+      "eval_samples_per_second": 20.36,
+      "eval_steps_per_second": 0.641,
+      "step": 200
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 40.009657639886804,
+      "learning_rate": 3.4376480090239047e-07,
+      "logits/chosen": 119.9991683959961,
+      "logits/rejected": 104.25712585449219,
+      "logps/chosen": -1026.137451171875,
+      "logps/rejected": -1069.336669921875,
+      "loss": 0.4974,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.1238807439804077,
+      "rewards/margins": 0.9766911268234253,
+      "rewards/rejected": -2.100571870803833,
+      "step": 210
+    },
+    {
+      "epoch": 0.46,
+      "grad_norm": 37.28316867834671,
+      "learning_rate": 3.265574537815398e-07,
+      "logits/chosen": 124.17179107666016,
+      "logits/rejected": 104.73355865478516,
+      "logps/chosen": -962.50244140625,
+      "logps/rejected": -1003.451171875,
+      "loss": 0.4961,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.7499821782112122,
+      "rewards/margins": 0.8301311731338501,
+      "rewards/rejected": -1.580113172531128,
+      "step": 220
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 54.558673930247686,
+      "learning_rate": 3.0893973387735683e-07,
+      "logits/chosen": 130.6039581298828,
+      "logits/rejected": 105.5712661743164,
+      "logps/chosen": -944.1550903320312,
+      "logps/rejected": -1030.9761962890625,
+      "loss": 0.4864,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.652384877204895,
+      "rewards/margins": 1.2353802919387817,
+      "rewards/rejected": -1.8877651691436768,
+      "step": 230
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 44.93883051182552,
+      "learning_rate": 2.910060778827554e-07,
+      "logits/chosen": 124.2732162475586,
+      "logits/rejected": 106.26287841796875,
+      "logps/chosen": -974.0313720703125,
+      "logps/rejected": -1065.3079833984375,
+      "loss": 0.4814,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.758819043636322,
+      "rewards/margins": 1.1402368545532227,
+      "rewards/rejected": -1.8990558385849,
+      "step": 240
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 44.64312531732698,
+      "learning_rate": 2.7285261601056697e-07,
+      "logits/chosen": 116.86869812011719,
+      "logits/rejected": 101.10758972167969,
+      "logps/chosen": -952.7760620117188,
+      "logps/rejected": -1048.2093505859375,
+      "loss": 0.4998,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.9232794046401978,
+      "rewards/margins": 1.1742398738861084,
+      "rewards/rejected": -2.0975191593170166,
+      "step": 250
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 40.68735732627764,
+      "learning_rate": 2.5457665670441937e-07,
+      "logits/chosen": 123.28076171875,
+      "logits/rejected": 102.635986328125,
+      "logps/chosen": -893.8860473632812,
+      "logps/rejected": -1004.1375732421875,
+      "loss": 0.4785,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.23751500248908997,
+      "rewards/margins": 1.1629530191421509,
+      "rewards/rejected": -1.4004679918289185,
+      "step": 260
+    },
+    {
+      "epoch": 0.57,
+      "grad_norm": 38.53534572807841,
+      "learning_rate": 2.3627616503391812e-07,
+      "logits/chosen": 118.7505874633789,
+      "logits/rejected": 110.69478607177734,
+      "logps/chosen": -951.8123779296875,
+      "logps/rejected": -1055.506103515625,
+      "loss": 0.4888,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.709445595741272,
+      "rewards/margins": 1.0170724391937256,
+      "rewards/rejected": -1.7265180349349976,
+      "step": 270
+    },
+    {
+      "epoch": 0.59,
+      "grad_norm": 39.95351928619943,
+      "learning_rate": 2.1804923757009882e-07,
+      "logits/chosen": 130.1121063232422,
+      "logits/rejected": 109.01686096191406,
+      "logps/chosen": -1051.2080078125,
+      "logps/rejected": -1150.492431640625,
+      "loss": 0.4577,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.1860597133636475,
+      "rewards/margins": 1.2530558109283447,
+      "rewards/rejected": -2.439115285873413,
+      "step": 280
+    },
+    {
+      "epoch": 0.61,
+      "grad_norm": 36.79184277903521,
+      "learning_rate": 1.9999357655598891e-07,
+      "logits/chosen": 125.7824935913086,
+      "logits/rejected": 115.7079086303711,
+      "logps/chosen": -1027.8924560546875,
+      "logps/rejected": -1112.7578125,
+      "loss": 0.4803,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.5402090549468994,
+      "rewards/margins": 1.2050487995147705,
+      "rewards/rejected": -2.74525785446167,
+      "step": 290
+    },
+    {
+      "epoch": 0.63,
+      "grad_norm": 42.82758424377552,
+      "learning_rate": 1.8220596619089573e-07,
+      "logits/chosen": 122.8412094116211,
+      "logits/rejected": 110.88859558105469,
+      "logps/chosen": -1007.1027221679688,
+      "logps/rejected": -1088.6361083984375,
+      "loss": 0.4893,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.636523962020874,
+      "rewards/margins": 1.004431962966919,
+      "rewards/rejected": -2.640956163406372,
+      "step": 300
+    },
+    {
+      "epoch": 0.63,
+      "eval_logits/chosen": 101.97990417480469,
+      "eval_logits/rejected": 87.42365264892578,
+      "eval_logps/chosen": -1071.85498046875,
+      "eval_logps/rejected": -1169.447998046875,
+      "eval_loss": 0.48103758692741394,
+      "eval_rewards/accuracies": 0.761904776096344,
+      "eval_rewards/chosen": -1.6603679656982422,
+      "eval_rewards/margins": 1.1793973445892334,
+      "eval_rewards/rejected": -2.8397650718688965,
+      "eval_runtime": 101.0727,
+      "eval_samples_per_second": 19.788,
+      "eval_steps_per_second": 0.623,
+      "step": 300
+    },
+    {
+      "epoch": 0.65,
+      "grad_norm": 38.54631864265249,
+      "learning_rate": 1.647817538357072e-07,
+      "logits/chosen": 113.38005065917969,
+      "logits/rejected": 110.6764144897461,
+      "logps/chosen": -995.9137573242188,
+      "logps/rejected": -1127.708251953125,
+      "loss": 0.4446,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.5860974788665771,
+      "rewards/margins": 1.1870568990707397,
+      "rewards/rejected": -2.7731544971466064,
+      "step": 310
+    },
+    {
+      "epoch": 0.67,
+      "grad_norm": 44.46063064657573,
+      "learning_rate": 1.478143389201113e-07,
+      "logits/chosen": 119.3581771850586,
+      "logits/rejected": 113.27059173583984,
+      "logps/chosen": -1008.1510620117188,
+      "logps/rejected": -1137.126220703125,
+      "loss": 0.4747,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.4959969520568848,
+      "rewards/margins": 1.4020049571990967,
+      "rewards/rejected": -2.898001194000244,
+      "step": 320
+    },
+    {
+      "epoch": 0.69,
+      "grad_norm": 34.37130611293822,
+      "learning_rate": 1.3139467229135998e-07,
+      "logits/chosen": 129.7974090576172,
+      "logits/rejected": 115.29783630371094,
+      "logps/chosen": -1069.2001953125,
+      "logps/rejected": -1156.9002685546875,
+      "loss": 0.4666,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.5326635837554932,
+      "rewards/margins": 1.1574156284332275,
+      "rewards/rejected": -2.6900792121887207,
+      "step": 330
+    },
+    {
+      "epoch": 0.71,
+      "grad_norm": 37.51413061347318,
+      "learning_rate": 1.1561076868822755e-07,
+      "logits/chosen": 135.8931884765625,
+      "logits/rejected": 121.91889953613281,
+      "logps/chosen": -1037.731689453125,
+      "logps/rejected": -1122.47412109375,
+      "loss": 0.466,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.680929183959961,
+      "rewards/margins": 1.011832356452942,
+      "rewards/rejected": -2.6927616596221924,
+      "step": 340
+    },
+    {
+      "epoch": 0.73,
+      "grad_norm": 44.78811791068193,
+      "learning_rate": 1.0054723495346482e-07,
+      "logits/chosen": 131.4265899658203,
+      "logits/rejected": 113.61830139160156,
+      "logps/chosen": -1038.681396484375,
+      "logps/rejected": -1120.1895751953125,
+      "loss": 0.4816,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.5251903533935547,
+      "rewards/margins": 1.061643362045288,
+      "rewards/rejected": -2.5868337154388428,
+      "step": 350
+    },
+    {
+      "epoch": 0.75,
+      "grad_norm": 48.873185347547825,
+      "learning_rate": 8.628481651367875e-08,
+      "logits/chosen": 131.0110321044922,
+      "logits/rejected": 106.7862548828125,
+      "logps/chosen": -1003.0614013671875,
+      "logps/rejected": -1117.0228271484375,
+      "loss": 0.4628,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.315259337425232,
+      "rewards/margins": 1.2398252487182617,
+      "rewards/rejected": -2.555084466934204,
+      "step": 360
+    },
+    {
+      "epoch": 0.77,
+      "grad_norm": 43.10852524272028,
+      "learning_rate": 7.289996455765748e-08,
+      "logits/chosen": 119.69869232177734,
+      "logits/rejected": 114.07867431640625,
+      "logps/chosen": -1036.690673828125,
+      "logps/rejected": -1146.2222900390625,
+      "loss": 0.4818,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.3266334533691406,
+      "rewards/margins": 1.4441391229629517,
+      "rewards/rejected": -2.7707724571228027,
+      "step": 370
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 38.151146819157155,
+      "learning_rate": 6.046442623320145e-08,
+      "logits/chosen": 122.61408996582031,
+      "logits/rejected": 126.60284423828125,
+      "logps/chosen": -1007.2545166015625,
+      "logps/rejected": -1128.876220703125,
+      "loss": 0.4616,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.2509572505950928,
+      "rewards/margins": 1.2325677871704102,
+      "rewards/rejected": -2.483525037765503,
+      "step": 380
+    },
+    {
+      "epoch": 0.82,
+      "grad_norm": 42.436222296241255,
+      "learning_rate": 4.904486005914027e-08,
+      "logits/chosen": 134.9565887451172,
+      "logits/rejected": 112.53916931152344,
+      "logps/chosen": -989.7498779296875,
+      "logps/rejected": -1100.5008544921875,
+      "loss": 0.466,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.100543737411499,
+      "rewards/margins": 1.3375900983810425,
+      "rewards/rejected": -2.438133955001831,
+      "step": 390
+    },
+    {
+      "epoch": 0.84,
+      "grad_norm": 45.12243805369195,
+      "learning_rate": 3.8702478614051345e-08,
+      "logits/chosen": 124.96537780761719,
+      "logits/rejected": 101.06147766113281,
+      "logps/chosen": -953.6491088867188,
+      "logps/rejected": -1035.303466796875,
+      "loss": 0.4759,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.9369322061538696,
+      "rewards/margins": 1.2312959432601929,
+      "rewards/rejected": -2.1682283878326416,
+      "step": 400
+    },
+    {
+      "epoch": 0.84,
+      "eval_logits/chosen": 104.01083374023438,
+      "eval_logits/rejected": 89.1600341796875,
+      "eval_logps/chosen": -990.8949584960938,
+      "eval_logps/rejected": -1100.8470458984375,
+      "eval_loss": 0.47177109122276306,
+      "eval_rewards/accuracies": 0.7817460298538208,
+      "eval_rewards/chosen": -0.8507668972015381,
+      "eval_rewards/margins": 1.3029894828796387,
+      "eval_rewards/rejected": -2.153756618499756,
+      "eval_runtime": 100.3183,
+      "eval_samples_per_second": 19.937,
+      "eval_steps_per_second": 0.628,
+      "step": 400
+    },
+    {
+      "epoch": 0.86,
+      "grad_norm": 41.02552262522525,
+      "learning_rate": 2.9492720416985e-08,
+      "logits/chosen": 134.49705505371094,
+      "logits/rejected": 125.9228744506836,
+      "logps/chosen": -977.2227783203125,
+      "logps/rejected": -1063.806396484375,
+      "loss": 0.4743,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.9780263900756836,
+      "rewards/margins": 0.9129725694656372,
+      "rewards/rejected": -1.8909988403320312,
+      "step": 410
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 38.58815640645097,
+      "learning_rate": 2.1464952759020856e-08,
+      "logits/chosen": 125.35734558105469,
+      "logits/rejected": 111.65289306640625,
+      "logps/chosen": -949.1236572265625,
+      "logps/rejected": -1051.959228515625,
+      "loss": 0.4823,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.9723941683769226,
+      "rewards/margins": 1.144566297531128,
+      "rewards/rejected": -2.116960287094116,
+      "step": 420
+    },
+    {
+      "epoch": 0.9,
+      "grad_norm": 31.91004161837456,
+      "learning_rate": 1.4662207078575684e-08,
+      "logits/chosen": 121.03411865234375,
+      "logits/rejected": 110.73746490478516,
+      "logps/chosen": -962.5640869140625,
+      "logps/rejected": -1089.4044189453125,
+      "loss": 0.4505,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.0843942165374756,
+      "rewards/margins": 1.454674243927002,
+      "rewards/rejected": -2.5390686988830566,
+      "step": 430
+    },
+    {
+      "epoch": 0.92,
+      "grad_norm": 37.24587161270191,
+      "learning_rate": 9.12094829893642e-09,
+      "logits/chosen": 122.1268081665039,
+      "logits/rejected": 124.87750244140625,
+      "logps/chosen": -947.4508666992188,
+      "logps/rejected": -1111.0018310546875,
+      "loss": 0.4731,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.1550958156585693,
+      "rewards/margins": 1.279443621635437,
+      "rewards/rejected": -2.434539318084717,
+      "step": 440
+    },
+    {
+      "epoch": 0.94,
+      "grad_norm": 33.88329943101718,
+      "learning_rate": 4.8708793644441086e-09,
+      "logits/chosen": 123.2929916381836,
+      "logits/rejected": 118.64930725097656,
+      "logps/chosen": -1003.01220703125,
+      "logps/rejected": -1144.7047119140625,
+      "loss": 0.4683,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.1160547733306885,
+      "rewards/margins": 1.316855549812317,
+      "rewards/rejected": -2.432910442352295,
+      "step": 450
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 40.762224983725595,
+      "learning_rate": 1.9347820230782295e-09,
+      "logits/chosen": 126.6349105834961,
+      "logits/rejected": 116.65708923339844,
+      "logps/chosen": -975.12548828125,
+      "logps/rejected": -1110.140625,
+      "loss": 0.4745,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.0417004823684692,
+      "rewards/margins": 1.3975470066070557,
+      "rewards/rejected": -2.4392473697662354,
+      "step": 460
+    },
+    {
+      "epoch": 0.98,
+      "grad_norm": 47.51695448791754,
+      "learning_rate": 3.2839470889836627e-10,
+      "logits/chosen": 136.34127807617188,
+      "logits/rejected": 115.61125183105469,
+      "logps/chosen": -1022.4205322265625,
+      "logps/rejected": -1111.9298095703125,
+      "loss": 0.4773,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.1639817953109741,
+      "rewards/margins": 1.1374759674072266,
+      "rewards/rejected": -2.3014578819274902,
+      "step": 470
+    },
+    {
+      "epoch": 1.0,
+      "step": 477,
+      "total_flos": 0.0,
+      "train_loss": 0.5152078939433867,
+      "train_runtime": 5321.4428,
+      "train_samples_per_second": 11.488,
+      "train_steps_per_second": 0.09
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 477,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}