commit from TX

README.md

@@ -0,0 +1,61 @@
+---
+license: apache-2.0
+base_model: alignment-handbook/zephyr-7b-sft-full
+tags:
+- alignment_handbook-handbook
+- generated_from_trainer
+datasets:
+- HuggingFaceH4/ultrafeedback_binarized
+model-index:
+- name: zephyr-7b-SimPO-SELM
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# zephyr-7b-SimPO-SELM
+
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the HuggingFaceH4/ultrafeedback_binarized dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 2
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 16
+- total_train_batch_size: 128
+- total_eval_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.40.2
+- Pytorch 2.1.2+cu121
+- Datasets 2.14.6
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.998691442030882,
+    "total_flos": 0.0,
+    "train_loss": 1.3288766333142166,
+    "train_runtime": 7370.4172,
+    "train_samples": 61135,
+    "train_samples_per_second": 8.295,
+    "train_steps_per_second": 0.065
+}

config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "alignment-handbook/zephyr-7b-sft-full",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.40.2",
+  "use_cache": true,
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.40.2"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e8cca466dddc3035d6308bab0f74adfba0aa7cdeb4b04f46b42291e2297cd5b
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9deb8d641a8e56416c73820b88386c5091f4a865a89ec0c1b21ba2012184896
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95b4010a7430665dc3ad781bf72f21e896575ac01a6db7ef77d74c73a62be9c2
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 2048,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.998691442030882,
+    "total_flos": 0.0,
+    "train_loss": 1.3288766333142166,
+    "train_runtime": 7370.4172,
+    "train_samples": 61135,
+    "train_samples_per_second": 8.295,
+    "train_steps_per_second": 0.065
+}

trainer_state.json

@@ -0,0 +1,1455 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.998691442030882,
+  "eval_steps": 500,
+  "global_step": 477,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.010468463752944255,
+      "grad_norm": 22.273845933282402,
+      "learning_rate": 5.208333333333333e-08,
+      "logits/chosen": -2.770733118057251,
+      "logits/rejected": -2.7281923294067383,
+      "logps/chosen": -1.0278276205062866,
+      "logps/rejected": -1.1733704805374146,
+      "loss": 1.7067,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.0278276205062866,
+      "rewards/margins": 0.14554281532764435,
+      "rewards/rejected": -1.1733704805374146,
+      "step": 5
+    },
+    {
+      "epoch": 0.02093692750588851,
+      "grad_norm": 19.65262772274273,
+      "learning_rate": 1.0416666666666667e-07,
+      "logits/chosen": -2.7513110637664795,
+      "logits/rejected": -2.725446939468384,
+      "logps/chosen": -0.9947978258132935,
+      "logps/rejected": -1.0709822177886963,
+      "loss": 1.7052,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.9947978258132935,
+      "rewards/margins": 0.07618431746959686,
+      "rewards/rejected": -1.0709822177886963,
+      "step": 10
+    },
+    {
+      "epoch": 0.031405391258832765,
+      "grad_norm": 20.86060525388248,
+      "learning_rate": 1.5624999999999999e-07,
+      "logits/chosen": -2.7313146591186523,
+      "logits/rejected": -2.697767496109009,
+      "logps/chosen": -0.9808584451675415,
+      "logps/rejected": -1.1414306163787842,
+      "loss": 1.6534,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.9808584451675415,
+      "rewards/margins": 0.16057202219963074,
+      "rewards/rejected": -1.1414306163787842,
+      "step": 15
+    },
+    {
+      "epoch": 0.04187385501177702,
+      "grad_norm": 18.455889774407584,
+      "learning_rate": 2.0833333333333333e-07,
+      "logits/chosen": -2.8040003776550293,
+      "logits/rejected": -2.7113003730773926,
+      "logps/chosen": -0.997894287109375,
+      "logps/rejected": -1.098174810409546,
+      "loss": 1.6781,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.997894287109375,
+      "rewards/margins": 0.1002805083990097,
+      "rewards/rejected": -1.098174810409546,
+      "step": 20
+    },
+    {
+      "epoch": 0.05234231876472128,
+      "grad_norm": 17.169851595349584,
+      "learning_rate": 2.604166666666667e-07,
+      "logits/chosen": -2.8004283905029297,
+      "logits/rejected": -2.777601718902588,
+      "logps/chosen": -0.9775938987731934,
+      "logps/rejected": -1.0392240285873413,
+      "loss": 1.6816,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.9775938987731934,
+      "rewards/margins": 0.061630137264728546,
+      "rewards/rejected": -1.0392240285873413,
+      "step": 25
+    },
+    {
+      "epoch": 0.06281078251766553,
+      "grad_norm": 17.60907134514165,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": -2.7743477821350098,
+      "logits/rejected": -2.734339714050293,
+      "logps/chosen": -0.8839586973190308,
+      "logps/rejected": -0.9886126518249512,
+      "loss": 1.7146,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.8839586973190308,
+      "rewards/margins": 0.1046539768576622,
+      "rewards/rejected": -0.9886126518249512,
+      "step": 30
+    },
+    {
+      "epoch": 0.07327924627060979,
+      "grad_norm": 19.779646214864712,
+      "learning_rate": 3.645833333333333e-07,
+      "logits/chosen": -2.7794618606567383,
+      "logits/rejected": -2.6966075897216797,
+      "logps/chosen": -0.9109536409378052,
+      "logps/rejected": -1.0334917306900024,
+      "loss": 1.6813,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.9109536409378052,
+      "rewards/margins": 0.12253817170858383,
+      "rewards/rejected": -1.0334917306900024,
+      "step": 35
+    },
+    {
+      "epoch": 0.08374771002355404,
+      "grad_norm": 17.435155033915933,
+      "learning_rate": 4.1666666666666667e-07,
+      "logits/chosen": -2.796264171600342,
+      "logits/rejected": -2.706550121307373,
+      "logps/chosen": -0.8977090120315552,
+      "logps/rejected": -1.1700526475906372,
+      "loss": 1.58,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.8977090120315552,
+      "rewards/margins": 0.2723436951637268,
+      "rewards/rejected": -1.1700526475906372,
+      "step": 40
+    },
+    {
+      "epoch": 0.0942161737764983,
+      "grad_norm": 20.376725274104686,
+      "learning_rate": 4.6874999999999996e-07,
+      "logits/chosen": -2.7916207313537598,
+      "logits/rejected": -2.7479615211486816,
+      "logps/chosen": -0.921708881855011,
+      "logps/rejected": -1.0657963752746582,
+      "loss": 1.5991,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.921708881855011,
+      "rewards/margins": 0.14408749341964722,
+      "rewards/rejected": -1.0657963752746582,
+      "step": 45
+    },
+    {
+      "epoch": 0.10468463752944256,
+      "grad_norm": 24.688089874039832,
+      "learning_rate": 4.999731868769026e-07,
+      "logits/chosen": -2.7189624309539795,
+      "logits/rejected": -2.657292604446411,
+      "logps/chosen": -0.9823735952377319,
+      "logps/rejected": -1.2429628372192383,
+      "loss": 1.5995,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.9823735952377319,
+      "rewards/margins": 0.2605893015861511,
+      "rewards/rejected": -1.2429628372192383,
+      "step": 50
+    },
+    {
+      "epoch": 0.11515310128238682,
+      "grad_norm": 20.10805225669153,
+      "learning_rate": 4.996716052911017e-07,
+      "logits/chosen": -2.685032367706299,
+      "logits/rejected": -2.6509592533111572,
+      "logps/chosen": -1.0144091844558716,
+      "logps/rejected": -1.2263678312301636,
+      "loss": 1.6066,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.0144091844558716,
+      "rewards/margins": 0.211958646774292,
+      "rewards/rejected": -1.2263678312301636,
+      "step": 55
+    },
+    {
+      "epoch": 0.12562156503533106,
+      "grad_norm": 23.40387121073354,
+      "learning_rate": 4.990353313429303e-07,
+      "logits/chosen": -2.717456817626953,
+      "logits/rejected": -2.669968843460083,
+      "logps/chosen": -1.071919322013855,
+      "logps/rejected": -1.261800765991211,
+      "loss": 1.5192,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.071919322013855,
+      "rewards/margins": 0.18988142907619476,
+      "rewards/rejected": -1.261800765991211,
+      "step": 60
+    },
+    {
+      "epoch": 0.1360900287882753,
+      "grad_norm": 22.52237455325428,
+      "learning_rate": 4.980652179769217e-07,
+      "logits/chosen": -2.681735038757324,
+      "logits/rejected": -2.568328857421875,
+      "logps/chosen": -1.0160045623779297,
+      "logps/rejected": -1.500519037246704,
+      "loss": 1.5548,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.0160045623779297,
+      "rewards/margins": 0.48451438546180725,
+      "rewards/rejected": -1.500519037246704,
+      "step": 65
+    },
+    {
+      "epoch": 0.14655849254121958,
+      "grad_norm": 29.253552274403233,
+      "learning_rate": 4.967625656594781e-07,
+      "logits/chosen": -2.5312721729278564,
+      "logits/rejected": -2.480713129043579,
+      "logps/chosen": -1.1166826486587524,
+      "logps/rejected": -1.405466914176941,
+      "loss": 1.5402,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.1166826486587524,
+      "rewards/margins": 0.2887844145298004,
+      "rewards/rejected": -1.405466914176941,
+      "step": 70
+    },
+    {
+      "epoch": 0.15702695629416383,
+      "grad_norm": 31.175214085255675,
+      "learning_rate": 4.951291206355559e-07,
+      "logits/chosen": -2.567188024520874,
+      "logits/rejected": -2.501024007797241,
+      "logps/chosen": -1.0800148248672485,
+      "logps/rejected": -1.4306553602218628,
+      "loss": 1.5108,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.0800148248672485,
+      "rewards/margins": 0.3506406545639038,
+      "rewards/rejected": -1.4306553602218628,
+      "step": 75
+    },
+    {
+      "epoch": 0.16749542004710807,
+      "grad_norm": 23.89446565720433,
+      "learning_rate": 4.93167072587771e-07,
+      "logits/chosen": -2.5460524559020996,
+      "logits/rejected": -2.4932408332824707,
+      "logps/chosen": -1.083786964416504,
+      "logps/rejected": -1.4139994382858276,
+      "loss": 1.557,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.083786964416504,
+      "rewards/margins": 0.3302125036716461,
+      "rewards/rejected": -1.4139994382858276,
+      "step": 80
+    },
+    {
+      "epoch": 0.17796388380005235,
+      "grad_norm": 35.04760138274803,
+      "learning_rate": 4.908790517010636e-07,
+      "logits/chosen": -2.555619239807129,
+      "logits/rejected": -2.49599289894104,
+      "logps/chosen": -1.027801275253296,
+      "logps/rejected": -1.3224855661392212,
+      "loss": 1.599,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.027801275253296,
+      "rewards/margins": 0.29468411207199097,
+      "rewards/rejected": -1.3224855661392212,
+      "step": 85
+    },
+    {
+      "epoch": 0.1884323475529966,
+      "grad_norm": 24.7603865878364,
+      "learning_rate": 4.882681251368548e-07,
+      "logits/chosen": -2.4858529567718506,
+      "logits/rejected": -2.458529472351074,
+      "logps/chosen": -1.0844331979751587,
+      "logps/rejected": -1.3636631965637207,
+      "loss": 1.5441,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.0844331979751587,
+      "rewards/margins": 0.27922993898391724,
+      "rewards/rejected": -1.3636631965637207,
+      "step": 90
+    },
+    {
+      "epoch": 0.19890081130594087,
+      "grad_norm": 31.023544965925865,
+      "learning_rate": 4.853377929214243e-07,
+      "logits/chosen": -2.50390887260437,
+      "logits/rejected": -2.40189790725708,
+      "logps/chosen": -1.1321955919265747,
+      "logps/rejected": -1.4152982234954834,
+      "loss": 1.5605,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.1321955919265747,
+      "rewards/margins": 0.28310269117355347,
+      "rewards/rejected": -1.4152982234954834,
+      "step": 95
+    },
+    {
+      "epoch": 0.2093692750588851,
+      "grad_norm": 29.51747892906268,
+      "learning_rate": 4.820919832540181e-07,
+      "logits/chosen": -2.4223530292510986,
+      "logits/rejected": -2.3233771324157715,
+      "logps/chosen": -1.153446912765503,
+      "logps/rejected": -1.5384576320648193,
+      "loss": 1.5601,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.153446912765503,
+      "rewards/margins": 0.3850107789039612,
+      "rewards/rejected": -1.5384576320648193,
+      "step": 100
+    },
+    {
+      "epoch": 0.21983773881182936,
+      "grad_norm": 30.939558746298886,
+      "learning_rate": 4.785350472409791e-07,
+      "logits/chosen": -2.3625779151916504,
+      "logits/rejected": -2.3349967002868652,
+      "logps/chosen": -1.1278746128082275,
+      "logps/rejected": -1.5877573490142822,
+      "loss": 1.4734,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.1278746128082275,
+      "rewards/margins": 0.4598826766014099,
+      "rewards/rejected": -1.5877573490142822,
+      "step": 105
+    },
+    {
+      "epoch": 0.23030620256477363,
+      "grad_norm": 45.105416576816324,
+      "learning_rate": 4.7467175306295647e-07,
+      "logits/chosen": -2.367917537689209,
+      "logits/rejected": -2.291100025177002,
+      "logps/chosen": -1.2823549509048462,
+      "logps/rejected": -1.7042672634124756,
+      "loss": 1.5012,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.2823549509048462,
+      "rewards/margins": 0.42191243171691895,
+      "rewards/rejected": -1.7042672634124756,
+      "step": 110
+    },
+    {
+      "epoch": 0.24077466631771788,
+      "grad_norm": 26.508438238557048,
+      "learning_rate": 4.70507279583015e-07,
+      "logits/chosen": -2.2769744396209717,
+      "logits/rejected": -2.1966700553894043,
+      "logps/chosen": -1.2124507427215576,
+      "logps/rejected": -1.7381147146224976,
+      "loss": 1.4243,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.2124507427215576,
+      "rewards/margins": 0.5256639719009399,
+      "rewards/rejected": -1.7381147146224976,
+      "step": 115
+    },
+    {
+      "epoch": 0.2512431300706621,
+      "grad_norm": 36.57968166514704,
+      "learning_rate": 4.6604720940421207e-07,
+      "logits/chosen": -2.2149672508239746,
+      "logits/rejected": -2.1592419147491455,
+      "logps/chosen": -1.2770636081695557,
+      "logps/rejected": -1.6595499515533447,
+      "loss": 1.5083,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2770636081695557,
+      "rewards/margins": 0.3824862837791443,
+      "rewards/rejected": -1.6595499515533447,
+      "step": 120
+    },
+    {
+      "epoch": 0.26171159382360637,
+      "grad_norm": 30.33542228792605,
+      "learning_rate": 4.612975213859487e-07,
+      "logits/chosen": -2.2143490314483643,
+      "logits/rejected": -2.177022695541382,
+      "logps/chosen": -1.2711435556411743,
+      "logps/rejected": -1.5601117610931396,
+      "loss": 1.5047,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2711435556411743,
+      "rewards/margins": 0.28896820545196533,
+      "rewards/rejected": -1.5601117610931396,
+      "step": 125
+    },
+    {
+      "epoch": 0.2721800575765506,
+      "grad_norm": 31.4132000967526,
+      "learning_rate": 4.5626458262912735e-07,
+      "logits/chosen": -2.0743236541748047,
+      "logits/rejected": -2.0040838718414307,
+      "logps/chosen": -1.3241310119628906,
+      "logps/rejected": -1.5913293361663818,
+      "loss": 1.47,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.3241310119628906,
+      "rewards/margins": 0.2671981453895569,
+      "rewards/rejected": -1.5913293361663818,
+      "step": 130
+    },
+    {
+      "epoch": 0.2826485213294949,
+      "grad_norm": 54.25569743941121,
+      "learning_rate": 4.5095513994085974e-07,
+      "logits/chosen": -1.8961169719696045,
+      "logits/rejected": -1.7147821187973022,
+      "logps/chosen": -1.5041354894638062,
+      "logps/rejected": -2.126206398010254,
+      "loss": 1.4478,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.5041354894638062,
+      "rewards/margins": 0.6220710277557373,
+      "rewards/rejected": -2.126206398010254,
+      "step": 135
+    },
+    {
+      "epoch": 0.29311698508243916,
+      "grad_norm": 43.983222727893036,
+      "learning_rate": 4.453763107901675e-07,
+      "logits/chosen": -1.5513172149658203,
+      "logits/rejected": -1.3982717990875244,
+      "logps/chosen": -1.5197192430496216,
+      "logps/rejected": -2.097294330596924,
+      "loss": 1.3929,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.5197192430496216,
+      "rewards/margins": 0.5775750279426575,
+      "rewards/rejected": -2.097294330596924,
+      "step": 140
+    },
+    {
+      "epoch": 0.3035854488353834,
+      "grad_norm": 28.469754678991702,
+      "learning_rate": 4.395355737667985e-07,
+      "logits/chosen": -1.4272167682647705,
+      "logits/rejected": -1.1418006420135498,
+      "logps/chosen": -1.4365965127944946,
+      "logps/rejected": -1.9005510807037354,
+      "loss": 1.442,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.4365965127944946,
+      "rewards/margins": 0.46395477652549744,
+      "rewards/rejected": -1.9005510807037354,
+      "step": 145
+    },
+    {
+      "epoch": 0.31405391258832765,
+      "grad_norm": 41.18537586594856,
+      "learning_rate": 4.3344075855595097e-07,
+      "logits/chosen": -1.3107974529266357,
+      "logits/rejected": -1.1333634853363037,
+      "logps/chosen": -1.333939552307129,
+      "logps/rejected": -1.8191452026367188,
+      "loss": 1.3703,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.333939552307129,
+      "rewards/margins": 0.4852055609226227,
+      "rewards/rejected": -1.8191452026367188,
+      "step": 150
+    },
+    {
+      "epoch": 0.3245223763412719,
+      "grad_norm": 43.62414965682813,
+      "learning_rate": 4.271000354423425e-07,
+      "logits/chosen": -0.9065883755683899,
+      "logits/rejected": -0.6428723931312561,
+      "logps/chosen": -1.6046466827392578,
+      "logps/rejected": -2.088477373123169,
+      "loss": 1.393,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.6046466827392578,
+      "rewards/margins": 0.48383063077926636,
+      "rewards/rejected": -2.088477373123169,
+      "step": 155
+    },
+    {
+      "epoch": 0.33499084009421615,
+      "grad_norm": 58.236255602056396,
+      "learning_rate": 4.2052190435769554e-07,
+      "logits/chosen": -0.5077437162399292,
+      "logits/rejected": -0.2604050636291504,
+      "logps/chosen": -1.8002322912216187,
+      "logps/rejected": -2.378002882003784,
+      "loss": 1.3526,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.8002322912216187,
+      "rewards/margins": 0.5777708292007446,
+      "rewards/rejected": -2.378002882003784,
+      "step": 160
+    },
+    {
+      "epoch": 0.34545930384716045,
+      "grad_norm": 37.97133552354845,
+      "learning_rate": 4.137151834863213e-07,
+      "logits/chosen": -0.9249609112739563,
+      "logits/rejected": -0.7287668585777283,
+      "logps/chosen": -1.729589819908142,
+      "logps/rejected": -2.2937121391296387,
+      "loss": 1.3021,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.729589819908142,
+      "rewards/margins": 0.5641222596168518,
+      "rewards/rejected": -2.2937121391296387,
+      "step": 165
+    },
+    {
+      "epoch": 0.3559277676001047,
+      "grad_norm": 58.781010614086675,
+      "learning_rate": 4.0668899744407567e-07,
+      "logits/chosen": -1.0361697673797607,
+      "logits/rejected": -0.7538817524909973,
+      "logps/chosen": -1.8726847171783447,
+      "logps/rejected": -2.472113847732544,
+      "loss": 1.4168,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.8726847171783447,
+      "rewards/margins": 0.5994290709495544,
+      "rewards/rejected": -2.472113847732544,
+      "step": 170
+    },
+    {
+      "epoch": 0.36639623135304894,
+      "grad_norm": 43.77533333071548,
+      "learning_rate": 3.994527650465352e-07,
+      "logits/chosen": -1.0444451570510864,
+      "logits/rejected": -0.6541527509689331,
+      "logps/chosen": -1.9884567260742188,
+      "logps/rejected": -2.6078011989593506,
+      "loss": 1.4179,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.9884567260742188,
+      "rewards/margins": 0.6193446516990662,
+      "rewards/rejected": -2.6078011989593506,
+      "step": 175
+    },
+    {
+      "epoch": 0.3768646951059932,
+      "grad_norm": 39.30408636565025,
+      "learning_rate": 3.920161866827889e-07,
+      "logits/chosen": -1.1477383375167847,
+      "logits/rejected": -0.9928515553474426,
+      "logps/chosen": -2.031057596206665,
+      "logps/rejected": -2.6500778198242188,
+      "loss": 1.3292,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.031057596206665,
+      "rewards/margins": 0.6190204620361328,
+      "rewards/rejected": -2.6500778198242188,
+      "step": 180
+    },
+    {
+      "epoch": 0.38733315885893743,
+      "grad_norm": 49.18397826699053,
+      "learning_rate": 3.8438923131177237e-07,
+      "logits/chosen": -1.3022983074188232,
+      "logits/rejected": -1.1097471714019775,
+      "logps/chosen": -2.1825501918792725,
+      "logps/rejected": -2.8575053215026855,
+      "loss": 1.3409,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.1825501918792725,
+      "rewards/margins": 0.6749551892280579,
+      "rewards/rejected": -2.8575053215026855,
+      "step": 185
+    },
+    {
+      "epoch": 0.39780162261188173,
+      "grad_norm": 59.443069428032004,
+      "learning_rate": 3.765821230985757e-07,
+      "logits/chosen": -1.309631586074829,
+      "logits/rejected": -1.1841962337493896,
+      "logps/chosen": -2.1491549015045166,
+      "logps/rejected": -2.7087011337280273,
+      "loss": 1.4201,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -2.1491549015045166,
+      "rewards/margins": 0.5595459938049316,
+      "rewards/rejected": -2.7087011337280273,
+      "step": 190
+    },
+    {
+      "epoch": 0.408270086364826,
+      "grad_norm": 58.81635631771188,
+      "learning_rate": 3.6860532770864005e-07,
+      "logits/chosen": -0.9547362327575684,
+      "logits/rejected": -0.6742358803749084,
+      "logps/chosen": -2.285407066345215,
+      "logps/rejected": -2.962996006011963,
+      "loss": 1.3681,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.285407066345215,
+      "rewards/margins": 0.6775889992713928,
+      "rewards/rejected": -2.962996006011963,
+      "step": 195
+    },
+    {
+      "epoch": 0.4187385501177702,
+      "grad_norm": 61.19133999845696,
+      "learning_rate": 3.604695382782159e-07,
+      "logits/chosen": -0.5743089914321899,
+      "logits/rejected": -0.5020885467529297,
+      "logps/chosen": -2.337622880935669,
+      "logps/rejected": -2.9621822834014893,
+      "loss": 1.3002,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.337622880935669,
+      "rewards/margins": 0.6245594620704651,
+      "rewards/rejected": -2.9621822834014893,
+      "step": 200
+    },
+    {
+      "epoch": 0.42920701387071447,
+      "grad_norm": 65.1623208567972,
+      "learning_rate": 3.5218566107988867e-07,
+      "logits/chosen": -0.27199602127075195,
+      "logits/rejected": 0.03203292936086655,
+      "logps/chosen": -2.4965696334838867,
+      "logps/rejected": -3.200573682785034,
+      "loss": 1.2788,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.4965696334838867,
+      "rewards/margins": 0.7040041089057922,
+      "rewards/rejected": -3.200573682785034,
+      "step": 205
+    },
+    {
+      "epoch": 0.4396754776236587,
+      "grad_norm": 81.87887955731682,
+      "learning_rate": 3.4376480090239047e-07,
+      "logits/chosen": -0.3396000266075134,
+      "logits/rejected": -0.0072940438985824585,
+      "logps/chosen": -2.886606216430664,
+      "logps/rejected": -3.522667407989502,
+      "loss": 1.3707,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.886606216430664,
+      "rewards/margins": 0.6360612511634827,
+      "rewards/rejected": -3.522667407989502,
+      "step": 210
+    },
+    {
+      "epoch": 0.45014394137660296,
+      "grad_norm": 54.53418228916395,
+      "learning_rate": 3.3521824616429284e-07,
+      "logits/chosen": -0.3693031072616577,
+      "logits/rejected": -0.05546772480010986,
+      "logps/chosen": -2.6358675956726074,
+      "logps/rejected": -3.3469290733337402,
+      "loss": 1.2589,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -2.6358675956726074,
+      "rewards/margins": 0.7110615968704224,
+      "rewards/rejected": -3.3469290733337402,
+      "step": 215
+    },
+    {
+      "epoch": 0.46061240512954726,
+      "grad_norm": 61.5068583100674,
+      "learning_rate": 3.265574537815398e-07,
+      "logits/chosen": -0.5512332320213318,
+      "logits/rejected": -0.04919959604740143,
+      "logps/chosen": -2.361748218536377,
+      "logps/rejected": -3.2458934783935547,
+      "loss": 1.2447,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.361748218536377,
+      "rewards/margins": 0.8841449022293091,
+      "rewards/rejected": -3.2458934783935547,
+      "step": 220
+    },
+    {
+      "epoch": 0.4710808688824915,
+      "grad_norm": 62.18444756933803,
+      "learning_rate": 3.1779403380910425e-07,
+      "logits/chosen": -0.4573966860771179,
+      "logits/rejected": 0.10396180301904678,
+      "logps/chosen": -2.5001068115234375,
+      "logps/rejected": -3.3893027305603027,
+      "loss": 1.2645,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -2.5001068115234375,
+      "rewards/margins": 0.8891960978507996,
+      "rewards/rejected": -3.3893027305603027,
+      "step": 225
+    },
+    {
+      "epoch": 0.48154933263543576,
+      "grad_norm": 69.58783982000695,
+      "learning_rate": 3.0893973387735683e-07,
+      "logits/chosen": -0.23679371178150177,
+      "logits/rejected": 0.2879559099674225,
+      "logps/chosen": -3.0258288383483887,
+      "logps/rejected": -4.09028959274292,
+      "loss": 1.1493,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -3.0258288383483887,
+      "rewards/margins": 1.0644607543945312,
+      "rewards/rejected": -4.09028959274292,
+      "step": 230
+    },
+    {
+      "epoch": 0.49201779638838,
+      "grad_norm": 65.85335888107983,
+      "learning_rate": 3.000064234440111e-07,
+      "logits/chosen": -0.09591199457645416,
+      "logits/rejected": 0.3684416711330414,
+      "logps/chosen": -2.861621379852295,
+      "logps/rejected": -3.635110855102539,
+      "loss": 1.2158,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.861621379852295,
+      "rewards/margins": 0.7734889984130859,
+      "rewards/rejected": -3.635110855102539,
+      "step": 235
+    },
+    {
+      "epoch": 0.5024862601413242,
+      "grad_norm": 51.47234347289653,
+      "learning_rate": 2.910060778827554e-07,
+      "logits/chosen": -0.214645653963089,
+      "logits/rejected": 0.15288104116916656,
+      "logps/chosen": -2.2756009101867676,
+      "logps/rejected": -3.0374677181243896,
+      "loss": 1.2342,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.2756009101867676,
+      "rewards/margins": 0.7618668079376221,
+      "rewards/rejected": -3.0374677181243896,
+      "step": 240
+    },
+    {
+      "epoch": 0.5129547238942685,
+      "grad_norm": 67.41003053954276,
+      "learning_rate": 2.8195076242990116e-07,
+      "logits/chosen": -0.233026385307312,
+      "logits/rejected": 0.1556404083967209,
+      "logps/chosen": -2.4698057174682617,
+      "logps/rejected": -3.129883050918579,
+      "loss": 1.2922,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -2.4698057174682617,
+      "rewards/margins": 0.6600776314735413,
+      "rewards/rejected": -3.129883050918579,
+      "step": 245
+    },
+    {
+      "epoch": 0.5234231876472127,
+      "grad_norm": 70.04740584020762,
+      "learning_rate": 2.7285261601056697e-07,
+      "logits/chosen": -0.32400113344192505,
+      "logits/rejected": 0.12020448595285416,
+      "logps/chosen": -2.376567840576172,
+      "logps/rejected": -3.175973653793335,
+      "loss": 1.3275,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -2.376567840576172,
+      "rewards/margins": 0.799405574798584,
+      "rewards/rejected": -3.175973653793335,
+      "step": 250
+    },
+    {
+      "epoch": 0.533891651400157,
+      "grad_norm": 49.05614284788625,
+      "learning_rate": 2.6372383496608186e-07,
+      "logits/chosen": -0.07061919569969177,
+      "logits/rejected": 0.49763208627700806,
+      "logps/chosen": -2.67657470703125,
+      "logps/rejected": -3.5274453163146973,
+      "loss": 1.1737,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -2.67657470703125,
+      "rewards/margins": 0.8508706092834473,
+      "rewards/rejected": -3.5274453163146973,
+      "step": 255
+    },
+    {
+      "epoch": 0.5443601151531012,
+      "grad_norm": 73.89578979282193,
+      "learning_rate": 2.5457665670441937e-07,
+      "logits/chosen": -0.18951067328453064,
+      "logits/rejected": 0.2982691824436188,
+      "logps/chosen": -2.413672924041748,
+      "logps/rejected": -3.579845905303955,
+      "loss": 1.2239,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.413672924041748,
+      "rewards/margins": 1.166172981262207,
+      "rewards/rejected": -3.579845905303955,
+      "step": 260
+    },
+    {
+      "epoch": 0.5548285789060455,
+      "grad_norm": 54.444851596761815,
+      "learning_rate": 2.454233432955807e-07,
+      "logits/chosen": -0.05852239206433296,
+      "logits/rejected": 0.3520205318927765,
+      "logps/chosen": -2.7941486835479736,
+      "logps/rejected": -3.663743495941162,
+      "loss": 1.1459,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.7941486835479736,
+      "rewards/margins": 0.8695951700210571,
+      "rewards/rejected": -3.663743495941162,
+      "step": 265
+    },
+    {
+      "epoch": 0.5652970426589898,
+      "grad_norm": 79.13341556564741,
+      "learning_rate": 2.3627616503391812e-07,
+      "logits/chosen": 0.12447645515203476,
+      "logits/rejected": 0.48517122864723206,
+      "logps/chosen": -3.074282169342041,
+      "logps/rejected": -3.778022289276123,
+      "loss": 1.2406,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -3.074282169342041,
+      "rewards/margins": 0.7037407159805298,
+      "rewards/rejected": -3.778022289276123,
+      "step": 270
+    },
+    {
+      "epoch": 0.575765506411934,
+      "grad_norm": 65.77375872071654,
+      "learning_rate": 2.2714738398943308e-07,
+      "logits/chosen": 0.10315855592489243,
+      "logits/rejected": 0.6628378629684448,
+      "logps/chosen": -2.9850118160247803,
+      "logps/rejected": -3.925368070602417,
+      "loss": 1.1927,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.9850118160247803,
+      "rewards/margins": 0.9403561353683472,
+      "rewards/rejected": -3.925368070602417,
+      "step": 275
+    },
+    {
+      "epoch": 0.5862339701648783,
+      "grad_norm": 68.7072404828179,
+      "learning_rate": 2.1804923757009882e-07,
+      "logits/chosen": 0.14654329419136047,
+      "logits/rejected": 0.7450932264328003,
+      "logps/chosen": -2.682091474533081,
+      "logps/rejected": -3.504145860671997,
+      "loss": 1.224,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -2.682091474533081,
+      "rewards/margins": 0.8220542669296265,
+      "rewards/rejected": -3.504145860671997,
+      "step": 280
+    },
+    {
+      "epoch": 0.5967024339178225,
+      "grad_norm": 61.507759714231206,
+      "learning_rate": 2.089939221172446e-07,
+      "logits/chosen": 0.048857107758522034,
+      "logits/rejected": 0.40524905920028687,
+      "logps/chosen": -2.660057306289673,
+      "logps/rejected": -3.4508941173553467,
+      "loss": 1.2054,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.660057306289673,
+      "rewards/margins": 0.7908371090888977,
+      "rewards/rejected": -3.4508941173553467,
+      "step": 285
+    },
+    {
+      "epoch": 0.6071708976707668,
+      "grad_norm": 68.54045280221443,
+      "learning_rate": 1.9999357655598891e-07,
+      "logits/chosen": 0.09905894100666046,
+      "logits/rejected": 0.6364008784294128,
+      "logps/chosen": -2.7884061336517334,
+      "logps/rejected": -3.744817018508911,
+      "loss": 1.1811,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.7884061336517334,
+      "rewards/margins": 0.9564107060432434,
+      "rewards/rejected": -3.744817018508911,
+      "step": 290
+    },
+    {
+      "epoch": 0.6176393614237111,
+      "grad_norm": 69.13964326129287,
+      "learning_rate": 1.9106026612264315e-07,
+      "logits/chosen": 0.03760681673884392,
+      "logits/rejected": 0.6295174956321716,
+      "logps/chosen": -2.8681061267852783,
+      "logps/rejected": -3.6628737449645996,
+      "loss": 1.2487,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.8681061267852783,
+      "rewards/margins": 0.7947677373886108,
+      "rewards/rejected": -3.6628737449645996,
+      "step": 295
+    },
+    {
+      "epoch": 0.6281078251766553,
+      "grad_norm": 68.24880568918918,
+      "learning_rate": 1.8220596619089573e-07,
+      "logits/chosen": 0.05892186239361763,
+      "logits/rejected": 0.4875253140926361,
+      "logps/chosen": -2.8865182399749756,
+      "logps/rejected": -3.7262775897979736,
+      "loss": 1.2566,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -2.8865182399749756,
+      "rewards/margins": 0.8397592306137085,
+      "rewards/rejected": -3.7262775897979736,
+      "step": 300
+    },
+    {
+      "epoch": 0.6385762889295996,
+      "grad_norm": 63.303760044846996,
+      "learning_rate": 1.7344254621846017e-07,
+      "logits/chosen": 0.2723705768585205,
+      "logits/rejected": 0.7761790752410889,
+      "logps/chosen": -2.930652141571045,
+      "logps/rejected": -3.947206974029541,
+      "loss": 1.1113,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.930652141571045,
+      "rewards/margins": 1.0165544748306274,
+      "rewards/rejected": -3.947206974029541,
+      "step": 305
+    },
+    {
+      "epoch": 0.6490447526825438,
+      "grad_norm": 74.87714108878664,
+      "learning_rate": 1.647817538357072e-07,
+      "logits/chosen": 0.40727514028549194,
+      "logits/rejected": 0.6944658160209656,
+      "logps/chosen": -2.684722900390625,
+      "logps/rejected": -3.821376085281372,
+      "loss": 1.1187,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.684722900390625,
+      "rewards/margins": 1.1366530656814575,
+      "rewards/rejected": -3.821376085281372,
+      "step": 310
+    },
+    {
+      "epoch": 0.6595132164354881,
+      "grad_norm": 78.50549857938152,
+      "learning_rate": 1.562351990976095e-07,
+      "logits/chosen": 0.38405269384384155,
+      "logits/rejected": 0.9985488653182983,
+      "logps/chosen": -2.8799889087677,
+      "logps/rejected": -4.003491401672363,
+      "loss": 1.1249,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -2.8799889087677,
+      "rewards/margins": 1.1235021352767944,
+      "rewards/rejected": -4.003491401672363,
+      "step": 315
+    },
+    {
+      "epoch": 0.6699816801884323,
+      "grad_norm": 80.43258089949055,
+      "learning_rate": 1.478143389201113e-07,
+      "logits/chosen": 0.6803662776947021,
+      "logits/rejected": 0.8349927067756653,
+      "logps/chosen": -2.9795212745666504,
+      "logps/rejected": -3.9721953868865967,
+      "loss": 1.1932,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.9795212745666504,
+      "rewards/margins": 0.9926745295524597,
+      "rewards/rejected": -3.9721953868865967,
+      "step": 320
+    },
+    {
+      "epoch": 0.6804501439413766,
+      "grad_norm": 73.34968580085386,
+      "learning_rate": 1.3953046172178413e-07,
+      "logits/chosen": 0.5375081300735474,
+      "logits/rejected": 1.211456060409546,
+      "logps/chosen": -2.997009754180908,
+      "logps/rejected": -3.9822990894317627,
+      "loss": 1.1884,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -2.997009754180908,
+      "rewards/margins": 0.9852892160415649,
+      "rewards/rejected": -3.9822990894317627,
+      "step": 325
+    },
+    {
+      "epoch": 0.6909186076943209,
+      "grad_norm": 67.18394255987927,
+      "learning_rate": 1.3139467229135998e-07,
+      "logits/chosen": 0.46189865469932556,
+      "logits/rejected": 1.051397681236267,
+      "logps/chosen": -2.7415473461151123,
+      "logps/rejected": -3.6691060066223145,
+      "loss": 1.1945,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.7415473461151123,
+      "rewards/margins": 0.9275587201118469,
+      "rewards/rejected": -3.6691060066223145,
+      "step": 330
+    },
+    {
+      "epoch": 0.7013870714472651,
+      "grad_norm": 64.70715932336967,
+      "learning_rate": 1.2341787690142435e-07,
+      "logits/chosen": 0.2790513038635254,
+      "logits/rejected": 0.8592203259468079,
+      "logps/chosen": -2.72259783744812,
+      "logps/rejected": -3.6879966259002686,
+      "loss": 1.2498,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -2.72259783744812,
+      "rewards/margins": 0.965398907661438,
+      "rewards/rejected": -3.6879966259002686,
+      "step": 335
+    },
+    {
+      "epoch": 0.7118555352002094,
+      "grad_norm": 61.63125001377349,
+      "learning_rate": 1.1561076868822755e-07,
+      "logits/chosen": 0.34762030839920044,
+      "logits/rejected": 0.8974772691726685,
+      "logps/chosen": -2.9279873371124268,
+      "logps/rejected": -3.637186050415039,
+      "loss": 1.1831,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -2.9279873371124268,
+      "rewards/margins": 0.7091987133026123,
+      "rewards/rejected": -3.637186050415039,
+      "step": 340
+    },
+    {
+      "epoch": 0.7223239989531536,
+      "grad_norm": 69.9867150019491,
+      "learning_rate": 1.0798381331721107e-07,
+      "logits/chosen": 0.42011842131614685,
+      "logits/rejected": 0.9746605753898621,
+      "logps/chosen": -2.681163787841797,
+      "logps/rejected": -3.532362699508667,
+      "loss": 1.1815,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -2.681163787841797,
+      "rewards/margins": 0.8511991500854492,
+      "rewards/rejected": -3.532362699508667,
+      "step": 345
+    },
+    {
+      "epoch": 0.7327924627060979,
+      "grad_norm": 75.66340151463851,
+      "learning_rate": 1.0054723495346482e-07,
+      "logits/chosen": 0.5741020441055298,
+      "logits/rejected": 0.8616167902946472,
+      "logps/chosen": -2.8164191246032715,
+      "logps/rejected": -3.6091017723083496,
+      "loss": 1.2396,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -2.8164191246032715,
+      "rewards/margins": 0.7926827669143677,
+      "rewards/rejected": -3.6091017723083496,
+      "step": 350
+    },
+    {
+      "epoch": 0.7432609264590422,
+      "grad_norm": 74.17047524574353,
+      "learning_rate": 9.331100255592436e-08,
+      "logits/chosen": 0.5733584761619568,
+      "logits/rejected": 1.1586692333221436,
+      "logps/chosen": -2.8743791580200195,
+      "logps/rejected": -4.042096138000488,
+      "loss": 1.1432,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -2.8743791580200195,
+      "rewards/margins": 1.1677173376083374,
+      "rewards/rejected": -4.042096138000488,
+      "step": 355
+    },
+    {
+      "epoch": 0.7537293902119864,
+      "grad_norm": 73.14407398619424,
+      "learning_rate": 8.628481651367875e-08,
+      "logits/chosen": 0.29708194732666016,
+      "logits/rejected": 0.8764042854309082,
+      "logps/chosen": -3.0110630989074707,
+      "logps/rejected": -3.7496750354766846,
+      "loss": 1.1891,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -3.0110630989074707,
+      "rewards/margins": 0.7386118173599243,
+      "rewards/rejected": -3.7496750354766846,
+      "step": 360
+    },
+    {
+      "epoch": 0.7641978539649307,
+      "grad_norm": 84.79816410361819,
+      "learning_rate": 7.947809564230445e-08,
+      "logits/chosen": 0.5876302719116211,
+      "logits/rejected": 1.1461758613586426,
+      "logps/chosen": -3.0007166862487793,
+      "logps/rejected": -4.093684673309326,
+      "loss": 1.2364,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -3.0007166862487793,
+      "rewards/margins": 1.0929677486419678,
+      "rewards/rejected": -4.093684673309326,
+      "step": 365
+    },
+    {
+      "epoch": 0.7746663177178749,
+      "grad_norm": 76.4181977449788,
+      "learning_rate": 7.289996455765748e-08,
+      "logits/chosen": 0.7314077615737915,
+      "logits/rejected": 1.2388323545455933,
+      "logps/chosen": -3.2148640155792236,
+      "logps/rejected": -4.329017639160156,
+      "loss": 1.1397,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -3.2148640155792236,
+      "rewards/margins": 1.1141533851623535,
+      "rewards/rejected": -4.329017639160156,
+      "step": 370
+    },
+    {
+      "epoch": 0.7851347814708192,
+      "grad_norm": 79.59752890916117,
+      "learning_rate": 6.655924144404906e-08,
+      "logits/chosen": 0.7917232513427734,
+      "logits/rejected": 0.8404657244682312,
+      "logps/chosen": -3.1535632610321045,
+      "logps/rejected": -4.065492153167725,
+      "loss": 1.117,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -3.1535632610321045,
+      "rewards/margins": 0.9119284749031067,
+      "rewards/rejected": -4.065492153167725,
+      "step": 375
+    },
+    {
+      "epoch": 0.7956032452237635,
+      "grad_norm": 83.98695473040011,
+      "learning_rate": 6.046442623320145e-08,
+      "logits/chosen": 0.9315813779830933,
+      "logits/rejected": 1.060347557067871,
+      "logps/chosen": -3.2030982971191406,
+      "logps/rejected": -4.173519134521484,
+      "loss": 1.2339,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -3.2030982971191406,
+      "rewards/margins": 0.9704204797744751,
+      "rewards/rejected": -4.173519134521484,
+      "step": 380
+    },
+    {
+      "epoch": 0.8060717089767077,
+      "grad_norm": 73.80815123078858,
+      "learning_rate": 5.4623689209832484e-08,
+      "logits/chosen": 0.6560049653053284,
+      "logits/rejected": 1.1292173862457275,
+      "logps/chosen": -3.0976970195770264,
+      "logps/rejected": -4.078602313995361,
+      "loss": 1.1986,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -3.0976970195770264,
+      "rewards/margins": 0.9809052348136902,
+      "rewards/rejected": -4.078602313995361,
+      "step": 385
+    },
+    {
+      "epoch": 0.816540172729652,
+      "grad_norm": 69.90570951076663,
+      "learning_rate": 4.904486005914027e-08,
+      "logits/chosen": 0.3861357569694519,
+      "logits/rejected": 1.0230625867843628,
+      "logps/chosen": -3.0694098472595215,
+      "logps/rejected": -4.231044769287109,
+      "loss": 1.12,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -3.0694098472595215,
+      "rewards/margins": 1.161635160446167,
+      "rewards/rejected": -4.231044769287109,
+      "step": 390
+    },
+    {
+      "epoch": 0.8270086364825961,
+      "grad_norm": 71.22815378604164,
+      "learning_rate": 4.373541737087263e-08,
+      "logits/chosen": 0.35766056180000305,
+      "logits/rejected": 1.0771682262420654,
+      "logps/chosen": -2.899632692337036,
+      "logps/rejected": -3.877995252609253,
+      "loss": 1.1684,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -2.899632692337036,
+      "rewards/margins": 0.9783622622489929,
+      "rewards/rejected": -3.877995252609253,
+      "step": 395
+    },
+    {
+      "epoch": 0.8374771002355405,
+      "grad_norm": 97.45446724327614,
+      "learning_rate": 3.8702478614051345e-08,
+      "logits/chosen": 0.5966876745223999,
+      "logits/rejected": 1.0948045253753662,
+      "logps/chosen": -2.8743724822998047,
+      "logps/rejected": -3.8696448802948,
+      "loss": 1.2227,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -2.8743724822998047,
+      "rewards/margins": 0.9952724575996399,
+      "rewards/rejected": -3.8696448802948,
+      "step": 400
+    },
+    {
+      "epoch": 0.8479455639884846,
+      "grad_norm": 77.52250073936997,
+      "learning_rate": 3.3952790595787986e-08,
+      "logits/chosen": 0.5455753207206726,
+      "logits/rejected": 0.8719021081924438,
+      "logps/chosen": -3.101245880126953,
+      "logps/rejected": -3.933109998703003,
+      "loss": 1.2031,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -3.101245880126953,
+      "rewards/margins": 0.8318643569946289,
+      "rewards/rejected": -3.933109998703003,
+      "step": 405
+    },
+    {
+      "epoch": 0.8584140277414289,
+      "grad_norm": 72.54119105678649,
+      "learning_rate": 2.9492720416985e-08,
+      "logits/chosen": 0.6000219583511353,
+      "logits/rejected": 0.9464153051376343,
+      "logps/chosen": -2.9545369148254395,
+      "logps/rejected": -3.8140132427215576,
+      "loss": 1.2519,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.9545369148254395,
+      "rewards/margins": 0.8594767451286316,
+      "rewards/rejected": -3.8140132427215576,
+      "step": 410
+    },
+    {
+      "epoch": 0.8688824914943732,
+      "grad_norm": 57.55112621238877,
+      "learning_rate": 2.5328246937043525e-08,
+      "logits/chosen": 0.657649576663971,
+      "logits/rejected": 1.3006607294082642,
+      "logps/chosen": -2.929166078567505,
+      "logps/rejected": -3.863920211791992,
+      "loss": 1.1329,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -2.929166078567505,
+      "rewards/margins": 0.9347543716430664,
+      "rewards/rejected": -3.863920211791992,
+      "step": 415
+    },
+    {
+      "epoch": 0.8793509552473174,
+      "grad_norm": 67.15006871791722,
+      "learning_rate": 2.1464952759020856e-08,
+      "logits/chosen": 0.557062029838562,
+      "logits/rejected": 0.9161618947982788,
+      "logps/chosen": -2.8673954010009766,
+      "logps/rejected": -3.752131938934326,
+      "loss": 1.2637,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.8673954010009766,
+      "rewards/margins": 0.8847367167472839,
+      "rewards/rejected": -3.752131938934326,
+      "step": 420
+    },
+    {
+      "epoch": 0.8898194190002617,
+      "grad_norm": 62.826606471582075,
+      "learning_rate": 1.7908016745981856e-08,
+      "logits/chosen": 0.6115803122520447,
+      "logits/rejected": 1.1207200288772583,
+      "logps/chosen": -2.983302354812622,
+      "logps/rejected": -3.9271881580352783,
+      "loss": 1.1823,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.983302354812622,
+      "rewards/margins": 0.9438859820365906,
+      "rewards/rejected": -3.9271881580352783,
+      "step": 425
+    },
+    {
+      "epoch": 0.9002878827532059,
+      "grad_norm": 59.68175243856826,
+      "learning_rate": 1.4662207078575684e-08,
+      "logits/chosen": 0.5250687599182129,
+      "logits/rejected": 1.0637626647949219,
+      "logps/chosen": -2.930884599685669,
+      "logps/rejected": -4.001204490661621,
+      "loss": 1.1121,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -2.930884599685669,
+      "rewards/margins": 1.0703200101852417,
+      "rewards/rejected": -4.001204490661621,
+      "step": 430
+    },
+    {
+      "epoch": 0.9107563465061502,
+      "grad_norm": 68.60209990488998,
+      "learning_rate": 1.1731874863145142e-08,
+      "logits/chosen": 0.5153384208679199,
+      "logits/rejected": 0.7662239670753479,
+      "logps/chosen": -2.8936798572540283,
+      "logps/rejected": -3.7913997173309326,
+      "loss": 1.1818,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.8936798572540283,
+      "rewards/margins": 0.8977203369140625,
+      "rewards/rejected": -3.7913997173309326,
+      "step": 435
+    },
+    {
+      "epoch": 0.9212248102590945,
+      "grad_norm": 61.87029001897422,
+      "learning_rate": 9.12094829893642e-09,
+      "logits/chosen": 0.7237823009490967,
+      "logits/rejected": 1.0966639518737793,
+      "logps/chosen": -2.946139097213745,
+      "logps/rejected": -4.009475231170654,
+      "loss": 1.1835,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.946139097213745,
+      "rewards/margins": 1.0633361339569092,
+      "rewards/rejected": -4.009475231170654,
+      "step": 440
+    },
+    {
+      "epoch": 0.9316932740120387,
+      "grad_norm": 64.1896852190105,
+      "learning_rate": 6.832927412229017e-09,
+      "logits/chosen": 0.6214802861213684,
+      "logits/rejected": 1.0805151462554932,
+      "logps/chosen": -3.0665371417999268,
+      "logps/rejected": -4.000259876251221,
+      "loss": 1.1936,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -3.0665371417999268,
+      "rewards/margins": 0.9337228536605835,
+      "rewards/rejected": -4.000259876251221,
+      "step": 445
+    },
+    {
+      "epoch": 0.942161737764983,
+      "grad_norm": 67.09074927088828,
+      "learning_rate": 4.8708793644441086e-09,
+      "logits/chosen": 0.6587177515029907,
+      "logits/rejected": 0.9696787595748901,
+      "logps/chosen": -2.8653368949890137,
+      "logps/rejected": -3.8535594940185547,
+      "loss": 1.1292,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -2.8653368949890137,
+      "rewards/margins": 0.9882224798202515,
+      "rewards/rejected": -3.8535594940185547,
+      "step": 450
+    },
+    {
+      "epoch": 0.9526302015179272,
+      "grad_norm": 74.52088387896268,
+      "learning_rate": 3.2374343405217884e-09,
+      "logits/chosen": 0.6812163591384888,
+      "logits/rejected": 1.1406841278076172,
+      "logps/chosen": -2.954162120819092,
+      "logps/rejected": -3.8345863819122314,
+      "loss": 1.2136,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -2.954162120819092,
+      "rewards/margins": 0.880423903465271,
+      "rewards/rejected": -3.8345863819122314,
+      "step": 455
+    },
+    {
+      "epoch": 0.9630986652708715,
+      "grad_norm": 57.70785985857215,
+      "learning_rate": 1.9347820230782295e-09,
+      "logits/chosen": 0.8318542242050171,
+      "logits/rejected": 1.231487512588501,
+      "logps/chosen": -2.9270339012145996,
+      "logps/rejected": -3.9501945972442627,
+      "loss": 1.1763,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.9270339012145996,
+      "rewards/margins": 1.0231605768203735,
+      "rewards/rejected": -3.9501945972442627,
+      "step": 460
+    },
+    {
+      "epoch": 0.9735671290238157,
+      "grad_norm": 66.60199874410606,
+      "learning_rate": 9.64668657069706e-10,
+      "logits/chosen": 0.6092048287391663,
+      "logits/rejected": 1.2739137411117554,
+      "logps/chosen": -2.9308724403381348,
+      "logps/rejected": -4.049551963806152,
+      "loss": 1.12,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -2.9308724403381348,
+      "rewards/margins": 1.1186802387237549,
+      "rewards/rejected": -4.049551963806152,
+      "step": 465
+    },
+    {
+      "epoch": 0.98403559277676,
+      "grad_norm": 72.73204833476578,
+      "learning_rate": 3.2839470889836627e-10,
+      "logits/chosen": 0.5203122496604919,
+      "logits/rejected": 0.9083112478256226,
+      "logps/chosen": -3.0056755542755127,
+      "logps/rejected": -3.833085536956787,
+      "loss": 1.1925,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -3.0056755542755127,
+      "rewards/margins": 0.8274103999137878,
+      "rewards/rejected": -3.833085536956787,
+      "step": 470
+    },
+    {
+      "epoch": 0.9945040565297043,
+      "grad_norm": 81.94973190630537,
+      "learning_rate": 2.6813123097352287e-11,
+      "logits/chosen": 0.4619743824005127,
+      "logits/rejected": 1.1849265098571777,
+      "logps/chosen": -2.943962574005127,
+      "logps/rejected": -3.8440794944763184,
+      "loss": 1.2154,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.943962574005127,
+      "rewards/margins": 0.9001175761222839,
+      "rewards/rejected": -3.8440794944763184,
+      "step": 475
+    },
+    {
+      "epoch": 0.998691442030882,
+      "step": 477,
+      "total_flos": 0.0,
+      "train_loss": 1.3288766333142166,
+      "train_runtime": 7370.4172,
+      "train_samples_per_second": 8.295,
+      "train_steps_per_second": 0.065
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 477,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000000,
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee74e17ad8006d7fd4154422768595646f13807cbfc14b1e6486015a025fa0a2
+size 7160