Model save

README.md

@@ -0,0 +1,60 @@
+---
+license: apache-2.0
+base_model: alignment-handbook/zephyr-7b-sft-full
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: zephyr-7b-SimPO-SELM3
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# zephyr-7b-SimPO-SELM3
+
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 2
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 16
+- total_train_batch_size: 128
+- total_eval_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.40.2
+- Pytorch 2.1.2+cu121
+- Datasets 2.14.6
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.998691442030882,
+    "total_flos": 0.0,
+    "train_loss": 1.3340201452843048,
+    "train_runtime": 7646.4301,
+    "train_samples": 61135,
+    "train_samples_per_second": 7.995,
+    "train_steps_per_second": 0.062
+}

config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "alignment-handbook/zephyr-7b-sft-full",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.40.2",
+  "use_cache": false,
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.40.2"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a58c56dca1080fd776bc799d45eadc7a36409e9a2fc96526c679356fdaf2ff48
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ad5d87d7bbca8d2fbd306fc0d173c5b22576cb363b927e9f22470d2d57e9a35
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47a8889d1cd92a410965adeb0a44633f89c90e9afd109ca2ec293952a0fe183a
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 2048,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.998691442030882,
+    "total_flos": 0.0,
+    "train_loss": 1.3340201452843048,
+    "train_runtime": 7646.4301,
+    "train_samples": 61135,
+    "train_samples_per_second": 7.995,
+    "train_steps_per_second": 0.062
+}

trainer_state.json

@@ -0,0 +1,1455 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.998691442030882,
+  "eval_steps": 500,
+  "global_step": 477,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.010468463752944255,
+      "grad_norm": 22.33847861457317,
+      "learning_rate": 5.208333333333333e-08,
+      "logits/chosen": -2.7707886695861816,
+      "logits/rejected": -2.7283411026000977,
+      "logps/chosen": -1.0281651020050049,
+      "logps/rejected": -1.1735057830810547,
+      "loss": 1.7068,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.0281651020050049,
+      "rewards/margins": 0.14534088969230652,
+      "rewards/rejected": -1.1735057830810547,
+      "step": 5
+    },
+    {
+      "epoch": 0.02093692750588851,
+      "grad_norm": 19.58286489214486,
+      "learning_rate": 1.0416666666666667e-07,
+      "logits/chosen": -2.7509849071502686,
+      "logits/rejected": -2.725268840789795,
+      "logps/chosen": -0.9945869445800781,
+      "logps/rejected": -1.070472002029419,
+      "loss": 1.7055,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.9945869445800781,
+      "rewards/margins": 0.07588515430688858,
+      "rewards/rejected": -1.070472002029419,
+      "step": 10
+    },
+    {
+      "epoch": 0.031405391258832765,
+      "grad_norm": 20.875691978403097,
+      "learning_rate": 1.5624999999999999e-07,
+      "logits/chosen": -2.731562852859497,
+      "logits/rejected": -2.698035478591919,
+      "logps/chosen": -0.9806415438652039,
+      "logps/rejected": -1.1409623622894287,
+      "loss": 1.6533,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.9806415438652039,
+      "rewards/margins": 0.1603206843137741,
+      "rewards/rejected": -1.1409623622894287,
+      "step": 15
+    },
+    {
+      "epoch": 0.04187385501177702,
+      "grad_norm": 18.447110198802093,
+      "learning_rate": 2.0833333333333333e-07,
+      "logits/chosen": -2.804091453552246,
+      "logits/rejected": -2.711153268814087,
+      "logps/chosen": -0.9976784586906433,
+      "logps/rejected": -1.0985018014907837,
+      "loss": 1.6778,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.9976784586906433,
+      "rewards/margins": 0.10082335770130157,
+      "rewards/rejected": -1.0985018014907837,
+      "step": 20
+    },
+    {
+      "epoch": 0.05234231876472128,
+      "grad_norm": 17.24388250585097,
+      "learning_rate": 2.604166666666667e-07,
+      "logits/chosen": -2.800624132156372,
+      "logits/rejected": -2.777677536010742,
+      "logps/chosen": -0.977279007434845,
+      "logps/rejected": -1.0392786264419556,
+      "loss": 1.6818,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.977279007434845,
+      "rewards/margins": 0.061999619007110596,
+      "rewards/rejected": -1.0392786264419556,
+      "step": 25
+    },
+    {
+      "epoch": 0.06281078251766553,
+      "grad_norm": 17.54927195272229,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": -2.774305820465088,
+      "logits/rejected": -2.734163284301758,
+      "logps/chosen": -0.8836237192153931,
+      "logps/rejected": -0.9883272051811218,
+      "loss": 1.7148,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.8836237192153931,
+      "rewards/margins": 0.10470354557037354,
+      "rewards/rejected": -0.9883272051811218,
+      "step": 30
+    },
+    {
+      "epoch": 0.07327924627060979,
+      "grad_norm": 19.597204446648743,
+      "learning_rate": 3.645833333333333e-07,
+      "logits/chosen": -2.7799019813537598,
+      "logits/rejected": -2.6970131397247314,
+      "logps/chosen": -0.9105981588363647,
+      "logps/rejected": -1.032915711402893,
+      "loss": 1.6819,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.9105981588363647,
+      "rewards/margins": 0.12231750786304474,
+      "rewards/rejected": -1.032915711402893,
+      "step": 35
+    },
+    {
+      "epoch": 0.08374771002355404,
+      "grad_norm": 17.382375782510174,
+      "learning_rate": 4.1666666666666667e-07,
+      "logits/chosen": -2.7961745262145996,
+      "logits/rejected": -2.7064435482025146,
+      "logps/chosen": -0.8972692489624023,
+      "logps/rejected": -1.1690887212753296,
+      "loss": 1.5799,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.8972692489624023,
+      "rewards/margins": 0.27181947231292725,
+      "rewards/rejected": -1.1690887212753296,
+      "step": 40
+    },
+    {
+      "epoch": 0.0942161737764983,
+      "grad_norm": 20.280760501242256,
+      "learning_rate": 4.6874999999999996e-07,
+      "logits/chosen": -2.792088031768799,
+      "logits/rejected": -2.748499870300293,
+      "logps/chosen": -0.9211010932922363,
+      "logps/rejected": -1.065104365348816,
+      "loss": 1.5989,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.9211010932922363,
+      "rewards/margins": 0.14400319755077362,
+      "rewards/rejected": -1.065104365348816,
+      "step": 45
+    },
+    {
+      "epoch": 0.10468463752944256,
+      "grad_norm": 24.965329955439582,
+      "learning_rate": 4.999731868769026e-07,
+      "logits/chosen": -2.7186241149902344,
+      "logits/rejected": -2.6570615768432617,
+      "logps/chosen": -0.9831829071044922,
+      "logps/rejected": -1.2431915998458862,
+      "loss": 1.5992,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.9831829071044922,
+      "rewards/margins": 0.2600088119506836,
+      "rewards/rejected": -1.2431915998458862,
+      "step": 50
+    },
+    {
+      "epoch": 0.11515310128238682,
+      "grad_norm": 20.11572375657384,
+      "learning_rate": 4.996716052911017e-07,
+      "logits/chosen": -2.6839053630828857,
+      "logits/rejected": -2.6501219272613525,
+      "logps/chosen": -1.0190128087997437,
+      "logps/rejected": -1.2308984994888306,
+      "loss": 1.6076,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.0190128087997437,
+      "rewards/margins": 0.21188561618328094,
+      "rewards/rejected": -1.2308984994888306,
+      "step": 55
+    },
+    {
+      "epoch": 0.12562156503533106,
+      "grad_norm": 23.21505244201069,
+      "learning_rate": 4.990353313429303e-07,
+      "logits/chosen": -2.7174434661865234,
+      "logits/rejected": -2.6702983379364014,
+      "logps/chosen": -1.0736011266708374,
+      "logps/rejected": -1.2630236148834229,
+      "loss": 1.5192,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.0736011266708374,
+      "rewards/margins": 0.18942244350910187,
+      "rewards/rejected": -1.2630236148834229,
+      "step": 60
+    },
+    {
+      "epoch": 0.1360900287882753,
+      "grad_norm": 23.25106134127669,
+      "learning_rate": 4.980652179769217e-07,
+      "logits/chosen": -2.6810200214385986,
+      "logits/rejected": -2.5684618949890137,
+      "logps/chosen": -1.0137274265289307,
+      "logps/rejected": -1.498494267463684,
+      "loss": 1.5546,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.0137274265289307,
+      "rewards/margins": 0.4847669005393982,
+      "rewards/rejected": -1.498494267463684,
+      "step": 65
+    },
+    {
+      "epoch": 0.14655849254121958,
+      "grad_norm": 27.792367557668214,
+      "learning_rate": 4.967625656594781e-07,
+      "logits/chosen": -2.530665636062622,
+      "logits/rejected": -2.4802050590515137,
+      "logps/chosen": -1.1090538501739502,
+      "logps/rejected": -1.3939735889434814,
+      "loss": 1.5395,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.1090538501739502,
+      "rewards/margins": 0.28491973876953125,
+      "rewards/rejected": -1.3939735889434814,
+      "step": 70
+    },
+    {
+      "epoch": 0.15702695629416383,
+      "grad_norm": 31.80202351677776,
+      "learning_rate": 4.951291206355559e-07,
+      "logits/chosen": -2.56596302986145,
+      "logits/rejected": -2.4995336532592773,
+      "logps/chosen": -1.0895880460739136,
+      "logps/rejected": -1.4386751651763916,
+      "loss": 1.5145,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.0895880460739136,
+      "rewards/margins": 0.3490869700908661,
+      "rewards/rejected": -1.4386751651763916,
+      "step": 75
+    },
+    {
+      "epoch": 0.16749542004710807,
+      "grad_norm": 24.649835098743253,
+      "learning_rate": 4.93167072587771e-07,
+      "logits/chosen": -2.5505566596984863,
+      "logits/rejected": -2.499436140060425,
+      "logps/chosen": -1.066165804862976,
+      "logps/rejected": -1.3876395225524902,
+      "loss": 1.5557,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.066165804862976,
+      "rewards/margins": 0.3214736878871918,
+      "rewards/rejected": -1.3876395225524902,
+      "step": 80
+    },
+    {
+      "epoch": 0.17796388380005235,
+      "grad_norm": 36.00370335117123,
+      "learning_rate": 4.908790517010636e-07,
+      "logits/chosen": -2.5619356632232666,
+      "logits/rejected": -2.5041964054107666,
+      "logps/chosen": -1.0125417709350586,
+      "logps/rejected": -1.3010904788970947,
+      "loss": 1.5994,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.0125417709350586,
+      "rewards/margins": 0.2885487377643585,
+      "rewards/rejected": -1.3010904788970947,
+      "step": 85
+    },
+    {
+      "epoch": 0.1884323475529966,
+      "grad_norm": 24.146045834642553,
+      "learning_rate": 4.882681251368548e-07,
+      "logits/chosen": -2.4920132160186768,
+      "logits/rejected": -2.465853691101074,
+      "logps/chosen": -1.0736846923828125,
+      "logps/rejected": -1.3465595245361328,
+      "loss": 1.5492,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.0736846923828125,
+      "rewards/margins": 0.27287474274635315,
+      "rewards/rejected": -1.3465595245361328,
+      "step": 90
+    },
+    {
+      "epoch": 0.19890081130594087,
+      "grad_norm": 28.073142968648483,
+      "learning_rate": 4.853377929214243e-07,
+      "logits/chosen": -2.5122618675231934,
+      "logits/rejected": -2.4115467071533203,
+      "logps/chosen": -1.0956408977508545,
+      "logps/rejected": -1.3623732328414917,
+      "loss": 1.5645,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.0956408977508545,
+      "rewards/margins": 0.26673227548599243,
+      "rewards/rejected": -1.3623732328414917,
+      "step": 95
+    },
+    {
+      "epoch": 0.2093692750588851,
+      "grad_norm": 31.739558342111017,
+      "learning_rate": 4.820919832540181e-07,
+      "logits/chosen": -2.4314770698547363,
+      "logits/rejected": -2.3331286907196045,
+      "logps/chosen": -1.1069271564483643,
+      "logps/rejected": -1.4718701839447021,
+      "loss": 1.5661,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.1069271564483643,
+      "rewards/margins": 0.3649430274963379,
+      "rewards/rejected": -1.4718701839447021,
+      "step": 100
+    },
+    {
+      "epoch": 0.21983773881182936,
+      "grad_norm": 30.25619066044505,
+      "learning_rate": 4.785350472409791e-07,
+      "logits/chosen": -2.374175548553467,
+      "logits/rejected": -2.345848321914673,
+      "logps/chosen": -1.1067227125167847,
+      "logps/rejected": -1.5379221439361572,
+      "loss": 1.4913,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.1067227125167847,
+      "rewards/margins": 0.43119925260543823,
+      "rewards/rejected": -1.5379221439361572,
+      "step": 105
+    },
+    {
+      "epoch": 0.23030620256477363,
+      "grad_norm": 37.413658539880686,
+      "learning_rate": 4.7467175306295647e-07,
+      "logits/chosen": -2.3754255771636963,
+      "logits/rejected": -2.298557996749878,
+      "logps/chosen": -1.2783015966415405,
+      "logps/rejected": -1.6868267059326172,
+      "loss": 1.4914,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.2783015966415405,
+      "rewards/margins": 0.4085250496864319,
+      "rewards/rejected": -1.6868267059326172,
+      "step": 110
+    },
+    {
+      "epoch": 0.24077466631771788,
+      "grad_norm": 30.692988045641766,
+      "learning_rate": 4.70507279583015e-07,
+      "logits/chosen": -2.295192241668701,
+      "logits/rejected": -2.2188549041748047,
+      "logps/chosen": -1.2483012676239014,
+      "logps/rejected": -1.789584755897522,
+      "loss": 1.422,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.2483012676239014,
+      "rewards/margins": 0.541283369064331,
+      "rewards/rejected": -1.789584755897522,
+      "step": 115
+    },
+    {
+      "epoch": 0.2512431300706621,
+      "grad_norm": 30.61675572148456,
+      "learning_rate": 4.6604720940421207e-07,
+      "logits/chosen": -2.2375972270965576,
+      "logits/rejected": -2.188506603240967,
+      "logps/chosen": -1.2778130769729614,
+      "logps/rejected": -1.6632368564605713,
+      "loss": 1.5023,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.2778130769729614,
+      "rewards/margins": 0.3854238986968994,
+      "rewards/rejected": -1.6632368564605713,
+      "step": 120
+    },
+    {
+      "epoch": 0.26171159382360637,
+      "grad_norm": 27.143222504991545,
+      "learning_rate": 4.612975213859487e-07,
+      "logits/chosen": -2.2686073780059814,
+      "logits/rejected": -2.240734815597534,
+      "logps/chosen": -1.2156823873519897,
+      "logps/rejected": -1.5097521543502808,
+      "loss": 1.5016,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.2156823873519897,
+      "rewards/margins": 0.2940698564052582,
+      "rewards/rejected": -1.5097521543502808,
+      "step": 125
+    },
+    {
+      "epoch": 0.2721800575765506,
+      "grad_norm": 28.662390853422256,
+      "learning_rate": 4.5626458262912735e-07,
+      "logits/chosen": -2.1936447620391846,
+      "logits/rejected": -2.1442556381225586,
+      "logps/chosen": -1.2718418836593628,
+      "logps/rejected": -1.5303620100021362,
+      "loss": 1.4909,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.2718418836593628,
+      "rewards/margins": 0.25852006673812866,
+      "rewards/rejected": -1.5303620100021362,
+      "step": 130
+    },
+    {
+      "epoch": 0.2826485213294949,
+      "grad_norm": 44.28036575523435,
+      "learning_rate": 4.5095513994085974e-07,
+      "logits/chosen": -2.1270031929016113,
+      "logits/rejected": -2.0248324871063232,
+      "logps/chosen": -1.3885653018951416,
+      "logps/rejected": -2.0145115852355957,
+      "loss": 1.4337,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.3885653018951416,
+      "rewards/margins": 0.6259465217590332,
+      "rewards/rejected": -2.0145115852355957,
+      "step": 135
+    },
+    {
+      "epoch": 0.29311698508243916,
+      "grad_norm": 36.60577846523631,
+      "learning_rate": 4.453763107901675e-07,
+      "logits/chosen": -2.1079370975494385,
+      "logits/rejected": -2.0384957790374756,
+      "logps/chosen": -1.5038011074066162,
+      "logps/rejected": -1.9894378185272217,
+      "loss": 1.4093,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.5038011074066162,
+      "rewards/margins": 0.48563677072525024,
+      "rewards/rejected": -1.9894378185272217,
+      "step": 140
+    },
+    {
+      "epoch": 0.3035854488353834,
+      "grad_norm": 28.487957762616777,
+      "learning_rate": 4.395355737667985e-07,
+      "logits/chosen": -2.0164198875427246,
+      "logits/rejected": -1.9161510467529297,
+      "logps/chosen": -1.5493210554122925,
+      "logps/rejected": -2.018698215484619,
+      "loss": 1.451,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.5493210554122925,
+      "rewards/margins": 0.4693775177001953,
+      "rewards/rejected": -2.018698215484619,
+      "step": 145
+    },
+    {
+      "epoch": 0.31405391258832765,
+      "grad_norm": 41.06239885865088,
+      "learning_rate": 4.3344075855595097e-07,
+      "logits/chosen": -1.9734079837799072,
+      "logits/rejected": -1.9259965419769287,
+      "logps/chosen": -1.4789108037948608,
+      "logps/rejected": -1.9842697381973267,
+      "loss": 1.3495,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.4789108037948608,
+      "rewards/margins": 0.5053588151931763,
+      "rewards/rejected": -1.9842697381973267,
+      "step": 150
+    },
+    {
+      "epoch": 0.3245223763412719,
+      "grad_norm": 48.830011945922514,
+      "learning_rate": 4.271000354423425e-07,
+      "logits/chosen": -1.8841253519058228,
+      "logits/rejected": -1.8244024515151978,
+      "logps/chosen": -1.7202045917510986,
+      "logps/rejected": -2.2293524742126465,
+      "loss": 1.3858,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.7202045917510986,
+      "rewards/margins": 0.5091480016708374,
+      "rewards/rejected": -2.2293524742126465,
+      "step": 155
+    },
+    {
+      "epoch": 0.33499084009421615,
+      "grad_norm": 58.39848893185564,
+      "learning_rate": 4.2052190435769554e-07,
+      "logits/chosen": -1.7686011791229248,
+      "logits/rejected": -1.7215473651885986,
+      "logps/chosen": -1.7806098461151123,
+      "logps/rejected": -2.337874174118042,
+      "loss": 1.3384,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.7806098461151123,
+      "rewards/margins": 0.5572644472122192,
+      "rewards/rejected": -2.337874174118042,
+      "step": 160
+    },
+    {
+      "epoch": 0.34545930384716045,
+      "grad_norm": 39.19480392174481,
+      "learning_rate": 4.137151834863213e-07,
+      "logits/chosen": -1.7493757009506226,
+      "logits/rejected": -1.6483466625213623,
+      "logps/chosen": -1.6905717849731445,
+      "logps/rejected": -2.2200164794921875,
+      "loss": 1.3194,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.6905717849731445,
+      "rewards/margins": 0.5294445753097534,
+      "rewards/rejected": -2.2200164794921875,
+      "step": 165
+    },
+    {
+      "epoch": 0.3559277676001047,
+      "grad_norm": 43.29327849827671,
+      "learning_rate": 4.0668899744407567e-07,
+      "logits/chosen": -1.7271289825439453,
+      "logits/rejected": -1.6169135570526123,
+      "logps/chosen": -1.7594356536865234,
+      "logps/rejected": -2.3671681880950928,
+      "loss": 1.4272,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.7594356536865234,
+      "rewards/margins": 0.6077327132225037,
+      "rewards/rejected": -2.3671681880950928,
+      "step": 170
+    },
+    {
+      "epoch": 0.36639623135304894,
+      "grad_norm": 51.0545099532057,
+      "learning_rate": 3.994527650465352e-07,
+      "logits/chosen": -1.6730191707611084,
+      "logits/rejected": -1.5045350790023804,
+      "logps/chosen": -1.7539478540420532,
+      "logps/rejected": -2.3421220779418945,
+      "loss": 1.4166,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.7539478540420532,
+      "rewards/margins": 0.5881742835044861,
+      "rewards/rejected": -2.3421220779418945,
+      "step": 175
+    },
+    {
+      "epoch": 0.3768646951059932,
+      "grad_norm": 39.83826698098843,
+      "learning_rate": 3.920161866827889e-07,
+      "logits/chosen": -1.5326063632965088,
+      "logits/rejected": -1.4709655046463013,
+      "logps/chosen": -1.8253322839736938,
+      "logps/rejected": -2.5203299522399902,
+      "loss": 1.327,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.8253322839736938,
+      "rewards/margins": 0.6949977278709412,
+      "rewards/rejected": -2.5203299522399902,
+      "step": 180
+    },
+    {
+      "epoch": 0.38733315885893743,
+      "grad_norm": 54.751515663792574,
+      "learning_rate": 3.8438923131177237e-07,
+      "logits/chosen": -1.6529709100723267,
+      "logits/rejected": -1.547048807144165,
+      "logps/chosen": -2.0327372550964355,
+      "logps/rejected": -2.799630880355835,
+      "loss": 1.3481,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -2.0327372550964355,
+      "rewards/margins": 0.7668935656547546,
+      "rewards/rejected": -2.799630880355835,
+      "step": 185
+    },
+    {
+      "epoch": 0.39780162261188173,
+      "grad_norm": 55.01879734493922,
+      "learning_rate": 3.765821230985757e-07,
+      "logits/chosen": -1.7503044605255127,
+      "logits/rejected": -1.7193371057510376,
+      "logps/chosen": -2.0020930767059326,
+      "logps/rejected": -2.541438341140747,
+      "loss": 1.4457,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.0020930767059326,
+      "rewards/margins": 0.5393451452255249,
+      "rewards/rejected": -2.541438341140747,
+      "step": 190
+    },
+    {
+      "epoch": 0.408270086364826,
+      "grad_norm": 56.01695405727775,
+      "learning_rate": 3.6860532770864005e-07,
+      "logits/chosen": -1.6532561779022217,
+      "logits/rejected": -1.5889365673065186,
+      "logps/chosen": -2.2126450538635254,
+      "logps/rejected": -2.833956241607666,
+      "loss": 1.3837,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -2.2126450538635254,
+      "rewards/margins": 0.6213110685348511,
+      "rewards/rejected": -2.833956241607666,
+      "step": 195
+    },
+    {
+      "epoch": 0.4187385501177702,
+      "grad_norm": 53.693962657799496,
+      "learning_rate": 3.604695382782159e-07,
+      "logits/chosen": -1.5952329635620117,
+      "logits/rejected": -1.5742504596710205,
+      "logps/chosen": -2.298196315765381,
+      "logps/rejected": -2.9012575149536133,
+      "loss": 1.3376,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.298196315765381,
+      "rewards/margins": 0.6030609607696533,
+      "rewards/rejected": -2.9012575149536133,
+      "step": 200
+    },
+    {
+      "epoch": 0.42920701387071447,
+      "grad_norm": 64.50276884709064,
+      "learning_rate": 3.5218566107988867e-07,
+      "logits/chosen": -1.3948825597763062,
+      "logits/rejected": -1.3095804452896118,
+      "logps/chosen": -2.4898180961608887,
+      "logps/rejected": -3.191925525665283,
+      "loss": 1.2815,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -2.4898180961608887,
+      "rewards/margins": 0.7021073698997498,
+      "rewards/rejected": -3.191925525665283,
+      "step": 205
+    },
+    {
+      "epoch": 0.4396754776236587,
+      "grad_norm": 84.53595036055246,
+      "learning_rate": 3.4376480090239047e-07,
+      "logits/chosen": -1.3462097644805908,
+      "logits/rejected": -1.2632777690887451,
+      "logps/chosen": -2.999131679534912,
+      "logps/rejected": -3.611553192138672,
+      "loss": 1.3852,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -2.999131679534912,
+      "rewards/margins": 0.6124216318130493,
+      "rewards/rejected": -3.611553192138672,
+      "step": 210
+    },
+    {
+      "epoch": 0.45014394137660296,
+      "grad_norm": 46.66563883333029,
+      "learning_rate": 3.3521824616429284e-07,
+      "logits/chosen": -1.3600023984909058,
+      "logits/rejected": -1.2664874792099,
+      "logps/chosen": -2.6604790687561035,
+      "logps/rejected": -3.3327317237854004,
+      "loss": 1.286,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -2.6604790687561035,
+      "rewards/margins": 0.6722527742385864,
+      "rewards/rejected": -3.3327317237854004,
+      "step": 215
+    },
+    {
+      "epoch": 0.46061240512954726,
+      "grad_norm": 52.30084280147944,
+      "learning_rate": 3.265574537815398e-07,
+      "logits/chosen": -1.328802466392517,
+      "logits/rejected": -1.1486713886260986,
+      "logps/chosen": -2.2492258548736572,
+      "logps/rejected": -3.048722743988037,
+      "loss": 1.2381,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -2.2492258548736572,
+      "rewards/margins": 0.7994968295097351,
+      "rewards/rejected": -3.048722743988037,
+      "step": 220
+    },
+    {
+      "epoch": 0.4710808688824915,
+      "grad_norm": 59.5983284817177,
+      "learning_rate": 3.1779403380910425e-07,
+      "logits/chosen": -1.2022250890731812,
+      "logits/rejected": -0.9333807229995728,
+      "logps/chosen": -2.3682289123535156,
+      "logps/rejected": -3.2058632373809814,
+      "loss": 1.2818,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -2.3682289123535156,
+      "rewards/margins": 0.8376340866088867,
+      "rewards/rejected": -3.2058632373809814,
+      "step": 225
+    },
+    {
+      "epoch": 0.48154933263543576,
+      "grad_norm": 57.79020776140021,
+      "learning_rate": 3.0893973387735683e-07,
+      "logits/chosen": -0.9690738916397095,
+      "logits/rejected": -0.6763086915016174,
+      "logps/chosen": -2.678581714630127,
+      "logps/rejected": -3.652540922164917,
+      "loss": 1.1691,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -2.678581714630127,
+      "rewards/margins": 0.9739594459533691,
+      "rewards/rejected": -3.652540922164917,
+      "step": 230
+    },
+    {
+      "epoch": 0.49201779638838,
+      "grad_norm": 68.31261443984954,
+      "learning_rate": 3.000064234440111e-07,
+      "logits/chosen": -0.7249783277511597,
+      "logits/rejected": -0.4497829079627991,
+      "logps/chosen": -2.929415225982666,
+      "logps/rejected": -3.6940486431121826,
+      "loss": 1.219,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -2.929415225982666,
+      "rewards/margins": 0.7646334171295166,
+      "rewards/rejected": -3.6940486431121826,
+      "step": 235
+    },
+    {
+      "epoch": 0.5024862601413242,
+      "grad_norm": 58.51770384505626,
+      "learning_rate": 2.910060778827554e-07,
+      "logits/chosen": -0.6646004915237427,
+      "logits/rejected": -0.4312285780906677,
+      "logps/chosen": -2.8067305088043213,
+      "logps/rejected": -3.5952727794647217,
+      "loss": 1.2149,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.8067305088043213,
+      "rewards/margins": 0.7885428071022034,
+      "rewards/rejected": -3.5952727794647217,
+      "step": 240
+    },
+    {
+      "epoch": 0.5129547238942685,
+      "grad_norm": 77.9188973058883,
+      "learning_rate": 2.8195076242990116e-07,
+      "logits/chosen": -0.8215748071670532,
+      "logits/rejected": -0.6058939695358276,
+      "logps/chosen": -3.0396523475646973,
+      "logps/rejected": -3.7490572929382324,
+      "loss": 1.3009,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -3.0396523475646973,
+      "rewards/margins": 0.7094049453735352,
+      "rewards/rejected": -3.7490572929382324,
+      "step": 245
+    },
+    {
+      "epoch": 0.5234231876472127,
+      "grad_norm": 63.40691188463422,
+      "learning_rate": 2.7285261601056697e-07,
+      "logits/chosen": -0.9382761716842651,
+      "logits/rejected": -0.729052722454071,
+      "logps/chosen": -2.595768451690674,
+      "logps/rejected": -3.3881466388702393,
+      "loss": 1.3493,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -2.595768451690674,
+      "rewards/margins": 0.792378306388855,
+      "rewards/rejected": -3.3881466388702393,
+      "step": 250
+    },
+    {
+      "epoch": 0.533891651400157,
+      "grad_norm": 48.6873954516319,
+      "learning_rate": 2.6372383496608186e-07,
+      "logits/chosen": -0.9191045761108398,
+      "logits/rejected": -0.6374125480651855,
+      "logps/chosen": -2.6228957176208496,
+      "logps/rejected": -3.4444642066955566,
+      "loss": 1.1933,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.6228957176208496,
+      "rewards/margins": 0.8215683698654175,
+      "rewards/rejected": -3.4444642066955566,
+      "step": 255
+    },
+    {
+      "epoch": 0.5443601151531012,
+      "grad_norm": 58.54153465550475,
+      "learning_rate": 2.5457665670441937e-07,
+      "logits/chosen": -1.0062066316604614,
+      "logits/rejected": -0.7524072527885437,
+      "logps/chosen": -2.226409435272217,
+      "logps/rejected": -3.2910056114196777,
+      "loss": 1.2637,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.226409435272217,
+      "rewards/margins": 1.0645958185195923,
+      "rewards/rejected": -3.2910056114196777,
+      "step": 260
+    },
+    {
+      "epoch": 0.5548285789060455,
+      "grad_norm": 56.27740231168225,
+      "learning_rate": 2.454233432955807e-07,
+      "logits/chosen": -0.9759531021118164,
+      "logits/rejected": -0.7592412233352661,
+      "logps/chosen": -2.4869279861450195,
+      "logps/rejected": -3.270657777786255,
+      "loss": 1.1763,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.4869279861450195,
+      "rewards/margins": 0.7837298512458801,
+      "rewards/rejected": -3.270657777786255,
+      "step": 265
+    },
+    {
+      "epoch": 0.5652970426589898,
+      "grad_norm": 79.16628471833214,
+      "learning_rate": 2.3627616503391812e-07,
+      "logits/chosen": -0.7810020446777344,
+      "logits/rejected": -0.563759446144104,
+      "logps/chosen": -2.9826416969299316,
+      "logps/rejected": -3.696744918823242,
+      "loss": 1.2307,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.9826416969299316,
+      "rewards/margins": 0.7141033411026001,
+      "rewards/rejected": -3.696744918823242,
+      "step": 270
+    },
+    {
+      "epoch": 0.575765506411934,
+      "grad_norm": 67.9940381335306,
+      "learning_rate": 2.2714738398943308e-07,
+      "logits/chosen": -0.6166077256202698,
+      "logits/rejected": -0.255386084318161,
+      "logps/chosen": -3.5992627143859863,
+      "logps/rejected": -4.62840461730957,
+      "loss": 1.2213,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -3.5992627143859863,
+      "rewards/margins": 1.029141902923584,
+      "rewards/rejected": -4.62840461730957,
+      "step": 275
+    },
+    {
+      "epoch": 0.5862339701648783,
+      "grad_norm": 77.77171483631982,
+      "learning_rate": 2.1804923757009882e-07,
+      "logits/chosen": -0.4341735243797302,
+      "logits/rejected": -0.08086974173784256,
+      "logps/chosen": -3.569580554962158,
+      "logps/rejected": -4.482806205749512,
+      "loss": 1.2253,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -3.569580554962158,
+      "rewards/margins": 0.9132259488105774,
+      "rewards/rejected": -4.482806205749512,
+      "step": 280
+    },
+    {
+      "epoch": 0.5967024339178225,
+      "grad_norm": 78.23503462159874,
+      "learning_rate": 2.089939221172446e-07,
+      "logits/chosen": -0.4769046902656555,
+      "logits/rejected": -0.28695839643478394,
+      "logps/chosen": -3.2708067893981934,
+      "logps/rejected": -4.225908279418945,
+      "loss": 1.2069,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -3.2708067893981934,
+      "rewards/margins": 0.955101490020752,
+      "rewards/rejected": -4.225908279418945,
+      "step": 285
+    },
+    {
+      "epoch": 0.6071708976707668,
+      "grad_norm": 60.888870909608286,
+      "learning_rate": 1.9999357655598891e-07,
+      "logits/chosen": -0.629644513130188,
+      "logits/rejected": -0.3126750886440277,
+      "logps/chosen": -3.1302852630615234,
+      "logps/rejected": -4.096064567565918,
+      "loss": 1.1961,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -3.1302852630615234,
+      "rewards/margins": 0.9657794833183289,
+      "rewards/rejected": -4.096064567565918,
+      "step": 290
+    },
+    {
+      "epoch": 0.6176393614237111,
+      "grad_norm": 69.95748727333331,
+      "learning_rate": 1.9106026612264315e-07,
+      "logits/chosen": -0.6755684614181519,
+      "logits/rejected": -0.30471256375312805,
+      "logps/chosen": -2.978564739227295,
+      "logps/rejected": -3.7914657592773438,
+      "loss": 1.2554,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.978564739227295,
+      "rewards/margins": 0.8129006624221802,
+      "rewards/rejected": -3.7914657592773438,
+      "step": 295
+    },
+    {
+      "epoch": 0.6281078251766553,
+      "grad_norm": 70.38289921062774,
+      "learning_rate": 1.8220596619089573e-07,
+      "logits/chosen": -0.5232299566268921,
+      "logits/rejected": -0.24573859572410583,
+      "logps/chosen": -2.9739651679992676,
+      "logps/rejected": -3.8225860595703125,
+      "loss": 1.2417,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.9739651679992676,
+      "rewards/margins": 0.8486205339431763,
+      "rewards/rejected": -3.8225860595703125,
+      "step": 300
+    },
+    {
+      "epoch": 0.6385762889295996,
+      "grad_norm": 72.4502519540176,
+      "learning_rate": 1.7344254621846017e-07,
+      "logits/chosen": -0.25070467591285706,
+      "logits/rejected": 0.04965158551931381,
+      "logps/chosen": -3.1987884044647217,
+      "logps/rejected": -4.245509147644043,
+      "loss": 1.1282,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -3.1987884044647217,
+      "rewards/margins": 1.046720266342163,
+      "rewards/rejected": -4.245509147644043,
+      "step": 305
+    },
+    {
+      "epoch": 0.6490447526825438,
+      "grad_norm": 74.82154328000124,
+      "learning_rate": 1.647817538357072e-07,
+      "logits/chosen": -0.21263869106769562,
+      "logits/rejected": -0.07299887388944626,
+      "logps/chosen": -3.0970096588134766,
+      "logps/rejected": -4.266674995422363,
+      "loss": 1.1228,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -3.0970096588134766,
+      "rewards/margins": 1.1696654558181763,
+      "rewards/rejected": -4.266674995422363,
+      "step": 310
+    },
+    {
+      "epoch": 0.6595132164354881,
+      "grad_norm": 104.80383724137711,
+      "learning_rate": 1.562351990976095e-07,
+      "logits/chosen": -0.2823619246482849,
+      "logits/rejected": 0.06330037117004395,
+      "logps/chosen": -3.170875072479248,
+      "logps/rejected": -4.283278465270996,
+      "loss": 1.1544,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -3.170875072479248,
+      "rewards/margins": 1.1124036312103271,
+      "rewards/rejected": -4.283278465270996,
+      "step": 315
+    },
+    {
+      "epoch": 0.6699816801884323,
+      "grad_norm": 77.84225661924857,
+      "learning_rate": 1.478143389201113e-07,
+      "logits/chosen": -0.15583737194538116,
+      "logits/rejected": -0.0702785775065422,
+      "logps/chosen": -3.0332465171813965,
+      "logps/rejected": -3.9285430908203125,
+      "loss": 1.2128,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -3.0332465171813965,
+      "rewards/margins": 0.8952968716621399,
+      "rewards/rejected": -3.9285430908203125,
+      "step": 320
+    },
+    {
+      "epoch": 0.6804501439413766,
+      "grad_norm": 82.4050976791357,
+      "learning_rate": 1.3953046172178413e-07,
+      "logits/chosen": -0.19347061216831207,
+      "logits/rejected": 0.27237311005592346,
+      "logps/chosen": -2.916917324066162,
+      "logps/rejected": -3.891824245452881,
+      "loss": 1.1941,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.916917324066162,
+      "rewards/margins": 0.9749069213867188,
+      "rewards/rejected": -3.891824245452881,
+      "step": 325
+    },
+    {
+      "epoch": 0.6909186076943209,
+      "grad_norm": 74.25892962169908,
+      "learning_rate": 1.3139467229135998e-07,
+      "logits/chosen": -0.2500423491001129,
+      "logits/rejected": 0.1266798973083496,
+      "logps/chosen": -2.7360358238220215,
+      "logps/rejected": -3.647095203399658,
+      "loss": 1.2222,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -2.7360358238220215,
+      "rewards/margins": 0.9110593795776367,
+      "rewards/rejected": -3.647095203399658,
+      "step": 330
+    },
+    {
+      "epoch": 0.7013870714472651,
+      "grad_norm": 68.6281193094518,
+      "learning_rate": 1.2341787690142435e-07,
+      "logits/chosen": -0.4035646915435791,
+      "logits/rejected": -0.033549416810274124,
+      "logps/chosen": -2.763370990753174,
+      "logps/rejected": -3.6889405250549316,
+      "loss": 1.2685,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -2.763370990753174,
+      "rewards/margins": 0.9255691766738892,
+      "rewards/rejected": -3.6889405250549316,
+      "step": 335
+    },
+    {
+      "epoch": 0.7118555352002094,
+      "grad_norm": 64.3244472190874,
+      "learning_rate": 1.1561076868822755e-07,
+      "logits/chosen": -0.36889219284057617,
+      "logits/rejected": -0.027694886550307274,
+      "logps/chosen": -2.9837875366210938,
+      "logps/rejected": -3.666727066040039,
+      "loss": 1.2109,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -2.9837875366210938,
+      "rewards/margins": 0.6829396486282349,
+      "rewards/rejected": -3.666727066040039,
+      "step": 340
+    },
+    {
+      "epoch": 0.7223239989531536,
+      "grad_norm": 67.8756323193863,
+      "learning_rate": 1.0798381331721107e-07,
+      "logits/chosen": -0.3677563965320587,
+      "logits/rejected": -0.05767295882105827,
+      "logps/chosen": -2.810889959335327,
+      "logps/rejected": -3.620894193649292,
+      "loss": 1.1996,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.810889959335327,
+      "rewards/margins": 0.8100040555000305,
+      "rewards/rejected": -3.620894193649292,
+      "step": 345
+    },
+    {
+      "epoch": 0.7327924627060979,
+      "grad_norm": 75.08030338233884,
+      "learning_rate": 1.0054723495346482e-07,
+      "logits/chosen": -0.2898016571998596,
+      "logits/rejected": -0.137650728225708,
+      "logps/chosen": -2.9614298343658447,
+      "logps/rejected": -3.739614963531494,
+      "loss": 1.2604,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -2.9614298343658447,
+      "rewards/margins": 0.7781847715377808,
+      "rewards/rejected": -3.739614963531494,
+      "step": 350
+    },
+    {
+      "epoch": 0.7432609264590422,
+      "grad_norm": 74.74858163258845,
+      "learning_rate": 9.331100255592436e-08,
+      "logits/chosen": -0.2886708378791809,
+      "logits/rejected": 0.029823053628206253,
+      "logps/chosen": -3.0250728130340576,
+      "logps/rejected": -4.2197394371032715,
+      "loss": 1.1354,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -3.0250728130340576,
+      "rewards/margins": 1.194666862487793,
+      "rewards/rejected": -4.2197394371032715,
+      "step": 355
+    },
+    {
+      "epoch": 0.7537293902119864,
+      "grad_norm": 70.24673714433773,
+      "learning_rate": 8.628481651367875e-08,
+      "logits/chosen": -0.40003472566604614,
+      "logits/rejected": -0.07148544490337372,
+      "logps/chosen": -3.224444627761841,
+      "logps/rejected": -3.998364210128784,
+      "loss": 1.1942,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -3.224444627761841,
+      "rewards/margins": 0.7739196419715881,
+      "rewards/rejected": -3.998364210128784,
+      "step": 360
+    },
+    {
+      "epoch": 0.7641978539649307,
+      "grad_norm": 99.18869677889904,
+      "learning_rate": 7.947809564230445e-08,
+      "logits/chosen": -0.20483890175819397,
+      "logits/rejected": 0.11580769717693329,
+      "logps/chosen": -3.247260570526123,
+      "logps/rejected": -4.369873046875,
+      "loss": 1.2124,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -3.247260570526123,
+      "rewards/margins": 1.122612714767456,
+      "rewards/rejected": -4.369873046875,
+      "step": 365
+    },
+    {
+      "epoch": 0.7746663177178749,
+      "grad_norm": 91.27768360001403,
+      "learning_rate": 7.289996455765748e-08,
+      "logits/chosen": -0.2206389605998993,
+      "logits/rejected": 0.08472562581300735,
+      "logps/chosen": -3.4089035987854004,
+      "logps/rejected": -4.553833961486816,
+      "loss": 1.1347,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -3.4089035987854004,
+      "rewards/margins": 1.1449302434921265,
+      "rewards/rejected": -4.553833961486816,
+      "step": 370
+    },
+    {
+      "epoch": 0.7851347814708192,
+      "grad_norm": 76.20144058868142,
+      "learning_rate": 6.655924144404906e-08,
+      "logits/chosen": -0.2008267343044281,
+      "logits/rejected": -0.1611969918012619,
+      "logps/chosen": -3.3004608154296875,
+      "logps/rejected": -4.173122406005859,
+      "loss": 1.1354,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -3.3004608154296875,
+      "rewards/margins": 0.8726619482040405,
+      "rewards/rejected": -4.173122406005859,
+      "step": 375
+    },
+    {
+      "epoch": 0.7956032452237635,
+      "grad_norm": 74.59904017410364,
+      "learning_rate": 6.046442623320145e-08,
+      "logits/chosen": -0.1321212500333786,
+      "logits/rejected": -0.01653924025595188,
+      "logps/chosen": -3.2044310569763184,
+      "logps/rejected": -4.154183864593506,
+      "loss": 1.1999,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -3.2044310569763184,
+      "rewards/margins": 0.9497528076171875,
+      "rewards/rejected": -4.154183864593506,
+      "step": 380
+    },
+    {
+      "epoch": 0.8060717089767077,
+      "grad_norm": 72.89488713559955,
+      "learning_rate": 5.4623689209832484e-08,
+      "logits/chosen": -0.2955227494239807,
+      "logits/rejected": -0.03774283826351166,
+      "logps/chosen": -3.0728917121887207,
+      "logps/rejected": -4.051039695739746,
+      "loss": 1.1963,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -3.0728917121887207,
+      "rewards/margins": 0.9781482815742493,
+      "rewards/rejected": -4.051039695739746,
+      "step": 385
+    },
+    {
+      "epoch": 0.816540172729652,
+      "grad_norm": 67.63752264326851,
+      "learning_rate": 4.904486005914027e-08,
+      "logits/chosen": -0.45038795471191406,
+      "logits/rejected": -0.09373664855957031,
+      "logps/chosen": -3.1700501441955566,
+      "logps/rejected": -4.2985382080078125,
+      "loss": 1.1136,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -3.1700501441955566,
+      "rewards/margins": 1.1284879446029663,
+      "rewards/rejected": -4.2985382080078125,
+      "step": 390
+    },
+    {
+      "epoch": 0.8270086364825961,
+      "grad_norm": 74.45872340513813,
+      "learning_rate": 4.373541737087263e-08,
+      "logits/chosen": -0.49037042260169983,
+      "logits/rejected": -0.09940163046121597,
+      "logps/chosen": -3.04799222946167,
+      "logps/rejected": -4.031525135040283,
+      "loss": 1.1544,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -3.04799222946167,
+      "rewards/margins": 0.9835329055786133,
+      "rewards/rejected": -4.031525135040283,
+      "step": 395
+    },
+    {
+      "epoch": 0.8374771002355405,
+      "grad_norm": 78.72742631851618,
+      "learning_rate": 3.8702478614051345e-08,
+      "logits/chosen": -0.32979053258895874,
+      "logits/rejected": -0.04894006997346878,
+      "logps/chosen": -3.051708698272705,
+      "logps/rejected": -4.071520805358887,
+      "loss": 1.2236,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -3.051708698272705,
+      "rewards/margins": 1.0198121070861816,
+      "rewards/rejected": -4.071520805358887,
+      "step": 400
+    },
+    {
+      "epoch": 0.8479455639884846,
+      "grad_norm": 84.34790591849121,
+      "learning_rate": 3.3952790595787986e-08,
+      "logits/chosen": -0.38359755277633667,
+      "logits/rejected": -0.2002539187669754,
+      "logps/chosen": -3.3085391521453857,
+      "logps/rejected": -4.132696151733398,
+      "loss": 1.1961,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -3.3085391521453857,
+      "rewards/margins": 0.8241568803787231,
+      "rewards/rejected": -4.132696151733398,
+      "step": 405
+    },
+    {
+      "epoch": 0.8584140277414289,
+      "grad_norm": 73.80761771512901,
+      "learning_rate": 2.9492720416985e-08,
+      "logits/chosen": -0.3136187195777893,
+      "logits/rejected": -0.1394844651222229,
+      "logps/chosen": -3.2027900218963623,
+      "logps/rejected": -4.078015327453613,
+      "loss": 1.2319,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -3.2027900218963623,
+      "rewards/margins": 0.8752256631851196,
+      "rewards/rejected": -4.078015327453613,
+      "step": 410
+    },
+    {
+      "epoch": 0.8688824914943732,
+      "grad_norm": 55.42101250348971,
+      "learning_rate": 2.5328246937043525e-08,
+      "logits/chosen": -0.3292819857597351,
+      "logits/rejected": 0.0064947158098220825,
+      "logps/chosen": -3.1560277938842773,
+      "logps/rejected": -4.08798360824585,
+      "loss": 1.1358,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -3.1560277938842773,
+      "rewards/margins": 0.9319561123847961,
+      "rewards/rejected": -4.08798360824585,
+      "step": 415
+    },
+    {
+      "epoch": 0.8793509552473174,
+      "grad_norm": 67.75077864144087,
+      "learning_rate": 2.1464952759020856e-08,
+      "logits/chosen": -0.36769989132881165,
+      "logits/rejected": -0.17563530802726746,
+      "logps/chosen": -3.0870118141174316,
+      "logps/rejected": -3.987121105194092,
+      "loss": 1.2481,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -3.0870118141174316,
+      "rewards/margins": 0.9001096487045288,
+      "rewards/rejected": -3.987121105194092,
+      "step": 420
+    },
+    {
+      "epoch": 0.8898194190002617,
+      "grad_norm": 73.97322762411653,
+      "learning_rate": 1.7908016745981856e-08,
+      "logits/chosen": -0.3043641448020935,
+      "logits/rejected": -0.044486187398433685,
+      "logps/chosen": -3.249401569366455,
+      "logps/rejected": -4.1566572189331055,
+      "loss": 1.1883,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -3.249401569366455,
+      "rewards/margins": 0.9072558283805847,
+      "rewards/rejected": -4.1566572189331055,
+      "step": 425
+    },
+    {
+      "epoch": 0.9002878827532059,
+      "grad_norm": 62.93268497369251,
+      "learning_rate": 1.4662207078575684e-08,
+      "logits/chosen": -0.3604608178138733,
+      "logits/rejected": -0.05810839682817459,
+      "logps/chosen": -3.14021635055542,
+      "logps/rejected": -4.223752975463867,
+      "loss": 1.1337,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -3.14021635055542,
+      "rewards/margins": 1.0835367441177368,
+      "rewards/rejected": -4.223752975463867,
+      "step": 430
+    },
+    {
+      "epoch": 0.9107563465061502,
+      "grad_norm": 72.41742830338748,
+      "learning_rate": 1.1731874863145142e-08,
+      "logits/chosen": -0.3878116011619568,
+      "logits/rejected": -0.25898757576942444,
+      "logps/chosen": -3.085711717605591,
+      "logps/rejected": -4.0059380531311035,
+      "loss": 1.1752,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -3.085711717605591,
+      "rewards/margins": 0.9202262163162231,
+      "rewards/rejected": -4.0059380531311035,
+      "step": 435
+    },
+    {
+      "epoch": 0.9212248102590945,
+      "grad_norm": 67.55543823572617,
+      "learning_rate": 9.12094829893642e-09,
+      "logits/chosen": -0.26049160957336426,
+      "logits/rejected": -0.0604906901717186,
+      "logps/chosen": -3.136883497238159,
+      "logps/rejected": -4.199965000152588,
+      "loss": 1.1765,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -3.136883497238159,
+      "rewards/margins": 1.0630815029144287,
+      "rewards/rejected": -4.199965000152588,
+      "step": 440
+    },
+    {
+      "epoch": 0.9316932740120387,
+      "grad_norm": 68.54556826267816,
+      "learning_rate": 6.832927412229017e-09,
+      "logits/chosen": -0.3159419894218445,
+      "logits/rejected": -0.0612739734351635,
+      "logps/chosen": -3.3173446655273438,
+      "logps/rejected": -4.260361194610596,
+      "loss": 1.1964,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -3.3173446655273438,
+      "rewards/margins": 0.9430168271064758,
+      "rewards/rejected": -4.260361194610596,
+      "step": 445
+    },
+    {
+      "epoch": 0.942161737764983,
+      "grad_norm": 67.1017202252172,
+      "learning_rate": 4.8708793644441086e-09,
+      "logits/chosen": -0.2727358341217041,
+      "logits/rejected": -0.0900549367070198,
+      "logps/chosen": -3.1065354347229004,
+      "logps/rejected": -4.069243431091309,
+      "loss": 1.1456,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -3.1065354347229004,
+      "rewards/margins": 0.9627076983451843,
+      "rewards/rejected": -4.069243431091309,
+      "step": 450
+    },
+    {
+      "epoch": 0.9526302015179272,
+      "grad_norm": 74.36090301118229,
+      "learning_rate": 3.2374343405217884e-09,
+      "logits/chosen": -0.2751519978046417,
+      "logits/rejected": 0.001943744719028473,
+      "logps/chosen": -3.195307493209839,
+      "logps/rejected": -4.041089057922363,
+      "loss": 1.2193,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -3.195307493209839,
+      "rewards/margins": 0.8457815051078796,
+      "rewards/rejected": -4.041089057922363,
+      "step": 455
+    },
+    {
+      "epoch": 0.9630986652708715,
+      "grad_norm": 63.718344779873874,
+      "learning_rate": 1.9347820230782295e-09,
+      "logits/chosen": -0.1701783835887909,
+      "logits/rejected": 0.05854835361242294,
+      "logps/chosen": -3.1413817405700684,
+      "logps/rejected": -4.193324565887451,
+      "loss": 1.1655,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -3.1413817405700684,
+      "rewards/margins": 1.0519429445266724,
+      "rewards/rejected": -4.193324565887451,
+      "step": 460
+    },
+    {
+      "epoch": 0.9735671290238157,
+      "grad_norm": 66.29809600281665,
+      "learning_rate": 9.64668657069706e-10,
+      "logits/chosen": -0.3206351101398468,
+      "logits/rejected": 0.049534112215042114,
+      "logps/chosen": -3.1361141204833984,
+      "logps/rejected": -4.281296730041504,
+      "loss": 1.1251,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -3.1361141204833984,
+      "rewards/margins": 1.1451822519302368,
+      "rewards/rejected": -4.281296730041504,
+      "step": 465
+    },
+    {
+      "epoch": 0.98403559277676,
+      "grad_norm": 74.93911331685558,
+      "learning_rate": 3.2839470889836627e-10,
+      "logits/chosen": -0.34328070282936096,
+      "logits/rejected": -0.11956997960805893,
+      "logps/chosen": -3.241725206375122,
+      "logps/rejected": -4.065664291381836,
+      "loss": 1.2111,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -3.241725206375122,
+      "rewards/margins": 0.8239390254020691,
+      "rewards/rejected": -4.065664291381836,
+      "step": 470
+    },
+    {
+      "epoch": 0.9945040565297043,
+      "grad_norm": 69.55111369299213,
+      "learning_rate": 2.6813123097352287e-11,
+      "logits/chosen": -0.4041665494441986,
+      "logits/rejected": 0.02163061499595642,
+      "logps/chosen": -3.192171573638916,
+      "logps/rejected": -4.112965106964111,
+      "loss": 1.2172,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -3.192171573638916,
+      "rewards/margins": 0.9207934141159058,
+      "rewards/rejected": -4.112965106964111,
+      "step": 475
+    },
+    {
+      "epoch": 0.998691442030882,
+      "step": 477,
+      "total_flos": 0.0,
+      "train_loss": 1.3340201452843048,
+      "train_runtime": 7646.4301,
+      "train_samples_per_second": 7.995,
+      "train_steps_per_second": 0.062
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 477,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000000,
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03f9c9de99606789fb43b901c1471ebd3d0fd7215741bc80432673954776472e
+size 7160