TX

README.md

@@ -0,0 +1,61 @@
+---
+license: apache-2.0
+base_model: alignment-handbook/zephyr-7b-sft-full
+tags:
+- alignment_handbook-handbook
+- generated_from_trainer
+datasets:
+- HuggingFaceH4/ultrafeedback_binarized
+model-index:
+- name: zephyr-7b-MI05-SELM
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# zephyr-7b-MI05-SELM
+
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the HuggingFaceH4/ultrafeedback_binarized dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 2
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 16
+- total_train_batch_size: 128
+- total_eval_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.40.2
+- Pytorch 2.1.2+cu121
+- Datasets 2.14.6
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.998691442030882,
+    "total_flos": 0.0,
+    "train_loss": 1.3747819329707627,
+    "train_runtime": 7204.7044,
+    "train_samples": 61135,
+    "train_samples_per_second": 8.485,
+    "train_steps_per_second": 0.066
+}

config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "alignment-handbook/zephyr-7b-sft-full",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.40.2",
+  "use_cache": true,
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.40.2"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8dc80a479260355c77643051271170c98b5028f6e08ce339c2734d2e1771211f
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:323da549c1e4d637e7a67bb1ed7c59c4f68567c28b645ea71fb605f0f1709682
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7dd6ad501575f323467473a7c1f9f74beefc8d8972f6e17e5b4323b982b3642
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 2048,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.998691442030882,
+    "total_flos": 0.0,
+    "train_loss": 1.3747819329707627,
+    "train_runtime": 7204.7044,
+    "train_samples": 61135,
+    "train_samples_per_second": 8.485,
+    "train_steps_per_second": 0.066
+}

trainer_state.json

@@ -0,0 +1,1455 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.998691442030882,
+  "eval_steps": 500,
+  "global_step": 477,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.010468463752944255,
+      "grad_norm": 7.631302590698798,
+      "learning_rate": 5.208333333333333e-08,
+      "logits/chosen": -2.7707834243774414,
+      "logits/rejected": -2.7284507751464844,
+      "logps/chosen": -1.027999758720398,
+      "logps/rejected": -1.1731758117675781,
+      "loss": 1.5142,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.027999758720398,
+      "rewards/margins": 0.145175963640213,
+      "rewards/rejected": -1.1731758117675781,
+      "step": 5
+    },
+    {
+      "epoch": 0.02093692750588851,
+      "grad_norm": 6.949481214555281,
+      "learning_rate": 1.0416666666666667e-07,
+      "logits/chosen": -2.751293659210205,
+      "logits/rejected": -2.7253918647766113,
+      "logps/chosen": -0.9958034753799438,
+      "logps/rejected": -1.0718666315078735,
+      "loss": 1.5011,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.9958034753799438,
+      "rewards/margins": 0.07606318593025208,
+      "rewards/rejected": -1.0718666315078735,
+      "step": 10
+    },
+    {
+      "epoch": 0.031405391258832765,
+      "grad_norm": 6.93246149046264,
+      "learning_rate": 1.5624999999999999e-07,
+      "logits/chosen": -2.7307419776916504,
+      "logits/rejected": -2.6973299980163574,
+      "logps/chosen": -0.9896200299263,
+      "logps/rejected": -1.1479737758636475,
+      "loss": 1.4936,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.9896200299263,
+      "rewards/margins": 0.15835365653038025,
+      "rewards/rejected": -1.1479737758636475,
+      "step": 15
+    },
+    {
+      "epoch": 0.04187385501177702,
+      "grad_norm": 6.847416841047133,
+      "learning_rate": 2.0833333333333333e-07,
+      "logits/chosen": -2.8032145500183105,
+      "logits/rejected": -2.709728717803955,
+      "logps/chosen": -1.023134469985962,
+      "logps/rejected": -1.1238592863082886,
+      "loss": 1.5132,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.023134469985962,
+      "rewards/margins": 0.10072481632232666,
+      "rewards/rejected": -1.1238592863082886,
+      "step": 20
+    },
+    {
+      "epoch": 0.05234231876472128,
+      "grad_norm": 7.005806187796597,
+      "learning_rate": 2.604166666666667e-07,
+      "logits/chosen": -2.799395799636841,
+      "logits/rejected": -2.775847911834717,
+      "logps/chosen": -1.0309282541275024,
+      "logps/rejected": -1.0878545045852661,
+      "loss": 1.5044,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.0309282541275024,
+      "rewards/margins": 0.056926220655441284,
+      "rewards/rejected": -1.0878545045852661,
+      "step": 25
+    },
+    {
+      "epoch": 0.06281078251766553,
+      "grad_norm": 7.350121902518634,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": -2.7731544971466064,
+      "logits/rejected": -2.7317252159118652,
+      "logps/chosen": -0.9808300733566284,
+      "logps/rejected": -1.0866905450820923,
+      "loss": 1.5044,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.9808300733566284,
+      "rewards/margins": 0.10586042702198029,
+      "rewards/rejected": -1.0866905450820923,
+      "step": 30
+    },
+    {
+      "epoch": 0.07327924627060979,
+      "grad_norm": 7.700359687001099,
+      "learning_rate": 3.645833333333333e-07,
+      "logits/chosen": -2.7761082649230957,
+      "logits/rejected": -2.689614772796631,
+      "logps/chosen": -1.0087323188781738,
+      "logps/rejected": -1.151500940322876,
+      "loss": 1.4925,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.0087323188781738,
+      "rewards/margins": 0.1427687108516693,
+      "rewards/rejected": -1.151500940322876,
+      "step": 35
+    },
+    {
+      "epoch": 0.08374771002355404,
+      "grad_norm": 13.186452732989158,
+      "learning_rate": 4.1666666666666667e-07,
+      "logits/chosen": -2.791355848312378,
+      "logits/rejected": -2.694056749343872,
+      "logps/chosen": -0.9970605969429016,
+      "logps/rejected": -1.2966585159301758,
+      "loss": 1.4668,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.9970605969429016,
+      "rewards/margins": 0.29959791898727417,
+      "rewards/rejected": -1.2966585159301758,
+      "step": 40
+    },
+    {
+      "epoch": 0.0942161737764983,
+      "grad_norm": 7.907271081451181,
+      "learning_rate": 4.6874999999999996e-07,
+      "logits/chosen": -2.7956533432006836,
+      "logits/rejected": -2.7485368251800537,
+      "logps/chosen": -0.9793591499328613,
+      "logps/rejected": -1.1734071969985962,
+      "loss": 1.4659,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.9793591499328613,
+      "rewards/margins": 0.1940479576587677,
+      "rewards/rejected": -1.1734071969985962,
+      "step": 45
+    },
+    {
+      "epoch": 0.10468463752944256,
+      "grad_norm": 10.523506634944091,
+      "learning_rate": 4.999731868769026e-07,
+      "logits/chosen": -2.740318775177002,
+      "logits/rejected": -2.669680118560791,
+      "logps/chosen": -1.023005723953247,
+      "logps/rejected": -1.3500120639801025,
+      "loss": 1.4663,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.023005723953247,
+      "rewards/margins": 0.3270064890384674,
+      "rewards/rejected": -1.3500120639801025,
+      "step": 50
+    },
+    {
+      "epoch": 0.11515310128238682,
+      "grad_norm": 9.075126792521907,
+      "learning_rate": 4.996716052911017e-07,
+      "logits/chosen": -2.7170307636260986,
+      "logits/rejected": -2.678213596343994,
+      "logps/chosen": -1.0640136003494263,
+      "logps/rejected": -1.3454773426055908,
+      "loss": 1.4617,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.0640136003494263,
+      "rewards/margins": 0.28146374225616455,
+      "rewards/rejected": -1.3454773426055908,
+      "step": 55
+    },
+    {
+      "epoch": 0.12562156503533106,
+      "grad_norm": 9.439762477880253,
+      "learning_rate": 4.990353313429303e-07,
+      "logits/chosen": -2.754854917526245,
+      "logits/rejected": -2.7064061164855957,
+      "logps/chosen": -1.103760838508606,
+      "logps/rejected": -1.3274078369140625,
+      "loss": 1.4332,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.103760838508606,
+      "rewards/margins": 0.22364696860313416,
+      "rewards/rejected": -1.3274078369140625,
+      "step": 60
+    },
+    {
+      "epoch": 0.1360900287882753,
+      "grad_norm": 9.434075067475233,
+      "learning_rate": 4.980652179769217e-07,
+      "logits/chosen": -2.7506463527679443,
+      "logits/rejected": -2.6285061836242676,
+      "logps/chosen": -1.0410432815551758,
+      "logps/rejected": -1.671651840209961,
+      "loss": 1.4215,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.0410432815551758,
+      "rewards/margins": 0.6306084394454956,
+      "rewards/rejected": -1.671651840209961,
+      "step": 65
+    },
+    {
+      "epoch": 0.14655849254121958,
+      "grad_norm": 13.786310447291475,
+      "learning_rate": 4.967625656594781e-07,
+      "logits/chosen": -2.656280994415283,
+      "logits/rejected": -2.609356641769409,
+      "logps/chosen": -1.0239083766937256,
+      "logps/rejected": -1.3759796619415283,
+      "loss": 1.4285,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.0239083766937256,
+      "rewards/margins": 0.35207125544548035,
+      "rewards/rejected": -1.3759796619415283,
+      "step": 70
+    },
+    {
+      "epoch": 0.15702695629416383,
+      "grad_norm": 19.70767709035438,
+      "learning_rate": 4.951291206355559e-07,
+      "logits/chosen": -2.755265712738037,
+      "logits/rejected": -2.683993101119995,
+      "logps/chosen": -1.0507580041885376,
+      "logps/rejected": -1.4722309112548828,
+      "loss": 1.4151,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.0507580041885376,
+      "rewards/margins": 0.42147302627563477,
+      "rewards/rejected": -1.4722309112548828,
+      "step": 75
+    },
+    {
+      "epoch": 0.16749542004710807,
+      "grad_norm": 13.640696435085882,
+      "learning_rate": 4.93167072587771e-07,
+      "logits/chosen": -2.719999313354492,
+      "logits/rejected": -2.659780979156494,
+      "logps/chosen": -1.1385526657104492,
+      "logps/rejected": -1.6847137212753296,
+      "loss": 1.4244,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.1385526657104492,
+      "rewards/margins": 0.5461611747741699,
+      "rewards/rejected": -1.6847137212753296,
+      "step": 80
+    },
+    {
+      "epoch": 0.17796388380005235,
+      "grad_norm": 15.610448795236834,
+      "learning_rate": 4.908790517010636e-07,
+      "logits/chosen": -2.7185451984405518,
+      "logits/rejected": -2.6570563316345215,
+      "logps/chosen": -1.0404447317123413,
+      "logps/rejected": -1.5335460901260376,
+      "loss": 1.4359,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.0404447317123413,
+      "rewards/margins": 0.4931013584136963,
+      "rewards/rejected": -1.5335460901260376,
+      "step": 85
+    },
+    {
+      "epoch": 0.1884323475529966,
+      "grad_norm": 15.498065363599855,
+      "learning_rate": 4.882681251368548e-07,
+      "logits/chosen": -2.705440044403076,
+      "logits/rejected": -2.6757359504699707,
+      "logps/chosen": -1.0957661867141724,
+      "logps/rejected": -1.541998267173767,
+      "loss": 1.4046,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.0957661867141724,
+      "rewards/margins": 0.4462320804595947,
+      "rewards/rejected": -1.541998267173767,
+      "step": 90
+    },
+    {
+      "epoch": 0.19890081130594087,
+      "grad_norm": 18.034112089469748,
+      "learning_rate": 4.853377929214243e-07,
+      "logits/chosen": -2.779350996017456,
+      "logits/rejected": -2.6815435886383057,
+      "logps/chosen": -1.1388349533081055,
+      "logps/rejected": -1.6295349597930908,
+      "loss": 1.4191,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.1388349533081055,
+      "rewards/margins": 0.490699827671051,
+      "rewards/rejected": -1.6295349597930908,
+      "step": 95
+    },
+    {
+      "epoch": 0.2093692750588851,
+      "grad_norm": 16.923919026891507,
+      "learning_rate": 4.820919832540181e-07,
+      "logits/chosen": -2.7381911277770996,
+      "logits/rejected": -2.637225389480591,
+      "logps/chosen": -1.1354267597198486,
+      "logps/rejected": -1.7485253810882568,
+      "loss": 1.4134,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.1354267597198486,
+      "rewards/margins": 0.6130987405776978,
+      "rewards/rejected": -1.7485253810882568,
+      "step": 100
+    },
+    {
+      "epoch": 0.21983773881182936,
+      "grad_norm": 18.172912956493622,
+      "learning_rate": 4.785350472409791e-07,
+      "logits/chosen": -2.6725711822509766,
+      "logits/rejected": -2.6469922065734863,
+      "logps/chosen": -1.0028823614120483,
+      "logps/rejected": -1.6847069263458252,
+      "loss": 1.3943,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.0028823614120483,
+      "rewards/margins": 0.6818245649337769,
+      "rewards/rejected": -1.6847069263458252,
+      "step": 105
+    },
+    {
+      "epoch": 0.23030620256477363,
+      "grad_norm": 47.211058537190446,
+      "learning_rate": 4.7467175306295647e-07,
+      "logits/chosen": -2.722909688949585,
+      "logits/rejected": -2.664407730102539,
+      "logps/chosen": -1.1426756381988525,
+      "logps/rejected": -1.676659345626831,
+      "loss": 1.3917,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.1426756381988525,
+      "rewards/margins": 0.5339838266372681,
+      "rewards/rejected": -1.676659345626831,
+      "step": 110
+    },
+    {
+      "epoch": 0.24077466631771788,
+      "grad_norm": 24.055123672850197,
+      "learning_rate": 4.70507279583015e-07,
+      "logits/chosen": -2.680541515350342,
+      "logits/rejected": -2.6233205795288086,
+      "logps/chosen": -1.0654175281524658,
+      "logps/rejected": -1.791318655014038,
+      "loss": 1.3876,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.0654175281524658,
+      "rewards/margins": 0.7259014248847961,
+      "rewards/rejected": -1.791318655014038,
+      "step": 115
+    },
+    {
+      "epoch": 0.2512431300706621,
+      "grad_norm": 26.57015894685055,
+      "learning_rate": 4.6604720940421207e-07,
+      "logits/chosen": -2.6764163970947266,
+      "logits/rejected": -2.651352643966675,
+      "logps/chosen": -1.1118848323822021,
+      "logps/rejected": -1.6098695993423462,
+      "loss": 1.4094,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.1118848323822021,
+      "rewards/margins": 0.49798470735549927,
+      "rewards/rejected": -1.6098695993423462,
+      "step": 120
+    },
+    {
+      "epoch": 0.26171159382360637,
+      "grad_norm": 18.624223201288878,
+      "learning_rate": 4.612975213859487e-07,
+      "logits/chosen": -2.74927020072937,
+      "logits/rejected": -2.7295844554901123,
+      "logps/chosen": -1.156049132347107,
+      "logps/rejected": -1.56387460231781,
+      "loss": 1.4148,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.156049132347107,
+      "rewards/margins": 0.4078255295753479,
+      "rewards/rejected": -1.56387460231781,
+      "step": 125
+    },
+    {
+      "epoch": 0.2721800575765506,
+      "grad_norm": 27.721075966231663,
+      "learning_rate": 4.5626458262912735e-07,
+      "logits/chosen": -2.710625171661377,
+      "logits/rejected": -2.6833271980285645,
+      "logps/chosen": -1.1802499294281006,
+      "logps/rejected": -1.4933916330337524,
+      "loss": 1.4308,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.1802499294281006,
+      "rewards/margins": 0.31314152479171753,
+      "rewards/rejected": -1.4933916330337524,
+      "step": 130
+    },
+    {
+      "epoch": 0.2826485213294949,
+      "grad_norm": 25.646654099025422,
+      "learning_rate": 4.5095513994085974e-07,
+      "logits/chosen": -2.690138578414917,
+      "logits/rejected": -2.59498929977417,
+      "logps/chosen": -1.2173268795013428,
+      "logps/rejected": -2.0789878368377686,
+      "loss": 1.4068,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.2173268795013428,
+      "rewards/margins": 0.8616611361503601,
+      "rewards/rejected": -2.0789878368377686,
+      "step": 135
+    },
+    {
+      "epoch": 0.29311698508243916,
+      "grad_norm": 15.591514935900287,
+      "learning_rate": 4.453763107901675e-07,
+      "logits/chosen": -2.715076208114624,
+      "logits/rejected": -2.666903018951416,
+      "logps/chosen": -1.191408634185791,
+      "logps/rejected": -1.7779796123504639,
+      "loss": 1.3784,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.191408634185791,
+      "rewards/margins": 0.5865710377693176,
+      "rewards/rejected": -1.7779796123504639,
+      "step": 140
+    },
+    {
+      "epoch": 0.3035854488353834,
+      "grad_norm": 12.601893786620954,
+      "learning_rate": 4.395355737667985e-07,
+      "logits/chosen": -2.7022361755371094,
+      "logits/rejected": -2.650947093963623,
+      "logps/chosen": -1.1326510906219482,
+      "logps/rejected": -1.6125431060791016,
+      "loss": 1.3973,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.1326510906219482,
+      "rewards/margins": 0.4798920154571533,
+      "rewards/rejected": -1.6125431060791016,
+      "step": 145
+    },
+    {
+      "epoch": 0.31405391258832765,
+      "grad_norm": 15.497858847663315,
+      "learning_rate": 4.3344075855595097e-07,
+      "logits/chosen": -2.723766326904297,
+      "logits/rejected": -2.704010486602783,
+      "logps/chosen": -1.0249402523040771,
+      "logps/rejected": -1.5360591411590576,
+      "loss": 1.3712,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.0249402523040771,
+      "rewards/margins": 0.51111900806427,
+      "rewards/rejected": -1.5360591411590576,
+      "step": 150
+    },
+    {
+      "epoch": 0.3245223763412719,
+      "grad_norm": 17.077156505391155,
+      "learning_rate": 4.271000354423425e-07,
+      "logits/chosen": -2.6389071941375732,
+      "logits/rejected": -2.5978379249572754,
+      "logps/chosen": -1.1586675643920898,
+      "logps/rejected": -1.6560027599334717,
+      "loss": 1.3901,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.1586675643920898,
+      "rewards/margins": 0.497335284948349,
+      "rewards/rejected": -1.6560027599334717,
+      "step": 155
+    },
+    {
+      "epoch": 0.33499084009421615,
+      "grad_norm": 22.945534571845954,
+      "learning_rate": 4.2052190435769554e-07,
+      "logits/chosen": -2.5543599128723145,
+      "logits/rejected": -2.5258305072784424,
+      "logps/chosen": -1.284404993057251,
+      "logps/rejected": -1.9095481634140015,
+      "loss": 1.3994,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.284404993057251,
+      "rewards/margins": 0.62514328956604,
+      "rewards/rejected": -1.9095481634140015,
+      "step": 160
+    },
+    {
+      "epoch": 0.34545930384716045,
+      "grad_norm": 22.9080407885994,
+      "learning_rate": 4.137151834863213e-07,
+      "logits/chosen": -2.6129097938537598,
+      "logits/rejected": -2.5123066902160645,
+      "logps/chosen": -1.2131001949310303,
+      "logps/rejected": -1.8760910034179688,
+      "loss": 1.3639,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.2131001949310303,
+      "rewards/margins": 0.6629905104637146,
+      "rewards/rejected": -1.8760910034179688,
+      "step": 165
+    },
+    {
+      "epoch": 0.3559277676001047,
+      "grad_norm": 23.012826789095357,
+      "learning_rate": 4.0668899744407567e-07,
+      "logits/chosen": -2.6876988410949707,
+      "logits/rejected": -2.6080994606018066,
+      "logps/chosen": -1.1881000995635986,
+      "logps/rejected": -1.9103021621704102,
+      "loss": 1.3735,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.1881000995635986,
+      "rewards/margins": 0.7222020626068115,
+      "rewards/rejected": -1.9103021621704102,
+      "step": 170
+    },
+    {
+      "epoch": 0.36639623135304894,
+      "grad_norm": 22.848365401534277,
+      "learning_rate": 3.994527650465352e-07,
+      "logits/chosen": -2.6223888397216797,
+      "logits/rejected": -2.510727643966675,
+      "logps/chosen": -1.2682397365570068,
+      "logps/rejected": -1.8920142650604248,
+      "loss": 1.4095,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.2682397365570068,
+      "rewards/margins": 0.623774528503418,
+      "rewards/rejected": -1.8920142650604248,
+      "step": 175
+    },
+    {
+      "epoch": 0.3768646951059932,
+      "grad_norm": 16.908175946250164,
+      "learning_rate": 3.920161866827889e-07,
+      "logits/chosen": -2.576324462890625,
+      "logits/rejected": -2.541231632232666,
+      "logps/chosen": -1.1460561752319336,
+      "logps/rejected": -1.8999898433685303,
+      "loss": 1.3719,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.1460561752319336,
+      "rewards/margins": 0.7539336085319519,
+      "rewards/rejected": -1.8999898433685303,
+      "step": 180
+    },
+    {
+      "epoch": 0.38733315885893743,
+      "grad_norm": 21.471414104048915,
+      "learning_rate": 3.8438923131177237e-07,
+      "logits/chosen": -2.453334331512451,
+      "logits/rejected": -2.260716438293457,
+      "logps/chosen": -1.1970899105072021,
+      "logps/rejected": -2.2250547409057617,
+      "loss": 1.3519,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.1970899105072021,
+      "rewards/margins": 1.0279648303985596,
+      "rewards/rejected": -2.2250547409057617,
+      "step": 185
+    },
+    {
+      "epoch": 0.39780162261188173,
+      "grad_norm": 25.009337039540963,
+      "learning_rate": 3.765821230985757e-07,
+      "logits/chosen": -2.283473253250122,
+      "logits/rejected": -2.178825855255127,
+      "logps/chosen": -1.2332468032836914,
+      "logps/rejected": -1.9305975437164307,
+      "loss": 1.4198,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.2332468032836914,
+      "rewards/margins": 0.6973507404327393,
+      "rewards/rejected": -1.9305975437164307,
+      "step": 190
+    },
+    {
+      "epoch": 0.408270086364826,
+      "grad_norm": 17.444378318038115,
+      "learning_rate": 3.6860532770864005e-07,
+      "logits/chosen": -2.3795723915100098,
+      "logits/rejected": -2.2288522720336914,
+      "logps/chosen": -1.1638405323028564,
+      "logps/rejected": -1.7543007135391235,
+      "loss": 1.3983,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.1638405323028564,
+      "rewards/margins": 0.5904603004455566,
+      "rewards/rejected": -1.7543007135391235,
+      "step": 195
+    },
+    {
+      "epoch": 0.4187385501177702,
+      "grad_norm": 15.428251714459405,
+      "learning_rate": 3.604695382782159e-07,
+      "logits/chosen": -2.31533145904541,
+      "logits/rejected": -2.2262110710144043,
+      "logps/chosen": -1.1396164894104004,
+      "logps/rejected": -1.730507493019104,
+      "loss": 1.3902,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.1396164894104004,
+      "rewards/margins": 0.5908910036087036,
+      "rewards/rejected": -1.730507493019104,
+      "step": 200
+    },
+    {
+      "epoch": 0.42920701387071447,
+      "grad_norm": 19.194522439269107,
+      "learning_rate": 3.5218566107988867e-07,
+      "logits/chosen": -1.926476240158081,
+      "logits/rejected": -1.6759207248687744,
+      "logps/chosen": -1.0454155206680298,
+      "logps/rejected": -1.7578277587890625,
+      "loss": 1.3552,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.0454155206680298,
+      "rewards/margins": 0.7124123573303223,
+      "rewards/rejected": -1.7578277587890625,
+      "step": 205
+    },
+    {
+      "epoch": 0.4396754776236587,
+      "grad_norm": 25.69506263243762,
+      "learning_rate": 3.4376480090239047e-07,
+      "logits/chosen": -1.7014926671981812,
+      "logits/rejected": -1.372465968132019,
+      "logps/chosen": -1.1878430843353271,
+      "logps/rejected": -1.839996576309204,
+      "loss": 1.3821,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.1878430843353271,
+      "rewards/margins": 0.6521533131599426,
+      "rewards/rejected": -1.839996576309204,
+      "step": 210
+    },
+    {
+      "epoch": 0.45014394137660296,
+      "grad_norm": 19.66701196453165,
+      "learning_rate": 3.3521824616429284e-07,
+      "logits/chosen": -1.6308574676513672,
+      "logits/rejected": -1.2923319339752197,
+      "logps/chosen": -1.1453461647033691,
+      "logps/rejected": -1.9198291301727295,
+      "loss": 1.3574,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.1453461647033691,
+      "rewards/margins": 0.774483323097229,
+      "rewards/rejected": -1.9198291301727295,
+      "step": 215
+    },
+    {
+      "epoch": 0.46061240512954726,
+      "grad_norm": 36.150180674302824,
+      "learning_rate": 3.265574537815398e-07,
+      "logits/chosen": -1.6699841022491455,
+      "logits/rejected": -1.1387865543365479,
+      "logps/chosen": -1.1862000226974487,
+      "logps/rejected": -2.100661516189575,
+      "loss": 1.3618,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.1862000226974487,
+      "rewards/margins": 0.9144614934921265,
+      "rewards/rejected": -2.100661516189575,
+      "step": 220
+    },
+    {
+      "epoch": 0.4710808688824915,
+      "grad_norm": 21.20657529926906,
+      "learning_rate": 3.1779403380910425e-07,
+      "logits/chosen": -1.573144793510437,
+      "logits/rejected": -1.014370322227478,
+      "logps/chosen": -1.1252361536026,
+      "logps/rejected": -2.136768102645874,
+      "loss": 1.3361,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.1252361536026,
+      "rewards/margins": 1.0115318298339844,
+      "rewards/rejected": -2.136768102645874,
+      "step": 225
+    },
+    {
+      "epoch": 0.48154933263543576,
+      "grad_norm": 20.81339747835316,
+      "learning_rate": 3.0893973387735683e-07,
+      "logits/chosen": -1.475577473640442,
+      "logits/rejected": -0.8431582450866699,
+      "logps/chosen": -1.1428128480911255,
+      "logps/rejected": -2.3163764476776123,
+      "loss": 1.345,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.1428128480911255,
+      "rewards/margins": 1.1735635995864868,
+      "rewards/rejected": -2.3163764476776123,
+      "step": 230
+    },
+    {
+      "epoch": 0.49201779638838,
+      "grad_norm": 25.711748027315434,
+      "learning_rate": 3.000064234440111e-07,
+      "logits/chosen": -1.3059219121932983,
+      "logits/rejected": -0.7800277471542358,
+      "logps/chosen": -1.089871883392334,
+      "logps/rejected": -1.943511962890625,
+      "loss": 1.3701,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.089871883392334,
+      "rewards/margins": 0.8536401987075806,
+      "rewards/rejected": -1.943511962890625,
+      "step": 235
+    },
+    {
+      "epoch": 0.5024862601413242,
+      "grad_norm": 15.99544959403641,
+      "learning_rate": 2.910060778827554e-07,
+      "logits/chosen": -1.499245285987854,
+      "logits/rejected": -1.0100138187408447,
+      "logps/chosen": -1.0744446516036987,
+      "logps/rejected": -1.84768807888031,
+      "loss": 1.3668,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.0744446516036987,
+      "rewards/margins": 0.7732433080673218,
+      "rewards/rejected": -1.84768807888031,
+      "step": 240
+    },
+    {
+      "epoch": 0.5129547238942685,
+      "grad_norm": 16.81971929638616,
+      "learning_rate": 2.8195076242990116e-07,
+      "logits/chosen": -1.5549015998840332,
+      "logits/rejected": -1.0168583393096924,
+      "logps/chosen": -1.208172082901001,
+      "logps/rejected": -1.8721519708633423,
+      "loss": 1.3684,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.208172082901001,
+      "rewards/margins": 0.6639797687530518,
+      "rewards/rejected": -1.8721519708633423,
+      "step": 245
+    },
+    {
+      "epoch": 0.5234231876472127,
+      "grad_norm": 25.668874451608698,
+      "learning_rate": 2.7285261601056697e-07,
+      "logits/chosen": -1.4319603443145752,
+      "logits/rejected": -0.8526409268379211,
+      "logps/chosen": -1.1322458982467651,
+      "logps/rejected": -1.981615662574768,
+      "loss": 1.3785,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.1322458982467651,
+      "rewards/margins": 0.8493697047233582,
+      "rewards/rejected": -1.981615662574768,
+      "step": 250
+    },
+    {
+      "epoch": 0.533891651400157,
+      "grad_norm": 24.7591899309445,
+      "learning_rate": 2.6372383496608186e-07,
+      "logits/chosen": -0.9910232424736023,
+      "logits/rejected": -0.1822575181722641,
+      "logps/chosen": -1.297499418258667,
+      "logps/rejected": -2.4289941787719727,
+      "loss": 1.3136,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.297499418258667,
+      "rewards/margins": 1.1314946413040161,
+      "rewards/rejected": -2.4289941787719727,
+      "step": 255
+    },
+    {
+      "epoch": 0.5443601151531012,
+      "grad_norm": 20.005217624077922,
+      "learning_rate": 2.5457665670441937e-07,
+      "logits/chosen": -1.2755597829818726,
+      "logits/rejected": -0.5895196199417114,
+      "logps/chosen": -1.0270662307739258,
+      "logps/rejected": -2.185426712036133,
+      "loss": 1.348,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.0270662307739258,
+      "rewards/margins": 1.158360481262207,
+      "rewards/rejected": -2.185426712036133,
+      "step": 260
+    },
+    {
+      "epoch": 0.5548285789060455,
+      "grad_norm": 23.193814761017656,
+      "learning_rate": 2.454233432955807e-07,
+      "logits/chosen": -1.4398943185806274,
+      "logits/rejected": -0.8867106437683105,
+      "logps/chosen": -1.1513334512710571,
+      "logps/rejected": -1.719251036643982,
+      "loss": 1.3446,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.1513334512710571,
+      "rewards/margins": 0.5679178237915039,
+      "rewards/rejected": -1.719251036643982,
+      "step": 265
+    },
+    {
+      "epoch": 0.5652970426589898,
+      "grad_norm": 34.7459644126849,
+      "learning_rate": 2.3627616503391812e-07,
+      "logits/chosen": -1.335096001625061,
+      "logits/rejected": -0.7483940124511719,
+      "logps/chosen": -1.2410709857940674,
+      "logps/rejected": -1.920100212097168,
+      "loss": 1.3635,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.2410709857940674,
+      "rewards/margins": 0.6790293455123901,
+      "rewards/rejected": -1.920100212097168,
+      "step": 270
+    },
+    {
+      "epoch": 0.575765506411934,
+      "grad_norm": 24.36604189795506,
+      "learning_rate": 2.2714738398943308e-07,
+      "logits/chosen": -1.1961032152175903,
+      "logits/rejected": -0.2931798994541168,
+      "logps/chosen": -1.1527847051620483,
+      "logps/rejected": -2.2658238410949707,
+      "loss": 1.3351,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.1527847051620483,
+      "rewards/margins": 1.1130387783050537,
+      "rewards/rejected": -2.2658238410949707,
+      "step": 275
+    },
+    {
+      "epoch": 0.5862339701648783,
+      "grad_norm": 21.064933326366333,
+      "learning_rate": 2.1804923757009882e-07,
+      "logits/chosen": -1.2238754034042358,
+      "logits/rejected": -0.43058863282203674,
+      "logps/chosen": -1.0913419723510742,
+      "logps/rejected": -1.885957956314087,
+      "loss": 1.3521,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.0913419723510742,
+      "rewards/margins": 0.7946157455444336,
+      "rewards/rejected": -1.885957956314087,
+      "step": 280
+    },
+    {
+      "epoch": 0.5967024339178225,
+      "grad_norm": 22.882367631876512,
+      "learning_rate": 2.089939221172446e-07,
+      "logits/chosen": -1.3524372577667236,
+      "logits/rejected": -0.9207743406295776,
+      "logps/chosen": -1.1918188333511353,
+      "logps/rejected": -2.069408893585205,
+      "loss": 1.3471,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.1918188333511353,
+      "rewards/margins": 0.8775898814201355,
+      "rewards/rejected": -2.069408893585205,
+      "step": 285
+    },
+    {
+      "epoch": 0.6071708976707668,
+      "grad_norm": 25.205431418510916,
+      "learning_rate": 1.9999357655598891e-07,
+      "logits/chosen": -1.240950345993042,
+      "logits/rejected": -0.5687157511711121,
+      "logps/chosen": -1.2093303203582764,
+      "logps/rejected": -2.087651252746582,
+      "loss": 1.3354,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.2093303203582764,
+      "rewards/margins": 0.8783208131790161,
+      "rewards/rejected": -2.087651252746582,
+      "step": 290
+    },
+    {
+      "epoch": 0.6176393614237111,
+      "grad_norm": 28.00791593703933,
+      "learning_rate": 1.9106026612264315e-07,
+      "logits/chosen": -1.1928002834320068,
+      "logits/rejected": -0.4924652576446533,
+      "logps/chosen": -1.2354944944381714,
+      "logps/rejected": -2.1065988540649414,
+      "loss": 1.3707,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.2354944944381714,
+      "rewards/margins": 0.8711041212081909,
+      "rewards/rejected": -2.1065988540649414,
+      "step": 295
+    },
+    {
+      "epoch": 0.6281078251766553,
+      "grad_norm": 24.457847868426004,
+      "learning_rate": 1.8220596619089573e-07,
+      "logits/chosen": -1.2340965270996094,
+      "logits/rejected": -0.6534504294395447,
+      "logps/chosen": -1.186089038848877,
+      "logps/rejected": -2.029968738555908,
+      "loss": 1.3421,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.186089038848877,
+      "rewards/margins": 0.8438796997070312,
+      "rewards/rejected": -2.029968738555908,
+      "step": 300
+    },
+    {
+      "epoch": 0.6385762889295996,
+      "grad_norm": 27.972140846529875,
+      "learning_rate": 1.7344254621846017e-07,
+      "logits/chosen": -1.0477724075317383,
+      "logits/rejected": -0.3579182028770447,
+      "logps/chosen": -1.2306467294692993,
+      "logps/rejected": -2.213597059249878,
+      "loss": 1.3325,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.2306467294692993,
+      "rewards/margins": 0.9829503893852234,
+      "rewards/rejected": -2.213597059249878,
+      "step": 305
+    },
+    {
+      "epoch": 0.6490447526825438,
+      "grad_norm": 21.126429643053488,
+      "learning_rate": 1.647817538357072e-07,
+      "logits/chosen": -0.967767596244812,
+      "logits/rejected": -0.5430553555488586,
+      "logps/chosen": -1.1055750846862793,
+      "logps/rejected": -1.997175931930542,
+      "loss": 1.3201,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.1055750846862793,
+      "rewards/margins": 0.8916007280349731,
+      "rewards/rejected": -1.997175931930542,
+      "step": 310
+    },
+    {
+      "epoch": 0.6595132164354881,
+      "grad_norm": 37.81135873666922,
+      "learning_rate": 1.562351990976095e-07,
+      "logits/chosen": -0.9042571783065796,
+      "logits/rejected": -0.11697681248188019,
+      "logps/chosen": -1.1413376331329346,
+      "logps/rejected": -2.2313427925109863,
+      "loss": 1.335,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.1413376331329346,
+      "rewards/margins": 1.0900049209594727,
+      "rewards/rejected": -2.2313427925109863,
+      "step": 315
+    },
+    {
+      "epoch": 0.6699816801884323,
+      "grad_norm": 22.549506139806287,
+      "learning_rate": 1.478143389201113e-07,
+      "logits/chosen": -0.7293549180030823,
+      "logits/rejected": -0.27204781770706177,
+      "logps/chosen": -1.185328722000122,
+      "logps/rejected": -2.089491605758667,
+      "loss": 1.308,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.185328722000122,
+      "rewards/margins": 0.9041631817817688,
+      "rewards/rejected": -2.089491605758667,
+      "step": 320
+    },
+    {
+      "epoch": 0.6804501439413766,
+      "grad_norm": 22.04232630753836,
+      "learning_rate": 1.3953046172178413e-07,
+      "logits/chosen": -0.9256671071052551,
+      "logits/rejected": -0.02759646438062191,
+      "logps/chosen": -1.1488409042358398,
+      "logps/rejected": -2.169890880584717,
+      "loss": 1.3024,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.1488409042358398,
+      "rewards/margins": 1.021050214767456,
+      "rewards/rejected": -2.169890880584717,
+      "step": 325
+    },
+    {
+      "epoch": 0.6909186076943209,
+      "grad_norm": 23.07193462400401,
+      "learning_rate": 1.3139467229135998e-07,
+      "logits/chosen": -0.845870852470398,
+      "logits/rejected": -0.20406107604503632,
+      "logps/chosen": -1.1633460521697998,
+      "logps/rejected": -1.813515067100525,
+      "loss": 1.3519,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.1633460521697998,
+      "rewards/margins": 0.6501691341400146,
+      "rewards/rejected": -1.813515067100525,
+      "step": 330
+    },
+    {
+      "epoch": 0.7013870714472651,
+      "grad_norm": 23.166253111385615,
+      "learning_rate": 1.2341787690142435e-07,
+      "logits/chosen": -1.0633478164672852,
+      "logits/rejected": -0.322580486536026,
+      "logps/chosen": -1.0509016513824463,
+      "logps/rejected": -1.967553734779358,
+      "loss": 1.3448,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.0509016513824463,
+      "rewards/margins": 0.9166520833969116,
+      "rewards/rejected": -1.967553734779358,
+      "step": 335
+    },
+    {
+      "epoch": 0.7118555352002094,
+      "grad_norm": 36.55063112089001,
+      "learning_rate": 1.1561076868822755e-07,
+      "logits/chosen": -0.9787310361862183,
+      "logits/rejected": -0.23488755524158478,
+      "logps/chosen": -1.3403594493865967,
+      "logps/rejected": -2.003267288208008,
+      "loss": 1.3737,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.3403594493865967,
+      "rewards/margins": 0.6629079580307007,
+      "rewards/rejected": -2.003267288208008,
+      "step": 340
+    },
+    {
+      "epoch": 0.7223239989531536,
+      "grad_norm": 20.352017367464722,
+      "learning_rate": 1.0798381331721107e-07,
+      "logits/chosen": -0.9965354800224304,
+      "logits/rejected": -0.1640423834323883,
+      "logps/chosen": -1.143103003501892,
+      "logps/rejected": -2.10261869430542,
+      "loss": 1.3502,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.143103003501892,
+      "rewards/margins": 0.9595154523849487,
+      "rewards/rejected": -2.10261869430542,
+      "step": 345
+    },
+    {
+      "epoch": 0.7327924627060979,
+      "grad_norm": 25.25408282143297,
+      "learning_rate": 1.0054723495346482e-07,
+      "logits/chosen": -0.7439842224121094,
+      "logits/rejected": -0.326732337474823,
+      "logps/chosen": -1.1874338388442993,
+      "logps/rejected": -1.9638007879257202,
+      "loss": 1.3647,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.1874338388442993,
+      "rewards/margins": 0.7763670086860657,
+      "rewards/rejected": -1.9638007879257202,
+      "step": 350
+    },
+    {
+      "epoch": 0.7432609264590422,
+      "grad_norm": 18.046932480362155,
+      "learning_rate": 9.331100255592436e-08,
+      "logits/chosen": -0.8028404116630554,
+      "logits/rejected": 0.0012960076564922929,
+      "logps/chosen": -1.001501441001892,
+      "logps/rejected": -2.1421008110046387,
+      "loss": 1.3092,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.001501441001892,
+      "rewards/margins": 1.1405994892120361,
+      "rewards/rejected": -2.1421008110046387,
+      "step": 355
+    },
+    {
+      "epoch": 0.7537293902119864,
+      "grad_norm": 24.86376502778227,
+      "learning_rate": 8.628481651367875e-08,
+      "logits/chosen": -1.0760581493377686,
+      "logits/rejected": -0.36155739426612854,
+      "logps/chosen": -1.0425236225128174,
+      "logps/rejected": -1.7828603982925415,
+      "loss": 1.3051,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.0425236225128174,
+      "rewards/margins": 0.7403370141983032,
+      "rewards/rejected": -1.7828603982925415,
+      "step": 360
+    },
+    {
+      "epoch": 0.7641978539649307,
+      "grad_norm": 21.60526931897869,
+      "learning_rate": 7.947809564230445e-08,
+      "logits/chosen": -0.7209302186965942,
+      "logits/rejected": 0.01974741742014885,
+      "logps/chosen": -1.1903411149978638,
+      "logps/rejected": -2.1893298625946045,
+      "loss": 1.3752,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.1903411149978638,
+      "rewards/margins": 0.9989888072013855,
+      "rewards/rejected": -2.1893298625946045,
+      "step": 365
+    },
+    {
+      "epoch": 0.7746663177178749,
+      "grad_norm": 22.57059768777603,
+      "learning_rate": 7.289996455765748e-08,
+      "logits/chosen": -0.7033840417861938,
+      "logits/rejected": 0.10633859783411026,
+      "logps/chosen": -1.241659164428711,
+      "logps/rejected": -2.4531188011169434,
+      "loss": 1.3325,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.241659164428711,
+      "rewards/margins": 1.2114596366882324,
+      "rewards/rejected": -2.4531188011169434,
+      "step": 370
+    },
+    {
+      "epoch": 0.7851347814708192,
+      "grad_norm": 23.77470848150414,
+      "learning_rate": 6.655924144404906e-08,
+      "logits/chosen": -0.5643360614776611,
+      "logits/rejected": -0.4250950217247009,
+      "logps/chosen": -1.1594164371490479,
+      "logps/rejected": -2.0758109092712402,
+      "loss": 1.319,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.1594164371490479,
+      "rewards/margins": 0.9163944125175476,
+      "rewards/rejected": -2.0758109092712402,
+      "step": 375
+    },
+    {
+      "epoch": 0.7956032452237635,
+      "grad_norm": 31.139993849555335,
+      "learning_rate": 6.046442623320145e-08,
+      "logits/chosen": -0.4887496531009674,
+      "logits/rejected": -0.15603157877922058,
+      "logps/chosen": -1.2021204233169556,
+      "logps/rejected": -2.0649125576019287,
+      "loss": 1.3035,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.2021204233169556,
+      "rewards/margins": 0.8627923130989075,
+      "rewards/rejected": -2.0649125576019287,
+      "step": 380
+    },
+    {
+      "epoch": 0.8060717089767077,
+      "grad_norm": 21.401361972850502,
+      "learning_rate": 5.4623689209832484e-08,
+      "logits/chosen": -0.7876440286636353,
+      "logits/rejected": -0.1371804028749466,
+      "logps/chosen": -1.1165800094604492,
+      "logps/rejected": -1.9990746974945068,
+      "loss": 1.3148,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.1165800094604492,
+      "rewards/margins": 0.8824948072433472,
+      "rewards/rejected": -1.9990746974945068,
+      "step": 385
+    },
+    {
+      "epoch": 0.816540172729652,
+      "grad_norm": 27.233602498877953,
+      "learning_rate": 4.904486005914027e-08,
+      "logits/chosen": -0.9208453297615051,
+      "logits/rejected": 0.03846040368080139,
+      "logps/chosen": -1.142458200454712,
+      "logps/rejected": -2.2543492317199707,
+      "loss": 1.2986,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.142458200454712,
+      "rewards/margins": 1.1118909120559692,
+      "rewards/rejected": -2.2543492317199707,
+      "step": 390
+    },
+    {
+      "epoch": 0.8270086364825961,
+      "grad_norm": 27.619241835201624,
+      "learning_rate": 4.373541737087263e-08,
+      "logits/chosen": -0.8224166631698608,
+      "logits/rejected": 0.22443437576293945,
+      "logps/chosen": -1.1223267316818237,
+      "logps/rejected": -2.0982825756073,
+      "loss": 1.3196,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.1223267316818237,
+      "rewards/margins": 0.9759558439254761,
+      "rewards/rejected": -2.0982825756073,
+      "step": 395
+    },
+    {
+      "epoch": 0.8374771002355405,
+      "grad_norm": 23.574179623539557,
+      "learning_rate": 3.8702478614051345e-08,
+      "logits/chosen": -0.5700989961624146,
+      "logits/rejected": 0.09446726739406586,
+      "logps/chosen": -1.0945535898208618,
+      "logps/rejected": -2.0572116374969482,
+      "loss": 1.3444,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.0945535898208618,
+      "rewards/margins": 0.9626580476760864,
+      "rewards/rejected": -2.0572116374969482,
+      "step": 400
+    },
+    {
+      "epoch": 0.8479455639884846,
+      "grad_norm": 27.16821359859113,
+      "learning_rate": 3.3952790595787986e-08,
+      "logits/chosen": -0.5356675982475281,
+      "logits/rejected": -0.06621427834033966,
+      "logps/chosen": -1.2879823446273804,
+      "logps/rejected": -2.23671555519104,
+      "loss": 1.3459,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.2879823446273804,
+      "rewards/margins": 0.9487331509590149,
+      "rewards/rejected": -2.23671555519104,
+      "step": 405
+    },
+    {
+      "epoch": 0.8584140277414289,
+      "grad_norm": 24.735499686166555,
+      "learning_rate": 2.9492720416985e-08,
+      "logits/chosen": -0.6337080597877502,
+      "logits/rejected": -0.15622717142105103,
+      "logps/chosen": -1.1116082668304443,
+      "logps/rejected": -1.9464092254638672,
+      "loss": 1.3254,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.1116082668304443,
+      "rewards/margins": 0.8348008990287781,
+      "rewards/rejected": -1.9464092254638672,
+      "step": 410
+    },
+    {
+      "epoch": 0.8688824914943732,
+      "grad_norm": 27.139165593959646,
+      "learning_rate": 2.5328246937043525e-08,
+      "logits/chosen": -0.5524105429649353,
+      "logits/rejected": 0.30828729271888733,
+      "logps/chosen": -1.1661826372146606,
+      "logps/rejected": -2.2065200805664062,
+      "loss": 1.2887,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.1661826372146606,
+      "rewards/margins": 1.0403375625610352,
+      "rewards/rejected": -2.2065200805664062,
+      "step": 415
+    },
+    {
+      "epoch": 0.8793509552473174,
+      "grad_norm": 21.202013882918227,
+      "learning_rate": 2.1464952759020856e-08,
+      "logits/chosen": -0.5271106958389282,
+      "logits/rejected": 0.04782446473836899,
+      "logps/chosen": -1.1561858654022217,
+      "logps/rejected": -2.1839046478271484,
+      "loss": 1.3408,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.1561858654022217,
+      "rewards/margins": 1.0277187824249268,
+      "rewards/rejected": -2.1839046478271484,
+      "step": 420
+    },
+    {
+      "epoch": 0.8898194190002617,
+      "grad_norm": 21.7364989280612,
+      "learning_rate": 1.7908016745981856e-08,
+      "logits/chosen": -0.5031063556671143,
+      "logits/rejected": 0.18595072627067566,
+      "logps/chosen": -1.3663603067398071,
+      "logps/rejected": -2.1731326580047607,
+      "loss": 1.3373,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.3663603067398071,
+      "rewards/margins": 0.8067724108695984,
+      "rewards/rejected": -2.1731326580047607,
+      "step": 425
+    },
+    {
+      "epoch": 0.9002878827532059,
+      "grad_norm": 28.976870006935112,
+      "learning_rate": 1.4662207078575684e-08,
+      "logits/chosen": -0.6391962170600891,
+      "logits/rejected": 0.05445007234811783,
+      "logps/chosen": -1.2463514804840088,
+      "logps/rejected": -2.253527879714966,
+      "loss": 1.3176,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.2463514804840088,
+      "rewards/margins": 1.007176399230957,
+      "rewards/rejected": -2.253527879714966,
+      "step": 430
+    },
+    {
+      "epoch": 0.9107563465061502,
+      "grad_norm": 29.575680170367956,
+      "learning_rate": 1.1731874863145142e-08,
+      "logits/chosen": -0.534844696521759,
+      "logits/rejected": -0.06693878024816513,
+      "logps/chosen": -1.227757215499878,
+      "logps/rejected": -2.329684019088745,
+      "loss": 1.3387,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.227757215499878,
+      "rewards/margins": 1.1019268035888672,
+      "rewards/rejected": -2.329684019088745,
+      "step": 435
+    },
+    {
+      "epoch": 0.9212248102590945,
+      "grad_norm": 26.81740086509168,
+      "learning_rate": 9.12094829893642e-09,
+      "logits/chosen": -0.4514737129211426,
+      "logits/rejected": 0.175465926527977,
+      "logps/chosen": -1.1389667987823486,
+      "logps/rejected": -2.100003719329834,
+      "loss": 1.333,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.1389667987823486,
+      "rewards/margins": 0.9610370397567749,
+      "rewards/rejected": -2.100003719329834,
+      "step": 440
+    },
+    {
+      "epoch": 0.9316932740120387,
+      "grad_norm": 23.37770169459381,
+      "learning_rate": 6.832927412229017e-09,
+      "logits/chosen": -0.6318598985671997,
+      "logits/rejected": 0.05266062542796135,
+      "logps/chosen": -1.1773350238800049,
+      "logps/rejected": -2.1383471488952637,
+      "loss": 1.3499,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.1773350238800049,
+      "rewards/margins": 0.9610121846199036,
+      "rewards/rejected": -2.1383471488952637,
+      "step": 445
+    },
+    {
+      "epoch": 0.942161737764983,
+      "grad_norm": 28.112053810154443,
+      "learning_rate": 4.8708793644441086e-09,
+      "logits/chosen": -0.49654707312583923,
+      "logits/rejected": -0.05771613121032715,
+      "logps/chosen": -1.1556320190429688,
+      "logps/rejected": -2.0353646278381348,
+      "loss": 1.3102,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.1556320190429688,
+      "rewards/margins": 0.879732608795166,
+      "rewards/rejected": -2.0353646278381348,
+      "step": 450
+    },
+    {
+      "epoch": 0.9526302015179272,
+      "grad_norm": 27.512930833592254,
+      "learning_rate": 3.2374343405217884e-09,
+      "logits/chosen": -0.47800731658935547,
+      "logits/rejected": 0.14576876163482666,
+      "logps/chosen": -1.3641871213912964,
+      "logps/rejected": -1.9644696712493896,
+      "loss": 1.3398,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.3641871213912964,
+      "rewards/margins": 0.6002823710441589,
+      "rewards/rejected": -1.9644696712493896,
+      "step": 455
+    },
+    {
+      "epoch": 0.9630986652708715,
+      "grad_norm": 21.866991790036977,
+      "learning_rate": 1.9347820230782295e-09,
+      "logits/chosen": -0.4405391216278076,
+      "logits/rejected": 0.19559261202812195,
+      "logps/chosen": -1.115911602973938,
+      "logps/rejected": -2.104705333709717,
+      "loss": 1.3035,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.115911602973938,
+      "rewards/margins": 0.9887935519218445,
+      "rewards/rejected": -2.104705333709717,
+      "step": 460
+    },
+    {
+      "epoch": 0.9735671290238157,
+      "grad_norm": 25.927634444403026,
+      "learning_rate": 9.64668657069706e-10,
+      "logits/chosen": -0.6560274362564087,
+      "logits/rejected": 0.26568812131881714,
+      "logps/chosen": -1.2110966444015503,
+      "logps/rejected": -2.1687331199645996,
+      "loss": 1.3275,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.2110966444015503,
+      "rewards/margins": 0.9576365351676941,
+      "rewards/rejected": -2.1687331199645996,
+      "step": 465
+    },
+    {
+      "epoch": 0.98403559277676,
+      "grad_norm": 23.70864962079994,
+      "learning_rate": 3.2839470889836627e-10,
+      "logits/chosen": -0.6618033647537231,
+      "logits/rejected": -0.13337016105651855,
+      "logps/chosen": -1.159348726272583,
+      "logps/rejected": -1.9146608114242554,
+      "loss": 1.3688,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.159348726272583,
+      "rewards/margins": 0.7553119659423828,
+      "rewards/rejected": -1.9146608114242554,
+      "step": 470
+    },
+    {
+      "epoch": 0.9945040565297043,
+      "grad_norm": 21.250598362207704,
+      "learning_rate": 2.6813123097352287e-11,
+      "logits/chosen": -0.7468453645706177,
+      "logits/rejected": 0.1613294780254364,
+      "logps/chosen": -1.1621391773223877,
+      "logps/rejected": -2.1066856384277344,
+      "loss": 1.3287,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.1621391773223877,
+      "rewards/margins": 0.9445463418960571,
+      "rewards/rejected": -2.1066856384277344,
+      "step": 475
+    },
+    {
+      "epoch": 0.998691442030882,
+      "step": 477,
+      "total_flos": 0.0,
+      "train_loss": 1.3747819329707627,
+      "train_runtime": 7204.7044,
+      "train_samples_per_second": 8.485,
+      "train_steps_per_second": 0.066
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 477,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000000,
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6528cefdc319ce43df83e11fce3533e94cae10f00fca20f9014cceca5931f59d
+size 7160