TX

README.md

@@ -0,0 +1,75 @@
+---
+license: apache-2.0
+base_model: mistralai/Mistral-7B-Instruct-v0.2
+tags:
+- alignment_handbook-handbook
+- generated_from_trainer
+datasets:
+- princeton-nlp/mistral-instruct-ultrafeedback
+model-index:
+- name: Mistral-7B-Instruct-v0.2-MI-6e-7
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/tengxiao01/huggingface/runs/4cfe527t)
+# Mistral-7B-Instruct-v0.2-MI-6e-7
+
+This model is a fine-tuned version of [mistralai/Mistral-7B-Instruct-v0.2](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2) on the princeton-nlp/mistral-instruct-ultrafeedback dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.5201
+- Rewards/chosen: -0.5693
+- Rewards/rejected: -0.6258
+- Rewards/accuracies: 0.5691
+- Rewards/margins: 0.0565
+- Logps/rejected: -0.6258
+- Logps/chosen: -0.5693
+- Logits/rejected: -3.4036
+- Logits/chosen: -3.4095
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 6e-07
+- train_batch_size: 2
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 16
+- total_train_batch_size: 128
+- total_eval_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 1.4828        | 0.8573 | 400  | 1.5201          | -0.5693        | -0.6258          | 0.5691             | 0.0565          | -0.6258        | -0.5693      | -3.4036         | -3.4095       |
+
+
+### Framework versions
+
+- Transformers 4.42.0
+- Pytorch 2.1.2+cu121
+- Datasets 2.14.6
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,22 @@
+{
+    "epoch": 0.9987943737441393,
+    "eval_logits/chosen": -3.4056549072265625,
+    "eval_logits/rejected": -3.3998796939849854,
+    "eval_logps/chosen": -0.5611971616744995,
+    "eval_logps/rejected": -0.6183008551597595,
+    "eval_loss": 1.5195728540420532,
+    "eval_rewards/accuracies": 0.5691489577293396,
+    "eval_rewards/chosen": -0.5611971616744995,
+    "eval_rewards/margins": 0.057103704661130905,
+    "eval_rewards/rejected": -0.6183008551597595,
+    "eval_runtime": 434.9992,
+    "eval_samples": 2994,
+    "eval_samples_per_second": 6.883,
+    "eval_steps_per_second": 0.432,
+    "total_flos": 0.0,
+    "train_loss": 1.5151263257976253,
+    "train_runtime": 19305.9847,
+    "train_samples": 59720,
+    "train_samples_per_second": 3.093,
+    "train_steps_per_second": 0.024
+}

config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "mistralai/Mistral-7B-Instruct-v0.2",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.42.0",
+  "use_cache": true,
+  "vocab_size": 32000
+}

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 0.9987943737441393,
+    "eval_logits/chosen": -3.4056549072265625,
+    "eval_logits/rejected": -3.3998796939849854,
+    "eval_logps/chosen": -0.5611971616744995,
+    "eval_logps/rejected": -0.6183008551597595,
+    "eval_loss": 1.5195728540420532,
+    "eval_rewards/accuracies": 0.5691489577293396,
+    "eval_rewards/chosen": -0.5611971616744995,
+    "eval_rewards/margins": 0.057103704661130905,
+    "eval_rewards/rejected": -0.6183008551597595,
+    "eval_runtime": 434.9992,
+    "eval_samples": 2994,
+    "eval_samples_per_second": 6.883,
+    "eval_steps_per_second": 0.432
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.42.0"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3292ea6482a3a8316b79c9c2f150a3d801559153a698ac9623b4d24ce1368478
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f1c0a868e96a5551ffb513f0f36a2ef3b13caa3f372458f3100626161139da6
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eccee0333f62d3af926d6fa71eecc62c59fb9f480805396eddc03a110ec71f78
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content'] %}\n    {%- set loop_messages = messages[1:] %}\n{%- else %}\n    {%- set loop_messages = messages %}\n{%- endif %}\n\n{{- bos_token }}\n{%- for message in loop_messages %}\n    {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}\n        {{- raise_exception('After the optional system message, conversation roles must alternate user/assistant/user/assistant/...') }}\n    {%- endif %}\n    {%- if message['role'] == 'user' %}\n        {%- if loop.first and system_message is defined %}\n            {{- ' [INST] ' + system_message + '\\n\\n' + message['content'] + ' [/INST]' }}\n        {%- else %}\n            {{- ' [INST] ' + message['content'] + ' [/INST]' }}\n        {%- endif %}\n    {%- elif message['role'] == 'assistant' %}\n        {{- ' ' + message['content'] + eos_token}}\n    {%- else %}\n        {{- raise_exception('Only user and assistant roles are supported, with the exception of an initial optional system message!') }}\n    {%- endif %}\n{%- endfor %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": false,
+  "model_max_length": 2048,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9987943737441393,
+    "total_flos": 0.0,
+    "train_loss": 1.5151263257976253,
+    "train_runtime": 19305.9847,
+    "train_samples": 59720,
+    "train_samples_per_second": 3.093,
+    "train_steps_per_second": 0.024
+}

trainer_state.json

@@ -0,0 +1,1453 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9987943737441393,
+  "eval_steps": 400,
+  "global_step": 466,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.010716677829872739,
+      "grad_norm": 38.81959429763923,
+      "learning_rate": 6.382978723404255e-08,
+      "logits/chosen": -2.397952079772949,
+      "logits/rejected": -2.391846179962158,
+      "logps/chosen": -0.5666699409484863,
+      "logps/rejected": -0.5553711652755737,
+      "loss": 1.5469,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.5666699409484863,
+      "rewards/margins": -0.01129874400794506,
+      "rewards/rejected": -0.5553711652755737,
+      "step": 5
+    },
+    {
+      "epoch": 0.021433355659745478,
+      "grad_norm": 17.957819802244767,
+      "learning_rate": 1.276595744680851e-07,
+      "logits/chosen": -2.402738571166992,
+      "logits/rejected": -2.3730971813201904,
+      "logps/chosen": -0.5517541766166687,
+      "logps/rejected": -0.5785264372825623,
+      "loss": 1.5538,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.5517541766166687,
+      "rewards/margins": 0.026772266253829002,
+      "rewards/rejected": -0.5785264372825623,
+      "step": 10
+    },
+    {
+      "epoch": 0.032150033489618215,
+      "grad_norm": 17.06492283094742,
+      "learning_rate": 1.9148936170212767e-07,
+      "logits/chosen": -2.4437928199768066,
+      "logits/rejected": -2.449697732925415,
+      "logps/chosen": -0.5636163353919983,
+      "logps/rejected": -0.5669411420822144,
+      "loss": 1.5619,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.5636163353919983,
+      "rewards/margins": 0.00332476943731308,
+      "rewards/rejected": -0.5669411420822144,
+      "step": 15
+    },
+    {
+      "epoch": 0.042866711319490956,
+      "grad_norm": 17.478232600769196,
+      "learning_rate": 2.553191489361702e-07,
+      "logits/chosen": -2.383941650390625,
+      "logits/rejected": -2.3943183422088623,
+      "logps/chosen": -0.5459321737289429,
+      "logps/rejected": -0.5427771806716919,
+      "loss": 1.5322,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.5459321737289429,
+      "rewards/margins": -0.00315500283613801,
+      "rewards/rejected": -0.5427771806716919,
+      "step": 20
+    },
+    {
+      "epoch": 0.0535833891493637,
+      "grad_norm": 14.134950451452564,
+      "learning_rate": 3.1914893617021275e-07,
+      "logits/chosen": -2.2786340713500977,
+      "logits/rejected": -2.2805464267730713,
+      "logps/chosen": -0.5260549783706665,
+      "logps/rejected": -0.5430394411087036,
+      "loss": 1.5298,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.5260549783706665,
+      "rewards/margins": 0.016984451562166214,
+      "rewards/rejected": -0.5430394411087036,
+      "step": 25
+    },
+    {
+      "epoch": 0.06430006697923643,
+      "grad_norm": 19.57863908597214,
+      "learning_rate": 3.8297872340425535e-07,
+      "logits/chosen": -2.3897128105163574,
+      "logits/rejected": -2.4030909538269043,
+      "logps/chosen": -0.5465933680534363,
+      "logps/rejected": -0.5372768640518188,
+      "loss": 1.5509,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -0.5465933680534363,
+      "rewards/margins": -0.009316539391875267,
+      "rewards/rejected": -0.5372768640518188,
+      "step": 30
+    },
+    {
+      "epoch": 0.07501674480910918,
+      "grad_norm": 24.218016837268095,
+      "learning_rate": 4.4680851063829783e-07,
+      "logits/chosen": -2.453273296356201,
+      "logits/rejected": -2.424668788909912,
+      "logps/chosen": -0.5341351628303528,
+      "logps/rejected": -0.5890725255012512,
+      "loss": 1.5479,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.5341351628303528,
+      "rewards/margins": 0.054937295615673065,
+      "rewards/rejected": -0.5890725255012512,
+      "step": 35
+    },
+    {
+      "epoch": 0.08573342263898191,
+      "grad_norm": 20.81509422651472,
+      "learning_rate": 5.106382978723404e-07,
+      "logits/chosen": -2.3677382469177246,
+      "logits/rejected": -2.3493103981018066,
+      "logps/chosen": -0.565592885017395,
+      "logps/rejected": -0.5375810861587524,
+      "loss": 1.5573,
+      "rewards/accuracies": 0.4312500059604645,
+      "rewards/chosen": -0.565592885017395,
+      "rewards/margins": -0.02801181748509407,
+      "rewards/rejected": -0.5375810861587524,
+      "step": 40
+    },
+    {
+      "epoch": 0.09645010046885466,
+      "grad_norm": 17.50647386551691,
+      "learning_rate": 5.74468085106383e-07,
+      "logits/chosen": -2.3343653678894043,
+      "logits/rejected": -2.32906436920166,
+      "logps/chosen": -0.5402032732963562,
+      "logps/rejected": -0.5591766238212585,
+      "loss": 1.5421,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.5402032732963562,
+      "rewards/margins": 0.018973344936966896,
+      "rewards/rejected": -0.5591766238212585,
+      "step": 45
+    },
+    {
+      "epoch": 0.1071667782987274,
+      "grad_norm": 16.375037037224466,
+      "learning_rate": 5.999241095449976e-07,
+      "logits/chosen": -2.3641974925994873,
+      "logits/rejected": -2.3596482276916504,
+      "logps/chosen": -0.5401940941810608,
+      "logps/rejected": -0.525315523147583,
+      "loss": 1.5284,
+      "rewards/accuracies": 0.45625001192092896,
+      "rewards/chosen": -0.5401940941810608,
+      "rewards/margins": -0.01487857848405838,
+      "rewards/rejected": -0.525315523147583,
+      "step": 50
+    },
+    {
+      "epoch": 0.11788345612860013,
+      "grad_norm": 26.218018133925373,
+      "learning_rate": 5.994604735812144e-07,
+      "logits/chosen": -2.4210665225982666,
+      "logits/rejected": -2.424318790435791,
+      "logps/chosen": -0.565641462802887,
+      "logps/rejected": -0.5864871740341187,
+      "loss": 1.5392,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.565641462802887,
+      "rewards/margins": 0.020845741033554077,
+      "rewards/rejected": -0.5864871740341187,
+      "step": 55
+    },
+    {
+      "epoch": 0.12860013395847286,
+      "grad_norm": 42.6456644243847,
+      "learning_rate": 5.985760137627685e-07,
+      "logits/chosen": -2.325913906097412,
+      "logits/rejected": -2.3350510597229004,
+      "logps/chosen": -0.4915548861026764,
+      "logps/rejected": -0.5130532383918762,
+      "loss": 1.5405,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.4915548861026764,
+      "rewards/margins": 0.02149834856390953,
+      "rewards/rejected": -0.5130532383918762,
+      "step": 60
+    },
+    {
+      "epoch": 0.13931681178834562,
+      "grad_norm": 17.49728999516173,
+      "learning_rate": 5.972719729975655e-07,
+      "logits/chosen": -2.3687386512756348,
+      "logits/rejected": -2.3732752799987793,
+      "logps/chosen": -0.5264291167259216,
+      "logps/rejected": -0.5606903433799744,
+      "loss": 1.5441,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.5264291167259216,
+      "rewards/margins": 0.03426120802760124,
+      "rewards/rejected": -0.5606903433799744,
+      "step": 65
+    },
+    {
+      "epoch": 0.15003348961821836,
+      "grad_norm": 19.04975638080615,
+      "learning_rate": 5.955501838194784e-07,
+      "logits/chosen": -2.2692012786865234,
+      "logits/rejected": -2.2734649181365967,
+      "logps/chosen": -0.5329629778862,
+      "logps/rejected": -0.585782527923584,
+      "loss": 1.5335,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.5329629778862,
+      "rewards/margins": 0.05281956121325493,
+      "rewards/rejected": -0.585782527923584,
+      "step": 70
+    },
+    {
+      "epoch": 0.1607501674480911,
+      "grad_norm": 16.46150164067359,
+      "learning_rate": 5.934130658131361e-07,
+      "logits/chosen": -2.3084473609924316,
+      "logits/rejected": -2.303145408630371,
+      "logps/chosen": -0.4908691346645355,
+      "logps/rejected": -0.5239783525466919,
+      "loss": 1.5327,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.4908691346645355,
+      "rewards/margins": 0.03310922160744667,
+      "rewards/rejected": -0.5239783525466919,
+      "step": 75
+    },
+    {
+      "epoch": 0.17146684527796383,
+      "grad_norm": 21.613971984342516,
+      "learning_rate": 5.908636222137454e-07,
+      "logits/chosen": -2.291396141052246,
+      "logits/rejected": -2.3133578300476074,
+      "logps/chosen": -0.48883646726608276,
+      "logps/rejected": -0.5628662109375,
+      "loss": 1.5301,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.48883646726608276,
+      "rewards/margins": 0.07402969151735306,
+      "rewards/rejected": -0.5628662109375,
+      "step": 80
+    },
+    {
+      "epoch": 0.18218352310783656,
+      "grad_norm": 22.2008339670987,
+      "learning_rate": 5.879054356867243e-07,
+      "logits/chosen": -2.328059673309326,
+      "logits/rejected": -2.3216350078582764,
+      "logps/chosen": -0.5081610679626465,
+      "logps/rejected": -0.5643308162689209,
+      "loss": 1.5335,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.5081610679626465,
+      "rewards/margins": 0.05616975575685501,
+      "rewards/rejected": -0.5643308162689209,
+      "step": 85
+    },
+    {
+      "epoch": 0.19290020093770932,
+      "grad_norm": 19.64419890416597,
+      "learning_rate": 5.84542663293077e-07,
+      "logits/chosen": -2.272433280944824,
+      "logits/rejected": -2.2766337394714355,
+      "logps/chosen": -0.5117042660713196,
+      "logps/rejected": -0.558184802532196,
+      "loss": 1.5348,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.5117042660713196,
+      "rewards/margins": 0.046480584889650345,
+      "rewards/rejected": -0.558184802532196,
+      "step": 90
+    },
+    {
+      "epoch": 0.20361687876758205,
+      "grad_norm": 17.22227696113398,
+      "learning_rate": 5.807800306475876e-07,
+      "logits/chosen": -2.3275113105773926,
+      "logits/rejected": -2.3395214080810547,
+      "logps/chosen": -0.6069667935371399,
+      "logps/rejected": -0.6348728537559509,
+      "loss": 1.5217,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.6069667935371399,
+      "rewards/margins": 0.027906125411391258,
+      "rewards/rejected": -0.6348728537559509,
+      "step": 95
+    },
+    {
+      "epoch": 0.2143335565974548,
+      "grad_norm": 24.396288528469587,
+      "learning_rate": 5.766228252780373e-07,
+      "logits/chosen": -2.368147373199463,
+      "logits/rejected": -2.377194881439209,
+      "logps/chosen": -0.5941327214241028,
+      "logps/rejected": -0.6152836680412292,
+      "loss": 1.5435,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.5941327214241028,
+      "rewards/margins": 0.0211509857326746,
+      "rewards/rejected": -0.6152836680412292,
+      "step": 100
+    },
+    {
+      "epoch": 0.22505023442732752,
+      "grad_norm": 16.23048110894587,
+      "learning_rate": 5.720768891947834e-07,
+      "logits/chosen": -2.3831636905670166,
+      "logits/rejected": -2.383808135986328,
+      "logps/chosen": -0.5236924886703491,
+      "logps/rejected": -0.5740348100662231,
+      "loss": 1.517,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.5236924886703491,
+      "rewards/margins": 0.05034228041768074,
+      "rewards/rejected": -0.5740348100662231,
+      "step": 105
+    },
+    {
+      "epoch": 0.23576691225720026,
+      "grad_norm": 22.95053096013821,
+      "learning_rate": 5.671486106811365e-07,
+      "logits/chosen": -2.4293274879455566,
+      "logits/rejected": -2.4386584758758545,
+      "logps/chosen": -0.5232604146003723,
+      "logps/rejected": -0.5748019218444824,
+      "loss": 1.54,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.5232604146003723,
+      "rewards/margins": 0.051541589200496674,
+      "rewards/rejected": -0.5748019218444824,
+      "step": 110
+    },
+    {
+      "epoch": 0.24648359008707302,
+      "grad_norm": 30.43281833851228,
+      "learning_rate": 5.618449153160763e-07,
+      "logits/chosen": -2.521904945373535,
+      "logits/rejected": -2.5165414810180664,
+      "logps/chosen": -0.5098231434822083,
+      "logps/rejected": -0.5272140502929688,
+      "loss": 1.5652,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.5098231434822083,
+      "rewards/margins": 0.01739095151424408,
+      "rewards/rejected": -0.5272140502929688,
+      "step": 115
+    },
+    {
+      "epoch": 0.2572002679169457,
+      "grad_norm": 18.218447276406668,
+      "learning_rate": 5.56173256241918e-07,
+      "logits/chosen": -2.6067259311676025,
+      "logits/rejected": -2.594320774078369,
+      "logps/chosen": -0.5535318851470947,
+      "logps/rejected": -0.5515246987342834,
+      "loss": 1.553,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.5535318851470947,
+      "rewards/margins": -0.002007170347496867,
+      "rewards/rejected": -0.5515246987342834,
+      "step": 120
+    },
+    {
+      "epoch": 0.2679169457468185,
+      "grad_norm": 19.047294066362046,
+      "learning_rate": 5.501416036906106e-07,
+      "logits/chosen": -2.599743366241455,
+      "logits/rejected": -2.602095603942871,
+      "logps/chosen": -0.5660097599029541,
+      "logps/rejected": -0.5823434591293335,
+      "loss": 1.5462,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.5660097599029541,
+      "rewards/margins": 0.01633365824818611,
+      "rewards/rejected": -0.5823434591293335,
+      "step": 125
+    },
+    {
+      "epoch": 0.27863362357669125,
+      "grad_norm": 25.91707976400477,
+      "learning_rate": 5.437584337833803e-07,
+      "logits/chosen": -2.6727051734924316,
+      "logits/rejected": -2.6547203063964844,
+      "logps/chosen": -0.5484704375267029,
+      "logps/rejected": -0.5791813731193542,
+      "loss": 1.5217,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.5484704375267029,
+      "rewards/margins": 0.030710989609360695,
+      "rewards/rejected": -0.5791813731193542,
+      "step": 130
+    },
+    {
+      "epoch": 0.289350301406564,
+      "grad_norm": 24.930436449789845,
+      "learning_rate": 5.370327166194635e-07,
+      "logits/chosen": -2.6759390830993652,
+      "logits/rejected": -2.688563346862793,
+      "logps/chosen": -0.5326634049415588,
+      "logps/rejected": -0.5721167325973511,
+      "loss": 1.5358,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.5326634049415588,
+      "rewards/margins": 0.03945332020521164,
+      "rewards/rejected": -0.5721167325973511,
+      "step": 135
+    },
+    {
+      "epoch": 0.3000669792364367,
+      "grad_norm": 25.343122236521907,
+      "learning_rate": 5.299739036706635e-07,
+      "logits/chosen": -2.6917319297790527,
+      "logits/rejected": -2.6787917613983154,
+      "logps/chosen": -0.5328460931777954,
+      "logps/rejected": -0.5668941736221313,
+      "loss": 1.5162,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.5328460931777954,
+      "rewards/margins": 0.034048013389110565,
+      "rewards/rejected": -0.5668941736221313,
+      "step": 140
+    },
+    {
+      "epoch": 0.31078365706630945,
+      "grad_norm": 20.469732566491704,
+      "learning_rate": 5.225919144994487e-07,
+      "logits/chosen": -2.7843101024627686,
+      "logits/rejected": -2.7595150470733643,
+      "logps/chosen": -0.5282408595085144,
+      "logps/rejected": -0.5839791893959045,
+      "loss": 1.5106,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.5282408595085144,
+      "rewards/margins": 0.05573834106326103,
+      "rewards/rejected": -0.5839791893959045,
+      "step": 145
+    },
+    {
+      "epoch": 0.3215003348961822,
+      "grad_norm": 22.910568604755916,
+      "learning_rate": 5.148971228192543e-07,
+      "logits/chosen": -2.769007921218872,
+      "logits/rejected": -2.7604851722717285,
+      "logps/chosen": -0.5120500326156616,
+      "logps/rejected": -0.5518966913223267,
+      "loss": 1.5387,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.5120500326156616,
+      "rewards/margins": 0.03984668105840683,
+      "rewards/rejected": -0.5518966913223267,
+      "step": 150
+    },
+    {
+      "epoch": 0.3322170127260549,
+      "grad_norm": 21.1001003359098,
+      "learning_rate": 5.069003419165781e-07,
+      "logits/chosen": -2.8498682975769043,
+      "logits/rejected": -2.8591020107269287,
+      "logps/chosen": -0.5589969754219055,
+      "logps/rejected": -0.5984258651733398,
+      "loss": 1.5176,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.5589969754219055,
+      "rewards/margins": 0.039428871124982834,
+      "rewards/rejected": -0.5984258651733398,
+      "step": 155
+    },
+    {
+      "epoch": 0.34293369055592765,
+      "grad_norm": 26.259271435648458,
+      "learning_rate": 4.986128094553569e-07,
+      "logits/chosen": -2.8450496196746826,
+      "logits/rejected": -2.8269691467285156,
+      "logps/chosen": -0.5768808126449585,
+      "logps/rejected": -0.6496576070785522,
+      "loss": 1.5296,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.5768808126449585,
+      "rewards/margins": 0.07277677953243256,
+      "rewards/rejected": -0.6496576070785522,
+      "step": 160
+    },
+    {
+      "epoch": 0.3536503683858004,
+      "grad_norm": 22.95727659422549,
+      "learning_rate": 4.900461716849745e-07,
+      "logits/chosen": -2.9189038276672363,
+      "logits/rejected": -2.8916220664978027,
+      "logps/chosen": -0.5302075147628784,
+      "logps/rejected": -0.5729304552078247,
+      "loss": 1.5205,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.5302075147628784,
+      "rewards/margins": 0.042722832411527634,
+      "rewards/rejected": -0.5729304552078247,
+      "step": 165
+    },
+    {
+      "epoch": 0.3643670462156731,
+      "grad_norm": 17.76635841368691,
+      "learning_rate": 4.812124670740974e-07,
+      "logits/chosen": -2.9066505432128906,
+      "logits/rejected": -2.910203456878662,
+      "logps/chosen": -0.5274362564086914,
+      "logps/rejected": -0.6013033986091614,
+      "loss": 1.5096,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.5274362564086914,
+      "rewards/margins": 0.0738670751452446,
+      "rewards/rejected": -0.6013033986091614,
+      "step": 170
+    },
+    {
+      "epoch": 0.3750837240455459,
+      "grad_norm": 22.708920984772448,
+      "learning_rate": 4.7212410939333393e-07,
+      "logits/chosen": -2.9743309020996094,
+      "logits/rejected": -2.9548959732055664,
+      "logps/chosen": -0.5641797780990601,
+      "logps/rejected": -0.594096302986145,
+      "loss": 1.5387,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.5641797780990601,
+      "rewards/margins": 0.029916446655988693,
+      "rewards/rejected": -0.594096302986145,
+      "step": 175
+    },
+    {
+      "epoch": 0.38580040187541864,
+      "grad_norm": 22.445712735196388,
+      "learning_rate": 4.6279387027049207e-07,
+      "logits/chosen": -3.0900559425354004,
+      "logits/rejected": -3.091893434524536,
+      "logps/chosen": -0.5932881236076355,
+      "logps/rejected": -0.6334934234619141,
+      "loss": 1.5474,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.5932881236076355,
+      "rewards/margins": 0.040205273777246475,
+      "rewards/rejected": -0.6334934234619141,
+      "step": 180
+    },
+    {
+      "epoch": 0.3965170797052914,
+      "grad_norm": 30.045592149378802,
+      "learning_rate": 4.5323486124294974e-07,
+      "logits/chosen": -3.1286568641662598,
+      "logits/rejected": -3.1514105796813965,
+      "logps/chosen": -0.5893043279647827,
+      "logps/rejected": -0.6178286075592041,
+      "loss": 1.52,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.5893043279647827,
+      "rewards/margins": 0.02852421998977661,
+      "rewards/rejected": -0.6178286075592041,
+      "step": 185
+    },
+    {
+      "epoch": 0.4072337575351641,
+      "grad_norm": 19.66079670307951,
+      "learning_rate": 4.434605153323596e-07,
+      "logits/chosen": -3.0138182640075684,
+      "logits/rejected": -3.027487277984619,
+      "logps/chosen": -0.5490652322769165,
+      "logps/rejected": -0.7248018383979797,
+      "loss": 1.5211,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.5490652322769165,
+      "rewards/margins": 0.17573660612106323,
+      "rewards/rejected": -0.7248018383979797,
+      "step": 190
+    },
+    {
+      "epoch": 0.41795043536503684,
+      "grad_norm": 23.76319559347926,
+      "learning_rate": 4.334845681675802e-07,
+      "logits/chosen": -3.263240098953247,
+      "logits/rejected": -3.234492063522339,
+      "logps/chosen": -0.523744523525238,
+      "logps/rejected": -0.5552490949630737,
+      "loss": 1.5291,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.523744523525238,
+      "rewards/margins": 0.03150450438261032,
+      "rewards/rejected": -0.5552490949630737,
+      "step": 195
+    },
+    {
+      "epoch": 0.4286671131949096,
+      "grad_norm": 22.67820361436796,
+      "learning_rate": 4.233210386823613e-07,
+      "logits/chosen": -3.1373372077941895,
+      "logits/rejected": -3.1572506427764893,
+      "logps/chosen": -0.49712926149368286,
+      "logps/rejected": -0.5306284427642822,
+      "loss": 1.5161,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.49712926149368286,
+      "rewards/margins": 0.033499158918857574,
+      "rewards/rejected": -0.5306284427642822,
+      "step": 200
+    },
+    {
+      "epoch": 0.4393837910247823,
+      "grad_norm": 26.07083589291384,
+      "learning_rate": 4.129842094149083e-07,
+      "logits/chosen": -3.277681827545166,
+      "logits/rejected": -3.2762560844421387,
+      "logps/chosen": -0.4885168969631195,
+      "logps/rejected": -0.5218795537948608,
+      "loss": 1.5331,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.4885168969631195,
+      "rewards/margins": 0.03336270898580551,
+      "rewards/rejected": -0.5218795537948608,
+      "step": 205
+    },
+    {
+      "epoch": 0.45010046885465504,
+      "grad_norm": 21.312251857603005,
+      "learning_rate": 4.024886064370107e-07,
+      "logits/chosen": -3.2972412109375,
+      "logits/rejected": -3.2872118949890137,
+      "logps/chosen": -0.509524941444397,
+      "logps/rejected": -0.5547453761100769,
+      "loss": 1.5062,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.509524941444397,
+      "rewards/margins": 0.04522045701742172,
+      "rewards/rejected": -0.5547453761100769,
+      "step": 210
+    },
+    {
+      "epoch": 0.4608171466845278,
+      "grad_norm": 22.199858140310962,
+      "learning_rate": 3.9184897894093836e-07,
+      "logits/chosen": -3.3184287548065186,
+      "logits/rejected": -3.3149967193603516,
+      "logps/chosen": -0.5457042455673218,
+      "logps/rejected": -0.5951209664344788,
+      "loss": 1.4947,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.5457042455673218,
+      "rewards/margins": 0.049416683614254,
+      "rewards/rejected": -0.5951209664344788,
+      "step": 215
+    },
+    {
+      "epoch": 0.4715338245144005,
+      "grad_norm": 26.637536554337164,
+      "learning_rate": 3.8108027851279425e-07,
+      "logits/chosen": -3.3705334663391113,
+      "logits/rejected": -3.348128080368042,
+      "logps/chosen": -0.5251081585884094,
+      "logps/rejected": -0.6058255434036255,
+      "loss": 1.533,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.5251081585884094,
+      "rewards/margins": 0.08071742951869965,
+      "rewards/rejected": -0.6058255434036255,
+      "step": 220
+    },
+    {
+      "epoch": 0.4822505023442733,
+      "grad_norm": 20.985181328903547,
+      "learning_rate": 3.701976381214462e-07,
+      "logits/chosen": -3.441849946975708,
+      "logits/rejected": -3.436166286468506,
+      "logps/chosen": -0.5670623183250427,
+      "logps/rejected": -0.6455426812171936,
+      "loss": 1.5056,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.5670623183250427,
+      "rewards/margins": 0.0784803032875061,
+      "rewards/rejected": -0.6455426812171936,
+      "step": 225
+    },
+    {
+      "epoch": 0.49296718017414604,
+      "grad_norm": 26.37006096745831,
+      "learning_rate": 3.5921635085256784e-07,
+      "logits/chosen": -3.3313636779785156,
+      "logits/rejected": -3.316943407058716,
+      "logps/chosen": -0.5642744898796082,
+      "logps/rejected": -0.6056590676307678,
+      "loss": 1.4952,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.5642744898796082,
+      "rewards/margins": 0.041384514421224594,
+      "rewards/rejected": -0.6056590676307678,
+      "step": 230
+    },
+    {
+      "epoch": 0.5036838580040187,
+      "grad_norm": 21.351561473671772,
+      "learning_rate": 3.4815184841767167e-07,
+      "logits/chosen": -3.3021767139434814,
+      "logits/rejected": -3.2890796661376953,
+      "logps/chosen": -0.5324856042861938,
+      "logps/rejected": -0.6350933313369751,
+      "loss": 1.5016,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.5324856042861938,
+      "rewards/margins": 0.10260789096355438,
+      "rewards/rejected": -0.6350933313369751,
+      "step": 235
+    },
+    {
+      "epoch": 0.5144005358338914,
+      "grad_norm": 25.25074061630777,
+      "learning_rate": 3.3701967946833387e-07,
+      "logits/chosen": -3.2254951000213623,
+      "logits/rejected": -3.232588529586792,
+      "logps/chosen": -0.5533393621444702,
+      "logps/rejected": -0.6351491212844849,
+      "loss": 1.5163,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.5533393621444702,
+      "rewards/margins": 0.08180973678827286,
+      "rewards/rejected": -0.6351491212844849,
+      "step": 240
+    },
+    {
+      "epoch": 0.5251172136637642,
+      "grad_norm": 25.884047164441235,
+      "learning_rate": 3.258354877460875e-07,
+      "logits/chosen": -3.2459404468536377,
+      "logits/rejected": -3.2240214347839355,
+      "logps/chosen": -0.5927519202232361,
+      "logps/rejected": -0.6273369193077087,
+      "loss": 1.5018,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.5927519202232361,
+      "rewards/margins": 0.03458496928215027,
+      "rewards/rejected": -0.6273369193077087,
+      "step": 245
+    },
+    {
+      "epoch": 0.535833891493637,
+      "grad_norm": 24.128535335441807,
+      "learning_rate": 3.1461499009868705e-07,
+      "logits/chosen": -3.229731321334839,
+      "logits/rejected": -3.2193870544433594,
+      "logps/chosen": -0.6217538118362427,
+      "logps/rejected": -0.6148039102554321,
+      "loss": 1.5303,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.6217538118362427,
+      "rewards/margins": -0.006949885282665491,
+      "rewards/rejected": -0.6148039102554321,
+      "step": 250
+    },
+    {
+      "epoch": 0.5465505693235098,
+      "grad_norm": 22.132762654520068,
+      "learning_rate": 3.033739543936404e-07,
+      "logits/chosen": -3.251239776611328,
+      "logits/rejected": -3.251615047454834,
+      "logps/chosen": -0.5319584608078003,
+      "logps/rejected": -0.6038156747817993,
+      "loss": 1.4956,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.5319584608078003,
+      "rewards/margins": 0.0718572586774826,
+      "rewards/rejected": -0.6038156747817993,
+      "step": 255
+    },
+    {
+      "epoch": 0.5572672471533825,
+      "grad_norm": 20.467314286083344,
+      "learning_rate": 2.921281773600424e-07,
+      "logits/chosen": -3.172785520553589,
+      "logits/rejected": -3.191011905670166,
+      "logps/chosen": -0.5374451875686646,
+      "logps/rejected": -0.6392644047737122,
+      "loss": 1.5245,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.5374451875686646,
+      "rewards/margins": 0.10181926190853119,
+      "rewards/rejected": -0.6392644047737122,
+      "step": 260
+    },
+    {
+      "epoch": 0.5679839249832552,
+      "grad_norm": 21.60647847703085,
+      "learning_rate": 2.808934623898511e-07,
+      "logits/chosen": -3.1863551139831543,
+      "logits/rejected": -3.1833932399749756,
+      "logps/chosen": -0.5692937970161438,
+      "logps/rejected": -0.6735215783119202,
+      "loss": 1.4938,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.5692937970161438,
+      "rewards/margins": 0.10422778129577637,
+      "rewards/rejected": -0.6735215783119202,
+      "step": 265
+    },
+    {
+      "epoch": 0.578700602813128,
+      "grad_norm": 25.07152854176611,
+      "learning_rate": 2.696855973298007e-07,
+      "logits/chosen": -3.1901869773864746,
+      "logits/rejected": -3.183385133743286,
+      "logps/chosen": -0.536370575428009,
+      "logps/rejected": -0.5907222628593445,
+      "loss": 1.5085,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.536370575428009,
+      "rewards/margins": 0.05435168743133545,
+      "rewards/rejected": -0.5907222628593445,
+      "step": 270
+    },
+    {
+      "epoch": 0.5894172806430007,
+      "grad_norm": 18.910155061928734,
+      "learning_rate": 2.585203322951589e-07,
+      "logits/chosen": -3.274017810821533,
+      "logits/rejected": -3.2836010456085205,
+      "logps/chosen": -0.4976142942905426,
+      "logps/rejected": -0.5548876523971558,
+      "loss": 1.5118,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.4976142942905426,
+      "rewards/margins": 0.05727345868945122,
+      "rewards/rejected": -0.5548876523971558,
+      "step": 275
+    },
+    {
+      "epoch": 0.6001339584728734,
+      "grad_norm": 21.74570057366724,
+      "learning_rate": 2.47413357536509e-07,
+      "logits/chosen": -3.2025809288024902,
+      "logits/rejected": -3.1752185821533203,
+      "logps/chosen": -0.569342851638794,
+      "logps/rejected": -0.623855471611023,
+      "loss": 1.5032,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.569342851638794,
+      "rewards/margins": 0.054512638598680496,
+      "rewards/rejected": -0.623855471611023,
+      "step": 280
+    },
+    {
+      "epoch": 0.6108506363027462,
+      "grad_norm": 25.233010095872693,
+      "learning_rate": 2.3638028139065624e-07,
+      "logits/chosen": -3.230616331100464,
+      "logits/rejected": -3.2365059852600098,
+      "logps/chosen": -0.5778087973594666,
+      "logps/rejected": -0.5905576944351196,
+      "loss": 1.521,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.5778087973594666,
+      "rewards/margins": 0.012748857028782368,
+      "rewards/rejected": -0.5905576944351196,
+      "step": 285
+    },
+    {
+      "epoch": 0.6215673141326189,
+      "grad_norm": 52.99096570461333,
+      "learning_rate": 2.2543660834664724e-07,
+      "logits/chosen": -3.3016669750213623,
+      "logits/rejected": -3.28556489944458,
+      "logps/chosen": -0.5023082494735718,
+      "logps/rejected": -0.5799704790115356,
+      "loss": 1.496,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.5023082494735718,
+      "rewards/margins": 0.07766219973564148,
+      "rewards/rejected": -0.5799704790115356,
+      "step": 290
+    },
+    {
+      "epoch": 0.6322839919624916,
+      "grad_norm": 22.86414304018135,
+      "learning_rate": 2.1459771725772267e-07,
+      "logits/chosen": -3.2204766273498535,
+      "logits/rejected": -3.243717908859253,
+      "logps/chosen": -0.5589109063148499,
+      "logps/rejected": -0.6346697807312012,
+      "loss": 1.4948,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.5589109063148499,
+      "rewards/margins": 0.07575888931751251,
+      "rewards/rejected": -0.6346697807312012,
+      "step": 295
+    },
+    {
+      "epoch": 0.6430006697923644,
+      "grad_norm": 24.245750925813,
+      "learning_rate": 2.0387883972982259e-07,
+      "logits/chosen": -3.3650691509246826,
+      "logits/rejected": -3.367690324783325,
+      "logps/chosen": -0.5385848879814148,
+      "logps/rejected": -0.6212387681007385,
+      "loss": 1.4876,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.5385848879814148,
+      "rewards/margins": 0.08265385776758194,
+      "rewards/rejected": -0.6212387681007385,
+      "step": 300
+    },
+    {
+      "epoch": 0.6537173476222371,
+      "grad_norm": 23.847848751632885,
+      "learning_rate": 1.9329503871701592e-07,
+      "logits/chosen": -3.309741973876953,
+      "logits/rejected": -3.281573534011841,
+      "logps/chosen": -0.5459524989128113,
+      "logps/rejected": -0.5997665524482727,
+      "loss": 1.497,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.5459524989128113,
+      "rewards/margins": 0.05381406098604202,
+      "rewards/rejected": -0.5997665524482727,
+      "step": 305
+    },
+    {
+      "epoch": 0.6644340254521098,
+      "grad_norm": 27.809403889861738,
+      "learning_rate": 1.8286118735393015e-07,
+      "logits/chosen": -3.3402085304260254,
+      "logits/rejected": -3.3407912254333496,
+      "logps/chosen": -0.5383692979812622,
+      "logps/rejected": -0.5971530079841614,
+      "loss": 1.4859,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.5383692979812622,
+      "rewards/margins": 0.058783747255802155,
+      "rewards/rejected": -0.5971530079841614,
+      "step": 310
+    },
+    {
+      "epoch": 0.6751507032819826,
+      "grad_norm": 20.70533413800174,
+      "learning_rate": 1.7259194805493042e-07,
+      "logits/chosen": -3.2731971740722656,
+      "logits/rejected": -3.2823867797851562,
+      "logps/chosen": -0.5446811318397522,
+      "logps/rejected": -0.606468915939331,
+      "loss": 1.4954,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.5446811318397522,
+      "rewards/margins": 0.06178779527544975,
+      "rewards/rejected": -0.606468915939331,
+      "step": 315
+    },
+    {
+      "epoch": 0.6858673811118553,
+      "grad_norm": 23.184720457564282,
+      "learning_rate": 1.6250175190941725e-07,
+      "logits/chosen": -3.3182265758514404,
+      "logits/rejected": -3.2908051013946533,
+      "logps/chosen": -0.5561486482620239,
+      "logps/rejected": -0.5695281028747559,
+      "loss": 1.5159,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.5561486482620239,
+      "rewards/margins": 0.0133795365691185,
+      "rewards/rejected": -0.5695281028747559,
+      "step": 320
+    },
+    {
+      "epoch": 0.696584058941728,
+      "grad_norm": 27.421134529600376,
+      "learning_rate": 1.5260477840220057e-07,
+      "logits/chosen": -3.309216260910034,
+      "logits/rejected": -3.318588972091675,
+      "logps/chosen": -0.5043013095855713,
+      "logps/rejected": -0.5974953770637512,
+      "loss": 1.4727,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.5043013095855713,
+      "rewards/margins": 0.09319403767585754,
+      "rewards/rejected": -0.5974953770637512,
+      "step": 325
+    },
+    {
+      "epoch": 0.7073007367716008,
+      "grad_norm": 24.907218358327214,
+      "learning_rate": 1.4291493548744542e-07,
+      "logits/chosen": -3.2981224060058594,
+      "logits/rejected": -3.269383668899536,
+      "logps/chosen": -0.5354940295219421,
+      "logps/rejected": -0.5932218432426453,
+      "loss": 1.492,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.5354940295219421,
+      "rewards/margins": 0.05772777646780014,
+      "rewards/rejected": -0.5932218432426453,
+      "step": 330
+    },
+    {
+      "epoch": 0.7180174146014735,
+      "grad_norm": 40.51550261895159,
+      "learning_rate": 1.334458400441933e-07,
+      "logits/chosen": -3.3821797370910645,
+      "logits/rejected": -3.373931884765625,
+      "logps/chosen": -0.5368712544441223,
+      "logps/rejected": -0.6348738670349121,
+      "loss": 1.5027,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.5368712544441223,
+      "rewards/margins": 0.09800264984369278,
+      "rewards/rejected": -0.6348738670349121,
+      "step": 335
+    },
+    {
+      "epoch": 0.7287340924313462,
+      "grad_norm": 25.305322827743687,
+      "learning_rate": 1.2421079874092336e-07,
+      "logits/chosen": -3.311006546020508,
+      "logits/rejected": -3.2969226837158203,
+      "logps/chosen": -0.5727181434631348,
+      "logps/rejected": -0.6479278802871704,
+      "loss": 1.516,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.5727181434631348,
+      "rewards/margins": 0.07520972192287445,
+      "rewards/rejected": -0.6479278802871704,
+      "step": 340
+    },
+    {
+      "epoch": 0.739450770261219,
+      "grad_norm": 22.209687488331966,
+      "learning_rate": 1.1522278933604484e-07,
+      "logits/chosen": -3.3381361961364746,
+      "logits/rejected": -3.3484432697296143,
+      "logps/chosen": -0.5813694596290588,
+      "logps/rejected": -0.6797500848770142,
+      "loss": 1.4932,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.5813694596290588,
+      "rewards/margins": 0.09838052839040756,
+      "rewards/rejected": -0.6797500848770142,
+      "step": 345
+    },
+    {
+      "epoch": 0.7501674480910918,
+      "grad_norm": 22.960003013518822,
+      "learning_rate": 1.0649444244059717e-07,
+      "logits/chosen": -3.32041597366333,
+      "logits/rejected": -3.3459019660949707,
+      "logps/chosen": -0.5482354164123535,
+      "logps/rejected": -0.6205809712409973,
+      "loss": 1.4932,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.5482354164123535,
+      "rewards/margins": 0.07234560698270798,
+      "rewards/rejected": -0.6205809712409973,
+      "step": 350
+    },
+    {
+      "epoch": 0.7608841259209645,
+      "grad_norm": 25.349782143414405,
+      "learning_rate": 9.803802376878795e-08,
+      "logits/chosen": -3.3137733936309814,
+      "logits/rejected": -3.3020172119140625,
+      "logps/chosen": -0.5893365740776062,
+      "logps/rejected": -0.6166855096817017,
+      "loss": 1.4957,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.5893365740776062,
+      "rewards/margins": 0.02734885737299919,
+      "rewards/rejected": -0.6166855096817017,
+      "step": 355
+    },
+    {
+      "epoch": 0.7716008037508373,
+      "grad_norm": 22.556111949917664,
+      "learning_rate": 8.98654169013098e-08,
+      "logits/chosen": -3.3032424449920654,
+      "logits/rejected": -3.288992404937744,
+      "logps/chosen": -0.521614670753479,
+      "logps/rejected": -0.5875999331474304,
+      "loss": 1.5004,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.521614670753479,
+      "rewards/margins": 0.06598522514104843,
+      "rewards/rejected": -0.5875999331474304,
+      "step": 360
+    },
+    {
+      "epoch": 0.78231748158071,
+      "grad_norm": 23.52734286974739,
+      "learning_rate": 8.198810658566058e-08,
+      "logits/chosen": -3.3537094593048096,
+      "logits/rejected": -3.348142147064209,
+      "logps/chosen": -0.5639868974685669,
+      "logps/rejected": -0.6276763677597046,
+      "loss": 1.5179,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.5639868974685669,
+      "rewards/margins": 0.06368952244520187,
+      "rewards/rejected": -0.6276763677597046,
+      "step": 365
+    },
+    {
+      "epoch": 0.7930341594105828,
+      "grad_norm": 35.138524177158146,
+      "learning_rate": 7.441716259693182e-08,
+      "logits/chosen": -3.3430557250976562,
+      "logits/rejected": -3.365880250930786,
+      "logps/chosen": -0.5710283517837524,
+      "logps/rejected": -0.6515873670578003,
+      "loss": 1.5075,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.5710283517837524,
+      "rewards/margins": 0.08055897057056427,
+      "rewards/rejected": -0.6515873670578003,
+      "step": 370
+    },
+    {
+      "epoch": 0.8037508372404555,
+      "grad_norm": 23.46216038728244,
+      "learning_rate": 6.716322418174835e-08,
+      "logits/chosen": -3.309415102005005,
+      "logits/rejected": -3.2904000282287598,
+      "logps/chosen": -0.6181533336639404,
+      "logps/rejected": -0.7165501713752747,
+      "loss": 1.4916,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.6181533336639404,
+      "rewards/margins": 0.09839687496423721,
+      "rewards/rejected": -0.7165501713752747,
+      "step": 375
+    },
+    {
+      "epoch": 0.8144675150703282,
+      "grad_norm": 26.38220306063447,
+      "learning_rate": 6.023648510721696e-08,
+      "logits/chosen": -3.391897678375244,
+      "logits/rejected": -3.358309268951416,
+      "logps/chosen": -0.5505380630493164,
+      "logps/rejected": -0.6369754076004028,
+      "loss": 1.5034,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.5505380630493164,
+      "rewards/margins": 0.08643738180398941,
+      "rewards/rejected": -0.6369754076004028,
+      "step": 380
+    },
+    {
+      "epoch": 0.825184192900201,
+      "grad_norm": 24.084433663112158,
+      "learning_rate": 5.364667933589596e-08,
+      "logits/chosen": -3.2913315296173096,
+      "logits/rejected": -3.3047938346862793,
+      "logps/chosen": -0.5729898810386658,
+      "logps/rejected": -0.6550789475440979,
+      "loss": 1.4915,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.5729898810386658,
+      "rewards/margins": 0.08208902180194855,
+      "rewards/rejected": -0.6550789475440979,
+      "step": 385
+    },
+    {
+      "epoch": 0.8359008707300737,
+      "grad_norm": 25.406153364539314,
+      "learning_rate": 4.74030673469165e-08,
+      "logits/chosen": -3.3330624103546143,
+      "logits/rejected": -3.293489933013916,
+      "logps/chosen": -0.613795280456543,
+      "logps/rejected": -0.6435045003890991,
+      "loss": 1.5266,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.613795280456543,
+      "rewards/margins": 0.029709184542298317,
+      "rewards/rejected": -0.6435045003890991,
+      "step": 390
+    },
+    {
+      "epoch": 0.8466175485599464,
+      "grad_norm": 24.427651638617984,
+      "learning_rate": 4.1514423122476606e-08,
+      "logits/chosen": -3.3390536308288574,
+      "logits/rejected": -3.3218231201171875,
+      "logps/chosen": -0.5534718632698059,
+      "logps/rejected": -0.600857138633728,
+      "loss": 1.507,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.5534718632698059,
+      "rewards/margins": 0.04738527163863182,
+      "rewards/rejected": -0.600857138633728,
+      "step": 395
+    },
+    {
+      "epoch": 0.8573342263898192,
+      "grad_norm": 26.673472503173954,
+      "learning_rate": 3.598902181799717e-08,
+      "logits/chosen": -3.298213481903076,
+      "logits/rejected": -3.2662785053253174,
+      "logps/chosen": -0.5154682397842407,
+      "logps/rejected": -0.6383693814277649,
+      "loss": 1.4828,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.5154682397842407,
+      "rewards/margins": 0.12290115654468536,
+      "rewards/rejected": -0.6383693814277649,
+      "step": 400
+    },
+    {
+      "epoch": 0.8573342263898192,
+      "eval_logits/chosen": -3.409546136856079,
+      "eval_logits/rejected": -3.403578758239746,
+      "eval_logps/chosen": -0.5692862868309021,
+      "eval_logps/rejected": -0.6257904171943665,
+      "eval_loss": 1.5201373100280762,
+      "eval_rewards/accuracies": 0.5691489577293396,
+      "eval_rewards/chosen": -0.5692862868309021,
+      "eval_rewards/margins": 0.05650414153933525,
+      "eval_rewards/rejected": -0.6257904171943665,
+      "eval_runtime": 432.4468,
+      "eval_samples_per_second": 6.923,
+      "eval_steps_per_second": 0.435,
+      "step": 400
+    },
+    {
+      "epoch": 0.8680509042196919,
+      "grad_norm": 24.460303851950272,
+      "learning_rate": 3.0834628133265293e-08,
+      "logits/chosen": -3.308946132659912,
+      "logits/rejected": -3.293513536453247,
+      "logps/chosen": -0.5684244632720947,
+      "logps/rejected": -0.6289744973182678,
+      "loss": 1.5056,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.5684244632720947,
+      "rewards/margins": 0.06054999679327011,
+      "rewards/rejected": -0.6289744973182678,
+      "step": 405
+    },
+    {
+      "epoch": 0.8787675820495646,
+      "grad_norm": 22.9693279472293,
+      "learning_rate": 2.6058485400908248e-08,
+      "logits/chosen": -3.358743190765381,
+      "logits/rejected": -3.3271114826202393,
+      "logps/chosen": -0.5511162877082825,
+      "logps/rejected": -0.5816215872764587,
+      "loss": 1.4873,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.5511162877082825,
+      "rewards/margins": 0.030505258589982986,
+      "rewards/rejected": -0.5816215872764587,
+      "step": 410
+    },
+    {
+      "epoch": 0.8894842598794374,
+      "grad_norm": 25.421672297355457,
+      "learning_rate": 2.1667305407530255e-08,
+      "logits/chosen": -3.2762393951416016,
+      "logits/rejected": -3.2448742389678955,
+      "logps/chosen": -0.548682689666748,
+      "logps/rejected": -0.6257365942001343,
+      "loss": 1.4819,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.548682689666748,
+      "rewards/margins": 0.07705400884151459,
+      "rewards/rejected": -0.6257365942001343,
+      "step": 415
+    },
+    {
+      "epoch": 0.9002009377093101,
+      "grad_norm": 27.04902626819421,
+      "learning_rate": 1.7667258961816723e-08,
+      "logits/chosen": -3.2720954418182373,
+      "logits/rejected": -3.2802345752716064,
+      "logps/chosen": -0.5331937074661255,
+      "logps/rejected": -0.573731541633606,
+      "loss": 1.5172,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.5331937074661255,
+      "rewards/margins": 0.04053787142038345,
+      "rewards/rejected": -0.573731541633606,
+      "step": 420
+    },
+    {
+      "epoch": 0.9109176155391828,
+      "grad_norm": 27.138507466839215,
+      "learning_rate": 1.4063967222860872e-08,
+      "logits/chosen": -3.280641555786133,
+      "logits/rejected": -3.268662214279175,
+      "logps/chosen": -0.5173559188842773,
+      "logps/rejected": -0.6271434426307678,
+      "loss": 1.4903,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.5173559188842773,
+      "rewards/margins": 0.10978756844997406,
+      "rewards/rejected": -0.6271434426307678,
+      "step": 425
+    },
+    {
+      "epoch": 0.9216342933690556,
+      "grad_norm": 36.01846846843038,
+      "learning_rate": 1.086249380089782e-08,
+      "logits/chosen": -3.33141827583313,
+      "logits/rejected": -3.3434956073760986,
+      "logps/chosen": -0.615702748298645,
+      "logps/rejected": -0.7053866386413574,
+      "loss": 1.4914,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.615702748298645,
+      "rewards/margins": 0.0896839126944542,
+      "rewards/rejected": -0.7053866386413574,
+      "step": 430
+    },
+    {
+      "epoch": 0.9323509711989283,
+      "grad_norm": 29.36505146378418,
+      "learning_rate": 8.067337641547777e-09,
+      "logits/chosen": -3.3964333534240723,
+      "logits/rejected": -3.4042282104492188,
+      "logps/chosen": -0.5159146189689636,
+      "logps/rejected": -0.6444130539894104,
+      "loss": 1.4866,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.5159146189689636,
+      "rewards/margins": 0.12849843502044678,
+      "rewards/rejected": -0.6444130539894104,
+      "step": 435
+    },
+    {
+      "epoch": 0.943067649028801,
+      "grad_norm": 23.999063503219116,
+      "learning_rate": 5.682426703567034e-09,
+      "logits/chosen": -3.2346031665802,
+      "logits/rejected": -3.2309417724609375,
+      "logps/chosen": -0.52639240026474,
+      "logps/rejected": -0.6208442449569702,
+      "loss": 1.4676,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.52639240026474,
+      "rewards/margins": 0.09445185959339142,
+      "rewards/rejected": -0.6208442449569702,
+      "step": 440
+    },
+    {
+      "epoch": 0.9537843268586738,
+      "grad_norm": 28.590598764799935,
+      "learning_rate": 3.7111124389918146e-09,
+      "logits/chosen": -3.2654852867126465,
+      "logits/rejected": -3.264702558517456,
+      "logps/chosen": -0.5419159531593323,
+      "logps/rejected": -0.6305166482925415,
+      "loss": 1.4876,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.5419159531593323,
+      "rewards/margins": 0.08860062062740326,
+      "rewards/rejected": -0.6305166482925415,
+      "step": 445
+    },
+    {
+      "epoch": 0.9645010046885466,
+      "grad_norm": 23.071429282207497,
+      "learning_rate": 2.156165083431627e-09,
+      "logits/chosen": -3.2962241172790527,
+      "logits/rejected": -3.283967971801758,
+      "logps/chosen": -0.5432751774787903,
+      "logps/rejected": -0.6289895176887512,
+      "loss": 1.4804,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.5432751774787903,
+      "rewards/margins": 0.08571438491344452,
+      "rewards/rejected": -0.6289895176887512,
+      "step": 450
+    },
+    {
+      "epoch": 0.9752176825184193,
+      "grad_norm": 25.76239166835448,
+      "learning_rate": 1.019769763130851e-09,
+      "logits/chosen": -3.2721996307373047,
+      "logits/rejected": -3.271106243133545,
+      "logps/chosen": -0.5389679670333862,
+      "logps/rejected": -0.6317923665046692,
+      "loss": 1.4855,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.5389679670333862,
+      "rewards/margins": 0.09282433986663818,
+      "rewards/rejected": -0.6317923665046692,
+      "step": 455
+    },
+    {
+      "epoch": 0.9859343603482921,
+      "grad_norm": 29.67085990506561,
+      "learning_rate": 3.0352342426868125e-10,
+      "logits/chosen": -3.287473678588867,
+      "logits/rejected": -3.29301381111145,
+      "logps/chosen": -0.5480049252510071,
+      "logps/rejected": -0.6534655690193176,
+      "loss": 1.4972,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.5480049252510071,
+      "rewards/margins": 0.10546054691076279,
+      "rewards/rejected": -0.6534655690193176,
+      "step": 460
+    },
+    {
+      "epoch": 0.9966510381781648,
+      "grad_norm": 29.944439651933386,
+      "learning_rate": 8.432588813089836e-12,
+      "logits/chosen": -3.3211536407470703,
+      "logits/rejected": -3.304069995880127,
+      "logps/chosen": -0.583086371421814,
+      "logps/rejected": -0.6369927525520325,
+      "loss": 1.5004,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.583086371421814,
+      "rewards/margins": 0.05390629172325134,
+      "rewards/rejected": -0.6369927525520325,
+      "step": 465
+    },
+    {
+      "epoch": 0.9987943737441393,
+      "step": 466,
+      "total_flos": 0.0,
+      "train_loss": 1.5151263257976253,
+      "train_runtime": 19305.9847,
+      "train_samples_per_second": 3.093,
+      "train_steps_per_second": 0.024
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 466,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:491a81ffad9726366156d66b5b7bb300817102b2dc6cb26d3103dff8402c9d52
+size 7416