Model save

README.md

@@ -0,0 +1,84 @@
+---
+license: apache-2.0
+base_model: hZzy/qwen2.5-0.5b-sft-news-IFT
+tags:
+- trl
+- expo
+- generated_from_trainer
+model-index:
+- name: qwen2.5-0.5b-expo-DPO-noES5-1
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/zhiyuzha-university-of-florida/huggingface/runs/ka5w2jn7)
+# qwen2.5-0.5b-expo-DPO-noES5-1
+
+This model is a fine-tuned version of [hZzy/qwen2.5-0.5b-sft-news-IFT](https://huggingface.co/hZzy/qwen2.5-0.5b-sft-news-IFT) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.9399
+- Logps: -81.1684
+- Logits: -0.8509
+- Objective: 1.8787
+- Dpo Loss: 1.8787
+- Ranking Simple: 0.5347
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-06
+- train_batch_size: 4
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 3
+- gradient_accumulation_steps: 12
+- total_train_batch_size: 144
+- total_eval_batch_size: 12
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 2
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss | Logps    | Logits  | Objective | Dpo Loss | Ranking Simple |
+|:-------------:|:------:|:----:|:---------------:|:--------:|:-------:|:---------:|:--------:|:--------------:|
+| 1.1219        | 0.1417 | 50   | 1.1407          | -91.1504 | -1.3894 | 1.1280    | 1.1280   | 0.5274         |
+| 1.3821        | 0.2834 | 100  | 1.5304          | -81.2234 | -1.3774 | 1.4947    | 1.4947   | 0.5290         |
+| 1.4062        | 0.4251 | 150  | 1.8818          | -79.7787 | -1.1641 | 1.8192    | 1.8192   | 0.5430         |
+| 1.2275        | 0.5668 | 200  | 2.0358          | -77.9854 | -1.1289 | 1.9717    | 1.9717   | 0.5347         |
+| 1.1914        | 0.7085 | 250  | 2.0084          | -78.3385 | -1.0883 | 1.9461    | 1.9461   | 0.5347         |
+| 1.0378        | 0.8503 | 300  | 2.0918          | -83.4707 | -0.9324 | 2.0357    | 2.0357   | 0.5352         |
+| 0.8334        | 0.9920 | 350  | 2.1143          | -81.1740 | -0.8755 | 1.9975    | 1.9975   | 0.5388         |
+| 0.4251        | 1.1337 | 400  | 2.0641          | -81.1689 | -0.8003 | 2.0241    | 2.0241   | 0.5435         |
+| 0.3886        | 1.2754 | 450  | 2.0085          | -79.8813 | -0.8999 | 1.9598    | 1.9598   | 0.5388         |
+| 0.4352        | 1.4171 | 500  | 2.0449          | -80.7357 | -0.8634 | 1.9819    | 1.9819   | 0.5367         |
+| 0.3103        | 1.5588 | 550  | 1.9784          | -80.8827 | -0.8672 | 1.9073    | 1.9073   | 0.5373         |
+| 0.2489        | 1.7005 | 600  | 1.9488          | -81.0833 | -0.8421 | 1.8851    | 1.8851   | 0.5367         |
+| 0.3631        | 1.8422 | 650  | 1.9417          | -81.1721 | -0.8529 | 1.8805    | 1.8805   | 0.5347         |
+| 0.3009        | 1.9839 | 700  | 1.9399          | -81.1684 | -0.8509 | 1.8787    | 1.8787   | 0.5347         |
+
+
+### Framework versions
+
+- Transformers 4.42.0
+- Pytorch 2.3.0+cu121
+- Datasets 3.2.0
+- Tokenizers 0.19.1

added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.995276334435522,
+    "total_flos": 0.0,
+    "train_loss": 0.7561936149881645,
+    "train_runtime": 36938.8609,
+    "train_samples": 50802,
+    "train_samples_per_second": 2.751,
+    "train_steps_per_second": 0.019
+}

config.json

@@ -0,0 +1,30 @@
+{
+  "_name_or_path": "hZzy/qwen2.5-0.5b-sft-news-IFT",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151644,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 896,
+  "initializer_range": 0.02,
+  "intermediate_size": 4864,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 24,
+  "model_type": "qwen2",
+  "num_attention_heads": 14,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 2,
+  "pad_token_id": 151645,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": 32768,
+  "tie_word_embeddings": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.42.0",
+  "use_cache": false,
+  "use_mrope": false,
+  "use_sliding_window": false,
+  "vocab_size": 151665
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "bos_token_id": 151644,
+  "eos_token_id": 151645,
+  "max_new_tokens": 2048,
+  "pad_token_id": 151645,
+  "transformers_version": "4.42.0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a6927a8e3a0aa891c25b491010fd4c0e7b8ba06c89a3a2661df8f80b2921567
+size 1975192208

special_tokens_map.json

@@ -0,0 +1,27 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": {
+    "content": "<|im_start|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,196 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": "<|im_start|>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 2048,
+  "pad_token": "<|im_end|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.995276334435522,
+    "total_flos": 0.0,
+    "train_loss": 0.7561936149881645,
+    "train_runtime": 36938.8609,
+    "train_samples": 50802,
+    "train_samples_per_second": 2.751,
+    "train_steps_per_second": 0.019
+}

trainer_state.json

@@ -0,0 +1,1916 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.995276334435522,
+  "eval_steps": 50,
+  "global_step": 704,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "dpo_loss": 0.6931471824645996,
+      "epoch": 0.002834199338686821,
+      "grad_norm": 183.97025063743072,
+      "learning_rate": 7.042253521126761e-08,
+      "logits": -1.2867579460144043,
+      "logps": -84.34933471679688,
+      "loss": 0.6931,
+      "objective": 0.6931471824645996,
+      "ranking_simple": 0.5833333134651184,
+      "step": 1
+    },
+    {
+      "dpo_loss": 0.6891745924949646,
+      "epoch": 0.014170996693434105,
+      "grad_norm": 175.97134717956817,
+      "learning_rate": 3.521126760563381e-07,
+      "logits": -1.430897831916809,
+      "logps": -83.63750457763672,
+      "loss": 0.6935,
+      "objective": 0.6891745924949646,
+      "ranking_simple": 0.4895833432674408,
+      "step": 5
+    },
+    {
+      "dpo_loss": 0.6994127631187439,
+      "epoch": 0.02834199338686821,
+      "grad_norm": 210.24216843481642,
+      "learning_rate": 7.042253521126762e-07,
+      "logits": -1.4024795293807983,
+      "logps": -84.05953216552734,
+      "loss": 0.7084,
+      "objective": 0.6994127631187439,
+      "ranking_simple": 0.5791666507720947,
+      "step": 10
+    },
+    {
+      "dpo_loss": 0.7201023101806641,
+      "epoch": 0.042512990080302314,
+      "grad_norm": 217.21520494236623,
+      "learning_rate": 1.0563380281690142e-06,
+      "logits": -1.5396863222122192,
+      "logps": -83.98353576660156,
+      "loss": 0.6992,
+      "objective": 0.7201023101806641,
+      "ranking_simple": 0.5708333253860474,
+      "step": 15
+    },
+    {
+      "dpo_loss": 0.6394654512405396,
+      "epoch": 0.05668398677373642,
+      "grad_norm": 178.00062840147226,
+      "learning_rate": 1.4084507042253523e-06,
+      "logits": -1.3805947303771973,
+      "logps": -82.83494567871094,
+      "loss": 0.6982,
+      "objective": 0.6394654512405396,
+      "ranking_simple": 0.4833333194255829,
+      "step": 20
+    },
+    {
+      "dpo_loss": 0.7543553113937378,
+      "epoch": 0.07085498346717052,
+      "grad_norm": 232.13126995990035,
+      "learning_rate": 1.7605633802816902e-06,
+      "logits": -1.406501054763794,
+      "logps": -83.180908203125,
+      "loss": 0.7413,
+      "objective": 0.7543553113937378,
+      "ranking_simple": 0.5833333134651184,
+      "step": 25
+    },
+    {
+      "dpo_loss": 0.8550678491592407,
+      "epoch": 0.08502598016060463,
+      "grad_norm": 178.51107981018083,
+      "learning_rate": 2.1126760563380285e-06,
+      "logits": -1.450936198234558,
+      "logps": -83.1689682006836,
+      "loss": 0.8038,
+      "objective": 0.8550678491592407,
+      "ranking_simple": 0.4958333373069763,
+      "step": 30
+    },
+    {
+      "dpo_loss": 0.8976810574531555,
+      "epoch": 0.09919697685403873,
+      "grad_norm": 159.63407046559615,
+      "learning_rate": 2.4647887323943666e-06,
+      "logits": -1.4125274419784546,
+      "logps": -83.51966857910156,
+      "loss": 0.8446,
+      "objective": 0.8976810574531555,
+      "ranking_simple": 0.550000011920929,
+      "step": 35
+    },
+    {
+      "dpo_loss": 0.9427976608276367,
+      "epoch": 0.11336797354747284,
+      "grad_norm": 204.8220203673265,
+      "learning_rate": 2.8169014084507046e-06,
+      "logits": -1.370052456855774,
+      "logps": -85.57767486572266,
+      "loss": 0.8988,
+      "objective": 0.9427976608276367,
+      "ranking_simple": 0.5,
+      "step": 40
+    },
+    {
+      "dpo_loss": 0.9266307950019836,
+      "epoch": 0.12753897024090693,
+      "grad_norm": 166.47204979724842,
+      "learning_rate": 3.1690140845070427e-06,
+      "logits": -1.3562687635421753,
+      "logps": -82.7292251586914,
+      "loss": 0.9677,
+      "objective": 0.9266307950019836,
+      "ranking_simple": 0.5041666626930237,
+      "step": 45
+    },
+    {
+      "dpo_loss": 1.1937092542648315,
+      "epoch": 0.14170996693434104,
+      "grad_norm": 162.3910297296546,
+      "learning_rate": 3.5211267605633804e-06,
+      "logits": -1.3756095170974731,
+      "logps": -83.8311538696289,
+      "loss": 1.1219,
+      "objective": 1.1937092542648315,
+      "ranking_simple": 0.5249999761581421,
+      "step": 50
+    },
+    {
+      "epoch": 0.14170996693434104,
+      "eval_dpo_loss": 1.127982258796692,
+      "eval_logits": -1.3893742561340332,
+      "eval_logps": -91.15036010742188,
+      "eval_loss": 1.1406893730163574,
+      "eval_objective": 1.127982258796692,
+      "eval_ranking_simple": 0.5274327397346497,
+      "eval_runtime": 605.2256,
+      "eval_samples_per_second": 9.567,
+      "eval_steps_per_second": 0.798,
+      "step": 50
+    },
+    {
+      "dpo_loss": 1.2814911603927612,
+      "epoch": 0.15588096362777515,
+      "grad_norm": 163.57940740751664,
+      "learning_rate": 3.873239436619718e-06,
+      "logits": -1.3777908086776733,
+      "logps": -84.10151672363281,
+      "loss": 1.1332,
+      "objective": 1.2814911603927612,
+      "ranking_simple": 0.5041666626930237,
+      "step": 55
+    },
+    {
+      "dpo_loss": 1.0759367942810059,
+      "epoch": 0.17005196032120926,
+      "grad_norm": 161.38779700951656,
+      "learning_rate": 4.225352112676057e-06,
+      "logits": -1.3657281398773193,
+      "logps": -80.9452133178711,
+      "loss": 1.1561,
+      "objective": 1.0759367942810059,
+      "ranking_simple": 0.5541666746139526,
+      "step": 60
+    },
+    {
+      "dpo_loss": 1.2120552062988281,
+      "epoch": 0.18422295701464336,
+      "grad_norm": 160.82431949475915,
+      "learning_rate": 4.577464788732395e-06,
+      "logits": -1.3958569765090942,
+      "logps": -77.98729705810547,
+      "loss": 1.1297,
+      "objective": 1.2120552062988281,
+      "ranking_simple": 0.574999988079071,
+      "step": 65
+    },
+    {
+      "dpo_loss": 1.182289958000183,
+      "epoch": 0.19839395370807747,
+      "grad_norm": 153.19784131936885,
+      "learning_rate": 4.929577464788733e-06,
+      "logits": -1.3969266414642334,
+      "logps": -81.52046203613281,
+      "loss": 1.1094,
+      "objective": 1.182289958000183,
+      "ranking_simple": 0.5791666507720947,
+      "step": 70
+    },
+    {
+      "dpo_loss": 1.1960618495941162,
+      "epoch": 0.21256495040151158,
+      "grad_norm": 146.00340475417016,
+      "learning_rate": 4.999507384516835e-06,
+      "logits": -1.3441658020019531,
+      "logps": -80.9335708618164,
+      "loss": 1.2518,
+      "objective": 1.1960618495941162,
+      "ranking_simple": 0.5833333134651184,
+      "step": 75
+    },
+    {
+      "dpo_loss": 1.1959224939346313,
+      "epoch": 0.22673594709494568,
+      "grad_norm": 164.14111542380238,
+      "learning_rate": 4.997506466835171e-06,
+      "logits": -1.4696060419082642,
+      "logps": -78.75235748291016,
+      "loss": 1.2283,
+      "objective": 1.1959224939346313,
+      "ranking_simple": 0.612500011920929,
+      "step": 80
+    },
+    {
+      "dpo_loss": 1.3274264335632324,
+      "epoch": 0.2409069437883798,
+      "grad_norm": 116.34306089867205,
+      "learning_rate": 4.9939676896203576e-06,
+      "logits": -1.3812180757522583,
+      "logps": -77.95438385009766,
+      "loss": 1.2276,
+      "objective": 1.3274264335632324,
+      "ranking_simple": 0.5833333134651184,
+      "step": 85
+    },
+    {
+      "dpo_loss": 1.0321040153503418,
+      "epoch": 0.25507794048181387,
+      "grad_norm": 130.12375600026587,
+      "learning_rate": 4.9888932319026994e-06,
+      "logits": -1.406469702720642,
+      "logps": -78.24427795410156,
+      "loss": 1.2313,
+      "objective": 1.0321040153503418,
+      "ranking_simple": 0.5916666388511658,
+      "step": 90
+    },
+    {
+      "dpo_loss": 1.0937633514404297,
+      "epoch": 0.269248937175248,
+      "grad_norm": 138.54409786316114,
+      "learning_rate": 4.982286218320023e-06,
+      "logits": -1.3431053161621094,
+      "logps": -78.20173645019531,
+      "loss": 1.0961,
+      "objective": 1.0937633514404297,
+      "ranking_simple": 0.5333333611488342,
+      "step": 95
+    },
+    {
+      "dpo_loss": 1.2458388805389404,
+      "epoch": 0.2834199338686821,
+      "grad_norm": 135.72867701135627,
+      "learning_rate": 4.974150717193654e-06,
+      "logits": -1.2822611331939697,
+      "logps": -75.05729675292969,
+      "loss": 1.3821,
+      "objective": 1.2458388805389404,
+      "ranking_simple": 0.5666666626930237,
+      "step": 100
+    },
+    {
+      "epoch": 0.2834199338686821,
+      "eval_dpo_loss": 1.4947212934494019,
+      "eval_logits": -1.3773702383041382,
+      "eval_logps": -81.2234115600586,
+      "eval_loss": 1.530389428138733,
+      "eval_objective": 1.4947212934494019,
+      "eval_ranking_simple": 0.5289855003356934,
+      "eval_runtime": 599.6473,
+      "eval_samples_per_second": 9.656,
+      "eval_steps_per_second": 0.805,
+      "step": 100
+    },
+    {
+      "dpo_loss": 1.673114538192749,
+      "epoch": 0.2975909305621162,
+      "grad_norm": 165.43372686768916,
+      "learning_rate": 4.964491738023321e-06,
+      "logits": -1.4061511754989624,
+      "logps": -76.94784545898438,
+      "loss": 1.4977,
+      "objective": 1.673114538192749,
+      "ranking_simple": 0.5916666388511658,
+      "step": 105
+    },
+    {
+      "dpo_loss": 1.381745457649231,
+      "epoch": 0.3117619272555503,
+      "grad_norm": 134.94435406621707,
+      "learning_rate": 4.953315228402512e-06,
+      "logits": -1.2966018915176392,
+      "logps": -75.49461364746094,
+      "loss": 1.4307,
+      "objective": 1.381745457649231,
+      "ranking_simple": 0.5541666746139526,
+      "step": 110
+    },
+    {
+      "dpo_loss": 1.525147795677185,
+      "epoch": 0.32593292394898443,
+      "grad_norm": 146.42714865280993,
+      "learning_rate": 4.9406280703561944e-06,
+      "logits": -1.203226923942566,
+      "logps": -73.4133529663086,
+      "loss": 1.4613,
+      "objective": 1.525147795677185,
+      "ranking_simple": 0.5708333253860474,
+      "step": 115
+    },
+    {
+      "dpo_loss": 1.2190510034561157,
+      "epoch": 0.3401039206424185,
+      "grad_norm": 145.8596102648603,
+      "learning_rate": 4.926438076103162e-06,
+      "logits": -1.192163109779358,
+      "logps": -72.815673828125,
+      "loss": 1.3059,
+      "objective": 1.2190510034561157,
+      "ranking_simple": 0.5708333253860474,
+      "step": 120
+    },
+    {
+      "dpo_loss": 1.5624465942382812,
+      "epoch": 0.35427491733585265,
+      "grad_norm": 131.5829174213295,
+      "learning_rate": 4.910753983245589e-06,
+      "logits": -1.190179467201233,
+      "logps": -73.50247192382812,
+      "loss": 1.4015,
+      "objective": 1.5624465942382812,
+      "ranking_simple": 0.5541666746139526,
+      "step": 125
+    },
+    {
+      "dpo_loss": 1.3345781564712524,
+      "epoch": 0.3684459140292867,
+      "grad_norm": 121.07722918376957,
+      "learning_rate": 4.893585449388786e-06,
+      "logits": -1.1411052942276,
+      "logps": -70.5329818725586,
+      "loss": 1.4099,
+      "objective": 1.3345781564712524,
+      "ranking_simple": 0.6416666507720947,
+      "step": 130
+    },
+    {
+      "dpo_loss": 1.0789151191711426,
+      "epoch": 0.3826169107227208,
+      "grad_norm": 118.85794111154456,
+      "learning_rate": 4.8749430461944536e-06,
+      "logits": -1.088476300239563,
+      "logps": -71.05619812011719,
+      "loss": 1.2611,
+      "objective": 1.0789151191711426,
+      "ranking_simple": 0.5874999761581421,
+      "step": 135
+    },
+    {
+      "dpo_loss": 1.8989228010177612,
+      "epoch": 0.39678790741615494,
+      "grad_norm": 131.90266764687357,
+      "learning_rate": 4.854838252871097e-06,
+      "logits": -1.0452989339828491,
+      "logps": -71.00385284423828,
+      "loss": 1.5343,
+      "objective": 1.8989228010177612,
+      "ranking_simple": 0.574999988079071,
+      "step": 140
+    },
+    {
+      "dpo_loss": 1.576548457145691,
+      "epoch": 0.410958904109589,
+      "grad_norm": 140.01764826482653,
+      "learning_rate": 4.833283449105609e-06,
+      "logits": -1.1004178524017334,
+      "logps": -73.08991241455078,
+      "loss": 1.6065,
+      "objective": 1.576548457145691,
+      "ranking_simple": 0.625,
+      "step": 145
+    },
+    {
+      "dpo_loss": 1.3776977062225342,
+      "epoch": 0.42512990080302315,
+      "grad_norm": 126.62570019000385,
+      "learning_rate": 4.810291907440382e-06,
+      "logits": -1.0991231203079224,
+      "logps": -74.44639587402344,
+      "loss": 1.4062,
+      "objective": 1.3776977062225342,
+      "ranking_simple": 0.574999988079071,
+      "step": 150
+    },
+    {
+      "epoch": 0.42512990080302315,
+      "eval_dpo_loss": 1.8192129135131836,
+      "eval_logits": -1.1641029119491577,
+      "eval_logps": -79.7786636352539,
+      "eval_loss": 1.8817615509033203,
+      "eval_objective": 1.8192129135131836,
+      "eval_ranking_simple": 0.5429606437683105,
+      "eval_runtime": 600.2453,
+      "eval_samples_per_second": 9.646,
+      "eval_steps_per_second": 0.805,
+      "step": 150
+    },
+    {
+      "dpo_loss": 1.2696728706359863,
+      "epoch": 0.43930089749645723,
+      "grad_norm": 122.22748192353309,
+      "learning_rate": 4.785877785100633e-06,
+      "logits": -1.187027096748352,
+      "logps": -75.60051727294922,
+      "loss": 1.3364,
+      "objective": 1.2696728706359863,
+      "ranking_simple": 0.5708333253860474,
+      "step": 155
+    },
+    {
+      "dpo_loss": 1.7951911687850952,
+      "epoch": 0.45347189418989137,
+      "grad_norm": 125.17292960366125,
+      "learning_rate": 4.7600561152769795e-06,
+      "logits": -1.1456806659698486,
+      "logps": -74.9968032836914,
+      "loss": 1.5773,
+      "objective": 1.7951911687850952,
+      "ranking_simple": 0.550000011920929,
+      "step": 160
+    },
+    {
+      "dpo_loss": 1.2346725463867188,
+      "epoch": 0.46764289088332545,
+      "grad_norm": 108.82881778945708,
+      "learning_rate": 4.732842797868631e-06,
+      "logits": -1.2308270931243896,
+      "logps": -75.75040435791016,
+      "loss": 1.3947,
+      "objective": 1.2346725463867188,
+      "ranking_simple": 0.6083333492279053,
+      "step": 165
+    },
+    {
+      "dpo_loss": 1.3481464385986328,
+      "epoch": 0.4818138875767596,
+      "grad_norm": 108.10724884444645,
+      "learning_rate": 4.704254589692903e-06,
+      "logits": -1.2144527435302734,
+      "logps": -75.97994232177734,
+      "loss": 1.305,
+      "objective": 1.3481464385986328,
+      "ranking_simple": 0.5583333373069763,
+      "step": 170
+    },
+    {
+      "dpo_loss": 1.4393560886383057,
+      "epoch": 0.49598488427019366,
+      "grad_norm": 104.8074907192406,
+      "learning_rate": 4.6743090941670675e-06,
+      "logits": -1.147178292274475,
+      "logps": -76.960205078125,
+      "loss": 1.3896,
+      "objective": 1.4393560886383057,
+      "ranking_simple": 0.5583333373069763,
+      "step": 175
+    },
+    {
+      "dpo_loss": 1.6493242979049683,
+      "epoch": 0.5101558809636277,
+      "grad_norm": 107.1323631395759,
+      "learning_rate": 4.643024750468913e-06,
+      "logits": -1.1330630779266357,
+      "logps": -75.9026870727539,
+      "loss": 1.3334,
+      "objective": 1.6493242979049683,
+      "ranking_simple": 0.5291666388511658,
+      "step": 180
+    },
+    {
+      "dpo_loss": 1.1641370058059692,
+      "epoch": 0.5243268776570619,
+      "grad_norm": 113.77830856113329,
+      "learning_rate": 4.610420822182671e-06,
+      "logits": -1.1537328958511353,
+      "logps": -74.29253387451172,
+      "loss": 1.1361,
+      "objective": 1.1641370058059692,
+      "ranking_simple": 0.5916666388511658,
+      "step": 185
+    },
+    {
+      "dpo_loss": 1.2755895853042603,
+      "epoch": 0.538497874350496,
+      "grad_norm": 109.44630427090762,
+      "learning_rate": 4.576517385437315e-06,
+      "logits": -1.1797126531600952,
+      "logps": -74.75035858154297,
+      "loss": 1.2583,
+      "objective": 1.2755895853042603,
+      "ranking_simple": 0.637499988079071,
+      "step": 190
+    },
+    {
+      "dpo_loss": 1.073716402053833,
+      "epoch": 0.5526688710439301,
+      "grad_norm": 98.88901766370496,
+      "learning_rate": 4.541335316544514e-06,
+      "logits": -1.1745376586914062,
+      "logps": -74.90966796875,
+      "loss": 1.2475,
+      "objective": 1.073716402053833,
+      "ranking_simple": 0.637499988079071,
+      "step": 195
+    },
+    {
+      "dpo_loss": 1.185225248336792,
+      "epoch": 0.5668398677373642,
+      "grad_norm": 94.03051977106067,
+      "learning_rate": 4.5048962791438885e-06,
+      "logits": -1.1321061849594116,
+      "logps": -74.69174194335938,
+      "loss": 1.2275,
+      "objective": 1.185225248336792,
+      "ranking_simple": 0.5791666507720947,
+      "step": 200
+    },
+    {
+      "epoch": 0.5668398677373642,
+      "eval_dpo_loss": 1.9716603755950928,
+      "eval_logits": -1.1289294958114624,
+      "eval_logps": -77.9853515625,
+      "eval_loss": 2.0358171463012695,
+      "eval_objective": 1.9716603755950928,
+      "eval_ranking_simple": 0.534679114818573,
+      "eval_runtime": 601.7719,
+      "eval_samples_per_second": 9.622,
+      "eval_steps_per_second": 0.803,
+      "step": 200
+    },
+    {
+      "dpo_loss": 1.2547378540039062,
+      "epoch": 0.5810108644307983,
+      "grad_norm": 109.12199093644038,
+      "learning_rate": 4.467222710863444e-06,
+      "logits": -0.9467021226882935,
+      "logps": -74.9859390258789,
+      "loss": 1.2613,
+      "objective": 1.2547378540039062,
+      "ranking_simple": 0.5874999761581421,
+      "step": 205
+    },
+    {
+      "dpo_loss": 0.9887279868125916,
+      "epoch": 0.5951818611242324,
+      "grad_norm": 110.59494347158504,
+      "learning_rate": 4.428337809503425e-06,
+      "logits": -0.9453111290931702,
+      "logps": -75.48457336425781,
+      "loss": 1.2147,
+      "objective": 0.9887279868125916,
+      "ranking_simple": 0.5958333611488342,
+      "step": 210
+    },
+    {
+      "dpo_loss": 1.297851800918579,
+      "epoch": 0.6093528578176665,
+      "grad_norm": 95.93031839243162,
+      "learning_rate": 4.388265518752085e-06,
+      "logits": -0.9826800227165222,
+      "logps": -75.59996032714844,
+      "loss": 1.1658,
+      "objective": 1.297851800918579,
+      "ranking_simple": 0.574999988079071,
+      "step": 215
+    },
+    {
+      "dpo_loss": 1.0760146379470825,
+      "epoch": 0.6235238545111006,
+      "grad_norm": 93.17621109620359,
+      "learning_rate": 4.347030513442168e-06,
+      "logits": -1.0486137866973877,
+      "logps": -75.58748626708984,
+      "loss": 1.0822,
+      "objective": 1.0760146379470825,
+      "ranking_simple": 0.5333333611488342,
+      "step": 220
+    },
+    {
+      "dpo_loss": 1.1693772077560425,
+      "epoch": 0.6376948512045347,
+      "grad_norm": 97.88558873864356,
+      "learning_rate": 4.304658184357186e-06,
+      "logits": -1.0480105876922607,
+      "logps": -75.87112426757812,
+      "loss": 1.2411,
+      "objective": 1.1693772077560425,
+      "ranking_simple": 0.5791666507720947,
+      "step": 225
+    },
+    {
+      "dpo_loss": 1.0544065237045288,
+      "epoch": 0.6518658478979689,
+      "grad_norm": 96.24074077951633,
+      "learning_rate": 4.261174622596835e-06,
+      "logits": -0.9322254657745361,
+      "logps": -74.73289489746094,
+      "loss": 1.2067,
+      "objective": 1.0544065237045288,
+      "ranking_simple": 0.6499999761581421,
+      "step": 230
+    },
+    {
+      "dpo_loss": 1.3109701871871948,
+      "epoch": 0.6660368445914029,
+      "grad_norm": 99.32952244494642,
+      "learning_rate": 4.216606603511202e-06,
+      "logits": -0.8552966713905334,
+      "logps": -76.02928161621094,
+      "loss": 1.1741,
+      "objective": 1.3109701871871948,
+      "ranking_simple": 0.5916666388511658,
+      "step": 235
+    },
+    {
+      "dpo_loss": 1.1290283203125,
+      "epoch": 0.680207841284837,
+      "grad_norm": 90.5291171770579,
+      "learning_rate": 4.170981570213621e-06,
+      "logits": -1.0387074947357178,
+      "logps": -74.89482116699219,
+      "loss": 1.1403,
+      "objective": 1.1290283203125,
+      "ranking_simple": 0.5916666388511658,
+      "step": 240
+    },
+    {
+      "dpo_loss": 1.3167603015899658,
+      "epoch": 0.6943788379782712,
+      "grad_norm": 111.7529832893929,
+      "learning_rate": 4.124327616682362e-06,
+      "logits": -1.05294930934906,
+      "logps": -73.70958709716797,
+      "loss": 1.1308,
+      "objective": 1.3167603015899658,
+      "ranking_simple": 0.612500011920929,
+      "step": 245
+    },
+    {
+      "dpo_loss": 1.0333999395370483,
+      "epoch": 0.7085498346717053,
+      "grad_norm": 103.1933415885334,
+      "learning_rate": 4.076673470461538e-06,
+      "logits": -0.8845551609992981,
+      "logps": -73.25338745117188,
+      "loss": 1.1914,
+      "objective": 1.0333999395370483,
+      "ranking_simple": 0.6208333373069763,
+      "step": 250
+    },
+    {
+      "epoch": 0.7085498346717053,
+      "eval_dpo_loss": 1.9460629224777222,
+      "eval_logits": -1.0883058309555054,
+      "eval_logps": -78.33851623535156,
+      "eval_loss": 2.0083987712860107,
+      "eval_objective": 1.9460629224777222,
+      "eval_ranking_simple": 0.534679114818573,
+      "eval_runtime": 607.1642,
+      "eval_samples_per_second": 9.536,
+      "eval_steps_per_second": 0.796,
+      "step": 250
+    },
+    {
+      "dpo_loss": 1.0360175371170044,
+      "epoch": 0.7227208313651393,
+      "grad_norm": 86.03208381847595,
+      "learning_rate": 4.028048474971889e-06,
+      "logits": -1.0679165124893188,
+      "logps": -73.73800659179688,
+      "loss": 1.0854,
+      "objective": 1.0360175371170044,
+      "ranking_simple": 0.5458333492279053,
+      "step": 255
+    },
+    {
+      "dpo_loss": 1.0874184370040894,
+      "epoch": 0.7368918280585735,
+      "grad_norm": 103.07844444269477,
+      "learning_rate": 3.978482571442339e-06,
+      "logits": -1.0024687051773071,
+      "logps": -77.03482818603516,
+      "loss": 1.12,
+      "objective": 1.0874184370040894,
+      "ranking_simple": 0.612500011920929,
+      "step": 260
+    },
+    {
+      "dpo_loss": 0.8361913561820984,
+      "epoch": 0.7510628247520076,
+      "grad_norm": 99.23229150586383,
+      "learning_rate": 3.928006280473445e-06,
+      "logits": -0.9436743259429932,
+      "logps": -76.39049530029297,
+      "loss": 1.0349,
+      "objective": 0.8361913561820984,
+      "ranking_simple": 0.6708333492279053,
+      "step": 265
+    },
+    {
+      "dpo_loss": 1.0280269384384155,
+      "epoch": 0.7652338214454416,
+      "grad_norm": 95.81187840710191,
+      "learning_rate": 3.876650683244093e-06,
+      "logits": -0.9881510138511658,
+      "logps": -77.2256851196289,
+      "loss": 1.0768,
+      "objective": 1.0280269384384155,
+      "ranking_simple": 0.5458333492279053,
+      "step": 270
+    },
+    {
+      "dpo_loss": 0.9129766821861267,
+      "epoch": 0.7794048181388757,
+      "grad_norm": 89.52663856813324,
+      "learning_rate": 3.8244474023730155e-06,
+      "logits": -0.8726529479026794,
+      "logps": -76.8287124633789,
+      "loss": 1.0978,
+      "objective": 0.9129766821861267,
+      "ranking_simple": 0.5874999761581421,
+      "step": 275
+    },
+    {
+      "dpo_loss": 0.8938360810279846,
+      "epoch": 0.7935758148323099,
+      "grad_norm": 88.2587417355247,
+      "learning_rate": 3.771428582446908e-06,
+      "logits": -0.9136671423912048,
+      "logps": -76.3465347290039,
+      "loss": 0.9242,
+      "objective": 0.8938360810279846,
+      "ranking_simple": 0.6083333492279053,
+      "step": 280
+    },
+    {
+      "dpo_loss": 1.147481083869934,
+      "epoch": 0.807746811525744,
+      "grad_norm": 106.04454752282797,
+      "learning_rate": 3.7176268702271468e-06,
+      "logits": -0.8765575885772705,
+      "logps": -77.12137603759766,
+      "loss": 0.9999,
+      "objective": 1.147481083869934,
+      "ranking_simple": 0.5583333373069763,
+      "step": 285
+    },
+    {
+      "dpo_loss": 0.9309877157211304,
+      "epoch": 0.821917808219178,
+      "grad_norm": 97.07071302996712,
+      "learning_rate": 3.6630753945472854e-06,
+      "logits": -0.8515520095825195,
+      "logps": -76.22865295410156,
+      "loss": 0.8822,
+      "objective": 0.9309877157211304,
+      "ranking_simple": 0.5916666388511658,
+      "step": 290
+    },
+    {
+      "dpo_loss": 0.8610218167304993,
+      "epoch": 0.8360888049126122,
+      "grad_norm": 95.1741444029312,
+      "learning_rate": 3.6078077459137097e-06,
+      "logits": -0.9089216589927673,
+      "logps": -76.60446166992188,
+      "loss": 1.093,
+      "objective": 0.8610218167304993,
+      "ranking_simple": 0.550000011920929,
+      "step": 295
+    },
+    {
+      "dpo_loss": 0.7812207341194153,
+      "epoch": 0.8502598016060463,
+      "grad_norm": 86.74577338149517,
+      "learning_rate": 3.5518579558220144e-06,
+      "logits": -0.885455846786499,
+      "logps": -76.61499786376953,
+      "loss": 1.0378,
+      "objective": 0.7812207341194153,
+      "ranking_simple": 0.6499999761581421,
+      "step": 300
+    },
+    {
+      "epoch": 0.8502598016060463,
+      "eval_dpo_loss": 2.0357348918914795,
+      "eval_logits": -0.9324438571929932,
+      "eval_logps": -83.47068786621094,
+      "eval_loss": 2.091830015182495,
+      "eval_objective": 2.0357348918914795,
+      "eval_ranking_simple": 0.5351966619491577,
+      "eval_runtime": 611.0424,
+      "eval_samples_per_second": 9.476,
+      "eval_steps_per_second": 0.79,
+      "step": 300
+    },
+    {
+      "dpo_loss": 1.2579916715621948,
+      "epoch": 0.8644307982994804,
+      "grad_norm": 85.57616018620313,
+      "learning_rate": 3.495260475801841e-06,
+      "logits": -0.8382306694984436,
+      "logps": -79.23892974853516,
+      "loss": 0.9276,
+      "objective": 1.2579916715621948,
+      "ranking_simple": 0.637499988079071,
+      "step": 305
+    },
+    {
+      "dpo_loss": 0.6779700517654419,
+      "epoch": 0.8786017949929145,
+      "grad_norm": 99.75714946886596,
+      "learning_rate": 3.4380501562030704e-06,
+      "logits": -0.7922627329826355,
+      "logps": -78.47576141357422,
+      "loss": 0.8132,
+      "objective": 0.6779700517654419,
+      "ranking_simple": 0.6166666746139526,
+      "step": 310
+    },
+    {
+      "dpo_loss": 0.8982964754104614,
+      "epoch": 0.8927727916863486,
+      "grad_norm": 77.85557552766669,
+      "learning_rate": 3.3802622247364446e-06,
+      "logits": -0.7617653608322144,
+      "logps": -78.67091369628906,
+      "loss": 0.9992,
+      "objective": 0.8982964754104614,
+      "ranking_simple": 0.5874999761581421,
+      "step": 315
+    },
+    {
+      "dpo_loss": 0.8807361125946045,
+      "epoch": 0.9069437883797827,
+      "grad_norm": 97.29848811322167,
+      "learning_rate": 3.321932264781822e-06,
+      "logits": -0.7270026803016663,
+      "logps": -80.3343734741211,
+      "loss": 0.9675,
+      "objective": 0.8807361125946045,
+      "ranking_simple": 0.6208333373069763,
+      "step": 320
+    },
+    {
+      "dpo_loss": 0.7385057806968689,
+      "epoch": 0.9211147850732169,
+      "grad_norm": 80.05333271753152,
+      "learning_rate": 3.2630961934774265e-06,
+      "logits": -0.8284817934036255,
+      "logps": -80.14983367919922,
+      "loss": 0.8071,
+      "objective": 0.7385057806968689,
+      "ranking_simple": 0.6041666865348816,
+      "step": 325
+    },
+    {
+      "dpo_loss": 0.8387654423713684,
+      "epoch": 0.9352857817666509,
+      "grad_norm": 85.76862598705932,
+      "learning_rate": 3.203790239603583e-06,
+      "logits": -0.7765657305717468,
+      "logps": -79.01445007324219,
+      "loss": 0.9652,
+      "objective": 0.8387654423713684,
+      "ranking_simple": 0.5958333611488342,
+      "step": 330
+    },
+    {
+      "dpo_loss": 0.838505208492279,
+      "epoch": 0.949456778460085,
+      "grad_norm": 88.47139227896243,
+      "learning_rate": 3.1440509212745584e-06,
+      "logits": -0.7588701844215393,
+      "logps": -78.80551147460938,
+      "loss": 0.8953,
+      "objective": 0.838505208492279,
+      "ranking_simple": 0.6166666746139526,
+      "step": 335
+    },
+    {
+      "dpo_loss": 1.108252763748169,
+      "epoch": 0.9636277751535192,
+      "grad_norm": 69.45012344359534,
+      "learning_rate": 3.0839150234522404e-06,
+      "logits": -0.8102169036865234,
+      "logps": -77.67143249511719,
+      "loss": 0.9601,
+      "objective": 1.108252763748169,
+      "ranking_simple": 0.5874999761581421,
+      "step": 340
+    },
+    {
+      "dpo_loss": 0.9010140299797058,
+      "epoch": 0.9777987718469532,
+      "grad_norm": 82.40549141111858,
+      "learning_rate": 3.0234195752955032e-06,
+      "logits": -0.8673346638679504,
+      "logps": -75.89323425292969,
+      "loss": 0.969,
+      "objective": 0.9010140299797058,
+      "ranking_simple": 0.6541666388511658,
+      "step": 345
+    },
+    {
+      "dpo_loss": 0.7752221822738647,
+      "epoch": 0.9919697685403873,
+      "grad_norm": 80.80020112901609,
+      "learning_rate": 2.962601827359208e-06,
+      "logits": -0.7917401790618896,
+      "logps": -77.88353729248047,
+      "loss": 0.8334,
+      "objective": 0.7752221822738647,
+      "ranking_simple": 0.6875,
+      "step": 350
+    },
+    {
+      "epoch": 0.9919697685403873,
+      "eval_dpo_loss": 1.9975330829620361,
+      "eval_logits": -0.8755112886428833,
+      "eval_logps": -81.17398071289062,
+      "eval_loss": 2.1142992973327637,
+      "eval_objective": 1.9975330829620361,
+      "eval_ranking_simple": 0.5388198494911194,
+      "eval_runtime": 610.9883,
+      "eval_samples_per_second": 9.476,
+      "eval_steps_per_second": 0.791,
+      "step": 350
+    },
+    {
+      "dpo_loss": 0.602699875831604,
+      "epoch": 1.0061407652338215,
+      "grad_norm": 47.23395645047123,
+      "learning_rate": 2.9014992286568773e-06,
+      "logits": -0.8253915309906006,
+      "logps": -76.2964096069336,
+      "loss": 0.6347,
+      "objective": 0.602699875831604,
+      "ranking_simple": 0.6708333492279053,
+      "step": 355
+    },
+    {
+      "dpo_loss": 0.6031454205513,
+      "epoch": 1.0203117619272555,
+      "grad_norm": 58.81365077909613,
+      "learning_rate": 2.840149403601166e-06,
+      "logits": -0.7789632081985474,
+      "logps": -77.5028305053711,
+      "loss": 0.4974,
+      "objective": 0.6031454205513,
+      "ranking_simple": 0.6499999761581421,
+      "step": 360
+    },
+    {
+      "dpo_loss": 0.45107781887054443,
+      "epoch": 1.0344827586206897,
+      "grad_norm": 53.96463042562765,
+      "learning_rate": 2.7785901288363253e-06,
+      "logits": -0.7713039517402649,
+      "logps": -77.14903259277344,
+      "loss": 0.4946,
+      "objective": 0.45107781887054443,
+      "ranking_simple": 0.625,
+      "step": 365
+    },
+    {
+      "dpo_loss": 0.4809951186180115,
+      "epoch": 1.0486537553141237,
+      "grad_norm": 64.07795252956707,
+      "learning_rate": 2.7168593099769414e-06,
+      "logits": -0.7404767870903015,
+      "logps": -74.05150604248047,
+      "loss": 0.4291,
+      "objective": 0.4809951186180115,
+      "ranking_simple": 0.6208333373069763,
+      "step": 370
+    },
+    {
+      "dpo_loss": 0.4239761233329773,
+      "epoch": 1.0628247520075578,
+      "grad_norm": 55.70979139140137,
+      "learning_rate": 2.654994958267241e-06,
+      "logits": -0.8401414752006531,
+      "logps": -75.16132354736328,
+      "loss": 0.4015,
+      "objective": 0.4239761233329773,
+      "ranking_simple": 0.6416666507720947,
+      "step": 375
+    },
+    {
+      "dpo_loss": 0.4185657799243927,
+      "epoch": 1.076995748700992,
+      "grad_norm": 57.189539552967226,
+      "learning_rate": 2.5930351671753707e-06,
+      "logits": -0.8457431793212891,
+      "logps": -76.84170532226562,
+      "loss": 0.4414,
+      "objective": 0.4185657799243927,
+      "ranking_simple": 0.6875,
+      "step": 380
+    },
+    {
+      "dpo_loss": 0.49603796005249023,
+      "epoch": 1.091166745394426,
+      "grad_norm": 47.834305883433586,
+      "learning_rate": 2.5310180889370374e-06,
+      "logits": -0.778562605381012,
+      "logps": -75.62349700927734,
+      "loss": 0.4482,
+      "objective": 0.49603796005249023,
+      "ranking_simple": 0.6333333253860474,
+      "step": 385
+    },
+    {
+      "dpo_loss": 0.42004406452178955,
+      "epoch": 1.10533774208786,
+      "grad_norm": 48.511623077068286,
+      "learning_rate": 2.468981911062964e-06,
+      "logits": -0.7501899600028992,
+      "logps": -77.29354095458984,
+      "loss": 0.3694,
+      "objective": 0.42004406452178955,
+      "ranking_simple": 0.6916666626930237,
+      "step": 390
+    },
+    {
+      "dpo_loss": 0.4247871935367584,
+      "epoch": 1.1195087387812943,
+      "grad_norm": 57.912950107999485,
+      "learning_rate": 2.4069648328246305e-06,
+      "logits": -0.7564467787742615,
+      "logps": -76.87413024902344,
+      "loss": 0.5005,
+      "objective": 0.4247871935367584,
+      "ranking_simple": 0.637499988079071,
+      "step": 395
+    },
+    {
+      "dpo_loss": 0.3325573801994324,
+      "epoch": 1.1336797354747283,
+      "grad_norm": 58.38348871817592,
+      "learning_rate": 2.3450050417327593e-06,
+      "logits": -0.7802504301071167,
+      "logps": -75.82648468017578,
+      "loss": 0.4251,
+      "objective": 0.3325573801994324,
+      "ranking_simple": 0.6583333611488342,
+      "step": 400
+    },
+    {
+      "epoch": 1.1336797354747283,
+      "eval_dpo_loss": 2.0240566730499268,
+      "eval_logits": -0.8002918362617493,
+      "eval_logps": -81.16890716552734,
+      "eval_loss": 2.06406831741333,
+      "eval_objective": 2.0240566730499268,
+      "eval_ranking_simple": 0.54347825050354,
+      "eval_runtime": 604.5313,
+      "eval_samples_per_second": 9.578,
+      "eval_steps_per_second": 0.799,
+      "step": 400
+    },
+    {
+      "dpo_loss": 0.6402778029441833,
+      "epoch": 1.1478507321681626,
+      "grad_norm": 57.999816664931764,
+      "learning_rate": 2.2831406900230586e-06,
+      "logits": -0.7289463877677917,
+      "logps": -77.95418548583984,
+      "loss": 0.4798,
+      "objective": 0.6402778029441833,
+      "ranking_simple": 0.6166666746139526,
+      "step": 405
+    },
+    {
+      "dpo_loss": 0.5912865400314331,
+      "epoch": 1.1620217288615966,
+      "grad_norm": 50.45480604272446,
+      "learning_rate": 2.221409871163675e-06,
+      "logits": -0.747395932674408,
+      "logps": -77.24584197998047,
+      "loss": 0.4554,
+      "objective": 0.5912865400314331,
+      "ranking_simple": 0.6541666388511658,
+      "step": 410
+    },
+    {
+      "dpo_loss": 0.3779014050960541,
+      "epoch": 1.1761927255550306,
+      "grad_norm": 64.88698660251025,
+      "learning_rate": 2.1598505963988354e-06,
+      "logits": -0.6771641969680786,
+      "logps": -75.90592956542969,
+      "loss": 0.4899,
+      "objective": 0.3779014050960541,
+      "ranking_simple": 0.6291666626930237,
+      "step": 415
+    },
+    {
+      "dpo_loss": 0.35035714507102966,
+      "epoch": 1.1903637222484649,
+      "grad_norm": 57.50741792623655,
+      "learning_rate": 2.098500771343124e-06,
+      "logits": -0.6913995742797852,
+      "logps": -76.21831512451172,
+      "loss": 0.4117,
+      "objective": 0.35035714507102966,
+      "ranking_simple": 0.6291666626930237,
+      "step": 420
+    },
+    {
+      "dpo_loss": 0.48492512106895447,
+      "epoch": 1.204534718941899,
+      "grad_norm": 59.717637187759834,
+      "learning_rate": 2.037398172640793e-06,
+      "logits": -0.7741472125053406,
+      "logps": -74.0752944946289,
+      "loss": 0.4692,
+      "objective": 0.48492512106895447,
+      "ranking_simple": 0.6541666388511658,
+      "step": 425
+    },
+    {
+      "dpo_loss": 0.613711953163147,
+      "epoch": 1.2187057156353331,
+      "grad_norm": 58.710504416860545,
+      "learning_rate": 1.976580424704498e-06,
+      "logits": -0.7667457461357117,
+      "logps": -76.66746520996094,
+      "loss": 0.3957,
+      "objective": 0.613711953163147,
+      "ranking_simple": 0.6291666626930237,
+      "step": 430
+    },
+    {
+      "dpo_loss": 0.28608372807502747,
+      "epoch": 1.2328767123287672,
+      "grad_norm": 88.90691891983907,
+      "learning_rate": 1.9160849765477604e-06,
+      "logits": -0.6787567734718323,
+      "logps": -76.04117584228516,
+      "loss": 0.4339,
+      "objective": 0.28608372807502747,
+      "ranking_simple": 0.699999988079071,
+      "step": 435
+    },
+    {
+      "dpo_loss": 0.4388173222541809,
+      "epoch": 1.2470477090222012,
+      "grad_norm": 63.01131335623436,
+      "learning_rate": 1.8559490787254423e-06,
+      "logits": -0.7236205339431763,
+      "logps": -76.31954956054688,
+      "loss": 0.4457,
+      "objective": 0.4388173222541809,
+      "ranking_simple": 0.7083333134651184,
+      "step": 440
+    },
+    {
+      "dpo_loss": 0.5651527643203735,
+      "epoch": 1.2612187057156352,
+      "grad_norm": 65.93704260731457,
+      "learning_rate": 1.7962097603964177e-06,
+      "logits": -0.7464514374732971,
+      "logps": -76.93856048583984,
+      "loss": 0.4889,
+      "objective": 0.5651527643203735,
+      "ranking_simple": 0.699999988079071,
+      "step": 445
+    },
+    {
+      "dpo_loss": 0.4833088517189026,
+      "epoch": 1.2753897024090695,
+      "grad_norm": 56.25706142129644,
+      "learning_rate": 1.7369038065225743e-06,
+      "logits": -0.8744809627532959,
+      "logps": -76.16983795166016,
+      "loss": 0.3886,
+      "objective": 0.4833088517189026,
+      "ranking_simple": 0.6416666507720947,
+      "step": 450
+    },
+    {
+      "epoch": 1.2753897024090695,
+      "eval_dpo_loss": 1.9598380327224731,
+      "eval_logits": -0.8998873233795166,
+      "eval_logps": -79.8813247680664,
+      "eval_loss": 2.0085067749023438,
+      "eval_objective": 1.9598380327224731,
+      "eval_ranking_simple": 0.5388198494911194,
+      "eval_runtime": 617.7574,
+      "eval_samples_per_second": 9.373,
+      "eval_steps_per_second": 0.782,
+      "step": 450
+    },
+    {
+      "dpo_loss": 0.5061320662498474,
+      "epoch": 1.2895606991025035,
+      "grad_norm": 54.696067906167414,
+      "learning_rate": 1.6780677352181781e-06,
+      "logits": -0.7727634310722351,
+      "logps": -76.43891906738281,
+      "loss": 0.4525,
+      "objective": 0.5061320662498474,
+      "ranking_simple": 0.5874999761581421,
+      "step": 455
+    },
+    {
+      "dpo_loss": 0.45641031861305237,
+      "epoch": 1.3037316957959377,
+      "grad_norm": 57.70332958198004,
+      "learning_rate": 1.6197377752635563e-06,
+      "logits": -0.8627158999443054,
+      "logps": -75.46236419677734,
+      "loss": 0.3956,
+      "objective": 0.45641031861305237,
+      "ranking_simple": 0.6333333253860474,
+      "step": 460
+    },
+    {
+      "dpo_loss": 0.35006803274154663,
+      "epoch": 1.3179026924893718,
+      "grad_norm": 60.819739611222175,
+      "learning_rate": 1.5619498437969302e-06,
+      "logits": -0.8722900748252869,
+      "logps": -73.59480285644531,
+      "loss": 0.4224,
+      "objective": 0.35006803274154663,
+      "ranking_simple": 0.637499988079071,
+      "step": 465
+    },
+    {
+      "dpo_loss": 0.4906507432460785,
+      "epoch": 1.3320736891828058,
+      "grad_norm": 49.64412942112177,
+      "learning_rate": 1.5047395241981606e-06,
+      "logits": -0.9396683573722839,
+      "logps": -73.02540588378906,
+      "loss": 0.5003,
+      "objective": 0.4906507432460785,
+      "ranking_simple": 0.6499999761581421,
+      "step": 470
+    },
+    {
+      "dpo_loss": 0.30290400981903076,
+      "epoch": 1.34624468587624,
+      "grad_norm": 79.3006324136739,
+      "learning_rate": 1.4481420441779862e-06,
+      "logits": -0.8443289995193481,
+      "logps": -75.5423583984375,
+      "loss": 0.3808,
+      "objective": 0.30290400981903076,
+      "ranking_simple": 0.6958333253860474,
+      "step": 475
+    },
+    {
+      "dpo_loss": 0.5529125928878784,
+      "epoch": 1.360415682569674,
+      "grad_norm": 70.13856035677702,
+      "learning_rate": 1.3921922540862907e-06,
+      "logits": -0.8475839495658875,
+      "logps": -76.10958862304688,
+      "loss": 0.476,
+      "objective": 0.5529125928878784,
+      "ranking_simple": 0.637499988079071,
+      "step": 480
+    },
+    {
+      "dpo_loss": 0.4004639685153961,
+      "epoch": 1.3745866792631083,
+      "grad_norm": 78.28600119390228,
+      "learning_rate": 1.3369246054527152e-06,
+      "logits": -0.8136020302772522,
+      "logps": -74.83834838867188,
+      "loss": 0.4712,
+      "objective": 0.4004639685153961,
+      "ranking_simple": 0.625,
+      "step": 485
+    },
+    {
+      "dpo_loss": 0.54802006483078,
+      "epoch": 1.3887576759565423,
+      "grad_norm": 61.45793050447296,
+      "learning_rate": 1.2823731297728536e-06,
+      "logits": -0.7907751798629761,
+      "logps": -77.5755844116211,
+      "loss": 0.3945,
+      "objective": 0.54802006483078,
+      "ranking_simple": 0.625,
+      "step": 490
+    },
+    {
+      "dpo_loss": 0.2221187800168991,
+      "epoch": 1.4029286726499763,
+      "grad_norm": 66.41905989355433,
+      "learning_rate": 1.2285714175530936e-06,
+      "logits": -0.805623471736908,
+      "logps": -77.69929504394531,
+      "loss": 0.3891,
+      "objective": 0.2221187800168991,
+      "ranking_simple": 0.625,
+      "step": 495
+    },
+    {
+      "dpo_loss": 0.42266911268234253,
+      "epoch": 1.4170996693434104,
+      "grad_norm": 66.25308663743711,
+      "learning_rate": 1.1755525976269851e-06,
+      "logits": -0.750647783279419,
+      "logps": -76.04014587402344,
+      "loss": 0.4352,
+      "objective": 0.42266911268234253,
+      "ranking_simple": 0.6583333611488342,
+      "step": 500
+    },
+    {
+      "epoch": 1.4170996693434104,
+      "eval_dpo_loss": 1.9818733930587769,
+      "eval_logits": -0.8633737564086914,
+      "eval_logps": -80.73566436767578,
+      "eval_loss": 2.0448944568634033,
+      "eval_objective": 1.9818733930587769,
+      "eval_ranking_simple": 0.5367494821548462,
+      "eval_runtime": 608.89,
+      "eval_samples_per_second": 9.509,
+      "eval_steps_per_second": 0.793,
+      "step": 500
+    },
+    {
+      "dpo_loss": 0.31959182024002075,
+      "epoch": 1.4312706660368446,
+      "grad_norm": 52.73552059138801,
+      "learning_rate": 1.1233493167559065e-06,
+      "logits": -0.7827561497688293,
+      "logps": -76.80120086669922,
+      "loss": 0.3302,
+      "objective": 0.31959182024002075,
+      "ranking_simple": 0.6458333134651184,
+      "step": 505
+    },
+    {
+      "dpo_loss": 0.3922279179096222,
+      "epoch": 1.4454416627302786,
+      "grad_norm": 61.50180104681149,
+      "learning_rate": 1.0719937195265555e-06,
+      "logits": -0.8109835386276245,
+      "logps": -76.9507827758789,
+      "loss": 0.3945,
+      "objective": 0.3922279179096222,
+      "ranking_simple": 0.6625000238418579,
+      "step": 510
+    },
+    {
+      "dpo_loss": 0.3581198453903198,
+      "epoch": 1.4596126594237129,
+      "grad_norm": 44.945879303706135,
+      "learning_rate": 1.0215174285576615e-06,
+      "logits": -0.8377180695533752,
+      "logps": -78.14236450195312,
+      "loss": 0.4044,
+      "objective": 0.3581198453903198,
+      "ranking_simple": 0.6083333492279053,
+      "step": 515
+    },
+    {
+      "dpo_loss": 0.3747193217277527,
+      "epoch": 1.473783656117147,
+      "grad_norm": 52.10326588262369,
+      "learning_rate": 9.719515250281122e-07,
+      "logits": -0.79498291015625,
+      "logps": -75.8031234741211,
+      "loss": 0.3132,
+      "objective": 0.3747193217277527,
+      "ranking_simple": 0.675000011920929,
+      "step": 520
+    },
+    {
+      "dpo_loss": 0.3443489670753479,
+      "epoch": 1.487954652810581,
+      "grad_norm": 51.25471401996847,
+      "learning_rate": 9.233265295384624e-07,
+      "logits": -0.6887346506118774,
+      "logps": -75.2328872680664,
+      "loss": 0.3972,
+      "objective": 0.3443489670753479,
+      "ranking_simple": 0.6541666388511658,
+      "step": 525
+    },
+    {
+      "dpo_loss": 0.3036381006240845,
+      "epoch": 1.5021256495040152,
+      "grad_norm": 58.55859781711994,
+      "learning_rate": 8.756723833176376e-07,
+      "logits": -0.8320663571357727,
+      "logps": -77.42900085449219,
+      "loss": 0.3414,
+      "objective": 0.3036381006240845,
+      "ranking_simple": 0.637499988079071,
+      "step": 530
+    },
+    {
+      "dpo_loss": 0.2670062184333801,
+      "epoch": 1.5162966461974492,
+      "grad_norm": 47.62625405878976,
+      "learning_rate": 8.290184297863793e-07,
+      "logits": -0.8374890685081482,
+      "logps": -76.14564514160156,
+      "loss": 0.3198,
+      "objective": 0.2670062184333801,
+      "ranking_simple": 0.6499999761581421,
+      "step": 535
+    },
+    {
+      "dpo_loss": 0.49031582474708557,
+      "epoch": 1.5304676428908834,
+      "grad_norm": 62.51314897054348,
+      "learning_rate": 7.833933964887985e-07,
+      "logits": -0.7449190020561218,
+      "logps": -76.09394073486328,
+      "loss": 0.3338,
+      "objective": 0.49031582474708557,
+      "ranking_simple": 0.699999988079071,
+      "step": 540
+    },
+    {
+      "dpo_loss": 0.4148392975330353,
+      "epoch": 1.5446386395843175,
+      "grad_norm": 52.636990202390535,
+      "learning_rate": 7.388253774031659e-07,
+      "logits": -0.8080311417579651,
+      "logps": -76.78816986083984,
+      "loss": 0.3883,
+      "objective": 0.4148392975330353,
+      "ranking_simple": 0.7250000238418579,
+      "step": 545
+    },
+    {
+      "dpo_loss": 0.22362883388996124,
+      "epoch": 1.5588096362777515,
+      "grad_norm": 46.69497634920009,
+      "learning_rate": 6.953418156428152e-07,
+      "logits": -0.8403748869895935,
+      "logps": -76.63642883300781,
+      "loss": 0.3103,
+      "objective": 0.22362883388996124,
+      "ranking_simple": 0.6499999761581421,
+      "step": 550
+    },
+    {
+      "epoch": 1.5588096362777515,
+      "eval_dpo_loss": 1.9073150157928467,
+      "eval_logits": -0.8671907186508179,
+      "eval_logps": -80.88265228271484,
+      "eval_loss": 1.9783891439437866,
+      "eval_objective": 1.9073150157928467,
+      "eval_ranking_simple": 0.5372670888900757,
+      "eval_runtime": 612.8866,
+      "eval_samples_per_second": 9.447,
+      "eval_steps_per_second": 0.788,
+      "step": 550
+    },
+    {
+      "dpo_loss": 0.41253456473350525,
+      "epoch": 1.5729806329711855,
+      "grad_norm": 50.727318491856984,
+      "learning_rate": 6.529694865578318e-07,
+      "logits": -0.6723602414131165,
+      "logps": -77.01154327392578,
+      "loss": 0.3626,
+      "objective": 0.41253456473350525,
+      "ranking_simple": 0.6291666626930237,
+      "step": 555
+    },
+    {
+      "dpo_loss": 0.3846481144428253,
+      "epoch": 1.5871516296646198,
+      "grad_norm": 69.96953957426896,
+      "learning_rate": 6.117344812479154e-07,
+      "logits": -0.7329820990562439,
+      "logps": -75.9783935546875,
+      "loss": 0.3651,
+      "objective": 0.3846481144428253,
+      "ranking_simple": 0.6791666746139526,
+      "step": 560
+    },
+    {
+      "dpo_loss": 0.3101910650730133,
+      "epoch": 1.601322626358054,
+      "grad_norm": 66.15790086947261,
+      "learning_rate": 5.71662190496575e-07,
+      "logits": -0.7286636233329773,
+      "logps": -76.75426483154297,
+      "loss": 0.349,
+      "objective": 0.3101910650730133,
+      "ranking_simple": 0.612500011920929,
+      "step": 565
+    },
+    {
+      "dpo_loss": 0.2726050615310669,
+      "epoch": 1.615493623051488,
+      "grad_norm": 56.0158605948958,
+      "learning_rate": 5.327772891365565e-07,
+      "logits": -0.8405836224555969,
+      "logps": -77.85334014892578,
+      "loss": 0.2979,
+      "objective": 0.2726050615310669,
+      "ranking_simple": 0.6541666388511658,
+      "step": 570
+    },
+    {
+      "dpo_loss": 0.4396015405654907,
+      "epoch": 1.629664619744922,
+      "grad_norm": 67.51231381478947,
+      "learning_rate": 4.951037208561116e-07,
+      "logits": -0.8009175658226013,
+      "logps": -75.7006607055664,
+      "loss": 0.3867,
+      "objective": 0.4396015405654907,
+      "ranking_simple": 0.6833333373069763,
+      "step": 575
+    },
+    {
+      "dpo_loss": 0.3268950283527374,
+      "epoch": 1.643835616438356,
+      "grad_norm": 55.29442626424949,
+      "learning_rate": 4.586646834554864e-07,
+      "logits": -0.8186151385307312,
+      "logps": -75.92921447753906,
+      "loss": 0.3658,
+      "objective": 0.3268950283527374,
+      "ranking_simple": 0.6625000238418579,
+      "step": 580
+    },
+    {
+      "dpo_loss": 0.2519194185733795,
+      "epoch": 1.6580066131317903,
+      "grad_norm": 53.89781546053762,
+      "learning_rate": 4.234826145626855e-07,
+      "logits": -0.7085616588592529,
+      "logps": -75.17672729492188,
+      "loss": 0.2996,
+      "objective": 0.2519194185733795,
+      "ranking_simple": 0.5874999761581421,
+      "step": 585
+    },
+    {
+      "dpo_loss": 0.31272074580192566,
+      "epoch": 1.6721776098252243,
+      "grad_norm": 59.39748892759291,
+      "learning_rate": 3.8957917781732883e-07,
+      "logits": -0.7312564253807068,
+      "logps": -76.37480163574219,
+      "loss": 0.3241,
+      "objective": 0.31272074580192566,
+      "ranking_simple": 0.675000011920929,
+      "step": 590
+    },
+    {
+      "dpo_loss": 0.27419552206993103,
+      "epoch": 1.6863486065186586,
+      "grad_norm": 53.45471180782799,
+      "learning_rate": 3.569752495310877e-07,
+      "logits": -0.7288424372673035,
+      "logps": -76.92625427246094,
+      "loss": 0.2888,
+      "objective": 0.27419552206993103,
+      "ranking_simple": 0.6458333134651184,
+      "step": 595
+    },
+    {
+      "dpo_loss": 0.1944553703069687,
+      "epoch": 1.7005196032120926,
+      "grad_norm": 50.57431920897434,
+      "learning_rate": 3.2569090583293356e-07,
+      "logits": -0.7772687673568726,
+      "logps": -77.40592193603516,
+      "loss": 0.2489,
+      "objective": 0.1944553703069687,
+      "ranking_simple": 0.699999988079071,
+      "step": 600
+    },
+    {
+      "epoch": 1.7005196032120926,
+      "eval_dpo_loss": 1.8850661516189575,
+      "eval_logits": -0.8421351313591003,
+      "eval_logps": -81.0832748413086,
+      "eval_loss": 1.9488178491592407,
+      "eval_objective": 1.8850661516189575,
+      "eval_ranking_simple": 0.5367494821548462,
+      "eval_runtime": 603.5072,
+      "eval_samples_per_second": 9.594,
+      "eval_steps_per_second": 0.8,
+      "step": 600
+    },
+    {
+      "dpo_loss": 0.35472404956817627,
+      "epoch": 1.7146905999055266,
+      "grad_norm": 65.83086255827398,
+      "learning_rate": 2.957454103070978e-07,
+      "logits": -0.7188993096351624,
+      "logps": -78.14205169677734,
+      "loss": 0.3777,
+      "objective": 0.35472404956817627,
+      "ranking_simple": 0.6499999761581421,
+      "step": 605
+    },
+    {
+      "dpo_loss": 0.286739319562912,
+      "epoch": 1.7288615965989607,
+      "grad_norm": 69.02122554997335,
+      "learning_rate": 2.6715720213136955e-07,
+      "logits": -0.7877374887466431,
+      "logps": -76.47672271728516,
+      "loss": 0.3096,
+      "objective": 0.286739319562912,
+      "ranking_simple": 0.6791666746139526,
+      "step": 610
+    },
+    {
+      "dpo_loss": 0.35429847240448,
+      "epoch": 1.743032593292395,
+      "grad_norm": 58.94519405559986,
+      "learning_rate": 2.399438847230212e-07,
+      "logits": -0.8013178110122681,
+      "logps": -76.39897155761719,
+      "loss": 0.3637,
+      "objective": 0.35429847240448,
+      "ranking_simple": 0.6166666746139526,
+      "step": 615
+    },
+    {
+      "dpo_loss": 0.2548845112323761,
+      "epoch": 1.7572035899858292,
+      "grad_norm": 53.576700184407876,
+      "learning_rate": 2.1412221489936796e-07,
+      "logits": -0.8240512609481812,
+      "logps": -76.49982452392578,
+      "loss": 0.3083,
+      "objective": 0.2548845112323761,
+      "ranking_simple": 0.6875,
+      "step": 620
+    },
+    {
+      "dpo_loss": 0.2307281196117401,
+      "epoch": 1.7713745866792632,
+      "grad_norm": 64.55540353999235,
+      "learning_rate": 1.897080925596187e-07,
+      "logits": -0.7509959936141968,
+      "logps": -75.8747329711914,
+      "loss": 0.321,
+      "objective": 0.2307281196117401,
+      "ranking_simple": 0.6416666507720947,
+      "step": 625
+    },
+    {
+      "dpo_loss": 0.5376826524734497,
+      "epoch": 1.7855455833726972,
+      "grad_norm": 61.55828638335153,
+      "learning_rate": 1.6671655089439186e-07,
+      "logits": -0.7904254794120789,
+      "logps": -77.4958267211914,
+      "loss": 0.4286,
+      "objective": 0.5376826524734497,
+      "ranking_simple": 0.6166666746139526,
+      "step": 630
+    },
+    {
+      "dpo_loss": 0.32618287205696106,
+      "epoch": 1.7997165800661312,
+      "grad_norm": 67.7723818410267,
+      "learning_rate": 1.4516174712890406e-07,
+      "logits": -0.8496752381324768,
+      "logps": -77.06031036376953,
+      "loss": 0.3605,
+      "objective": 0.32618287205696106,
+      "ranking_simple": 0.7041666507720947,
+      "step": 635
+    },
+    {
+      "dpo_loss": 0.264717161655426,
+      "epoch": 1.8138875767595655,
+      "grad_norm": 49.987295020521714,
+      "learning_rate": 1.2505695380554712e-07,
+      "logits": -0.8106148838996887,
+      "logps": -77.50927734375,
+      "loss": 0.2895,
+      "objective": 0.264717161655426,
+      "ranking_simple": 0.6499999761581421,
+      "step": 640
+    },
+    {
+      "dpo_loss": 0.3132435977458954,
+      "epoch": 1.8280585734529995,
+      "grad_norm": 57.443846637348,
+      "learning_rate": 1.0641455061121519e-07,
+      "logits": -0.8152769804000854,
+      "logps": -78.20152282714844,
+      "loss": 0.4079,
+      "objective": 0.3132435977458954,
+      "ranking_simple": 0.6958333253860474,
+      "step": 645
+    },
+    {
+      "dpo_loss": 0.3075469732284546,
+      "epoch": 1.8422295701464337,
+      "grad_norm": 63.674814735474534,
+      "learning_rate": 8.924601675441207e-08,
+      "logits": -0.8521133065223694,
+      "logps": -78.01354217529297,
+      "loss": 0.3631,
+      "objective": 0.3075469732284546,
+      "ranking_simple": 0.6708333492279053,
+      "step": 650
+    },
+    {
+      "epoch": 1.8422295701464337,
+      "eval_dpo_loss": 1.8805330991744995,
+      "eval_logits": -0.8528652787208557,
+      "eval_logps": -81.17214965820312,
+      "eval_loss": 1.9417303800582886,
+      "eval_objective": 1.8805330991744995,
+      "eval_ranking_simple": 0.534679114818573,
+      "eval_runtime": 608.284,
+      "eval_samples_per_second": 9.519,
+      "eval_steps_per_second": 0.794,
+      "step": 650
+    },
+    {
+      "dpo_loss": 0.2915767729282379,
+      "epoch": 1.8564005668398678,
+      "grad_norm": 51.326241873653245,
+      "learning_rate": 7.356192389683825e-08,
+      "logits": -0.7731481790542603,
+      "logps": -75.27713012695312,
+      "loss": 0.3407,
+      "objective": 0.2915767729282379,
+      "ranking_simple": 0.6166666746139526,
+      "step": 655
+    },
+    {
+      "dpo_loss": 0.40089327096939087,
+      "epoch": 1.8705715635333018,
+      "grad_norm": 51.09671637748102,
+      "learning_rate": 5.937192964380556e-08,
+      "logits": -0.7996050715446472,
+      "logps": -78.20768737792969,
+      "loss": 0.3594,
+      "objective": 0.40089327096939087,
+      "ranking_simple": 0.6583333611488342,
+      "step": 660
+    },
+    {
+      "dpo_loss": 0.32784855365753174,
+      "epoch": 1.8847425602267358,
+      "grad_norm": 59.65383844014325,
+      "learning_rate": 4.668477159748858e-08,
+      "logits": -0.824116587638855,
+      "logps": -76.31925201416016,
+      "loss": 0.3228,
+      "objective": 0.32784855365753174,
+      "ranking_simple": 0.7041666507720947,
+      "step": 665
+    },
+    {
+      "dpo_loss": 0.4064708948135376,
+      "epoch": 1.89891355692017,
+      "grad_norm": 52.69018895294125,
+      "learning_rate": 3.5508261976678894e-08,
+      "logits": -0.7589130997657776,
+      "logps": -75.46687316894531,
+      "loss": 0.366,
+      "objective": 0.4064708948135376,
+      "ranking_simple": 0.625,
+      "step": 670
+    },
+    {
+      "dpo_loss": 0.3538086712360382,
+      "epoch": 1.9130845536136043,
+      "grad_norm": 57.27565478185342,
+      "learning_rate": 2.5849282806345855e-08,
+      "logits": -0.7437042593955994,
+      "logps": -76.25951385498047,
+      "loss": 0.3801,
+      "objective": 0.3538086712360382,
+      "ranking_simple": 0.675000011920929,
+      "step": 675
+    },
+    {
+      "dpo_loss": 0.2823149859905243,
+      "epoch": 1.9272555503070383,
+      "grad_norm": 55.3247140739336,
+      "learning_rate": 1.771378167997745e-08,
+      "logits": -0.788720428943634,
+      "logps": -78.25997161865234,
+      "loss": 0.389,
+      "objective": 0.2823149859905243,
+      "ranking_simple": 0.6875,
+      "step": 680
+    },
+    {
+      "dpo_loss": 0.30807915329933167,
+      "epoch": 1.9414265470004723,
+      "grad_norm": 42.581364917436595,
+      "learning_rate": 1.1106768097300657e-08,
+      "logits": -0.769125759601593,
+      "logps": -75.96378326416016,
+      "loss": 0.3183,
+      "objective": 0.30807915329933167,
+      "ranking_simple": 0.6083333492279053,
+      "step": 685
+    },
+    {
+      "dpo_loss": 0.21873274445533752,
+      "epoch": 1.9555975436939064,
+      "grad_norm": 37.39828456518223,
+      "learning_rate": 6.032310379642803e-09,
+      "logits": -0.7863041758537292,
+      "logps": -76.51734924316406,
+      "loss": 0.3202,
+      "objective": 0.21873274445533752,
+      "ranking_simple": 0.6708333492279053,
+      "step": 690
+    },
+    {
+      "dpo_loss": 0.24407407641410828,
+      "epoch": 1.9697685403873406,
+      "grad_norm": 75.42620047856866,
+      "learning_rate": 2.4935331648298644e-09,
+      "logits": -0.7426345944404602,
+      "logps": -77.34226989746094,
+      "loss": 0.3361,
+      "objective": 0.24407407641410828,
+      "ranking_simple": 0.625,
+      "step": 695
+    },
+    {
+      "dpo_loss": 0.2809065580368042,
+      "epoch": 1.9839395370807746,
+      "grad_norm": 63.61763495221277,
+      "learning_rate": 4.926154831655372e-10,
+      "logits": -0.6787734627723694,
+      "logps": -78.1474609375,
+      "loss": 0.3009,
+      "objective": 0.2809065580368042,
+      "ranking_simple": 0.6541666388511658,
+      "step": 700
+    },
+    {
+      "epoch": 1.9839395370807746,
+      "eval_dpo_loss": 1.8786609172821045,
+      "eval_logits": -0.850897490978241,
+      "eval_logps": -81.16839599609375,
+      "eval_loss": 1.9399291276931763,
+      "eval_objective": 1.8786609172821045,
+      "eval_ranking_simple": 0.534679114818573,
+      "eval_runtime": 605.8989,
+      "eval_samples_per_second": 9.556,
+      "eval_steps_per_second": 0.797,
+      "step": 700
+    },
+    {
+      "epoch": 1.995276334435522,
+      "step": 704,
+      "total_flos": 0.0,
+      "train_loss": 0.7561936149881645,
+      "train_runtime": 36938.8609,
+      "train_samples_per_second": 2.751,
+      "train_steps_per_second": 0.019
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 704,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6bba08646559c28cfce6c7b3860c6f96553c410db9f55571ebac85cbdab56839
+size 8120