Model save

README.md

@@ -0,0 +1,76 @@
+---
+license: apache-2.0
+base_model: hZzy/qwen2.5-0.5b-sft-news-IFT
+tags:
+- trl
+- expo
+- generated_from_trainer
+model-index:
+- name: qwen2.5-0.5b-expo-DPO-EXPERIMENT-1W
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/zhiyuzha-university-of-florida/huggingface/runs/vdo32atw)
+# qwen2.5-0.5b-expo-DPO-EXPERIMENT-1W
+
+This model is a fine-tuned version of [hZzy/qwen2.5-0.5b-sft-news-IFT](https://huggingface.co/hZzy/qwen2.5-0.5b-sft-news-IFT) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 12664.2070
+- Logps: -80.5751
+- Logits: -1.3556
+- Objective: 12926.7227
+- Dpo Loss: 12926.7227
+- Regularize: 12926.7227
+- Ranking Simple: 0.5227
+- Ranking Idealized: 0.5093
+- Ranking Idealized Expo: 0.5093
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-06
+- train_batch_size: 4
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 6
+- gradient_accumulation_steps: 12
+- total_train_batch_size: 288
+- total_eval_batch_size: 24
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss | Logps    | Logits  | Objective  | Dpo Loss   | Regularize | Ranking Simple | Ranking Idealized | Ranking Idealized Expo |
+|:-------------:|:------:|:----:|:---------------:|:--------:|:-------:|:----------:|:----------:|:----------:|:--------------:|:-----------------:|:----------------------:|
+| 10470.1625    | 0.2834 | 50   | 11322.9775      | -84.9886 | -1.4586 | 11815.3389 | 11815.3389 | 11815.3389 | 0.5165         | 0.5093            | 0.5093                 |
+| 8558.2836     | 0.5668 | 100  | 12877.2754      | -82.3385 | -1.3720 | 13248.5166 | 13248.5166 | 13248.5166 | 0.5176         | 0.5093            | 0.5093                 |
+| 7521.95       | 0.8503 | 150  | 12664.2070      | -80.5751 | -1.3556 | 12926.7227 | 12926.7227 | 12926.7227 | 0.5227         | 0.5093            | 0.5093                 |
+
+
+### Framework versions
+
+- Transformers 4.42.0
+- Pytorch 2.3.0+cu121
+- Datasets 2.19.1
+- Tokenizers 0.19.1

added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.997638167217761,
+    "total_flos": 0.0,
+    "train_loss": 8072.862625997175,
+    "train_runtime": 10876.2953,
+    "train_samples": 50802,
+    "train_samples_per_second": 4.671,
+    "train_steps_per_second": 0.016
+}

config.json

@@ -0,0 +1,30 @@
+{
+  "_name_or_path": "hZzy/qwen2.5-0.5b-sft-news-IFT",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151644,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 896,
+  "initializer_range": 0.02,
+  "intermediate_size": 4864,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 24,
+  "model_type": "qwen2",
+  "num_attention_heads": 14,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 2,
+  "pad_token_id": 151645,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": 32768,
+  "tie_word_embeddings": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.42.0",
+  "use_cache": false,
+  "use_mrope": false,
+  "use_sliding_window": false,
+  "vocab_size": 151665
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "bos_token_id": 151644,
+  "eos_token_id": 151645,
+  "max_new_tokens": 2048,
+  "pad_token_id": 151645,
+  "transformers_version": "4.42.0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e25a672a0099634dac3b7237a79c2dd5bce57c5d814f490409b8430f859fe07e
+size 1975192208

special_tokens_map.json

@@ -0,0 +1,27 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": {
+    "content": "<|im_start|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,196 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": "<|im_start|>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 2048,
+  "pad_token": "<|im_end|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.997638167217761,
+    "total_flos": 0.0,
+    "train_loss": 8072.862625997175,
+    "train_runtime": 10876.2953,
+    "train_samples": 50802,
+    "train_samples_per_second": 4.671,
+    "train_steps_per_second": 0.016
+}

trainer_state.json

@@ -0,0 +1,630 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.997638167217761,
+  "eval_steps": 50,
+  "global_step": 176,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "dpo_loss": 0.6931471824645996,
+      "epoch": 0.005668398677373642,
+      "grad_norm": 1341367.1350394282,
+      "learning_rate": 2.7777777777777776e-07,
+      "logits": -1.3147305250167847,
+      "logps": -88.0877456665039,
+      "loss": 0.6931,
+      "objective": 0.6931471824645996,
+      "ranking_idealized": 0.5208333134651184,
+      "ranking_idealized_expo": 0.5208333134651184,
+      "ranking_simple": 0.5208333134651184,
+      "regularize": 0.6931471824645996,
+      "step": 1
+    },
+    {
+      "dpo_loss": 1280.619140625,
+      "epoch": 0.02834199338686821,
+      "grad_norm": 2623988.3634039685,
+      "learning_rate": 1.3888888888888892e-06,
+      "logits": -1.3680986166000366,
+      "logps": -84.14160919189453,
+      "loss": 1112.7681,
+      "objective": 1280.619140625,
+      "ranking_idealized": 0.546875,
+      "ranking_idealized_expo": 0.546875,
+      "ranking_simple": 0.5520833134651184,
+      "regularize": 1280.619140625,
+      "step": 5
+    },
+    {
+      "dpo_loss": 2070.714599609375,
+      "epoch": 0.05668398677373642,
+      "grad_norm": 2330391.207726475,
+      "learning_rate": 2.7777777777777783e-06,
+      "logits": -1.45966374874115,
+      "logps": -83.2505111694336,
+      "loss": 2407.7941,
+      "objective": 2070.714599609375,
+      "ranking_idealized": 0.5083333253860474,
+      "ranking_idealized_expo": 0.5083333253860474,
+      "ranking_simple": 0.5166666507720947,
+      "regularize": 2070.714599609375,
+      "step": 10
+    },
+    {
+      "dpo_loss": 5679.9853515625,
+      "epoch": 0.08502598016060463,
+      "grad_norm": 1644564.4852830889,
+      "learning_rate": 4.166666666666667e-06,
+      "logits": -1.4328060150146484,
+      "logps": -82.17012023925781,
+      "loss": 4264.0652,
+      "objective": 5679.9853515625,
+      "ranking_idealized": 0.5041666626930237,
+      "ranking_idealized_expo": 0.5041666626930237,
+      "ranking_simple": 0.5166666507720947,
+      "regularize": 5679.9853515625,
+      "step": 15
+    },
+    {
+      "dpo_loss": 7525.20068359375,
+      "epoch": 0.11336797354747284,
+      "grad_norm": 1515613.7017780922,
+      "learning_rate": 4.998023493068255e-06,
+      "logits": -1.4038361310958862,
+      "logps": -82.86460876464844,
+      "loss": 6967.5602,
+      "objective": 7525.20068359375,
+      "ranking_idealized": 0.5375000238418579,
+      "ranking_idealized_expo": 0.5375000238418579,
+      "ranking_simple": 0.5583333373069763,
+      "regularize": 7525.20068359375,
+      "step": 20
+    },
+    {
+      "dpo_loss": 8746.1123046875,
+      "epoch": 0.14170996693434104,
+      "grad_norm": 1687436.4259306653,
+      "learning_rate": 4.975823666181256e-06,
+      "logits": -1.371711254119873,
+      "logps": -82.0951156616211,
+      "loss": 9527.1836,
+      "objective": 8746.1123046875,
+      "ranking_idealized": 0.4958333373069763,
+      "ranking_idealized_expo": 0.4958333373069763,
+      "ranking_simple": 0.512499988079071,
+      "regularize": 8746.1123046875,
+      "step": 25
+    },
+    {
+      "dpo_loss": 11099.513671875,
+      "epoch": 0.17005196032120926,
+      "grad_norm": 1294226.520732075,
+      "learning_rate": 4.929173350101025e-06,
+      "logits": -1.3348666429519653,
+      "logps": -81.38406372070312,
+      "loss": 10099.4609,
+      "objective": 11099.513671875,
+      "ranking_idealized": 0.5416666865348816,
+      "ranking_idealized_expo": 0.5416666865348816,
+      "ranking_simple": 0.5458333492279053,
+      "regularize": 11099.513671875,
+      "step": 30
+    },
+    {
+      "dpo_loss": 12696.7705078125,
+      "epoch": 0.19839395370807747,
+      "grad_norm": 1391829.2894247118,
+      "learning_rate": 4.858533249305337e-06,
+      "logits": -1.411891222000122,
+      "logps": -77.1667251586914,
+      "loss": 9874.5609,
+      "objective": 12696.7705078125,
+      "ranking_idealized": 0.5041666626930237,
+      "ranking_idealized_expo": 0.5041666626930237,
+      "ranking_simple": 0.5375000238418579,
+      "regularize": 12696.7705078125,
+      "step": 35
+    },
+    {
+      "dpo_loss": 8431.9521484375,
+      "epoch": 0.22673594709494568,
+      "grad_norm": 1186854.0601841968,
+      "learning_rate": 4.764600984163809e-06,
+      "logits": -1.3430523872375488,
+      "logps": -77.41344451904297,
+      "loss": 10212.3687,
+      "objective": 8431.9521484375,
+      "ranking_idealized": 0.5208333134651184,
+      "ranking_idealized_expo": 0.5208333134651184,
+      "ranking_simple": 0.5874999761581421,
+      "regularize": 8431.9521484375,
+      "step": 40
+    },
+    {
+      "dpo_loss": 11319.8056640625,
+      "epoch": 0.25507794048181387,
+      "grad_norm": 1072714.7178744846,
+      "learning_rate": 4.648304201449153e-06,
+      "logits": -1.3935389518737793,
+      "logps": -79.65689086914062,
+      "loss": 10434.6648,
+      "objective": 11319.8056640625,
+      "ranking_idealized": 0.5375000238418579,
+      "ranking_idealized_expo": 0.5375000238418579,
+      "ranking_simple": 0.574999988079071,
+      "regularize": 11319.8056640625,
+      "step": 45
+    },
+    {
+      "dpo_loss": 9569.685546875,
+      "epoch": 0.2834199338686821,
+      "grad_norm": 950921.657886262,
+      "learning_rate": 4.510791413176912e-06,
+      "logits": -1.3682667016983032,
+      "logps": -77.96815490722656,
+      "loss": 10470.1625,
+      "objective": 9569.685546875,
+      "ranking_idealized": 0.42500001192092896,
+      "ranking_idealized_expo": 0.42500001192092896,
+      "ranking_simple": 0.5,
+      "regularize": 9569.685546875,
+      "step": 50
+    },
+    {
+      "epoch": 0.2834199338686821,
+      "eval_dpo_loss": 11815.3388671875,
+      "eval_logits": -1.4585951566696167,
+      "eval_logps": -84.9886474609375,
+      "eval_loss": 11322.9775390625,
+      "eval_objective": 11815.3388671875,
+      "eval_ranking_idealized": 0.5092975497245789,
+      "eval_ranking_idealized_expo": 0.5092975497245789,
+      "eval_ranking_simple": 0.5165289044380188,
+      "eval_regularize": 11815.3388671875,
+      "eval_runtime": 1211.6497,
+      "eval_samples_per_second": 4.779,
+      "eval_steps_per_second": 0.2,
+      "step": 50
+    },
+    {
+      "dpo_loss": 12371.0048828125,
+      "epoch": 0.3117619272555503,
+      "grad_norm": 1109923.0216191527,
+      "learning_rate": 4.353420654246546e-06,
+      "logits": -1.4771044254302979,
+      "logps": -79.130126953125,
+      "loss": 11113.0828,
+      "objective": 12371.0048828125,
+      "ranking_idealized": 0.5416666865348816,
+      "ranking_idealized_expo": 0.5416666865348816,
+      "ranking_simple": 0.5541666746139526,
+      "regularize": 12371.0048828125,
+      "step": 55
+    },
+    {
+      "dpo_loss": 8734.4072265625,
+      "epoch": 0.3401039206424185,
+      "grad_norm": 1211359.9780672283,
+      "learning_rate": 4.177746070897593e-06,
+      "logits": -1.4155789613723755,
+      "logps": -77.57923126220703,
+      "loss": 11080.9961,
+      "objective": 8734.4072265625,
+      "ranking_idealized": 0.512499988079071,
+      "ranking_idealized_expo": 0.512499988079071,
+      "ranking_simple": 0.5708333253860474,
+      "regularize": 8734.4072265625,
+      "step": 60
+    },
+    {
+      "dpo_loss": 9739.5908203125,
+      "epoch": 0.3684459140292867,
+      "grad_norm": 1057255.008796819,
+      "learning_rate": 3.985502572429277e-06,
+      "logits": -1.3343302011489868,
+      "logps": -77.00284576416016,
+      "loss": 11711.2734,
+      "objective": 9739.5908203125,
+      "ranking_idealized": 0.5083333253860474,
+      "ranking_idealized_expo": 0.5083333253860474,
+      "ranking_simple": 0.5541666746139526,
+      "regularize": 9739.5908203125,
+      "step": 65
+    },
+    {
+      "dpo_loss": 9313.849609375,
+      "epoch": 0.39678790741615494,
+      "grad_norm": 1088725.4948180364,
+      "learning_rate": 3.7785886977585562e-06,
+      "logits": -1.3323314189910889,
+      "logps": -76.60472106933594,
+      "loss": 11694.0484,
+      "objective": 9313.849609375,
+      "ranking_idealized": 0.5166666507720947,
+      "ranking_idealized_expo": 0.5166666507720947,
+      "ranking_simple": 0.5416666865348816,
+      "regularize": 9313.849609375,
+      "step": 70
+    },
+    {
+      "dpo_loss": 10158.4462890625,
+      "epoch": 0.42512990080302315,
+      "grad_norm": 999614.3040187941,
+      "learning_rate": 3.5590478660213214e-06,
+      "logits": -1.2671582698822021,
+      "logps": -77.10884094238281,
+      "loss": 11305.65,
+      "objective": 10158.4462890625,
+      "ranking_idealized": 0.5458333492279053,
+      "ranking_idealized_expo": 0.5458333492279053,
+      "ranking_simple": 0.5916666388511658,
+      "regularize": 10158.4462890625,
+      "step": 75
+    },
+    {
+      "dpo_loss": 13367.1591796875,
+      "epoch": 0.45347189418989137,
+      "grad_norm": 1036547.2837553191,
+      "learning_rate": 3.32904819638017e-06,
+      "logits": -1.2884188890457153,
+      "logps": -76.65129852294922,
+      "loss": 11108.9961,
+      "objective": 13367.1591796875,
+      "ranking_idealized": 0.5166666507720947,
+      "ranking_idealized_expo": 0.5166666507720947,
+      "ranking_simple": 0.5583333373069763,
+      "regularize": 13367.1591796875,
+      "step": 80
+    },
+    {
+      "dpo_loss": 9063.92578125,
+      "epoch": 0.4818138875767596,
+      "grad_norm": 878439.4100295339,
+      "learning_rate": 3.090861096332263e-06,
+      "logits": -1.2684974670410156,
+      "logps": -75.37580108642578,
+      "loss": 9492.8891,
+      "objective": 9063.92578125,
+      "ranking_idealized": 0.5249999761581421,
+      "ranking_idealized_expo": 0.5249999761581421,
+      "ranking_simple": 0.5916666388511658,
+      "regularize": 9063.92578125,
+      "step": 85
+    },
+    {
+      "dpo_loss": 12140.421875,
+      "epoch": 0.5101558809636277,
+      "grad_norm": 839896.1614839052,
+      "learning_rate": 2.8468388299726714e-06,
+      "logits": -1.1826907396316528,
+      "logps": -77.36532592773438,
+      "loss": 8699.0336,
+      "objective": 12140.421875,
+      "ranking_idealized": 0.4791666567325592,
+      "ranking_idealized_expo": 0.4791666567325592,
+      "ranking_simple": 0.5166666507720947,
+      "regularize": 12140.421875,
+      "step": 90
+    },
+    {
+      "dpo_loss": 7882.3046875,
+      "epoch": 0.538497874350496,
+      "grad_norm": 1169515.5696928622,
+      "learning_rate": 2.599391287742315e-06,
+      "logits": -1.2656235694885254,
+      "logps": -76.93645477294922,
+      "loss": 8873.2109,
+      "objective": 7882.3046875,
+      "ranking_idealized": 0.5416666865348816,
+      "ranking_idealized_expo": 0.5416666865348816,
+      "ranking_simple": 0.6166666746139526,
+      "regularize": 7882.3046875,
+      "step": 95
+    },
+    {
+      "dpo_loss": 7251.060546875,
+      "epoch": 0.5668398677373642,
+      "grad_norm": 1006084.9040205956,
+      "learning_rate": 2.3509621870754505e-06,
+      "logits": -1.342369794845581,
+      "logps": -78.5166244506836,
+      "loss": 8558.2836,
+      "objective": 7251.060546875,
+      "ranking_idealized": 0.5416666865348816,
+      "ranking_idealized_expo": 0.5416666865348816,
+      "ranking_simple": 0.6041666865348816,
+      "regularize": 7251.060546875,
+      "step": 100
+    },
+    {
+      "epoch": 0.5668398677373642,
+      "eval_dpo_loss": 13248.5166015625,
+      "eval_logits": -1.3720418214797974,
+      "eval_logps": -82.3385238647461,
+      "eval_loss": 12877.275390625,
+      "eval_objective": 13248.5166015625,
+      "eval_ranking_idealized": 0.5092975497245789,
+      "eval_ranking_idealized_expo": 0.5092975497245789,
+      "eval_ranking_simple": 0.5175619721412659,
+      "eval_regularize": 13248.5166015625,
+      "eval_runtime": 258.8572,
+      "eval_samples_per_second": 22.368,
+      "eval_steps_per_second": 0.935,
+      "step": 100
+    },
+    {
+      "dpo_loss": 9016.806640625,
+      "epoch": 0.5951818611242324,
+      "grad_norm": 861666.5853674684,
+      "learning_rate": 2.1040049389819628e-06,
+      "logits": -1.238232970237732,
+      "logps": -78.26189422607422,
+      "loss": 8298.018,
+      "objective": 9016.806640625,
+      "ranking_idealized": 0.5333333611488342,
+      "ranking_idealized_expo": 0.5333333611488342,
+      "ranking_simple": 0.612500011920929,
+      "regularize": 9016.806640625,
+      "step": 105
+    },
+    {
+      "dpo_loss": 7970.41748046875,
+      "epoch": 0.6235238545111006,
+      "grad_norm": 865124.6869036607,
+      "learning_rate": 1.8609584188988135e-06,
+      "logits": -1.330941081047058,
+      "logps": -76.76597595214844,
+      "loss": 7414.1281,
+      "objective": 7970.41748046875,
+      "ranking_idealized": 0.44999998807907104,
+      "ranking_idealized_expo": 0.44999998807907104,
+      "ranking_simple": 0.5416666865348816,
+      "regularize": 7970.41748046875,
+      "step": 110
+    },
+    {
+      "dpo_loss": 9534.12890625,
+      "epoch": 0.6518658478979689,
+      "grad_norm": 840576.3563797984,
+      "learning_rate": 1.6242228810904393e-06,
+      "logits": -1.3068194389343262,
+      "logps": -76.78305053710938,
+      "loss": 8769.2398,
+      "objective": 9534.12890625,
+      "ranking_idealized": 0.4833333194255829,
+      "ranking_idealized_expo": 0.4833333194255829,
+      "ranking_simple": 0.5458333492279053,
+      "regularize": 9534.12890625,
+      "step": 115
+    },
+    {
+      "dpo_loss": 8552.21484375,
+      "epoch": 0.680207841284837,
+      "grad_norm": 794346.4700274817,
+      "learning_rate": 1.3961362544602215e-06,
+      "logits": -1.2461904287338257,
+      "logps": -77.25305938720703,
+      "loss": 7377.8172,
+      "objective": 8552.21484375,
+      "ranking_idealized": 0.5333333611488342,
+      "ranking_idealized_expo": 0.5333333611488342,
+      "ranking_simple": 0.5874999761581421,
+      "regularize": 8552.21484375,
+      "step": 120
+    },
+    {
+      "dpo_loss": 9080.310546875,
+      "epoch": 0.7085498346717053,
+      "grad_norm": 866794.9526252571,
+      "learning_rate": 1.1789510538684524e-06,
+      "logits": -1.2470247745513916,
+      "logps": -75.66565704345703,
+      "loss": 7754.6125,
+      "objective": 9080.310546875,
+      "ranking_idealized": 0.550000011920929,
+      "ranking_idealized_expo": 0.550000011920929,
+      "ranking_simple": 0.6000000238418579,
+      "regularize": 9080.310546875,
+      "step": 125
+    },
+    {
+      "dpo_loss": 7816.40869140625,
+      "epoch": 0.7368918280585735,
+      "grad_norm": 914484.5347620883,
+      "learning_rate": 9.74812134973689e-07,
+      "logits": -1.362328290939331,
+      "logps": -74.44100952148438,
+      "loss": 7071.1984,
+      "objective": 7816.40869140625,
+      "ranking_idealized": 0.44999998807907104,
+      "ranking_idealized_expo": 0.44999998807907104,
+      "ranking_simple": 0.5041666626930237,
+      "regularize": 7816.40869140625,
+      "step": 130
+    },
+    {
+      "dpo_loss": 6297.37939453125,
+      "epoch": 0.7652338214454416,
+      "grad_norm": 908053.7687284376,
+      "learning_rate": 7.857355122839674e-07,
+      "logits": -1.3475323915481567,
+      "logps": -76.51807403564453,
+      "loss": 7334.7656,
+      "objective": 6297.37939453125,
+      "ranking_idealized": 0.5291666388511658,
+      "ranking_idealized_expo": 0.5291666388511658,
+      "ranking_simple": 0.5833333134651184,
+      "regularize": 6297.37939453125,
+      "step": 135
+    },
+    {
+      "dpo_loss": 5713.158203125,
+      "epoch": 0.7935758148323099,
+      "grad_norm": 765346.0794896972,
+      "learning_rate": 6.135884496044245e-07,
+      "logits": -1.2585749626159668,
+      "logps": -75.7166519165039,
+      "loss": 6126.0992,
+      "objective": 5713.158203125,
+      "ranking_idealized": 0.5041666626930237,
+      "ranking_idealized_expo": 0.5041666626930237,
+      "ranking_simple": 0.5708333253860474,
+      "regularize": 5713.158203125,
+      "step": 140
+    },
+    {
+      "dpo_loss": 6023.36669921875,
+      "epoch": 0.821917808219178,
+      "grad_norm": 864232.6098968883,
+      "learning_rate": 4.6007101950209827e-07,
+      "logits": -1.311624526977539,
+      "logps": -75.54283905029297,
+      "loss": 6440.6867,
+      "objective": 6023.36669921875,
+      "ranking_idealized": 0.4833333194255829,
+      "ranking_idealized_expo": 0.4833333194255829,
+      "ranking_simple": 0.5333333611488342,
+      "regularize": 6023.36669921875,
+      "step": 145
+    },
+    {
+      "dpo_loss": 9931.4228515625,
+      "epoch": 0.8502598016060463,
+      "grad_norm": 789706.2833060493,
+      "learning_rate": 3.266993139010438e-07,
+      "logits": -1.3248921632766724,
+      "logps": -73.5119857788086,
+      "loss": 7521.95,
+      "objective": 9931.4228515625,
+      "ranking_idealized": 0.5375000238418579,
+      "ranking_idealized_expo": 0.5375000238418579,
+      "ranking_simple": 0.5833333134651184,
+      "regularize": 9931.4228515625,
+      "step": 150
+    },
+    {
+      "epoch": 0.8502598016060463,
+      "eval_dpo_loss": 12926.72265625,
+      "eval_logits": -1.3555928468704224,
+      "eval_logps": -80.5750961303711,
+      "eval_loss": 12664.20703125,
+      "eval_objective": 12926.72265625,
+      "eval_ranking_idealized": 0.5092975497245789,
+      "eval_ranking_idealized_expo": 0.5092975497245789,
+      "eval_ranking_simple": 0.5227272510528564,
+      "eval_regularize": 12926.72265625,
+      "eval_runtime": 258.9258,
+      "eval_samples_per_second": 22.362,
+      "eval_steps_per_second": 0.935,
+      "step": 150
+    },
+    {
+      "dpo_loss": 5853.70361328125,
+      "epoch": 0.8786017949929145,
+      "grad_norm": 805936.4979433398,
+      "learning_rate": 2.1479047161491351e-07,
+      "logits": -1.3135905265808105,
+      "logps": -73.38835144042969,
+      "loss": 5683.3855,
+      "objective": 5853.70361328125,
+      "ranking_idealized": 0.512499988079071,
+      "ranking_idealized_expo": 0.512499988079071,
+      "ranking_simple": 0.612500011920929,
+      "regularize": 5853.70361328125,
+      "step": 155
+    },
+    {
+      "dpo_loss": 6223.60693359375,
+      "epoch": 0.9069437883797827,
+      "grad_norm": 771613.5624330686,
+      "learning_rate": 1.2544967068054332e-07,
+      "logits": -1.2275891304016113,
+      "logps": -74.46540069580078,
+      "loss": 6799.2719,
+      "objective": 6223.60693359375,
+      "ranking_idealized": 0.4958333373069763,
+      "ranking_idealized_expo": 0.4958333373069763,
+      "ranking_simple": 0.574999988079071,
+      "regularize": 6223.60693359375,
+      "step": 160
+    },
+    {
+      "dpo_loss": 4967.171875,
+      "epoch": 0.9352857817666509,
+      "grad_norm": 1004063.6300908878,
+      "learning_rate": 5.955921395237318e-08,
+      "logits": -1.2412575483322144,
+      "logps": -75.6294174194336,
+      "loss": 6197.7492,
+      "objective": 4967.171875,
+      "ranking_idealized": 0.5208333134651184,
+      "ranking_idealized_expo": 0.5208333134651184,
+      "ranking_simple": 0.5958333611488342,
+      "regularize": 4967.171875,
+      "step": 165
+    },
+    {
+      "dpo_loss": 7402.99267578125,
+      "epoch": 0.9636277751535192,
+      "grad_norm": 778643.4373397075,
+      "learning_rate": 1.7769815745066476e-08,
+      "logits": -1.2604658603668213,
+      "logps": -75.88407135009766,
+      "loss": 6623.1367,
+      "objective": 7402.99267578125,
+      "ranking_idealized": 0.5083333253860474,
+      "ranking_idealized_expo": 0.5083333253860474,
+      "ranking_simple": 0.5666666626930237,
+      "regularize": 7402.99267578125,
+      "step": 170
+    },
+    {
+      "dpo_loss": 5726.1806640625,
+      "epoch": 0.9919697685403873,
+      "grad_norm": 762884.4185589707,
+      "learning_rate": 4.941755748361088e-10,
+      "logits": -1.2217018604278564,
+      "logps": -76.01050567626953,
+      "loss": 5562.8855,
+      "objective": 5726.1806640625,
+      "ranking_idealized": 0.574999988079071,
+      "ranking_idealized_expo": 0.574999988079071,
+      "ranking_simple": 0.6333333253860474,
+      "regularize": 5726.1806640625,
+      "step": 175
+    },
+    {
+      "epoch": 0.997638167217761,
+      "step": 176,
+      "total_flos": 0.0,
+      "train_loss": 8072.862625997175,
+      "train_runtime": 10876.2953,
+      "train_samples_per_second": 4.671,
+      "train_steps_per_second": 0.016
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 176,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b8198cad37243392c8c785ac399571ba23fcba30addbad1b0966add4517521a
+size 8120