Hugging Face's logo

hZzy
/
qwen2.5-0.5b-expo-L1EXPO-ES-1

Safetensors
qwen2
alignment-handbook
ndcg
trl
expo
Generated from Trainer
Model card Files Community
hZzy commited on
Commit
cad4783
•
1 Parent(s): 1677e49

Model save

Browse files
Files changed (6) hide show
  1. README.md +88 -0
  2. all_results.json +9 -0
  3. generation_config.json +7 -0
  4. model.safetensors +1 -1
  5. train_results.json +9 -0
  6. trainer_state.json +529 -0
README.md ADDED
@@ -0,0 +1,88 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ base_model: hZzy/qwen2.5-0.5b-sft-news-IFT
4
+ tags:
5
+ - trl
6
+ - expo
7
+ - generated_from_trainer
8
+ model-index:
9
+ - name: qwen2.5-0.5b-expo-L1EXPO-ES-1
10
+ results: []
11
+ ---
12
+
13
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
14
+ should probably proofread and complete it, then remove this comment. -->
15
+
16
+ [<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/zhiyuzha-university-of-florida/huggingface/runs/3epxudpc)
17
+ # qwen2.5-0.5b-expo-L1EXPO-ES-1
18
+
19
+ This model is a fine-tuned version of [hZzy/qwen2.5-0.5b-sft-news-IFT](https://huggingface.co/hZzy/qwen2.5-0.5b-sft-news-IFT) on an unknown dataset.
20
+ It achieves the following results on the evaluation set:
21
+ - Loss: 4.9659
22
+ - Logps: -79.7376
23
+ - Logits: -0.5594
24
+ - Objective: 4.9951
25
+ - Dpo Loss: 2.6292
26
+ - Regularize: 4.9951
27
+ - Ranking Simple: 0.5212
28
+ - Ranking Idealized: 0.5295
29
+ - Ranking Idealized Expo: 0.5212
30
+ - Wo Beta: 14.1755
31
+
32
+ ## Model description
33
+
34
+ More information needed
35
+
36
+ ## Intended uses & limitations
37
+
38
+ More information needed
39
+
40
+ ## Training and evaluation data
41
+
42
+ More information needed
43
+
44
+ ## Training procedure
45
+
46
+ ### Training hyperparameters
47
+
48
+ The following hyperparameters were used during training:
49
+ - learning_rate: 5e-06
50
+ - train_batch_size: 4
51
+ - eval_batch_size: 4
52
+ - seed: 42
53
+ - distributed_type: multi-GPU
54
+ - num_devices: 3
55
+ - gradient_accumulation_steps: 12
56
+ - total_train_batch_size: 144
57
+ - total_eval_batch_size: 12
58
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
59
+ - lr_scheduler_type: cosine
60
+ - lr_scheduler_warmup_ratio: 0.1
61
+ - num_epochs: 5
62
+
63
+ ### Training results
64
+
65
+ | Training Loss | Epoch | Step | Validation Loss | Logps | Logits | Objective | Dpo Loss | Regularize | Ranking Simple | Ranking Idealized | Ranking Idealized Expo | Wo Beta |
66
+ |:-------------:|:------:|:----:|:---------------:|:--------:|:-------:|:---------:|:--------:|:----------:|:--------------:|:-----------------:|:----------------------:|:-------:|
67
+ | 0.4306 | 0.1417 | 50 | 0.5493 | -90.4264 | -1.4289 | 0.5433 | 0.7632 | 0.5433 | 0.5212 | 0.5295 | 0.5212 | 16.2237 |
68
+ | 1.748 | 0.2834 | 100 | 1.6975 | -88.0491 | -1.2535 | 1.6864 | 1.1354 | 1.6864 | 0.5228 | 0.5295 | 0.5212 | 15.6834 |
69
+ | 2.8697 | 0.4251 | 150 | 2.9624 | -82.4967 | -1.2524 | 2.8923 | 1.6846 | 2.8923 | 0.5243 | 0.5295 | 0.5212 | 15.1970 |
70
+ | 3.5268 | 0.5668 | 200 | 4.0302 | -75.9716 | -0.9581 | 3.9597 | 2.1590 | 3.9597 | 0.5238 | 0.5295 | 0.5212 | 14.5792 |
71
+ | 3.7241 | 0.7085 | 250 | 4.2694 | -81.3047 | -0.7680 | 4.2728 | 2.3310 | 4.2728 | 0.5259 | 0.5295 | 0.5212 | 14.5615 |
72
+ | 3.6109 | 0.8503 | 300 | 4.4908 | -83.9815 | -0.6388 | 4.4573 | 2.4072 | 4.4573 | 0.5264 | 0.5295 | 0.5212 | 14.3464 |
73
+ | 3.36 | 0.9920 | 350 | 4.6586 | -80.7491 | -0.5030 | 4.6212 | 2.4991 | 4.6212 | 0.5212 | 0.5295 | 0.5212 | 14.3467 |
74
+ | 3.112 | 1.1337 | 400 | 4.7244 | -82.4974 | -0.5664 | 4.7293 | 2.5403 | 4.7293 | 0.5186 | 0.5295 | 0.5212 | 14.4038 |
75
+ | 2.9448 | 1.2754 | 450 | 4.8354 | -80.1753 | -0.6936 | 4.8114 | 2.5735 | 4.8114 | 0.5248 | 0.5295 | 0.5212 | 13.9356 |
76
+ | 2.8517 | 1.4171 | 500 | 5.0044 | -80.7676 | -0.5973 | 5.0058 | 2.6782 | 5.0058 | 0.5269 | 0.5295 | 0.5212 | 14.2626 |
77
+ | 2.632 | 1.5588 | 550 | 4.8777 | -80.5219 | -0.6149 | 4.8844 | 2.5752 | 4.8844 | 0.5223 | 0.5295 | 0.5212 | 14.1469 |
78
+ | 2.5208 | 1.7005 | 600 | 4.9258 | -80.1775 | -0.5875 | 4.9621 | 2.5974 | 4.9621 | 0.5243 | 0.5295 | 0.5212 | 14.2669 |
79
+ | 2.4198 | 1.8422 | 650 | 5.0327 | -81.0550 | -0.5441 | 5.0454 | 2.6345 | 5.0454 | 0.5269 | 0.5295 | 0.5212 | 14.2479 |
80
+ | 2.2699 | 1.9839 | 700 | 4.9659 | -79.7376 | -0.5594 | 4.9951 | 2.6292 | 4.9951 | 0.5212 | 0.5295 | 0.5212 | 14.1755 |
81
+
82
+
83
+ ### Framework versions
84
+
85
+ - Transformers 4.42.0
86
+ - Pytorch 2.3.0+cu121
87
+ - Datasets 3.2.0
88
+ - Tokenizers 0.19.1
all_results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 1.9839395370807746,
3
+ "total_flos": 0.0,
4
+ "train_loss": 2.7151880131661894,
5
+ "train_runtime": 19357.2061,
6
+ "train_samples": 50802,
7
+ "train_samples_per_second": 13.122,
8
+ "train_steps_per_second": 0.091
9
+ }
generation_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 151644,
3
+ "eos_token_id": 151645,
4
+ "max_new_tokens": 2048,
5
+ "pad_token_id": 151645,
6
+ "transformers_version": "4.42.0"
7
+ }
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:c0a543918c8313be3148b1e4155e0527a6f880153f4fa1cb66a933e8c3c1f111
3
  size 1975192208
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3fdbcb5bbd340aba62603d5a1ba7bc4a76f860e5763cc95de9a9740474665e85
3
  size 1975192208
train_results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 1.9839395370807746,
3
+ "total_flos": 0.0,
4
+ "train_loss": 2.7151880131661894,
5
+ "train_runtime": 19357.2061,
6
+ "train_samples": 50802,
7
+ "train_samples_per_second": 13.122,
8
+ "train_steps_per_second": 0.091
9
+ }
trainer_state.json ADDED
@@ -0,0 +1,529 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": 13.93560791015625,
3
+ "best_model_checkpoint": "./qwen2.5-0.5b/qwen2.5-0.5b-expo-L1EXPO-ES-1/checkpoint-450",
4
+ "epoch": 1.9839395370807746,
5
+ "eval_steps": 50,
6
+ "global_step": 700,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "dpo_loss": 0.6931471824645996,
13
+ "epoch": 0.002834199338686821,
14
+ "grad_norm": 368.8500800599453,
15
+ "learning_rate": 2.840909090909091e-08,
16
+ "logits": -1.359458565711975,
17
+ "logps": -84.69721221923828,
18
+ "loss": 0.0051,
19
+ "objective": 0.0046141319908201694,
20
+ "ranking_idealized": 0.5833333134651184,
21
+ "ranking_idealized_expo": 0.5833333134651184,
22
+ "ranking_simple": 0.5833333134651184,
23
+ "regularize": 0.0046141319908201694,
24
+ "step": 1,
25
+ "wo_beta": 14.840873718261719
26
+ },
27
+ {
28
+ "dpo_loss": 0.7411907315254211,
29
+ "epoch": 0.14170996693434104,
30
+ "grad_norm": 326.1369710639882,
31
+ "learning_rate": 1.4204545454545458e-06,
32
+ "logits": -1.4541096687316895,
33
+ "logps": -84.35031127929688,
34
+ "loss": 0.4306,
35
+ "objective": 0.4069998860359192,
36
+ "ranking_idealized": 0.5289115905761719,
37
+ "ranking_idealized_expo": 0.5221088528633118,
38
+ "ranking_simple": 0.5225340127944946,
39
+ "regularize": 0.4069998860359192,
40
+ "step": 50,
41
+ "wo_beta": 15.642387390136719
42
+ },
43
+ {
44
+ "epoch": 0.14170996693434104,
45
+ "eval_dpo_loss": 0.7632195949554443,
46
+ "eval_logits": -1.428945779800415,
47
+ "eval_logps": -90.4263687133789,
48
+ "eval_loss": 0.5493111610412598,
49
+ "eval_objective": 0.5433010458946228,
50
+ "eval_ranking_idealized": 0.5295031070709229,
51
+ "eval_ranking_idealized_expo": 0.5212215185165405,
52
+ "eval_ranking_simple": 0.5212215185165405,
53
+ "eval_regularize": 0.5433010458946228,
54
+ "eval_runtime": 308.9459,
55
+ "eval_samples_per_second": 18.741,
56
+ "eval_steps_per_second": 1.563,
57
+ "eval_wo_beta": 16.223697662353516,
58
+ "step": 50
59
+ },
60
+ {
61
+ "dpo_loss": 1.189667820930481,
62
+ "epoch": 0.2834199338686821,
63
+ "grad_norm": 271.670123611671,
64
+ "learning_rate": 2.8409090909090916e-06,
65
+ "logits": -1.3798158168792725,
66
+ "logps": -82.86579895019531,
67
+ "loss": 1.748,
68
+ "objective": 1.777085542678833,
69
+ "ranking_idealized": 0.5241666436195374,
70
+ "ranking_idealized_expo": 0.5137500166893005,
71
+ "ranking_simple": 0.5174999833106995,
72
+ "regularize": 1.777085542678833,
73
+ "step": 100,
74
+ "wo_beta": 15.245094299316406
75
+ },
76
+ {
77
+ "epoch": 0.2834199338686821,
78
+ "eval_dpo_loss": 1.1354432106018066,
79
+ "eval_logits": -1.2535114288330078,
80
+ "eval_logps": -88.04913330078125,
81
+ "eval_loss": 1.6975041627883911,
82
+ "eval_objective": 1.6864262819290161,
83
+ "eval_ranking_idealized": 0.5295031070709229,
84
+ "eval_ranking_idealized_expo": 0.5212215185165405,
85
+ "eval_ranking_simple": 0.522774338722229,
86
+ "eval_regularize": 1.6864262819290161,
87
+ "eval_runtime": 308.5261,
88
+ "eval_samples_per_second": 18.767,
89
+ "eval_steps_per_second": 1.566,
90
+ "eval_wo_beta": 15.683449745178223,
91
+ "step": 100
92
+ },
93
+ {
94
+ "dpo_loss": 1.6895211935043335,
95
+ "epoch": 0.42512990080302315,
96
+ "grad_norm": 221.9711378891906,
97
+ "learning_rate": 4.2613636363636365e-06,
98
+ "logits": -1.2630772590637207,
99
+ "logps": -79.18020629882812,
100
+ "loss": 2.8697,
101
+ "objective": 2.862943172454834,
102
+ "ranking_idealized": 0.5333333611488342,
103
+ "ranking_idealized_expo": 0.527916669845581,
104
+ "ranking_simple": 0.5199999809265137,
105
+ "regularize": 2.862943172454834,
106
+ "step": 150,
107
+ "wo_beta": 15.206230163574219
108
+ },
109
+ {
110
+ "epoch": 0.42512990080302315,
111
+ "eval_dpo_loss": 1.6845829486846924,
112
+ "eval_logits": -1.252400279045105,
113
+ "eval_logps": -82.49666595458984,
114
+ "eval_loss": 2.9623751640319824,
115
+ "eval_objective": 2.892326593399048,
116
+ "eval_ranking_idealized": 0.5295031070709229,
117
+ "eval_ranking_idealized_expo": 0.5212215185165405,
118
+ "eval_ranking_simple": 0.5243270993232727,
119
+ "eval_regularize": 2.892326593399048,
120
+ "eval_runtime": 308.7897,
121
+ "eval_samples_per_second": 18.751,
122
+ "eval_steps_per_second": 1.564,
123
+ "eval_wo_beta": 15.196976661682129,
124
+ "step": 150
125
+ },
126
+ {
127
+ "dpo_loss": 2.04150390625,
128
+ "epoch": 0.5668398677373642,
129
+ "grad_norm": 196.4932514581971,
130
+ "learning_rate": 4.997168347957521e-06,
131
+ "logits": -1.0472546815872192,
132
+ "logps": -75.09263610839844,
133
+ "loss": 3.5268,
134
+ "objective": 3.5377087593078613,
135
+ "ranking_idealized": 0.5204166769981384,
136
+ "ranking_idealized_expo": 0.51541668176651,
137
+ "ranking_simple": 0.5066666603088379,
138
+ "regularize": 3.5377087593078613,
139
+ "step": 200,
140
+ "wo_beta": 15.402359962463379
141
+ },
142
+ {
143
+ "epoch": 0.5668398677373642,
144
+ "eval_dpo_loss": 2.158970594406128,
145
+ "eval_logits": -0.9581480622291565,
146
+ "eval_logps": -75.9715576171875,
147
+ "eval_loss": 4.030220031738281,
148
+ "eval_objective": 3.9596574306488037,
149
+ "eval_ranking_idealized": 0.5295031070709229,
150
+ "eval_ranking_idealized_expo": 0.5212215185165405,
151
+ "eval_ranking_simple": 0.523809552192688,
152
+ "eval_regularize": 3.9596574306488037,
153
+ "eval_runtime": 308.8005,
154
+ "eval_samples_per_second": 18.75,
155
+ "eval_steps_per_second": 1.564,
156
+ "eval_wo_beta": 14.579182624816895,
157
+ "step": 200
158
+ },
159
+ {
160
+ "dpo_loss": 2.0967283248901367,
161
+ "epoch": 0.7085498346717053,
162
+ "grad_norm": 186.71108857655122,
163
+ "learning_rate": 4.973122855144066e-06,
164
+ "logits": -0.8511989116668701,
165
+ "logps": -75.66019439697266,
166
+ "loss": 3.7241,
167
+ "objective": 3.6993343830108643,
168
+ "ranking_idealized": 0.5249999761581421,
169
+ "ranking_idealized_expo": 0.5162500143051147,
170
+ "ranking_simple": 0.5104166865348816,
171
+ "regularize": 3.6993343830108643,
172
+ "step": 250,
173
+ "wo_beta": 15.4879789352417
174
+ },
175
+ {
176
+ "epoch": 0.7085498346717053,
177
+ "eval_dpo_loss": 2.3309643268585205,
178
+ "eval_logits": -0.7680178880691528,
179
+ "eval_logps": -81.3046875,
180
+ "eval_loss": 4.269361972808838,
181
+ "eval_objective": 4.272838115692139,
182
+ "eval_ranking_idealized": 0.5295031070709229,
183
+ "eval_ranking_idealized_expo": 0.5212215185165405,
184
+ "eval_ranking_simple": 0.5258799195289612,
185
+ "eval_regularize": 4.272838115692139,
186
+ "eval_runtime": 308.4403,
187
+ "eval_samples_per_second": 18.772,
188
+ "eval_steps_per_second": 1.566,
189
+ "eval_wo_beta": 14.561494827270508,
190
+ "step": 250
191
+ },
192
+ {
193
+ "dpo_loss": 1.9671478271484375,
194
+ "epoch": 0.8502598016060463,
195
+ "grad_norm": 175.55668911243956,
196
+ "learning_rate": 4.924776641419513e-06,
197
+ "logits": -0.6268683671951294,
198
+ "logps": -79.04060363769531,
199
+ "loss": 3.6109,
200
+ "objective": 3.6469531059265137,
201
+ "ranking_idealized": 0.5062500238418579,
202
+ "ranking_idealized_expo": 0.4950000047683716,
203
+ "ranking_simple": 0.5049999952316284,
204
+ "regularize": 3.6469531059265137,
205
+ "step": 300,
206
+ "wo_beta": 15.15662956237793
207
+ },
208
+ {
209
+ "epoch": 0.8502598016060463,
210
+ "eval_dpo_loss": 2.4072353839874268,
211
+ "eval_logits": -0.6388247609138489,
212
+ "eval_logps": -83.98152923583984,
213
+ "eval_loss": 4.49080228805542,
214
+ "eval_objective": 4.45733642578125,
215
+ "eval_ranking_idealized": 0.5295031070709229,
216
+ "eval_ranking_idealized_expo": 0.5212215185165405,
217
+ "eval_ranking_simple": 0.5263975262641907,
218
+ "eval_regularize": 4.45733642578125,
219
+ "eval_runtime": 308.8236,
220
+ "eval_samples_per_second": 18.749,
221
+ "eval_steps_per_second": 1.564,
222
+ "eval_wo_beta": 14.346380233764648,
223
+ "step": 300
224
+ },
225
+ {
226
+ "dpo_loss": 1.8654496669769287,
227
+ "epoch": 0.9919697685403873,
228
+ "grad_norm": 165.9769670968443,
229
+ "learning_rate": 4.8526047530778175e-06,
230
+ "logits": -0.5482619404792786,
231
+ "logps": -77.96940612792969,
232
+ "loss": 3.36,
233
+ "objective": 3.306919574737549,
234
+ "ranking_idealized": 0.5354166626930237,
235
+ "ranking_idealized_expo": 0.5254166722297668,
236
+ "ranking_simple": 0.5224999785423279,
237
+ "regularize": 3.306919574737549,
238
+ "step": 350,
239
+ "wo_beta": 15.03686237335205
240
+ },
241
+ {
242
+ "epoch": 0.9919697685403873,
243
+ "eval_dpo_loss": 2.499128580093384,
244
+ "eval_logits": -0.5030297636985779,
245
+ "eval_logps": -80.74913024902344,
246
+ "eval_loss": 4.658600807189941,
247
+ "eval_objective": 4.6211676597595215,
248
+ "eval_ranking_idealized": 0.5295031070709229,
249
+ "eval_ranking_idealized_expo": 0.5212215185165405,
250
+ "eval_ranking_simple": 0.5212215185165405,
251
+ "eval_regularize": 4.6211676597595215,
252
+ "eval_runtime": 314.3306,
253
+ "eval_samples_per_second": 18.42,
254
+ "eval_steps_per_second": 1.537,
255
+ "eval_wo_beta": 14.346673965454102,
256
+ "step": 350
257
+ },
258
+ {
259
+ "dpo_loss": 1.732424259185791,
260
+ "epoch": 1.1336797354747283,
261
+ "grad_norm": 163.81688300949705,
262
+ "learning_rate": 4.757316345716554e-06,
263
+ "logits": -0.5219219326972961,
264
+ "logps": -77.66575622558594,
265
+ "loss": 3.112,
266
+ "objective": 3.121299982070923,
267
+ "ranking_idealized": 0.5412499904632568,
268
+ "ranking_idealized_expo": 0.5320833325386047,
269
+ "ranking_simple": 0.53125,
270
+ "regularize": 3.121299982070923,
271
+ "step": 400,
272
+ "wo_beta": 15.248611450195312
273
+ },
274
+ {
275
+ "epoch": 1.1336797354747283,
276
+ "eval_dpo_loss": 2.5402870178222656,
277
+ "eval_logits": -0.5664270520210266,
278
+ "eval_logps": -82.49735260009766,
279
+ "eval_loss": 4.724371433258057,
280
+ "eval_objective": 4.729303359985352,
281
+ "eval_ranking_idealized": 0.5295031070709229,
282
+ "eval_ranking_idealized_expo": 0.5212215185165405,
283
+ "eval_ranking_simple": 0.5186335444450378,
284
+ "eval_regularize": 4.729303359985352,
285
+ "eval_runtime": 310.1606,
286
+ "eval_samples_per_second": 18.668,
287
+ "eval_steps_per_second": 1.557,
288
+ "eval_wo_beta": 14.403800964355469,
289
+ "step": 400
290
+ },
291
+ {
292
+ "dpo_loss": 1.6898671388626099,
293
+ "epoch": 1.2753897024090695,
294
+ "grad_norm": 190.1252094592616,
295
+ "learning_rate": 4.639847716126855e-06,
296
+ "logits": -0.5678730010986328,
297
+ "logps": -78.37519836425781,
298
+ "loss": 2.9448,
299
+ "objective": 3.0088882446289062,
300
+ "ranking_idealized": 0.5245833396911621,
301
+ "ranking_idealized_expo": 0.5191666483879089,
302
+ "ranking_simple": 0.518750011920929,
303
+ "regularize": 3.0088882446289062,
304
+ "step": 450,
305
+ "wo_beta": 15.990630149841309
306
+ },
307
+ {
308
+ "epoch": 1.2753897024090695,
309
+ "eval_dpo_loss": 2.573472738265991,
310
+ "eval_logits": -0.6935593485832214,
311
+ "eval_logps": -80.17532348632812,
312
+ "eval_loss": 4.835423946380615,
313
+ "eval_objective": 4.811424732208252,
314
+ "eval_ranking_idealized": 0.5295031070709229,
315
+ "eval_ranking_idealized_expo": 0.5212215185165405,
316
+ "eval_ranking_simple": 0.5248447060585022,
317
+ "eval_regularize": 4.811424732208252,
318
+ "eval_runtime": 308.2089,
319
+ "eval_samples_per_second": 18.786,
320
+ "eval_steps_per_second": 1.567,
321
+ "eval_wo_beta": 13.93560791015625,
322
+ "step": 450
323
+ },
324
+ {
325
+ "dpo_loss": 1.5981500148773193,
326
+ "epoch": 1.4170996693434104,
327
+ "grad_norm": 156.9344817637022,
328
+ "learning_rate": 4.501353102310901e-06,
329
+ "logits": -0.58225017786026,
330
+ "logps": -76.96984100341797,
331
+ "loss": 2.8517,
332
+ "objective": 2.828920602798462,
333
+ "ranking_idealized": 0.5054166913032532,
334
+ "ranking_idealized_expo": 0.4970833361148834,
335
+ "ranking_simple": 0.502916693687439,
336
+ "regularize": 2.828920602798462,
337
+ "step": 500,
338
+ "wo_beta": 15.1870756149292
339
+ },
340
+ {
341
+ "epoch": 1.4170996693434104,
342
+ "eval_dpo_loss": 2.6782028675079346,
343
+ "eval_logits": -0.597258448600769,
344
+ "eval_logps": -80.76759338378906,
345
+ "eval_loss": 5.004448413848877,
346
+ "eval_objective": 5.005820274353027,
347
+ "eval_ranking_idealized": 0.5295031070709229,
348
+ "eval_ranking_idealized_expo": 0.5212215185165405,
349
+ "eval_ranking_simple": 0.5269151329994202,
350
+ "eval_regularize": 5.005820274353027,
351
+ "eval_runtime": 308.5472,
352
+ "eval_samples_per_second": 18.765,
353
+ "eval_steps_per_second": 1.565,
354
+ "eval_wo_beta": 14.262606620788574,
355
+ "step": 500
356
+ },
357
+ {
358
+ "dpo_loss": 1.574170708656311,
359
+ "epoch": 1.5588096362777515,
360
+ "grad_norm": 157.3420281051892,
361
+ "learning_rate": 4.34319334202531e-06,
362
+ "logits": -0.5697833299636841,
363
+ "logps": -77.83871459960938,
364
+ "loss": 2.632,
365
+ "objective": 2.6769964694976807,
366
+ "ranking_idealized": 0.5199999809265137,
367
+ "ranking_idealized_expo": 0.5108333230018616,
368
+ "ranking_simple": 0.5104166865348816,
369
+ "regularize": 2.6769964694976807,
370
+ "step": 550,
371
+ "wo_beta": 15.042858123779297
372
+ },
373
+ {
374
+ "epoch": 1.5588096362777515,
375
+ "eval_dpo_loss": 2.5752413272857666,
376
+ "eval_logits": -0.6148820519447327,
377
+ "eval_logps": -80.52192687988281,
378
+ "eval_loss": 4.877658843994141,
379
+ "eval_objective": 4.884354114532471,
380
+ "eval_ranking_idealized": 0.5295031070709229,
381
+ "eval_ranking_idealized_expo": 0.5212215185165405,
382
+ "eval_ranking_simple": 0.5222567319869995,
383
+ "eval_regularize": 4.884354114532471,
384
+ "eval_runtime": 309.1726,
385
+ "eval_samples_per_second": 18.727,
386
+ "eval_steps_per_second": 1.562,
387
+ "eval_wo_beta": 14.146936416625977,
388
+ "step": 550
389
+ },
390
+ {
391
+ "dpo_loss": 1.4848108291625977,
392
+ "epoch": 1.7005196032120926,
393
+ "grad_norm": 158.42202757367144,
394
+ "learning_rate": 4.16692250129073e-06,
395
+ "logits": -0.5375781655311584,
396
+ "logps": -77.73197937011719,
397
+ "loss": 2.5208,
398
+ "objective": 2.518798828125,
399
+ "ranking_idealized": 0.5220833420753479,
400
+ "ranking_idealized_expo": 0.5149999856948853,
401
+ "ranking_simple": 0.5058333277702332,
402
+ "regularize": 2.518798828125,
403
+ "step": 600,
404
+ "wo_beta": 15.039664268493652
405
+ },
406
+ {
407
+ "epoch": 1.7005196032120926,
408
+ "eval_dpo_loss": 2.5973868370056152,
409
+ "eval_logits": -0.5874757766723633,
410
+ "eval_logps": -80.17750549316406,
411
+ "eval_loss": 4.925836086273193,
412
+ "eval_objective": 4.962131500244141,
413
+ "eval_ranking_idealized": 0.5295031070709229,
414
+ "eval_ranking_idealized_expo": 0.5212215185165405,
415
+ "eval_ranking_simple": 0.5243270993232727,
416
+ "eval_regularize": 4.962131500244141,
417
+ "eval_runtime": 308.6109,
418
+ "eval_samples_per_second": 18.761,
419
+ "eval_steps_per_second": 1.565,
420
+ "eval_wo_beta": 14.266931533813477,
421
+ "step": 600
422
+ },
423
+ {
424
+ "dpo_loss": 1.4892569780349731,
425
+ "epoch": 1.8422295701464337,
426
+ "grad_norm": 161.21497067375822,
427
+ "learning_rate": 3.974272604254906e-06,
428
+ "logits": -0.4858429431915283,
429
+ "logps": -77.9110107421875,
430
+ "loss": 2.4198,
431
+ "objective": 2.448315143585205,
432
+ "ranking_idealized": 0.5408333539962769,
433
+ "ranking_idealized_expo": 0.527916669845581,
434
+ "ranking_simple": 0.527916669845581,
435
+ "regularize": 2.448315143585205,
436
+ "step": 650,
437
+ "wo_beta": 15.80264663696289
438
+ },
439
+ {
440
+ "epoch": 1.8422295701464337,
441
+ "eval_dpo_loss": 2.6345465183258057,
442
+ "eval_logits": -0.5440609455108643,
443
+ "eval_logps": -81.05504608154297,
444
+ "eval_loss": 5.032698154449463,
445
+ "eval_objective": 5.045432090759277,
446
+ "eval_ranking_idealized": 0.5295031070709229,
447
+ "eval_ranking_idealized_expo": 0.5212215185165405,
448
+ "eval_ranking_simple": 0.5269151329994202,
449
+ "eval_regularize": 5.045432090759277,
450
+ "eval_runtime": 315.2786,
451
+ "eval_samples_per_second": 18.365,
452
+ "eval_steps_per_second": 1.532,
453
+ "eval_wo_beta": 14.247919082641602,
454
+ "step": 650
455
+ },
456
+ {
457
+ "dpo_loss": 1.3786677122116089,
458
+ "epoch": 1.9839395370807746,
459
+ "grad_norm": 165.2682511618685,
460
+ "learning_rate": 3.767136614452458e-06,
461
+ "logits": -0.4978749454021454,
462
+ "logps": -77.35163116455078,
463
+ "loss": 2.2699,
464
+ "objective": 2.302328586578369,
465
+ "ranking_idealized": 0.5170833468437195,
466
+ "ranking_idealized_expo": 0.5112500190734863,
467
+ "ranking_simple": 0.5095833539962769,
468
+ "regularize": 2.302328586578369,
469
+ "step": 700,
470
+ "wo_beta": 15.442806243896484
471
+ },
472
+ {
473
+ "epoch": 1.9839395370807746,
474
+ "eval_dpo_loss": 2.629160165786743,
475
+ "eval_logits": -0.559353232383728,
476
+ "eval_logps": -79.73757934570312,
477
+ "eval_loss": 4.9659104347229,
478
+ "eval_objective": 4.995144844055176,
479
+ "eval_ranking_idealized": 0.5295031070709229,
480
+ "eval_ranking_idealized_expo": 0.5212215185165405,
481
+ "eval_ranking_simple": 0.5212215185165405,
482
+ "eval_regularize": 4.995144844055176,
483
+ "eval_runtime": 308.0921,
484
+ "eval_samples_per_second": 18.793,
485
+ "eval_steps_per_second": 1.568,
486
+ "eval_wo_beta": 14.175456047058105,
487
+ "step": 700
488
+ },
489
+ {
490
+ "epoch": 1.9839395370807746,
491
+ "step": 700,
492
+ "total_flos": 0.0,
493
+ "train_loss": 2.7151880131661894,
494
+ "train_runtime": 19357.2061,
495
+ "train_samples_per_second": 13.122,
496
+ "train_steps_per_second": 0.091
497
+ }
498
+ ],
499
+ "logging_steps": 50,
500
+ "max_steps": 1760,
501
+ "num_input_tokens_seen": 0,
502
+ "num_train_epochs": 5,
503
+ "save_steps": 50,
504
+ "stateful_callbacks": {
505
+ "EarlyStoppingCallback": {
506
+ "args": {
507
+ "early_stopping_patience": 5,
508
+ "early_stopping_threshold": 0.0
509
+ },
510
+ "attributes": {
511
+ "early_stopping_patience_counter": 0
512
+ }
513
+ },
514
+ "TrainerControl": {
515
+ "args": {
516
+ "should_epoch_stop": false,
517
+ "should_evaluate": false,
518
+ "should_log": false,
519
+ "should_save": true,
520
+ "should_training_stop": true
521
+ },
522
+ "attributes": {}
523
+ }
524
+ },
525
+ "total_flos": 0.0,
526
+ "train_batch_size": 4,
527
+ "trial_name": null,
528
+ "trial_params": null
529
+ }