xinlai commited on
Commit
9c522b3
1 Parent(s): 759a067

upload model

Browse files
.gitattributes CHANGED
@@ -33,3 +33,26 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ model-00005-of-00023.safetensors filter=lfs diff=lfs merge=lfs -text
37
+ model-00006-of-00023.safetensors filter=lfs diff=lfs merge=lfs -text
38
+ model-00014-of-00023.safetensors filter=lfs diff=lfs merge=lfs -text
39
+ model-00016-of-00023.safetensors filter=lfs diff=lfs merge=lfs -text
40
+ model-00017-of-00023.safetensors filter=lfs diff=lfs merge=lfs -text
41
+ model-00019-of-00023.safetensors filter=lfs diff=lfs merge=lfs -text
42
+ model-00002-of-00023.safetensors filter=lfs diff=lfs merge=lfs -text
43
+ model-00003-of-00023.safetensors filter=lfs diff=lfs merge=lfs -text
44
+ model-00004-of-00023.safetensors filter=lfs diff=lfs merge=lfs -text
45
+ model-00007-of-00023.safetensors filter=lfs diff=lfs merge=lfs -text
46
+ model-00008-of-00023.safetensors filter=lfs diff=lfs merge=lfs -text
47
+ model-00010-of-00023.safetensors filter=lfs diff=lfs merge=lfs -text
48
+ model-00021-of-00023.safetensors filter=lfs diff=lfs merge=lfs -text
49
+ model-00011-of-00023.safetensors filter=lfs diff=lfs merge=lfs -text
50
+ model-00012-of-00023.safetensors filter=lfs diff=lfs merge=lfs -text
51
+ model-00013-of-00023.safetensors filter=lfs diff=lfs merge=lfs -text
52
+ model-00001-of-00023.safetensors filter=lfs diff=lfs merge=lfs -text
53
+ model-00009-of-00023.safetensors filter=lfs diff=lfs merge=lfs -text
54
+ model-00015-of-00023.safetensors filter=lfs diff=lfs merge=lfs -text
55
+ model-00018-of-00023.safetensors filter=lfs diff=lfs merge=lfs -text
56
+ model-00020-of-00023.safetensors filter=lfs diff=lfs merge=lfs -text
57
+ model-00022-of-00023.safetensors filter=lfs diff=lfs merge=lfs -text
58
+ model-00023-of-00023.safetensors filter=lfs diff=lfs merge=lfs -text
added_tokens.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "<|endoftext|>": 151643,
3
+ "<|im_end|>": 151645,
4
+ "<|im_start|>": 151644
5
+ }
all_results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 3.9493670886075947,
3
+ "total_flos": 0.0,
4
+ "train_loss": 0.3210896818110576,
5
+ "train_runtime": 45702.9058,
6
+ "train_samples": 5050,
7
+ "train_samples_per_second": 0.442,
8
+ "train_steps_per_second": 0.003
9
+ }
config.json ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "/dataset/industry_gpt/llm_infer/MetaMath/output/qwen2-57b-distill-data-374k-bs32-lr5e-6-linear-fix-sample0.8-sharded",
3
+ "architectures": [
4
+ "Qwen2MoeForCausalLM"
5
+ ],
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 151643,
8
+ "decoder_sparse_step": 1,
9
+ "eos_token_id": 151643,
10
+ "hidden_act": "silu",
11
+ "hidden_size": 3584,
12
+ "initializer_range": 0.02,
13
+ "intermediate_size": 18944,
14
+ "max_position_embeddings": 131072,
15
+ "max_window_layers": 28,
16
+ "model_type": "qwen2_moe",
17
+ "moe_intermediate_size": 2560,
18
+ "norm_topk_prob": false,
19
+ "num_attention_heads": 28,
20
+ "num_experts": 64,
21
+ "num_experts_per_tok": 8,
22
+ "num_hidden_layers": 28,
23
+ "num_key_value_heads": 4,
24
+ "output_router_logits": false,
25
+ "rms_norm_eps": 1e-06,
26
+ "rope_scaling": {
27
+ "factor": 2.0,
28
+ "original_max_position_embeddings": 32768,
29
+ "type": "yarn"
30
+ },
31
+ "rope_theta": 1000000.0,
32
+ "router_aux_loss_coef": 0.001,
33
+ "shared_expert_intermediate_size": 20480,
34
+ "sliding_window": 131072,
35
+ "tie_word_embeddings": false,
36
+ "torch_dtype": "bfloat16",
37
+ "transformers_version": "4.40.0",
38
+ "use_cache": true,
39
+ "use_sliding_window": false,
40
+ "vocab_size": 151936
41
+ }
generation_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 151643,
3
+ "eos_token_id": [
4
+ 151645,
5
+ 151643
6
+ ],
7
+ "pad_token_id": 151643,
8
+ "transformers_version": "4.40.0"
9
+ }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
model-00001-of-00023.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cce015ec3ad8fc90662529503d754a71c512949a292b0d5d93547a31ddb4723d
3
+ size 4965107248
model-00002-of-00023.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1592d8d4affd6f8592753c1a6474be7ffa5b3a7be85a6e3fd4eadc38eb1414b6
3
+ size 4999572712
model-00003-of-00023.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b25e9dcfc3088f84622e1f0a72eb1bc6a75ea8e8d69b7ddfcb99d3ce93fa1a59
3
+ size 4995413224
model-00004-of-00023.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:27f061170c7dbcf455b8944cdba026960d95928086ffccba79105579a3017b19
3
+ size 4995413224
model-00005-of-00023.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6dc4d80778d4cc8897afa103f1b0efe4d7f6ac5c22139d189b2780550b270ac2
3
+ size 4977062168
model-00006-of-00023.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:66bfcdc0737225ddd26fbc99fbc41aecf01d56fba09e91d5440b2f74797b1cae
3
+ size 4999571856
model-00007-of-00023.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:de90a5c5df2da9b92fe7ac9a3c807580b0b2d16ec0a275f5f43e6f5e489a49c4
3
+ size 4995413224
model-00008-of-00023.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5b4b406d7bbb7e8df930802376bb52b03fc8b434babe515c375e6d25cc692991
3
+ size 4995413224
model-00009-of-00023.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:83c5eed3408b6521d7d606745f5cbb001f65ce36591d3a99ae395e70199844d2
3
+ size 4977063232
model-00010-of-00023.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9f8f9a13fa5e3cddf20cf678d41d8576f5378a190b9002a1f85cb0db0722815a
3
+ size 4999571256
model-00011-of-00023.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:90a1e2abb1ff9ec6d4220f384345b5d86b003835a4b19f248c8c08e3b8774455
3
+ size 4995413480
model-00012-of-00023.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8b810bd6323f6a9e7b99b4ca858705d22e1bf430190ddd42b7dfce62b940b7e8
3
+ size 4995413480
model-00013-of-00023.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d1f0b39df3fda4616d1be00b86a543df6c410066a613e5ef965108854b3df5b1
3
+ size 4995413480
model-00014-of-00023.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:221e605017498d1454797eb53a747289c5d759ee5516a759e0355061394afcc8
3
+ size 4999571256
model-00015-of-00023.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7aa01c9ae215839c022000883e5dd35b3462882b3ce0e82c6d8d9c81a2bc3d09
3
+ size 4995413472
model-00016-of-00023.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c6dedc7cd0255da13802bca87baee89d12ce85792d5daf36c0afaced90351fb8
3
+ size 4995413480
model-00017-of-00023.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bdeeed3ad06db7d5e54dd3fe8b8b0c1658305c3ee94f15f985ad8023dd826c38
3
+ size 4995413480
model-00018-of-00023.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a179356947faf3e3bd53f8f1fdf4584e1430e6263bfdee7514e6480082aafe8f
3
+ size 4999571256
model-00019-of-00023.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:573b67d3f71dc80c6b20279ae4b0a27716c549345ce86b10993d39ba0023cb70
3
+ size 4995413464
model-00020-of-00023.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:62ec4a3e843beb602df717e90da7f5c440a31f7ed62deea1824e65b09c0459d2
3
+ size 4995413480
model-00021-of-00023.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1fd491390cda65f1fa19e2ea3fd1cc7dcaf28cb75a29fbbb84b2d6b3ee393ffd
3
+ size 4995413480
model-00022-of-00023.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8c8b67803a1e21aafd9f891ee3e544ea1f5cd324ce05af4b43b743148cff749d
3
+ size 4999571264
model-00023-of-00023.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3e49f36dd46c59a9353abc2c81d078de4c88aeb9bdb350119d348633ba83aa89
3
+ size 4960997176
model.safetensors.index.json ADDED
The diff for this file is too large to render. See raw diff
 
special_tokens_map.json ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<|im_start|>",
4
+ "<|im_end|>"
5
+ ],
6
+ "eos_token": {
7
+ "content": "<|endoftext|>",
8
+ "lstrip": false,
9
+ "normalized": false,
10
+ "rstrip": false,
11
+ "single_word": false
12
+ },
13
+ "pad_token": {
14
+ "content": "<|endoftext|>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false
19
+ }
20
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "added_tokens_decoder": {
4
+ "151643": {
5
+ "content": "<|endoftext|>",
6
+ "lstrip": false,
7
+ "normalized": false,
8
+ "rstrip": false,
9
+ "single_word": false,
10
+ "special": true
11
+ },
12
+ "151644": {
13
+ "content": "<|im_start|>",
14
+ "lstrip": false,
15
+ "normalized": false,
16
+ "rstrip": false,
17
+ "single_word": false,
18
+ "special": true
19
+ },
20
+ "151645": {
21
+ "content": "<|im_end|>",
22
+ "lstrip": false,
23
+ "normalized": false,
24
+ "rstrip": false,
25
+ "single_word": false,
26
+ "special": true
27
+ }
28
+ },
29
+ "additional_special_tokens": [
30
+ "<|im_start|>",
31
+ "<|im_end|>"
32
+ ],
33
+ "bos_token": null,
34
+ "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
35
+ "clean_up_tokenization_spaces": false,
36
+ "eos_token": "<|endoftext|>",
37
+ "errors": "replace",
38
+ "model_max_length": 512,
39
+ "pad_token": "<|endoftext|>",
40
+ "padding_side": "right",
41
+ "split_special_tokens": false,
42
+ "tokenizer_class": "Qwen2Tokenizer",
43
+ "unk_token": null
44
+ }
train_results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 3.9493670886075947,
3
+ "total_flos": 0.0,
4
+ "train_loss": 0.3210896818110576,
5
+ "train_runtime": 45702.9058,
6
+ "train_samples": 5050,
7
+ "train_samples_per_second": 0.442,
8
+ "train_steps_per_second": 0.003
9
+ }
trainer_state.json ADDED
@@ -0,0 +1,2370 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 3.9493670886075947,
5
+ "eval_steps": 1,
6
+ "global_step": 156,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.02531645569620253,
13
+ "grad_norm": 57.114834738699926,
14
+ "learning_rate": 3.125e-08,
15
+ "logits/chosen": 3.3286685943603516,
16
+ "logits/rejected": 3.4021615982055664,
17
+ "logps/chosen": -32.21625900268555,
18
+ "logps/rejected": -38.764957427978516,
19
+ "loss": 0.6931,
20
+ "rewards/accuracies": 0.0,
21
+ "rewards/chosen": 0.0,
22
+ "rewards/margins": 0.0,
23
+ "rewards/rejected": 0.0,
24
+ "step": 1
25
+ },
26
+ {
27
+ "epoch": 0.05063291139240506,
28
+ "grad_norm": 55.56377943215762,
29
+ "learning_rate": 6.25e-08,
30
+ "logits/chosen": 2.647796630859375,
31
+ "logits/rejected": 2.733036994934082,
32
+ "logps/chosen": -24.994104385375977,
33
+ "logps/rejected": -29.78559112548828,
34
+ "loss": 0.6931,
35
+ "rewards/accuracies": 0.0,
36
+ "rewards/chosen": 0.0,
37
+ "rewards/margins": 0.0,
38
+ "rewards/rejected": 0.0,
39
+ "step": 2
40
+ },
41
+ {
42
+ "epoch": 0.0759493670886076,
43
+ "grad_norm": 61.65658878594894,
44
+ "learning_rate": 9.375e-08,
45
+ "logits/chosen": 2.33656644821167,
46
+ "logits/rejected": 2.518265962600708,
47
+ "logps/chosen": -28.91141128540039,
48
+ "logps/rejected": -35.79951477050781,
49
+ "loss": 0.7163,
50
+ "rewards/accuracies": 0.4375,
51
+ "rewards/chosen": -0.046910468488931656,
52
+ "rewards/margins": -0.0570392943918705,
53
+ "rewards/rejected": 0.01012883335351944,
54
+ "step": 3
55
+ },
56
+ {
57
+ "epoch": 0.10126582278481013,
58
+ "grad_norm": 54.332498272912595,
59
+ "learning_rate": 1.25e-07,
60
+ "logits/chosen": 2.9658162593841553,
61
+ "logits/rejected": 3.0012216567993164,
62
+ "logps/chosen": -29.552040100097656,
63
+ "logps/rejected": -34.8863639831543,
64
+ "loss": 0.7065,
65
+ "rewards/accuracies": 0.4375,
66
+ "rewards/chosen": -0.047019001096487045,
67
+ "rewards/margins": -0.010130930691957474,
68
+ "rewards/rejected": -0.03688807412981987,
69
+ "step": 4
70
+ },
71
+ {
72
+ "epoch": 0.12658227848101267,
73
+ "grad_norm": 55.132631513210136,
74
+ "learning_rate": 1.5624999999999999e-07,
75
+ "logits/chosen": 3.340583086013794,
76
+ "logits/rejected": 3.325289487838745,
77
+ "logps/chosen": -29.905162811279297,
78
+ "logps/rejected": -38.20769500732422,
79
+ "loss": 0.7069,
80
+ "rewards/accuracies": 0.4375,
81
+ "rewards/chosen": -0.06609320640563965,
82
+ "rewards/margins": -0.07786056399345398,
83
+ "rewards/rejected": 0.01176736131310463,
84
+ "step": 5
85
+ },
86
+ {
87
+ "epoch": 0.1518987341772152,
88
+ "grad_norm": 57.59086219970169,
89
+ "learning_rate": 1.875e-07,
90
+ "logits/chosen": 2.2882869243621826,
91
+ "logits/rejected": 2.7739901542663574,
92
+ "logps/chosen": -23.262107849121094,
93
+ "logps/rejected": -38.5238151550293,
94
+ "loss": 0.7044,
95
+ "rewards/accuracies": 0.4375,
96
+ "rewards/chosen": -0.004146900027990341,
97
+ "rewards/margins": -0.010677304118871689,
98
+ "rewards/rejected": 0.006530404556542635,
99
+ "step": 6
100
+ },
101
+ {
102
+ "epoch": 0.17721518987341772,
103
+ "grad_norm": 51.18146497947048,
104
+ "learning_rate": 2.1875e-07,
105
+ "logits/chosen": 2.7440547943115234,
106
+ "logits/rejected": 2.5763754844665527,
107
+ "logps/chosen": -29.953731536865234,
108
+ "logps/rejected": -33.62974166870117,
109
+ "loss": 0.7102,
110
+ "rewards/accuracies": 0.375,
111
+ "rewards/chosen": -0.038678932934999466,
112
+ "rewards/margins": -0.042138125747442245,
113
+ "rewards/rejected": 0.0034591909497976303,
114
+ "step": 7
115
+ },
116
+ {
117
+ "epoch": 0.20253164556962025,
118
+ "grad_norm": 53.68918501282459,
119
+ "learning_rate": 2.5e-07,
120
+ "logits/chosen": 3.2939491271972656,
121
+ "logits/rejected": 3.003464698791504,
122
+ "logps/chosen": -28.833343505859375,
123
+ "logps/rejected": -23.717449188232422,
124
+ "loss": 0.6967,
125
+ "rewards/accuracies": 0.625,
126
+ "rewards/chosen": 0.013088438659906387,
127
+ "rewards/margins": 0.03873240202665329,
128
+ "rewards/rejected": -0.025643955916166306,
129
+ "step": 8
130
+ },
131
+ {
132
+ "epoch": 0.22784810126582278,
133
+ "grad_norm": 56.10725202078908,
134
+ "learning_rate": 2.8125e-07,
135
+ "logits/chosen": 2.243440866470337,
136
+ "logits/rejected": 2.4725394248962402,
137
+ "logps/chosen": -22.95738983154297,
138
+ "logps/rejected": -27.134145736694336,
139
+ "loss": 0.684,
140
+ "rewards/accuracies": 0.625,
141
+ "rewards/chosen": 0.04334143549203873,
142
+ "rewards/margins": 0.07354923337697983,
143
+ "rewards/rejected": -0.03020780347287655,
144
+ "step": 9
145
+ },
146
+ {
147
+ "epoch": 0.25316455696202533,
148
+ "grad_norm": 52.307052055168896,
149
+ "learning_rate": 3.1249999999999997e-07,
150
+ "logits/chosen": 3.2282490730285645,
151
+ "logits/rejected": 3.332897424697876,
152
+ "logps/chosen": -26.25811004638672,
153
+ "logps/rejected": -33.90528869628906,
154
+ "loss": 0.69,
155
+ "rewards/accuracies": 0.75,
156
+ "rewards/chosen": 0.05821816250681877,
157
+ "rewards/margins": 0.11382907629013062,
158
+ "rewards/rejected": -0.05561092123389244,
159
+ "step": 10
160
+ },
161
+ {
162
+ "epoch": 0.27848101265822783,
163
+ "grad_norm": 51.178561062412925,
164
+ "learning_rate": 3.4375e-07,
165
+ "logits/chosen": 3.0024123191833496,
166
+ "logits/rejected": 2.8223249912261963,
167
+ "logps/chosen": -25.597423553466797,
168
+ "logps/rejected": -27.973861694335938,
169
+ "loss": 0.6937,
170
+ "rewards/accuracies": 0.375,
171
+ "rewards/chosen": -0.006550817750394344,
172
+ "rewards/margins": 0.020815890282392502,
173
+ "rewards/rejected": -0.02736670896410942,
174
+ "step": 11
175
+ },
176
+ {
177
+ "epoch": 0.3037974683544304,
178
+ "grad_norm": 55.45441942378304,
179
+ "learning_rate": 3.75e-07,
180
+ "logits/chosen": 2.8794424533843994,
181
+ "logits/rejected": 2.8852243423461914,
182
+ "logps/chosen": -24.754030227661133,
183
+ "logps/rejected": -37.79888916015625,
184
+ "loss": 0.6747,
185
+ "rewards/accuracies": 0.75,
186
+ "rewards/chosen": -0.028544973582029343,
187
+ "rewards/margins": 0.1005675345659256,
188
+ "rewards/rejected": -0.12911249697208405,
189
+ "step": 12
190
+ },
191
+ {
192
+ "epoch": 0.3291139240506329,
193
+ "grad_norm": 53.09392363916735,
194
+ "learning_rate": 4.0625e-07,
195
+ "logits/chosen": 2.4571850299835205,
196
+ "logits/rejected": 2.484537363052368,
197
+ "logps/chosen": -25.80780601501465,
198
+ "logps/rejected": -29.53820037841797,
199
+ "loss": 0.6911,
200
+ "rewards/accuracies": 0.6875,
201
+ "rewards/chosen": 0.03155774995684624,
202
+ "rewards/margins": 0.07387223094701767,
203
+ "rewards/rejected": -0.04231448099017143,
204
+ "step": 13
205
+ },
206
+ {
207
+ "epoch": 0.35443037974683544,
208
+ "grad_norm": 52.83620354330049,
209
+ "learning_rate": 4.375e-07,
210
+ "logits/chosen": 2.509875535964966,
211
+ "logits/rejected": 2.7940757274627686,
212
+ "logps/chosen": -23.630701065063477,
213
+ "logps/rejected": -30.94167709350586,
214
+ "loss": 0.6538,
215
+ "rewards/accuracies": 0.6875,
216
+ "rewards/chosen": 0.0901968702673912,
217
+ "rewards/margins": 0.12045808881521225,
218
+ "rewards/rejected": -0.030261218547821045,
219
+ "step": 14
220
+ },
221
+ {
222
+ "epoch": 0.379746835443038,
223
+ "grad_norm": 54.43440442749398,
224
+ "learning_rate": 4.6874999999999996e-07,
225
+ "logits/chosen": 2.544405460357666,
226
+ "logits/rejected": 2.4906702041625977,
227
+ "logps/chosen": -31.145732879638672,
228
+ "logps/rejected": -35.83900451660156,
229
+ "loss": 0.6699,
230
+ "rewards/accuracies": 0.6875,
231
+ "rewards/chosen": 0.015418028458952904,
232
+ "rewards/margins": 0.1095210611820221,
233
+ "rewards/rejected": -0.09410304576158524,
234
+ "step": 15
235
+ },
236
+ {
237
+ "epoch": 0.4050632911392405,
238
+ "grad_norm": 56.228300664828,
239
+ "learning_rate": 5e-07,
240
+ "logits/chosen": 2.400423049926758,
241
+ "logits/rejected": 2.620924949645996,
242
+ "logps/chosen": -19.128610610961914,
243
+ "logps/rejected": -33.12566375732422,
244
+ "loss": 0.6456,
245
+ "rewards/accuracies": 0.5,
246
+ "rewards/chosen": -0.02211858704686165,
247
+ "rewards/margins": 0.07731227576732635,
248
+ "rewards/rejected": -0.0994308590888977,
249
+ "step": 16
250
+ },
251
+ {
252
+ "epoch": 0.43037974683544306,
253
+ "grad_norm": 55.89193538913802,
254
+ "learning_rate": 4.999370587356267e-07,
255
+ "logits/chosen": 2.4614853858947754,
256
+ "logits/rejected": 2.2287514209747314,
257
+ "logps/chosen": -30.090726852416992,
258
+ "logps/rejected": -36.996307373046875,
259
+ "loss": 0.6714,
260
+ "rewards/accuracies": 0.8125,
261
+ "rewards/chosen": 0.010743052698671818,
262
+ "rewards/margins": 0.19052858650684357,
263
+ "rewards/rejected": -0.17978551983833313,
264
+ "step": 17
265
+ },
266
+ {
267
+ "epoch": 0.45569620253164556,
268
+ "grad_norm": 47.67611457430014,
269
+ "learning_rate": 4.997482666353286e-07,
270
+ "logits/chosen": 3.0267255306243896,
271
+ "logits/rejected": 3.2108960151672363,
272
+ "logps/chosen": -24.398426055908203,
273
+ "logps/rejected": -36.53913879394531,
274
+ "loss": 0.6385,
275
+ "rewards/accuracies": 0.8125,
276
+ "rewards/chosen": 0.06351403892040253,
277
+ "rewards/margins": 0.35269594192504883,
278
+ "rewards/rejected": -0.2891818583011627,
279
+ "step": 18
280
+ },
281
+ {
282
+ "epoch": 0.4810126582278481,
283
+ "grad_norm": 52.87018041987955,
284
+ "learning_rate": 4.99433718761614e-07,
285
+ "logits/chosen": 1.927664041519165,
286
+ "logits/rejected": 2.412555694580078,
287
+ "logps/chosen": -23.279611587524414,
288
+ "logps/rejected": -35.23127746582031,
289
+ "loss": 0.6107,
290
+ "rewards/accuracies": 0.75,
291
+ "rewards/chosen": 0.028891097754240036,
292
+ "rewards/margins": 0.2483980357646942,
293
+ "rewards/rejected": -0.21950694918632507,
294
+ "step": 19
295
+ },
296
+ {
297
+ "epoch": 0.5063291139240507,
298
+ "grad_norm": 48.602746524439674,
299
+ "learning_rate": 4.989935734988097e-07,
300
+ "logits/chosen": 2.6588048934936523,
301
+ "logits/rejected": 2.663198232650757,
302
+ "logps/chosen": -27.881851196289062,
303
+ "logps/rejected": -40.07163619995117,
304
+ "loss": 0.6026,
305
+ "rewards/accuracies": 0.6875,
306
+ "rewards/chosen": 0.008863937109708786,
307
+ "rewards/margins": 0.33766672015190125,
308
+ "rewards/rejected": -0.32880276441574097,
309
+ "step": 20
310
+ },
311
+ {
312
+ "epoch": 0.5316455696202531,
313
+ "grad_norm": 54.16428877770723,
314
+ "learning_rate": 4.984280524733107e-07,
315
+ "logits/chosen": 2.961024284362793,
316
+ "logits/rejected": 2.982067584991455,
317
+ "logps/chosen": -33.65000915527344,
318
+ "logps/rejected": -44.283145904541016,
319
+ "loss": 0.6255,
320
+ "rewards/accuracies": 0.6875,
321
+ "rewards/chosen": -0.0565052255988121,
322
+ "rewards/margins": 0.3312607407569885,
323
+ "rewards/rejected": -0.387766033411026,
324
+ "step": 21
325
+ },
326
+ {
327
+ "epoch": 0.5569620253164557,
328
+ "grad_norm": 45.46343620685162,
329
+ "learning_rate": 4.977374404419837e-07,
330
+ "logits/chosen": 2.731034278869629,
331
+ "logits/rejected": 2.8560142517089844,
332
+ "logps/chosen": -28.184062957763672,
333
+ "logps/rejected": -29.667505264282227,
334
+ "loss": 0.5798,
335
+ "rewards/accuracies": 0.6875,
336
+ "rewards/chosen": 0.03637414053082466,
337
+ "rewards/margins": 0.30975764989852905,
338
+ "rewards/rejected": -0.2733834683895111,
339
+ "step": 22
340
+ },
341
+ {
342
+ "epoch": 0.5822784810126582,
343
+ "grad_norm": 44.12365829762132,
344
+ "learning_rate": 4.969220851487844e-07,
345
+ "logits/chosen": 2.936208963394165,
346
+ "logits/rejected": 2.651503086090088,
347
+ "logps/chosen": -33.56488037109375,
348
+ "logps/rejected": -41.19189453125,
349
+ "loss": 0.5772,
350
+ "rewards/accuracies": 0.625,
351
+ "rewards/chosen": 0.06098739802837372,
352
+ "rewards/margins": 0.4615795612335205,
353
+ "rewards/rejected": -0.4005921483039856,
354
+ "step": 23
355
+ },
356
+ {
357
+ "epoch": 0.6075949367088608,
358
+ "grad_norm": 45.017464152313536,
359
+ "learning_rate": 4.959823971496574e-07,
360
+ "logits/chosen": 2.6883764266967773,
361
+ "logits/rejected": 2.8039445877075195,
362
+ "logps/chosen": -25.476293563842773,
363
+ "logps/rejected": -27.924108505249023,
364
+ "loss": 0.5669,
365
+ "rewards/accuracies": 0.875,
366
+ "rewards/chosen": 0.003826950676739216,
367
+ "rewards/margins": 0.374502956867218,
368
+ "rewards/rejected": -0.37067604064941406,
369
+ "step": 24
370
+ },
371
+ {
372
+ "epoch": 0.6329113924050633,
373
+ "grad_norm": 42.27141207917208,
374
+ "learning_rate": 4.949188496058089e-07,
375
+ "logits/chosen": 2.4697136878967285,
376
+ "logits/rejected": 2.4843673706054688,
377
+ "logps/chosen": -29.330278396606445,
378
+ "logps/rejected": -34.346717834472656,
379
+ "loss": 0.5643,
380
+ "rewards/accuracies": 0.6875,
381
+ "rewards/chosen": -0.015232307836413383,
382
+ "rewards/margins": 0.2725878059864044,
383
+ "rewards/rejected": -0.28782013058662415,
384
+ "step": 25
385
+ },
386
+ {
387
+ "epoch": 0.6582278481012658,
388
+ "grad_norm": 43.50860864037927,
389
+ "learning_rate": 4.937319780454559e-07,
390
+ "logits/chosen": 2.4729130268096924,
391
+ "logits/rejected": 2.728221893310547,
392
+ "logps/chosen": -15.063579559326172,
393
+ "logps/rejected": -29.431615829467773,
394
+ "loss": 0.5485,
395
+ "rewards/accuracies": 1.0,
396
+ "rewards/chosen": 0.010220827534794807,
397
+ "rewards/margins": 0.6472443342208862,
398
+ "rewards/rejected": -0.6370234489440918,
399
+ "step": 26
400
+ },
401
+ {
402
+ "epoch": 0.6835443037974683,
403
+ "grad_norm": 44.10326676433438,
404
+ "learning_rate": 4.924223800941717e-07,
405
+ "logits/chosen": 2.506294012069702,
406
+ "logits/rejected": 2.611067295074463,
407
+ "logps/chosen": -27.957015991210938,
408
+ "logps/rejected": -34.36131286621094,
409
+ "loss": 0.5338,
410
+ "rewards/accuracies": 0.8125,
411
+ "rewards/chosen": 0.09378986805677414,
412
+ "rewards/margins": 0.7992368936538696,
413
+ "rewards/rejected": -0.7054470777511597,
414
+ "step": 27
415
+ },
416
+ {
417
+ "epoch": 0.7088607594936709,
418
+ "grad_norm": 42.53652604653026,
419
+ "learning_rate": 4.909907151739633e-07,
420
+ "logits/chosen": 2.370758295059204,
421
+ "logits/rejected": 2.3593924045562744,
422
+ "logps/chosen": -24.23151969909668,
423
+ "logps/rejected": -28.833358764648438,
424
+ "loss": 0.5266,
425
+ "rewards/accuracies": 0.75,
426
+ "rewards/chosen": 0.015818338841199875,
427
+ "rewards/margins": 0.35423219203948975,
428
+ "rewards/rejected": -0.3384138345718384,
429
+ "step": 28
430
+ },
431
+ {
432
+ "epoch": 0.7341772151898734,
433
+ "grad_norm": 43.92462963258578,
434
+ "learning_rate": 4.894377041712326e-07,
435
+ "logits/chosen": 2.85575532913208,
436
+ "logits/rejected": 2.9239468574523926,
437
+ "logps/chosen": -24.336238861083984,
438
+ "logps/rejected": -32.19173049926758,
439
+ "loss": 0.5788,
440
+ "rewards/accuracies": 0.625,
441
+ "rewards/chosen": -0.1497105062007904,
442
+ "rewards/margins": 0.41176801919937134,
443
+ "rewards/rejected": -0.5614784955978394,
444
+ "step": 29
445
+ },
446
+ {
447
+ "epoch": 0.759493670886076,
448
+ "grad_norm": 45.52462540230979,
449
+ "learning_rate": 4.877641290737883e-07,
450
+ "logits/chosen": 2.577241897583008,
451
+ "logits/rejected": 2.6502695083618164,
452
+ "logps/chosen": -27.839168548583984,
453
+ "logps/rejected": -36.543941497802734,
454
+ "loss": 0.5261,
455
+ "rewards/accuracies": 0.75,
456
+ "rewards/chosen": -0.1546507328748703,
457
+ "rewards/margins": 1.282554268836975,
458
+ "rewards/rejected": -1.4372049570083618,
459
+ "step": 30
460
+ },
461
+ {
462
+ "epoch": 0.7848101265822784,
463
+ "grad_norm": 40.84962297403327,
464
+ "learning_rate": 4.859708325770919e-07,
465
+ "logits/chosen": 2.821880340576172,
466
+ "logits/rejected": 2.7289583683013916,
467
+ "logps/chosen": -32.00937271118164,
468
+ "logps/rejected": -32.9297981262207,
469
+ "loss": 0.5013,
470
+ "rewards/accuracies": 0.75,
471
+ "rewards/chosen": -0.05743463337421417,
472
+ "rewards/margins": 0.7840094566345215,
473
+ "rewards/rejected": -0.8414440155029297,
474
+ "step": 31
475
+ },
476
+ {
477
+ "epoch": 0.810126582278481,
478
+ "grad_norm": 42.08408504733762,
479
+ "learning_rate": 4.840587176599343e-07,
480
+ "logits/chosen": 2.684711456298828,
481
+ "logits/rejected": 3.0184218883514404,
482
+ "logps/chosen": -29.447864532470703,
483
+ "logps/rejected": -45.021995544433594,
484
+ "loss": 0.5126,
485
+ "rewards/accuracies": 0.875,
486
+ "rewards/chosen": 0.03239244967699051,
487
+ "rewards/margins": 1.4539411067962646,
488
+ "rewards/rejected": -1.4215487241744995,
489
+ "step": 32
490
+ },
491
+ {
492
+ "epoch": 0.8354430379746836,
493
+ "grad_norm": 39.28461553560508,
494
+ "learning_rate": 4.820287471297597e-07,
495
+ "logits/chosen": 2.2539427280426025,
496
+ "logits/rejected": 2.2139289379119873,
497
+ "logps/chosen": -28.69852066040039,
498
+ "logps/rejected": -31.549945831298828,
499
+ "loss": 0.5182,
500
+ "rewards/accuracies": 0.875,
501
+ "rewards/chosen": -0.13107189536094666,
502
+ "rewards/margins": 1.036413311958313,
503
+ "rewards/rejected": -1.167485237121582,
504
+ "step": 33
505
+ },
506
+ {
507
+ "epoch": 0.8607594936708861,
508
+ "grad_norm": 40.986833656434555,
509
+ "learning_rate": 4.798819431378626e-07,
510
+ "logits/chosen": 2.653698444366455,
511
+ "logits/rejected": 2.910414218902588,
512
+ "logps/chosen": -21.828556060791016,
513
+ "logps/rejected": -38.98440170288086,
514
+ "loss": 0.4764,
515
+ "rewards/accuracies": 0.75,
516
+ "rewards/chosen": 0.08699348568916321,
517
+ "rewards/margins": 0.9170618057250977,
518
+ "rewards/rejected": -0.8300682306289673,
519
+ "step": 34
520
+ },
521
+ {
522
+ "epoch": 0.8860759493670886,
523
+ "grad_norm": 44.21046799920892,
524
+ "learning_rate": 4.776193866647039e-07,
525
+ "logits/chosen": 2.2878611087799072,
526
+ "logits/rejected": 2.6529476642608643,
527
+ "logps/chosen": -32.880462646484375,
528
+ "logps/rejected": -48.58502197265625,
529
+ "loss": 0.5087,
530
+ "rewards/accuracies": 0.8125,
531
+ "rewards/chosen": -0.1624097228050232,
532
+ "rewards/margins": 0.6843761205673218,
533
+ "rewards/rejected": -0.8467859029769897,
534
+ "step": 35
535
+ },
536
+ {
537
+ "epoch": 0.9113924050632911,
538
+ "grad_norm": 37.027834114234714,
539
+ "learning_rate": 4.752422169756047e-07,
540
+ "logits/chosen": 1.9958323240280151,
541
+ "logits/rejected": 2.1072514057159424,
542
+ "logps/chosen": -25.151729583740234,
543
+ "logps/rejected": -34.56462478637695,
544
+ "loss": 0.448,
545
+ "rewards/accuracies": 0.8125,
546
+ "rewards/chosen": 0.035374294966459274,
547
+ "rewards/margins": 0.7525378465652466,
548
+ "rewards/rejected": -0.7171635627746582,
549
+ "step": 36
550
+ },
551
+ {
552
+ "epoch": 0.9367088607594937,
553
+ "grad_norm": 42.723620077832024,
554
+ "learning_rate": 4.7275163104709194e-07,
555
+ "logits/chosen": 2.223971128463745,
556
+ "logits/rejected": 2.2867026329040527,
557
+ "logps/chosen": -28.47149658203125,
558
+ "logps/rejected": -41.30598831176758,
559
+ "loss": 0.4752,
560
+ "rewards/accuracies": 0.75,
561
+ "rewards/chosen": -0.04645688831806183,
562
+ "rewards/margins": 0.7158762812614441,
563
+ "rewards/rejected": -0.7623331546783447,
564
+ "step": 37
565
+ },
566
+ {
567
+ "epoch": 0.9620253164556962,
568
+ "grad_norm": 37.84394079738309,
569
+ "learning_rate": 4.7014888296418447e-07,
570
+ "logits/chosen": 2.7294678688049316,
571
+ "logits/rejected": 2.7926061153411865,
572
+ "logps/chosen": -27.44596290588379,
573
+ "logps/rejected": -37.265743255615234,
574
+ "loss": 0.4448,
575
+ "rewards/accuracies": 0.8125,
576
+ "rewards/chosen": 0.049863241612911224,
577
+ "rewards/margins": 1.077908992767334,
578
+ "rewards/rejected": -1.0280457735061646,
579
+ "step": 38
580
+ },
581
+ {
582
+ "epoch": 0.9873417721518988,
583
+ "grad_norm": 37.542515394049275,
584
+ "learning_rate": 4.6743528328892384e-07,
585
+ "logits/chosen": 2.9349310398101807,
586
+ "logits/rejected": 2.7823617458343506,
587
+ "logps/chosen": -28.974937438964844,
588
+ "logps/rejected": -31.127620697021484,
589
+ "loss": 0.4247,
590
+ "rewards/accuracies": 0.875,
591
+ "rewards/chosen": 0.06356163322925568,
592
+ "rewards/margins": 0.8182165622711182,
593
+ "rewards/rejected": -0.7546550035476685,
594
+ "step": 39
595
+ },
596
+ {
597
+ "epoch": 1.0126582278481013,
598
+ "grad_norm": 34.306915155273884,
599
+ "learning_rate": 4.646121984004665e-07,
600
+ "logits/chosen": 2.93210768699646,
601
+ "logits/rejected": 2.952573299407959,
602
+ "logps/chosen": -22.667604446411133,
603
+ "logps/rejected": -33.4359130859375,
604
+ "loss": 0.4023,
605
+ "rewards/accuracies": 1.0,
606
+ "rewards/chosen": 0.15648899972438812,
607
+ "rewards/margins": 1.7330085039138794,
608
+ "rewards/rejected": -1.5765196084976196,
609
+ "step": 40
610
+ },
611
+ {
612
+ "epoch": 1.0379746835443038,
613
+ "grad_norm": 34.583417240102314,
614
+ "learning_rate": 4.6168104980707103e-07,
615
+ "logits/chosen": 1.8353512287139893,
616
+ "logits/rejected": 1.8640844821929932,
617
+ "logps/chosen": -30.39456558227539,
618
+ "logps/rejected": -39.79166793823242,
619
+ "loss": 0.3791,
620
+ "rewards/accuracies": 0.9375,
621
+ "rewards/chosen": -0.04578995332121849,
622
+ "rewards/margins": 1.216768741607666,
623
+ "rewards/rejected": -1.2625586986541748,
624
+ "step": 41
625
+ },
626
+ {
627
+ "epoch": 1.0632911392405062,
628
+ "grad_norm": 34.082479241604844,
629
+ "learning_rate": 4.5864331343032565e-07,
630
+ "logits/chosen": 1.8534090518951416,
631
+ "logits/rejected": 2.3674142360687256,
632
+ "logps/chosen": -24.82038116455078,
633
+ "logps/rejected": -36.29607009887695,
634
+ "loss": 0.3826,
635
+ "rewards/accuracies": 0.875,
636
+ "rewards/chosen": 0.017678476870059967,
637
+ "rewards/margins": 1.1888235807418823,
638
+ "rewards/rejected": -1.1711452007293701,
639
+ "step": 42
640
+ },
641
+ {
642
+ "epoch": 1.0886075949367089,
643
+ "grad_norm": 34.56856221568917,
644
+ "learning_rate": 4.555005188619775e-07,
645
+ "logits/chosen": 2.3347442150115967,
646
+ "logits/rejected": 2.896552562713623,
647
+ "logps/chosen": -27.94430923461914,
648
+ "logps/rejected": -56.70208740234375,
649
+ "loss": 0.3719,
650
+ "rewards/accuracies": 0.9375,
651
+ "rewards/chosen": 0.003501923754811287,
652
+ "rewards/margins": 1.5741243362426758,
653
+ "rewards/rejected": -1.5706223249435425,
654
+ "step": 43
655
+ },
656
+ {
657
+ "epoch": 1.1139240506329113,
658
+ "grad_norm": 31.370937274347394,
659
+ "learning_rate": 4.5225424859373684e-07,
660
+ "logits/chosen": 2.6383707523345947,
661
+ "logits/rejected": 2.5485527515411377,
662
+ "logps/chosen": -19.298486709594727,
663
+ "logps/rejected": -25.37928009033203,
664
+ "loss": 0.3744,
665
+ "rewards/accuracies": 0.8125,
666
+ "rewards/chosen": 0.22318463027477264,
667
+ "rewards/margins": 0.6971991658210754,
668
+ "rewards/rejected": -0.474014550447464,
669
+ "step": 44
670
+ },
671
+ {
672
+ "epoch": 1.139240506329114,
673
+ "grad_norm": 34.10062660077868,
674
+ "learning_rate": 4.489061372204452e-07,
675
+ "logits/chosen": 2.376051664352417,
676
+ "logits/rejected": 2.3887717723846436,
677
+ "logps/chosen": -27.828815460205078,
678
+ "logps/rejected": -35.97361755371094,
679
+ "loss": 0.3561,
680
+ "rewards/accuracies": 0.9375,
681
+ "rewards/chosen": 0.13765862584114075,
682
+ "rewards/margins": 0.8190426826477051,
683
+ "rewards/rejected": -0.6813840866088867,
684
+ "step": 45
685
+ },
686
+ {
687
+ "epoch": 1.1645569620253164,
688
+ "grad_norm": 34.550138211560416,
689
+ "learning_rate": 4.4545787061700746e-07,
690
+ "logits/chosen": 1.5758471488952637,
691
+ "logits/rejected": 1.6474454402923584,
692
+ "logps/chosen": -26.35508155822754,
693
+ "logps/rejected": -33.66876983642578,
694
+ "loss": 0.3738,
695
+ "rewards/accuracies": 0.9375,
696
+ "rewards/chosen": 0.11953898519277573,
697
+ "rewards/margins": 1.4261627197265625,
698
+ "rewards/rejected": -1.3066238164901733,
699
+ "step": 46
700
+ },
701
+ {
702
+ "epoch": 1.189873417721519,
703
+ "grad_norm": 32.97226834077475,
704
+ "learning_rate": 4.4191118508950277e-07,
705
+ "logits/chosen": 2.352506160736084,
706
+ "logits/rejected": 2.5829250812530518,
707
+ "logps/chosen": -30.464948654174805,
708
+ "logps/rejected": -48.602386474609375,
709
+ "loss": 0.3773,
710
+ "rewards/accuracies": 0.9375,
711
+ "rewards/chosen": 0.28972989320755005,
712
+ "rewards/margins": 1.533113956451416,
713
+ "rewards/rejected": -1.2433841228485107,
714
+ "step": 47
715
+ },
716
+ {
717
+ "epoch": 1.2151898734177216,
718
+ "grad_norm": 31.173818641433147,
719
+ "learning_rate": 4.3826786650090273e-07,
720
+ "logits/chosen": 2.6230416297912598,
721
+ "logits/rejected": 2.9404265880584717,
722
+ "logps/chosen": -26.442947387695312,
723
+ "logps/rejected": -40.41877746582031,
724
+ "loss": 0.3218,
725
+ "rewards/accuracies": 0.9375,
726
+ "rewards/chosen": 0.13079282641410828,
727
+ "rewards/margins": 1.8220040798187256,
728
+ "rewards/rejected": -1.6912113428115845,
729
+ "step": 48
730
+ },
731
+ {
732
+ "epoch": 1.240506329113924,
733
+ "grad_norm": 31.65330353079338,
734
+ "learning_rate": 4.345297493718352e-07,
735
+ "logits/chosen": 2.0961148738861084,
736
+ "logits/rejected": 2.31225848197937,
737
+ "logps/chosen": -28.76830291748047,
738
+ "logps/rejected": -41.471527099609375,
739
+ "loss": 0.324,
740
+ "rewards/accuracies": 0.9375,
741
+ "rewards/chosen": 0.17320382595062256,
742
+ "rewards/margins": 1.8003501892089844,
743
+ "rewards/rejected": -1.6271462440490723,
744
+ "step": 49
745
+ },
746
+ {
747
+ "epoch": 1.2658227848101267,
748
+ "grad_norm": 32.0517210362914,
749
+ "learning_rate": 4.3069871595684787e-07,
750
+ "logits/chosen": 2.477304458618164,
751
+ "logits/rejected": 2.607898473739624,
752
+ "logps/chosen": -25.64235496520996,
753
+ "logps/rejected": -30.789039611816406,
754
+ "loss": 0.3365,
755
+ "rewards/accuracies": 0.875,
756
+ "rewards/chosen": 0.04965633898973465,
757
+ "rewards/margins": 1.5173263549804688,
758
+ "rewards/rejected": -1.4676700830459595,
759
+ "step": 50
760
+ },
761
+ {
762
+ "epoch": 1.2911392405063291,
763
+ "grad_norm": 34.75297133689012,
764
+ "learning_rate": 4.2677669529663686e-07,
765
+ "logits/chosen": 2.4322314262390137,
766
+ "logits/rejected": 2.3749780654907227,
767
+ "logps/chosen": -28.2196102142334,
768
+ "logps/rejected": -36.66654968261719,
769
+ "loss": 0.376,
770
+ "rewards/accuracies": 0.875,
771
+ "rewards/chosen": -0.014162863604724407,
772
+ "rewards/margins": 1.2780933380126953,
773
+ "rewards/rejected": -1.292256236076355,
774
+ "step": 51
775
+ },
776
+ {
777
+ "epoch": 1.3164556962025316,
778
+ "grad_norm": 30.679462903169394,
779
+ "learning_rate": 4.227656622467162e-07,
780
+ "logits/chosen": 2.16550350189209,
781
+ "logits/rejected": 2.483671188354492,
782
+ "logps/chosen": -27.87743377685547,
783
+ "logps/rejected": -35.79537582397461,
784
+ "loss": 0.328,
785
+ "rewards/accuracies": 0.6875,
786
+ "rewards/chosen": -0.016814690083265305,
787
+ "rewards/margins": 0.7990709543228149,
788
+ "rewards/rejected": -0.8158857226371765,
789
+ "step": 52
790
+ },
791
+ {
792
+ "epoch": 1.3417721518987342,
793
+ "grad_norm": 33.362419000257866,
794
+ "learning_rate": 4.186676364830186e-07,
795
+ "logits/chosen": 1.6940243244171143,
796
+ "logits/rejected": 1.8324474096298218,
797
+ "logps/chosen": -22.575485229492188,
798
+ "logps/rejected": -37.813812255859375,
799
+ "loss": 0.3899,
800
+ "rewards/accuracies": 0.9375,
801
+ "rewards/chosen": -0.18489199876785278,
802
+ "rewards/margins": 1.3045916557312012,
803
+ "rewards/rejected": -1.4894835948944092,
804
+ "step": 53
805
+ },
806
+ {
807
+ "epoch": 1.3670886075949367,
808
+ "grad_norm": 34.27040807040801,
809
+ "learning_rate": 4.1448468148492814e-07,
810
+ "logits/chosen": 2.4027748107910156,
811
+ "logits/rejected": 2.6370065212249756,
812
+ "logps/chosen": -25.746681213378906,
813
+ "logps/rejected": -39.262149810791016,
814
+ "loss": 0.3439,
815
+ "rewards/accuracies": 0.75,
816
+ "rewards/chosen": -0.14556269347667694,
817
+ "rewards/margins": 1.5188615322113037,
818
+ "rewards/rejected": -1.664424180984497,
819
+ "step": 54
820
+ },
821
+ {
822
+ "epoch": 1.3924050632911391,
823
+ "grad_norm": 30.395014985650786,
824
+ "learning_rate": 4.10218903496256e-07,
825
+ "logits/chosen": 1.9827089309692383,
826
+ "logits/rejected": 2.0387563705444336,
827
+ "logps/chosen": -23.929346084594727,
828
+ "logps/rejected": -31.56798553466797,
829
+ "loss": 0.3347,
830
+ "rewards/accuracies": 0.875,
831
+ "rewards/chosen": 0.012682847678661346,
832
+ "rewards/margins": 1.1076587438583374,
833
+ "rewards/rejected": -1.0949759483337402,
834
+ "step": 55
835
+ },
836
+ {
837
+ "epoch": 1.4177215189873418,
838
+ "grad_norm": 30.07232120551146,
839
+ "learning_rate": 4.058724504646834e-07,
840
+ "logits/chosen": 2.144460916519165,
841
+ "logits/rejected": 2.3883056640625,
842
+ "logps/chosen": -29.003910064697266,
843
+ "logps/rejected": -48.30259704589844,
844
+ "loss": 0.3069,
845
+ "rewards/accuracies": 0.875,
846
+ "rewards/chosen": 0.12483976036310196,
847
+ "rewards/margins": 2.0643203258514404,
848
+ "rewards/rejected": -1.9394805431365967,
849
+ "step": 56
850
+ },
851
+ {
852
+ "epoch": 1.4430379746835442,
853
+ "grad_norm": 29.37804816237709,
854
+ "learning_rate": 4.0144751096020497e-07,
855
+ "logits/chosen": 2.4025626182556152,
856
+ "logits/rejected": 2.6997711658477783,
857
+ "logps/chosen": -20.084449768066406,
858
+ "logps/rejected": -41.12477493286133,
859
+ "loss": 0.2932,
860
+ "rewards/accuracies": 0.8125,
861
+ "rewards/chosen": -0.05090276896953583,
862
+ "rewards/margins": 1.5857845544815063,
863
+ "rewards/rejected": -1.6366872787475586,
864
+ "step": 57
865
+ },
866
+ {
867
+ "epoch": 1.4683544303797469,
868
+ "grad_norm": 28.707949914480356,
869
+ "learning_rate": 3.9694631307311825e-07,
870
+ "logits/chosen": 1.7879891395568848,
871
+ "logits/rejected": 1.9085421562194824,
872
+ "logps/chosen": -24.52581214904785,
873
+ "logps/rejected": -42.650333404541016,
874
+ "loss": 0.2848,
875
+ "rewards/accuracies": 0.875,
876
+ "rewards/chosen": -0.09958541393280029,
877
+ "rewards/margins": 1.8922325372695923,
878
+ "rewards/rejected": -1.991817831993103,
879
+ "step": 58
880
+ },
881
+ {
882
+ "epoch": 1.4936708860759493,
883
+ "grad_norm": 30.2249376141746,
884
+ "learning_rate": 3.92371123292113e-07,
885
+ "logits/chosen": 2.3484342098236084,
886
+ "logits/rejected": 2.6477420330047607,
887
+ "logps/chosen": -25.54952621459961,
888
+ "logps/rejected": -42.36656951904297,
889
+ "loss": 0.2884,
890
+ "rewards/accuracies": 0.9375,
891
+ "rewards/chosen": -0.025862978771328926,
892
+ "rewards/margins": 2.4160983562469482,
893
+ "rewards/rejected": -2.4419612884521484,
894
+ "step": 59
895
+ },
896
+ {
897
+ "epoch": 1.518987341772152,
898
+ "grad_norm": 29.710974044035055,
899
+ "learning_rate": 3.877242453630256e-07,
900
+ "logits/chosen": 2.434805154800415,
901
+ "logits/rejected": 2.456228256225586,
902
+ "logps/chosen": -20.521100997924805,
903
+ "logps/rejected": -28.69353485107422,
904
+ "loss": 0.3295,
905
+ "rewards/accuracies": 0.9375,
906
+ "rewards/chosen": 0.3444966673851013,
907
+ "rewards/margins": 2.016552448272705,
908
+ "rewards/rejected": -1.672055959701538,
909
+ "step": 60
910
+ },
911
+ {
912
+ "epoch": 1.5443037974683544,
913
+ "grad_norm": 33.402955887536194,
914
+ "learning_rate": 3.8300801912883414e-07,
915
+ "logits/chosen": 2.511925220489502,
916
+ "logits/rejected": 2.43487286567688,
917
+ "logps/chosen": -30.91568946838379,
918
+ "logps/rejected": -38.48870086669922,
919
+ "loss": 0.297,
920
+ "rewards/accuracies": 0.9375,
921
+ "rewards/chosen": -0.009598510339856148,
922
+ "rewards/margins": 1.2743103504180908,
923
+ "rewards/rejected": -1.2839089632034302,
924
+ "step": 61
925
+ },
926
+ {
927
+ "epoch": 1.5696202531645569,
928
+ "grad_norm": 34.221583251712744,
929
+ "learning_rate": 3.7822481935147655e-07,
930
+ "logits/chosen": 2.2030210494995117,
931
+ "logits/rejected": 2.4195468425750732,
932
+ "logps/chosen": -26.423166275024414,
933
+ "logps/rejected": -36.79461669921875,
934
+ "loss": 0.3281,
935
+ "rewards/accuracies": 0.8125,
936
+ "rewards/chosen": -0.2358216643333435,
937
+ "rewards/margins": 1.8056204319000244,
938
+ "rewards/rejected": -2.0414421558380127,
939
+ "step": 62
940
+ },
941
+ {
942
+ "epoch": 1.5949367088607596,
943
+ "grad_norm": 27.39193509195394,
944
+ "learning_rate": 3.7337705451608667e-07,
945
+ "logits/chosen": 2.442267894744873,
946
+ "logits/rejected": 2.6541049480438232,
947
+ "logps/chosen": -24.891948699951172,
948
+ "logps/rejected": -40.318904876708984,
949
+ "loss": 0.2564,
950
+ "rewards/accuracies": 0.9375,
951
+ "rewards/chosen": -0.08293318003416061,
952
+ "rewards/margins": 1.7536163330078125,
953
+ "rewards/rejected": -1.8365494012832642,
954
+ "step": 63
955
+ },
956
+ {
957
+ "epoch": 1.620253164556962,
958
+ "grad_norm": 32.95811212676831,
959
+ "learning_rate": 3.6846716561824967e-07,
960
+ "logits/chosen": 2.404311180114746,
961
+ "logits/rejected": 2.50986385345459,
962
+ "logps/chosen": -24.17348861694336,
963
+ "logps/rejected": -35.517120361328125,
964
+ "loss": 0.3643,
965
+ "rewards/accuracies": 0.9375,
966
+ "rewards/chosen": -0.00808901246637106,
967
+ "rewards/margins": 1.9171310663223267,
968
+ "rewards/rejected": -1.925220251083374,
969
+ "step": 64
970
+ },
971
+ {
972
+ "epoch": 1.6455696202531644,
973
+ "grad_norm": 31.89102457068134,
974
+ "learning_rate": 3.634976249348867e-07,
975
+ "logits/chosen": 1.8697429895401,
976
+ "logits/rejected": 1.9369537830352783,
977
+ "logps/chosen": -26.27237319946289,
978
+ "logps/rejected": -43.584659576416016,
979
+ "loss": 0.3047,
980
+ "rewards/accuracies": 0.875,
981
+ "rewards/chosen": 0.018543783575296402,
982
+ "rewards/margins": 1.540452480316162,
983
+ "rewards/rejected": -1.5219086408615112,
984
+ "step": 65
985
+ },
986
+ {
987
+ "epoch": 1.6708860759493671,
988
+ "grad_norm": 29.55969704854734,
989
+ "learning_rate": 3.584709347793895e-07,
990
+ "logits/chosen": 2.12164044380188,
991
+ "logits/rejected": 2.12618350982666,
992
+ "logps/chosen": -19.538429260253906,
993
+ "logps/rejected": -34.374759674072266,
994
+ "loss": 0.3159,
995
+ "rewards/accuracies": 0.875,
996
+ "rewards/chosen": 0.1228712797164917,
997
+ "rewards/margins": 1.97737455368042,
998
+ "rewards/rejected": -1.8545031547546387,
999
+ "step": 66
1000
+ },
1001
+ {
1002
+ "epoch": 1.6962025316455698,
1003
+ "grad_norm": 29.63374295371732,
1004
+ "learning_rate": 3.5338962624163016e-07,
1005
+ "logits/chosen": 2.3299381732940674,
1006
+ "logits/rejected": 2.5256855487823486,
1007
+ "logps/chosen": -24.86815643310547,
1008
+ "logps/rejected": -32.63747024536133,
1009
+ "loss": 0.304,
1010
+ "rewards/accuracies": 0.75,
1011
+ "rewards/chosen": -0.14013400673866272,
1012
+ "rewards/margins": 1.9178043603897095,
1013
+ "rewards/rejected": -2.05793833732605,
1014
+ "step": 67
1015
+ },
1016
+ {
1017
+ "epoch": 1.721518987341772,
1018
+ "grad_norm": 29.816270030062956,
1019
+ "learning_rate": 3.482562579134809e-07,
1020
+ "logits/chosen": 2.947877883911133,
1021
+ "logits/rejected": 2.919872760772705,
1022
+ "logps/chosen": -24.169422149658203,
1023
+ "logps/rejected": -35.171504974365234,
1024
+ "loss": 0.2947,
1025
+ "rewards/accuracies": 0.9375,
1026
+ "rewards/chosen": 0.18604925274848938,
1027
+ "rewards/margins": 2.742734432220459,
1028
+ "rewards/rejected": -2.556685209274292,
1029
+ "step": 68
1030
+ },
1031
+ {
1032
+ "epoch": 1.7468354430379747,
1033
+ "grad_norm": 28.204359911989563,
1034
+ "learning_rate": 3.430734146004863e-07,
1035
+ "logits/chosen": 2.15493106842041,
1036
+ "logits/rejected": 2.3170015811920166,
1037
+ "logps/chosen": -22.2800350189209,
1038
+ "logps/rejected": -35.93194580078125,
1039
+ "loss": 0.2844,
1040
+ "rewards/accuracies": 0.875,
1041
+ "rewards/chosen": -0.17889335751533508,
1042
+ "rewards/margins": 2.3182311058044434,
1043
+ "rewards/rejected": -2.497124433517456,
1044
+ "step": 69
1045
+ },
1046
+ {
1047
+ "epoch": 1.7721518987341773,
1048
+ "grad_norm": 26.787627596958465,
1049
+ "learning_rate": 3.378437060203357e-07,
1050
+ "logits/chosen": 2.5400497913360596,
1051
+ "logits/rejected": 2.5072226524353027,
1052
+ "logps/chosen": -21.618453979492188,
1053
+ "logps/rejected": -33.122589111328125,
1054
+ "loss": 0.2766,
1055
+ "rewards/accuracies": 1.0,
1056
+ "rewards/chosen": 0.05331309512257576,
1057
+ "rewards/margins": 2.7737677097320557,
1058
+ "rewards/rejected": -2.720454692840576,
1059
+ "step": 70
1060
+ },
1061
+ {
1062
+ "epoch": 1.7974683544303798,
1063
+ "grad_norm": 31.205788121930055,
1064
+ "learning_rate": 3.325697654887918e-07,
1065
+ "logits/chosen": 1.4359736442565918,
1066
+ "logits/rejected": 1.0823308229446411,
1067
+ "logps/chosen": -26.589662551879883,
1068
+ "logps/rejected": -38.18647766113281,
1069
+ "loss": 0.2822,
1070
+ "rewards/accuracies": 0.9375,
1071
+ "rewards/chosen": 0.2071526050567627,
1072
+ "rewards/margins": 2.4934191703796387,
1073
+ "rewards/rejected": -2.286266565322876,
1074
+ "step": 71
1075
+ },
1076
+ {
1077
+ "epoch": 1.8227848101265822,
1078
+ "grad_norm": 29.788415172955876,
1079
+ "learning_rate": 3.272542485937368e-07,
1080
+ "logits/chosen": 2.800675868988037,
1081
+ "logits/rejected": 2.9707438945770264,
1082
+ "logps/chosen": -25.928173065185547,
1083
+ "logps/rejected": -38.25574493408203,
1084
+ "loss": 0.2955,
1085
+ "rewards/accuracies": 1.0,
1086
+ "rewards/chosen": 0.0182593185454607,
1087
+ "rewards/margins": 2.1046228408813477,
1088
+ "rewards/rejected": -2.0863637924194336,
1089
+ "step": 72
1090
+ },
1091
+ {
1092
+ "epoch": 1.8481012658227849,
1093
+ "grad_norm": 31.197362620805283,
1094
+ "learning_rate": 3.218998318580043e-07,
1095
+ "logits/chosen": 2.2215137481689453,
1096
+ "logits/rejected": 2.1676788330078125,
1097
+ "logps/chosen": -23.131969451904297,
1098
+ "logps/rejected": -34.484920501708984,
1099
+ "loss": 0.276,
1100
+ "rewards/accuracies": 0.9375,
1101
+ "rewards/chosen": 0.19679217040538788,
1102
+ "rewards/margins": 2.1039047241210938,
1103
+ "rewards/rejected": -1.9071124792099,
1104
+ "step": 73
1105
+ },
1106
+ {
1107
+ "epoch": 1.8734177215189873,
1108
+ "grad_norm": 27.14206740038897,
1109
+ "learning_rate": 3.1650921139166874e-07,
1110
+ "logits/chosen": 2.0885462760925293,
1111
+ "logits/rejected": 2.1695892810821533,
1112
+ "logps/chosen": -24.332992553710938,
1113
+ "logps/rejected": -39.35213088989258,
1114
+ "loss": 0.2598,
1115
+ "rewards/accuracies": 1.0,
1116
+ "rewards/chosen": 0.13339604437351227,
1117
+ "rewards/margins": 2.6932549476623535,
1118
+ "rewards/rejected": -2.559858798980713,
1119
+ "step": 74
1120
+ },
1121
+ {
1122
+ "epoch": 1.8987341772151898,
1123
+ "grad_norm": 31.93987675909966,
1124
+ "learning_rate": 3.110851015344735e-07,
1125
+ "logits/chosen": 2.1219115257263184,
1126
+ "logits/rejected": 2.1965231895446777,
1127
+ "logps/chosen": -25.27581024169922,
1128
+ "logps/rejected": -34.003536224365234,
1129
+ "loss": 0.3142,
1130
+ "rewards/accuracies": 1.0,
1131
+ "rewards/chosen": 0.1961415410041809,
1132
+ "rewards/margins": 2.4378857612609863,
1133
+ "rewards/rejected": -2.24174427986145,
1134
+ "step": 75
1135
+ },
1136
+ {
1137
+ "epoch": 1.9240506329113924,
1138
+ "grad_norm": 29.021478328579693,
1139
+ "learning_rate": 3.056302334890786e-07,
1140
+ "logits/chosen": 1.8746654987335205,
1141
+ "logits/rejected": 2.389371871948242,
1142
+ "logps/chosen": -22.446157455444336,
1143
+ "logps/rejected": -39.833770751953125,
1144
+ "loss": 0.2671,
1145
+ "rewards/accuracies": 1.0,
1146
+ "rewards/chosen": 0.18668973445892334,
1147
+ "rewards/margins": 3.256779909133911,
1148
+ "rewards/rejected": -3.0700900554656982,
1149
+ "step": 76
1150
+ },
1151
+ {
1152
+ "epoch": 1.9493670886075949,
1153
+ "grad_norm": 31.861473137885586,
1154
+ "learning_rate": 3.001473539458182e-07,
1155
+ "logits/chosen": 2.0852279663085938,
1156
+ "logits/rejected": 2.2871861457824707,
1157
+ "logps/chosen": -27.48174285888672,
1158
+ "logps/rejected": -38.38823699951172,
1159
+ "loss": 0.2623,
1160
+ "rewards/accuracies": 0.9375,
1161
+ "rewards/chosen": 0.12376511096954346,
1162
+ "rewards/margins": 2.879195213317871,
1163
+ "rewards/rejected": -2.755429983139038,
1164
+ "step": 77
1165
+ },
1166
+ {
1167
+ "epoch": 1.9746835443037973,
1168
+ "grad_norm": 30.580581160948984,
1169
+ "learning_rate": 2.9463922369965915e-07,
1170
+ "logits/chosen": 2.183192253112793,
1171
+ "logits/rejected": 1.8703649044036865,
1172
+ "logps/chosen": -25.529539108276367,
1173
+ "logps/rejected": -38.35371780395508,
1174
+ "loss": 0.2812,
1175
+ "rewards/accuracies": 0.875,
1176
+ "rewards/chosen": -0.09511762112379074,
1177
+ "rewards/margins": 1.8313283920288086,
1178
+ "rewards/rejected": -1.9264459609985352,
1179
+ "step": 78
1180
+ },
1181
+ {
1182
+ "epoch": 2.0,
1183
+ "grad_norm": 28.28932497860918,
1184
+ "learning_rate": 2.8910861626005773e-07,
1185
+ "logits/chosen": 2.1175851821899414,
1186
+ "logits/rejected": 2.7737534046173096,
1187
+ "logps/chosen": -22.948108673095703,
1188
+ "logps/rejected": -54.805484771728516,
1189
+ "loss": 0.2737,
1190
+ "rewards/accuracies": 1.0,
1191
+ "rewards/chosen": -0.07266075909137726,
1192
+ "rewards/margins": 2.9845314025878906,
1193
+ "rewards/rejected": -3.057192087173462,
1194
+ "step": 79
1195
+ },
1196
+ {
1197
+ "epoch": 2.0253164556962027,
1198
+ "grad_norm": 22.84435575128268,
1199
+ "learning_rate": 2.8355831645441387e-07,
1200
+ "logits/chosen": 2.129598617553711,
1201
+ "logits/rejected": 2.214559316635132,
1202
+ "logps/chosen": -33.85695266723633,
1203
+ "logps/rejected": -40.15086364746094,
1204
+ "loss": 0.2116,
1205
+ "rewards/accuracies": 1.0,
1206
+ "rewards/chosen": -0.04292801395058632,
1207
+ "rewards/margins": 2.009500741958618,
1208
+ "rewards/rejected": -2.052428722381592,
1209
+ "step": 80
1210
+ },
1211
+ {
1212
+ "epoch": 2.050632911392405,
1213
+ "grad_norm": 22.68444850198704,
1214
+ "learning_rate": 2.7799111902582693e-07,
1215
+ "logits/chosen": 2.512767791748047,
1216
+ "logits/rejected": 2.5582704544067383,
1217
+ "logps/chosen": -25.28483009338379,
1218
+ "logps/rejected": -36.232017517089844,
1219
+ "loss": 0.2011,
1220
+ "rewards/accuracies": 1.0,
1221
+ "rewards/chosen": 0.14101354777812958,
1222
+ "rewards/margins": 2.3776185512542725,
1223
+ "rewards/rejected": -2.236604690551758,
1224
+ "step": 81
1225
+ },
1226
+ {
1227
+ "epoch": 2.0759493670886076,
1228
+ "grad_norm": 23.440024916384186,
1229
+ "learning_rate": 2.7240982722585837e-07,
1230
+ "logits/chosen": 1.941485047340393,
1231
+ "logits/rejected": 2.076082944869995,
1232
+ "logps/chosen": -25.91452407836914,
1233
+ "logps/rejected": -34.40084457397461,
1234
+ "loss": 0.2306,
1235
+ "rewards/accuracies": 0.9375,
1236
+ "rewards/chosen": -0.1705533266067505,
1237
+ "rewards/margins": 1.7886393070220947,
1238
+ "rewards/rejected": -1.9591926336288452,
1239
+ "step": 82
1240
+ },
1241
+ {
1242
+ "epoch": 2.1012658227848102,
1243
+ "grad_norm": 24.39463073368113,
1244
+ "learning_rate": 2.6681725140300995e-07,
1245
+ "logits/chosen": 1.5756129026412964,
1246
+ "logits/rejected": 1.9195802211761475,
1247
+ "logps/chosen": -21.2039737701416,
1248
+ "logps/rejected": -43.85890579223633,
1249
+ "loss": 0.2048,
1250
+ "rewards/accuracies": 1.0,
1251
+ "rewards/chosen": -0.012569274753332138,
1252
+ "rewards/margins": 3.1985719203948975,
1253
+ "rewards/rejected": -3.2111411094665527,
1254
+ "step": 83
1255
+ },
1256
+ {
1257
+ "epoch": 2.1265822784810124,
1258
+ "grad_norm": 21.14214956646525,
1259
+ "learning_rate": 2.6121620758762875e-07,
1260
+ "logits/chosen": 2.5736641883850098,
1261
+ "logits/rejected": 2.7494866847991943,
1262
+ "logps/chosen": -27.16839027404785,
1263
+ "logps/rejected": -37.205440521240234,
1264
+ "loss": 0.1933,
1265
+ "rewards/accuracies": 1.0,
1266
+ "rewards/chosen": 0.010486233979463577,
1267
+ "rewards/margins": 2.571336269378662,
1268
+ "rewards/rejected": -2.560849905014038,
1269
+ "step": 84
1270
+ },
1271
+ {
1272
+ "epoch": 2.151898734177215,
1273
+ "grad_norm": 21.59512651806526,
1274
+ "learning_rate": 2.5560951607395126e-07,
1275
+ "logits/chosen": 2.4533567428588867,
1276
+ "logits/rejected": 2.3579037189483643,
1277
+ "logps/chosen": -28.007740020751953,
1278
+ "logps/rejected": -36.33196258544922,
1279
+ "loss": 0.192,
1280
+ "rewards/accuracies": 0.875,
1281
+ "rewards/chosen": 0.04641330614686012,
1282
+ "rewards/margins": 1.9481606483459473,
1283
+ "rewards/rejected": -1.901747465133667,
1284
+ "step": 85
1285
+ },
1286
+ {
1287
+ "epoch": 2.1772151898734178,
1288
+ "grad_norm": 24.879344750048478,
1289
+ "learning_rate": 2.5e-07,
1290
+ "logits/chosen": 2.1627023220062256,
1291
+ "logits/rejected": 2.059532403945923,
1292
+ "logps/chosen": -32.905269622802734,
1293
+ "logps/rejected": -31.235652923583984,
1294
+ "loss": 0.2379,
1295
+ "rewards/accuracies": 0.9375,
1296
+ "rewards/chosen": -0.02136234939098358,
1297
+ "rewards/margins": 1.5867081880569458,
1298
+ "rewards/rejected": -1.6080706119537354,
1299
+ "step": 86
1300
+ },
1301
+ {
1302
+ "epoch": 2.2025316455696204,
1303
+ "grad_norm": 22.69000503277292,
1304
+ "learning_rate": 2.4439048392604877e-07,
1305
+ "logits/chosen": 2.3072502613067627,
1306
+ "logits/rejected": 2.6580121517181396,
1307
+ "logps/chosen": -23.412647247314453,
1308
+ "logps/rejected": -42.0001106262207,
1309
+ "loss": 0.2005,
1310
+ "rewards/accuracies": 1.0,
1311
+ "rewards/chosen": -0.16801214218139648,
1312
+ "rewards/margins": 2.332923412322998,
1313
+ "rewards/rejected": -2.5009355545043945,
1314
+ "step": 87
1315
+ },
1316
+ {
1317
+ "epoch": 2.2278481012658227,
1318
+ "grad_norm": 24.693555115896494,
1319
+ "learning_rate": 2.3878379241237134e-07,
1320
+ "logits/chosen": 2.4124932289123535,
1321
+ "logits/rejected": 2.2819831371307373,
1322
+ "logps/chosen": -25.710378646850586,
1323
+ "logps/rejected": -32.972537994384766,
1324
+ "loss": 0.2211,
1325
+ "rewards/accuracies": 1.0,
1326
+ "rewards/chosen": 0.0028405077755451202,
1327
+ "rewards/margins": 2.1923816204071045,
1328
+ "rewards/rejected": -2.1895413398742676,
1329
+ "step": 88
1330
+ },
1331
+ {
1332
+ "epoch": 2.2531645569620253,
1333
+ "grad_norm": 22.755579101890042,
1334
+ "learning_rate": 2.3318274859699008e-07,
1335
+ "logits/chosen": 2.006254196166992,
1336
+ "logits/rejected": 2.4240238666534424,
1337
+ "logps/chosen": -26.35239028930664,
1338
+ "logps/rejected": -42.78788757324219,
1339
+ "loss": 0.2437,
1340
+ "rewards/accuracies": 0.875,
1341
+ "rewards/chosen": -0.381151020526886,
1342
+ "rewards/margins": 2.395555019378662,
1343
+ "rewards/rejected": -2.7767062187194824,
1344
+ "step": 89
1345
+ },
1346
+ {
1347
+ "epoch": 2.278481012658228,
1348
+ "grad_norm": 23.73790882415418,
1349
+ "learning_rate": 2.2759017277414164e-07,
1350
+ "logits/chosen": 2.482501745223999,
1351
+ "logits/rejected": 2.456974744796753,
1352
+ "logps/chosen": -32.156795501708984,
1353
+ "logps/rejected": -35.610713958740234,
1354
+ "loss": 0.2328,
1355
+ "rewards/accuracies": 1.0,
1356
+ "rewards/chosen": -0.08202718943357468,
1357
+ "rewards/margins": 1.4416414499282837,
1358
+ "rewards/rejected": -1.523668646812439,
1359
+ "step": 90
1360
+ },
1361
+ {
1362
+ "epoch": 2.3037974683544302,
1363
+ "grad_norm": 22.47212835812806,
1364
+ "learning_rate": 2.2200888097417302e-07,
1365
+ "logits/chosen": 2.5853986740112305,
1366
+ "logits/rejected": 2.6601390838623047,
1367
+ "logps/chosen": -33.39643859863281,
1368
+ "logps/rejected": -43.778709411621094,
1369
+ "loss": 0.1828,
1370
+ "rewards/accuracies": 0.9375,
1371
+ "rewards/chosen": -0.010579407215118408,
1372
+ "rewards/margins": 2.473262310028076,
1373
+ "rewards/rejected": -2.483841896057129,
1374
+ "step": 91
1375
+ },
1376
+ {
1377
+ "epoch": 2.329113924050633,
1378
+ "grad_norm": 22.509363583762184,
1379
+ "learning_rate": 2.164416835455862e-07,
1380
+ "logits/chosen": 2.3972582817077637,
1381
+ "logits/rejected": 2.534813642501831,
1382
+ "logps/chosen": -22.962604522705078,
1383
+ "logps/rejected": -37.55415344238281,
1384
+ "loss": 0.2213,
1385
+ "rewards/accuracies": 1.0,
1386
+ "rewards/chosen": -0.0858815610408783,
1387
+ "rewards/margins": 2.756964683532715,
1388
+ "rewards/rejected": -2.842846155166626,
1389
+ "step": 92
1390
+ },
1391
+ {
1392
+ "epoch": 2.3544303797468356,
1393
+ "grad_norm": 22.253217260948635,
1394
+ "learning_rate": 2.1089138373994222e-07,
1395
+ "logits/chosen": 2.153351306915283,
1396
+ "logits/rejected": 2.3578603267669678,
1397
+ "logps/chosen": -22.811275482177734,
1398
+ "logps/rejected": -39.044639587402344,
1399
+ "loss": 0.1953,
1400
+ "rewards/accuracies": 1.0,
1401
+ "rewards/chosen": -0.2400507628917694,
1402
+ "rewards/margins": 2.045787811279297,
1403
+ "rewards/rejected": -2.2858386039733887,
1404
+ "step": 93
1405
+ },
1406
+ {
1407
+ "epoch": 2.379746835443038,
1408
+ "grad_norm": 23.13742394622883,
1409
+ "learning_rate": 2.0536077630034085e-07,
1410
+ "logits/chosen": 1.7368831634521484,
1411
+ "logits/rejected": 1.9436362981796265,
1412
+ "logps/chosen": -26.176815032958984,
1413
+ "logps/rejected": -40.83625411987305,
1414
+ "loss": 0.2061,
1415
+ "rewards/accuracies": 1.0,
1416
+ "rewards/chosen": 0.39153048396110535,
1417
+ "rewards/margins": 2.8905537128448486,
1418
+ "rewards/rejected": -2.499023199081421,
1419
+ "step": 94
1420
+ },
1421
+ {
1422
+ "epoch": 2.4050632911392404,
1423
+ "grad_norm": 21.94757471208971,
1424
+ "learning_rate": 1.998526460541818e-07,
1425
+ "logits/chosen": 2.3502449989318848,
1426
+ "logits/rejected": 2.3544671535491943,
1427
+ "logps/chosen": -26.833280563354492,
1428
+ "logps/rejected": -37.404605865478516,
1429
+ "loss": 0.1962,
1430
+ "rewards/accuracies": 0.875,
1431
+ "rewards/chosen": 0.0008228495717048645,
1432
+ "rewards/margins": 2.2684872150421143,
1433
+ "rewards/rejected": -2.2676644325256348,
1434
+ "step": 95
1435
+ },
1436
+ {
1437
+ "epoch": 2.430379746835443,
1438
+ "grad_norm": 20.124080148253146,
1439
+ "learning_rate": 1.9436976651092142e-07,
1440
+ "logits/chosen": 1.1564311981201172,
1441
+ "logits/rejected": 1.8700743913650513,
1442
+ "logps/chosen": -25.664215087890625,
1443
+ "logps/rejected": -42.25444030761719,
1444
+ "loss": 0.1803,
1445
+ "rewards/accuracies": 1.0,
1446
+ "rewards/chosen": 0.013121239840984344,
1447
+ "rewards/margins": 2.5192627906799316,
1448
+ "rewards/rejected": -2.5061416625976562,
1449
+ "step": 96
1450
+ },
1451
+ {
1452
+ "epoch": 2.4556962025316453,
1453
+ "grad_norm": 22.336736253170702,
1454
+ "learning_rate": 1.8891489846552644e-07,
1455
+ "logits/chosen": 2.3158211708068848,
1456
+ "logits/rejected": 2.718118667602539,
1457
+ "logps/chosen": -23.182619094848633,
1458
+ "logps/rejected": -41.77816390991211,
1459
+ "loss": 0.2237,
1460
+ "rewards/accuracies": 1.0,
1461
+ "rewards/chosen": 0.010374765843153,
1462
+ "rewards/margins": 2.8450756072998047,
1463
+ "rewards/rejected": -2.8347012996673584,
1464
+ "step": 97
1465
+ },
1466
+ {
1467
+ "epoch": 2.481012658227848,
1468
+ "grad_norm": 21.95104798778307,
1469
+ "learning_rate": 1.8349078860833124e-07,
1470
+ "logits/chosen": 2.2794296741485596,
1471
+ "logits/rejected": 2.176607847213745,
1472
+ "logps/chosen": -33.21621322631836,
1473
+ "logps/rejected": -38.940025329589844,
1474
+ "loss": 0.1762,
1475
+ "rewards/accuracies": 1.0,
1476
+ "rewards/chosen": -0.013390496373176575,
1477
+ "rewards/margins": 2.8369154930114746,
1478
+ "rewards/rejected": -2.8503060340881348,
1479
+ "step": 98
1480
+ },
1481
+ {
1482
+ "epoch": 2.5063291139240507,
1483
+ "grad_norm": 22.129805040414322,
1484
+ "learning_rate": 1.781001681419957e-07,
1485
+ "logits/chosen": 1.8799982070922852,
1486
+ "logits/rejected": 2.1679162979125977,
1487
+ "logps/chosen": -20.39841079711914,
1488
+ "logps/rejected": -40.38523864746094,
1489
+ "loss": 0.2032,
1490
+ "rewards/accuracies": 0.875,
1491
+ "rewards/chosen": 0.04029037803411484,
1492
+ "rewards/margins": 2.2781383991241455,
1493
+ "rewards/rejected": -2.2378478050231934,
1494
+ "step": 99
1495
+ },
1496
+ {
1497
+ "epoch": 2.5316455696202533,
1498
+ "grad_norm": 18.828604860640993,
1499
+ "learning_rate": 1.7274575140626315e-07,
1500
+ "logits/chosen": 2.4270401000976562,
1501
+ "logits/rejected": 2.768435478210449,
1502
+ "logps/chosen": -28.433048248291016,
1503
+ "logps/rejected": -47.266197204589844,
1504
+ "loss": 0.1502,
1505
+ "rewards/accuracies": 1.0,
1506
+ "rewards/chosen": -0.2632801830768585,
1507
+ "rewards/margins": 2.7442736625671387,
1508
+ "rewards/rejected": -3.007554054260254,
1509
+ "step": 100
1510
+ },
1511
+ {
1512
+ "epoch": 2.5569620253164556,
1513
+ "grad_norm": 20.880074717956166,
1514
+ "learning_rate": 1.674302345112083e-07,
1515
+ "logits/chosen": 1.9880956411361694,
1516
+ "logits/rejected": 2.2008445262908936,
1517
+ "logps/chosen": -24.262271881103516,
1518
+ "logps/rejected": -45.60854721069336,
1519
+ "loss": 0.2089,
1520
+ "rewards/accuracies": 1.0,
1521
+ "rewards/chosen": 0.04222775995731354,
1522
+ "rewards/margins": 3.01908802986145,
1523
+ "rewards/rejected": -2.9768600463867188,
1524
+ "step": 101
1525
+ },
1526
+ {
1527
+ "epoch": 2.5822784810126582,
1528
+ "grad_norm": 21.838535925363054,
1529
+ "learning_rate": 1.621562939796643e-07,
1530
+ "logits/chosen": 2.205430746078491,
1531
+ "logits/rejected": 2.4456026554107666,
1532
+ "logps/chosen": -24.633563995361328,
1533
+ "logps/rejected": -37.809478759765625,
1534
+ "loss": 0.2187,
1535
+ "rewards/accuracies": 1.0,
1536
+ "rewards/chosen": 0.2103780210018158,
1537
+ "rewards/margins": 2.4951250553131104,
1538
+ "rewards/rejected": -2.2847468852996826,
1539
+ "step": 102
1540
+ },
1541
+ {
1542
+ "epoch": 2.607594936708861,
1543
+ "grad_norm": 22.277589914673204,
1544
+ "learning_rate": 1.569265853995137e-07,
1545
+ "logits/chosen": 1.4377330541610718,
1546
+ "logits/rejected": 1.4871635437011719,
1547
+ "logps/chosen": -24.512638092041016,
1548
+ "logps/rejected": -37.595733642578125,
1549
+ "loss": 0.1964,
1550
+ "rewards/accuracies": 0.9375,
1551
+ "rewards/chosen": -0.3236038088798523,
1552
+ "rewards/margins": 2.621075391769409,
1553
+ "rewards/rejected": -2.9446792602539062,
1554
+ "step": 103
1555
+ },
1556
+ {
1557
+ "epoch": 2.632911392405063,
1558
+ "grad_norm": 21.07481872439906,
1559
+ "learning_rate": 1.517437420865191e-07,
1560
+ "logits/chosen": 1.6525717973709106,
1561
+ "logits/rejected": 2.0419435501098633,
1562
+ "logps/chosen": -24.287813186645508,
1563
+ "logps/rejected": -43.26752853393555,
1564
+ "loss": 0.1873,
1565
+ "rewards/accuracies": 1.0,
1566
+ "rewards/chosen": 0.2492070347070694,
1567
+ "rewards/margins": 3.5699872970581055,
1568
+ "rewards/rejected": -3.3207802772521973,
1569
+ "step": 104
1570
+ },
1571
+ {
1572
+ "epoch": 2.6582278481012658,
1573
+ "grad_norm": 20.677444088444823,
1574
+ "learning_rate": 1.4661037375836987e-07,
1575
+ "logits/chosen": 2.6604502201080322,
1576
+ "logits/rejected": 2.587705373764038,
1577
+ "logps/chosen": -25.429487228393555,
1578
+ "logps/rejected": -36.87110137939453,
1579
+ "loss": 0.1855,
1580
+ "rewards/accuracies": 0.9375,
1581
+ "rewards/chosen": -0.07871632277965546,
1582
+ "rewards/margins": 2.1279850006103516,
1583
+ "rewards/rejected": -2.2067012786865234,
1584
+ "step": 105
1585
+ },
1586
+ {
1587
+ "epoch": 2.6835443037974684,
1588
+ "grad_norm": 23.5751426913938,
1589
+ "learning_rate": 1.4152906522061047e-07,
1590
+ "logits/chosen": 2.1078615188598633,
1591
+ "logits/rejected": 2.089411735534668,
1592
+ "logps/chosen": -25.450345993041992,
1593
+ "logps/rejected": -33.47675323486328,
1594
+ "loss": 0.1944,
1595
+ "rewards/accuracies": 0.9375,
1596
+ "rewards/chosen": 0.11998379230499268,
1597
+ "rewards/margins": 2.037046432495117,
1598
+ "rewards/rejected": -1.917062759399414,
1599
+ "step": 106
1600
+ },
1601
+ {
1602
+ "epoch": 2.708860759493671,
1603
+ "grad_norm": 22.767643882863975,
1604
+ "learning_rate": 1.365023750651133e-07,
1605
+ "logits/chosen": 1.7534555196762085,
1606
+ "logits/rejected": 2.2760560512542725,
1607
+ "logps/chosen": -27.695234298706055,
1608
+ "logps/rejected": -46.436767578125,
1609
+ "loss": 0.1638,
1610
+ "rewards/accuracies": 1.0,
1611
+ "rewards/chosen": -0.030735721811652184,
1612
+ "rewards/margins": 3.136977434158325,
1613
+ "rewards/rejected": -3.1677134037017822,
1614
+ "step": 107
1615
+ },
1616
+ {
1617
+ "epoch": 2.7341772151898733,
1618
+ "grad_norm": 20.58426575505033,
1619
+ "learning_rate": 1.3153283438175034e-07,
1620
+ "logits/chosen": 2.3597166538238525,
1621
+ "logits/rejected": 2.5851235389709473,
1622
+ "logps/chosen": -23.936431884765625,
1623
+ "logps/rejected": -47.1025390625,
1624
+ "loss": 0.1785,
1625
+ "rewards/accuracies": 1.0,
1626
+ "rewards/chosen": -0.1647489368915558,
1627
+ "rewards/margins": 3.3675107955932617,
1628
+ "rewards/rejected": -3.532259941101074,
1629
+ "step": 108
1630
+ },
1631
+ {
1632
+ "epoch": 2.759493670886076,
1633
+ "grad_norm": 21.349116101829402,
1634
+ "learning_rate": 1.2662294548391328e-07,
1635
+ "logits/chosen": 1.9581003189086914,
1636
+ "logits/rejected": 1.994449496269226,
1637
+ "logps/chosen": -28.851770401000977,
1638
+ "logps/rejected": -39.374019622802734,
1639
+ "loss": 0.1772,
1640
+ "rewards/accuracies": 0.9375,
1641
+ "rewards/chosen": 0.023888343945145607,
1642
+ "rewards/margins": 2.033907890319824,
1643
+ "rewards/rejected": -2.0100197792053223,
1644
+ "step": 109
1645
+ },
1646
+ {
1647
+ "epoch": 2.7848101265822782,
1648
+ "grad_norm": 20.523600951939002,
1649
+ "learning_rate": 1.2177518064852348e-07,
1650
+ "logits/chosen": 2.149280071258545,
1651
+ "logits/rejected": 2.338773012161255,
1652
+ "logps/chosen": -24.151710510253906,
1653
+ "logps/rejected": -44.250370025634766,
1654
+ "loss": 0.197,
1655
+ "rewards/accuracies": 1.0,
1656
+ "rewards/chosen": 0.008578717708587646,
1657
+ "rewards/margins": 2.208198308944702,
1658
+ "rewards/rejected": -2.1996195316314697,
1659
+ "step": 110
1660
+ },
1661
+ {
1662
+ "epoch": 2.810126582278481,
1663
+ "grad_norm": 21.793053109415474,
1664
+ "learning_rate": 1.1699198087116588e-07,
1665
+ "logits/chosen": 1.9670238494873047,
1666
+ "logits/rejected": 2.2674317359924316,
1667
+ "logps/chosen": -19.423011779785156,
1668
+ "logps/rejected": -43.466705322265625,
1669
+ "loss": 0.1912,
1670
+ "rewards/accuracies": 1.0,
1671
+ "rewards/chosen": -0.0942595824599266,
1672
+ "rewards/margins": 2.482712984085083,
1673
+ "rewards/rejected": -2.576972484588623,
1674
+ "step": 111
1675
+ },
1676
+ {
1677
+ "epoch": 2.8354430379746836,
1678
+ "grad_norm": 18.7267582468967,
1679
+ "learning_rate": 1.1227575463697439e-07,
1680
+ "logits/chosen": 2.125753402709961,
1681
+ "logits/rejected": 2.3489747047424316,
1682
+ "logps/chosen": -24.716629028320312,
1683
+ "logps/rejected": -35.32937240600586,
1684
+ "loss": 0.1497,
1685
+ "rewards/accuracies": 1.0,
1686
+ "rewards/chosen": -0.06388971209526062,
1687
+ "rewards/margins": 2.4911532402038574,
1688
+ "rewards/rejected": -2.5550432205200195,
1689
+ "step": 112
1690
+ },
1691
+ {
1692
+ "epoch": 2.8607594936708862,
1693
+ "grad_norm": 21.953907512144703,
1694
+ "learning_rate": 1.0762887670788701e-07,
1695
+ "logits/chosen": 1.75847327709198,
1696
+ "logits/rejected": 1.766452431678772,
1697
+ "logps/chosen": -33.99777603149414,
1698
+ "logps/rejected": -49.58311080932617,
1699
+ "loss": 0.1887,
1700
+ "rewards/accuracies": 1.0,
1701
+ "rewards/chosen": -0.09381230175495148,
1702
+ "rewards/margins": 3.5883729457855225,
1703
+ "rewards/rejected": -3.682185649871826,
1704
+ "step": 113
1705
+ },
1706
+ {
1707
+ "epoch": 2.8860759493670884,
1708
+ "grad_norm": 21.263544038283367,
1709
+ "learning_rate": 1.0305368692688174e-07,
1710
+ "logits/chosen": 1.9139025211334229,
1711
+ "logits/rejected": 1.8650161027908325,
1712
+ "logps/chosen": -17.992523193359375,
1713
+ "logps/rejected": -38.388519287109375,
1714
+ "loss": 0.1854,
1715
+ "rewards/accuracies": 1.0,
1716
+ "rewards/chosen": 0.09737363457679749,
1717
+ "rewards/margins": 3.4405698776245117,
1718
+ "rewards/rejected": -3.3431966304779053,
1719
+ "step": 114
1720
+ },
1721
+ {
1722
+ "epoch": 2.911392405063291,
1723
+ "grad_norm": 19.070922554256914,
1724
+ "learning_rate": 9.855248903979505e-08,
1725
+ "logits/chosen": 2.087536096572876,
1726
+ "logits/rejected": 2.118912935256958,
1727
+ "logps/chosen": -30.79559326171875,
1728
+ "logps/rejected": -47.3471565246582,
1729
+ "loss": 0.1693,
1730
+ "rewards/accuracies": 1.0,
1731
+ "rewards/chosen": -0.07975523173809052,
1732
+ "rewards/margins": 3.129413366317749,
1733
+ "rewards/rejected": -3.2091684341430664,
1734
+ "step": 115
1735
+ },
1736
+ {
1737
+ "epoch": 2.9367088607594938,
1738
+ "grad_norm": 21.46961371171142,
1739
+ "learning_rate": 9.412754953531663e-08,
1740
+ "logits/chosen": 1.7869651317596436,
1741
+ "logits/rejected": 1.881705641746521,
1742
+ "logps/chosen": -24.043615341186523,
1743
+ "logps/rejected": -38.01569366455078,
1744
+ "loss": 0.1852,
1745
+ "rewards/accuracies": 0.9375,
1746
+ "rewards/chosen": 0.10924415290355682,
1747
+ "rewards/margins": 2.661862373352051,
1748
+ "rewards/rejected": -2.5526182651519775,
1749
+ "step": 116
1750
+ },
1751
+ {
1752
+ "epoch": 2.962025316455696,
1753
+ "grad_norm": 19.929759669575105,
1754
+ "learning_rate": 8.978109650374396e-08,
1755
+ "logits/chosen": 1.7048578262329102,
1756
+ "logits/rejected": 1.9161901473999023,
1757
+ "logps/chosen": -26.87795639038086,
1758
+ "logps/rejected": -42.6606330871582,
1759
+ "loss": 0.1589,
1760
+ "rewards/accuracies": 1.0,
1761
+ "rewards/chosen": 0.13115011155605316,
1762
+ "rewards/margins": 3.121182441711426,
1763
+ "rewards/rejected": -2.990032196044922,
1764
+ "step": 117
1765
+ },
1766
+ {
1767
+ "epoch": 2.9873417721518987,
1768
+ "grad_norm": 25.118856703851232,
1769
+ "learning_rate": 8.551531851507185e-08,
1770
+ "logits/chosen": 2.3607137203216553,
1771
+ "logits/rejected": 2.5181257724761963,
1772
+ "logps/chosen": -30.176773071289062,
1773
+ "logps/rejected": -44.625003814697266,
1774
+ "loss": 0.224,
1775
+ "rewards/accuracies": 0.875,
1776
+ "rewards/chosen": -0.30477192997932434,
1777
+ "rewards/margins": 2.4585647583007812,
1778
+ "rewards/rejected": -2.763336658477783,
1779
+ "step": 118
1780
+ },
1781
+ {
1782
+ "epoch": 3.0126582278481013,
1783
+ "grad_norm": 24.215057475486347,
1784
+ "learning_rate": 8.133236351698142e-08,
1785
+ "logits/chosen": 1.6108707189559937,
1786
+ "logits/rejected": 1.7527028322219849,
1787
+ "logps/chosen": -18.345888137817383,
1788
+ "logps/rejected": -42.66941833496094,
1789
+ "loss": 0.2033,
1790
+ "rewards/accuracies": 1.0,
1791
+ "rewards/chosen": 0.02492370456457138,
1792
+ "rewards/margins": 3.3315927982330322,
1793
+ "rewards/rejected": -3.306669235229492,
1794
+ "step": 119
1795
+ },
1796
+ {
1797
+ "epoch": 3.037974683544304,
1798
+ "grad_norm": 18.940438234622814,
1799
+ "learning_rate": 7.723433775328384e-08,
1800
+ "logits/chosen": 2.0556139945983887,
1801
+ "logits/rejected": 2.3014981746673584,
1802
+ "logps/chosen": -21.665407180786133,
1803
+ "logps/rejected": -41.777896881103516,
1804
+ "loss": 0.1836,
1805
+ "rewards/accuracies": 1.0,
1806
+ "rewards/chosen": 0.17341355979442596,
1807
+ "rewards/margins": 2.674577236175537,
1808
+ "rewards/rejected": -2.5011634826660156,
1809
+ "step": 120
1810
+ },
1811
+ {
1812
+ "epoch": 3.0632911392405062,
1813
+ "grad_norm": 18.297883681094888,
1814
+ "learning_rate": 7.322330470336313e-08,
1815
+ "logits/chosen": 2.301410675048828,
1816
+ "logits/rejected": 2.2454984188079834,
1817
+ "logps/chosen": -23.815502166748047,
1818
+ "logps/rejected": -41.840091705322266,
1819
+ "loss": 0.1505,
1820
+ "rewards/accuracies": 1.0,
1821
+ "rewards/chosen": 0.02547144889831543,
1822
+ "rewards/margins": 3.206540584564209,
1823
+ "rewards/rejected": -3.1810688972473145,
1824
+ "step": 121
1825
+ },
1826
+ {
1827
+ "epoch": 3.088607594936709,
1828
+ "grad_norm": 18.507140827349733,
1829
+ "learning_rate": 6.930128404315214e-08,
1830
+ "logits/chosen": 2.0005404949188232,
1831
+ "logits/rejected": 2.2419371604919434,
1832
+ "logps/chosen": -23.189014434814453,
1833
+ "logps/rejected": -44.778133392333984,
1834
+ "loss": 0.1697,
1835
+ "rewards/accuracies": 1.0,
1836
+ "rewards/chosen": -0.21837717294692993,
1837
+ "rewards/margins": 2.497182846069336,
1838
+ "rewards/rejected": -2.715559959411621,
1839
+ "step": 122
1840
+ },
1841
+ {
1842
+ "epoch": 3.1139240506329116,
1843
+ "grad_norm": 20.458085524449526,
1844
+ "learning_rate": 6.547025062816486e-08,
1845
+ "logits/chosen": 2.5110156536102295,
1846
+ "logits/rejected": 2.835902452468872,
1847
+ "logps/chosen": -21.34715461730957,
1848
+ "logps/rejected": -42.06149673461914,
1849
+ "loss": 0.1769,
1850
+ "rewards/accuracies": 1.0,
1851
+ "rewards/chosen": 0.2778359651565552,
1852
+ "rewards/margins": 3.1256825923919678,
1853
+ "rewards/rejected": -2.847846508026123,
1854
+ "step": 123
1855
+ },
1856
+ {
1857
+ "epoch": 3.1392405063291138,
1858
+ "grad_norm": 19.657986671859284,
1859
+ "learning_rate": 6.173213349909728e-08,
1860
+ "logits/chosen": 2.4499258995056152,
1861
+ "logits/rejected": 2.4804461002349854,
1862
+ "logps/chosen": -24.912944793701172,
1863
+ "logps/rejected": -38.67371368408203,
1864
+ "loss": 0.172,
1865
+ "rewards/accuracies": 0.9375,
1866
+ "rewards/chosen": -0.06854649633169174,
1867
+ "rewards/margins": 2.4865663051605225,
1868
+ "rewards/rejected": -2.5551130771636963,
1869
+ "step": 124
1870
+ },
1871
+ {
1872
+ "epoch": 3.1645569620253164,
1873
+ "grad_norm": 18.87674548749799,
1874
+ "learning_rate": 5.808881491049722e-08,
1875
+ "logits/chosen": 1.3738226890563965,
1876
+ "logits/rejected": 1.6148171424865723,
1877
+ "logps/chosen": -25.101566314697266,
1878
+ "logps/rejected": -40.51878356933594,
1879
+ "loss": 0.1786,
1880
+ "rewards/accuracies": 1.0,
1881
+ "rewards/chosen": -0.33920496702194214,
1882
+ "rewards/margins": 2.1792898178100586,
1883
+ "rewards/rejected": -2.5184948444366455,
1884
+ "step": 125
1885
+ },
1886
+ {
1887
+ "epoch": 3.189873417721519,
1888
+ "grad_norm": 20.113483471649342,
1889
+ "learning_rate": 5.454212938299255e-08,
1890
+ "logits/chosen": 2.0692553520202637,
1891
+ "logits/rejected": 2.1005959510803223,
1892
+ "logps/chosen": -26.763628005981445,
1893
+ "logps/rejected": -37.69628143310547,
1894
+ "loss": 0.1624,
1895
+ "rewards/accuracies": 1.0,
1896
+ "rewards/chosen": -0.04393099620938301,
1897
+ "rewards/margins": 2.375842809677124,
1898
+ "rewards/rejected": -2.419773817062378,
1899
+ "step": 126
1900
+ },
1901
+ {
1902
+ "epoch": 3.2151898734177213,
1903
+ "grad_norm": 19.33193488722329,
1904
+ "learning_rate": 5.109386277955477e-08,
1905
+ "logits/chosen": 1.9354777336120605,
1906
+ "logits/rejected": 2.3222734928131104,
1907
+ "logps/chosen": -22.444690704345703,
1908
+ "logps/rejected": -42.732322692871094,
1909
+ "loss": 0.1615,
1910
+ "rewards/accuracies": 1.0,
1911
+ "rewards/chosen": 0.1692490577697754,
1912
+ "rewards/margins": 2.950117349624634,
1913
+ "rewards/rejected": -2.7808682918548584,
1914
+ "step": 127
1915
+ },
1916
+ {
1917
+ "epoch": 3.240506329113924,
1918
+ "grad_norm": 17.657011863002797,
1919
+ "learning_rate": 4.774575140626316e-08,
1920
+ "logits/chosen": 1.7825671434402466,
1921
+ "logits/rejected": 2.0206806659698486,
1922
+ "logps/chosen": -26.314062118530273,
1923
+ "logps/rejected": -44.959896087646484,
1924
+ "loss": 0.1292,
1925
+ "rewards/accuracies": 1.0,
1926
+ "rewards/chosen": 0.18020519614219666,
1927
+ "rewards/margins": 3.4240498542785645,
1928
+ "rewards/rejected": -3.243844747543335,
1929
+ "step": 128
1930
+ },
1931
+ {
1932
+ "epoch": 3.2658227848101267,
1933
+ "grad_norm": 17.36688426753045,
1934
+ "learning_rate": 4.449948113802254e-08,
1935
+ "logits/chosen": 1.963316559791565,
1936
+ "logits/rejected": 2.073986530303955,
1937
+ "logps/chosen": -29.986591339111328,
1938
+ "logps/rejected": -46.73766326904297,
1939
+ "loss": 0.1476,
1940
+ "rewards/accuracies": 1.0,
1941
+ "rewards/chosen": -0.1175435334444046,
1942
+ "rewards/margins": 3.3601157665252686,
1943
+ "rewards/rejected": -3.477659225463867,
1944
+ "step": 129
1945
+ },
1946
+ {
1947
+ "epoch": 3.291139240506329,
1948
+ "grad_norm": 17.66376104354918,
1949
+ "learning_rate": 4.1356686569674335e-08,
1950
+ "logits/chosen": 1.4700018167495728,
1951
+ "logits/rejected": 1.7809115648269653,
1952
+ "logps/chosen": -27.53030014038086,
1953
+ "logps/rejected": -43.54850387573242,
1954
+ "loss": 0.1642,
1955
+ "rewards/accuracies": 1.0,
1956
+ "rewards/chosen": 0.1137082502245903,
1957
+ "rewards/margins": 2.9248647689819336,
1958
+ "rewards/rejected": -2.811156749725342,
1959
+ "step": 130
1960
+ },
1961
+ {
1962
+ "epoch": 3.3164556962025316,
1963
+ "grad_norm": 19.797305830292228,
1964
+ "learning_rate": 3.831895019292897e-08,
1965
+ "logits/chosen": 1.8914071321487427,
1966
+ "logits/rejected": 1.8628309965133667,
1967
+ "logps/chosen": -25.410337448120117,
1968
+ "logps/rejected": -42.06156921386719,
1969
+ "loss": 0.1525,
1970
+ "rewards/accuracies": 1.0,
1971
+ "rewards/chosen": -0.044954825192689896,
1972
+ "rewards/margins": 2.8561882972717285,
1973
+ "rewards/rejected": -2.9011433124542236,
1974
+ "step": 131
1975
+ },
1976
+ {
1977
+ "epoch": 3.3417721518987342,
1978
+ "grad_norm": 18.851413557062596,
1979
+ "learning_rate": 3.538780159953347e-08,
1980
+ "logits/chosen": 2.0251123905181885,
1981
+ "logits/rejected": 2.210965156555176,
1982
+ "logps/chosen": -20.657623291015625,
1983
+ "logps/rejected": -45.97637939453125,
1984
+ "loss": 0.1488,
1985
+ "rewards/accuracies": 1.0,
1986
+ "rewards/chosen": 0.21129505336284637,
1987
+ "rewards/margins": 3.6020712852478027,
1988
+ "rewards/rejected": -3.3907763957977295,
1989
+ "step": 132
1990
+ },
1991
+ {
1992
+ "epoch": 3.367088607594937,
1993
+ "grad_norm": 18.383406639630245,
1994
+ "learning_rate": 3.256471671107616e-08,
1995
+ "logits/chosen": 1.5468469858169556,
1996
+ "logits/rejected": 1.8942471742630005,
1997
+ "logps/chosen": -34.759246826171875,
1998
+ "logps/rejected": -54.94669723510742,
1999
+ "loss": 0.161,
2000
+ "rewards/accuracies": 1.0,
2001
+ "rewards/chosen": 0.08364612609148026,
2002
+ "rewards/margins": 3.432290554046631,
2003
+ "rewards/rejected": -3.348644495010376,
2004
+ "step": 133
2005
+ },
2006
+ {
2007
+ "epoch": 3.392405063291139,
2008
+ "grad_norm": 18.442661691122066,
2009
+ "learning_rate": 2.98511170358155e-08,
2010
+ "logits/chosen": 1.9410444498062134,
2011
+ "logits/rejected": 1.9713021516799927,
2012
+ "logps/chosen": -33.468505859375,
2013
+ "logps/rejected": -40.28277587890625,
2014
+ "loss": 0.1517,
2015
+ "rewards/accuracies": 1.0,
2016
+ "rewards/chosen": -0.1791667342185974,
2017
+ "rewards/margins": 2.8069636821746826,
2018
+ "rewards/rejected": -2.986130475997925,
2019
+ "step": 134
2020
+ },
2021
+ {
2022
+ "epoch": 3.4177215189873418,
2023
+ "grad_norm": 19.123016466489233,
2024
+ "learning_rate": 2.724836895290805e-08,
2025
+ "logits/chosen": 2.172494649887085,
2026
+ "logits/rejected": 2.3771188259124756,
2027
+ "logps/chosen": -18.799209594726562,
2028
+ "logps/rejected": -27.414329528808594,
2029
+ "loss": 0.1574,
2030
+ "rewards/accuracies": 0.9375,
2031
+ "rewards/chosen": 0.05184905230998993,
2032
+ "rewards/margins": 1.7008121013641357,
2033
+ "rewards/rejected": -1.648963212966919,
2034
+ "step": 135
2035
+ },
2036
+ {
2037
+ "epoch": 3.4430379746835444,
2038
+ "grad_norm": 19.389448097902036,
2039
+ "learning_rate": 2.475778302439524e-08,
2040
+ "logits/chosen": 1.5435059070587158,
2041
+ "logits/rejected": 2.0342299938201904,
2042
+ "logps/chosen": -23.09898567199707,
2043
+ "logps/rejected": -41.35944366455078,
2044
+ "loss": 0.1673,
2045
+ "rewards/accuracies": 1.0,
2046
+ "rewards/chosen": -0.01025979220867157,
2047
+ "rewards/margins": 3.0842089653015137,
2048
+ "rewards/rejected": -3.094468832015991,
2049
+ "step": 136
2050
+ },
2051
+ {
2052
+ "epoch": 3.4683544303797467,
2053
+ "grad_norm": 18.168150870329598,
2054
+ "learning_rate": 2.2380613335296033e-08,
2055
+ "logits/chosen": 1.5661985874176025,
2056
+ "logits/rejected": 1.6321345567703247,
2057
+ "logps/chosen": -21.557811737060547,
2058
+ "logps/rejected": -37.444087982177734,
2059
+ "loss": 0.1457,
2060
+ "rewards/accuracies": 1.0,
2061
+ "rewards/chosen": 0.32843151688575745,
2062
+ "rewards/margins": 3.217196464538574,
2063
+ "rewards/rejected": -2.888765335083008,
2064
+ "step": 137
2065
+ },
2066
+ {
2067
+ "epoch": 3.4936708860759493,
2068
+ "grad_norm": 18.10048373008008,
2069
+ "learning_rate": 2.0118056862137354e-08,
2070
+ "logits/chosen": 2.2138497829437256,
2071
+ "logits/rejected": 2.3827362060546875,
2072
+ "logps/chosen": -25.050493240356445,
2073
+ "logps/rejected": -40.5790901184082,
2074
+ "loss": 0.1574,
2075
+ "rewards/accuracies": 1.0,
2076
+ "rewards/chosen": -0.05445308983325958,
2077
+ "rewards/margins": 2.6866414546966553,
2078
+ "rewards/rejected": -2.7410943508148193,
2079
+ "step": 138
2080
+ },
2081
+ {
2082
+ "epoch": 3.518987341772152,
2083
+ "grad_norm": 17.844608513376855,
2084
+ "learning_rate": 1.797125287024029e-08,
2085
+ "logits/chosen": 2.252528667449951,
2086
+ "logits/rejected": 2.517775297164917,
2087
+ "logps/chosen": -25.718944549560547,
2088
+ "logps/rejected": -40.703636169433594,
2089
+ "loss": 0.1633,
2090
+ "rewards/accuracies": 0.9375,
2091
+ "rewards/chosen": -0.23676270246505737,
2092
+ "rewards/margins": 2.2277328968048096,
2093
+ "rewards/rejected": -2.4644956588745117,
2094
+ "step": 139
2095
+ },
2096
+ {
2097
+ "epoch": 3.5443037974683547,
2098
+ "grad_norm": 19.82721673855706,
2099
+ "learning_rate": 1.5941282340065697e-08,
2100
+ "logits/chosen": 2.1378254890441895,
2101
+ "logits/rejected": 2.3837673664093018,
2102
+ "logps/chosen": -21.759735107421875,
2103
+ "logps/rejected": -35.555789947509766,
2104
+ "loss": 0.1725,
2105
+ "rewards/accuracies": 1.0,
2106
+ "rewards/chosen": 0.32154813408851624,
2107
+ "rewards/margins": 2.5137619972229004,
2108
+ "rewards/rejected": -2.192214012145996,
2109
+ "step": 140
2110
+ },
2111
+ {
2112
+ "epoch": 3.569620253164557,
2113
+ "grad_norm": 19.687984330411137,
2114
+ "learning_rate": 1.4029167422908105e-08,
2115
+ "logits/chosen": 1.8691318035125732,
2116
+ "logits/rejected": 1.951183557510376,
2117
+ "logps/chosen": -22.21922492980957,
2118
+ "logps/rejected": -38.082176208496094,
2119
+ "loss": 0.1587,
2120
+ "rewards/accuracies": 1.0,
2121
+ "rewards/chosen": 0.12926092743873596,
2122
+ "rewards/margins": 3.5831055641174316,
2123
+ "rewards/rejected": -3.4538450241088867,
2124
+ "step": 141
2125
+ },
2126
+ {
2127
+ "epoch": 3.5949367088607596,
2128
+ "grad_norm": 19.210992875859024,
2129
+ "learning_rate": 1.2235870926211616e-08,
2130
+ "logits/chosen": 1.878167748451233,
2131
+ "logits/rejected": 2.1185381412506104,
2132
+ "logps/chosen": -25.917736053466797,
2133
+ "logps/rejected": -45.47472381591797,
2134
+ "loss": 0.1359,
2135
+ "rewards/accuracies": 1.0,
2136
+ "rewards/chosen": 0.0160946324467659,
2137
+ "rewards/margins": 3.645671844482422,
2138
+ "rewards/rejected": -3.629577398300171,
2139
+ "step": 142
2140
+ },
2141
+ {
2142
+ "epoch": 3.620253164556962,
2143
+ "grad_norm": 20.116883252259765,
2144
+ "learning_rate": 1.0562295828767387e-08,
2145
+ "logits/chosen": 1.2207794189453125,
2146
+ "logits/rejected": 1.1344801187515259,
2147
+ "logps/chosen": -23.36668586730957,
2148
+ "logps/rejected": -31.341829299926758,
2149
+ "loss": 0.1841,
2150
+ "rewards/accuracies": 0.9375,
2151
+ "rewards/chosen": -0.16667915880680084,
2152
+ "rewards/margins": 1.5721771717071533,
2153
+ "rewards/rejected": -1.7388561964035034,
2154
+ "step": 143
2155
+ },
2156
+ {
2157
+ "epoch": 3.6455696202531644,
2158
+ "grad_norm": 19.720241233310634,
2159
+ "learning_rate": 9.009284826036689e-09,
2160
+ "logits/chosen": 2.4453952312469482,
2161
+ "logits/rejected": 2.662018299102783,
2162
+ "logps/chosen": -22.387048721313477,
2163
+ "logps/rejected": -36.92119216918945,
2164
+ "loss": 0.1617,
2165
+ "rewards/accuracies": 1.0,
2166
+ "rewards/chosen": 0.031851306557655334,
2167
+ "rewards/margins": 2.5965640544891357,
2168
+ "rewards/rejected": -2.5647125244140625,
2169
+ "step": 144
2170
+ },
2171
+ {
2172
+ "epoch": 3.670886075949367,
2173
+ "grad_norm": 17.86661138635714,
2174
+ "learning_rate": 7.577619905828281e-09,
2175
+ "logits/chosen": 2.745617628097534,
2176
+ "logits/rejected": 2.844001531600952,
2177
+ "logps/chosen": -26.597410202026367,
2178
+ "logps/rejected": -36.82072448730469,
2179
+ "loss": 0.142,
2180
+ "rewards/accuracies": 1.0,
2181
+ "rewards/chosen": -0.17600667476654053,
2182
+ "rewards/margins": 2.553907632827759,
2183
+ "rewards/rejected": -2.7299141883850098,
2184
+ "step": 145
2185
+ },
2186
+ {
2187
+ "epoch": 3.6962025316455698,
2188
+ "grad_norm": 16.0477696571449,
2189
+ "learning_rate": 6.268021954544095e-09,
2190
+ "logits/chosen": 2.1746973991394043,
2191
+ "logits/rejected": 2.2613565921783447,
2192
+ "logps/chosen": -32.3970947265625,
2193
+ "logps/rejected": -40.7056884765625,
2194
+ "loss": 0.1372,
2195
+ "rewards/accuracies": 1.0,
2196
+ "rewards/chosen": -0.09012872725725174,
2197
+ "rewards/margins": 4.017242431640625,
2198
+ "rewards/rejected": -4.1073713302612305,
2199
+ "step": 146
2200
+ },
2201
+ {
2202
+ "epoch": 3.721518987341772,
2203
+ "grad_norm": 18.112204362761403,
2204
+ "learning_rate": 5.08115039419113e-09,
2205
+ "logits/chosen": 2.2588939666748047,
2206
+ "logits/rejected": 2.217837333679199,
2207
+ "logps/chosen": -23.21999740600586,
2208
+ "logps/rejected": -38.22819519042969,
2209
+ "loss": 0.1574,
2210
+ "rewards/accuracies": 1.0,
2211
+ "rewards/chosen": -0.07647465169429779,
2212
+ "rewards/margins": 3.384047031402588,
2213
+ "rewards/rejected": -3.460521697998047,
2214
+ "step": 147
2215
+ },
2216
+ {
2217
+ "epoch": 3.7468354430379747,
2218
+ "grad_norm": 19.146595468250382,
2219
+ "learning_rate": 4.0176028503425826e-09,
2220
+ "logits/chosen": 2.2898073196411133,
2221
+ "logits/rejected": 2.612645149230957,
2222
+ "logps/chosen": -21.40774917602539,
2223
+ "logps/rejected": -39.644142150878906,
2224
+ "loss": 0.1575,
2225
+ "rewards/accuracies": 1.0,
2226
+ "rewards/chosen": 0.14267057180404663,
2227
+ "rewards/margins": 2.3787121772766113,
2228
+ "rewards/rejected": -2.23604154586792,
2229
+ "step": 148
2230
+ },
2231
+ {
2232
+ "epoch": 3.7721518987341773,
2233
+ "grad_norm": 16.066181981705437,
2234
+ "learning_rate": 3.077914851215585e-09,
2235
+ "logits/chosen": 2.0213255882263184,
2236
+ "logits/rejected": 2.22232723236084,
2237
+ "logps/chosen": -20.719486236572266,
2238
+ "logps/rejected": -33.82309341430664,
2239
+ "loss": 0.1448,
2240
+ "rewards/accuracies": 1.0,
2241
+ "rewards/chosen": -0.04364713281393051,
2242
+ "rewards/margins": 2.438537836074829,
2243
+ "rewards/rejected": -2.482184886932373,
2244
+ "step": 149
2245
+ },
2246
+ {
2247
+ "epoch": 3.7974683544303796,
2248
+ "grad_norm": 20.168906914662397,
2249
+ "learning_rate": 2.2625595580163247e-09,
2250
+ "logits/chosen": 1.8664175271987915,
2251
+ "logits/rejected": 2.135633945465088,
2252
+ "logps/chosen": -24.14510726928711,
2253
+ "logps/rejected": -39.59516906738281,
2254
+ "loss": 0.1844,
2255
+ "rewards/accuracies": 0.875,
2256
+ "rewards/chosen": -0.004626031965017319,
2257
+ "rewards/margins": 2.253840446472168,
2258
+ "rewards/rejected": -2.2584664821624756,
2259
+ "step": 150
2260
+ },
2261
+ {
2262
+ "epoch": 3.8227848101265822,
2263
+ "grad_norm": 21.962898348700495,
2264
+ "learning_rate": 1.5719475266893489e-09,
2265
+ "logits/chosen": 1.7901225090026855,
2266
+ "logits/rejected": 1.9218878746032715,
2267
+ "logps/chosen": -22.082956314086914,
2268
+ "logps/rejected": -43.000667572021484,
2269
+ "loss": 0.192,
2270
+ "rewards/accuracies": 0.9375,
2271
+ "rewards/chosen": 0.07910831272602081,
2272
+ "rewards/margins": 2.6802215576171875,
2273
+ "rewards/rejected": -2.6011130809783936,
2274
+ "step": 151
2275
+ },
2276
+ {
2277
+ "epoch": 3.848101265822785,
2278
+ "grad_norm": 16.511033653339577,
2279
+ "learning_rate": 1.0064265011902328e-09,
2280
+ "logits/chosen": 2.4072799682617188,
2281
+ "logits/rejected": 2.4377200603485107,
2282
+ "logps/chosen": -25.803665161132812,
2283
+ "logps/rejected": -40.92607879638672,
2284
+ "loss": 0.1549,
2285
+ "rewards/accuracies": 1.0,
2286
+ "rewards/chosen": 0.035745151340961456,
2287
+ "rewards/margins": 3.1956663131713867,
2288
+ "rewards/rejected": -3.159921169281006,
2289
+ "step": 152
2290
+ },
2291
+ {
2292
+ "epoch": 3.8734177215189876,
2293
+ "grad_norm": 18.636452488465395,
2294
+ "learning_rate": 5.662812383859794e-10,
2295
+ "logits/chosen": 2.361091136932373,
2296
+ "logits/rejected": 2.3107473850250244,
2297
+ "logps/chosen": -27.91613006591797,
2298
+ "logps/rejected": -39.84597396850586,
2299
+ "loss": 0.1544,
2300
+ "rewards/accuracies": 1.0,
2301
+ "rewards/chosen": -0.1522400677204132,
2302
+ "rewards/margins": 3.036912441253662,
2303
+ "rewards/rejected": -3.189152240753174,
2304
+ "step": 153
2305
+ },
2306
+ {
2307
+ "epoch": 3.8987341772151898,
2308
+ "grad_norm": 17.763028400462222,
2309
+ "learning_rate": 2.5173336467135263e-10,
2310
+ "logits/chosen": 2.0824742317199707,
2311
+ "logits/rejected": 2.3250155448913574,
2312
+ "logps/chosen": -23.416114807128906,
2313
+ "logps/rejected": -38.985511779785156,
2314
+ "loss": 0.1542,
2315
+ "rewards/accuracies": 0.9375,
2316
+ "rewards/chosen": -0.06766586005687714,
2317
+ "rewards/margins": 2.1996846199035645,
2318
+ "rewards/rejected": -2.267350673675537,
2319
+ "step": 154
2320
+ },
2321
+ {
2322
+ "epoch": 3.9240506329113924,
2323
+ "grad_norm": 19.457837238018815,
2324
+ "learning_rate": 6.294126437336733e-11,
2325
+ "logits/chosen": 2.00872540473938,
2326
+ "logits/rejected": 2.4769511222839355,
2327
+ "logps/chosen": -24.827199935913086,
2328
+ "logps/rejected": -44.62668991088867,
2329
+ "loss": 0.1614,
2330
+ "rewards/accuracies": 1.0,
2331
+ "rewards/chosen": -0.19776426255702972,
2332
+ "rewards/margins": 2.720571756362915,
2333
+ "rewards/rejected": -2.9183361530303955,
2334
+ "step": 155
2335
+ },
2336
+ {
2337
+ "epoch": 3.9493670886075947,
2338
+ "grad_norm": 19.565085246341297,
2339
+ "learning_rate": 0.0,
2340
+ "logits/chosen": 1.8402495384216309,
2341
+ "logits/rejected": 2.033839464187622,
2342
+ "logps/chosen": -28.689044952392578,
2343
+ "logps/rejected": -46.74669647216797,
2344
+ "loss": 0.1636,
2345
+ "rewards/accuracies": 1.0,
2346
+ "rewards/chosen": 0.04416623339056969,
2347
+ "rewards/margins": 3.22959041595459,
2348
+ "rewards/rejected": -3.185424327850342,
2349
+ "step": 156
2350
+ },
2351
+ {
2352
+ "epoch": 3.9493670886075947,
2353
+ "step": 156,
2354
+ "total_flos": 0.0,
2355
+ "train_loss": 0.3210896818110576,
2356
+ "train_runtime": 45702.9058,
2357
+ "train_samples_per_second": 0.442,
2358
+ "train_steps_per_second": 0.003
2359
+ }
2360
+ ],
2361
+ "logging_steps": 1,
2362
+ "max_steps": 156,
2363
+ "num_input_tokens_seen": 0,
2364
+ "num_train_epochs": 4,
2365
+ "save_steps": 200,
2366
+ "total_flos": 0.0,
2367
+ "train_batch_size": 2,
2368
+ "trial_name": null,
2369
+ "trial_params": null
2370
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:98c1657be13291df936f81398805d492e04063e837e20c4235adb640b8a307a1
3
+ size 6456
vocab.json ADDED
The diff for this file is too large to render. See raw diff