Model save

README.md

@@ -1,11 +1,9 @@
 ---
-base_model: Qwen/Qwen2.5-Math-7B
-datasets: DigitalLearningGmbH/MATH-lighteval
+base_model: Qwen/Qwen2.5-Math-7B-Instruct
 library_name: transformers
 model_name: Qwen-2.5-7B-Simple-RL
 tags:
 - generated_from_trainer
-- open-r1
 - trl
 - grpo
 licence: license
@@ -13,7 +11,7 @@ licence: license
 
 # Model Card for Qwen-2.5-7B-Simple-RL
 
-This model is a fine-tuned version of [Qwen/Qwen2.5-Math-7B](https://huggingface.co/Qwen/Qwen2.5-Math-7B) on the [DigitalLearningGmbH/MATH-lighteval](https://huggingface.co/datasets/DigitalLearningGmbH/MATH-lighteval) dataset.
+This model is a fine-tuned version of [Qwen/Qwen2.5-Math-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Math-7B-Instruct).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start
@@ -29,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/cyruskwan/open-r1/runs/hjcgoqm1) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/cyruskwan/open-r1/runs/xoeuq0c9) 
 
 
 This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.028255671029910445,
-    "train_runtime": 36935.2831,
+    "train_loss": 0.1694948914914559,
+    "train_runtime": 74169.3594,
     "train_samples": 7500,
-    "train_samples_per_second": 0.203,
-    "train_steps_per_second": 0.001
+    "train_samples_per_second": 0.101,
+    "train_steps_per_second": 0.002
 }

config.json

@@ -4,7 +4,7 @@
   ],
   "attention_dropout": 0.0,
   "bos_token_id": 151643,
-  "eos_token_id": 151643,
+  "eos_token_id": 151645,
   "hidden_act": "silu",
   "hidden_size": 3584,
   "initializer_range": 0.02,
@@ -17,13 +17,12 @@
   "num_key_value_heads": 4,
   "rms_norm_eps": 1e-06,
   "rope_scaling": null,
-  "rope_theta": 10000,
+  "rope_theta": 10000.0,
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.50.0",
-  "use_cache": true,
-  "use_mrope": false,
+  "use_cache": false,
   "use_sliding_window": false,
   "vocab_size": 152064
 }

generation_config.json

@@ -1,6 +1,9 @@
 {
   "bos_token_id": 151643,
-  "eos_token_id": 151643,
-  "max_new_tokens": 2048,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
   "transformers_version": "4.50.0"
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:67a84a79ce3e4b09530b80cacf3bfb34e7e032df6e3b16a781b2bc07188e8b3f
+oid sha256:b8843653ab97f66ef52ec60f955e758344b765b8d832a1d64ad1b59643e341fd
 size 4877660776

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6a159a8a5dcc8f9d3ecaa17119169e5886eff14df9e5ce9b0f19c653749082af
+oid sha256:479a1b36a162285708400e0fbebe6bd4a7d674cb6002d7442118bed9a2df00f4
 size 4932751008

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e53d996d4793e766dfbe9ffee667e58a1546bb7a52974160ff3f9720369ebfb3
+oid sha256:21ac08b7265fe178a5909fbd0eac4ae746c85c17552d2b43018834dbe519374f
 size 4330865200

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1c2667bb1def9db62c2af9c1447ef43fc4cafcaf67b8c9052c0ed695781a75dd
+oid sha256:de6f7ae8b22a48abeb54e5972dc2e6f5fd30dabec878c16d8a23bd5b13eda572
 size 1089994880

special_tokens_map.json

@@ -15,7 +15,7 @@
     "<|video_pad|>"
   ],
   "eos_token": {
-    "content": "<|endoftext|>",
+    "content": "<|im_end|>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,

tokenizer_config.json

@@ -195,9 +195,9 @@
     "<|video_pad|>"
   ],
   "bos_token": null,
-  "chat_template": "{{ bos_token }}{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful AI Assistant that provides well-reasoned and detailed responses. You first think about the reasoning process as an internal monologue and then provide the user with the answer. Respond in the following format: <think>\n...\n</think>\n<answer>\n...\n</answer><|im_end|>\n<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n' }}{% elif message['role'] == 'system' %}{{ '<|im_start|>system\n' + message['content'] + '<|im_end|>\n' }}{% elif message['role'] == 'user' %}{{ '<|im_start|>user\n' + message['content'] + '<|im_end|>' + '\n' }}{% elif message['role'] == 'assistant' %}{% if not loop.last %}{{ '<|im_start|>assistant\n' + message['content'] + '<|im_end|>' + '\n' + eos_token + '\n' }}{% else %}{{ '<|im_start|>assistant\n'  + message['content'] + '<|im_end|>' + '\n' + eos_token }}{% endif %}{% endif %}{% if loop.last and add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}{% endfor %}",
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'Please reason step by step, and put your final answer within \\\\boxed{}.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
   "clean_up_tokenization_spaces": false,
-  "eos_token": "<|endoftext|>",
+  "eos_token": "<|im_end|>",
   "errors": "replace",
   "extra_special_tokens": {},
   "model_max_length": 131072,

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.028255671029910445,
-    "train_runtime": 36935.2831,
+    "train_loss": 0.1694948914914559,
+    "train_runtime": 74169.3594,
     "train_samples": 7500,
-    "train_samples_per_second": 0.203,
-    "train_steps_per_second": 0.001
+    "train_samples_per_second": 0.101,
+    "train_steps_per_second": 0.002
 }

trainer_state.json

@@ -2,258 +2,918 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9898666666666667,
+  "epoch": 0.9984,
   "eval_steps": 500,
-  "global_step": 29,
+  "global_step": 117,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "clip_ratio": 0.0,
-      "completion_length": 609.2392578125,
-      "epoch": 0.034133333333333335,
-      "grad_norm": 0.23621266719831985,
+      "completion_length": 635.126953125,
+      "epoch": 0.008533333333333334,
+      "grad_norm": 0.6395311468390221,
       "kl": 0.0,
-      "learning_rate": 1e-06,
-      "loss": -0.0026,
-      "num_tokens": 785357.0,
-      "reward": 0.646484375,
-      "reward_std": 0.2832682034932077,
-      "rewards/accuracy_reward": 0.646484375,
+      "learning_rate": 2.5e-07,
+      "loss": 0.0603,
+      "num_tokens": 406601.0,
+      "reward": 0.6796875,
+      "reward_std": 0.17794983368366957,
+      "rewards/accuracy_reward": 0.6796875,
       "rewards/format_reward": 0.0,
       "step": 1
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 623.9384765625,
-      "epoch": 0.06826666666666667,
-      "grad_norm": 0.29521809256012294,
+      "completion_length": 568.509765625,
+      "epoch": 0.017066666666666667,
+      "grad_norm": 0.3978739586526697,
       "kl": 0.0,
-      "learning_rate": 2e-06,
-      "loss": 0.0061,
-      "num_tokens": 1593058.0,
-      "reward": 0.6142578125,
-      "reward_std": 0.295451024081558,
-      "rewards/accuracy_reward": 0.6142578125,
+      "learning_rate": 5e-07,
+      "loss": 0.0391,
+      "num_tokens": 776134.0,
+      "reward": 0.73046875,
+      "reward_std": 0.17123176017776132,
+      "rewards/accuracy_reward": 0.73046875,
       "rewards/format_reward": 0.0,
       "step": 2
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 598.8173828125,
-      "epoch": 0.13653333333333334,
-      "grad_norm": 0.42773393862897524,
-      "kl": 0.0004331320524215698,
-      "learning_rate": 2.989063311147081e-06,
-      "loss": 0.0084,
-      "num_tokens": 3136920.0,
-      "reward": 0.630859375,
-      "reward_std": 0.29506791406311095,
-      "rewards/accuracy_reward": 0.630859375,
+      "completion_length": 578.5009765625,
+      "epoch": 0.034133333333333335,
+      "grad_norm": 0.2928130055744145,
+      "kl": 1.120672095566988e-05,
+      "learning_rate": 1e-06,
+      "loss": 0.0263,
+      "num_tokens": 1531639.0,
+      "reward": 0.68359375,
+      "reward_std": 0.13874951610341668,
+      "rewards/accuracy_reward": 0.68359375,
       "rewards/format_reward": 0.0,
       "step": 4
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 602.5595703125,
-      "epoch": 0.2048,
-      "grad_norm": 0.17771809594185137,
-      "kl": 0.001168280839920044,
-      "learning_rate": 2.9025243640281224e-06,
-      "loss": 0.0428,
-      "num_tokens": 4677534.0,
-      "reward": 0.73291015625,
-      "reward_std": 0.23434827150776982,
-      "rewards/accuracy_reward": 0.73291015625,
+      "completion_length": 615.7861328125,
+      "epoch": 0.0512,
+      "grad_norm": 0.38001767737083486,
+      "kl": 7.296388503164053e-06,
+      "learning_rate": 1.5e-06,
+      "loss": 0.0361,
+      "num_tokens": 2334764.0,
+      "reward": 0.642578125,
+      "reward_std": 0.22008429956622422,
+      "rewards/accuracy_reward": 0.642578125,
       "rewards/format_reward": 0.0,
       "step": 6
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 602.4541015625,
-      "epoch": 0.2730666666666667,
-      "grad_norm": 0.11255873766865199,
-      "kl": 0.0025081783533096313,
-      "learning_rate": 2.7344757988404844e-06,
-      "loss": 0.0544,
-      "num_tokens": 6229636.0,
-      "reward": 0.73974609375,
-      "reward_std": 0.21338723483495414,
-      "rewards/accuracy_reward": 0.73974609375,
+      "completion_length": 606.6162109375,
+      "epoch": 0.06826666666666667,
+      "grad_norm": 0.36726993629552374,
+      "kl": 1.181039260700345e-05,
+      "learning_rate": 2e-06,
+      "loss": 0.0446,
+      "num_tokens": 3120955.0,
+      "reward": 0.67578125,
+      "reward_std": 0.18526801373809576,
+      "rewards/accuracy_reward": 0.67578125,
       "rewards/format_reward": 0.0,
       "step": 8
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 604.98681640625,
-      "epoch": 0.3413333333333333,
-      "grad_norm": 0.06599060323311819,
-      "kl": 0.002334892749786377,
-      "learning_rate": 2.4946839873611927e-06,
-      "loss": 0.0419,
-      "num_tokens": 7784865.0,
-      "reward": 0.75830078125,
-      "reward_std": 0.17949885036796331,
-      "rewards/accuracy_reward": 0.75830078125,
+      "completion_length": 587.7744140625,
+      "epoch": 0.08533333333333333,
+      "grad_norm": 0.5438420318338805,
+      "kl": 0.00024004688020795584,
+      "learning_rate": 2.5e-06,
+      "loss": 0.0401,
+      "num_tokens": 3883788.0,
+      "reward": 0.7041015625,
+      "reward_std": 0.15495382226072252,
+      "rewards/accuracy_reward": 0.7041015625,
       "rewards/format_reward": 0.0,
       "step": 10
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 609.39794921875,
-      "epoch": 0.4096,
-      "grad_norm": 0.06593463513171627,
-      "kl": 0.0026736706495285034,
-      "learning_rate": 2.1970847580656528e-06,
-      "loss": 0.0326,
-      "num_tokens": 9351732.0,
-      "reward": 0.7705078125,
-      "reward_std": 0.16446730284951627,
-      "rewards/accuracy_reward": 0.7705078125,
+      "completion_length": 567.94140625,
+      "epoch": 0.1024,
+      "grad_norm": 0.3374078187837519,
+      "kl": 0.0005977721302770078,
+      "learning_rate": 3e-06,
+      "loss": 0.03,
+      "num_tokens": 4627856.0,
+      "reward": 0.7080078125,
+      "reward_std": 0.14151287800632417,
+      "rewards/accuracy_reward": 0.7080078125,
       "rewards/format_reward": 0.0,
       "step": 12
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 598.2216796875,
-      "epoch": 0.47786666666666666,
-      "grad_norm": 0.06574801507589917,
-      "kl": 0.003045812249183655,
-      "learning_rate": 1.8589734964313368e-06,
-      "loss": 0.0238,
-      "num_tokens": 10890530.0,
-      "reward": 0.7509765625,
-      "reward_std": 0.15395715064369142,
-      "rewards/accuracy_reward": 0.7509765625,
+      "completion_length": 586.3916015625,
+      "epoch": 0.11946666666666667,
+      "grad_norm": 0.5380156747856304,
+      "kl": 0.005719492677599192,
+      "learning_rate": 2.9973151946516025e-06,
+      "loss": 0.06,
+      "num_tokens": 5372441.0,
+      "reward": 0.7119140625,
+      "reward_std": 0.18398740980774164,
+      "rewards/accuracy_reward": 0.7119140625,
       "rewards/format_reward": 0.0,
       "step": 14
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 612.27392578125,
-      "epoch": 0.5461333333333334,
-      "grad_norm": 0.07737476341941271,
-      "kl": 0.003218412399291992,
-      "learning_rate": 1.5e-06,
-      "loss": 0.0251,
-      "num_tokens": 12469023.0,
-      "reward": 0.7412109375,
-      "reward_std": 0.14749347674660385,
-      "rewards/accuracy_reward": 0.7412109375,
+      "completion_length": 613.46484375,
+      "epoch": 0.13653333333333334,
+      "grad_norm": 0.7673623447856024,
+      "kl": 0.015465203672647476,
+      "learning_rate": 2.989270389512756e-06,
+      "loss": 0.0697,
+      "num_tokens": 6168029.0,
+      "reward": 0.6435546875,
+      "reward_std": 0.2058832028415054,
+      "rewards/accuracy_reward": 0.6435546875,
       "rewards/format_reward": 0.0,
       "step": 16
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 598.642578125,
-      "epoch": 0.6144,
-      "grad_norm": 0.07278656613203825,
-      "kl": 0.0029258430004119873,
-      "learning_rate": 1.141026503568664e-06,
-      "loss": 0.0356,
-      "num_tokens": 14024751.0,
-      "reward": 0.73974609375,
-      "reward_std": 0.16665246314369142,
-      "rewards/accuracy_reward": 0.73974609375,
+      "completion_length": 566.7099609375,
+      "epoch": 0.1536,
+      "grad_norm": 0.8551080207596556,
+      "kl": 0.02153320051729679,
+      "learning_rate": 2.9758943828979446e-06,
+      "loss": 0.0669,
+      "num_tokens": 6893764.0,
+      "reward": 0.703125,
+      "reward_std": 0.21476810285821557,
+      "rewards/accuracy_reward": 0.703125,
       "rewards/format_reward": 0.0,
       "step": 18
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 592.80224609375,
-      "epoch": 0.6826666666666666,
-      "grad_norm": 0.075280235931183,
-      "kl": 0.0033763498067855835,
-      "learning_rate": 8.029152419343472e-07,
-      "loss": 0.0337,
-      "num_tokens": 15557946.0,
-      "reward": 0.76513671875,
-      "reward_std": 0.16409503924660385,
-      "rewards/accuracy_reward": 0.76513671875,
+      "completion_length": 617.9345703125,
+      "epoch": 0.17066666666666666,
+      "grad_norm": 2.416498946611759,
+      "kl": 0.04414602927863598,
+      "learning_rate": 2.957235057439301e-06,
+      "loss": 0.0841,
+      "num_tokens": 7688649.0,
+      "reward": 0.6337890625,
+      "reward_std": 0.20062413928098977,
+      "rewards/accuracy_reward": 0.6337890625,
       "rewards/format_reward": 0.0,
       "step": 20
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 609.025390625,
-      "epoch": 0.7509333333333333,
-      "grad_norm": 0.05918406556315966,
-      "kl": 0.0036567747592926025,
-      "learning_rate": 5.053160126388076e-07,
-      "loss": 0.0227,
-      "num_tokens": 17129102.0,
-      "reward": 0.7431640625,
-      "reward_std": 0.15508478786796331,
-      "rewards/accuracy_reward": 0.74462890625,
+      "completion_length": 647.6904296875,
+      "epoch": 0.18773333333333334,
+      "grad_norm": 1.3063605057043102,
+      "kl": 0.07265654765069485,
+      "learning_rate": 2.933359208679211e-06,
+      "loss": 0.0894,
+      "num_tokens": 8507764.0,
+      "reward": 0.6083984375,
+      "reward_std": 0.2147587512154132,
+      "rewards/accuracy_reward": 0.6083984375,
       "rewards/format_reward": 0.0,
       "step": 22
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 603.46875,
-      "epoch": 0.8192,
-      "grad_norm": 0.054654929027731385,
-      "kl": 0.0032880455255508423,
-      "learning_rate": 2.6552420115951547e-07,
-      "loss": 0.0254,
-      "num_tokens": 18693822.0,
-      "reward": 0.74755859375,
-      "reward_std": 0.14907433814369142,
-      "rewards/accuracy_reward": 0.74853515625,
+      "completion_length": 577.3544921875,
+      "epoch": 0.2048,
+      "grad_norm": 0.9576537994468717,
+      "kl": 0.0842108279466629,
+      "learning_rate": 2.904352305959606e-06,
+      "loss": 0.1019,
+      "num_tokens": 9248695.0,
+      "reward": 0.7001953125,
+      "reward_std": 0.2201498057693243,
+      "rewards/accuracy_reward": 0.7001953125,
       "rewards/format_reward": 0.0,
       "step": 24
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 605.34716796875,
-      "epoch": 0.8874666666666666,
-      "grad_norm": 0.1188197019666417,
-      "kl": 0.0029678642749786377,
-      "learning_rate": 9.747563597187792e-08,
-      "loss": 0.0229,
-      "num_tokens": 20251341.0,
-      "reward": 0.76611328125,
-      "reward_std": 0.14734240202233195,
-      "rewards/accuracy_reward": 0.76611328125,
+      "completion_length": 617.2255859375,
+      "epoch": 0.22186666666666666,
+      "grad_norm": 1.0236250726018012,
+      "kl": 0.1061122715473175,
+      "learning_rate": 2.8703181864639013e-06,
+      "loss": 0.1201,
+      "num_tokens": 10033030.0,
+      "reward": 0.6640625,
+      "reward_std": 0.25549834687262774,
+      "rewards/accuracy_reward": 0.6640625,
       "rewards/format_reward": 0.0,
       "step": 26
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 594.63916015625,
-      "epoch": 0.9557333333333333,
-      "grad_norm": 0.06119972355000096,
-      "kl": 0.0031557083129882812,
-      "learning_rate": 1.093668885291904e-08,
-      "loss": 0.0283,
-      "num_tokens": 21794738.0,
-      "reward": 0.7734375,
-      "reward_std": 0.15282951341941953,
-      "rewards/accuracy_reward": 0.7734375,
+      "completion_length": 624.8525390625,
+      "epoch": 0.23893333333333333,
+      "grad_norm": 0.7738985933722767,
+      "kl": 0.1274673491716385,
+      "learning_rate": 2.8313786835068315e-06,
+      "loss": 0.1069,
+      "num_tokens": 10825831.0,
+      "reward": 0.6279296875,
+      "reward_std": 0.22281860560178757,
+      "rewards/accuracy_reward": 0.6279296875,
       "rewards/format_reward": 0.0,
       "step": 28
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 595.41796875,
+      "completion_length": 607.80859375,
+      "epoch": 0.256,
+      "grad_norm": 2.053309280558149,
+      "kl": 0.1347993239760399,
+      "learning_rate": 2.7876731904027993e-06,
+      "loss": 0.1112,
+      "num_tokens": 11625699.0,
+      "reward": 0.634765625,
+      "reward_std": 0.22420434933155775,
+      "rewards/accuracy_reward": 0.634765625,
+      "rewards/format_reward": 0.0,
+      "step": 30
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 603.4423828125,
+      "epoch": 0.2730666666666667,
+      "grad_norm": 4.067058887496009,
+      "kl": 0.16801568865776062,
+      "learning_rate": 2.7393581614739926e-06,
+      "loss": 0.1299,
+      "num_tokens": 12397456.0,
+      "reward": 0.654296875,
+      "reward_std": 0.27433712710626423,
+      "rewards/accuracy_reward": 0.654296875,
+      "rewards/format_reward": 0.0,
+      "step": 32
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 627.7119140625,
+      "epoch": 0.29013333333333335,
+      "grad_norm": 1.3847909943122017,
+      "kl": 0.19360647350549698,
+      "learning_rate": 2.6866065519845123e-06,
+      "loss": 0.1096,
+      "num_tokens": 13200929.0,
+      "reward": 0.634765625,
+      "reward_std": 0.21354426653124392,
+      "rewards/accuracy_reward": 0.634765625,
+      "rewards/format_reward": 0.0,
+      "step": 34
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 630.40625,
+      "epoch": 0.3072,
+      "grad_norm": 0.7646338265276826,
+      "kl": 0.1692635715007782,
+      "learning_rate": 2.6296071990054165e-06,
+      "loss": 0.1261,
+      "num_tokens": 14006033.0,
+      "reward": 0.6416015625,
+      "reward_std": 0.2696962603367865,
+      "rewards/accuracy_reward": 0.6416015625,
+      "rewards/format_reward": 0.0,
+      "step": 36
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 629.939453125,
+      "epoch": 0.32426666666666665,
+      "grad_norm": 1.6400416690599355,
+      "kl": 0.22570940852165222,
+      "learning_rate": 2.5685641454270174e-06,
+      "loss": 0.1629,
+      "num_tokens": 14805347.0,
+      "reward": 0.62890625,
+      "reward_std": 0.260196712333709,
+      "rewards/accuracy_reward": 0.62890625,
+      "rewards/format_reward": 0.0,
+      "step": 38
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 590.64453125,
+      "epoch": 0.3413333333333333,
+      "grad_norm": 1.9814122678587998,
+      "kl": 0.20644378662109375,
+      "learning_rate": 2.5036959095382875e-06,
+      "loss": 0.1376,
+      "num_tokens": 15568079.0,
+      "reward": 0.6396484375,
+      "reward_std": 0.2311337033752352,
+      "rewards/accuracy_reward": 0.6396484375,
+      "rewards/format_reward": 0.0,
+      "step": 40
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 624.5517578125,
+      "epoch": 0.3584,
+      "grad_norm": 0.7992402392720819,
+      "kl": 0.20878393948078156,
+      "learning_rate": 2.4352347027881005e-06,
+      "loss": 0.1235,
+      "num_tokens": 16374948.0,
+      "reward": 0.6328125,
+      "reward_std": 0.24743947107344866,
+      "rewards/accuracy_reward": 0.6328125,
+      "rewards/format_reward": 0.0,
+      "step": 42
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 616.9150390625,
+      "epoch": 0.37546666666666667,
+      "grad_norm": 1.0209831895837471,
+      "kl": 0.29173046350479126,
+      "learning_rate": 2.3634255985285104e-06,
+      "loss": 0.166,
+      "num_tokens": 17157997.0,
+      "reward": 0.640625,
+      "reward_std": 0.2668769913725555,
+      "rewards/accuracy_reward": 0.640625,
+      "rewards/format_reward": 0.0,
+      "step": 44
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 655.28125,
+      "epoch": 0.39253333333333335,
+      "grad_norm": 2.3877680127484027,
+      "kl": 0.38541412353515625,
+      "learning_rate": 2.288525654715757e-06,
+      "loss": 0.2105,
+      "num_tokens": 17986773.0,
+      "reward": 0.578125,
+      "reward_std": 0.318851436721161,
+      "rewards/accuracy_reward": 0.578125,
+      "rewards/format_reward": 0.0,
+      "step": 46
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 658.8642578125,
+      "epoch": 0.4096,
+      "grad_norm": 1.1534735045296605,
+      "kl": 0.5227391719818115,
+      "learning_rate": 2.210802993709498e-06,
+      "loss": 0.2131,
+      "num_tokens": 18822666.0,
+      "reward": 0.5625,
+      "reward_std": 0.2897139354608953,
+      "rewards/accuracy_reward": 0.5625,
+      "rewards/format_reward": 0.0,
+      "step": 48
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 686.6533203125,
+      "epoch": 0.4266666666666667,
+      "grad_norm": 0.8115361840628914,
+      "kl": 0.6134325265884399,
+      "learning_rate": 2.1305358424643485e-06,
+      "loss": 0.1922,
+      "num_tokens": 19682471.0,
+      "reward": 0.4873046875,
+      "reward_std": 0.2924760712776333,
+      "rewards/accuracy_reward": 0.4873046875,
+      "rewards/format_reward": 0.0,
+      "step": 50
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 677.859375,
+      "epoch": 0.4437333333333333,
+      "grad_norm": 2.335343150105781,
+      "kl": 0.6195580959320068,
+      "learning_rate": 2.048011536549593e-06,
+      "loss": 0.2115,
+      "num_tokens": 20534495.0,
+      "reward": 0.4921875,
+      "reward_std": 0.33811459480784833,
+      "rewards/accuracy_reward": 0.4921875,
+      "rewards/format_reward": 0.0,
+      "step": 52
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 695.298828125,
+      "epoch": 0.4608,
+      "grad_norm": 0.936446069249802,
+      "kl": 0.8193651437759399,
+      "learning_rate": 1.963525491562421e-06,
+      "loss": 0.198,
+      "num_tokens": 21401217.0,
+      "reward": 0.466796875,
+      "reward_std": 0.3250986794009805,
+      "rewards/accuracy_reward": 0.466796875,
+      "rewards/format_reward": 0.0,
+      "step": 54
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 670.49609375,
+      "epoch": 0.47786666666666666,
+      "grad_norm": 1.2585716542171583,
+      "kl": 0.8083813190460205,
+      "learning_rate": 1.877380145616763e-06,
+      "loss": 0.2072,
+      "num_tokens": 22245781.0,
+      "reward": 0.48046875,
+      "reward_std": 0.3280505524016917,
+      "rewards/accuracy_reward": 0.48046875,
+      "rewards/format_reward": 0.0,
+      "step": 56
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 682.578125,
+      "epoch": 0.49493333333333334,
+      "grad_norm": 1.730566930595855,
+      "kl": 1.0304815769195557,
+      "learning_rate": 1.7898838766933299e-06,
+      "loss": 0.2221,
+      "num_tokens": 23100117.0,
+      "reward": 0.4677734375,
+      "reward_std": 0.34290964691899717,
+      "rewards/accuracy_reward": 0.4677734375,
+      "rewards/format_reward": 0.0,
+      "step": 58
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 693.1591796875,
+      "epoch": 0.512,
+      "grad_norm": 1.438316022085548,
+      "kl": 0.9743777513504028,
+      "learning_rate": 1.7013498987264833e-06,
+      "loss": 0.1934,
+      "num_tokens": 23971624.0,
+      "reward": 0.455078125,
+      "reward_std": 0.31845546374097466,
+      "rewards/accuracy_reward": 0.455078125,
+      "rewards/format_reward": 0.0,
+      "step": 60
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 674.126953125,
+      "epoch": 0.5290666666666667,
+      "grad_norm": 0.7991321837464155,
+      "kl": 1.1657259464263916,
+      "learning_rate": 1.6120951403796365e-06,
+      "loss": 0.2183,
+      "num_tokens": 24829178.0,
+      "reward": 0.4619140625,
+      "reward_std": 0.32980443933047354,
+      "rewards/accuracy_reward": 0.4619140625,
+      "rewards/format_reward": 0.0,
+      "step": 62
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 683.255859375,
+      "epoch": 0.5461333333333334,
+      "grad_norm": 1.252900556331916,
+      "kl": 1.1991257667541504,
+      "learning_rate": 1.5224391105228955e-06,
+      "loss": 0.2287,
+      "num_tokens": 25693608.0,
+      "reward": 0.4541015625,
+      "reward_std": 0.3399870772846043,
+      "rewards/accuracy_reward": 0.4541015625,
+      "rewards/format_reward": 0.0,
+      "step": 64
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 682.7490234375,
+      "epoch": 0.5632,
+      "grad_norm": 1.0658219410533047,
+      "kl": 1.0920262336730957,
+      "learning_rate": 1.4327027544742282e-06,
+      "loss": 0.1729,
+      "num_tokens": 26565695.0,
+      "reward": 0.4716796875,
+      "reward_std": 0.29094083700329065,
+      "rewards/accuracy_reward": 0.4716796875,
+      "rewards/format_reward": 0.0,
+      "step": 66
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 679.49609375,
+      "epoch": 0.5802666666666667,
+      "grad_norm": 0.7213359939486753,
+      "kl": 1.1640419960021973,
+      "learning_rate": 1.3432073050985201e-06,
+      "loss": 0.233,
+      "num_tokens": 27428483.0,
+      "reward": 0.42578125,
+      "reward_std": 0.3486728884745389,
+      "rewards/accuracy_reward": 0.42578125,
+      "rewards/format_reward": 0.0,
+      "step": 68
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 719.8935546875,
+      "epoch": 0.5973333333333334,
+      "grad_norm": 1.141153692198079,
+      "kl": 1.179184913635254,
+      "learning_rate": 1.2542731328772936e-06,
+      "loss": 0.1907,
+      "num_tokens": 28320078.0,
+      "reward": 0.4365234375,
+      "reward_std": 0.3560307021252811,
+      "rewards/accuracy_reward": 0.4365234375,
+      "rewards/format_reward": 0.0,
+      "step": 70
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 663.5009765625,
+      "epoch": 0.6144,
+      "grad_norm": 0.6721068419339931,
+      "kl": 1.0740742683410645,
+      "learning_rate": 1.1662185990655286e-06,
+      "loss": 0.1856,
+      "num_tokens": 29164559.0,
+      "reward": 0.45703125,
+      "reward_std": 0.3046431930270046,
+      "rewards/accuracy_reward": 0.45703125,
+      "rewards/format_reward": 0.0,
+      "step": 72
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 659.466796875,
+      "epoch": 0.6314666666666666,
+      "grad_norm": 1.0601082808384605,
+      "kl": 1.3345909118652344,
+      "learning_rate": 1.079358916040996e-06,
+      "loss": 0.2408,
+      "num_tokens": 29999605.0,
+      "reward": 0.482421875,
+      "reward_std": 0.35783175751566887,
+      "rewards/accuracy_reward": 0.482421875,
+      "rewards/format_reward": 0.0,
+      "step": 74
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 650.8701171875,
+      "epoch": 0.6485333333333333,
+      "grad_norm": 0.7261564892244352,
+      "kl": 1.3090394735336304,
+      "learning_rate": 9.94005018925755e-07,
+      "loss": 0.2281,
+      "num_tokens": 30830072.0,
+      "reward": 0.5,
+      "reward_std": 0.3525468213483691,
+      "rewards/accuracy_reward": 0.5,
+      "rewards/format_reward": 0.0,
+      "step": 76
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 660.0498046875,
+      "epoch": 0.6656,
+      "grad_norm": 0.809244012044935,
+      "kl": 1.414804220199585,
+      "learning_rate": 9.104624525191147e-07,
+      "loss": 0.229,
+      "num_tokens": 31660627.0,
+      "reward": 0.4833984375,
+      "reward_std": 0.3342659214977175,
+      "rewards/accuracy_reward": 0.4833984375,
+      "rewards/format_reward": 0.0,
+      "step": 78
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 707.998046875,
+      "epoch": 0.6826666666666666,
+      "grad_norm": 0.829618860187009,
+      "kl": 1.2725417613983154,
+      "learning_rate": 8.290302775265509e-07,
+      "loss": 0.2243,
+      "num_tokens": 32545497.0,
+      "reward": 0.44140625,
+      "reward_std": 0.33390835602767766,
+      "rewards/accuracy_reward": 0.44140625,
+      "rewards/format_reward": 0.0,
+      "step": 80
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 686.3525390625,
+      "epoch": 0.6997333333333333,
+      "grad_norm": 1.0692886522584544,
+      "kl": 1.46016526222229,
+      "learning_rate": 7.500000000000003e-07,
+      "loss": 0.2172,
+      "num_tokens": 33399938.0,
+      "reward": 0.462890625,
+      "reward_std": 0.34255110286176205,
+      "rewards/accuracy_reward": 0.462890625,
+      "rewards/format_reward": 0.0,
+      "step": 82
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 681.7529296875,
+      "epoch": 0.7168,
+      "grad_norm": 0.7824851387467523,
+      "kl": 1.6537617444992065,
+      "learning_rate": 6.736545278218464e-07,
+      "loss": 0.2377,
+      "num_tokens": 34254333.0,
+      "reward": 0.458984375,
+      "reward_std": 0.35577166243456304,
+      "rewards/accuracy_reward": 0.458984375,
+      "rewards/format_reward": 0.0,
+      "step": 84
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 697.927734375,
+      "epoch": 0.7338666666666667,
+      "grad_norm": 0.6324560657515779,
+      "kl": 1.3420374393463135,
+      "learning_rate": 6.002671579681295e-07,
+      "loss": 0.2057,
+      "num_tokens": 35139235.0,
+      "reward": 0.4541015625,
+      "reward_std": 0.3485010163858533,
+      "rewards/accuracy_reward": NaN,
+      "rewards/format_reward": 0.0,
+      "step": 86
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 667.0712890625,
+      "epoch": 0.7509333333333333,
+      "grad_norm": 0.7581516198278789,
+      "kl": 1.462003469467163,
+      "learning_rate": 5.301005981763008e-07,
+      "loss": 0.2314,
+      "num_tokens": 35991940.0,
+      "reward": 0.4755859375,
+      "reward_std": 0.3375488743185997,
+      "rewards/accuracy_reward": 0.4755859375,
+      "rewards/format_reward": 0.0,
+      "step": 88
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 651.244140625,
+      "epoch": 0.768,
+      "grad_norm": 1.149771581664081,
+      "kl": 1.4573760032653809,
+      "learning_rate": 4.63406026519703e-07,
+      "loss": 0.2122,
+      "num_tokens": 36824670.0,
+      "reward": 0.4892578125,
+      "reward_std": 0.3413656740449369,
+      "rewards/accuracy_reward": 0.4892578125,
+      "rewards/format_reward": 0.0,
+      "step": 90
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 670.01171875,
+      "epoch": 0.7850666666666667,
+      "grad_norm": 0.8650230378477372,
+      "kl": 1.3401384353637695,
+      "learning_rate": 4.0042219225526084e-07,
+      "loss": 0.2158,
+      "num_tokens": 37675050.0,
+      "reward": 0.4619140625,
+      "reward_std": 0.3221384333446622,
+      "rewards/accuracy_reward": 0.4619140625,
+      "rewards/format_reward": 0.0,
+      "step": 92
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 658.0634765625,
+      "epoch": 0.8021333333333334,
+      "grad_norm": 1.069246045365052,
+      "kl": 1.3796547651290894,
+      "learning_rate": 3.4137456116310087e-07,
+      "loss": 0.2059,
+      "num_tokens": 38506451.0,
+      "reward": 0.4775390625,
+      "reward_std": 0.3411385079380125,
+      "rewards/accuracy_reward": 0.4775390625,
+      "rewards/format_reward": 0.0,
+      "step": 94
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 633.93359375,
+      "epoch": 0.8192,
+      "grad_norm": 1.3141678752688295,
+      "kl": 1.41829252243042,
+      "learning_rate": 2.86474508437579e-07,
+      "loss": 0.2136,
+      "num_tokens": 39325543.0,
+      "reward": 0.5146484375,
+      "reward_std": 0.3530385824851692,
+      "rewards/accuracy_reward": NaN,
+      "rewards/format_reward": 0.0,
+      "step": 96
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 638.5849609375,
+      "epoch": 0.8362666666666667,
+      "grad_norm": 0.6225949267416366,
+      "kl": 1.382025122642517,
+      "learning_rate": 2.3591856201894125e-07,
+      "loss": 0.1809,
+      "num_tokens": 40142158.0,
+      "reward": 0.5078125,
+      "reward_std": 0.3463376206345856,
+      "rewards/accuracy_reward": 0.5078125,
+      "rewards/format_reward": 0.0,
+      "step": 98
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 654.8212890625,
+      "epoch": 0.8533333333333334,
+      "grad_norm": 0.8100698859065342,
+      "kl": 1.5549572706222534,
+      "learning_rate": 1.8988769907430552e-07,
+      "loss": 0.2397,
+      "num_tokens": 40968791.0,
+      "reward": 0.49609375,
+      "reward_std": 0.34113691630773246,
+      "rewards/accuracy_reward": 0.49609375,
+      "rewards/format_reward": 0.0,
+      "step": 100
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 636.75,
+      "epoch": 0.8704,
+      "grad_norm": 0.7556943680962844,
+      "kl": 1.4161872863769531,
+      "learning_rate": 1.4854669814637145e-07,
+      "loss": 0.2239,
+      "num_tokens": 41776423.0,
+      "reward": 0.509765625,
+      "reward_std": 0.34553979174233973,
+      "rewards/accuracy_reward": 0.509765625,
+      "rewards/format_reward": 0.0,
+      "step": 102
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 680.8701171875,
+      "epoch": 0.8874666666666666,
+      "grad_norm": 1.864698019362775,
+      "kl": 1.522972583770752,
+      "learning_rate": 1.1204354928900495e-07,
+      "loss": 0.2215,
+      "num_tokens": 42634770.0,
+      "reward": 0.5244140625,
+      "reward_std": 0.37442070548422635,
+      "rewards/accuracy_reward": 0.5244140625,
+      "rewards/format_reward": 0.0,
+      "step": 104
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 641.76171875,
+      "epoch": 0.9045333333333333,
+      "grad_norm": 0.738823482177991,
+      "kl": 1.6065247058868408,
+      "learning_rate": 8.050892430125362e-08,
+      "loss": 0.2359,
+      "num_tokens": 43454382.0,
+      "reward": 0.513671875,
+      "reward_std": 0.31917500402778387,
+      "rewards/accuracy_reward": 0.513671875,
+      "rewards/format_reward": 0.0,
+      "step": 106
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 614.0791015625,
+      "epoch": 0.9216,
+      "grad_norm": 0.8114583750166938,
+      "kl": 1.5821502208709717,
+      "learning_rate": 5.405570895622014e-08,
+      "loss": 0.2511,
+      "num_tokens": 44243727.0,
+      "reward": 0.548828125,
+      "reward_std": 0.34129240782931447,
+      "rewards/accuracy_reward": 0.548828125,
+      "rewards/format_reward": 0.0,
+      "step": 108
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 644.2939453125,
+      "epoch": 0.9386666666666666,
+      "grad_norm": 1.1755921438495844,
+      "kl": 1.6895747184753418,
+      "learning_rate": 3.277859889929147e-08,
+      "loss": 0.2189,
+      "num_tokens": 45063404.0,
+      "reward": 0.525390625,
+      "reward_std": 0.366482344456017,
+      "rewards/accuracy_reward": 0.525390625,
+      "rewards/format_reward": 0.0,
+      "step": 110
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 659.8671875,
+      "epoch": 0.9557333333333333,
+      "grad_norm": 0.8029884872533015,
+      "kl": 1.5845341682434082,
+      "learning_rate": 1.6753760662307216e-08,
+      "loss": 0.2013,
+      "num_tokens": 45907364.0,
+      "reward": 0.484375,
+      "reward_std": 0.34991508489474654,
+      "rewards/accuracy_reward": 0.484375,
+      "rewards/format_reward": 0.0,
+      "step": 112
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 654.6015625,
+      "epoch": 0.9728,
+      "grad_norm": 1.0658581526050372,
+      "kl": 1.4549345970153809,
+      "learning_rate": 6.038559007141398e-09,
+      "loss": 0.2538,
+      "num_tokens": 46737148.0,
+      "reward": 0.5087890625,
+      "reward_std": 0.35136694787070155,
+      "rewards/accuracy_reward": 0.5087890625,
+      "rewards/format_reward": 0.0,
+      "step": 114
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 649.0869140625,
       "epoch": 0.9898666666666667,
-      "kl": 0.0034178495407104492,
-      "num_tokens": 22556823.0,
-      "reward": 0.7607421875,
-      "reward_std": 0.15357404062524438,
-      "rewards/accuracy_reward": 0.7607421875,
+      "grad_norm": 0.9380076341662866,
+      "kl": 1.4578795433044434,
+      "learning_rate": 6.71351574745016e-10,
+      "loss": 0.2334,
+      "num_tokens": 47562341.0,
+      "reward": 0.5068359375,
+      "reward_std": 0.3348612308036536,
+      "rewards/accuracy_reward": 0.5068359375,
+      "rewards/format_reward": 0.0,
+      "step": 116
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 697.40625,
+      "epoch": 0.9984,
+      "kl": 1.8018455505371094,
+      "num_tokens": 47993634.0,
+      "reward": 0.4453125,
+      "reward_std": 0.38571832375600934,
+      "rewards/accuracy_reward": 0.4453125,
       "rewards/format_reward": 0.0,
-      "step": 29,
+      "step": 117,
       "total_flos": 0.0,
-      "train_loss": 0.028255671029910445,
-      "train_runtime": 36935.2831,
-      "train_samples_per_second": 0.203,
-      "train_steps_per_second": 0.001
+      "train_loss": 0.1694948914914559,
+      "train_runtime": 74169.3594,
+      "train_samples_per_second": 0.101,
+      "train_steps_per_second": 0.002
     }
   ],
   "logging_steps": 2,
-  "max_steps": 29,
+  "max_steps": 117,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 500,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d3c54b8da18c9e54b15fbe6df3208ef9f72b48c83ce7d83ada6ae5bfebf9b5ce
-size 9656
+oid sha256:1f3efb0c43f6345d981a04831c0a4ff2a5b2109e1afc3d3843f248d150f1c8c0
+size 8568