Upload folder using huggingface_hub

checkpoint-175/config.json

@@ -0,0 +1,27 @@
+{
+  "_name_or_path": "/mnt/parscratch/users/acp23ay/private/models/OLMo-2-1124-7B-zgh_Tfng-coli-focus/",
+  "architectures": [
+    "Olmo2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "model_type": "olmo2",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 500000,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.48.0.dev0",
+  "use_cache": true,
+  "vocab_size": 100384
+}

checkpoint-175/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "eos_token_id": 100257,
+  "pad_token_id": 100277,
+  "transformers_version": "4.48.0.dev0"
+}

checkpoint-175/model-00001-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2077e977782c258012a577b37888fc8457f034a9060f992cba786dc7f1e7430c
+size 4950070472

checkpoint-175/model-00002-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de0b6a357d03676c4986588aaf4dc96afc3f44a0498514961d2b2cfa1940655e
+size 4991622816

checkpoint-175/model-00003-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1038a7d346a681a8fb6949e054c2246f3f3c80e826855c1cec6af1155cc607d
+size 4924546896

checkpoint-175/model-00004-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33c0a21146ace02d36928787ae7f34be868f7c81b894dea98ec7b0e5a3ea01b2
+size 4857404928

checkpoint-175/model-00005-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7f82a3014ccfc0f67208f63785ae4bc71f09aba86db48bf6139d18178abbfed
+size 4857404928

checkpoint-175/model-00006-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3fc5c982cb4aad5ba5a33476e892d46158f4cd083b37e1c9841d53f23ae1e44
+size 4614509080

checkpoint-175/model.safetensors.index.json

@@ -0,0 +1,362 @@
+{
+  "metadata": {
+    "total_size": 29195517952
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00006-of-00006.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.norm.weight": "model-00006-of-00006.safetensors"
+  }
+}

checkpoint-175/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a11053ee0db46e7ff9efcb7313131b2103a9af211287218cc38ce7299f6f3f36
+size 13055369358

checkpoint-175/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:386fcc8cc1089aade9450d86fb239ea3483f455fd2d78d8378645feecfec9d69
+size 14244

checkpoint-175/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9cc0900cfa767a9bb53ccd2d2cac5f08efc44427a2d2a7304595f8cf0c94b83
+size 1064

checkpoint-175/trainer_state.json

@@ -0,0 +1,1258 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 175,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.005714285714285714,
+      "grad_norm": 70.73185729980469,
+      "learning_rate": 5.555555555555556e-06,
+      "loss": 7.814,
+      "step": 1
+    },
+    {
+      "epoch": 0.011428571428571429,
+      "grad_norm": 67.59354400634766,
+      "learning_rate": 1.1111111111111112e-05,
+      "loss": 7.4984,
+      "step": 2
+    },
+    {
+      "epoch": 0.017142857142857144,
+      "grad_norm": 33.69651794433594,
+      "learning_rate": 1.6666666666666667e-05,
+      "loss": 5.9383,
+      "step": 3
+    },
+    {
+      "epoch": 0.022857142857142857,
+      "grad_norm": 13.683563232421875,
+      "learning_rate": 2.2222222222222223e-05,
+      "loss": 5.1633,
+      "step": 4
+    },
+    {
+      "epoch": 0.02857142857142857,
+      "grad_norm": 14.047224044799805,
+      "learning_rate": 2.777777777777778e-05,
+      "loss": 5.2954,
+      "step": 5
+    },
+    {
+      "epoch": 0.03428571428571429,
+      "grad_norm": 13.862848281860352,
+      "learning_rate": 3.3333333333333335e-05,
+      "loss": 5.2335,
+      "step": 6
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 11.201601028442383,
+      "learning_rate": 3.888888888888889e-05,
+      "loss": 5.1067,
+      "step": 7
+    },
+    {
+      "epoch": 0.045714285714285714,
+      "grad_norm": 9.068086624145508,
+      "learning_rate": 4.4444444444444447e-05,
+      "loss": 4.8728,
+      "step": 8
+    },
+    {
+      "epoch": 0.05142857142857143,
+      "grad_norm": 26.591472625732422,
+      "learning_rate": 5e-05,
+      "loss": 4.9554,
+      "step": 9
+    },
+    {
+      "epoch": 0.05714285714285714,
+      "grad_norm": 11.889716148376465,
+      "learning_rate": 5.555555555555556e-05,
+      "loss": 4.7776,
+      "step": 10
+    },
+    {
+      "epoch": 0.06285714285714286,
+      "grad_norm": 8.594311714172363,
+      "learning_rate": 6.111111111111112e-05,
+      "loss": 4.711,
+      "step": 11
+    },
+    {
+      "epoch": 0.06857142857142857,
+      "grad_norm": 6.407460689544678,
+      "learning_rate": 6.666666666666667e-05,
+      "loss": 4.5297,
+      "step": 12
+    },
+    {
+      "epoch": 0.07428571428571429,
+      "grad_norm": 5.835541248321533,
+      "learning_rate": 7.222222222222222e-05,
+      "loss": 4.5921,
+      "step": 13
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 5.524861812591553,
+      "learning_rate": 7.777777777777778e-05,
+      "loss": 4.6319,
+      "step": 14
+    },
+    {
+      "epoch": 0.08571428571428572,
+      "grad_norm": 5.910550117492676,
+      "learning_rate": 8.333333333333334e-05,
+      "loss": 4.5386,
+      "step": 15
+    },
+    {
+      "epoch": 0.09142857142857143,
+      "grad_norm": 3.423496723175049,
+      "learning_rate": 8.888888888888889e-05,
+      "loss": 4.481,
+      "step": 16
+    },
+    {
+      "epoch": 0.09714285714285714,
+      "grad_norm": 14.430135726928711,
+      "learning_rate": 9.444444444444444e-05,
+      "loss": 4.2537,
+      "step": 17
+    },
+    {
+      "epoch": 0.10285714285714286,
+      "grad_norm": 4.56686544418335,
+      "learning_rate": 0.0001,
+      "loss": 4.5151,
+      "step": 18
+    },
+    {
+      "epoch": 0.10857142857142857,
+      "grad_norm": 3.4966490268707275,
+      "learning_rate": 9.999776148326216e-05,
+      "loss": 4.332,
+      "step": 19
+    },
+    {
+      "epoch": 0.11428571428571428,
+      "grad_norm": 7.750316619873047,
+      "learning_rate": 9.999104613348688e-05,
+      "loss": 4.361,
+      "step": 20
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 4.739988803863525,
+      "learning_rate": 9.997985455197114e-05,
+      "loss": 4.3876,
+      "step": 21
+    },
+    {
+      "epoch": 0.12571428571428572,
+      "grad_norm": 6.870810508728027,
+      "learning_rate": 9.996418774081658e-05,
+      "loss": 3.8275,
+      "step": 22
+    },
+    {
+      "epoch": 0.13142857142857142,
+      "grad_norm": 3.8571834564208984,
+      "learning_rate": 9.994404710283998e-05,
+      "loss": 4.3033,
+      "step": 23
+    },
+    {
+      "epoch": 0.13714285714285715,
+      "grad_norm": 2.915670156478882,
+      "learning_rate": 9.991943444144757e-05,
+      "loss": 3.7222,
+      "step": 24
+    },
+    {
+      "epoch": 0.14285714285714285,
+      "grad_norm": 3.343397855758667,
+      "learning_rate": 9.98903519604735e-05,
+      "loss": 4.4069,
+      "step": 25
+    },
+    {
+      "epoch": 0.14857142857142858,
+      "grad_norm": 4.528430461883545,
+      "learning_rate": 9.985680226398261e-05,
+      "loss": 4.5253,
+      "step": 26
+    },
+    {
+      "epoch": 0.15428571428571428,
+      "grad_norm": 2.878030300140381,
+      "learning_rate": 9.981878835603717e-05,
+      "loss": 4.2524,
+      "step": 27
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 4.2011637687683105,
+      "learning_rate": 9.977631364042795e-05,
+      "loss": 4.5261,
+      "step": 28
+    },
+    {
+      "epoch": 0.1657142857142857,
+      "grad_norm": 3.751654624938965,
+      "learning_rate": 9.972938192036944e-05,
+      "loss": 4.1648,
+      "step": 29
+    },
+    {
+      "epoch": 0.17142857142857143,
+      "grad_norm": 5.761055946350098,
+      "learning_rate": 9.967799739815925e-05,
+      "loss": 4.2666,
+      "step": 30
+    },
+    {
+      "epoch": 0.17714285714285713,
+      "grad_norm": 2.6174347400665283,
+      "learning_rate": 9.962216467480193e-05,
+      "loss": 4.4525,
+      "step": 31
+    },
+    {
+      "epoch": 0.18285714285714286,
+      "grad_norm": 3.8841474056243896,
+      "learning_rate": 9.956188874959687e-05,
+      "loss": 4.0747,
+      "step": 32
+    },
+    {
+      "epoch": 0.18857142857142858,
+      "grad_norm": 2.7967708110809326,
+      "learning_rate": 9.94971750196908e-05,
+      "loss": 4.4146,
+      "step": 33
+    },
+    {
+      "epoch": 0.19428571428571428,
+      "grad_norm": 3.007628917694092,
+      "learning_rate": 9.942802927959443e-05,
+      "loss": 4.2302,
+      "step": 34
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 3.3915443420410156,
+      "learning_rate": 9.93544577206636e-05,
+      "loss": 4.3599,
+      "step": 35
+    },
+    {
+      "epoch": 0.2057142857142857,
+      "grad_norm": 3.1952192783355713,
+      "learning_rate": 9.927646693054496e-05,
+      "loss": 3.9553,
+      "step": 36
+    },
+    {
+      "epoch": 0.21142857142857144,
+      "grad_norm": 3.318091630935669,
+      "learning_rate": 9.919406389258607e-05,
+      "loss": 4.4174,
+      "step": 37
+    },
+    {
+      "epoch": 0.21714285714285714,
+      "grad_norm": 3.1411001682281494,
+      "learning_rate": 9.910725598521013e-05,
+      "loss": 4.3758,
+      "step": 38
+    },
+    {
+      "epoch": 0.22285714285714286,
+      "grad_norm": 2.8873131275177,
+      "learning_rate": 9.901605098125528e-05,
+      "loss": 4.0678,
+      "step": 39
+    },
+    {
+      "epoch": 0.22857142857142856,
+      "grad_norm": 2.7040855884552,
+      "learning_rate": 9.892045704727864e-05,
+      "loss": 4.1704,
+      "step": 40
+    },
+    {
+      "epoch": 0.2342857142857143,
+      "grad_norm": 2.8548882007598877,
+      "learning_rate": 9.882048274282505e-05,
+      "loss": 4.3153,
+      "step": 41
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 2.8522284030914307,
+      "learning_rate": 9.871613701966067e-05,
+      "loss": 4.2952,
+      "step": 42
+    },
+    {
+      "epoch": 0.24571428571428572,
+      "grad_norm": 3.3185341358184814,
+      "learning_rate": 9.860742922097141e-05,
+      "loss": 4.292,
+      "step": 43
+    },
+    {
+      "epoch": 0.25142857142857145,
+      "grad_norm": 2.78251051902771,
+      "learning_rate": 9.849436908052636e-05,
+      "loss": 4.0693,
+      "step": 44
+    },
+    {
+      "epoch": 0.2571428571428571,
+      "grad_norm": 3.5805132389068604,
+      "learning_rate": 9.837696672180618e-05,
+      "loss": 4.2146,
+      "step": 45
+    },
+    {
+      "epoch": 0.26285714285714284,
+      "grad_norm": 3.7008020877838135,
+      "learning_rate": 9.825523265709666e-05,
+      "loss": 4.2144,
+      "step": 46
+    },
+    {
+      "epoch": 0.26857142857142857,
+      "grad_norm": 3.876337766647339,
+      "learning_rate": 9.812917778654748e-05,
+      "loss": 4.3116,
+      "step": 47
+    },
+    {
+      "epoch": 0.2742857142857143,
+      "grad_norm": 3.8221843242645264,
+      "learning_rate": 9.799881339719615e-05,
+      "loss": 3.788,
+      "step": 48
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 3.6571123600006104,
+      "learning_rate": 9.786415116195732e-05,
+      "loss": 4.185,
+      "step": 49
+    },
+    {
+      "epoch": 0.2857142857142857,
+      "grad_norm": 3.0925450325012207,
+      "learning_rate": 9.772520313857775e-05,
+      "loss": 3.9539,
+      "step": 50
+    },
+    {
+      "epoch": 0.2914285714285714,
+      "grad_norm": 2.5549099445343018,
+      "learning_rate": 9.758198176855648e-05,
+      "loss": 3.8761,
+      "step": 51
+    },
+    {
+      "epoch": 0.29714285714285715,
+      "grad_norm": 3.2105484008789062,
+      "learning_rate": 9.743449987603083e-05,
+      "loss": 4.0286,
+      "step": 52
+    },
+    {
+      "epoch": 0.3028571428571429,
+      "grad_norm": 3.4165728092193604,
+      "learning_rate": 9.72827706666282e-05,
+      "loss": 3.9907,
+      "step": 53
+    },
+    {
+      "epoch": 0.30857142857142855,
+      "grad_norm": 3.0576295852661133,
+      "learning_rate": 9.712680772628364e-05,
+      "loss": 3.9951,
+      "step": 54
+    },
+    {
+      "epoch": 0.3142857142857143,
+      "grad_norm": 6.778007507324219,
+      "learning_rate": 9.69666250200232e-05,
+      "loss": 4.0592,
+      "step": 55
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 5.007216453552246,
+      "learning_rate": 9.680223689071364e-05,
+      "loss": 4.1202,
+      "step": 56
+    },
+    {
+      "epoch": 0.32571428571428573,
+      "grad_norm": 4.383210182189941,
+      "learning_rate": 9.663365805777814e-05,
+      "loss": 4.1462,
+      "step": 57
+    },
+    {
+      "epoch": 0.3314285714285714,
+      "grad_norm": 3.4334166049957275,
+      "learning_rate": 9.646090361587827e-05,
+      "loss": 3.9832,
+      "step": 58
+    },
+    {
+      "epoch": 0.33714285714285713,
+      "grad_norm": 3.4360697269439697,
+      "learning_rate": 9.628398903356239e-05,
+      "loss": 3.9158,
+      "step": 59
+    },
+    {
+      "epoch": 0.34285714285714286,
+      "grad_norm": 2.9482150077819824,
+      "learning_rate": 9.610293015188067e-05,
+      "loss": 3.6752,
+      "step": 60
+    },
+    {
+      "epoch": 0.3485714285714286,
+      "grad_norm": 4.406130790710449,
+      "learning_rate": 9.591774318296661e-05,
+      "loss": 4.1023,
+      "step": 61
+    },
+    {
+      "epoch": 0.35428571428571426,
+      "grad_norm": 3.4706637859344482,
+      "learning_rate": 9.572844470858537e-05,
+      "loss": 3.9855,
+      "step": 62
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 4.023850917816162,
+      "learning_rate": 9.553505167864908e-05,
+      "loss": 3.948,
+      "step": 63
+    },
+    {
+      "epoch": 0.3657142857142857,
+      "grad_norm": 2.9235880374908447,
+      "learning_rate": 9.533758140969912e-05,
+      "loss": 3.5046,
+      "step": 64
+    },
+    {
+      "epoch": 0.37142857142857144,
+      "grad_norm": 3.7228496074676514,
+      "learning_rate": 9.513605158335562e-05,
+      "loss": 3.6147,
+      "step": 65
+    },
+    {
+      "epoch": 0.37714285714285717,
+      "grad_norm": 3.482706308364868,
+      "learning_rate": 9.493048024473412e-05,
+      "loss": 3.5001,
+      "step": 66
+    },
+    {
+      "epoch": 0.38285714285714284,
+      "grad_norm": 4.889612674713135,
+      "learning_rate": 9.47208858008299e-05,
+      "loss": 3.941,
+      "step": 67
+    },
+    {
+      "epoch": 0.38857142857142857,
+      "grad_norm": 3.4616644382476807,
+      "learning_rate": 9.450728701886983e-05,
+      "loss": 3.7212,
+      "step": 68
+    },
+    {
+      "epoch": 0.3942857142857143,
+      "grad_norm": 3.7083253860473633,
+      "learning_rate": 9.428970302463185e-05,
+      "loss": 3.6287,
+      "step": 69
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 3.255087375640869,
+      "learning_rate": 9.406815330073244e-05,
+      "loss": 3.7913,
+      "step": 70
+    },
+    {
+      "epoch": 0.4057142857142857,
+      "grad_norm": 3.504664897918701,
+      "learning_rate": 9.384265768488225e-05,
+      "loss": 3.892,
+      "step": 71
+    },
+    {
+      "epoch": 0.4114285714285714,
+      "grad_norm": 3.181671380996704,
+      "learning_rate": 9.36132363681097e-05,
+      "loss": 3.7464,
+      "step": 72
+    },
+    {
+      "epoch": 0.41714285714285715,
+      "grad_norm": 5.243124961853027,
+      "learning_rate": 9.337990989295306e-05,
+      "loss": 3.8959,
+      "step": 73
+    },
+    {
+      "epoch": 0.4228571428571429,
+      "grad_norm": 3.619094133377075,
+      "learning_rate": 9.314269915162114e-05,
+      "loss": 3.8042,
+      "step": 74
+    },
+    {
+      "epoch": 0.42857142857142855,
+      "grad_norm": 4.521401405334473,
+      "learning_rate": 9.290162538412256e-05,
+      "loss": 3.7167,
+      "step": 75
+    },
+    {
+      "epoch": 0.4342857142857143,
+      "grad_norm": 5.233442306518555,
+      "learning_rate": 9.265671017636383e-05,
+      "loss": 3.8335,
+      "step": 76
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 4.076780796051025,
+      "learning_rate": 9.240797545821667e-05,
+      "loss": 3.7429,
+      "step": 77
+    },
+    {
+      "epoch": 0.44571428571428573,
+      "grad_norm": 3.2406275272369385,
+      "learning_rate": 9.215544350155422e-05,
+      "loss": 3.6701,
+      "step": 78
+    },
+    {
+      "epoch": 0.4514285714285714,
+      "grad_norm": 3.954871654510498,
+      "learning_rate": 9.1899136918257e-05,
+      "loss": 3.7933,
+      "step": 79
+    },
+    {
+      "epoch": 0.45714285714285713,
+      "grad_norm": 3.645928144454956,
+      "learning_rate": 9.163907865818806e-05,
+      "loss": 3.776,
+      "step": 80
+    },
+    {
+      "epoch": 0.46285714285714286,
+      "grad_norm": 2.8732380867004395,
+      "learning_rate": 9.13752920071381e-05,
+      "loss": 3.613,
+      "step": 81
+    },
+    {
+      "epoch": 0.4685714285714286,
+      "grad_norm": 2.751926898956299,
+      "learning_rate": 9.110780058474052e-05,
+      "loss": 3.5343,
+      "step": 82
+    },
+    {
+      "epoch": 0.4742857142857143,
+      "grad_norm": 3.1919469833374023,
+      "learning_rate": 9.08366283423563e-05,
+      "loss": 3.6823,
+      "step": 83
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 3.450768232345581,
+      "learning_rate": 9.056179956092962e-05,
+      "loss": 3.6727,
+      "step": 84
+    },
+    {
+      "epoch": 0.4857142857142857,
+      "grad_norm": 4.474507808685303,
+      "learning_rate": 9.028333884881357e-05,
+      "loss": 3.6091,
+      "step": 85
+    },
+    {
+      "epoch": 0.49142857142857144,
+      "grad_norm": 4.046672344207764,
+      "learning_rate": 9.000127113956674e-05,
+      "loss": 3.6397,
+      "step": 86
+    },
+    {
+      "epoch": 0.49714285714285716,
+      "grad_norm": 3.351008176803589,
+      "learning_rate": 8.971562168972064e-05,
+      "loss": 3.6661,
+      "step": 87
+    },
+    {
+      "epoch": 0.5028571428571429,
+      "grad_norm": 4.021505355834961,
+      "learning_rate": 8.94264160765183e-05,
+      "loss": 3.5036,
+      "step": 88
+    },
+    {
+      "epoch": 0.5085714285714286,
+      "grad_norm": 2.808837413787842,
+      "learning_rate": 8.913368019562391e-05,
+      "loss": 3.4524,
+      "step": 89
+    },
+    {
+      "epoch": 0.5142857142857142,
+      "grad_norm": 2.7998015880584717,
+      "learning_rate": 8.883744025880428e-05,
+      "loss": 3.6086,
+      "step": 90
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 2.3896753787994385,
+      "learning_rate": 8.853772279158166e-05,
+      "loss": 3.4425,
+      "step": 91
+    },
+    {
+      "epoch": 0.5257142857142857,
+      "grad_norm": 3.2213077545166016,
+      "learning_rate": 8.823455463085873e-05,
+      "loss": 3.5151,
+      "step": 92
+    },
+    {
+      "epoch": 0.5314285714285715,
+      "grad_norm": 4.131877422332764,
+      "learning_rate": 8.79279629225156e-05,
+      "loss": 3.6083,
+      "step": 93
+    },
+    {
+      "epoch": 0.5371428571428571,
+      "grad_norm": 3.334186553955078,
+      "learning_rate": 8.761797511897906e-05,
+      "loss": 3.488,
+      "step": 94
+    },
+    {
+      "epoch": 0.5428571428571428,
+      "grad_norm": 3.3978617191314697,
+      "learning_rate": 8.730461897676464e-05,
+      "loss": 3.7274,
+      "step": 95
+    },
+    {
+      "epoch": 0.5485714285714286,
+      "grad_norm": 3.625830888748169,
+      "learning_rate": 8.698792255399104e-05,
+      "loss": 3.6878,
+      "step": 96
+    },
+    {
+      "epoch": 0.5542857142857143,
+      "grad_norm": 3.0797600746154785,
+      "learning_rate": 8.666791420786803e-05,
+      "loss": 3.3222,
+      "step": 97
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 3.047025442123413,
+      "learning_rate": 8.634462259215719e-05,
+      "loss": 3.6581,
+      "step": 98
+    },
+    {
+      "epoch": 0.5657142857142857,
+      "grad_norm": 3.139619827270508,
+      "learning_rate": 8.60180766546062e-05,
+      "loss": 3.655,
+      "step": 99
+    },
+    {
+      "epoch": 0.5714285714285714,
+      "grad_norm": 3.2921836376190186,
+      "learning_rate": 8.568830563435694e-05,
+      "loss": 3.5063,
+      "step": 100
+    },
+    {
+      "epoch": 0.5771428571428572,
+      "grad_norm": 3.49196195602417,
+      "learning_rate": 8.535533905932738e-05,
+      "loss": 3.5654,
+      "step": 101
+    },
+    {
+      "epoch": 0.5828571428571429,
+      "grad_norm": 3.840716600418091,
+      "learning_rate": 8.501920674356754e-05,
+      "loss": 3.4673,
+      "step": 102
+    },
+    {
+      "epoch": 0.5885714285714285,
+      "grad_norm": 2.9666836261749268,
+      "learning_rate": 8.467993878459004e-05,
+      "loss": 3.6019,
+      "step": 103
+    },
+    {
+      "epoch": 0.5942857142857143,
+      "grad_norm": 2.98416805267334,
+      "learning_rate": 8.433756556067506e-05,
+      "loss": 3.403,
+      "step": 104
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 3.2488508224487305,
+      "learning_rate": 8.39921177281503e-05,
+      "loss": 3.4254,
+      "step": 105
+    },
+    {
+      "epoch": 0.6057142857142858,
+      "grad_norm": 4.834604263305664,
+      "learning_rate": 8.364362621864595e-05,
+      "loss": 3.6265,
+      "step": 106
+    },
+    {
+      "epoch": 0.6114285714285714,
+      "grad_norm": 3.29841947555542,
+      "learning_rate": 8.329212223632511e-05,
+      "loss": 3.5551,
+      "step": 107
+    },
+    {
+      "epoch": 0.6171428571428571,
+      "grad_norm": 3.66792631149292,
+      "learning_rate": 8.293763725508969e-05,
+      "loss": 3.5378,
+      "step": 108
+    },
+    {
+      "epoch": 0.6228571428571429,
+      "grad_norm": 3.1973721981048584,
+      "learning_rate": 8.258020301576224e-05,
+      "loss": 3.5083,
+      "step": 109
+    },
+    {
+      "epoch": 0.6285714285714286,
+      "grad_norm": 3.9324676990509033,
+      "learning_rate": 8.221985152324385e-05,
+      "loss": 3.4797,
+      "step": 110
+    },
+    {
+      "epoch": 0.6342857142857142,
+      "grad_norm": 3.004668951034546,
+      "learning_rate": 8.185661504364844e-05,
+      "loss": 3.5164,
+      "step": 111
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 2.747321605682373,
+      "learning_rate": 8.149052610141357e-05,
+      "loss": 3.2127,
+      "step": 112
+    },
+    {
+      "epoch": 0.6457142857142857,
+      "grad_norm": 4.0046162605285645,
+      "learning_rate": 8.112161747638823e-05,
+      "loss": 3.5309,
+      "step": 113
+    },
+    {
+      "epoch": 0.6514285714285715,
+      "grad_norm": 2.192357063293457,
+      "learning_rate": 8.074992220089769e-05,
+      "loss": 3.0268,
+      "step": 114
+    },
+    {
+      "epoch": 0.6571428571428571,
+      "grad_norm": 3.4939377307891846,
+      "learning_rate": 8.037547355678577e-05,
+      "loss": 3.5529,
+      "step": 115
+    },
+    {
+      "epoch": 0.6628571428571428,
+      "grad_norm": 2.790357828140259,
+      "learning_rate": 7.999830507243478e-05,
+      "loss": 3.3849,
+      "step": 116
+    },
+    {
+      "epoch": 0.6685714285714286,
+      "grad_norm": 3.2865593433380127,
+      "learning_rate": 7.961845051976334e-05,
+      "loss": 3.4702,
+      "step": 117
+    },
+    {
+      "epoch": 0.6742857142857143,
+      "grad_norm": 3.31089186668396,
+      "learning_rate": 7.923594391120236e-05,
+      "loss": 3.4549,
+      "step": 118
+    },
+    {
+      "epoch": 0.68,
+      "grad_norm": 3.0583114624023438,
+      "learning_rate": 7.88508194966497e-05,
+      "loss": 3.496,
+      "step": 119
+    },
+    {
+      "epoch": 0.6857142857142857,
+      "grad_norm": 2.981785297393799,
+      "learning_rate": 7.846311176040331e-05,
+      "loss": 3.4234,
+      "step": 120
+    },
+    {
+      "epoch": 0.6914285714285714,
+      "grad_norm": 2.7434380054473877,
+      "learning_rate": 7.80728554180734e-05,
+      "loss": 3.3552,
+      "step": 121
+    },
+    {
+      "epoch": 0.6971428571428572,
+      "grad_norm": 9.255672454833984,
+      "learning_rate": 7.768008541347423e-05,
+      "loss": 3.5707,
+      "step": 122
+    },
+    {
+      "epoch": 0.7028571428571428,
+      "grad_norm": 3.796801805496216,
+      "learning_rate": 7.728483691549491e-05,
+      "loss": 3.4186,
+      "step": 123
+    },
+    {
+      "epoch": 0.7085714285714285,
+      "grad_norm": 2.910283327102661,
+      "learning_rate": 7.688714531495061e-05,
+      "loss": 3.3844,
+      "step": 124
+    },
+    {
+      "epoch": 0.7142857142857143,
+      "grad_norm": 2.9970083236694336,
+      "learning_rate": 7.648704622141347e-05,
+      "loss": 3.5279,
+      "step": 125
+    },
+    {
+      "epoch": 0.72,
+      "grad_norm": 2.8664133548736572,
+      "learning_rate": 7.608457546002424e-05,
+      "loss": 3.4568,
+      "step": 126
+    },
+    {
+      "epoch": 0.7257142857142858,
+      "grad_norm": 3.1223716735839844,
+      "learning_rate": 7.567976906828431e-05,
+      "loss": 3.2902,
+      "step": 127
+    },
+    {
+      "epoch": 0.7314285714285714,
+      "grad_norm": 2.87223482131958,
+      "learning_rate": 7.527266329282905e-05,
+      "loss": 3.2975,
+      "step": 128
+    },
+    {
+      "epoch": 0.7371428571428571,
+      "grad_norm": 2.5925962924957275,
+      "learning_rate": 7.486329458618215e-05,
+      "loss": 3.3436,
+      "step": 129
+    },
+    {
+      "epoch": 0.7428571428571429,
+      "grad_norm": 3.0222792625427246,
+      "learning_rate": 7.445169960349167e-05,
+      "loss": 3.4291,
+      "step": 130
+    },
+    {
+      "epoch": 0.7485714285714286,
+      "grad_norm": 3.4972734451293945,
+      "learning_rate": 7.403791519924794e-05,
+      "loss": 3.3198,
+      "step": 131
+    },
+    {
+      "epoch": 0.7542857142857143,
+      "grad_norm": 2.552716016769409,
+      "learning_rate": 7.362197842398355e-05,
+      "loss": 3.4851,
+      "step": 132
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 3.033993721008301,
+      "learning_rate": 7.320392652095585e-05,
+      "loss": 3.2181,
+      "step": 133
+    },
+    {
+      "epoch": 0.7657142857142857,
+      "grad_norm": 3.41973876953125,
+      "learning_rate": 7.278379692281208e-05,
+      "loss": 3.319,
+      "step": 134
+    },
+    {
+      "epoch": 0.7714285714285715,
+      "grad_norm": 3.8678910732269287,
+      "learning_rate": 7.23616272482378e-05,
+      "loss": 3.4172,
+      "step": 135
+    },
+    {
+      "epoch": 0.7771428571428571,
+      "grad_norm": 2.930457830429077,
+      "learning_rate": 7.193745529858826e-05,
+      "loss": 3.3255,
+      "step": 136
+    },
+    {
+      "epoch": 0.7828571428571428,
+      "grad_norm": 2.9811675548553467,
+      "learning_rate": 7.151131905450386e-05,
+      "loss": 3.4066,
+      "step": 137
+    },
+    {
+      "epoch": 0.7885714285714286,
+      "grad_norm": 2.4796462059020996,
+      "learning_rate": 7.10832566725092e-05,
+      "loss": 3.4391,
+      "step": 138
+    },
+    {
+      "epoch": 0.7942857142857143,
+      "grad_norm": 3.213162899017334,
+      "learning_rate": 7.065330648159656e-05,
+      "loss": 3.2553,
+      "step": 139
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 4.176576614379883,
+      "learning_rate": 7.022150697979384e-05,
+      "loss": 3.1612,
+      "step": 140
+    },
+    {
+      "epoch": 0.8057142857142857,
+      "grad_norm": 2.9058704376220703,
+      "learning_rate": 6.97878968307176e-05,
+      "loss": 3.2411,
+      "step": 141
+    },
+    {
+      "epoch": 0.8114285714285714,
+      "grad_norm": 2.4541189670562744,
+      "learning_rate": 6.935251486011087e-05,
+      "loss": 3.2884,
+      "step": 142
+    },
+    {
+      "epoch": 0.8171428571428572,
+      "grad_norm": 2.880113363265991,
+      "learning_rate": 6.891540005236675e-05,
+      "loss": 3.2428,
+      "step": 143
+    },
+    {
+      "epoch": 0.8228571428571428,
+      "grad_norm": 2.3897929191589355,
+      "learning_rate": 6.847659154703785e-05,
+      "loss": 3.3205,
+      "step": 144
+    },
+    {
+      "epoch": 0.8285714285714286,
+      "grad_norm": 2.817380905151367,
+      "learning_rate": 6.803612863533148e-05,
+      "loss": 3.1432,
+      "step": 145
+    },
+    {
+      "epoch": 0.8342857142857143,
+      "grad_norm": 2.5644307136535645,
+      "learning_rate": 6.759405075659166e-05,
+      "loss": 3.1011,
+      "step": 146
+    },
+    {
+      "epoch": 0.84,
+      "grad_norm": 3.0853381156921387,
+      "learning_rate": 6.715039749476763e-05,
+      "loss": 3.3309,
+      "step": 147
+    },
+    {
+      "epoch": 0.8457142857142858,
+      "grad_norm": 2.8046741485595703,
+      "learning_rate": 6.67052085748695e-05,
+      "loss": 3.2116,
+      "step": 148
+    },
+    {
+      "epoch": 0.8514285714285714,
+      "grad_norm": 2.470891237258911,
+      "learning_rate": 6.625852385941119e-05,
+      "loss": 3.013,
+      "step": 149
+    },
+    {
+      "epoch": 0.8571428571428571,
+      "grad_norm": 2.9108824729919434,
+      "learning_rate": 6.58103833448412e-05,
+      "loss": 3.3611,
+      "step": 150
+    },
+    {
+      "epoch": 0.8628571428571429,
+      "grad_norm": 2.715352773666382,
+      "learning_rate": 6.536082715796125e-05,
+      "loss": 3.2395,
+      "step": 151
+    },
+    {
+      "epoch": 0.8685714285714285,
+      "grad_norm": 3.0429553985595703,
+      "learning_rate": 6.490989555233327e-05,
+      "loss": 3.2163,
+      "step": 152
+    },
+    {
+      "epoch": 0.8742857142857143,
+      "grad_norm": 3.281540870666504,
+      "learning_rate": 6.445762890467517e-05,
+      "loss": 3.2273,
+      "step": 153
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 2.3502442836761475,
+      "learning_rate": 6.400406771124536e-05,
+      "loss": 3.1729,
+      "step": 154
+    },
+    {
+      "epoch": 0.8857142857142857,
+      "grad_norm": 2.791484832763672,
+      "learning_rate": 6.354925258421675e-05,
+      "loss": 3.1035,
+      "step": 155
+    },
+    {
+      "epoch": 0.8914285714285715,
+      "grad_norm": 2.831284523010254,
+      "learning_rate": 6.309322424804034e-05,
+      "loss": 3.2658,
+      "step": 156
+    },
+    {
+      "epoch": 0.8971428571428571,
+      "grad_norm": 2.340580940246582,
+      "learning_rate": 6.263602353579868e-05,
+      "loss": 3.186,
+      "step": 157
+    },
+    {
+      "epoch": 0.9028571428571428,
+      "grad_norm": 2.8199057579040527,
+      "learning_rate": 6.21776913855496e-05,
+      "loss": 3.0878,
+      "step": 158
+    },
+    {
+      "epoch": 0.9085714285714286,
+      "grad_norm": 2.6800119876861572,
+      "learning_rate": 6.171826883666074e-05,
+      "loss": 3.2225,
+      "step": 159
+    },
+    {
+      "epoch": 0.9142857142857143,
+      "grad_norm": 3.132681369781494,
+      "learning_rate": 6.125779702613471e-05,
+      "loss": 3.0903,
+      "step": 160
+    },
+    {
+      "epoch": 0.92,
+      "grad_norm": 3.0272161960601807,
+      "learning_rate": 6.079631718492569e-05,
+      "loss": 3.2235,
+      "step": 161
+    },
+    {
+      "epoch": 0.9257142857142857,
+      "grad_norm": 2.942263603210449,
+      "learning_rate": 6.0333870634247645e-05,
+      "loss": 3.2786,
+      "step": 162
+    },
+    {
+      "epoch": 0.9314285714285714,
+      "grad_norm": 2.588752269744873,
+      "learning_rate": 5.9870498781874365e-05,
+      "loss": 3.1267,
+      "step": 163
+    },
+    {
+      "epoch": 0.9371428571428572,
+      "grad_norm": 2.386512279510498,
+      "learning_rate": 5.940624311843169e-05,
+      "loss": 3.1636,
+      "step": 164
+    },
+    {
+      "epoch": 0.9428571428571428,
+      "grad_norm": 2.2246899604797363,
+      "learning_rate": 5.8941145213682594e-05,
+      "loss": 3.1075,
+      "step": 165
+    },
+    {
+      "epoch": 0.9485714285714286,
+      "grad_norm": 2.6086435317993164,
+      "learning_rate": 5.847524671280484e-05,
+      "loss": 3.2346,
+      "step": 166
+    },
+    {
+      "epoch": 0.9542857142857143,
+      "grad_norm": 2.297578811645508,
+      "learning_rate": 5.8008589332662136e-05,
+      "loss": 3.1868,
+      "step": 167
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 2.551158905029297,
+      "learning_rate": 5.7541214858068705e-05,
+      "loss": 3.0412,
+      "step": 168
+    },
+    {
+      "epoch": 0.9657142857142857,
+      "grad_norm": 2.5748233795166016,
+      "learning_rate": 5.7073165138047924e-05,
+      "loss": 3.0838,
+      "step": 169
+    },
+    {
+      "epoch": 0.9714285714285714,
+      "grad_norm": 2.5943634510040283,
+      "learning_rate": 5.660448208208513e-05,
+      "loss": 3.0985,
+      "step": 170
+    },
+    {
+      "epoch": 0.9771428571428571,
+      "grad_norm": 2.3920516967773438,
+      "learning_rate": 5.613520765637489e-05,
+      "loss": 3.1383,
+      "step": 171
+    },
+    {
+      "epoch": 0.9828571428571429,
+      "grad_norm": 2.8342676162719727,
+      "learning_rate": 5.56653838800635e-05,
+      "loss": 3.0972,
+      "step": 172
+    },
+    {
+      "epoch": 0.9885714285714285,
+      "grad_norm": 2.9993340969085693,
+      "learning_rate": 5.519505282148644e-05,
+      "loss": 3.0928,
+      "step": 173
+    },
+    {
+      "epoch": 0.9942857142857143,
+      "grad_norm": 2.4510772228240967,
+      "learning_rate": 5.472425659440157e-05,
+      "loss": 3.0435,
+      "step": 174
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 2.8540945053100586,
+      "learning_rate": 5.425303735421828e-05,
+      "loss": 2.9636,
+      "step": 175
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 350,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 175,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.9559171544252416e+16,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-175/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ad0c704fe63e5ac279ea081f68cb3c977c8cde0304b52633f8383ee6a082eb6
+size 5432

checkpoint-350/config.json

@@ -0,0 +1,27 @@
+{
+  "_name_or_path": "/mnt/parscratch/users/acp23ay/private/models/OLMo-2-1124-7B-zgh_Tfng-coli-focus/",
+  "architectures": [
+    "Olmo2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "model_type": "olmo2",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 500000,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.48.0.dev0",
+  "use_cache": true,
+  "vocab_size": 100384
+}

checkpoint-350/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "eos_token_id": 100257,
+  "pad_token_id": 100277,
+  "transformers_version": "4.48.0.dev0"
+}

checkpoint-350/model-00001-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2457f8b271a2ce0838dc425f2603a78969df287a4d6e4615c18b4cf6b0a1d35b
+size 4950070472

checkpoint-350/model-00002-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de0b6a357d03676c4986588aaf4dc96afc3f44a0498514961d2b2cfa1940655e
+size 4991622816

checkpoint-350/model-00003-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1038a7d346a681a8fb6949e054c2246f3f3c80e826855c1cec6af1155cc607d
+size 4924546896

checkpoint-350/model-00004-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33c0a21146ace02d36928787ae7f34be868f7c81b894dea98ec7b0e5a3ea01b2
+size 4857404928

checkpoint-350/model-00005-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7f82a3014ccfc0f67208f63785ae4bc71f09aba86db48bf6139d18178abbfed
+size 4857404928

checkpoint-350/model-00006-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:693e409bf992725327578bb468a6e939979ae7439baad6c164fe835d29fb178e
+size 4614509080

checkpoint-350/model.safetensors.index.json

@@ -0,0 +1,362 @@
+{
+  "metadata": {
+    "total_size": 29195517952
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00006-of-00006.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.norm.weight": "model-00006-of-00006.safetensors"
+  }
+}

checkpoint-350/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8787342db6b2a4fb6bdd096fd1d5cbf2f3c8c3c5a5ccd12741b19e456ce13420
+size 13055369358

checkpoint-350/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ff264f99d31b522cc7e2a4eac9d38606d0c58a34c0adc74d71e0ca8b371dc36
+size 14244

checkpoint-350/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70732dec4f14211b78555ce1acfc091b59c12a05595924c42e5da10d3a62cd6d
+size 1064

checkpoint-350/trainer_state.json

@@ -0,0 +1,2483 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 500,
+  "global_step": 350,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.005714285714285714,
+      "grad_norm": 70.73185729980469,
+      "learning_rate": 5.555555555555556e-06,
+      "loss": 7.814,
+      "step": 1
+    },
+    {
+      "epoch": 0.011428571428571429,
+      "grad_norm": 67.59354400634766,
+      "learning_rate": 1.1111111111111112e-05,
+      "loss": 7.4984,
+      "step": 2
+    },
+    {
+      "epoch": 0.017142857142857144,
+      "grad_norm": 33.69651794433594,
+      "learning_rate": 1.6666666666666667e-05,
+      "loss": 5.9383,
+      "step": 3
+    },
+    {
+      "epoch": 0.022857142857142857,
+      "grad_norm": 13.683563232421875,
+      "learning_rate": 2.2222222222222223e-05,
+      "loss": 5.1633,
+      "step": 4
+    },
+    {
+      "epoch": 0.02857142857142857,
+      "grad_norm": 14.047224044799805,
+      "learning_rate": 2.777777777777778e-05,
+      "loss": 5.2954,
+      "step": 5
+    },
+    {
+      "epoch": 0.03428571428571429,
+      "grad_norm": 13.862848281860352,
+      "learning_rate": 3.3333333333333335e-05,
+      "loss": 5.2335,
+      "step": 6
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 11.201601028442383,
+      "learning_rate": 3.888888888888889e-05,
+      "loss": 5.1067,
+      "step": 7
+    },
+    {
+      "epoch": 0.045714285714285714,
+      "grad_norm": 9.068086624145508,
+      "learning_rate": 4.4444444444444447e-05,
+      "loss": 4.8728,
+      "step": 8
+    },
+    {
+      "epoch": 0.05142857142857143,
+      "grad_norm": 26.591472625732422,
+      "learning_rate": 5e-05,
+      "loss": 4.9554,
+      "step": 9
+    },
+    {
+      "epoch": 0.05714285714285714,
+      "grad_norm": 11.889716148376465,
+      "learning_rate": 5.555555555555556e-05,
+      "loss": 4.7776,
+      "step": 10
+    },
+    {
+      "epoch": 0.06285714285714286,
+      "grad_norm": 8.594311714172363,
+      "learning_rate": 6.111111111111112e-05,
+      "loss": 4.711,
+      "step": 11
+    },
+    {
+      "epoch": 0.06857142857142857,
+      "grad_norm": 6.407460689544678,
+      "learning_rate": 6.666666666666667e-05,
+      "loss": 4.5297,
+      "step": 12
+    },
+    {
+      "epoch": 0.07428571428571429,
+      "grad_norm": 5.835541248321533,
+      "learning_rate": 7.222222222222222e-05,
+      "loss": 4.5921,
+      "step": 13
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 5.524861812591553,
+      "learning_rate": 7.777777777777778e-05,
+      "loss": 4.6319,
+      "step": 14
+    },
+    {
+      "epoch": 0.08571428571428572,
+      "grad_norm": 5.910550117492676,
+      "learning_rate": 8.333333333333334e-05,
+      "loss": 4.5386,
+      "step": 15
+    },
+    {
+      "epoch": 0.09142857142857143,
+      "grad_norm": 3.423496723175049,
+      "learning_rate": 8.888888888888889e-05,
+      "loss": 4.481,
+      "step": 16
+    },
+    {
+      "epoch": 0.09714285714285714,
+      "grad_norm": 14.430135726928711,
+      "learning_rate": 9.444444444444444e-05,
+      "loss": 4.2537,
+      "step": 17
+    },
+    {
+      "epoch": 0.10285714285714286,
+      "grad_norm": 4.56686544418335,
+      "learning_rate": 0.0001,
+      "loss": 4.5151,
+      "step": 18
+    },
+    {
+      "epoch": 0.10857142857142857,
+      "grad_norm": 3.4966490268707275,
+      "learning_rate": 9.999776148326216e-05,
+      "loss": 4.332,
+      "step": 19
+    },
+    {
+      "epoch": 0.11428571428571428,
+      "grad_norm": 7.750316619873047,
+      "learning_rate": 9.999104613348688e-05,
+      "loss": 4.361,
+      "step": 20
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 4.739988803863525,
+      "learning_rate": 9.997985455197114e-05,
+      "loss": 4.3876,
+      "step": 21
+    },
+    {
+      "epoch": 0.12571428571428572,
+      "grad_norm": 6.870810508728027,
+      "learning_rate": 9.996418774081658e-05,
+      "loss": 3.8275,
+      "step": 22
+    },
+    {
+      "epoch": 0.13142857142857142,
+      "grad_norm": 3.8571834564208984,
+      "learning_rate": 9.994404710283998e-05,
+      "loss": 4.3033,
+      "step": 23
+    },
+    {
+      "epoch": 0.13714285714285715,
+      "grad_norm": 2.915670156478882,
+      "learning_rate": 9.991943444144757e-05,
+      "loss": 3.7222,
+      "step": 24
+    },
+    {
+      "epoch": 0.14285714285714285,
+      "grad_norm": 3.343397855758667,
+      "learning_rate": 9.98903519604735e-05,
+      "loss": 4.4069,
+      "step": 25
+    },
+    {
+      "epoch": 0.14857142857142858,
+      "grad_norm": 4.528430461883545,
+      "learning_rate": 9.985680226398261e-05,
+      "loss": 4.5253,
+      "step": 26
+    },
+    {
+      "epoch": 0.15428571428571428,
+      "grad_norm": 2.878030300140381,
+      "learning_rate": 9.981878835603717e-05,
+      "loss": 4.2524,
+      "step": 27
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 4.2011637687683105,
+      "learning_rate": 9.977631364042795e-05,
+      "loss": 4.5261,
+      "step": 28
+    },
+    {
+      "epoch": 0.1657142857142857,
+      "grad_norm": 3.751654624938965,
+      "learning_rate": 9.972938192036944e-05,
+      "loss": 4.1648,
+      "step": 29
+    },
+    {
+      "epoch": 0.17142857142857143,
+      "grad_norm": 5.761055946350098,
+      "learning_rate": 9.967799739815925e-05,
+      "loss": 4.2666,
+      "step": 30
+    },
+    {
+      "epoch": 0.17714285714285713,
+      "grad_norm": 2.6174347400665283,
+      "learning_rate": 9.962216467480193e-05,
+      "loss": 4.4525,
+      "step": 31
+    },
+    {
+      "epoch": 0.18285714285714286,
+      "grad_norm": 3.8841474056243896,
+      "learning_rate": 9.956188874959687e-05,
+      "loss": 4.0747,
+      "step": 32
+    },
+    {
+      "epoch": 0.18857142857142858,
+      "grad_norm": 2.7967708110809326,
+      "learning_rate": 9.94971750196908e-05,
+      "loss": 4.4146,
+      "step": 33
+    },
+    {
+      "epoch": 0.19428571428571428,
+      "grad_norm": 3.007628917694092,
+      "learning_rate": 9.942802927959443e-05,
+      "loss": 4.2302,
+      "step": 34
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 3.3915443420410156,
+      "learning_rate": 9.93544577206636e-05,
+      "loss": 4.3599,
+      "step": 35
+    },
+    {
+      "epoch": 0.2057142857142857,
+      "grad_norm": 3.1952192783355713,
+      "learning_rate": 9.927646693054496e-05,
+      "loss": 3.9553,
+      "step": 36
+    },
+    {
+      "epoch": 0.21142857142857144,
+      "grad_norm": 3.318091630935669,
+      "learning_rate": 9.919406389258607e-05,
+      "loss": 4.4174,
+      "step": 37
+    },
+    {
+      "epoch": 0.21714285714285714,
+      "grad_norm": 3.1411001682281494,
+      "learning_rate": 9.910725598521013e-05,
+      "loss": 4.3758,
+      "step": 38
+    },
+    {
+      "epoch": 0.22285714285714286,
+      "grad_norm": 2.8873131275177,
+      "learning_rate": 9.901605098125528e-05,
+      "loss": 4.0678,
+      "step": 39
+    },
+    {
+      "epoch": 0.22857142857142856,
+      "grad_norm": 2.7040855884552,
+      "learning_rate": 9.892045704727864e-05,
+      "loss": 4.1704,
+      "step": 40
+    },
+    {
+      "epoch": 0.2342857142857143,
+      "grad_norm": 2.8548882007598877,
+      "learning_rate": 9.882048274282505e-05,
+      "loss": 4.3153,
+      "step": 41
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 2.8522284030914307,
+      "learning_rate": 9.871613701966067e-05,
+      "loss": 4.2952,
+      "step": 42
+    },
+    {
+      "epoch": 0.24571428571428572,
+      "grad_norm": 3.3185341358184814,
+      "learning_rate": 9.860742922097141e-05,
+      "loss": 4.292,
+      "step": 43
+    },
+    {
+      "epoch": 0.25142857142857145,
+      "grad_norm": 2.78251051902771,
+      "learning_rate": 9.849436908052636e-05,
+      "loss": 4.0693,
+      "step": 44
+    },
+    {
+      "epoch": 0.2571428571428571,
+      "grad_norm": 3.5805132389068604,
+      "learning_rate": 9.837696672180618e-05,
+      "loss": 4.2146,
+      "step": 45
+    },
+    {
+      "epoch": 0.26285714285714284,
+      "grad_norm": 3.7008020877838135,
+      "learning_rate": 9.825523265709666e-05,
+      "loss": 4.2144,
+      "step": 46
+    },
+    {
+      "epoch": 0.26857142857142857,
+      "grad_norm": 3.876337766647339,
+      "learning_rate": 9.812917778654748e-05,
+      "loss": 4.3116,
+      "step": 47
+    },
+    {
+      "epoch": 0.2742857142857143,
+      "grad_norm": 3.8221843242645264,
+      "learning_rate": 9.799881339719615e-05,
+      "loss": 3.788,
+      "step": 48
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 3.6571123600006104,
+      "learning_rate": 9.786415116195732e-05,
+      "loss": 4.185,
+      "step": 49
+    },
+    {
+      "epoch": 0.2857142857142857,
+      "grad_norm": 3.0925450325012207,
+      "learning_rate": 9.772520313857775e-05,
+      "loss": 3.9539,
+      "step": 50
+    },
+    {
+      "epoch": 0.2914285714285714,
+      "grad_norm": 2.5549099445343018,
+      "learning_rate": 9.758198176855648e-05,
+      "loss": 3.8761,
+      "step": 51
+    },
+    {
+      "epoch": 0.29714285714285715,
+      "grad_norm": 3.2105484008789062,
+      "learning_rate": 9.743449987603083e-05,
+      "loss": 4.0286,
+      "step": 52
+    },
+    {
+      "epoch": 0.3028571428571429,
+      "grad_norm": 3.4165728092193604,
+      "learning_rate": 9.72827706666282e-05,
+      "loss": 3.9907,
+      "step": 53
+    },
+    {
+      "epoch": 0.30857142857142855,
+      "grad_norm": 3.0576295852661133,
+      "learning_rate": 9.712680772628364e-05,
+      "loss": 3.9951,
+      "step": 54
+    },
+    {
+      "epoch": 0.3142857142857143,
+      "grad_norm": 6.778007507324219,
+      "learning_rate": 9.69666250200232e-05,
+      "loss": 4.0592,
+      "step": 55
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 5.007216453552246,
+      "learning_rate": 9.680223689071364e-05,
+      "loss": 4.1202,
+      "step": 56
+    },
+    {
+      "epoch": 0.32571428571428573,
+      "grad_norm": 4.383210182189941,
+      "learning_rate": 9.663365805777814e-05,
+      "loss": 4.1462,
+      "step": 57
+    },
+    {
+      "epoch": 0.3314285714285714,
+      "grad_norm": 3.4334166049957275,
+      "learning_rate": 9.646090361587827e-05,
+      "loss": 3.9832,
+      "step": 58
+    },
+    {
+      "epoch": 0.33714285714285713,
+      "grad_norm": 3.4360697269439697,
+      "learning_rate": 9.628398903356239e-05,
+      "loss": 3.9158,
+      "step": 59
+    },
+    {
+      "epoch": 0.34285714285714286,
+      "grad_norm": 2.9482150077819824,
+      "learning_rate": 9.610293015188067e-05,
+      "loss": 3.6752,
+      "step": 60
+    },
+    {
+      "epoch": 0.3485714285714286,
+      "grad_norm": 4.406130790710449,
+      "learning_rate": 9.591774318296661e-05,
+      "loss": 4.1023,
+      "step": 61
+    },
+    {
+      "epoch": 0.35428571428571426,
+      "grad_norm": 3.4706637859344482,
+      "learning_rate": 9.572844470858537e-05,
+      "loss": 3.9855,
+      "step": 62
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 4.023850917816162,
+      "learning_rate": 9.553505167864908e-05,
+      "loss": 3.948,
+      "step": 63
+    },
+    {
+      "epoch": 0.3657142857142857,
+      "grad_norm": 2.9235880374908447,
+      "learning_rate": 9.533758140969912e-05,
+      "loss": 3.5046,
+      "step": 64
+    },
+    {
+      "epoch": 0.37142857142857144,
+      "grad_norm": 3.7228496074676514,
+      "learning_rate": 9.513605158335562e-05,
+      "loss": 3.6147,
+      "step": 65
+    },
+    {
+      "epoch": 0.37714285714285717,
+      "grad_norm": 3.482706308364868,
+      "learning_rate": 9.493048024473412e-05,
+      "loss": 3.5001,
+      "step": 66
+    },
+    {
+      "epoch": 0.38285714285714284,
+      "grad_norm": 4.889612674713135,
+      "learning_rate": 9.47208858008299e-05,
+      "loss": 3.941,
+      "step": 67
+    },
+    {
+      "epoch": 0.38857142857142857,
+      "grad_norm": 3.4616644382476807,
+      "learning_rate": 9.450728701886983e-05,
+      "loss": 3.7212,
+      "step": 68
+    },
+    {
+      "epoch": 0.3942857142857143,
+      "grad_norm": 3.7083253860473633,
+      "learning_rate": 9.428970302463185e-05,
+      "loss": 3.6287,
+      "step": 69
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 3.255087375640869,
+      "learning_rate": 9.406815330073244e-05,
+      "loss": 3.7913,
+      "step": 70
+    },
+    {
+      "epoch": 0.4057142857142857,
+      "grad_norm": 3.504664897918701,
+      "learning_rate": 9.384265768488225e-05,
+      "loss": 3.892,
+      "step": 71
+    },
+    {
+      "epoch": 0.4114285714285714,
+      "grad_norm": 3.181671380996704,
+      "learning_rate": 9.36132363681097e-05,
+      "loss": 3.7464,
+      "step": 72
+    },
+    {
+      "epoch": 0.41714285714285715,
+      "grad_norm": 5.243124961853027,
+      "learning_rate": 9.337990989295306e-05,
+      "loss": 3.8959,
+      "step": 73
+    },
+    {
+      "epoch": 0.4228571428571429,
+      "grad_norm": 3.619094133377075,
+      "learning_rate": 9.314269915162114e-05,
+      "loss": 3.8042,
+      "step": 74
+    },
+    {
+      "epoch": 0.42857142857142855,
+      "grad_norm": 4.521401405334473,
+      "learning_rate": 9.290162538412256e-05,
+      "loss": 3.7167,
+      "step": 75
+    },
+    {
+      "epoch": 0.4342857142857143,
+      "grad_norm": 5.233442306518555,
+      "learning_rate": 9.265671017636383e-05,
+      "loss": 3.8335,
+      "step": 76
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 4.076780796051025,
+      "learning_rate": 9.240797545821667e-05,
+      "loss": 3.7429,
+      "step": 77
+    },
+    {
+      "epoch": 0.44571428571428573,
+      "grad_norm": 3.2406275272369385,
+      "learning_rate": 9.215544350155422e-05,
+      "loss": 3.6701,
+      "step": 78
+    },
+    {
+      "epoch": 0.4514285714285714,
+      "grad_norm": 3.954871654510498,
+      "learning_rate": 9.1899136918257e-05,
+      "loss": 3.7933,
+      "step": 79
+    },
+    {
+      "epoch": 0.45714285714285713,
+      "grad_norm": 3.645928144454956,
+      "learning_rate": 9.163907865818806e-05,
+      "loss": 3.776,
+      "step": 80
+    },
+    {
+      "epoch": 0.46285714285714286,
+      "grad_norm": 2.8732380867004395,
+      "learning_rate": 9.13752920071381e-05,
+      "loss": 3.613,
+      "step": 81
+    },
+    {
+      "epoch": 0.4685714285714286,
+      "grad_norm": 2.751926898956299,
+      "learning_rate": 9.110780058474052e-05,
+      "loss": 3.5343,
+      "step": 82
+    },
+    {
+      "epoch": 0.4742857142857143,
+      "grad_norm": 3.1919469833374023,
+      "learning_rate": 9.08366283423563e-05,
+      "loss": 3.6823,
+      "step": 83
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 3.450768232345581,
+      "learning_rate": 9.056179956092962e-05,
+      "loss": 3.6727,
+      "step": 84
+    },
+    {
+      "epoch": 0.4857142857142857,
+      "grad_norm": 4.474507808685303,
+      "learning_rate": 9.028333884881357e-05,
+      "loss": 3.6091,
+      "step": 85
+    },
+    {
+      "epoch": 0.49142857142857144,
+      "grad_norm": 4.046672344207764,
+      "learning_rate": 9.000127113956674e-05,
+      "loss": 3.6397,
+      "step": 86
+    },
+    {
+      "epoch": 0.49714285714285716,
+      "grad_norm": 3.351008176803589,
+      "learning_rate": 8.971562168972064e-05,
+      "loss": 3.6661,
+      "step": 87
+    },
+    {
+      "epoch": 0.5028571428571429,
+      "grad_norm": 4.021505355834961,
+      "learning_rate": 8.94264160765183e-05,
+      "loss": 3.5036,
+      "step": 88
+    },
+    {
+      "epoch": 0.5085714285714286,
+      "grad_norm": 2.808837413787842,
+      "learning_rate": 8.913368019562391e-05,
+      "loss": 3.4524,
+      "step": 89
+    },
+    {
+      "epoch": 0.5142857142857142,
+      "grad_norm": 2.7998015880584717,
+      "learning_rate": 8.883744025880428e-05,
+      "loss": 3.6086,
+      "step": 90
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 2.3896753787994385,
+      "learning_rate": 8.853772279158166e-05,
+      "loss": 3.4425,
+      "step": 91
+    },
+    {
+      "epoch": 0.5257142857142857,
+      "grad_norm": 3.2213077545166016,
+      "learning_rate": 8.823455463085873e-05,
+      "loss": 3.5151,
+      "step": 92
+    },
+    {
+      "epoch": 0.5314285714285715,
+      "grad_norm": 4.131877422332764,
+      "learning_rate": 8.79279629225156e-05,
+      "loss": 3.6083,
+      "step": 93
+    },
+    {
+      "epoch": 0.5371428571428571,
+      "grad_norm": 3.334186553955078,
+      "learning_rate": 8.761797511897906e-05,
+      "loss": 3.488,
+      "step": 94
+    },
+    {
+      "epoch": 0.5428571428571428,
+      "grad_norm": 3.3978617191314697,
+      "learning_rate": 8.730461897676464e-05,
+      "loss": 3.7274,
+      "step": 95
+    },
+    {
+      "epoch": 0.5485714285714286,
+      "grad_norm": 3.625830888748169,
+      "learning_rate": 8.698792255399104e-05,
+      "loss": 3.6878,
+      "step": 96
+    },
+    {
+      "epoch": 0.5542857142857143,
+      "grad_norm": 3.0797600746154785,
+      "learning_rate": 8.666791420786803e-05,
+      "loss": 3.3222,
+      "step": 97
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 3.047025442123413,
+      "learning_rate": 8.634462259215719e-05,
+      "loss": 3.6581,
+      "step": 98
+    },
+    {
+      "epoch": 0.5657142857142857,
+      "grad_norm": 3.139619827270508,
+      "learning_rate": 8.60180766546062e-05,
+      "loss": 3.655,
+      "step": 99
+    },
+    {
+      "epoch": 0.5714285714285714,
+      "grad_norm": 3.2921836376190186,
+      "learning_rate": 8.568830563435694e-05,
+      "loss": 3.5063,
+      "step": 100
+    },
+    {
+      "epoch": 0.5771428571428572,
+      "grad_norm": 3.49196195602417,
+      "learning_rate": 8.535533905932738e-05,
+      "loss": 3.5654,
+      "step": 101
+    },
+    {
+      "epoch": 0.5828571428571429,
+      "grad_norm": 3.840716600418091,
+      "learning_rate": 8.501920674356754e-05,
+      "loss": 3.4673,
+      "step": 102
+    },
+    {
+      "epoch": 0.5885714285714285,
+      "grad_norm": 2.9666836261749268,
+      "learning_rate": 8.467993878459004e-05,
+      "loss": 3.6019,
+      "step": 103
+    },
+    {
+      "epoch": 0.5942857142857143,
+      "grad_norm": 2.98416805267334,
+      "learning_rate": 8.433756556067506e-05,
+      "loss": 3.403,
+      "step": 104
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 3.2488508224487305,
+      "learning_rate": 8.39921177281503e-05,
+      "loss": 3.4254,
+      "step": 105
+    },
+    {
+      "epoch": 0.6057142857142858,
+      "grad_norm": 4.834604263305664,
+      "learning_rate": 8.364362621864595e-05,
+      "loss": 3.6265,
+      "step": 106
+    },
+    {
+      "epoch": 0.6114285714285714,
+      "grad_norm": 3.29841947555542,
+      "learning_rate": 8.329212223632511e-05,
+      "loss": 3.5551,
+      "step": 107
+    },
+    {
+      "epoch": 0.6171428571428571,
+      "grad_norm": 3.66792631149292,
+      "learning_rate": 8.293763725508969e-05,
+      "loss": 3.5378,
+      "step": 108
+    },
+    {
+      "epoch": 0.6228571428571429,
+      "grad_norm": 3.1973721981048584,
+      "learning_rate": 8.258020301576224e-05,
+      "loss": 3.5083,
+      "step": 109
+    },
+    {
+      "epoch": 0.6285714285714286,
+      "grad_norm": 3.9324676990509033,
+      "learning_rate": 8.221985152324385e-05,
+      "loss": 3.4797,
+      "step": 110
+    },
+    {
+      "epoch": 0.6342857142857142,
+      "grad_norm": 3.004668951034546,
+      "learning_rate": 8.185661504364844e-05,
+      "loss": 3.5164,
+      "step": 111
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 2.747321605682373,
+      "learning_rate": 8.149052610141357e-05,
+      "loss": 3.2127,
+      "step": 112
+    },
+    {
+      "epoch": 0.6457142857142857,
+      "grad_norm": 4.0046162605285645,
+      "learning_rate": 8.112161747638823e-05,
+      "loss": 3.5309,
+      "step": 113
+    },
+    {
+      "epoch": 0.6514285714285715,
+      "grad_norm": 2.192357063293457,
+      "learning_rate": 8.074992220089769e-05,
+      "loss": 3.0268,
+      "step": 114
+    },
+    {
+      "epoch": 0.6571428571428571,
+      "grad_norm": 3.4939377307891846,
+      "learning_rate": 8.037547355678577e-05,
+      "loss": 3.5529,
+      "step": 115
+    },
+    {
+      "epoch": 0.6628571428571428,
+      "grad_norm": 2.790357828140259,
+      "learning_rate": 7.999830507243478e-05,
+      "loss": 3.3849,
+      "step": 116
+    },
+    {
+      "epoch": 0.6685714285714286,
+      "grad_norm": 3.2865593433380127,
+      "learning_rate": 7.961845051976334e-05,
+      "loss": 3.4702,
+      "step": 117
+    },
+    {
+      "epoch": 0.6742857142857143,
+      "grad_norm": 3.31089186668396,
+      "learning_rate": 7.923594391120236e-05,
+      "loss": 3.4549,
+      "step": 118
+    },
+    {
+      "epoch": 0.68,
+      "grad_norm": 3.0583114624023438,
+      "learning_rate": 7.88508194966497e-05,
+      "loss": 3.496,
+      "step": 119
+    },
+    {
+      "epoch": 0.6857142857142857,
+      "grad_norm": 2.981785297393799,
+      "learning_rate": 7.846311176040331e-05,
+      "loss": 3.4234,
+      "step": 120
+    },
+    {
+      "epoch": 0.6914285714285714,
+      "grad_norm": 2.7434380054473877,
+      "learning_rate": 7.80728554180734e-05,
+      "loss": 3.3552,
+      "step": 121
+    },
+    {
+      "epoch": 0.6971428571428572,
+      "grad_norm": 9.255672454833984,
+      "learning_rate": 7.768008541347423e-05,
+      "loss": 3.5707,
+      "step": 122
+    },
+    {
+      "epoch": 0.7028571428571428,
+      "grad_norm": 3.796801805496216,
+      "learning_rate": 7.728483691549491e-05,
+      "loss": 3.4186,
+      "step": 123
+    },
+    {
+      "epoch": 0.7085714285714285,
+      "grad_norm": 2.910283327102661,
+      "learning_rate": 7.688714531495061e-05,
+      "loss": 3.3844,
+      "step": 124
+    },
+    {
+      "epoch": 0.7142857142857143,
+      "grad_norm": 2.9970083236694336,
+      "learning_rate": 7.648704622141347e-05,
+      "loss": 3.5279,
+      "step": 125
+    },
+    {
+      "epoch": 0.72,
+      "grad_norm": 2.8664133548736572,
+      "learning_rate": 7.608457546002424e-05,
+      "loss": 3.4568,
+      "step": 126
+    },
+    {
+      "epoch": 0.7257142857142858,
+      "grad_norm": 3.1223716735839844,
+      "learning_rate": 7.567976906828431e-05,
+      "loss": 3.2902,
+      "step": 127
+    },
+    {
+      "epoch": 0.7314285714285714,
+      "grad_norm": 2.87223482131958,
+      "learning_rate": 7.527266329282905e-05,
+      "loss": 3.2975,
+      "step": 128
+    },
+    {
+      "epoch": 0.7371428571428571,
+      "grad_norm": 2.5925962924957275,
+      "learning_rate": 7.486329458618215e-05,
+      "loss": 3.3436,
+      "step": 129
+    },
+    {
+      "epoch": 0.7428571428571429,
+      "grad_norm": 3.0222792625427246,
+      "learning_rate": 7.445169960349167e-05,
+      "loss": 3.4291,
+      "step": 130
+    },
+    {
+      "epoch": 0.7485714285714286,
+      "grad_norm": 3.4972734451293945,
+      "learning_rate": 7.403791519924794e-05,
+      "loss": 3.3198,
+      "step": 131
+    },
+    {
+      "epoch": 0.7542857142857143,
+      "grad_norm": 2.552716016769409,
+      "learning_rate": 7.362197842398355e-05,
+      "loss": 3.4851,
+      "step": 132
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 3.033993721008301,
+      "learning_rate": 7.320392652095585e-05,
+      "loss": 3.2181,
+      "step": 133
+    },
+    {
+      "epoch": 0.7657142857142857,
+      "grad_norm": 3.41973876953125,
+      "learning_rate": 7.278379692281208e-05,
+      "loss": 3.319,
+      "step": 134
+    },
+    {
+      "epoch": 0.7714285714285715,
+      "grad_norm": 3.8678910732269287,
+      "learning_rate": 7.23616272482378e-05,
+      "loss": 3.4172,
+      "step": 135
+    },
+    {
+      "epoch": 0.7771428571428571,
+      "grad_norm": 2.930457830429077,
+      "learning_rate": 7.193745529858826e-05,
+      "loss": 3.3255,
+      "step": 136
+    },
+    {
+      "epoch": 0.7828571428571428,
+      "grad_norm": 2.9811675548553467,
+      "learning_rate": 7.151131905450386e-05,
+      "loss": 3.4066,
+      "step": 137
+    },
+    {
+      "epoch": 0.7885714285714286,
+      "grad_norm": 2.4796462059020996,
+      "learning_rate": 7.10832566725092e-05,
+      "loss": 3.4391,
+      "step": 138
+    },
+    {
+      "epoch": 0.7942857142857143,
+      "grad_norm": 3.213162899017334,
+      "learning_rate": 7.065330648159656e-05,
+      "loss": 3.2553,
+      "step": 139
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 4.176576614379883,
+      "learning_rate": 7.022150697979384e-05,
+      "loss": 3.1612,
+      "step": 140
+    },
+    {
+      "epoch": 0.8057142857142857,
+      "grad_norm": 2.9058704376220703,
+      "learning_rate": 6.97878968307176e-05,
+      "loss": 3.2411,
+      "step": 141
+    },
+    {
+      "epoch": 0.8114285714285714,
+      "grad_norm": 2.4541189670562744,
+      "learning_rate": 6.935251486011087e-05,
+      "loss": 3.2884,
+      "step": 142
+    },
+    {
+      "epoch": 0.8171428571428572,
+      "grad_norm": 2.880113363265991,
+      "learning_rate": 6.891540005236675e-05,
+      "loss": 3.2428,
+      "step": 143
+    },
+    {
+      "epoch": 0.8228571428571428,
+      "grad_norm": 2.3897929191589355,
+      "learning_rate": 6.847659154703785e-05,
+      "loss": 3.3205,
+      "step": 144
+    },
+    {
+      "epoch": 0.8285714285714286,
+      "grad_norm": 2.817380905151367,
+      "learning_rate": 6.803612863533148e-05,
+      "loss": 3.1432,
+      "step": 145
+    },
+    {
+      "epoch": 0.8342857142857143,
+      "grad_norm": 2.5644307136535645,
+      "learning_rate": 6.759405075659166e-05,
+      "loss": 3.1011,
+      "step": 146
+    },
+    {
+      "epoch": 0.84,
+      "grad_norm": 3.0853381156921387,
+      "learning_rate": 6.715039749476763e-05,
+      "loss": 3.3309,
+      "step": 147
+    },
+    {
+      "epoch": 0.8457142857142858,
+      "grad_norm": 2.8046741485595703,
+      "learning_rate": 6.67052085748695e-05,
+      "loss": 3.2116,
+      "step": 148
+    },
+    {
+      "epoch": 0.8514285714285714,
+      "grad_norm": 2.470891237258911,
+      "learning_rate": 6.625852385941119e-05,
+      "loss": 3.013,
+      "step": 149
+    },
+    {
+      "epoch": 0.8571428571428571,
+      "grad_norm": 2.9108824729919434,
+      "learning_rate": 6.58103833448412e-05,
+      "loss": 3.3611,
+      "step": 150
+    },
+    {
+      "epoch": 0.8628571428571429,
+      "grad_norm": 2.715352773666382,
+      "learning_rate": 6.536082715796125e-05,
+      "loss": 3.2395,
+      "step": 151
+    },
+    {
+      "epoch": 0.8685714285714285,
+      "grad_norm": 3.0429553985595703,
+      "learning_rate": 6.490989555233327e-05,
+      "loss": 3.2163,
+      "step": 152
+    },
+    {
+      "epoch": 0.8742857142857143,
+      "grad_norm": 3.281540870666504,
+      "learning_rate": 6.445762890467517e-05,
+      "loss": 3.2273,
+      "step": 153
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 2.3502442836761475,
+      "learning_rate": 6.400406771124536e-05,
+      "loss": 3.1729,
+      "step": 154
+    },
+    {
+      "epoch": 0.8857142857142857,
+      "grad_norm": 2.791484832763672,
+      "learning_rate": 6.354925258421675e-05,
+      "loss": 3.1035,
+      "step": 155
+    },
+    {
+      "epoch": 0.8914285714285715,
+      "grad_norm": 2.831284523010254,
+      "learning_rate": 6.309322424804034e-05,
+      "loss": 3.2658,
+      "step": 156
+    },
+    {
+      "epoch": 0.8971428571428571,
+      "grad_norm": 2.340580940246582,
+      "learning_rate": 6.263602353579868e-05,
+      "loss": 3.186,
+      "step": 157
+    },
+    {
+      "epoch": 0.9028571428571428,
+      "grad_norm": 2.8199057579040527,
+      "learning_rate": 6.21776913855496e-05,
+      "loss": 3.0878,
+      "step": 158
+    },
+    {
+      "epoch": 0.9085714285714286,
+      "grad_norm": 2.6800119876861572,
+      "learning_rate": 6.171826883666074e-05,
+      "loss": 3.2225,
+      "step": 159
+    },
+    {
+      "epoch": 0.9142857142857143,
+      "grad_norm": 3.132681369781494,
+      "learning_rate": 6.125779702613471e-05,
+      "loss": 3.0903,
+      "step": 160
+    },
+    {
+      "epoch": 0.92,
+      "grad_norm": 3.0272161960601807,
+      "learning_rate": 6.079631718492569e-05,
+      "loss": 3.2235,
+      "step": 161
+    },
+    {
+      "epoch": 0.9257142857142857,
+      "grad_norm": 2.942263603210449,
+      "learning_rate": 6.0333870634247645e-05,
+      "loss": 3.2786,
+      "step": 162
+    },
+    {
+      "epoch": 0.9314285714285714,
+      "grad_norm": 2.588752269744873,
+      "learning_rate": 5.9870498781874365e-05,
+      "loss": 3.1267,
+      "step": 163
+    },
+    {
+      "epoch": 0.9371428571428572,
+      "grad_norm": 2.386512279510498,
+      "learning_rate": 5.940624311843169e-05,
+      "loss": 3.1636,
+      "step": 164
+    },
+    {
+      "epoch": 0.9428571428571428,
+      "grad_norm": 2.2246899604797363,
+      "learning_rate": 5.8941145213682594e-05,
+      "loss": 3.1075,
+      "step": 165
+    },
+    {
+      "epoch": 0.9485714285714286,
+      "grad_norm": 2.6086435317993164,
+      "learning_rate": 5.847524671280484e-05,
+      "loss": 3.2346,
+      "step": 166
+    },
+    {
+      "epoch": 0.9542857142857143,
+      "grad_norm": 2.297578811645508,
+      "learning_rate": 5.8008589332662136e-05,
+      "loss": 3.1868,
+      "step": 167
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 2.551158905029297,
+      "learning_rate": 5.7541214858068705e-05,
+      "loss": 3.0412,
+      "step": 168
+    },
+    {
+      "epoch": 0.9657142857142857,
+      "grad_norm": 2.5748233795166016,
+      "learning_rate": 5.7073165138047924e-05,
+      "loss": 3.0838,
+      "step": 169
+    },
+    {
+      "epoch": 0.9714285714285714,
+      "grad_norm": 2.5943634510040283,
+      "learning_rate": 5.660448208208513e-05,
+      "loss": 3.0985,
+      "step": 170
+    },
+    {
+      "epoch": 0.9771428571428571,
+      "grad_norm": 2.3920516967773438,
+      "learning_rate": 5.613520765637489e-05,
+      "loss": 3.1383,
+      "step": 171
+    },
+    {
+      "epoch": 0.9828571428571429,
+      "grad_norm": 2.8342676162719727,
+      "learning_rate": 5.56653838800635e-05,
+      "loss": 3.0972,
+      "step": 172
+    },
+    {
+      "epoch": 0.9885714285714285,
+      "grad_norm": 2.9993340969085693,
+      "learning_rate": 5.519505282148644e-05,
+      "loss": 3.0928,
+      "step": 173
+    },
+    {
+      "epoch": 0.9942857142857143,
+      "grad_norm": 2.4510772228240967,
+      "learning_rate": 5.472425659440157e-05,
+      "loss": 3.0435,
+      "step": 174
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 2.8540945053100586,
+      "learning_rate": 5.425303735421828e-05,
+      "loss": 2.9636,
+      "step": 175
+    },
+    {
+      "epoch": 1.0057142857142858,
+      "grad_norm": 2.8723742961883545,
+      "learning_rate": 5.3781437294222845e-05,
+      "loss": 3.0461,
+      "step": 176
+    },
+    {
+      "epoch": 1.0114285714285713,
+      "grad_norm": 2.713038921356201,
+      "learning_rate": 5.3309498641800337e-05,
+      "loss": 3.0732,
+      "step": 177
+    },
+    {
+      "epoch": 1.0171428571428571,
+      "grad_norm": 2.4567737579345703,
+      "learning_rate": 5.2837263654653715e-05,
+      "loss": 2.7021,
+      "step": 178
+    },
+    {
+      "epoch": 1.022857142857143,
+      "grad_norm": 3.091270923614502,
+      "learning_rate": 5.236477461701985e-05,
+      "loss": 3.0731,
+      "step": 179
+    },
+    {
+      "epoch": 1.0285714285714285,
+      "grad_norm": 2.301541328430176,
+      "learning_rate": 5.1892073835883524e-05,
+      "loss": 2.8954,
+      "step": 180
+    },
+    {
+      "epoch": 1.0342857142857143,
+      "grad_norm": 3.6354727745056152,
+      "learning_rate": 5.141920363718916e-05,
+      "loss": 2.9446,
+      "step": 181
+    },
+    {
+      "epoch": 1.04,
+      "grad_norm": 2.500694990158081,
+      "learning_rate": 5.094620636205095e-05,
+      "loss": 3.0847,
+      "step": 182
+    },
+    {
+      "epoch": 1.0457142857142858,
+      "grad_norm": 3.361649751663208,
+      "learning_rate": 5.047312436296159e-05,
+      "loss": 3.083,
+      "step": 183
+    },
+    {
+      "epoch": 1.0514285714285714,
+      "grad_norm": 3.068362236022949,
+      "learning_rate": 5e-05,
+      "loss": 2.9202,
+      "step": 184
+    },
+    {
+      "epoch": 1.0571428571428572,
+      "grad_norm": 2.7910420894622803,
+      "learning_rate": 4.952687563703841e-05,
+      "loss": 2.7418,
+      "step": 185
+    },
+    {
+      "epoch": 1.062857142857143,
+      "grad_norm": 2.914522171020508,
+      "learning_rate": 4.9053793637949067e-05,
+      "loss": 2.8153,
+      "step": 186
+    },
+    {
+      "epoch": 1.0685714285714285,
+      "grad_norm": 3.4799089431762695,
+      "learning_rate": 4.858079636281085e-05,
+      "loss": 2.825,
+      "step": 187
+    },
+    {
+      "epoch": 1.0742857142857143,
+      "grad_norm": 2.342607259750366,
+      "learning_rate": 4.810792616411649e-05,
+      "loss": 2.4725,
+      "step": 188
+    },
+    {
+      "epoch": 1.08,
+      "grad_norm": 3.211456537246704,
+      "learning_rate": 4.7635225382980176e-05,
+      "loss": 2.9357,
+      "step": 189
+    },
+    {
+      "epoch": 1.0857142857142856,
+      "grad_norm": 2.878603219985962,
+      "learning_rate": 4.7162736345346303e-05,
+      "loss": 2.8111,
+      "step": 190
+    },
+    {
+      "epoch": 1.0914285714285714,
+      "grad_norm": 2.989529848098755,
+      "learning_rate": 4.669050135819966e-05,
+      "loss": 3.1117,
+      "step": 191
+    },
+    {
+      "epoch": 1.0971428571428572,
+      "grad_norm": 3.149132490158081,
+      "learning_rate": 4.621856270577718e-05,
+      "loss": 2.9542,
+      "step": 192
+    },
+    {
+      "epoch": 1.1028571428571428,
+      "grad_norm": 3.134085178375244,
+      "learning_rate": 4.5746962645781724e-05,
+      "loss": 2.9269,
+      "step": 193
+    },
+    {
+      "epoch": 1.1085714285714285,
+      "grad_norm": 2.8814001083374023,
+      "learning_rate": 4.527574340559844e-05,
+      "loss": 2.822,
+      "step": 194
+    },
+    {
+      "epoch": 1.1142857142857143,
+      "grad_norm": 2.7505338191986084,
+      "learning_rate": 4.480494717851359e-05,
+      "loss": 2.7407,
+      "step": 195
+    },
+    {
+      "epoch": 1.12,
+      "grad_norm": 2.8151917457580566,
+      "learning_rate": 4.433461611993651e-05,
+      "loss": 3.0281,
+      "step": 196
+    },
+    {
+      "epoch": 1.1257142857142857,
+      "grad_norm": 2.3902337551116943,
+      "learning_rate": 4.386479234362512e-05,
+      "loss": 2.5909,
+      "step": 197
+    },
+    {
+      "epoch": 1.1314285714285715,
+      "grad_norm": 2.2986514568328857,
+      "learning_rate": 4.3395517917914895e-05,
+      "loss": 2.4779,
+      "step": 198
+    },
+    {
+      "epoch": 1.1371428571428572,
+      "grad_norm": 2.6526057720184326,
+      "learning_rate": 4.292683486195208e-05,
+      "loss": 2.9049,
+      "step": 199
+    },
+    {
+      "epoch": 1.1428571428571428,
+      "grad_norm": 2.7401392459869385,
+      "learning_rate": 4.2458785141931314e-05,
+      "loss": 2.8861,
+      "step": 200
+    },
+    {
+      "epoch": 1.1485714285714286,
+      "grad_norm": 2.4267308712005615,
+      "learning_rate": 4.1991410667337896e-05,
+      "loss": 2.7445,
+      "step": 201
+    },
+    {
+      "epoch": 1.1542857142857144,
+      "grad_norm": 2.703918933868408,
+      "learning_rate": 4.1524753287195165e-05,
+      "loss": 2.8685,
+      "step": 202
+    },
+    {
+      "epoch": 1.16,
+      "grad_norm": 2.6457486152648926,
+      "learning_rate": 4.105885478631741e-05,
+      "loss": 2.9489,
+      "step": 203
+    },
+    {
+      "epoch": 1.1657142857142857,
+      "grad_norm": 2.595367193222046,
+      "learning_rate": 4.059375688156832e-05,
+      "loss": 2.8462,
+      "step": 204
+    },
+    {
+      "epoch": 1.1714285714285715,
+      "grad_norm": 2.519364833831787,
+      "learning_rate": 4.012950121812565e-05,
+      "loss": 2.7106,
+      "step": 205
+    },
+    {
+      "epoch": 1.177142857142857,
+      "grad_norm": 2.35355806350708,
+      "learning_rate": 3.966612936575235e-05,
+      "loss": 2.8384,
+      "step": 206
+    },
+    {
+      "epoch": 1.1828571428571428,
+      "grad_norm": 2.323267936706543,
+      "learning_rate": 3.9203682815074316e-05,
+      "loss": 2.7867,
+      "step": 207
+    },
+    {
+      "epoch": 1.1885714285714286,
+      "grad_norm": 2.6874446868896484,
+      "learning_rate": 3.87422029738653e-05,
+      "loss": 2.9452,
+      "step": 208
+    },
+    {
+      "epoch": 1.1942857142857144,
+      "grad_norm": 2.2298147678375244,
+      "learning_rate": 3.828173116333925e-05,
+      "loss": 2.8886,
+      "step": 209
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 3.020049571990967,
+      "learning_rate": 3.7822308614450406e-05,
+      "loss": 2.6171,
+      "step": 210
+    },
+    {
+      "epoch": 1.2057142857142857,
+      "grad_norm": 2.55083966255188,
+      "learning_rate": 3.736397646420135e-05,
+      "loss": 2.7687,
+      "step": 211
+    },
+    {
+      "epoch": 1.2114285714285715,
+      "grad_norm": 2.995971202850342,
+      "learning_rate": 3.690677575195967e-05,
+      "loss": 2.3852,
+      "step": 212
+    },
+    {
+      "epoch": 1.217142857142857,
+      "grad_norm": 2.295849084854126,
+      "learning_rate": 3.645074741578326e-05,
+      "loss": 2.5424,
+      "step": 213
+    },
+    {
+      "epoch": 1.2228571428571429,
+      "grad_norm": 2.757340908050537,
+      "learning_rate": 3.599593228875465e-05,
+      "loss": 2.8362,
+      "step": 214
+    },
+    {
+      "epoch": 1.2285714285714286,
+      "grad_norm": 1.7869561910629272,
+      "learning_rate": 3.554237109532483e-05,
+      "loss": 2.2078,
+      "step": 215
+    },
+    {
+      "epoch": 1.2342857142857142,
+      "grad_norm": 2.612844944000244,
+      "learning_rate": 3.509010444766674e-05,
+      "loss": 2.9,
+      "step": 216
+    },
+    {
+      "epoch": 1.24,
+      "grad_norm": 2.3267641067504883,
+      "learning_rate": 3.463917284203876e-05,
+      "loss": 2.5606,
+      "step": 217
+    },
+    {
+      "epoch": 1.2457142857142858,
+      "grad_norm": 3.3164138793945312,
+      "learning_rate": 3.41896166551588e-05,
+      "loss": 2.7651,
+      "step": 218
+    },
+    {
+      "epoch": 1.2514285714285713,
+      "grad_norm": 2.7278623580932617,
+      "learning_rate": 3.3741476140588824e-05,
+      "loss": 2.5427,
+      "step": 219
+    },
+    {
+      "epoch": 1.2571428571428571,
+      "grad_norm": 2.37326979637146,
+      "learning_rate": 3.329479142513051e-05,
+      "loss": 2.8948,
+      "step": 220
+    },
+    {
+      "epoch": 1.262857142857143,
+      "grad_norm": 2.3831706047058105,
+      "learning_rate": 3.284960250523237e-05,
+      "loss": 2.6675,
+      "step": 221
+    },
+    {
+      "epoch": 1.2685714285714287,
+      "grad_norm": 2.2831101417541504,
+      "learning_rate": 3.240594924340835e-05,
+      "loss": 2.5774,
+      "step": 222
+    },
+    {
+      "epoch": 1.2742857142857142,
+      "grad_norm": 2.1690053939819336,
+      "learning_rate": 3.196387136466853e-05,
+      "loss": 2.5916,
+      "step": 223
+    },
+    {
+      "epoch": 1.28,
+      "grad_norm": 1.8106428384780884,
+      "learning_rate": 3.152340845296216e-05,
+      "loss": 2.439,
+      "step": 224
+    },
+    {
+      "epoch": 1.2857142857142856,
+      "grad_norm": 2.4522788524627686,
+      "learning_rate": 3.1084599947633256e-05,
+      "loss": 2.7907,
+      "step": 225
+    },
+    {
+      "epoch": 1.2914285714285714,
+      "grad_norm": 2.239344358444214,
+      "learning_rate": 3.0647485139889145e-05,
+      "loss": 2.4848,
+      "step": 226
+    },
+    {
+      "epoch": 1.2971428571428572,
+      "grad_norm": 2.266481637954712,
+      "learning_rate": 3.0212103169282414e-05,
+      "loss": 2.8114,
+      "step": 227
+    },
+    {
+      "epoch": 1.302857142857143,
+      "grad_norm": 2.1474883556365967,
+      "learning_rate": 2.9778493020206154e-05,
+      "loss": 2.4393,
+      "step": 228
+    },
+    {
+      "epoch": 1.3085714285714285,
+      "grad_norm": 2.3610217571258545,
+      "learning_rate": 2.9346693518403458e-05,
+      "loss": 2.8522,
+      "step": 229
+    },
+    {
+      "epoch": 1.3142857142857143,
+      "grad_norm": 2.2677860260009766,
+      "learning_rate": 2.8916743327490803e-05,
+      "loss": 2.7967,
+      "step": 230
+    },
+    {
+      "epoch": 1.32,
+      "grad_norm": 2.3276822566986084,
+      "learning_rate": 2.848868094549615e-05,
+      "loss": 2.7512,
+      "step": 231
+    },
+    {
+      "epoch": 1.3257142857142856,
+      "grad_norm": 2.246767044067383,
+      "learning_rate": 2.8062544701411742e-05,
+      "loss": 2.5956,
+      "step": 232
+    },
+    {
+      "epoch": 1.3314285714285714,
+      "grad_norm": 2.389099359512329,
+      "learning_rate": 2.763837275176224e-05,
+      "loss": 2.9168,
+      "step": 233
+    },
+    {
+      "epoch": 1.3371428571428572,
+      "grad_norm": 2.155503511428833,
+      "learning_rate": 2.721620307718793e-05,
+      "loss": 2.1683,
+      "step": 234
+    },
+    {
+      "epoch": 1.342857142857143,
+      "grad_norm": 2.267953395843506,
+      "learning_rate": 2.6796073479044174e-05,
+      "loss": 2.4045,
+      "step": 235
+    },
+    {
+      "epoch": 1.3485714285714285,
+      "grad_norm": 2.522728204727173,
+      "learning_rate": 2.6378021576016466e-05,
+      "loss": 2.7082,
+      "step": 236
+    },
+    {
+      "epoch": 1.3542857142857143,
+      "grad_norm": 2.3800156116485596,
+      "learning_rate": 2.5962084800752063e-05,
+      "loss": 2.7085,
+      "step": 237
+    },
+    {
+      "epoch": 1.3599999999999999,
+      "grad_norm": 2.1050288677215576,
+      "learning_rate": 2.554830039650834e-05,
+      "loss": 2.6596,
+      "step": 238
+    },
+    {
+      "epoch": 1.3657142857142857,
+      "grad_norm": 2.215343952178955,
+      "learning_rate": 2.5136705413817875e-05,
+      "loss": 2.5746,
+      "step": 239
+    },
+    {
+      "epoch": 1.3714285714285714,
+      "grad_norm": 2.246169328689575,
+      "learning_rate": 2.4727336707170973e-05,
+      "loss": 2.4099,
+      "step": 240
+    },
+    {
+      "epoch": 1.3771428571428572,
+      "grad_norm": 2.3929522037506104,
+      "learning_rate": 2.43202309317157e-05,
+      "loss": 2.5922,
+      "step": 241
+    },
+    {
+      "epoch": 1.3828571428571428,
+      "grad_norm": 2.4444329738616943,
+      "learning_rate": 2.391542453997578e-05,
+      "loss": 2.7618,
+      "step": 242
+    },
+    {
+      "epoch": 1.3885714285714286,
+      "grad_norm": 2.7787318229675293,
+      "learning_rate": 2.3512953778586537e-05,
+      "loss": 2.8187,
+      "step": 243
+    },
+    {
+      "epoch": 1.3942857142857144,
+      "grad_norm": 2.3685426712036133,
+      "learning_rate": 2.3112854685049397e-05,
+      "loss": 2.7183,
+      "step": 244
+    },
+    {
+      "epoch": 1.4,
+      "grad_norm": 2.324232578277588,
+      "learning_rate": 2.271516308450511e-05,
+      "loss": 2.4545,
+      "step": 245
+    },
+    {
+      "epoch": 1.4057142857142857,
+      "grad_norm": 2.3053135871887207,
+      "learning_rate": 2.2319914586525777e-05,
+      "loss": 2.7865,
+      "step": 246
+    },
+    {
+      "epoch": 1.4114285714285715,
+      "grad_norm": 2.2500791549682617,
+      "learning_rate": 2.1927144581926596e-05,
+      "loss": 2.5543,
+      "step": 247
+    },
+    {
+      "epoch": 1.4171428571428573,
+      "grad_norm": 2.5529165267944336,
+      "learning_rate": 2.1536888239596714e-05,
+      "loss": 2.55,
+      "step": 248
+    },
+    {
+      "epoch": 1.4228571428571428,
+      "grad_norm": 2.7905304431915283,
+      "learning_rate": 2.114918050335029e-05,
+      "loss": 2.6568,
+      "step": 249
+    },
+    {
+      "epoch": 1.4285714285714286,
+      "grad_norm": 2.0310685634613037,
+      "learning_rate": 2.0764056088797645e-05,
+      "loss": 2.3341,
+      "step": 250
+    },
+    {
+      "epoch": 1.4342857142857142,
+      "grad_norm": 2.024231433868408,
+      "learning_rate": 2.0381549480236685e-05,
+      "loss": 2.5117,
+      "step": 251
+    },
+    {
+      "epoch": 1.44,
+      "grad_norm": 2.0035414695739746,
+      "learning_rate": 2.000169492756523e-05,
+      "loss": 2.578,
+      "step": 252
+    },
+    {
+      "epoch": 1.4457142857142857,
+      "grad_norm": 2.26760196685791,
+      "learning_rate": 1.9624526443214224e-05,
+      "loss": 2.7067,
+      "step": 253
+    },
+    {
+      "epoch": 1.4514285714285715,
+      "grad_norm": 2.4531280994415283,
+      "learning_rate": 1.9250077799102322e-05,
+      "loss": 2.8926,
+      "step": 254
+    },
+    {
+      "epoch": 1.457142857142857,
+      "grad_norm": 2.12389874458313,
+      "learning_rate": 1.8878382523611786e-05,
+      "loss": 2.5756,
+      "step": 255
+    },
+    {
+      "epoch": 1.4628571428571429,
+      "grad_norm": 2.2084004878997803,
+      "learning_rate": 1.850947389858643e-05,
+      "loss": 2.7307,
+      "step": 256
+    },
+    {
+      "epoch": 1.4685714285714286,
+      "grad_norm": 1.8847193717956543,
+      "learning_rate": 1.8143384956351578e-05,
+      "loss": 2.3994,
+      "step": 257
+    },
+    {
+      "epoch": 1.4742857142857142,
+      "grad_norm": 1.989594578742981,
+      "learning_rate": 1.7780148476756147e-05,
+      "loss": 2.4855,
+      "step": 258
+    },
+    {
+      "epoch": 1.48,
+      "grad_norm": 2.028862237930298,
+      "learning_rate": 1.7419796984237768e-05,
+      "loss": 2.4529,
+      "step": 259
+    },
+    {
+      "epoch": 1.4857142857142858,
+      "grad_norm": 2.2522597312927246,
+      "learning_rate": 1.7062362744910322e-05,
+      "loss": 2.8092,
+      "step": 260
+    },
+    {
+      "epoch": 1.4914285714285715,
+      "grad_norm": 2.2714245319366455,
+      "learning_rate": 1.670787776367489e-05,
+      "loss": 2.3553,
+      "step": 261
+    },
+    {
+      "epoch": 1.497142857142857,
+      "grad_norm": 2.354313373565674,
+      "learning_rate": 1.6356373781354058e-05,
+      "loss": 2.4792,
+      "step": 262
+    },
+    {
+      "epoch": 1.502857142857143,
+      "grad_norm": 2.238191604614258,
+      "learning_rate": 1.6007882271849716e-05,
+      "loss": 2.7143,
+      "step": 263
+    },
+    {
+      "epoch": 1.5085714285714285,
+      "grad_norm": 2.149390697479248,
+      "learning_rate": 1.566243443932496e-05,
+      "loss": 2.5726,
+      "step": 264
+    },
+    {
+      "epoch": 1.5142857142857142,
+      "grad_norm": 2.2486279010772705,
+      "learning_rate": 1.5320061215409958e-05,
+      "loss": 2.468,
+      "step": 265
+    },
+    {
+      "epoch": 1.52,
+      "grad_norm": 2.2732369899749756,
+      "learning_rate": 1.4980793256432474e-05,
+      "loss": 2.7752,
+      "step": 266
+    },
+    {
+      "epoch": 1.5257142857142858,
+      "grad_norm": 1.931206464767456,
+      "learning_rate": 1.4644660940672627e-05,
+      "loss": 2.3823,
+      "step": 267
+    },
+    {
+      "epoch": 1.5314285714285716,
+      "grad_norm": 2.2987496852874756,
+      "learning_rate": 1.4311694365643047e-05,
+      "loss": 2.7731,
+      "step": 268
+    },
+    {
+      "epoch": 1.5371428571428571,
+      "grad_norm": 2.0458099842071533,
+      "learning_rate": 1.3981923345393815e-05,
+      "loss": 2.6913,
+      "step": 269
+    },
+    {
+      "epoch": 1.5428571428571427,
+      "grad_norm": 1.9849002361297607,
+      "learning_rate": 1.3655377407842812e-05,
+      "loss": 2.4437,
+      "step": 270
+    },
+    {
+      "epoch": 1.5485714285714285,
+      "grad_norm": 2.2155039310455322,
+      "learning_rate": 1.3332085792131966e-05,
+      "loss": 2.6769,
+      "step": 271
+    },
+    {
+      "epoch": 1.5542857142857143,
+      "grad_norm": 1.882140874862671,
+      "learning_rate": 1.3012077446008968e-05,
+      "loss": 2.5275,
+      "step": 272
+    },
+    {
+      "epoch": 1.56,
+      "grad_norm": 2.107290267944336,
+      "learning_rate": 1.2695381023235386e-05,
+      "loss": 2.7014,
+      "step": 273
+    },
+    {
+      "epoch": 1.5657142857142858,
+      "grad_norm": 2.2918882369995117,
+      "learning_rate": 1.2382024881020937e-05,
+      "loss": 2.8029,
+      "step": 274
+    },
+    {
+      "epoch": 1.5714285714285714,
+      "grad_norm": 1.8708436489105225,
+      "learning_rate": 1.2072037077484416e-05,
+      "loss": 2.6464,
+      "step": 275
+    },
+    {
+      "epoch": 1.5771428571428572,
+      "grad_norm": 2.4698877334594727,
+      "learning_rate": 1.1765445369141276e-05,
+      "loss": 2.7066,
+      "step": 276
+    },
+    {
+      "epoch": 1.5828571428571427,
+      "grad_norm": 1.7984893321990967,
+      "learning_rate": 1.1462277208418338e-05,
+      "loss": 2.322,
+      "step": 277
+    },
+    {
+      "epoch": 1.5885714285714285,
+      "grad_norm": 2.1097710132598877,
+      "learning_rate": 1.1162559741195733e-05,
+      "loss": 2.7959,
+      "step": 278
+    },
+    {
+      "epoch": 1.5942857142857143,
+      "grad_norm": 2.0764143466949463,
+      "learning_rate": 1.0866319804376085e-05,
+      "loss": 2.5278,
+      "step": 279
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 1.9418200254440308,
+      "learning_rate": 1.0573583923481711e-05,
+      "loss": 2.5384,
+      "step": 280
+    },
+    {
+      "epoch": 1.6057142857142859,
+      "grad_norm": 1.9476945400238037,
+      "learning_rate": 1.0284378310279369e-05,
+      "loss": 2.4785,
+      "step": 281
+    },
+    {
+      "epoch": 1.6114285714285714,
+      "grad_norm": 2.0154948234558105,
+      "learning_rate": 9.998728860433276e-06,
+      "loss": 2.4682,
+      "step": 282
+    },
+    {
+      "epoch": 1.617142857142857,
+      "grad_norm": 1.8853057622909546,
+      "learning_rate": 9.71666115118644e-06,
+      "loss": 2.3503,
+      "step": 283
+    },
+    {
+      "epoch": 1.6228571428571428,
+      "grad_norm": 2.0594022274017334,
+      "learning_rate": 9.438200439070388e-06,
+      "loss": 2.5881,
+      "step": 284
+    },
+    {
+      "epoch": 1.6285714285714286,
+      "grad_norm": 2.1388227939605713,
+      "learning_rate": 9.163371657643716e-06,
+      "loss": 2.7675,
+      "step": 285
+    },
+    {
+      "epoch": 1.6342857142857143,
+      "grad_norm": 1.8516862392425537,
+      "learning_rate": 8.8921994152595e-06,
+      "loss": 2.2794,
+      "step": 286
+    },
+    {
+      "epoch": 1.6400000000000001,
+      "grad_norm": 2.1659090518951416,
+      "learning_rate": 8.624707992861897e-06,
+      "loss": 2.6779,
+      "step": 287
+    },
+    {
+      "epoch": 1.6457142857142857,
+      "grad_norm": 2.096539258956909,
+      "learning_rate": 8.360921341811956e-06,
+      "loss": 2.7319,
+      "step": 288
+    },
+    {
+      "epoch": 1.6514285714285715,
+      "grad_norm": 2.2054567337036133,
+      "learning_rate": 8.100863081742999e-06,
+      "loss": 2.6083,
+      "step": 289
+    },
+    {
+      "epoch": 1.657142857142857,
+      "grad_norm": 1.8694688081741333,
+      "learning_rate": 7.844556498445788e-06,
+      "loss": 2.5052,
+      "step": 290
+    },
+    {
+      "epoch": 1.6628571428571428,
+      "grad_norm": 1.9643840789794922,
+      "learning_rate": 7.592024541783343e-06,
+      "loss": 2.5618,
+      "step": 291
+    },
+    {
+      "epoch": 1.6685714285714286,
+      "grad_norm": 1.9020506143569946,
+      "learning_rate": 7.343289823636168e-06,
+      "loss": 2.6932,
+      "step": 292
+    },
+    {
+      "epoch": 1.6742857142857144,
+      "grad_norm": 1.81760835647583,
+      "learning_rate": 7.098374615877451e-06,
+      "loss": 2.5419,
+      "step": 293
+    },
+    {
+      "epoch": 1.6800000000000002,
+      "grad_norm": 2.1840879917144775,
+      "learning_rate": 6.857300848378856e-06,
+      "loss": 2.6564,
+      "step": 294
+    },
+    {
+      "epoch": 1.6857142857142857,
+      "grad_norm": 2.04762864112854,
+      "learning_rate": 6.62009010704695e-06,
+      "loss": 2.7176,
+      "step": 295
+    },
+    {
+      "epoch": 1.6914285714285713,
+      "grad_norm": 2.0075876712799072,
+      "learning_rate": 6.386763631890313e-06,
+      "loss": 2.5652,
+      "step": 296
+    },
+    {
+      "epoch": 1.697142857142857,
+      "grad_norm": 1.95602285861969,
+      "learning_rate": 6.1573423151177534e-06,
+      "loss": 2.7462,
+      "step": 297
+    },
+    {
+      "epoch": 1.7028571428571428,
+      "grad_norm": 2.0787439346313477,
+      "learning_rate": 5.931846699267557e-06,
+      "loss": 2.7423,
+      "step": 298
+    },
+    {
+      "epoch": 1.7085714285714286,
+      "grad_norm": 1.9398733377456665,
+      "learning_rate": 5.710296975368162e-06,
+      "loss": 2.6488,
+      "step": 299
+    },
+    {
+      "epoch": 1.7142857142857144,
+      "grad_norm": 2.0808677673339844,
+      "learning_rate": 5.4927129811301715e-06,
+      "loss": 2.6719,
+      "step": 300
+    },
+    {
+      "epoch": 1.72,
+      "grad_norm": 1.8372448682785034,
+      "learning_rate": 5.279114199170093e-06,
+      "loss": 2.4105,
+      "step": 301
+    },
+    {
+      "epoch": 1.7257142857142858,
+      "grad_norm": 2.204139232635498,
+      "learning_rate": 5.0695197552659e-06,
+      "loss": 2.7128,
+      "step": 302
+    },
+    {
+      "epoch": 1.7314285714285713,
+      "grad_norm": 1.9228308200836182,
+      "learning_rate": 4.863948416644382e-06,
+      "loss": 2.7051,
+      "step": 303
+    },
+    {
+      "epoch": 1.737142857142857,
+      "grad_norm": 1.8368226289749146,
+      "learning_rate": 4.662418590300871e-06,
+      "loss": 2.4436,
+      "step": 304
+    },
+    {
+      "epoch": 1.7428571428571429,
+      "grad_norm": 2.113672971725464,
+      "learning_rate": 4.464948321350925e-06,
+      "loss": 2.4788,
+      "step": 305
+    },
+    {
+      "epoch": 1.7485714285714287,
+      "grad_norm": 1.8933615684509277,
+      "learning_rate": 4.271555291414636e-06,
+      "loss": 2.4738,
+      "step": 306
+    },
+    {
+      "epoch": 1.7542857142857144,
+      "grad_norm": 2.203009605407715,
+      "learning_rate": 4.082256817033392e-06,
+      "loss": 2.6877,
+      "step": 307
+    },
+    {
+      "epoch": 1.76,
+      "grad_norm": 1.8428044319152832,
+      "learning_rate": 3.897069848119323e-06,
+      "loss": 2.4575,
+      "step": 308
+    },
+    {
+      "epoch": 1.7657142857142856,
+      "grad_norm": 4.350110054016113,
+      "learning_rate": 3.716010966437611e-06,
+      "loss": 2.4935,
+      "step": 309
+    },
+    {
+      "epoch": 1.7714285714285714,
+      "grad_norm": 3.4492435455322266,
+      "learning_rate": 3.539096384121743e-06,
+      "loss": 2.6133,
+      "step": 310
+    },
+    {
+      "epoch": 1.7771428571428571,
+      "grad_norm": 1.9938527345657349,
+      "learning_rate": 3.3663419422218677e-06,
+      "loss": 2.8958,
+      "step": 311
+    },
+    {
+      "epoch": 1.782857142857143,
+      "grad_norm": 2.0432679653167725,
+      "learning_rate": 3.1977631092863615e-06,
+      "loss": 2.7653,
+      "step": 312
+    },
+    {
+      "epoch": 1.7885714285714287,
+      "grad_norm": 1.9322545528411865,
+      "learning_rate": 3.0333749799768107e-06,
+      "loss": 2.4947,
+      "step": 313
+    },
+    {
+      "epoch": 1.7942857142857143,
+      "grad_norm": 2.0642244815826416,
+      "learning_rate": 2.8731922737163685e-06,
+      "loss": 2.4899,
+      "step": 314
+    },
+    {
+      "epoch": 1.8,
+      "grad_norm": 2.0744378566741943,
+      "learning_rate": 2.7172293333717848e-06,
+      "loss": 2.3971,
+      "step": 315
+    },
+    {
+      "epoch": 1.8057142857142856,
+      "grad_norm": 1.949622631072998,
+      "learning_rate": 2.5655001239691835e-06,
+      "loss": 2.5438,
+      "step": 316
+    },
+    {
+      "epoch": 1.8114285714285714,
+      "grad_norm": 1.918898105621338,
+      "learning_rate": 2.4180182314435307e-06,
+      "loss": 2.6909,
+      "step": 317
+    },
+    {
+      "epoch": 1.8171428571428572,
+      "grad_norm": 2.059469223022461,
+      "learning_rate": 2.274796861422246e-06,
+      "loss": 2.8429,
+      "step": 318
+    },
+    {
+      "epoch": 1.822857142857143,
+      "grad_norm": 2.2747437953948975,
+      "learning_rate": 2.1358488380426755e-06,
+      "loss": 2.6863,
+      "step": 319
+    },
+    {
+      "epoch": 1.8285714285714287,
+      "grad_norm": 2.4646449089050293,
+      "learning_rate": 2.0011866028038617e-06,
+      "loss": 2.452,
+      "step": 320
+    },
+    {
+      "epoch": 1.8342857142857143,
+      "grad_norm": 2.05745792388916,
+      "learning_rate": 1.8708222134525167e-06,
+      "loss": 2.6388,
+      "step": 321
+    },
+    {
+      "epoch": 1.8399999999999999,
+      "grad_norm": 1.9350451231002808,
+      "learning_rate": 1.7447673429033362e-06,
+      "loss": 2.5689,
+      "step": 322
+    },
+    {
+      "epoch": 1.8457142857142856,
+      "grad_norm": 13.557132720947266,
+      "learning_rate": 1.623033278193825e-06,
+      "loss": 2.5439,
+      "step": 323
+    },
+    {
+      "epoch": 1.8514285714285714,
+      "grad_norm": 1.9621270895004272,
+      "learning_rate": 1.5056309194736384e-06,
+      "loss": 2.7291,
+      "step": 324
+    },
+    {
+      "epoch": 1.8571428571428572,
+      "grad_norm": 1.8751730918884277,
+      "learning_rate": 1.3925707790285846e-06,
+      "loss": 2.4051,
+      "step": 325
+    },
+    {
+      "epoch": 1.862857142857143,
+      "grad_norm": 1.7616914510726929,
+      "learning_rate": 1.2838629803393342e-06,
+      "loss": 2.1836,
+      "step": 326
+    },
+    {
+      "epoch": 1.8685714285714285,
+      "grad_norm": 2.106416702270508,
+      "learning_rate": 1.1795172571749501e-06,
+      "loss": 2.6986,
+      "step": 327
+    },
+    {
+      "epoch": 1.8742857142857143,
+      "grad_norm": 1.8708751201629639,
+      "learning_rate": 1.0795429527213686e-06,
+      "loss": 2.7,
+      "step": 328
+    },
+    {
+      "epoch": 1.88,
+      "grad_norm": 2.9574902057647705,
+      "learning_rate": 9.839490187447177e-07,
+      "loss": 2.5177,
+      "step": 329
+    },
+    {
+      "epoch": 1.8857142857142857,
+      "grad_norm": 2.1311211585998535,
+      "learning_rate": 8.927440147898702e-07,
+      "loss": 2.6424,
+      "step": 330
+    },
+    {
+      "epoch": 1.8914285714285715,
+      "grad_norm": 1.7282516956329346,
+      "learning_rate": 8.059361074139293e-07,
+      "loss": 2.596,
+      "step": 331
+    },
+    {
+      "epoch": 1.8971428571428572,
+      "grad_norm": 1.961656928062439,
+      "learning_rate": 7.235330694550402e-07,
+      "loss": 2.4858,
+      "step": 332
+    },
+    {
+      "epoch": 1.9028571428571428,
+      "grad_norm": 1.8824467658996582,
+      "learning_rate": 6.45542279336403e-07,
+      "loss": 2.5617,
+      "step": 333
+    },
+    {
+      "epoch": 1.9085714285714286,
+      "grad_norm": 1.8931300640106201,
+      "learning_rate": 5.719707204055735e-07,
+      "loss": 2.6623,
+      "step": 334
+    },
+    {
+      "epoch": 1.9142857142857141,
+      "grad_norm": 2.0303523540496826,
+      "learning_rate": 5.028249803091966e-07,
+      "loss": 2.3588,
+      "step": 335
+    },
+    {
+      "epoch": 1.92,
+      "grad_norm": 1.995362401008606,
+      "learning_rate": 4.381112504031337e-07,
+      "loss": 2.5495,
+      "step": 336
+    },
+    {
+      "epoch": 1.9257142857142857,
+      "grad_norm": 1.7753702402114868,
+      "learning_rate": 3.778353251980837e-07,
+      "loss": 2.3391,
+      "step": 337
+    },
+    {
+      "epoch": 1.9314285714285715,
+      "grad_norm": 2.7760531902313232,
+      "learning_rate": 3.2200260184075406e-07,
+      "loss": 2.4219,
+      "step": 338
+    },
+    {
+      "epoch": 1.9371428571428573,
+      "grad_norm": 1.7815756797790527,
+      "learning_rate": 2.7061807963056906e-07,
+      "loss": 2.2366,
+      "step": 339
+    },
+    {
+      "epoch": 1.9428571428571428,
+      "grad_norm": 1.9269545078277588,
+      "learning_rate": 2.2368635957205618e-07,
+      "loss": 2.7267,
+      "step": 340
+    },
+    {
+      "epoch": 1.9485714285714286,
+      "grad_norm": 1.8720332384109497,
+      "learning_rate": 1.812116439628364e-07,
+      "loss": 2.6532,
+      "step": 341
+    },
+    {
+      "epoch": 1.9542857142857142,
+      "grad_norm": 1.8351267576217651,
+      "learning_rate": 1.431977360173975e-07,
+      "loss": 2.4401,
+      "step": 342
+    },
+    {
+      "epoch": 1.96,
+      "grad_norm": 1.8677483797073364,
+      "learning_rate": 1.0964803952650537e-07,
+      "loss": 2.6214,
+      "step": 343
+    },
+    {
+      "epoch": 1.9657142857142857,
+      "grad_norm": 2.0632822513580322,
+      "learning_rate": 8.056555855243675e-08,
+      "loss": 2.6871,
+      "step": 344
+    },
+    {
+      "epoch": 1.9714285714285715,
+      "grad_norm": 1.9952597618103027,
+      "learning_rate": 5.59528971600165e-08,
+      "loss": 2.7334,
+      "step": 345
+    },
+    {
+      "epoch": 1.977142857142857,
+      "grad_norm": 1.846852421760559,
+      "learning_rate": 3.581225918342646e-08,
+      "loss": 2.6545,
+      "step": 346
+    },
+    {
+      "epoch": 1.9828571428571429,
+      "grad_norm": 1.7492363452911377,
+      "learning_rate": 2.0145448028874304e-08,
+      "loss": 2.5111,
+      "step": 347
+    },
+    {
+      "epoch": 1.9885714285714284,
+      "grad_norm": 1.789237380027771,
+      "learning_rate": 8.953866513111697e-09,
+      "loss": 2.4304,
+      "step": 348
+    },
+    {
+      "epoch": 1.9942857142857142,
+      "grad_norm": 1.959470510482788,
+      "learning_rate": 2.2385167378513593e-09,
+      "loss": 2.4341,
+      "step": 349
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 2.1399502754211426,
+      "learning_rate": 0.0,
+      "loss": 2.3662,
+      "step": 350
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 350,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 175,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 5.911834308850483e+16,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-350/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ad0c704fe63e5ac279ea081f68cb3c977c8cde0304b52633f8383ee6a082eb6
+size 5432

config.json

@@ -0,0 +1,27 @@
+{
+  "_name_or_path": "/mnt/parscratch/users/acp23ay/private/models/OLMo-2-1124-7B-zgh_Tfng-coli-focus/",
+  "architectures": [
+    "Olmo2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "model_type": "olmo2",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 100277,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 500000,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.48.0.dev0",
+  "use_cache": true,
+  "vocab_size": 100384
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "eos_token_id": 100257,
+  "pad_token_id": 100277,
+  "transformers_version": "4.48.0.dev0"
+}

model-00001-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2457f8b271a2ce0838dc425f2603a78969df287a4d6e4615c18b4cf6b0a1d35b
+size 4950070472

model-00002-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de0b6a357d03676c4986588aaf4dc96afc3f44a0498514961d2b2cfa1940655e
+size 4991622816

model-00003-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1038a7d346a681a8fb6949e054c2246f3f3c80e826855c1cec6af1155cc607d
+size 4924546896

model-00004-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33c0a21146ace02d36928787ae7f34be868f7c81b894dea98ec7b0e5a3ea01b2
+size 4857404928

model-00005-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7f82a3014ccfc0f67208f63785ae4bc71f09aba86db48bf6139d18178abbfed
+size 4857404928

model-00006-of-00006.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:693e409bf992725327578bb468a6e939979ae7439baad6c164fe835d29fb178e
+size 4614509080

model.safetensors.index.json

@@ -0,0 +1,362 @@
+{
+  "metadata": {
+    "total_size": 29195517952
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00006-of-00006.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
+    "model.norm.weight": "model-00006-of-00006.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|pad|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,189 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "100256": {
+      "content": "<|extra_id_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100257": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100258": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100259": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100260": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100261": {
+      "content": "|||PHONE_NUMBER|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100262": {
+      "content": "|||EMAIL_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100263": {
+      "content": "|||IP_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100264": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100265": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100266": {
+      "content": "<|extra_id_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100267": {
+      "content": "<|extra_id_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100268": {
+      "content": "<|extra_id_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100269": {
+      "content": "<|extra_id_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100270": {
+      "content": "<|extra_id_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100271": {
+      "content": "<|extra_id_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100272": {
+      "content": "<|extra_id_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100273": {
+      "content": "<|extra_id_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100274": {
+      "content": "<|extra_id_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100275": {
+      "content": "<|extra_id_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100276": {
+      "content": "<|endofprompt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100277": {
+      "content": "<|pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|pad|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ad0c704fe63e5ac279ea081f68cb3c977c8cde0304b52633f8383ee6a082eb6
+size 5432