Upload folder using huggingface_hub

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/added_tokens.json

@@ -0,0 +1,6 @@
+{
+  "<image>": 151646,
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644
+}

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/added_tokens.json

@@ -0,0 +1,6 @@
+{
+  "<image>": 151646,
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644
+}

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/config.json

@@ -0,0 +1,221 @@
+{
+  "_name_or_path": "lmms-lab/llava-onevision-qwen2-7b-si",
+  "add_faster_video": false,
+  "add_time_instruction": true,
+  "architectures": [
+    "LlavaQwenForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "faster_token_stride": 10,
+  "force_sample": true,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "ignore_index": -100,
+  "image_aspect_ratio": "anyres_max_9",
+  "image_crop_resolution": null,
+  "image_grid_pinpoints": [
+    [
+      384,
+      384
+    ],
+    [
+      384,
+      768
+    ],
+    [
+      384,
+      1152
+    ],
+    [
+      384,
+      1536
+    ],
+    [
+      384,
+      1920
+    ],
+    [
+      384,
+      2304
+    ],
+    [
+      768,
+      384
+    ],
+    [
+      768,
+      768
+    ],
+    [
+      768,
+      1152
+    ],
+    [
+      768,
+      1536
+    ],
+    [
+      768,
+      1920
+    ],
+    [
+      768,
+      2304
+    ],
+    [
+      1152,
+      384
+    ],
+    [
+      1152,
+      768
+    ],
+    [
+      1152,
+      1152
+    ],
+    [
+      1152,
+      1536
+    ],
+    [
+      1152,
+      1920
+    ],
+    [
+      1152,
+      2304
+    ],
+    [
+      1536,
+      384
+    ],
+    [
+      1536,
+      768
+    ],
+    [
+      1536,
+      1152
+    ],
+    [
+      1536,
+      1536
+    ],
+    [
+      1536,
+      1920
+    ],
+    [
+      1536,
+      2304
+    ],
+    [
+      1920,
+      384
+    ],
+    [
+      1920,
+      768
+    ],
+    [
+      1920,
+      1152
+    ],
+    [
+      1920,
+      1536
+    ],
+    [
+      1920,
+      1920
+    ],
+    [
+      1920,
+      2304
+    ],
+    [
+      2304,
+      384
+    ],
+    [
+      2304,
+      768
+    ],
+    [
+      2304,
+      1152
+    ],
+    [
+      2304,
+      1536
+    ],
+    [
+      2304,
+      1920
+    ],
+    [
+      2304,
+      2304
+    ]
+  ],
+  "image_split_resolution": null,
+  "image_token_index": 151646,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "mm_hidden_size": 1152,
+  "mm_newline_position": "grid",
+  "mm_patch_merge_type": "spatial_unpad",
+  "mm_projector_lr": null,
+  "mm_projector_type": "mlp2x_gelu",
+  "mm_resampler_type": null,
+  "mm_spatial_pool_mode": "bilinear",
+  "mm_spatial_pool_stride": 2,
+  "mm_tunable_parts": "mm_vision_tower,mm_mlp_adapter,mm_language_model",
+  "mm_use_im_patch_token": false,
+  "mm_use_im_start_end": false,
+  "mm_vision_select_feature": "patch",
+  "mm_vision_select_layer": -2,
+  "mm_vision_tower": "google/siglip-so400m-patch14-384",
+  "mm_vision_tower_lr": 2e-06,
+  "model_type": "llava",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "pos_skipping_range": 4096,
+  "projector_hidden_act": "gelu",
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": 131072,
+  "text_config": {
+    "model_type": "llama"
+  },
+  "tie_word_embeddings": false,
+  "tokenizer_model_max_length": 32768,
+  "tokenizer_padding_side": "right",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.40.0.dev0",
+  "use_cache": false,
+  "use_mm_proj": true,
+  "use_pos_skipping": false,
+  "use_sliding_window": false,
+  "vision_config": {
+    "hidden_size": 1024,
+    "image_size": 336,
+    "intermediate_size": 4096,
+    "model_type": "clip_vision_model",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 24,
+    "patch_size": 14,
+    "projection_dim": 768,
+    "vocab_size": 32000
+  },
+  "vision_feature_layer": -2,
+  "vision_feature_select_strategy": "default",
+  "vision_tower_pretrained": null
+}

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.05,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "4.40.0.dev0"
+}

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/global_step500/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb0414e5cc5c727ac7ca93dd58b4498d1295c378288583c133422bf2bf8b2b03
+size 24091055846

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/global_step500/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b1bfe6f71325ca71e2dce3060023ce741018771fcce67d70ce47b8872481e75
+size 24091055846

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/global_step500/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79e3a8875331146d1454f778947d829dd7a51ef376fe87054b3d366a96747695
+size 24091055846

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/global_step500/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7995817f58750dcacb8b4b68d03bf6c82db08a1d1402e03a0f6e3172b1155cb0
+size 24091055846

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/global_step500/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d3009eb1490aef3931e316ec63030a529944f011418a919367481c98888d618
+size 419351

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/global_step500/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b78406943c6fb485f8fae89e4291616ce089eaeeae56cc70d671077b60c3fb4c
+size 419351

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/global_step500/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5406035a1c98d1ca12650b7d97bba5aa5724d60235e9ca6a4c70c185bfa5d138
+size 419351

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/global_step500/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3eadc5a1eccc5215acda5b04ec77e33be02e6c3545d924d9d0d64b1f37783ae2
+size 419351

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/latest

@@ -0,0 +1 @@
+global_step500

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f966ae2c87927d9413e33e504a7bb8e850655b5250ae7d66f27063d91c914448
+size 4877668032

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03cf039de0fb75cfaabd873f824eb7136d99bcb350175c80ce77e2da396e6f55
+size 4932751008

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4ab349e68cbecd2818266f4cbdedbd5a3fc0182f5c0ba70a693ea0441c5f9de
+size 4994571904

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11d909a1044aa3465efa12c5ce42d521ec75b8a71cdde25161c521d1ecaf64b8
+size 1255812224

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/model.safetensors.index.json

@@ -0,0 +1,772 @@
+{
+  "metadata": {
+    "total_size": 16060697664
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.image_newline": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.mm_projector.0.bias": "model-00004-of-00004.safetensors",
+    "model.mm_projector.0.weight": "model-00004-of-00004.safetensors",
+    "model.mm_projector.2.bias": "model-00004-of-00004.safetensors",
+    "model.mm_projector.2.weight": "model-00004-of-00004.safetensors",
+    "model.norm.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.embeddings.patch_embedding.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.embeddings.patch_embedding.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.embeddings.position_embedding.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.post_layernorm.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.post_layernorm.weight": "model-00004-of-00004.safetensors"
+  }
+}

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92cc13315f24c28015d695b6cde08bb1cd6fea4cbc435998485ed6fbe4c91285
+size 15024

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4c154b6a63e0b1f98f7d2847944398f99f1657d35e8eddf7fdf0ae2c24b0552
+size 15024

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f784c6a9507b51189f2caffbd178ea9882103b75852e31c15f47fdae6a43af1d
+size 15024

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34b023e05bc2d12b91dc436d4922b990d50ec8dc56d40dc3e36b3bb34fc81341
+size 15024

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce711c5162a77c8e3dc9e306c7c613224d95db6d1eb11552783a6124905667f7
+size 1064

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/special_tokens_map.json

@@ -0,0 +1,20 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/tokenizer_config.json

@@ -0,0 +1,53 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "processor_class": "LlavaProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/trainer_state.json

@@ -0,0 +1,3521 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.5714285714285714,
+  "eval_steps": 500,
+  "global_step": 500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "grad_norm": 8.812040662498683,
+      "learning_rate": 3.7037037037037036e-07,
+      "loss": 0.6324,
+      "step": 1
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 8.231021188121307,
+      "learning_rate": 7.407407407407407e-07,
+      "loss": 0.5088,
+      "step": 2
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 10.084697585340875,
+      "learning_rate": 1.111111111111111e-06,
+      "loss": 0.5952,
+      "step": 3
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 8.504686522060355,
+      "learning_rate": 1.4814814814814815e-06,
+      "loss": 0.5506,
+      "step": 4
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 7.78998673730679,
+      "learning_rate": 1.8518518518518519e-06,
+      "loss": 0.5389,
+      "step": 5
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 7.564344020278276,
+      "learning_rate": 2.222222222222222e-06,
+      "loss": 0.4419,
+      "step": 6
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 6.335657208121732,
+      "learning_rate": 2.5925925925925925e-06,
+      "loss": 0.4503,
+      "step": 7
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 5.736176328651262,
+      "learning_rate": 2.962962962962963e-06,
+      "loss": 0.4152,
+      "step": 8
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 6.646697141146249,
+      "learning_rate": 3.3333333333333333e-06,
+      "loss": 0.4766,
+      "step": 9
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 7.271376765925272,
+      "learning_rate": 3.7037037037037037e-06,
+      "loss": 0.3847,
+      "step": 10
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 11.816959033855433,
+      "learning_rate": 4.074074074074074e-06,
+      "loss": 0.3234,
+      "step": 11
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 17.376716600598993,
+      "learning_rate": 4.444444444444444e-06,
+      "loss": 0.2575,
+      "step": 12
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 13.166527782235265,
+      "learning_rate": 4.814814814814815e-06,
+      "loss": 0.3037,
+      "step": 13
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 8.131001484797347,
+      "learning_rate": 5.185185185185185e-06,
+      "loss": 0.1747,
+      "step": 14
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 9.174937982191583,
+      "learning_rate": 5.555555555555557e-06,
+      "loss": 0.2399,
+      "step": 15
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 8.592965454381307,
+      "learning_rate": 5.925925925925926e-06,
+      "loss": 0.1949,
+      "step": 16
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 6.872465363012901,
+      "learning_rate": 6.296296296296297e-06,
+      "loss": 0.1805,
+      "step": 17
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 5.792592320141116,
+      "learning_rate": 6.666666666666667e-06,
+      "loss": 0.1909,
+      "step": 18
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 2.288697729735019,
+      "learning_rate": 7.0370370370370375e-06,
+      "loss": 0.1502,
+      "step": 19
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 1.6947246687987911,
+      "learning_rate": 7.4074074074074075e-06,
+      "loss": 0.1174,
+      "step": 20
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 5.72465565098873,
+      "learning_rate": 7.77777777777778e-06,
+      "loss": 0.1739,
+      "step": 21
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 5.765453075575737,
+      "learning_rate": 8.148148148148148e-06,
+      "loss": 0.1894,
+      "step": 22
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 3.2240061796287334,
+      "learning_rate": 8.518518518518519e-06,
+      "loss": 0.1376,
+      "step": 23
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 5.983671039035109,
+      "learning_rate": 8.888888888888888e-06,
+      "loss": 0.1445,
+      "step": 24
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 6.056090509557002,
+      "learning_rate": 9.25925925925926e-06,
+      "loss": 0.1628,
+      "step": 25
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 3.667505927161921,
+      "learning_rate": 9.62962962962963e-06,
+      "loss": 0.1411,
+      "step": 26
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 9.96893801537625,
+      "learning_rate": 1e-05,
+      "loss": 0.1942,
+      "step": 27
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 3.081996759418418,
+      "learning_rate": 9.999965687879942e-06,
+      "loss": 0.1401,
+      "step": 28
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 3.992334648293423,
+      "learning_rate": 9.999862751990697e-06,
+      "loss": 0.1495,
+      "step": 29
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 3.446786827543344,
+      "learning_rate": 9.999691193745045e-06,
+      "loss": 0.1356,
+      "step": 30
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 2.452927122228086,
+      "learning_rate": 9.999451015497595e-06,
+      "loss": 0.1576,
+      "step": 31
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 4.8167704565956715,
+      "learning_rate": 9.999142220544759e-06,
+      "loss": 0.1237,
+      "step": 32
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 4.128302969077741,
+      "learning_rate": 9.9987648131247e-06,
+      "loss": 0.124,
+      "step": 33
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 3.7257757203533584,
+      "learning_rate": 9.998318798417276e-06,
+      "loss": 0.1622,
+      "step": 34
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 4.811332863461092,
+      "learning_rate": 9.997804182543973e-06,
+      "loss": 0.1373,
+      "step": 35
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 6.006945777469291,
+      "learning_rate": 9.997220972567815e-06,
+      "loss": 0.1617,
+      "step": 36
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 5.2290233962608275,
+      "learning_rate": 9.996569176493269e-06,
+      "loss": 0.1317,
+      "step": 37
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 4.887606275239368,
+      "learning_rate": 9.995848803266141e-06,
+      "loss": 0.1559,
+      "step": 38
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 2.649617040738647,
+      "learning_rate": 9.99505986277344e-06,
+      "loss": 0.1359,
+      "step": 39
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 3.6342992270813492,
+      "learning_rate": 9.994202365843256e-06,
+      "loss": 0.1444,
+      "step": 40
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 2.977502599851725,
+      "learning_rate": 9.993276324244606e-06,
+      "loss": 0.1367,
+      "step": 41
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 3.007957356864356,
+      "learning_rate": 9.992281750687265e-06,
+      "loss": 0.1297,
+      "step": 42
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 1.8406129718892825,
+      "learning_rate": 9.991218658821609e-06,
+      "loss": 0.129,
+      "step": 43
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 2.0044983411925275,
+      "learning_rate": 9.990087063238408e-06,
+      "loss": 0.107,
+      "step": 44
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 1.8795658720176402,
+      "learning_rate": 9.988886979468644e-06,
+      "loss": 0.1121,
+      "step": 45
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 2.356084791458523,
+      "learning_rate": 9.98761842398328e-06,
+      "loss": 0.0981,
+      "step": 46
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 4.534924570867148,
+      "learning_rate": 9.98628141419305e-06,
+      "loss": 0.1794,
+      "step": 47
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 2.3579321421491866,
+      "learning_rate": 9.984875968448212e-06,
+      "loss": 0.108,
+      "step": 48
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 2.2510325224717254,
+      "learning_rate": 9.983402106038292e-06,
+      "loss": 0.1117,
+      "step": 49
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 2.4788953336363653,
+      "learning_rate": 9.981859847191827e-06,
+      "loss": 0.1282,
+      "step": 50
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 1.8906655279037572,
+      "learning_rate": 9.980249213076085e-06,
+      "loss": 0.1208,
+      "step": 51
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 3.1073556178878188,
+      "learning_rate": 9.978570225796777e-06,
+      "loss": 0.1644,
+      "step": 52
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 1.9452742167713701,
+      "learning_rate": 9.97682290839775e-06,
+      "loss": 0.1285,
+      "step": 53
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 2.670121793189728,
+      "learning_rate": 9.975007284860664e-06,
+      "loss": 0.142,
+      "step": 54
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 1.8244790066486611,
+      "learning_rate": 9.97312338010468e-06,
+      "loss": 0.1182,
+      "step": 55
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 1.1114450183499889,
+      "learning_rate": 9.971171219986104e-06,
+      "loss": 0.1013,
+      "step": 56
+    },
+    {
+      "epoch": 0.07,
+      "grad_norm": 2.8735563274335463,
+      "learning_rate": 9.969150831298037e-06,
+      "loss": 0.1351,
+      "step": 57
+    },
+    {
+      "epoch": 0.07,
+      "grad_norm": 2.222351118905723,
+      "learning_rate": 9.967062241770008e-06,
+      "loss": 0.1122,
+      "step": 58
+    },
+    {
+      "epoch": 0.07,
+      "grad_norm": 1.7829363422208375,
+      "learning_rate": 9.964905480067585e-06,
+      "loss": 0.1299,
+      "step": 59
+    },
+    {
+      "epoch": 0.07,
+      "grad_norm": 2.187968659410326,
+      "learning_rate": 9.962680575792e-06,
+      "loss": 0.0957,
+      "step": 60
+    },
+    {
+      "epoch": 0.07,
+      "grad_norm": 2.846988563675764,
+      "learning_rate": 9.960387559479727e-06,
+      "loss": 0.1187,
+      "step": 61
+    },
+    {
+      "epoch": 0.07,
+      "grad_norm": 1.643896260993509,
+      "learning_rate": 9.958026462602062e-06,
+      "loss": 0.1086,
+      "step": 62
+    },
+    {
+      "epoch": 0.07,
+      "grad_norm": 2.3501454089832894,
+      "learning_rate": 9.955597317564705e-06,
+      "loss": 0.1181,
+      "step": 63
+    },
+    {
+      "epoch": 0.07,
+      "grad_norm": 2.3392801570363164,
+      "learning_rate": 9.953100157707299e-06,
+      "loss": 0.1283,
+      "step": 64
+    },
+    {
+      "epoch": 0.07,
+      "grad_norm": 2.215077177644609,
+      "learning_rate": 9.950535017302984e-06,
+      "loss": 0.1194,
+      "step": 65
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 2.298364445576207,
+      "learning_rate": 9.947901931557924e-06,
+      "loss": 0.1317,
+      "step": 66
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 2.8138148663788862,
+      "learning_rate": 9.945200936610821e-06,
+      "loss": 0.1241,
+      "step": 67
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 2.5107542165471934,
+      "learning_rate": 9.942432069532417e-06,
+      "loss": 0.1021,
+      "step": 68
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 2.5847605764691766,
+      "learning_rate": 9.939595368324996e-06,
+      "loss": 0.1432,
+      "step": 69
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 2.534356782579585,
+      "learning_rate": 9.936690871921849e-06,
+      "loss": 0.1407,
+      "step": 70
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 2.2605138628608623,
+      "learning_rate": 9.933718620186745e-06,
+      "loss": 0.1323,
+      "step": 71
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 4.239825098653746,
+      "learning_rate": 9.930678653913392e-06,
+      "loss": 0.1329,
+      "step": 72
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 3.4950058767667134,
+      "learning_rate": 9.927571014824862e-06,
+      "loss": 0.1259,
+      "step": 73
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 2.538742560460438,
+      "learning_rate": 9.924395745573029e-06,
+      "loss": 0.1276,
+      "step": 74
+    },
+    {
+      "epoch": 0.09,
+      "grad_norm": 2.8351129913937987,
+      "learning_rate": 9.921152889737985e-06,
+      "loss": 0.1333,
+      "step": 75
+    },
+    {
+      "epoch": 0.09,
+      "grad_norm": 3.6729191949488227,
+      "learning_rate": 9.91784249182743e-06,
+      "loss": 0.1209,
+      "step": 76
+    },
+    {
+      "epoch": 0.09,
+      "grad_norm": 3.5760153753569086,
+      "learning_rate": 9.914464597276072e-06,
+      "loss": 0.1443,
+      "step": 77
+    },
+    {
+      "epoch": 0.09,
+      "grad_norm": 2.057116589062098,
+      "learning_rate": 9.911019252445e-06,
+      "loss": 0.1082,
+      "step": 78
+    },
+    {
+      "epoch": 0.09,
+      "grad_norm": 2.6837432634831067,
+      "learning_rate": 9.907506504621052e-06,
+      "loss": 0.1133,
+      "step": 79
+    },
+    {
+      "epoch": 0.09,
+      "grad_norm": 2.118680599811244,
+      "learning_rate": 9.903926402016153e-06,
+      "loss": 0.1193,
+      "step": 80
+    },
+    {
+      "epoch": 0.09,
+      "grad_norm": 3.1410450154804663,
+      "learning_rate": 9.90027899376667e-06,
+      "loss": 0.1212,
+      "step": 81
+    },
+    {
+      "epoch": 0.09,
+      "grad_norm": 5.110344600127584,
+      "learning_rate": 9.896564329932727e-06,
+      "loss": 0.1318,
+      "step": 82
+    },
+    {
+      "epoch": 0.09,
+      "grad_norm": 2.33027322583564,
+      "learning_rate": 9.892782461497521e-06,
+      "loss": 0.1054,
+      "step": 83
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 1.9832010503747017,
+      "learning_rate": 9.88893344036662e-06,
+      "loss": 0.1113,
+      "step": 84
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 2.6705177259808988,
+      "learning_rate": 9.885017319367253e-06,
+      "loss": 0.1181,
+      "step": 85
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 1.6879356709331421,
+      "learning_rate": 9.88103415224759e-06,
+      "loss": 0.0843,
+      "step": 86
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 1.3641501934361187,
+      "learning_rate": 9.87698399367599e-06,
+      "loss": 0.0985,
+      "step": 87
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 2.0451464275509053,
+      "learning_rate": 9.872866899240265e-06,
+      "loss": 0.1057,
+      "step": 88
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 2.2974724008225484,
+      "learning_rate": 9.86868292544691e-06,
+      "loss": 0.1166,
+      "step": 89
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 2.1974043469355458,
+      "learning_rate": 9.864432129720332e-06,
+      "loss": 0.1007,
+      "step": 90
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 1.5880779234771585,
+      "learning_rate": 9.860114570402055e-06,
+      "loss": 0.1059,
+      "step": 91
+    },
+    {
+      "epoch": 0.11,
+      "grad_norm": 1.6210019246715581,
+      "learning_rate": 9.855730306749924e-06,
+      "loss": 0.0955,
+      "step": 92
+    },
+    {
+      "epoch": 0.11,
+      "grad_norm": 2.053943766699023,
+      "learning_rate": 9.85127939893729e-06,
+      "loss": 0.1223,
+      "step": 93
+    },
+    {
+      "epoch": 0.11,
+      "grad_norm": 1.5230542823178097,
+      "learning_rate": 9.846761908052188e-06,
+      "loss": 0.0969,
+      "step": 94
+    },
+    {
+      "epoch": 0.11,
+      "grad_norm": 1.9898055550048976,
+      "learning_rate": 9.842177896096495e-06,
+      "loss": 0.1359,
+      "step": 95
+    },
+    {
+      "epoch": 0.11,
+      "grad_norm": 1.5219298537997954,
+      "learning_rate": 9.837527425985074e-06,
+      "loss": 0.1049,
+      "step": 96
+    },
+    {
+      "epoch": 0.11,
+      "grad_norm": 1.5694384883915047,
+      "learning_rate": 9.832810561544925e-06,
+      "loss": 0.0885,
+      "step": 97
+    },
+    {
+      "epoch": 0.11,
+      "grad_norm": 1.8382074625592275,
+      "learning_rate": 9.828027367514296e-06,
+      "loss": 0.0789,
+      "step": 98
+    },
+    {
+      "epoch": 0.11,
+      "grad_norm": 3.094156783497333,
+      "learning_rate": 9.823177909541795e-06,
+      "loss": 0.125,
+      "step": 99
+    },
+    {
+      "epoch": 0.11,
+      "grad_norm": 2.1177801350294474,
+      "learning_rate": 9.818262254185495e-06,
+      "loss": 0.1286,
+      "step": 100
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 2.051759324600107,
+      "learning_rate": 9.813280468912024e-06,
+      "loss": 0.0867,
+      "step": 101
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 2.309052314502647,
+      "learning_rate": 9.808232622095621e-06,
+      "loss": 0.0836,
+      "step": 102
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 1.9691513403830945,
+      "learning_rate": 9.803118783017221e-06,
+      "loss": 0.1088,
+      "step": 103
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 1.8816504929896276,
+      "learning_rate": 9.797939021863487e-06,
+      "loss": 0.1263,
+      "step": 104
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 3.5814461055271196,
+      "learning_rate": 9.792693409725853e-06,
+      "loss": 0.1266,
+      "step": 105
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 2.804380821700463,
+      "learning_rate": 9.78738201859955e-06,
+      "loss": 0.1548,
+      "step": 106
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 2.0086077809415332,
+      "learning_rate": 9.782004921382612e-06,
+      "loss": 0.1312,
+      "step": 107
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 1.7232244244365362,
+      "learning_rate": 9.776562191874884e-06,
+      "loss": 0.1197,
+      "step": 108
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 1.3610013356293815,
+      "learning_rate": 9.771053904776998e-06,
+      "loss": 0.1139,
+      "step": 109
+    },
+    {
+      "epoch": 0.13,
+      "grad_norm": 3.0085151960644643,
+      "learning_rate": 9.765480135689357e-06,
+      "loss": 0.1049,
+      "step": 110
+    },
+    {
+      "epoch": 0.13,
+      "grad_norm": 2.94972805872663,
+      "learning_rate": 9.759840961111098e-06,
+      "loss": 0.0992,
+      "step": 111
+    },
+    {
+      "epoch": 0.13,
+      "grad_norm": 1.2797796597040898,
+      "learning_rate": 9.754136458439033e-06,
+      "loss": 0.0779,
+      "step": 112
+    },
+    {
+      "epoch": 0.13,
+      "grad_norm": 2.080098377998089,
+      "learning_rate": 9.748366705966595e-06,
+      "loss": 0.1192,
+      "step": 113
+    },
+    {
+      "epoch": 0.13,
+      "grad_norm": 3.026465124623598,
+      "learning_rate": 9.742531782882758e-06,
+      "loss": 0.1248,
+      "step": 114
+    },
+    {
+      "epoch": 0.13,
+      "grad_norm": 3.419083049509824,
+      "learning_rate": 9.736631769270958e-06,
+      "loss": 0.1284,
+      "step": 115
+    },
+    {
+      "epoch": 0.13,
+      "grad_norm": 1.9998224613269435,
+      "learning_rate": 9.730666746107982e-06,
+      "loss": 0.1106,
+      "step": 116
+    },
+    {
+      "epoch": 0.13,
+      "grad_norm": 1.7572924951035829,
+      "learning_rate": 9.724636795262868e-06,
+      "loss": 0.0874,
+      "step": 117
+    },
+    {
+      "epoch": 0.13,
+      "grad_norm": 2.448977492181066,
+      "learning_rate": 9.718541999495773e-06,
+      "loss": 0.0823,
+      "step": 118
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 4.110748776288409,
+      "learning_rate": 9.712382442456845e-06,
+      "loss": 0.1604,
+      "step": 119
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 2.09981079064145,
+      "learning_rate": 9.706158208685066e-06,
+      "loss": 0.0911,
+      "step": 120
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 1.9370008400745906,
+      "learning_rate": 9.6998693836071e-06,
+      "loss": 0.1122,
+      "step": 121
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 1.9316495646897083,
+      "learning_rate": 9.693516053536118e-06,
+      "loss": 0.0807,
+      "step": 122
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 3.16334665341287,
+      "learning_rate": 9.687098305670606e-06,
+      "loss": 0.1276,
+      "step": 123
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 2.341234757750837,
+      "learning_rate": 9.680616228093178e-06,
+      "loss": 0.1187,
+      "step": 124
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 2.223298454459867,
+      "learning_rate": 9.674069909769365e-06,
+      "loss": 0.1045,
+      "step": 125
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 3.4945449518796585,
+      "learning_rate": 9.66745944054639e-06,
+      "loss": 0.12,
+      "step": 126
+    },
+    {
+      "epoch": 0.15,
+      "grad_norm": 4.143984048508036,
+      "learning_rate": 9.66078491115194e-06,
+      "loss": 0.1013,
+      "step": 127
+    },
+    {
+      "epoch": 0.15,
+      "grad_norm": 2.7705806851123493,
+      "learning_rate": 9.654046413192916e-06,
+      "loss": 0.1123,
+      "step": 128
+    },
+    {
+      "epoch": 0.15,
+      "grad_norm": 1.2313407498327578,
+      "learning_rate": 9.647244039154178e-06,
+      "loss": 0.0846,
+      "step": 129
+    },
+    {
+      "epoch": 0.15,
+      "grad_norm": 1.7722835766792302,
+      "learning_rate": 9.640377882397276e-06,
+      "loss": 0.0965,
+      "step": 130
+    },
+    {
+      "epoch": 0.15,
+      "grad_norm": 3.4978175367401065,
+      "learning_rate": 9.633448037159167e-06,
+      "loss": 0.1218,
+      "step": 131
+    },
+    {
+      "epoch": 0.15,
+      "grad_norm": 3.9772864819418463,
+      "learning_rate": 9.626454598550927e-06,
+      "loss": 0.0919,
+      "step": 132
+    },
+    {
+      "epoch": 0.15,
+      "grad_norm": 1.9729154551618289,
+      "learning_rate": 9.619397662556434e-06,
+      "loss": 0.1104,
+      "step": 133
+    },
+    {
+      "epoch": 0.15,
+      "grad_norm": 1.9749670072044205,
+      "learning_rate": 9.612277326031065e-06,
+      "loss": 0.0879,
+      "step": 134
+    },
+    {
+      "epoch": 0.15,
+      "grad_norm": 2.46650550465165,
+      "learning_rate": 9.605093686700356e-06,
+      "loss": 0.1151,
+      "step": 135
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 2.1700672021295397,
+      "learning_rate": 9.597846843158663e-06,
+      "loss": 0.1192,
+      "step": 136
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 1.5484117589367528,
+      "learning_rate": 9.590536894867814e-06,
+      "loss": 0.1093,
+      "step": 137
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 3.159315025435744,
+      "learning_rate": 9.58316394215574e-06,
+      "loss": 0.1246,
+      "step": 138
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 2.3062290760264044,
+      "learning_rate": 9.575728086215093e-06,
+      "loss": 0.1044,
+      "step": 139
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 2.1924424597130647,
+      "learning_rate": 9.568229429101867e-06,
+      "loss": 0.1018,
+      "step": 140
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 1.4292889067167893,
+      "learning_rate": 9.560668073733993e-06,
+      "loss": 0.1102,
+      "step": 141
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 1.8452890182403634,
+      "learning_rate": 9.553044123889922e-06,
+      "loss": 0.0851,
+      "step": 142
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 1.686312012130693,
+      "learning_rate": 9.54535768420721e-06,
+      "loss": 0.0877,
+      "step": 143
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 1.9544040435460412,
+      "learning_rate": 9.537608860181069e-06,
+      "loss": 0.1401,
+      "step": 144
+    },
+    {
+      "epoch": 0.17,
+      "grad_norm": 2.5638709576265777,
+      "learning_rate": 9.529797758162935e-06,
+      "loss": 0.1092,
+      "step": 145
+    },
+    {
+      "epoch": 0.17,
+      "grad_norm": 3.282994763913454,
+      "learning_rate": 9.521924485358993e-06,
+      "loss": 0.1153,
+      "step": 146
+    },
+    {
+      "epoch": 0.17,
+      "grad_norm": 2.5304046735735803,
+      "learning_rate": 9.513989149828718e-06,
+      "loss": 0.1249,
+      "step": 147
+    },
+    {
+      "epoch": 0.17,
+      "grad_norm": 2.084105660790946,
+      "learning_rate": 9.505991860483384e-06,
+      "loss": 0.1127,
+      "step": 148
+    },
+    {
+      "epoch": 0.17,
+      "grad_norm": 2.0807266175742334,
+      "learning_rate": 9.497932727084571e-06,
+      "loss": 0.0923,
+      "step": 149
+    },
+    {
+      "epoch": 0.17,
+      "grad_norm": 2.535491149619429,
+      "learning_rate": 9.489811860242658e-06,
+      "loss": 0.1171,
+      "step": 150
+    },
+    {
+      "epoch": 0.17,
+      "grad_norm": 2.8577061917206694,
+      "learning_rate": 9.481629371415315e-06,
+      "loss": 0.1027,
+      "step": 151
+    },
+    {
+      "epoch": 0.17,
+      "grad_norm": 2.366848987135643,
+      "learning_rate": 9.47338537290595e-06,
+      "loss": 0.1461,
+      "step": 152
+    },
+    {
+      "epoch": 0.17,
+      "grad_norm": 1.370601085821976,
+      "learning_rate": 9.465079977862193e-06,
+      "loss": 0.1176,
+      "step": 153
+    },
+    {
+      "epoch": 0.18,
+      "grad_norm": 2.418308466667954,
+      "learning_rate": 9.45671330027433e-06,
+      "loss": 0.14,
+      "step": 154
+    },
+    {
+      "epoch": 0.18,
+      "grad_norm": 2.3789531359503155,
+      "learning_rate": 9.448285454973739e-06,
+      "loss": 0.1272,
+      "step": 155
+    },
+    {
+      "epoch": 0.18,
+      "grad_norm": 1.837535532086006,
+      "learning_rate": 9.439796557631311e-06,
+      "loss": 0.1018,
+      "step": 156
+    },
+    {
+      "epoch": 0.18,
+      "grad_norm": 2.26111781111186,
+      "learning_rate": 9.431246724755879e-06,
+      "loss": 0.1089,
+      "step": 157
+    },
+    {
+      "epoch": 0.18,
+      "grad_norm": 2.149991918092674,
+      "learning_rate": 9.422636073692595e-06,
+      "loss": 0.1224,
+      "step": 158
+    },
+    {
+      "epoch": 0.18,
+      "grad_norm": 1.7257130483628491,
+      "learning_rate": 9.413964722621339e-06,
+      "loss": 0.0829,
+      "step": 159
+    },
+    {
+      "epoch": 0.18,
+      "grad_norm": 1.8121675919179852,
+      "learning_rate": 9.405232790555083e-06,
+      "loss": 0.1082,
+      "step": 160
+    },
+    {
+      "epoch": 0.18,
+      "grad_norm": 1.1269467977377563,
+      "learning_rate": 9.396440397338273e-06,
+      "loss": 0.0735,
+      "step": 161
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 2.66841905851172,
+      "learning_rate": 9.387587663645163e-06,
+      "loss": 0.1171,
+      "step": 162
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 1.9979957509268915,
+      "learning_rate": 9.378674710978185e-06,
+      "loss": 0.0867,
+      "step": 163
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 1.3467231541181564,
+      "learning_rate": 9.369701661666255e-06,
+      "loss": 0.084,
+      "step": 164
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 1.5661741867693126,
+      "learning_rate": 9.36066863886311e-06,
+      "loss": 0.0879,
+      "step": 165
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 1.4781065347920526,
+      "learning_rate": 9.35157576654562e-06,
+      "loss": 0.081,
+      "step": 166
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 1.747486649844575,
+      "learning_rate": 9.342423169512072e-06,
+      "loss": 0.0998,
+      "step": 167
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 2.183425315531194,
+      "learning_rate": 9.333210973380469e-06,
+      "loss": 0.113,
+      "step": 168
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 2.6187200801222126,
+      "learning_rate": 9.323939304586806e-06,
+      "loss": 0.1211,
+      "step": 169
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 2.298579486613351,
+      "learning_rate": 9.314608290383324e-06,
+      "loss": 0.098,
+      "step": 170
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 2.3760059355851286,
+      "learning_rate": 9.305218058836778e-06,
+      "loss": 0.1072,
+      "step": 171
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 2.7212461615360892,
+      "learning_rate": 9.295768738826668e-06,
+      "loss": 0.1128,
+      "step": 172
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 1.7379020422811915,
+      "learning_rate": 9.286260460043475e-06,
+      "loss": 0.125,
+      "step": 173
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 3.1792312593675724,
+      "learning_rate": 9.27669335298688e-06,
+      "loss": 0.124,
+      "step": 174
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 2.0277111530618916,
+      "learning_rate": 9.267067548963975e-06,
+      "loss": 0.0942,
+      "step": 175
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 1.4469039936621195,
+      "learning_rate": 9.257383180087454e-06,
+      "loss": 0.0909,
+      "step": 176
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 2.051541742743329,
+      "learning_rate": 9.24764037927381e-06,
+      "loss": 0.1199,
+      "step": 177
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 1.4918383860547757,
+      "learning_rate": 9.237839280241504e-06,
+      "loss": 0.0948,
+      "step": 178
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 1.64559721928272,
+      "learning_rate": 9.22798001750913e-06,
+      "loss": 0.0926,
+      "step": 179
+    },
+    {
+      "epoch": 0.21,
+      "grad_norm": 2.7118844975893306,
+      "learning_rate": 9.218062726393572e-06,
+      "loss": 0.1006,
+      "step": 180
+    },
+    {
+      "epoch": 0.21,
+      "grad_norm": 1.9102131867075969,
+      "learning_rate": 9.208087543008142e-06,
+      "loss": 0.0971,
+      "step": 181
+    },
+    {
+      "epoch": 0.21,
+      "grad_norm": 1.1482279190562423,
+      "learning_rate": 9.198054604260714e-06,
+      "loss": 0.0856,
+      "step": 182
+    },
+    {
+      "epoch": 0.21,
+      "grad_norm": 0.8541863535744467,
+      "learning_rate": 9.187964047851851e-06,
+      "loss": 0.0744,
+      "step": 183
+    },
+    {
+      "epoch": 0.21,
+      "grad_norm": 1.6754271851342437,
+      "learning_rate": 9.177816012272904e-06,
+      "loss": 0.1079,
+      "step": 184
+    },
+    {
+      "epoch": 0.21,
+      "grad_norm": 7.473478229947605,
+      "learning_rate": 9.16761063680412e-06,
+      "loss": 0.0957,
+      "step": 185
+    },
+    {
+      "epoch": 0.21,
+      "grad_norm": 3.0632873310708706,
+      "learning_rate": 9.157348061512728e-06,
+      "loss": 0.1259,
+      "step": 186
+    },
+    {
+      "epoch": 0.21,
+      "grad_norm": 2.173245370297517,
+      "learning_rate": 9.14702842725101e-06,
+      "loss": 0.1064,
+      "step": 187
+    },
+    {
+      "epoch": 0.21,
+      "grad_norm": 2.5577392058893906,
+      "learning_rate": 9.136651875654381e-06,
+      "loss": 0.1018,
+      "step": 188
+    },
+    {
+      "epoch": 0.22,
+      "grad_norm": 2.4914753007974073,
+      "learning_rate": 9.126218549139434e-06,
+      "loss": 0.0766,
+      "step": 189
+    },
+    {
+      "epoch": 0.22,
+      "grad_norm": 2.607548007532628,
+      "learning_rate": 9.115728590901988e-06,
+      "loss": 0.1145,
+      "step": 190
+    },
+    {
+      "epoch": 0.22,
+      "grad_norm": 2.3652955179863704,
+      "learning_rate": 9.10518214491513e-06,
+      "loss": 0.1075,
+      "step": 191
+    },
+    {
+      "epoch": 0.22,
+      "grad_norm": 1.944852075113964,
+      "learning_rate": 9.094579355927224e-06,
+      "loss": 0.1237,
+      "step": 192
+    },
+    {
+      "epoch": 0.22,
+      "grad_norm": 2.531422123532763,
+      "learning_rate": 9.083920369459941e-06,
+      "loss": 0.0996,
+      "step": 193
+    },
+    {
+      "epoch": 0.22,
+      "grad_norm": 1.6840091229079706,
+      "learning_rate": 9.073205331806248e-06,
+      "loss": 0.0978,
+      "step": 194
+    },
+    {
+      "epoch": 0.22,
+      "grad_norm": 1.1180165313908608,
+      "learning_rate": 9.062434390028407e-06,
+      "loss": 0.1071,
+      "step": 195
+    },
+    {
+      "epoch": 0.22,
+      "grad_norm": 1.3064591316530667,
+      "learning_rate": 9.05160769195596e-06,
+      "loss": 0.0989,
+      "step": 196
+    },
+    {
+      "epoch": 0.23,
+      "grad_norm": 2.467504354977469,
+      "learning_rate": 9.040725386183691e-06,
+      "loss": 0.1042,
+      "step": 197
+    },
+    {
+      "epoch": 0.23,
+      "grad_norm": 2.050946499964558,
+      "learning_rate": 9.029787622069594e-06,
+      "loss": 0.0975,
+      "step": 198
+    },
+    {
+      "epoch": 0.23,
+      "grad_norm": 1.4499865119155375,
+      "learning_rate": 9.018794549732819e-06,
+      "loss": 0.085,
+      "step": 199
+    },
+    {
+      "epoch": 0.23,
+      "grad_norm": 1.3533643803726048,
+      "learning_rate": 9.007746320051612e-06,
+      "loss": 0.1035,
+      "step": 200
+    },
+    {
+      "epoch": 0.23,
+      "grad_norm": 1.6026335880678133,
+      "learning_rate": 8.996643084661245e-06,
+      "loss": 0.1027,
+      "step": 201
+    },
+    {
+      "epoch": 0.23,
+      "grad_norm": 1.9397686652721016,
+      "learning_rate": 8.985484995951941e-06,
+      "loss": 0.1186,
+      "step": 202
+    },
+    {
+      "epoch": 0.23,
+      "grad_norm": 1.985748151579965,
+      "learning_rate": 8.974272207066767e-06,
+      "loss": 0.1106,
+      "step": 203
+    },
+    {
+      "epoch": 0.23,
+      "grad_norm": 3.0494865642540487,
+      "learning_rate": 8.96300487189955e-06,
+      "loss": 0.1064,
+      "step": 204
+    },
+    {
+      "epoch": 0.23,
+      "grad_norm": 1.5961078519634195,
+      "learning_rate": 8.951683145092749e-06,
+      "loss": 0.107,
+      "step": 205
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 1.1110307819462288,
+      "learning_rate": 8.940307182035348e-06,
+      "loss": 0.0572,
+      "step": 206
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 1.5666032654284987,
+      "learning_rate": 8.928877138860708e-06,
+      "loss": 0.0855,
+      "step": 207
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 2.5628141099754145,
+      "learning_rate": 8.917393172444435e-06,
+      "loss": 0.1166,
+      "step": 208
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 2.1688262602344186,
+      "learning_rate": 8.905855440402225e-06,
+      "loss": 0.1282,
+      "step": 209
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 1.9861109578632914,
+      "learning_rate": 8.894264101087692e-06,
+      "loss": 0.114,
+      "step": 210
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 2.501921488076984,
+      "learning_rate": 8.882619313590212e-06,
+      "loss": 0.1097,
+      "step": 211
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 2.268674705211975,
+      "learning_rate": 8.870921237732722e-06,
+      "loss": 0.1071,
+      "step": 212
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 2.067986488761189,
+      "learning_rate": 8.859170034069533e-06,
+      "loss": 0.078,
+      "step": 213
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 1.848889166285347,
+      "learning_rate": 8.847365863884131e-06,
+      "loss": 0.0939,
+      "step": 214
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 1.633820214871211,
+      "learning_rate": 8.835508889186957e-06,
+      "loss": 0.0668,
+      "step": 215
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 1.762882965258614,
+      "learning_rate": 8.823599272713188e-06,
+      "loss": 0.1218,
+      "step": 216
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 1.320523619911371,
+      "learning_rate": 8.8116371779205e-06,
+      "loss": 0.0862,
+      "step": 217
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 2.2237720260978944,
+      "learning_rate": 8.799622768986826e-06,
+      "loss": 0.0818,
+      "step": 218
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 2.4364710210798948,
+      "learning_rate": 8.787556210808101e-06,
+      "loss": 0.1199,
+      "step": 219
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 1.6849135689019348,
+      "learning_rate": 8.775437668996004e-06,
+      "loss": 0.1075,
+      "step": 220
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 2.220372344984879,
+      "learning_rate": 8.76326730987568e-06,
+      "loss": 0.0939,
+      "step": 221
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 1.5555946268423595,
+      "learning_rate": 8.751045300483457e-06,
+      "loss": 0.0936,
+      "step": 222
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 2.1402335410022744,
+      "learning_rate": 8.738771808564555e-06,
+      "loss": 0.0934,
+      "step": 223
+    },
+    {
+      "epoch": 0.26,
+      "grad_norm": 1.0488112883529606,
+      "learning_rate": 8.726447002570791e-06,
+      "loss": 0.0984,
+      "step": 224
+    },
+    {
+      "epoch": 0.26,
+      "grad_norm": 1.6002022312116384,
+      "learning_rate": 8.714071051658247e-06,
+      "loss": 0.1182,
+      "step": 225
+    },
+    {
+      "epoch": 0.26,
+      "grad_norm": 1.1565321002590416,
+      "learning_rate": 8.701644125684972e-06,
+      "loss": 0.0936,
+      "step": 226
+    },
+    {
+      "epoch": 0.26,
+      "grad_norm": 2.185324721324382,
+      "learning_rate": 8.689166395208638e-06,
+      "loss": 0.083,
+      "step": 227
+    },
+    {
+      "epoch": 0.26,
+      "grad_norm": 2.030700715820133,
+      "learning_rate": 8.676638031484196e-06,
+      "loss": 0.117,
+      "step": 228
+    },
+    {
+      "epoch": 0.26,
+      "grad_norm": 1.984880186535839,
+      "learning_rate": 8.664059206461537e-06,
+      "loss": 0.1018,
+      "step": 229
+    },
+    {
+      "epoch": 0.26,
+      "grad_norm": 1.5970035045867854,
+      "learning_rate": 8.65143009278312e-06,
+      "loss": 0.0949,
+      "step": 230
+    },
+    {
+      "epoch": 0.26,
+      "grad_norm": 1.1816220314739803,
+      "learning_rate": 8.638750863781614e-06,
+      "loss": 0.0813,
+      "step": 231
+    },
+    {
+      "epoch": 0.27,
+      "grad_norm": 1.1265530666409853,
+      "learning_rate": 8.626021693477506e-06,
+      "loss": 0.0807,
+      "step": 232
+    },
+    {
+      "epoch": 0.27,
+      "grad_norm": 1.4798060764319763,
+      "learning_rate": 8.613242756576729e-06,
+      "loss": 0.1005,
+      "step": 233
+    },
+    {
+      "epoch": 0.27,
+      "grad_norm": 2.2654194933943224,
+      "learning_rate": 8.600414228468245e-06,
+      "loss": 0.1188,
+      "step": 234
+    },
+    {
+      "epoch": 0.27,
+      "grad_norm": 2.069029364915602,
+      "learning_rate": 8.587536285221656e-06,
+      "loss": 0.1174,
+      "step": 235
+    },
+    {
+      "epoch": 0.27,
+      "grad_norm": 1.575346902419333,
+      "learning_rate": 8.574609103584776e-06,
+      "loss": 0.1085,
+      "step": 236
+    },
+    {
+      "epoch": 0.27,
+      "grad_norm": 1.4098282704868057,
+      "learning_rate": 8.561632860981205e-06,
+      "loss": 0.0866,
+      "step": 237
+    },
+    {
+      "epoch": 0.27,
+      "grad_norm": 1.322192828067451,
+      "learning_rate": 8.548607735507903e-06,
+      "loss": 0.0989,
+      "step": 238
+    },
+    {
+      "epoch": 0.27,
+      "grad_norm": 2.503973593981987,
+      "learning_rate": 8.535533905932739e-06,
+      "loss": 0.0959,
+      "step": 239
+    },
+    {
+      "epoch": 0.27,
+      "grad_norm": 1.9979798111827445,
+      "learning_rate": 8.522411551692034e-06,
+      "loss": 0.1173,
+      "step": 240
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 2.331120139565699,
+      "learning_rate": 8.509240852888106e-06,
+      "loss": 0.1207,
+      "step": 241
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 2.6853298265250545,
+      "learning_rate": 8.496021990286799e-06,
+      "loss": 0.1062,
+      "step": 242
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 1.9768348877886996,
+      "learning_rate": 8.482755145314987e-06,
+      "loss": 0.1072,
+      "step": 243
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 2.6831710659010493,
+      "learning_rate": 8.469440500058104e-06,
+      "loss": 0.1415,
+      "step": 244
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 1.3075146767802872,
+      "learning_rate": 8.456078237257632e-06,
+      "loss": 0.0927,
+      "step": 245
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 1.6603378900652728,
+      "learning_rate": 8.442668540308597e-06,
+      "loss": 0.1199,
+      "step": 246
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 1.732311993009954,
+      "learning_rate": 8.429211593257054e-06,
+      "loss": 0.1162,
+      "step": 247
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 1.0678226132557302,
+      "learning_rate": 8.415707580797552e-06,
+      "loss": 0.0811,
+      "step": 248
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 1.637868487774632,
+      "learning_rate": 8.402156688270613e-06,
+      "loss": 0.1134,
+      "step": 249
+    },
+    {
+      "epoch": 0.29,
+      "grad_norm": 2.133244302298436,
+      "learning_rate": 8.388559101660177e-06,
+      "loss": 0.0926,
+      "step": 250
+    },
+    {
+      "epoch": 0.29,
+      "grad_norm": 2.0183612078901554,
+      "learning_rate": 8.374915007591053e-06,
+      "loss": 0.1111,
+      "step": 251
+    },
+    {
+      "epoch": 0.29,
+      "grad_norm": 1.9309545129494814,
+      "learning_rate": 8.361224593326359e-06,
+      "loss": 0.0935,
+      "step": 252
+    },
+    {
+      "epoch": 0.29,
+      "grad_norm": 1.9872264464717466,
+      "learning_rate": 8.347488046764948e-06,
+      "loss": 0.0943,
+      "step": 253
+    },
+    {
+      "epoch": 0.29,
+      "grad_norm": 3.421297893280941,
+      "learning_rate": 8.33370555643884e-06,
+      "loss": 0.1065,
+      "step": 254
+    },
+    {
+      "epoch": 0.29,
+      "grad_norm": 2.897377603677577,
+      "learning_rate": 8.319877311510614e-06,
+      "loss": 0.1002,
+      "step": 255
+    },
+    {
+      "epoch": 0.29,
+      "grad_norm": 2.3046015963660937,
+      "learning_rate": 8.30600350177083e-06,
+      "loss": 0.0855,
+      "step": 256
+    },
+    {
+      "epoch": 0.29,
+      "grad_norm": 2.124569836934495,
+      "learning_rate": 8.29208431763542e-06,
+      "loss": 0.1393,
+      "step": 257
+    },
+    {
+      "epoch": 0.29,
+      "grad_norm": 1.1578644869843313,
+      "learning_rate": 8.278119950143074e-06,
+      "loss": 0.0727,
+      "step": 258
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 1.854683458939737,
+      "learning_rate": 8.264110590952609e-06,
+      "loss": 0.0943,
+      "step": 259
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 1.4832089308281162,
+      "learning_rate": 8.250056432340352e-06,
+      "loss": 0.0874,
+      "step": 260
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 1.258489452559218,
+      "learning_rate": 8.235957667197494e-06,
+      "loss": 0.0813,
+      "step": 261
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 1.8439098458745395,
+      "learning_rate": 8.221814489027447e-06,
+      "loss": 0.1109,
+      "step": 262
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 2.044799826590439,
+      "learning_rate": 8.207627091943178e-06,
+      "loss": 0.1031,
+      "step": 263
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 1.0827682370362863,
+      "learning_rate": 8.193395670664555e-06,
+      "loss": 0.1009,
+      "step": 264
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 2.323956787231295,
+      "learning_rate": 8.179120420515675e-06,
+      "loss": 0.124,
+      "step": 265
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 10.023030336667428,
+      "learning_rate": 8.164801537422179e-06,
+      "loss": 0.0942,
+      "step": 266
+    },
+    {
+      "epoch": 0.31,
+      "grad_norm": 1.7106775924129292,
+      "learning_rate": 8.150439217908557e-06,
+      "loss": 0.1294,
+      "step": 267
+    },
+    {
+      "epoch": 0.31,
+      "grad_norm": 2.496815910955381,
+      "learning_rate": 8.136033659095462e-06,
+      "loss": 0.1046,
+      "step": 268
+    },
+    {
+      "epoch": 0.31,
+      "grad_norm": 1.646786991911152,
+      "learning_rate": 8.121585058697e-06,
+      "loss": 0.0861,
+      "step": 269
+    },
+    {
+      "epoch": 0.31,
+      "grad_norm": 1.8266200161202253,
+      "learning_rate": 8.107093615018017e-06,
+      "loss": 0.0949,
+      "step": 270
+    },
+    {
+      "epoch": 0.31,
+      "grad_norm": 2.5916169021461823,
+      "learning_rate": 8.092559526951374e-06,
+      "loss": 0.1169,
+      "step": 271
+    },
+    {
+      "epoch": 0.31,
+      "grad_norm": 1.3232237535990277,
+      "learning_rate": 8.077982993975222e-06,
+      "loss": 0.0972,
+      "step": 272
+    },
+    {
+      "epoch": 0.31,
+      "grad_norm": 1.5105056997202508,
+      "learning_rate": 8.063364216150258e-06,
+      "loss": 0.0838,
+      "step": 273
+    },
+    {
+      "epoch": 0.31,
+      "grad_norm": 1.422000688554186,
+      "learning_rate": 8.04870339411699e-06,
+      "loss": 0.0906,
+      "step": 274
+    },
+    {
+      "epoch": 0.31,
+      "grad_norm": 1.6597229035037346,
+      "learning_rate": 8.034000729092967e-06,
+      "loss": 0.1005,
+      "step": 275
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 2.0117580401941204,
+      "learning_rate": 8.019256422870036e-06,
+      "loss": 0.1114,
+      "step": 276
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 1.1092429217293147,
+      "learning_rate": 8.00447067781156e-06,
+      "loss": 0.0879,
+      "step": 277
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 0.7979548894678264,
+      "learning_rate": 7.98964369684964e-06,
+      "loss": 0.0802,
+      "step": 278
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 4.002883666778827,
+      "learning_rate": 7.974775683482337e-06,
+      "loss": 0.1201,
+      "step": 279
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 2.9158794261385665,
+      "learning_rate": 7.959866841770875e-06,
+      "loss": 0.0928,
+      "step": 280
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 1.8564270859008971,
+      "learning_rate": 7.94491737633684e-06,
+      "loss": 0.1002,
+      "step": 281
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 0.9683845320254458,
+      "learning_rate": 7.929927492359377e-06,
+      "loss": 0.076,
+      "step": 282
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 1.4081959120743777,
+      "learning_rate": 7.914897395572362e-06,
+      "loss": 0.1062,
+      "step": 283
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 2.5215551080759475,
+      "learning_rate": 7.899827292261589e-06,
+      "loss": 0.1165,
+      "step": 284
+    },
+    {
+      "epoch": 0.33,
+      "grad_norm": 1.2748703118190907,
+      "learning_rate": 7.884717389261935e-06,
+      "loss": 0.0815,
+      "step": 285
+    },
+    {
+      "epoch": 0.33,
+      "grad_norm": 1.7476333672006885,
+      "learning_rate": 7.869567893954523e-06,
+      "loss": 0.1036,
+      "step": 286
+    },
+    {
+      "epoch": 0.33,
+      "grad_norm": 2.0867624034561008,
+      "learning_rate": 7.854379014263877e-06,
+      "loss": 0.0975,
+      "step": 287
+    },
+    {
+      "epoch": 0.33,
+      "grad_norm": 1.505523226821333,
+      "learning_rate": 7.839150958655058e-06,
+      "loss": 0.0909,
+      "step": 288
+    },
+    {
+      "epoch": 0.33,
+      "grad_norm": 1.1729694098449017,
+      "learning_rate": 7.823883936130817e-06,
+      "loss": 0.0973,
+      "step": 289
+    },
+    {
+      "epoch": 0.33,
+      "grad_norm": 2.8384149448507516,
+      "learning_rate": 7.808578156228718e-06,
+      "loss": 0.0897,
+      "step": 290
+    },
+    {
+      "epoch": 0.33,
+      "grad_norm": 1.62588050433828,
+      "learning_rate": 7.793233829018263e-06,
+      "loss": 0.0743,
+      "step": 291
+    },
+    {
+      "epoch": 0.33,
+      "grad_norm": 1.3669908142605618,
+      "learning_rate": 7.777851165098012e-06,
+      "loss": 0.0905,
+      "step": 292
+    },
+    {
+      "epoch": 0.33,
+      "grad_norm": 2.808747017133476,
+      "learning_rate": 7.762430375592689e-06,
+      "loss": 0.1093,
+      "step": 293
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 1.6645286286403973,
+      "learning_rate": 7.746971672150286e-06,
+      "loss": 0.0904,
+      "step": 294
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 1.5994325863369407,
+      "learning_rate": 7.731475266939159e-06,
+      "loss": 0.1028,
+      "step": 295
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 1.5901303888596037,
+      "learning_rate": 7.715941372645115e-06,
+      "loss": 0.0956,
+      "step": 296
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 2.328737208260307,
+      "learning_rate": 7.70037020246849e-06,
+      "loss": 0.1105,
+      "step": 297
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 1.9563743002247425,
+      "learning_rate": 7.68476197012123e-06,
+      "loss": 0.1004,
+      "step": 298
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 2.0781455779725273,
+      "learning_rate": 7.669116889823955e-06,
+      "loss": 0.0996,
+      "step": 299
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 0.9436403770848232,
+      "learning_rate": 7.653435176303007e-06,
+      "loss": 0.0761,
+      "step": 300
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 1.6406510415551603,
+      "learning_rate": 7.637717044787527e-06,
+      "loss": 0.1147,
+      "step": 301
+    },
+    {
+      "epoch": 0.35,
+      "grad_norm": 1.7565088827574487,
+      "learning_rate": 7.621962711006479e-06,
+      "loss": 0.1064,
+      "step": 302
+    },
+    {
+      "epoch": 0.35,
+      "grad_norm": 1.744343584962407,
+      "learning_rate": 7.6061723911857e-06,
+      "loss": 0.1237,
+      "step": 303
+    },
+    {
+      "epoch": 0.35,
+      "grad_norm": 1.8449164911800016,
+      "learning_rate": 7.59034630204493e-06,
+      "loss": 0.0866,
+      "step": 304
+    },
+    {
+      "epoch": 0.35,
+      "grad_norm": 1.950623541879017,
+      "learning_rate": 7.574484660794836e-06,
+      "loss": 0.0933,
+      "step": 305
+    },
+    {
+      "epoch": 0.35,
+      "grad_norm": 2.1686948556536465,
+      "learning_rate": 7.558587685134035e-06,
+      "loss": 0.1025,
+      "step": 306
+    },
+    {
+      "epoch": 0.35,
+      "grad_norm": 1.7581227603244038,
+      "learning_rate": 7.542655593246103e-06,
+      "loss": 0.0875,
+      "step": 307
+    },
+    {
+      "epoch": 0.35,
+      "grad_norm": 2.067970224647301,
+      "learning_rate": 7.526688603796578e-06,
+      "loss": 0.0984,
+      "step": 308
+    },
+    {
+      "epoch": 0.35,
+      "grad_norm": 1.589283294901552,
+      "learning_rate": 7.510686935929963e-06,
+      "loss": 0.086,
+      "step": 309
+    },
+    {
+      "epoch": 0.35,
+      "grad_norm": 2.0302397726328767,
+      "learning_rate": 7.494650809266719e-06,
+      "loss": 0.099,
+      "step": 310
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 1.5989723290365598,
+      "learning_rate": 7.478580443900247e-06,
+      "loss": 0.0939,
+      "step": 311
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 1.7617760156145226,
+      "learning_rate": 7.4624760603938704e-06,
+      "loss": 0.1073,
+      "step": 312
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 2.399035341306831,
+      "learning_rate": 7.446337879777803e-06,
+      "loss": 0.0875,
+      "step": 313
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 1.7044340741166755,
+      "learning_rate": 7.430166123546122e-06,
+      "loss": 0.1059,
+      "step": 314
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 2.0742510793830444,
+      "learning_rate": 7.413961013653725e-06,
+      "loss": 0.121,
+      "step": 315
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 1.3710310648899238,
+      "learning_rate": 7.397722772513282e-06,
+      "loss": 0.099,
+      "step": 316
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 1.4093925149964166,
+      "learning_rate": 7.381451622992184e-06,
+      "loss": 0.0809,
+      "step": 317
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 1.0871938974349964,
+      "learning_rate": 7.3651477884094855e-06,
+      "loss": 0.0873,
+      "step": 318
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 2.6023982808593678,
+      "learning_rate": 7.34881149253284e-06,
+      "loss": 0.0827,
+      "step": 319
+    },
+    {
+      "epoch": 0.37,
+      "grad_norm": 2.467883768091289,
+      "learning_rate": 7.332442959575424e-06,
+      "loss": 0.1005,
+      "step": 320
+    },
+    {
+      "epoch": 0.37,
+      "grad_norm": 1.1212388969162672,
+      "learning_rate": 7.316042414192866e-06,
+      "loss": 0.0914,
+      "step": 321
+    },
+    {
+      "epoch": 0.37,
+      "grad_norm": 1.8358020056483744,
+      "learning_rate": 7.299610081480155e-06,
+      "loss": 0.1146,
+      "step": 322
+    },
+    {
+      "epoch": 0.37,
+      "grad_norm": 1.3636541396641189,
+      "learning_rate": 7.283146186968566e-06,
+      "loss": 0.1085,
+      "step": 323
+    },
+    {
+      "epoch": 0.37,
+      "grad_norm": 1.07569146070376,
+      "learning_rate": 7.266650956622546e-06,
+      "loss": 0.089,
+      "step": 324
+    },
+    {
+      "epoch": 0.37,
+      "grad_norm": 1.8768958532173836,
+      "learning_rate": 7.2501246168366235e-06,
+      "loss": 0.096,
+      "step": 325
+    },
+    {
+      "epoch": 0.37,
+      "grad_norm": 1.8347653173611815,
+      "learning_rate": 7.233567394432302e-06,
+      "loss": 0.1067,
+      "step": 326
+    },
+    {
+      "epoch": 0.37,
+      "grad_norm": 1.202389845113578,
+      "learning_rate": 7.216979516654944e-06,
+      "loss": 0.0754,
+      "step": 327
+    },
+    {
+      "epoch": 0.37,
+      "grad_norm": 1.0247024117157115,
+      "learning_rate": 7.200361211170648e-06,
+      "loss": 0.0958,
+      "step": 328
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 0.9758438803344642,
+      "learning_rate": 7.183712706063133e-06,
+      "loss": 0.073,
+      "step": 329
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 1.0927982452541785,
+      "learning_rate": 7.1670342298306e-06,
+      "loss": 0.0891,
+      "step": 330
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 1.3702951110134494,
+      "learning_rate": 7.1503260113826035e-06,
+      "loss": 0.102,
+      "step": 331
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 1.6194293721517825,
+      "learning_rate": 7.1335882800369e-06,
+      "loss": 0.0877,
+      "step": 332
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 1.3330024442939468,
+      "learning_rate": 7.116821265516307e-06,
+      "loss": 0.1038,
+      "step": 333
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 1.5082089283588793,
+      "learning_rate": 7.100025197945555e-06,
+      "loss": 0.0903,
+      "step": 334
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 0.918479171265335,
+      "learning_rate": 7.083200307848116e-06,
+      "loss": 0.0782,
+      "step": 335
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 1.270870459177953,
+      "learning_rate": 7.066346826143049e-06,
+      "loss": 0.078,
+      "step": 336
+    },
+    {
+      "epoch": 0.39,
+      "grad_norm": 2.808167569407084,
+      "learning_rate": 7.049464984141831e-06,
+      "loss": 0.1034,
+      "step": 337
+    },
+    {
+      "epoch": 0.39,
+      "grad_norm": 1.4850123445239072,
+      "learning_rate": 7.032555013545176e-06,
+      "loss": 0.0987,
+      "step": 338
+    },
+    {
+      "epoch": 0.39,
+      "grad_norm": 1.0857390354428968,
+      "learning_rate": 7.015617146439863e-06,
+      "loss": 0.0846,
+      "step": 339
+    },
+    {
+      "epoch": 0.39,
+      "grad_norm": 1.4019142617304745,
+      "learning_rate": 6.998651615295541e-06,
+      "loss": 0.0946,
+      "step": 340
+    },
+    {
+      "epoch": 0.39,
+      "grad_norm": 1.4419350588759805,
+      "learning_rate": 6.9816586529615475e-06,
+      "loss": 0.0925,
+      "step": 341
+    },
+    {
+      "epoch": 0.39,
+      "grad_norm": 2.122469313937723,
+      "learning_rate": 6.9646384926637076e-06,
+      "loss": 0.101,
+      "step": 342
+    },
+    {
+      "epoch": 0.39,
+      "grad_norm": 2.023493438132179,
+      "learning_rate": 6.947591368001138e-06,
+      "loss": 0.1049,
+      "step": 343
+    },
+    {
+      "epoch": 0.39,
+      "grad_norm": 1.8085392355883325,
+      "learning_rate": 6.930517512943029e-06,
+      "loss": 0.0906,
+      "step": 344
+    },
+    {
+      "epoch": 0.39,
+      "grad_norm": 1.8476500146983619,
+      "learning_rate": 6.913417161825449e-06,
+      "loss": 0.075,
+      "step": 345
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 1.3169871494739973,
+      "learning_rate": 6.896290549348119e-06,
+      "loss": 0.0999,
+      "step": 346
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 1.5793887545549115,
+      "learning_rate": 6.879137910571191e-06,
+      "loss": 0.0951,
+      "step": 347
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 1.8170746345841167,
+      "learning_rate": 6.861959480912027e-06,
+      "loss": 0.0872,
+      "step": 348
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 1.9489463937624985,
+      "learning_rate": 6.8447554961419615e-06,
+      "loss": 0.1034,
+      "step": 349
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 2.1832874089959704,
+      "learning_rate": 6.82752619238307e-06,
+      "loss": 0.1158,
+      "step": 350
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 1.3585943935815048,
+      "learning_rate": 6.810271806104931e-06,
+      "loss": 0.0801,
+      "step": 351
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 4.249666680648082,
+      "learning_rate": 6.792992574121374e-06,
+      "loss": 0.097,
+      "step": 352
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 2.597130012884428,
+      "learning_rate": 6.7756887335872275e-06,
+      "loss": 0.1166,
+      "step": 353
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 1.4936397575141562,
+      "learning_rate": 6.758360521995079e-06,
+      "loss": 0.1212,
+      "step": 354
+    },
+    {
+      "epoch": 0.41,
+      "grad_norm": 1.171091229180981,
+      "learning_rate": 6.741008177171995e-06,
+      "loss": 0.0908,
+      "step": 355
+    },
+    {
+      "epoch": 0.41,
+      "grad_norm": 0.9253783941058572,
+      "learning_rate": 6.723631937276271e-06,
+      "loss": 0.0715,
+      "step": 356
+    },
+    {
+      "epoch": 0.41,
+      "grad_norm": 1.1914008404922645,
+      "learning_rate": 6.706232040794162e-06,
+      "loss": 0.0753,
+      "step": 357
+    },
+    {
+      "epoch": 0.41,
+      "grad_norm": 1.6494093340509537,
+      "learning_rate": 6.6888087265365995e-06,
+      "loss": 0.1066,
+      "step": 358
+    },
+    {
+      "epoch": 0.41,
+      "grad_norm": 1.1812819049319463,
+      "learning_rate": 6.671362233635926e-06,
+      "loss": 0.0927,
+      "step": 359
+    },
+    {
+      "epoch": 0.41,
+      "grad_norm": 2.4559536958240527,
+      "learning_rate": 6.653892801542604e-06,
+      "loss": 0.0928,
+      "step": 360
+    },
+    {
+      "epoch": 0.41,
+      "grad_norm": 1.25389273102752,
+      "learning_rate": 6.636400670021934e-06,
+      "loss": 0.0999,
+      "step": 361
+    },
+    {
+      "epoch": 0.41,
+      "grad_norm": 1.7179424428138383,
+      "learning_rate": 6.618886079150764e-06,
+      "loss": 0.0921,
+      "step": 362
+    },
+    {
+      "epoch": 0.41,
+      "grad_norm": 1.135878355877357,
+      "learning_rate": 6.601349269314188e-06,
+      "loss": 0.082,
+      "step": 363
+    },
+    {
+      "epoch": 0.42,
+      "grad_norm": 1.362482025104223,
+      "learning_rate": 6.583790481202261e-06,
+      "loss": 0.0999,
+      "step": 364
+    },
+    {
+      "epoch": 0.42,
+      "grad_norm": 1.9432112152221168,
+      "learning_rate": 6.566209955806679e-06,
+      "loss": 0.1061,
+      "step": 365
+    },
+    {
+      "epoch": 0.42,
+      "grad_norm": 1.6175027458716444,
+      "learning_rate": 6.54860793441748e-06,
+      "loss": 0.1199,
+      "step": 366
+    },
+    {
+      "epoch": 0.42,
+      "grad_norm": 1.7771666465783587,
+      "learning_rate": 6.530984658619735e-06,
+      "loss": 0.1045,
+      "step": 367
+    },
+    {
+      "epoch": 0.42,
+      "grad_norm": 1.4987987433398124,
+      "learning_rate": 6.5133403702902245e-06,
+      "loss": 0.1011,
+      "step": 368
+    },
+    {
+      "epoch": 0.42,
+      "grad_norm": 2.024080574349673,
+      "learning_rate": 6.495675311594123e-06,
+      "loss": 0.0962,
+      "step": 369
+    },
+    {
+      "epoch": 0.42,
+      "grad_norm": 1.233698410707572,
+      "learning_rate": 6.477989724981679e-06,
+      "loss": 0.102,
+      "step": 370
+    },
+    {
+      "epoch": 0.42,
+      "grad_norm": 1.860410897978535,
+      "learning_rate": 6.46028385318488e-06,
+      "loss": 0.0882,
+      "step": 371
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 1.056573638183693,
+      "learning_rate": 6.442557939214125e-06,
+      "loss": 0.0768,
+      "step": 372
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 1.8634932101277937,
+      "learning_rate": 6.42481222635489e-06,
+      "loss": 0.1102,
+      "step": 373
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 1.624917843958734,
+      "learning_rate": 6.407046958164387e-06,
+      "loss": 0.1014,
+      "step": 374
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 1.3310298175257878,
+      "learning_rate": 6.389262378468219e-06,
+      "loss": 0.0828,
+      "step": 375
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 1.0614121621946018,
+      "learning_rate": 6.371458731357047e-06,
+      "loss": 0.0987,
+      "step": 376
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 2.7534907444752217,
+      "learning_rate": 6.353636261183214e-06,
+      "loss": 0.0884,
+      "step": 377
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 1.3293066218185532,
+      "learning_rate": 6.335795212557416e-06,
+      "loss": 0.1089,
+      "step": 378
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 1.8227195971889414,
+      "learning_rate": 6.3179358303453386e-06,
+      "loss": 0.1146,
+      "step": 379
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 1.8879414641081136,
+      "learning_rate": 6.300058359664285e-06,
+      "loss": 0.0771,
+      "step": 380
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 2.0221109980415646,
+      "learning_rate": 6.2821630458798236e-06,
+      "loss": 0.113,
+      "step": 381
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 1.9690496853348498,
+      "learning_rate": 6.264250134602415e-06,
+      "loss": 0.1076,
+      "step": 382
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 1.3353353790171576,
+      "learning_rate": 6.246319871684048e-06,
+      "loss": 0.0983,
+      "step": 383
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 0.8840122811422151,
+      "learning_rate": 6.228372503214853e-06,
+      "loss": 0.0784,
+      "step": 384
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 1.9223687815447972,
+      "learning_rate": 6.210408275519735e-06,
+      "loss": 0.1164,
+      "step": 385
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 1.3646046422183316,
+      "learning_rate": 6.192427435154991e-06,
+      "loss": 0.1062,
+      "step": 386
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 1.3010904867560287,
+      "learning_rate": 6.17443022890492e-06,
+      "loss": 0.0733,
+      "step": 387
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 0.9588966018297974,
+      "learning_rate": 6.156416903778443e-06,
+      "loss": 0.0937,
+      "step": 388
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 0.8176889337157417,
+      "learning_rate": 6.138387707005711e-06,
+      "loss": 0.0785,
+      "step": 389
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 1.3798947208822552,
+      "learning_rate": 6.1203428860347085e-06,
+      "loss": 0.1029,
+      "step": 390
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 1.2627266387294376,
+      "learning_rate": 6.10228268852786e-06,
+      "loss": 0.0853,
+      "step": 391
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 1.3636634430274013,
+      "learning_rate": 6.084207362358635e-06,
+      "loss": 0.1015,
+      "step": 392
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 1.7109591987525221,
+      "learning_rate": 6.066117155608136e-06,
+      "loss": 0.1118,
+      "step": 393
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 1.517961417989287,
+      "learning_rate": 6.048012316561699e-06,
+      "loss": 0.0852,
+      "step": 394
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 2.342629048761873,
+      "learning_rate": 6.029893093705492e-06,
+      "loss": 0.1187,
+      "step": 395
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 0.9294142148345402,
+      "learning_rate": 6.011759735723096e-06,
+      "loss": 0.0864,
+      "step": 396
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 1.1950075847793311,
+      "learning_rate": 5.993612491492088e-06,
+      "loss": 0.092,
+      "step": 397
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 1.3551545691072298,
+      "learning_rate": 5.975451610080643e-06,
+      "loss": 0.0693,
+      "step": 398
+    },
+    {
+      "epoch": 0.46,
+      "grad_norm": 1.216161129705157,
+      "learning_rate": 5.957277340744094e-06,
+      "loss": 0.0895,
+      "step": 399
+    },
+    {
+      "epoch": 0.46,
+      "grad_norm": 1.4076300298899016,
+      "learning_rate": 5.939089932921527e-06,
+      "loss": 0.085,
+      "step": 400
+    },
+    {
+      "epoch": 0.46,
+      "grad_norm": 1.465518309640694,
+      "learning_rate": 5.920889636232352e-06,
+      "loss": 0.0957,
+      "step": 401
+    },
+    {
+      "epoch": 0.46,
+      "grad_norm": 1.3444282299556731,
+      "learning_rate": 5.902676700472874e-06,
+      "loss": 0.1023,
+      "step": 402
+    },
+    {
+      "epoch": 0.46,
+      "grad_norm": 2.172246680213261,
+      "learning_rate": 5.884451375612865e-06,
+      "loss": 0.0859,
+      "step": 403
+    },
+    {
+      "epoch": 0.46,
+      "grad_norm": 1.0096049350842804,
+      "learning_rate": 5.8662139117921435e-06,
+      "loss": 0.0952,
+      "step": 404
+    },
+    {
+      "epoch": 0.46,
+      "grad_norm": 1.3924332642607742,
+      "learning_rate": 5.847964559317129e-06,
+      "loss": 0.075,
+      "step": 405
+    },
+    {
+      "epoch": 0.46,
+      "grad_norm": 1.555222595194896,
+      "learning_rate": 5.829703568657406e-06,
+      "loss": 0.1194,
+      "step": 406
+    },
+    {
+      "epoch": 0.47,
+      "grad_norm": 1.814069025220444,
+      "learning_rate": 5.8114311904423e-06,
+      "loss": 0.1114,
+      "step": 407
+    },
+    {
+      "epoch": 0.47,
+      "grad_norm": 2.63589934143905,
+      "learning_rate": 5.793147675457425e-06,
+      "loss": 0.0997,
+      "step": 408
+    },
+    {
+      "epoch": 0.47,
+      "grad_norm": 1.672874605967912,
+      "learning_rate": 5.774853274641243e-06,
+      "loss": 0.0936,
+      "step": 409
+    },
+    {
+      "epoch": 0.47,
+      "grad_norm": 1.293398379507954,
+      "learning_rate": 5.756548239081626e-06,
+      "loss": 0.0936,
+      "step": 410
+    },
+    {
+      "epoch": 0.47,
+      "grad_norm": 1.3509632172781578,
+      "learning_rate": 5.738232820012407e-06,
+      "loss": 0.1014,
+      "step": 411
+    },
+    {
+      "epoch": 0.47,
+      "grad_norm": 1.4424133826280734,
+      "learning_rate": 5.719907268809927e-06,
+      "loss": 0.0942,
+      "step": 412
+    },
+    {
+      "epoch": 0.47,
+      "grad_norm": 1.6011921811511678,
+      "learning_rate": 5.701571836989591e-06,
+      "loss": 0.0839,
+      "step": 413
+    },
+    {
+      "epoch": 0.47,
+      "grad_norm": 2.1072468499647248,
+      "learning_rate": 5.683226776202416e-06,
+      "loss": 0.1096,
+      "step": 414
+    },
+    {
+      "epoch": 0.47,
+      "grad_norm": 1.3764568994495563,
+      "learning_rate": 5.664872338231572e-06,
+      "loss": 0.1001,
+      "step": 415
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 2.53101947882973,
+      "learning_rate": 5.64650877498893e-06,
+      "loss": 0.0885,
+      "step": 416
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 1.9958749325869098,
+      "learning_rate": 5.628136338511607e-06,
+      "loss": 0.1067,
+      "step": 417
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 1.1068155120562846,
+      "learning_rate": 5.6097552809584996e-06,
+      "loss": 0.0862,
+      "step": 418
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 1.625671593043311,
+      "learning_rate": 5.591365854606829e-06,
+      "loss": 0.0954,
+      "step": 419
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 0.8652769478201479,
+      "learning_rate": 5.5729683118486785e-06,
+      "loss": 0.0848,
+      "step": 420
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 2.0731466858910217,
+      "learning_rate": 5.554562905187527e-06,
+      "loss": 0.089,
+      "step": 421
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 1.6660858543441386,
+      "learning_rate": 5.536149887234781e-06,
+      "loss": 0.0982,
+      "step": 422
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 1.552222550169945,
+      "learning_rate": 5.517729510706316e-06,
+      "loss": 0.1154,
+      "step": 423
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 1.3368802229091306,
+      "learning_rate": 5.499302028418998e-06,
+      "loss": 0.0976,
+      "step": 424
+    },
+    {
+      "epoch": 0.49,
+      "grad_norm": 2.181126697333057,
+      "learning_rate": 5.480867693287224e-06,
+      "loss": 0.1062,
+      "step": 425
+    },
+    {
+      "epoch": 0.49,
+      "grad_norm": 1.9205645782423137,
+      "learning_rate": 5.462426758319439e-06,
+      "loss": 0.0911,
+      "step": 426
+    },
+    {
+      "epoch": 0.49,
+      "grad_norm": 1.9632409923514775,
+      "learning_rate": 5.443979476614674e-06,
+      "loss": 0.1095,
+      "step": 427
+    },
+    {
+      "epoch": 0.49,
+      "grad_norm": 1.1362611723204445,
+      "learning_rate": 5.425526101359068e-06,
+      "loss": 0.0777,
+      "step": 428
+    },
+    {
+      "epoch": 0.49,
+      "grad_norm": 1.3238503828238188,
+      "learning_rate": 5.407066885822391e-06,
+      "loss": 0.0824,
+      "step": 429
+    },
+    {
+      "epoch": 0.49,
+      "grad_norm": 1.5813044674729215,
+      "learning_rate": 5.388602083354572e-06,
+      "loss": 0.1091,
+      "step": 430
+    },
+    {
+      "epoch": 0.49,
+      "grad_norm": 1.2656866672687868,
+      "learning_rate": 5.370131947382215e-06,
+      "loss": 0.1061,
+      "step": 431
+    },
+    {
+      "epoch": 0.49,
+      "grad_norm": 1.3312297948060596,
+      "learning_rate": 5.351656731405132e-06,
+      "loss": 0.0823,
+      "step": 432
+    },
+    {
+      "epoch": 0.49,
+      "grad_norm": 1.4157388997754832,
+      "learning_rate": 5.333176688992856e-06,
+      "loss": 0.0792,
+      "step": 433
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 1.4094464298882508,
+      "learning_rate": 5.314692073781157e-06,
+      "loss": 0.101,
+      "step": 434
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 1.4099498232807148,
+      "learning_rate": 5.296203139468572e-06,
+      "loss": 0.0865,
+      "step": 435
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 1.8502414411111636,
+      "learning_rate": 5.2777101398129135e-06,
+      "loss": 0.0889,
+      "step": 436
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 1.7560344587273293,
+      "learning_rate": 5.2592133286277925e-06,
+      "loss": 0.0916,
+      "step": 437
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 1.6710267518378974,
+      "learning_rate": 5.2407129597791295e-06,
+      "loss": 0.1047,
+      "step": 438
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 1.2201280272734387,
+      "learning_rate": 5.222209287181677e-06,
+      "loss": 0.0891,
+      "step": 439
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 1.4358152308072225,
+      "learning_rate": 5.203702564795528e-06,
+      "loss": 0.0802,
+      "step": 440
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 0.8914507091018575,
+      "learning_rate": 5.185193046622636e-06,
+      "loss": 0.0782,
+      "step": 441
+    },
+    {
+      "epoch": 0.51,
+      "grad_norm": 1.6165198764693776,
+      "learning_rate": 5.166680986703323e-06,
+      "loss": 0.1246,
+      "step": 442
+    },
+    {
+      "epoch": 0.51,
+      "grad_norm": 1.4298748482799117,
+      "learning_rate": 5.148166639112799e-06,
+      "loss": 0.105,
+      "step": 443
+    },
+    {
+      "epoch": 0.51,
+      "grad_norm": 1.535559818853895,
+      "learning_rate": 5.129650257957671e-06,
+      "loss": 0.1119,
+      "step": 444
+    },
+    {
+      "epoch": 0.51,
+      "grad_norm": 1.9197842071730942,
+      "learning_rate": 5.111132097372459e-06,
+      "loss": 0.1033,
+      "step": 445
+    },
+    {
+      "epoch": 0.51,
+      "grad_norm": 1.1599006332138146,
+      "learning_rate": 5.0926124115160976e-06,
+      "loss": 0.0896,
+      "step": 446
+    },
+    {
+      "epoch": 0.51,
+      "grad_norm": 1.3974819169152706,
+      "learning_rate": 5.074091454568464e-06,
+      "loss": 0.0969,
+      "step": 447
+    },
+    {
+      "epoch": 0.51,
+      "grad_norm": 0.9408016749032659,
+      "learning_rate": 5.055569480726876e-06,
+      "loss": 0.0895,
+      "step": 448
+    },
+    {
+      "epoch": 0.51,
+      "grad_norm": 1.9952244551309546,
+      "learning_rate": 5.0370467442026115e-06,
+      "loss": 0.0871,
+      "step": 449
+    },
+    {
+      "epoch": 0.51,
+      "grad_norm": 1.36586105745886,
+      "learning_rate": 5.0185234992174114e-06,
+      "loss": 0.0844,
+      "step": 450
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 1.7479502067142012,
+      "learning_rate": 5e-06,
+      "loss": 0.1044,
+      "step": 451
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 1.3573048964331627,
+      "learning_rate": 4.981476500782591e-06,
+      "loss": 0.0948,
+      "step": 452
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 1.473812452772602,
+      "learning_rate": 4.96295325579739e-06,
+      "loss": 0.0797,
+      "step": 453
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 2.0296959916896,
+      "learning_rate": 4.944430519273126e-06,
+      "loss": 0.0895,
+      "step": 454
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 1.2103951372488675,
+      "learning_rate": 4.925908545431537e-06,
+      "loss": 0.0915,
+      "step": 455
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 1.434571480202637,
+      "learning_rate": 4.907387588483903e-06,
+      "loss": 0.0962,
+      "step": 456
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 1.2049199119312812,
+      "learning_rate": 4.8888679026275436e-06,
+      "loss": 0.0909,
+      "step": 457
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 1.7639545815530557,
+      "learning_rate": 4.870349742042329e-06,
+      "loss": 0.1117,
+      "step": 458
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 1.1494030416261993,
+      "learning_rate": 4.8518333608872015e-06,
+      "loss": 0.0752,
+      "step": 459
+    },
+    {
+      "epoch": 0.53,
+      "grad_norm": 1.6445773991618207,
+      "learning_rate": 4.8333190132966794e-06,
+      "loss": 0.0861,
+      "step": 460
+    },
+    {
+      "epoch": 0.53,
+      "grad_norm": 1.6414731186074902,
+      "learning_rate": 4.814806953377366e-06,
+      "loss": 0.0829,
+      "step": 461
+    },
+    {
+      "epoch": 0.53,
+      "grad_norm": 1.8968387997428107,
+      "learning_rate": 4.796297435204473e-06,
+      "loss": 0.0887,
+      "step": 462
+    },
+    {
+      "epoch": 0.53,
+      "grad_norm": 2.25439526816161,
+      "learning_rate": 4.777790712818324e-06,
+      "loss": 0.075,
+      "step": 463
+    },
+    {
+      "epoch": 0.53,
+      "grad_norm": 1.1014332021036057,
+      "learning_rate": 4.759287040220872e-06,
+      "loss": 0.0719,
+      "step": 464
+    },
+    {
+      "epoch": 0.53,
+      "grad_norm": 1.2580923566775155,
+      "learning_rate": 4.740786671372209e-06,
+      "loss": 0.0955,
+      "step": 465
+    },
+    {
+      "epoch": 0.53,
+      "grad_norm": 1.7123882846797445,
+      "learning_rate": 4.7222898601870864e-06,
+      "loss": 0.0935,
+      "step": 466
+    },
+    {
+      "epoch": 0.53,
+      "grad_norm": 1.178774620445043,
+      "learning_rate": 4.703796860531429e-06,
+      "loss": 0.0987,
+      "step": 467
+    },
+    {
+      "epoch": 0.53,
+      "grad_norm": 1.6539278144919904,
+      "learning_rate": 4.6853079262188445e-06,
+      "loss": 0.1124,
+      "step": 468
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 1.0935968959871103,
+      "learning_rate": 4.666823311007145e-06,
+      "loss": 0.0871,
+      "step": 469
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 1.3166304048687854,
+      "learning_rate": 4.648343268594869e-06,
+      "loss": 0.0749,
+      "step": 470
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 1.8561652305018204,
+      "learning_rate": 4.629868052617786e-06,
+      "loss": 0.0847,
+      "step": 471
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 1.414637359104844,
+      "learning_rate": 4.61139791664543e-06,
+      "loss": 0.0887,
+      "step": 472
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 1.2510852736375488,
+      "learning_rate": 4.59293311417761e-06,
+      "loss": 0.0713,
+      "step": 473
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 1.404573593610514,
+      "learning_rate": 4.574473898640933e-06,
+      "loss": 0.0863,
+      "step": 474
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 1.1678998858209273,
+      "learning_rate": 4.556020523385326e-06,
+      "loss": 0.1042,
+      "step": 475
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 1.4292796447931189,
+      "learning_rate": 4.537573241680563e-06,
+      "loss": 0.0898,
+      "step": 476
+    },
+    {
+      "epoch": 0.55,
+      "grad_norm": 1.8305004593870045,
+      "learning_rate": 4.519132306712778e-06,
+      "loss": 0.0883,
+      "step": 477
+    },
+    {
+      "epoch": 0.55,
+      "grad_norm": 1.706985080173823,
+      "learning_rate": 4.500697971581003e-06,
+      "loss": 0.0628,
+      "step": 478
+    },
+    {
+      "epoch": 0.55,
+      "grad_norm": 1.5783162958105084,
+      "learning_rate": 4.482270489293685e-06,
+      "loss": 0.0899,
+      "step": 479
+    },
+    {
+      "epoch": 0.55,
+      "grad_norm": 1.2211595747222304,
+      "learning_rate": 4.463850112765221e-06,
+      "loss": 0.0754,
+      "step": 480
+    },
+    {
+      "epoch": 0.55,
+      "grad_norm": 1.995983190967539,
+      "learning_rate": 4.445437094812476e-06,
+      "loss": 0.0887,
+      "step": 481
+    },
+    {
+      "epoch": 0.55,
+      "grad_norm": 1.428898296054049,
+      "learning_rate": 4.4270316881513215e-06,
+      "loss": 0.1133,
+      "step": 482
+    },
+    {
+      "epoch": 0.55,
+      "grad_norm": 1.4225799563091075,
+      "learning_rate": 4.408634145393172e-06,
+      "loss": 0.1184,
+      "step": 483
+    },
+    {
+      "epoch": 0.55,
+      "grad_norm": 1.3918275141803165,
+      "learning_rate": 4.390244719041502e-06,
+      "loss": 0.0648,
+      "step": 484
+    },
+    {
+      "epoch": 0.55,
+      "grad_norm": 1.367999116075585,
+      "learning_rate": 4.371863661488394e-06,
+      "loss": 0.0853,
+      "step": 485
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 1.7079182185930346,
+      "learning_rate": 4.353491225011071e-06,
+      "loss": 0.0949,
+      "step": 486
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 1.5010142904615251,
+      "learning_rate": 4.335127661768429e-06,
+      "loss": 0.0641,
+      "step": 487
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 1.7284173282169324,
+      "learning_rate": 4.316773223797585e-06,
+      "loss": 0.0703,
+      "step": 488
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 1.5621635512490386,
+      "learning_rate": 4.298428163010411e-06,
+      "loss": 0.0869,
+      "step": 489
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 1.422341370982355,
+      "learning_rate": 4.280092731190075e-06,
+      "loss": 0.0799,
+      "step": 490
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 1.7501446138446721,
+      "learning_rate": 4.261767179987595e-06,
+      "loss": 0.0897,
+      "step": 491
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 1.2772834345032038,
+      "learning_rate": 4.243451760918376e-06,
+      "loss": 0.1065,
+      "step": 492
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 0.7832782732498063,
+      "learning_rate": 4.225146725358759e-06,
+      "loss": 0.0806,
+      "step": 493
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 0.9590851368962775,
+      "learning_rate": 4.206852324542578e-06,
+      "loss": 0.0834,
+      "step": 494
+    },
+    {
+      "epoch": 0.57,
+      "grad_norm": 1.5291845240945432,
+      "learning_rate": 4.188568809557701e-06,
+      "loss": 0.0935,
+      "step": 495
+    },
+    {
+      "epoch": 0.57,
+      "grad_norm": 1.774265127813176,
+      "learning_rate": 4.170296431342595e-06,
+      "loss": 0.0776,
+      "step": 496
+    },
+    {
+      "epoch": 0.57,
+      "grad_norm": 1.6751404930422793,
+      "learning_rate": 4.152035440682874e-06,
+      "loss": 0.1102,
+      "step": 497
+    },
+    {
+      "epoch": 0.57,
+      "grad_norm": 1.1017368315986533,
+      "learning_rate": 4.1337860882078564e-06,
+      "loss": 0.0842,
+      "step": 498
+    },
+    {
+      "epoch": 0.57,
+      "grad_norm": 1.0095647614965828,
+      "learning_rate": 4.115548624387136e-06,
+      "loss": 0.1033,
+      "step": 499
+    },
+    {
+      "epoch": 0.57,
+      "grad_norm": 1.1850618393493437,
+      "learning_rate": 4.097323299527129e-06,
+      "loss": 0.0935,
+      "step": 500
+    }
+  ],
+  "logging_steps": 1.0,
+  "max_steps": 875,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "total_flos": 27948467372032.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebfe8bb420da2eb16d19fd06f47558897a5e52c061378a3e71ac7060768cf078
+size 7864

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/zero_to_fp32.py

@@ -0,0 +1,604 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+
+    total_files = len(files)
+    state_dicts = []
+    for f in files:
+        state_dict = torch.load(f, map_location=device)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    if zero_stage <= 2:
+        fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    elif zero_stage == 3:
+        # if there is more than one param group, there will be multiple flattened tensors - one
+        # flattened tensor per group - for simplicity merge them into a single tensor
+        #
+        # XXX: could make the script more memory efficient for when there are multiple groups - it
+        # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+        fp32_flat_groups = [
+            torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+        ]
+
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = fp32_flat_groups[0].numel() * world_size
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    for name, shape in param_shapes.items():
+
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # XXX: memory usage doubles here
+        state_dict[name] = torch.cat(
+            tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+            0).narrow(0, 0, unpartitioned_numel).view(shape)
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+
+    Returns:
+        - pytorch ``state_dict``
+
+    Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+    you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters)
+    print(f"Saving fp32 state dict to {output_file}")
+    torch.save(state_dict, output_file)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument(
+        "output_file",
+        type=str,
+        help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_file,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/config.json

@@ -0,0 +1,221 @@
+{
+  "_name_or_path": "lmms-lab/llava-onevision-qwen2-7b-si",
+  "add_faster_video": false,
+  "add_time_instruction": true,
+  "architectures": [
+    "LlavaQwenForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "faster_token_stride": 10,
+  "force_sample": true,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "ignore_index": -100,
+  "image_aspect_ratio": "anyres_max_9",
+  "image_crop_resolution": null,
+  "image_grid_pinpoints": [
+    [
+      384,
+      384
+    ],
+    [
+      384,
+      768
+    ],
+    [
+      384,
+      1152
+    ],
+    [
+      384,
+      1536
+    ],
+    [
+      384,
+      1920
+    ],
+    [
+      384,
+      2304
+    ],
+    [
+      768,
+      384
+    ],
+    [
+      768,
+      768
+    ],
+    [
+      768,
+      1152
+    ],
+    [
+      768,
+      1536
+    ],
+    [
+      768,
+      1920
+    ],
+    [
+      768,
+      2304
+    ],
+    [
+      1152,
+      384
+    ],
+    [
+      1152,
+      768
+    ],
+    [
+      1152,
+      1152
+    ],
+    [
+      1152,
+      1536
+    ],
+    [
+      1152,
+      1920
+    ],
+    [
+      1152,
+      2304
+    ],
+    [
+      1536,
+      384
+    ],
+    [
+      1536,
+      768
+    ],
+    [
+      1536,
+      1152
+    ],
+    [
+      1536,
+      1536
+    ],
+    [
+      1536,
+      1920
+    ],
+    [
+      1536,
+      2304
+    ],
+    [
+      1920,
+      384
+    ],
+    [
+      1920,
+      768
+    ],
+    [
+      1920,
+      1152
+    ],
+    [
+      1920,
+      1536
+    ],
+    [
+      1920,
+      1920
+    ],
+    [
+      1920,
+      2304
+    ],
+    [
+      2304,
+      384
+    ],
+    [
+      2304,
+      768
+    ],
+    [
+      2304,
+      1152
+    ],
+    [
+      2304,
+      1536
+    ],
+    [
+      2304,
+      1920
+    ],
+    [
+      2304,
+      2304
+    ]
+  ],
+  "image_split_resolution": null,
+  "image_token_index": 151646,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "mm_hidden_size": 1152,
+  "mm_newline_position": "grid",
+  "mm_patch_merge_type": "spatial_unpad",
+  "mm_projector_lr": null,
+  "mm_projector_type": "mlp2x_gelu",
+  "mm_resampler_type": null,
+  "mm_spatial_pool_mode": "bilinear",
+  "mm_spatial_pool_stride": 2,
+  "mm_tunable_parts": "mm_vision_tower,mm_mlp_adapter,mm_language_model",
+  "mm_use_im_patch_token": false,
+  "mm_use_im_start_end": false,
+  "mm_vision_select_feature": "patch",
+  "mm_vision_select_layer": -2,
+  "mm_vision_tower": "google/siglip-so400m-patch14-384",
+  "mm_vision_tower_lr": 2e-06,
+  "model_type": "llava",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "pos_skipping_range": 4096,
+  "projector_hidden_act": "gelu",
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": 131072,
+  "text_config": {
+    "model_type": "llama"
+  },
+  "tie_word_embeddings": false,
+  "tokenizer_model_max_length": 32768,
+  "tokenizer_padding_side": "right",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.40.0.dev0",
+  "use_cache": true,
+  "use_mm_proj": true,
+  "use_pos_skipping": false,
+  "use_sliding_window": false,
+  "vision_config": {
+    "hidden_size": 1024,
+    "image_size": 336,
+    "intermediate_size": 4096,
+    "model_type": "clip_vision_model",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 24,
+    "patch_size": 14,
+    "projection_dim": 768,
+    "vocab_size": 32000
+  },
+  "vision_feature_layer": -2,
+  "vision_feature_select_strategy": "default",
+  "vision_tower_pretrained": null
+}

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.05,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "4.40.0.dev0"
+}

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1587f08ededb10afe215bfdd63f56ead468d14b4c5480e59353d7f6ad0a8aa33
+size 4877668032

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3689b2b26ddfe0ec9cb6c0eb46c7baff6f963537014c868d18f9baabd2e4e91e
+size 4932751008

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a2430cc05cb411e38188b8d2b262d02ab853d4f6a374b0b9cf799d41cb9a35a
+size 4994571904

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d220af7174045b40585aec2fa6f9f309b4e5a7850c5b923c51791304146c7d64
+size 1255812224

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/model.safetensors.index.json

@@ -0,0 +1,772 @@
+{
+  "metadata": {
+    "total_size": 16060697664
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.image_newline": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.mm_projector.0.bias": "model-00004-of-00004.safetensors",
+    "model.mm_projector.0.weight": "model-00004-of-00004.safetensors",
+    "model.mm_projector.2.bias": "model-00004-of-00004.safetensors",
+    "model.mm_projector.2.weight": "model-00004-of-00004.safetensors",
+    "model.norm.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.embeddings.patch_embedding.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.embeddings.patch_embedding.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.embeddings.position_embedding.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.post_layernorm.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.post_layernorm.weight": "model-00004-of-00004.safetensors"
+  }
+}

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/special_tokens_map.json

@@ -0,0 +1,20 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/tokenizer_config.json

@@ -0,0 +1,53 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "processor_class": "LlavaProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebfe8bb420da2eb16d19fd06f47558897a5e52c061378a3e71ac7060768cf078
+size 7864