diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 0000000000000000000000000000000000000000..a6344aac8c09253b3b630fb776ae94478aa0275b
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
diff --git a/README.md b/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..63058fa9752f1432c26c262481f3d46ea9e7fe0c
--- /dev/null
+++ b/README.md
@@ -0,0 +1,106 @@
+---
+license: apache-2.0
+---
+
+
+
+
+🏠 WizardLM-2 Release Blog
+
+🤗 HF Repo •🐱 Github Repo • 🐦 Twitter • 📃 [WizardLM] • 📃 [WizardCoder] • 📃 [WizardMath]
+
+
+ 👋 Join our Discord
+
+
+
+## News 🔥🔥🔥 [2024/04/15]
+
+We introduce and opensource WizardLM-2, our next generation state-of-the-art large language models,
+which have improved performance on complex chat, multilingual, reasoning and agent.
+New family includes three cutting-edge models: WizardLM-2 8x22B, WizardLM-2 70B, and WizardLM-2 7B.
+
+- WizardLM-2 8x22B is our most advanced model, demonstrates highly competitive performance compared to those leading proprietary works
+and consistently outperforms all the existing state-of-the-art opensource models.
+- WizardLM-2 70B reaches top-tier reasoning capabilities and is the first choice in the same size.
+- WizardLM-2 7B is the fastest and achieves comparable performance with existing 10x larger opensource leading models.
+
+For more details of WizardLM-2 please read our [release blog post](https://wizardlm.github.io/WizardLM2) and upcoming paper.
+
+
+## Model Details
+
+* **Model name**: WizardLM-2 8x22B
+* **Developed by**: WizardLM@Microsoft AI
+* **Model type**: Mixture of Experts (MoE)
+* **Base model**: [mistral-community/Mixtral-8x22B-v0.1](https://huggingface.co/mistral-community/Mixtral-8x22B-v0.1)
+* **Parameters**: 141B
+* **Language(s)**: Multilingual
+* **Blog**: [Introducing WizardLM-2](https://wizardlm.github.io/WizardLM2)
+* **Repository**: [https://github.com/nlpxucan/WizardLM](https://github.com/nlpxucan/WizardLM)
+* **Paper**: WizardLM-2 (Upcoming)
+* **License**: Apache2.0
+
+
+## Model Capacities
+
+
+**MT-Bench**
+
+We also adopt the automatic MT-Bench evaluation framework based on GPT-4 proposed by lmsys to assess the performance of models.
+The WizardLM-2 8x22B even demonstrates highly competitive performance compared to the most advanced proprietary models.
+Meanwhile, WizardLM-2 7B and WizardLM-2 70B are all the top-performing models among the other leading baselines at 7B to 70B model scales.
+
+
+
+
+
+
+**Human Preferences Evaluation**
+
+We carefully collected a complex and challenging set consisting of real-world instructions, which includes main requirements of humanity, such as writing, coding, math, reasoning, agent, and multilingual.
+We report the win:loss rate without tie:
+
+- WizardLM-2 8x22B is just slightly falling behind GPT-4-1106-preview, and significantly stronger than Command R Plus and GPT4-0314.
+- WizardLM-2 70B is better than GPT4-0613, Mistral-Large, and Qwen1.5-72B-Chat.
+- WizardLM-2 7B is comparable with Qwen1.5-32B-Chat, and surpasses Qwen1.5-14B-Chat and Starling-LM-7B-beta.
+
+
+
+
+
+
+
+
+
+## Method Overview
+We built a **fully AI powered synthetic training system** to train WizardLM-2 models, please refer to our [blog](https://wizardlm.github.io/WizardLM2) for more details of this system.
+
+
+
+
+
+
+
+
+
+## Usage
+
+❗Note for model system prompts usage:
+
+
+WizardLM-2 adopts the prompt format from Vicuna and supports **multi-turn** conversation. The prompt should be as following:
+
+```
+A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful,
+detailed, and polite answers to the user's questions. USER: Hi ASSISTANT: Hello.
+USER: Who are you? ASSISTANT: I am WizardLM.......
+```
+
+ Inference WizardLM-2 Demo Script
+
+We provide a WizardLM-2 inference demo [code](https://github.com/nlpxucan/WizardLM/tree/main/demo) on our github.
+
+
+
+
diff --git a/config.json b/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..c2e3aea022e7115bf62b41eda31fe363eb816ef1
--- /dev/null
+++ b/config.json
@@ -0,0 +1,31 @@
+{
+ "_name_or_path": "",
+ "architectures": [
+ "MixtralForCausalLM"
+ ],
+ "attention_dropout": 0.0,
+ "bos_token_id": 1,
+ "eos_token_id": 2,
+ "hidden_act": "silu",
+ "hidden_size": 6144,
+ "initializer_range": 0.02,
+ "intermediate_size": 16384,
+ "max_position_embeddings": 65536,
+ "model_type": "mixtral",
+ "num_attention_heads": 48,
+ "num_experts_per_tok": 2,
+ "num_hidden_layers": 56,
+ "num_key_value_heads": 8,
+ "num_local_experts": 8,
+ "output_router_logits": false,
+ "rms_norm_eps": 1e-05,
+ "rope_theta": 1000000,
+ "router_aux_loss_coef": 0.001,
+ "router_jitter_noise": 0.0,
+ "sliding_window": null,
+ "tie_word_embeddings": false,
+ "torch_dtype": "bfloat16",
+ "transformers_version": "4.36.2",
+ "use_cache": false,
+ "vocab_size": 32000
+}
diff --git a/generation_config.json b/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..c533f934c6359393a56a3ea067a0df118c14797e
--- /dev/null
+++ b/generation_config.json
@@ -0,0 +1,6 @@
+{
+ "_from_model_config": true,
+ "bos_token_id": 1,
+ "eos_token_id": 2,
+ "transformers_version": "4.36.2"
+}
diff --git a/huggingface-metadata.txt b/huggingface-metadata.txt
new file mode 100644
index 0000000000000000000000000000000000000000..60a6d8f36bc58bc125251e0d2252e7f5d7b5bafe
--- /dev/null
+++ b/huggingface-metadata.txt
@@ -0,0 +1,64 @@
+url: https://huggingface.co/microsoft/WizardLM-2-8x22B
+branch: main
+download date: 2024-04-15 16:48:15
+sha256sum:
+ 9e5cc2f91f7830364eff657b53df4f855091996cb2db7de2b69b8f11fa06be0a model-00001-of-00059.safetensors
+ 24dbc249686530b5bcf630c6edae25fff212900d340946fb868ebd3dd8c62851 model-00002-of-00059.safetensors
+ d8ebc25463564c00d8e5d3f8a08ca5ce60c0efd53dd23aecfa7a2e3b0112b941 model-00003-of-00059.safetensors
+ 81abed9dadae5877c49a57990fe5d8a449ed507987517728b1da7c195eacdd54 model-00004-of-00059.safetensors
+ 45a5914445b0d8a95e02881d342a37b7d7c69943980b373b78331cf5c406191f model-00005-of-00059.safetensors
+ ea96b16084f0b6fddf232343e2ea55734ec324aeaa2eb60759fca724b80e292d model-00006-of-00059.safetensors
+ a78d60bb853a77110a815f6a20c5819cef21bcc0b4d50e700b560be54d6f53b5 model-00007-of-00059.safetensors
+ ba757c9109937a28e912c10bdb18042d347c55e768ec2dc89859c0e25823fa35 model-00008-of-00059.safetensors
+ ab90762b6b58313c37b2d1e09a8f9ae956bc4b5d0cc52c379b0937ce752dca98 model-00009-of-00059.safetensors
+ 0b167436f7049226dbe7153e1ec602bf06d4cc384e5c16289e7207408c66068a model-00010-of-00059.safetensors
+ c4cc3a3f71b37f107b20f96d04067abbaee78f16e74196cdc68469d551bbe38c model-00011-of-00059.safetensors
+ b045740b8c2377430a5f42e06d9eec2606b8db45c3e5ac87a316fbc715569cc0 model-00012-of-00059.safetensors
+ 175977b687acd05686a22c0f2b89f0ab0afb79c03db3480fa7fd989aec786287 model-00013-of-00059.safetensors
+ 2703b8b32154f524baf105cb15f54ffbf134d13ccd923a30315e698fb85e54ff model-00014-of-00059.safetensors
+ bbe6004093d3a5e29f26c5e3f75e4e0b8a40c16498102dac31b0180746880671 model-00015-of-00059.safetensors
+ 54c8f475521181cb4b93031b17c2a5c08ad0de1a81b035c2864c2bce19234c21 model-00016-of-00059.safetensors
+ 0b4e356a83fc7e42a056d790213fee7624abc0d12b8ca113dbe734a139a99491 model-00017-of-00059.safetensors
+ 188a8a3611852f704dae7804b47cfabb5a2ccf6c7f19226ae4e3fce6a3aa0506 model-00018-of-00059.safetensors
+ 36d1e452bb1e962a83fdd712f4b018195a0a6102f378e9297291acf2d82cb3c0 model-00019-of-00059.safetensors
+ f99ae9d9cf903af58255d0774edec5ec1382c0d56a609108a59ce111518d40f0 model-00020-of-00059.safetensors
+ 1223e5b007d833bcb459b23f3d7f59f646b16acf2a1a685085389b8de2dc6128 model-00021-of-00059.safetensors
+ 48a99d78aaea1949ad85dce7847fa7d033173a22a58add4b2cd732410e7e15cf model-00022-of-00059.safetensors
+ db3c4baffbc4fc5a4eb34926e342d91fe4bcae59e0f712d06db4c2c4deb985da model-00023-of-00059.safetensors
+ 141a6b26fb0a959967c24629bbfcf0c970a475d7698d2125e622cb38fb385b9b model-00024-of-00059.safetensors
+ 433db989f798b99a87a64829f0a9d067e89daf1352cda3014e15c5414cbd33d2 model-00025-of-00059.safetensors
+ 6cc8f60655049f3c1cc04c74cc3f02f72a3f820dcac263b483fa8bcba026ac6c model-00026-of-00059.safetensors
+ 20b6cc117252f16fcb3d41f2372aca13a8f7426c13c86ad623c43cf0b96b5e72 model-00027-of-00059.safetensors
+ 8bbbb1329d0fbe43933a49d37dada0518609bc18cc762549c67f677051ac46e8 model-00028-of-00059.safetensors
+ 0734dbd916a3e5789d4338fdb053da6882fb7a6d636fd1495710557c032b5a24 model-00029-of-00059.safetensors
+ 01f5ace1d4086c96950f57604626a30821ff3ca6f60dddea74678d914d93dd72 model-00030-of-00059.safetensors
+ 8c362f13e9e0889f09bb552e57e5e7ac45e2f28510a9ad9fab5b50147b9c6dab model-00031-of-00059.safetensors
+ e93b983001b7b890da1a96405c630efb3eee7617482af1491822f213a6640b38 model-00032-of-00059.safetensors
+ 61bcaf0a63535ba8f756509e7cca4b6ce1c4b741aca01f2afba40f5c3a727cfd model-00033-of-00059.safetensors
+ 1c97dd5624ad3332e21c86836fc063059caab9262125a986e6f331c4ecf5aeff model-00034-of-00059.safetensors
+ 8e9e4be3a6296da50cc1b4af53115818aa75d266c430a6baa2f91709885d18df model-00035-of-00059.safetensors
+ a699b76e6a8633bd6ba0527270bb81a4c91604cd3a7118de09c9361fda3b144e model-00036-of-00059.safetensors
+ ff7bbbeb2f68415f368f2e99e91b905e145c87beadcf352d0311f85ec6ddaf6c model-00037-of-00059.safetensors
+ 2fe8533aac43a8c1c7b0885efef6a3080fdc6a61c880746c47520c25a10bf17d model-00038-of-00059.safetensors
+ 4846e39f398d04411859f18b2697f4e70e70ad33deeefa93135911d10c5a23e8 model-00039-of-00059.safetensors
+ 4b7de8e7af1d27c665f9e8dee5c904f9ee88ccd48c831361fc5999b979c16897 model-00040-of-00059.safetensors
+ 02c8320c607f11bbd377930a056873d84f0192d950176fc22e3c59d1dfb5995d model-00041-of-00059.safetensors
+ 68a8a3b2780336a8012b795aa6295fd9248c322f939e8131315a520fe4e43480 model-00042-of-00059.safetensors
+ d859d466ef5da5d0cf036bc6ed57fa0033d15c3921eb042d1a31617c6f0eef34 model-00043-of-00059.safetensors
+ 06d32c1ed652616e42fffdcce1735fad67b50a4f13397cfd030a4304ce035a08 model-00044-of-00059.safetensors
+ beb9181e4b827f0136d213c11a05c569e0bcb85b7e68ade06b38c5d3da2d1ae0 model-00045-of-00059.safetensors
+ 44a573a390174b0f0a41f8c0b198761384c46ae328a089f97d3e622468507224 model-00046-of-00059.safetensors
+ 32a3bc20ff1805ae3e14e0bcf7ac3a1f9c21a31fb04922d3d31e440e658c32b6 model-00047-of-00059.safetensors
+ 69a198153c9725aa0b6de50831b55fb0f04fc7a0f402731257836110358b7bde model-00048-of-00059.safetensors
+ a9d4adc81dbb3f12ec1300cb9c53bae070b7dbe41a66e30b5160841504fb5ba3 model-00049-of-00059.safetensors
+ 9d7cf6acb12c6907a102f9b836e3db345d2aa885493b3904084b95d0ac55c739 model-00050-of-00059.safetensors
+ 582322e446962d374f4b6fd5e469040d331150f7ef3d4a69fa460deded7c3ebc model-00051-of-00059.safetensors
+ d6fe7d0d82aa12dd5f40c083b4046ac6ef15ed0627433d63795bed9e50c55fb4 model-00052-of-00059.safetensors
+ af0c588e8ffe2e87ce5dee116d70ec083a3bfd2f4c4a09618d8a2791de5cd55f model-00053-of-00059.safetensors
+ ecda36174ea4e376919360f6e08c17d5b3d0a870469e9fd52cac0b7c8e12351e model-00054-of-00059.safetensors
+ 5d7c98c60576932850e8ef8945e38f24ca0f37cca895bf04c4a23d251081859c model-00055-of-00059.safetensors
+ 7410c9689fd0d2d384f4b2886707fb377327457937c91da914c725b7c35c2396 model-00056-of-00059.safetensors
+ 5c684ae59038495c44d01507e8047511db57fa720cc9997f7719fba76dbc29ee model-00057-of-00059.safetensors
+ b1234554df96e1f1e2a7adcef5f680e7b57e9b6ae4f4a16ceff8225b20921c5d model-00058-of-00059.safetensors
+ e36fa9e3f053db7a27e06f5b2804a2aa650a026a66e3f7db9efd35c5103cd9ae model-00059-of-00059.safetensors
+ dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055 tokenizer.model
diff --git a/model-00001-of-00059.safetensors b/model-00001-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..b11e559b24e26e085d1023a6494702419f1a7aac
--- /dev/null
+++ b/model-00001-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e5cc2f91f7830364eff657b53df4f855091996cb2db7de2b69b8f11fa06be0a
+size 4998663696
diff --git a/model-00002-of-00059.safetensors b/model-00002-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..8dee08c0aeda57e16f9e5c8be35ac6880368be86
--- /dev/null
+++ b/model-00002-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24dbc249686530b5bcf630c6edae25fff212900d340946fb868ebd3dd8c62851
+size 4806799120
diff --git a/model-00003-of-00059.safetensors b/model-00003-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..51ce78a4015562813a3c674d6acc9bb57de37220
--- /dev/null
+++ b/model-00003-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8ebc25463564c00d8e5d3f8a08ca5ce60c0efd53dd23aecfa7a2e3b0112b941
+size 4806799120
diff --git a/model-00004-of-00059.safetensors b/model-00004-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..1a294822eacab5fcc37bc842961f22bd0992f7a8
--- /dev/null
+++ b/model-00004-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81abed9dadae5877c49a57990fe5d8a449ed507987517728b1da7c195eacdd54
+size 4806799120
diff --git a/model-00005-of-00059.safetensors b/model-00005-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..d17beae3b27fe748795356f1265269dfadead52e
--- /dev/null
+++ b/model-00005-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45a5914445b0d8a95e02881d342a37b7d7c69943980b373b78331cf5c406191f
+size 4806799120
diff --git a/model-00006-of-00059.safetensors b/model-00006-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..467c3fc1ffa43510027c89d03000f2bd2dddae31
--- /dev/null
+++ b/model-00006-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea96b16084f0b6fddf232343e2ea55734ec324aeaa2eb60759fca724b80e292d
+size 4806799120
diff --git a/model-00007-of-00059.safetensors b/model-00007-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..5cc3a0521e1c590b76be00d2c9475b35ab609b58
--- /dev/null
+++ b/model-00007-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a78d60bb853a77110a815f6a20c5819cef21bcc0b4d50e700b560be54d6f53b5
+size 4806799120
diff --git a/model-00008-of-00059.safetensors b/model-00008-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..13a77efb85ad66b73857d0dcf8be3aa3dfd5a63b
--- /dev/null
+++ b/model-00008-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba757c9109937a28e912c10bdb18042d347c55e768ec2dc89859c0e25823fa35
+size 4806799120
diff --git a/model-00009-of-00059.safetensors b/model-00009-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..3dd40845937266317c2d620faadc6756b47c6091
--- /dev/null
+++ b/model-00009-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab90762b6b58313c37b2d1e09a8f9ae956bc4b5d0cc52c379b0937ce752dca98
+size 4806799120
diff --git a/model-00010-of-00059.safetensors b/model-00010-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..1d0c9712e4e34e6e7767a80f8d79fb0f980a6bcc
--- /dev/null
+++ b/model-00010-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b167436f7049226dbe7153e1ec602bf06d4cc384e5c16289e7207408c66068a
+size 4806799120
diff --git a/model-00011-of-00059.safetensors b/model-00011-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..12578cc18fca05c12f52ed9e3a4fb40c89b3ae33
--- /dev/null
+++ b/model-00011-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4cc3a3f71b37f107b20f96d04067abbaee78f16e74196cdc68469d551bbe38c
+size 4806799136
diff --git a/model-00012-of-00059.safetensors b/model-00012-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..64e287a5695c90ba5ab7fc4a9c7cffc18f0c491a
--- /dev/null
+++ b/model-00012-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b045740b8c2377430a5f42e06d9eec2606b8db45c3e5ac87a316fbc715569cc0
+size 4806799152
diff --git a/model-00013-of-00059.safetensors b/model-00013-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..4a311d864745af98499acac1ad73413cfa54c0ec
--- /dev/null
+++ b/model-00013-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:175977b687acd05686a22c0f2b89f0ab0afb79c03db3480fa7fd989aec786287
+size 4806799152
diff --git a/model-00014-of-00059.safetensors b/model-00014-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..11c87fa879833f39223050b4f74fae1645f8691d
--- /dev/null
+++ b/model-00014-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2703b8b32154f524baf105cb15f54ffbf134d13ccd923a30315e698fb85e54ff
+size 4806799152
diff --git a/model-00015-of-00059.safetensors b/model-00015-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..faca7ab06abb28d40a95a7a93013cb34e2b41096
--- /dev/null
+++ b/model-00015-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbe6004093d3a5e29f26c5e3f75e4e0b8a40c16498102dac31b0180746880671
+size 4806799152
diff --git a/model-00016-of-00059.safetensors b/model-00016-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..dec22288dd78df268ced26b68f8a78b372b478c4
--- /dev/null
+++ b/model-00016-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54c8f475521181cb4b93031b17c2a5c08ad0de1a81b035c2864c2bce19234c21
+size 4806799152
diff --git a/model-00017-of-00059.safetensors b/model-00017-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ef87d7c776f9d516713ec6c36c265e8dcb9819e8
--- /dev/null
+++ b/model-00017-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b4e356a83fc7e42a056d790213fee7624abc0d12b8ca113dbe734a139a99491
+size 4806799152
diff --git a/model-00018-of-00059.safetensors b/model-00018-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..5ae11f681b74e5c19964e527bee392db10c46c06
--- /dev/null
+++ b/model-00018-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:188a8a3611852f704dae7804b47cfabb5a2ccf6c7f19226ae4e3fce6a3aa0506
+size 4806799152
diff --git a/model-00019-of-00059.safetensors b/model-00019-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..140a37476601f4a73740d97dac6354c68ffd5753
--- /dev/null
+++ b/model-00019-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36d1e452bb1e962a83fdd712f4b018195a0a6102f378e9297291acf2d82cb3c0
+size 4806799152
diff --git a/model-00020-of-00059.safetensors b/model-00020-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..29f2306c92a05ceb214915f81300f34d66174af7
--- /dev/null
+++ b/model-00020-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f99ae9d9cf903af58255d0774edec5ec1382c0d56a609108a59ce111518d40f0
+size 4806799152
diff --git a/model-00021-of-00059.safetensors b/model-00021-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..d79b10871ddaa776d32312ab53d381b542f0100f
--- /dev/null
+++ b/model-00021-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1223e5b007d833bcb459b23f3d7f59f646b16acf2a1a685085389b8de2dc6128
+size 4806799152
diff --git a/model-00022-of-00059.safetensors b/model-00022-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ae899eff7297e7373615fe614fe3b24863c5fb8f
--- /dev/null
+++ b/model-00022-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48a99d78aaea1949ad85dce7847fa7d033173a22a58add4b2cd732410e7e15cf
+size 4806799152
diff --git a/model-00023-of-00059.safetensors b/model-00023-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ef245d428bc875b553b16ddcc8add86410884886
--- /dev/null
+++ b/model-00023-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db3c4baffbc4fc5a4eb34926e342d91fe4bcae59e0f712d06db4c2c4deb985da
+size 4806799152
diff --git a/model-00024-of-00059.safetensors b/model-00024-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..6781cce8650eeeb351953c44bb7dfa5c3f6b1f43
--- /dev/null
+++ b/model-00024-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:141a6b26fb0a959967c24629bbfcf0c970a475d7698d2125e622cb38fb385b9b
+size 4932529864
diff --git a/model-00025-of-00059.safetensors b/model-00025-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..d3b04661dba3ad373cf9eedec9cca08d341956c9
--- /dev/null
+++ b/model-00025-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:433db989f798b99a87a64829f0a9d067e89daf1352cda3014e15c5414cbd33d2
+size 4995542848
diff --git a/model-00026-of-00059.safetensors b/model-00026-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..1c1546d17ab6749486401cc033bd258f514964fa
--- /dev/null
+++ b/model-00026-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6cc8f60655049f3c1cc04c74cc3f02f72a3f820dcac263b483fa8bcba026ac6c
+size 4995542848
diff --git a/model-00027-of-00059.safetensors b/model-00027-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..23834f5919e2489c34f57ab5d53affa47c81966a
--- /dev/null
+++ b/model-00027-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20b6cc117252f16fcb3d41f2372aca13a8f7426c13c86ad623c43cf0b96b5e72
+size 4932628288
diff --git a/model-00028-of-00059.safetensors b/model-00028-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..7882f5c9fb87a75a6b4a2fdc15f3e3cafafccec3
--- /dev/null
+++ b/model-00028-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8bbbb1329d0fbe43933a49d37dada0518609bc18cc762549c67f677051ac46e8
+size 4806774344
diff --git a/model-00029-of-00059.safetensors b/model-00029-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..0dbf722c3e045fe31b66912382a2aa2fab62703a
--- /dev/null
+++ b/model-00029-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0734dbd916a3e5789d4338fdb053da6882fb7a6d636fd1495710557c032b5a24
+size 4806799144
diff --git a/model-00030-of-00059.safetensors b/model-00030-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..f6ac53bb59535e9351180351c9f470b760a76f3d
--- /dev/null
+++ b/model-00030-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01f5ace1d4086c96950f57604626a30821ff3ca6f60dddea74678d914d93dd72
+size 4806799144
diff --git a/model-00031-of-00059.safetensors b/model-00031-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..0328be54d06b3ffbb45b230f5be3458c9d5be743
--- /dev/null
+++ b/model-00031-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c362f13e9e0889f09bb552e57e5e7ac45e2f28510a9ad9fab5b50147b9c6dab
+size 4806799144
diff --git a/model-00032-of-00059.safetensors b/model-00032-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..32328a167aa3c3f4f0953aeae9545e834a790c63
--- /dev/null
+++ b/model-00032-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e93b983001b7b890da1a96405c630efb3eee7617482af1491822f213a6640b38
+size 4806799144
diff --git a/model-00033-of-00059.safetensors b/model-00033-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..583cb23cb550e5853311dbf6fd9c6acbe33f100e
--- /dev/null
+++ b/model-00033-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61bcaf0a63535ba8f756509e7cca4b6ce1c4b741aca01f2afba40f5c3a727cfd
+size 4806799152
diff --git a/model-00034-of-00059.safetensors b/model-00034-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..8276e6586ff1c109b6f533aa61523c013ea4fb6e
--- /dev/null
+++ b/model-00034-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c97dd5624ad3332e21c86836fc063059caab9262125a986e6f331c4ecf5aeff
+size 4806799152
diff --git a/model-00035-of-00059.safetensors b/model-00035-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..24c38ce9584dcb376452709e6bc466342b2376f0
--- /dev/null
+++ b/model-00035-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e9e4be3a6296da50cc1b4af53115818aa75d266c430a6baa2f91709885d18df
+size 4806799152
diff --git a/model-00036-of-00059.safetensors b/model-00036-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..d24375f88dc4ea172188429fe259ecbc5937b5b8
--- /dev/null
+++ b/model-00036-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a699b76e6a8633bd6ba0527270bb81a4c91604cd3a7118de09c9361fda3b144e
+size 4806799152
diff --git a/model-00037-of-00059.safetensors b/model-00037-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..42bc0e9d3706c9721819fc939b5485d2a6092c15
--- /dev/null
+++ b/model-00037-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff7bbbeb2f68415f368f2e99e91b905e145c87beadcf352d0311f85ec6ddaf6c
+size 4806799152
diff --git a/model-00038-of-00059.safetensors b/model-00038-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..82108a43cca7debc81d77c3b807ee4d32371e06c
--- /dev/null
+++ b/model-00038-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2fe8533aac43a8c1c7b0885efef6a3080fdc6a61c880746c47520c25a10bf17d
+size 4806799152
diff --git a/model-00039-of-00059.safetensors b/model-00039-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..d34e0f50b8dbf6d60ac5f55b2261e93f3e414be8
--- /dev/null
+++ b/model-00039-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4846e39f398d04411859f18b2697f4e70e70ad33deeefa93135911d10c5a23e8
+size 4806799152
diff --git a/model-00040-of-00059.safetensors b/model-00040-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..15fff101657cec38d43132f4f32d3f1af7d6d23a
--- /dev/null
+++ b/model-00040-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b7de8e7af1d27c665f9e8dee5c904f9ee88ccd48c831361fc5999b979c16897
+size 4806799152
diff --git a/model-00041-of-00059.safetensors b/model-00041-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..82aa41dac4a60e802e3a9b8a9aa3144875af2fb8
--- /dev/null
+++ b/model-00041-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02c8320c607f11bbd377930a056873d84f0192d950176fc22e3c59d1dfb5995d
+size 4806799152
diff --git a/model-00042-of-00059.safetensors b/model-00042-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..dfd78c36da54f5eee061f9d7824d0868fd9b9449
--- /dev/null
+++ b/model-00042-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68a8a3b2780336a8012b795aa6295fd9248c322f939e8131315a520fe4e43480
+size 4806799152
diff --git a/model-00043-of-00059.safetensors b/model-00043-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..d2694ecd2a19042013f758568bafed9bfc7be514
--- /dev/null
+++ b/model-00043-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d859d466ef5da5d0cf036bc6ed57fa0033d15c3921eb042d1a31617c6f0eef34
+size 4806799152
diff --git a/model-00044-of-00059.safetensors b/model-00044-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ea9f123aa6488972038543d085def843b91d0c0d
--- /dev/null
+++ b/model-00044-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06d32c1ed652616e42fffdcce1735fad67b50a4f13397cfd030a4304ce035a08
+size 4806799152
diff --git a/model-00045-of-00059.safetensors b/model-00045-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..53e304034afbd8f2e9569ddd847e467130328a2a
--- /dev/null
+++ b/model-00045-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:beb9181e4b827f0136d213c11a05c569e0bcb85b7e68ade06b38c5d3da2d1ae0
+size 4806799152
diff --git a/model-00046-of-00059.safetensors b/model-00046-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..a8fc7814f022e6ad28b57b9f640459f332255cf8
--- /dev/null
+++ b/model-00046-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44a573a390174b0f0a41f8c0b198761384c46ae328a089f97d3e622468507224
+size 4806799152
diff --git a/model-00047-of-00059.safetensors b/model-00047-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..469950eeada25a3cfa075daaa30b1b594e72f128
--- /dev/null
+++ b/model-00047-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32a3bc20ff1805ae3e14e0bcf7ac3a1f9c21a31fb04922d3d31e440e658c32b6
+size 4806799152
diff --git a/model-00048-of-00059.safetensors b/model-00048-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..edb44292c8676ddfd369efea9085c8b2b2926d0e
--- /dev/null
+++ b/model-00048-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69a198153c9725aa0b6de50831b55fb0f04fc7a0f402731257836110358b7bde
+size 4806799152
diff --git a/model-00049-of-00059.safetensors b/model-00049-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..3a5470d3bf0541283fb3a8699e23001557b3aed4
--- /dev/null
+++ b/model-00049-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9d4adc81dbb3f12ec1300cb9c53bae070b7dbe41a66e30b5160841504fb5ba3
+size 4806799152
diff --git a/model-00050-of-00059.safetensors b/model-00050-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..1f833101874758b78c7d362c643cb5b2cf31ebf7
--- /dev/null
+++ b/model-00050-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d7cf6acb12c6907a102f9b836e3db345d2aa885493b3904084b95d0ac55c739
+size 4806799152
diff --git a/model-00051-of-00059.safetensors b/model-00051-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..0058aa76e955beded5e5ab6df13bd9c2f8bd9841
--- /dev/null
+++ b/model-00051-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:582322e446962d374f4b6fd5e469040d331150f7ef3d4a69fa460deded7c3ebc
+size 4806799152
diff --git a/model-00052-of-00059.safetensors b/model-00052-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ebf75a92b824415a0b718afc220ea1a683e76245
--- /dev/null
+++ b/model-00052-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6fe7d0d82aa12dd5f40c083b4046ac6ef15ed0627433d63795bed9e50c55fb4
+size 4932529864
diff --git a/model-00053-of-00059.safetensors b/model-00053-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..5428f8cf262699d785f4e8fca1db15f76b689cc2
--- /dev/null
+++ b/model-00053-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af0c588e8ffe2e87ce5dee116d70ec083a3bfd2f4c4a09618d8a2791de5cd55f
+size 4995542848
diff --git a/model-00054-of-00059.safetensors b/model-00054-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..1c6ac9afb113d9a41ec114bc86b1c520557277c1
--- /dev/null
+++ b/model-00054-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ecda36174ea4e376919360f6e08c17d5b3d0a870469e9fd52cac0b7c8e12351e
+size 4995542848
diff --git a/model-00055-of-00059.safetensors b/model-00055-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..7f6502a9db2118b1d144dfd3ba370017a8edf602
--- /dev/null
+++ b/model-00055-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d7c98c60576932850e8ef8945e38f24ca0f37cca895bf04c4a23d251081859c
+size 4932628288
diff --git a/model-00056-of-00059.safetensors b/model-00056-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..228bd4146c753ebced26f6b885afa807cc7a337f
--- /dev/null
+++ b/model-00056-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7410c9689fd0d2d384f4b2886707fb377327457937c91da914c725b7c35c2396
+size 4806774344
diff --git a/model-00057-of-00059.safetensors b/model-00057-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..5995fbb8665a3e0d03deead7883c73030add3f8d
--- /dev/null
+++ b/model-00057-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c684ae59038495c44d01507e8047511db57fa720cc9997f7719fba76dbc29ee
+size 4806799144
diff --git a/model-00058-of-00059.safetensors b/model-00058-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..fa80e37f5bb5478eb443722f452e2bbbb013a580
--- /dev/null
+++ b/model-00058-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1234554df96e1f1e2a7adcef5f680e7b57e9b6ae4f4a16ceff8225b20921c5d
+size 4806799144
diff --git a/model-00059-of-00059.safetensors b/model-00059-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..e57a9c39369c13ca99ce0bfa15bf60f1b1da0fd7
--- /dev/null
+++ b/model-00059-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e36fa9e3f053db7a27e06f5b2804a2aa650a026a66e3f7db9efd35c5103cd9ae
+size 997233472
diff --git a/model.safetensors.index.json b/model.safetensors.index.json
new file mode 100644
index 0000000000000000000000000000000000000000..805c18819ee3c4a51164b634df4f4033c74583ad
--- /dev/null
+++ b/model.safetensors.index.json
@@ -0,0 +1,1746 @@
+{
+ "metadata": {
+ "total_size": 281241268224
+ },
+ "weight_map": {
+ "lm_head.weight": "model-00059-of-00059.safetensors",
+ "model.embed_tokens.weight": "model-00001-of-00059.safetensors",
+ "model.layers.0.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00059.safetensors",
+ "model.layers.0.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00059.safetensors",
+ "model.layers.0.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00059.safetensors",
+ "model.layers.0.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00059.safetensors",
+ "model.layers.0.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00059.safetensors",
+ "model.layers.0.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00059.safetensors",
+ "model.layers.0.block_sparse_moe.experts.2.w1.weight": "model-00001-of-00059.safetensors",
+ "model.layers.0.block_sparse_moe.experts.2.w2.weight": "model-00001-of-00059.safetensors",
+ "model.layers.0.block_sparse_moe.experts.2.w3.weight": "model-00001-of-00059.safetensors",
+ "model.layers.0.block_sparse_moe.experts.3.w1.weight": "model-00001-of-00059.safetensors",
+ "model.layers.0.block_sparse_moe.experts.3.w2.weight": "model-00001-of-00059.safetensors",
+ "model.layers.0.block_sparse_moe.experts.3.w3.weight": "model-00001-of-00059.safetensors",
+ "model.layers.0.block_sparse_moe.experts.4.w1.weight": "model-00001-of-00059.safetensors",
+ "model.layers.0.block_sparse_moe.experts.4.w2.weight": "model-00001-of-00059.safetensors",
+ "model.layers.0.block_sparse_moe.experts.4.w3.weight": "model-00001-of-00059.safetensors",
+ "model.layers.0.block_sparse_moe.experts.5.w1.weight": "model-00001-of-00059.safetensors",
+ "model.layers.0.block_sparse_moe.experts.5.w2.weight": "model-00001-of-00059.safetensors",
+ "model.layers.0.block_sparse_moe.experts.5.w3.weight": "model-00001-of-00059.safetensors",
+ "model.layers.0.block_sparse_moe.experts.6.w1.weight": "model-00001-of-00059.safetensors",
+ "model.layers.0.block_sparse_moe.experts.6.w2.weight": "model-00001-of-00059.safetensors",
+ "model.layers.0.block_sparse_moe.experts.6.w3.weight": "model-00001-of-00059.safetensors",
+ "model.layers.0.block_sparse_moe.experts.7.w1.weight": "model-00001-of-00059.safetensors",
+ "model.layers.0.block_sparse_moe.experts.7.w2.weight": "model-00002-of-00059.safetensors",
+ "model.layers.0.block_sparse_moe.experts.7.w3.weight": "model-00002-of-00059.safetensors",
+ "model.layers.0.block_sparse_moe.gate.weight": "model-00001-of-00059.safetensors",
+ "model.layers.0.input_layernorm.weight": "model-00002-of-00059.safetensors",
+ "model.layers.0.post_attention_layernorm.weight": "model-00002-of-00059.safetensors",
+ "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00059.safetensors",
+ "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00059.safetensors",
+ "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00059.safetensors",
+ "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00059.safetensors",
+ "model.layers.1.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00059.safetensors",
+ "model.layers.1.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00059.safetensors",
+ "model.layers.1.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00059.safetensors",
+ "model.layers.1.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00059.safetensors",
+ "model.layers.1.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00059.safetensors",
+ "model.layers.1.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00059.safetensors",
+ "model.layers.1.block_sparse_moe.experts.2.w1.weight": "model-00002-of-00059.safetensors",
+ "model.layers.1.block_sparse_moe.experts.2.w2.weight": "model-00002-of-00059.safetensors",
+ "model.layers.1.block_sparse_moe.experts.2.w3.weight": "model-00002-of-00059.safetensors",
+ "model.layers.1.block_sparse_moe.experts.3.w1.weight": "model-00002-of-00059.safetensors",
+ "model.layers.1.block_sparse_moe.experts.3.w2.weight": "model-00002-of-00059.safetensors",
+ "model.layers.1.block_sparse_moe.experts.3.w3.weight": "model-00002-of-00059.safetensors",
+ "model.layers.1.block_sparse_moe.experts.4.w1.weight": "model-00002-of-00059.safetensors",
+ "model.layers.1.block_sparse_moe.experts.4.w2.weight": "model-00002-of-00059.safetensors",
+ "model.layers.1.block_sparse_moe.experts.4.w3.weight": "model-00002-of-00059.safetensors",
+ "model.layers.1.block_sparse_moe.experts.5.w1.weight": "model-00002-of-00059.safetensors",
+ "model.layers.1.block_sparse_moe.experts.5.w2.weight": "model-00002-of-00059.safetensors",
+ "model.layers.1.block_sparse_moe.experts.5.w3.weight": "model-00002-of-00059.safetensors",
+ "model.layers.1.block_sparse_moe.experts.6.w1.weight": "model-00002-of-00059.safetensors",
+ "model.layers.1.block_sparse_moe.experts.6.w2.weight": "model-00002-of-00059.safetensors",
+ "model.layers.1.block_sparse_moe.experts.6.w3.weight": "model-00002-of-00059.safetensors",
+ "model.layers.1.block_sparse_moe.experts.7.w1.weight": "model-00003-of-00059.safetensors",
+ "model.layers.1.block_sparse_moe.experts.7.w2.weight": "model-00003-of-00059.safetensors",
+ "model.layers.1.block_sparse_moe.experts.7.w3.weight": "model-00003-of-00059.safetensors",
+ "model.layers.1.block_sparse_moe.gate.weight": "model-00002-of-00059.safetensors",
+ "model.layers.1.input_layernorm.weight": "model-00003-of-00059.safetensors",
+ "model.layers.1.post_attention_layernorm.weight": "model-00003-of-00059.safetensors",
+ "model.layers.1.self_attn.k_proj.weight": "model-00002-of-00059.safetensors",
+ "model.layers.1.self_attn.o_proj.weight": "model-00002-of-00059.safetensors",
+ "model.layers.1.self_attn.q_proj.weight": "model-00002-of-00059.safetensors",
+ "model.layers.1.self_attn.v_proj.weight": "model-00002-of-00059.safetensors",
+ "model.layers.10.block_sparse_moe.experts.0.w1.weight": "model-00011-of-00059.safetensors",
+ "model.layers.10.block_sparse_moe.experts.0.w2.weight": "model-00011-of-00059.safetensors",
+ "model.layers.10.block_sparse_moe.experts.0.w3.weight": "model-00011-of-00059.safetensors",
+ "model.layers.10.block_sparse_moe.experts.1.w1.weight": "model-00011-of-00059.safetensors",
+ "model.layers.10.block_sparse_moe.experts.1.w2.weight": "model-00011-of-00059.safetensors",
+ "model.layers.10.block_sparse_moe.experts.1.w3.weight": "model-00011-of-00059.safetensors",
+ "model.layers.10.block_sparse_moe.experts.2.w1.weight": "model-00011-of-00059.safetensors",
+ "model.layers.10.block_sparse_moe.experts.2.w2.weight": "model-00011-of-00059.safetensors",
+ "model.layers.10.block_sparse_moe.experts.2.w3.weight": "model-00011-of-00059.safetensors",
+ "model.layers.10.block_sparse_moe.experts.3.w1.weight": "model-00011-of-00059.safetensors",
+ "model.layers.10.block_sparse_moe.experts.3.w2.weight": "model-00011-of-00059.safetensors",
+ "model.layers.10.block_sparse_moe.experts.3.w3.weight": "model-00011-of-00059.safetensors",
+ "model.layers.10.block_sparse_moe.experts.4.w1.weight": "model-00012-of-00059.safetensors",
+ "model.layers.10.block_sparse_moe.experts.4.w2.weight": "model-00012-of-00059.safetensors",
+ "model.layers.10.block_sparse_moe.experts.4.w3.weight": "model-00012-of-00059.safetensors",
+ "model.layers.10.block_sparse_moe.experts.5.w1.weight": "model-00012-of-00059.safetensors",
+ "model.layers.10.block_sparse_moe.experts.5.w2.weight": "model-00012-of-00059.safetensors",
+ "model.layers.10.block_sparse_moe.experts.5.w3.weight": "model-00012-of-00059.safetensors",
+ "model.layers.10.block_sparse_moe.experts.6.w1.weight": "model-00012-of-00059.safetensors",
+ "model.layers.10.block_sparse_moe.experts.6.w2.weight": "model-00012-of-00059.safetensors",
+ "model.layers.10.block_sparse_moe.experts.6.w3.weight": "model-00012-of-00059.safetensors",
+ "model.layers.10.block_sparse_moe.experts.7.w1.weight": "model-00012-of-00059.safetensors",
+ "model.layers.10.block_sparse_moe.experts.7.w2.weight": "model-00012-of-00059.safetensors",
+ "model.layers.10.block_sparse_moe.experts.7.w3.weight": "model-00012-of-00059.safetensors",
+ "model.layers.10.block_sparse_moe.gate.weight": "model-00011-of-00059.safetensors",
+ "model.layers.10.input_layernorm.weight": "model-00012-of-00059.safetensors",
+ "model.layers.10.post_attention_layernorm.weight": "model-00012-of-00059.safetensors",
+ "model.layers.10.self_attn.k_proj.weight": "model-00011-of-00059.safetensors",
+ "model.layers.10.self_attn.o_proj.weight": "model-00011-of-00059.safetensors",
+ "model.layers.10.self_attn.q_proj.weight": "model-00011-of-00059.safetensors",
+ "model.layers.10.self_attn.v_proj.weight": "model-00011-of-00059.safetensors",
+ "model.layers.11.block_sparse_moe.experts.0.w1.weight": "model-00012-of-00059.safetensors",
+ "model.layers.11.block_sparse_moe.experts.0.w2.weight": "model-00012-of-00059.safetensors",
+ "model.layers.11.block_sparse_moe.experts.0.w3.weight": "model-00012-of-00059.safetensors",
+ "model.layers.11.block_sparse_moe.experts.1.w1.weight": "model-00012-of-00059.safetensors",
+ "model.layers.11.block_sparse_moe.experts.1.w2.weight": "model-00012-of-00059.safetensors",
+ "model.layers.11.block_sparse_moe.experts.1.w3.weight": "model-00012-of-00059.safetensors",
+ "model.layers.11.block_sparse_moe.experts.2.w1.weight": "model-00012-of-00059.safetensors",
+ "model.layers.11.block_sparse_moe.experts.2.w2.weight": "model-00012-of-00059.safetensors",
+ "model.layers.11.block_sparse_moe.experts.2.w3.weight": "model-00012-of-00059.safetensors",
+ "model.layers.11.block_sparse_moe.experts.3.w1.weight": "model-00012-of-00059.safetensors",
+ "model.layers.11.block_sparse_moe.experts.3.w2.weight": "model-00012-of-00059.safetensors",
+ "model.layers.11.block_sparse_moe.experts.3.w3.weight": "model-00013-of-00059.safetensors",
+ "model.layers.11.block_sparse_moe.experts.4.w1.weight": "model-00013-of-00059.safetensors",
+ "model.layers.11.block_sparse_moe.experts.4.w2.weight": "model-00013-of-00059.safetensors",
+ "model.layers.11.block_sparse_moe.experts.4.w3.weight": "model-00013-of-00059.safetensors",
+ "model.layers.11.block_sparse_moe.experts.5.w1.weight": "model-00013-of-00059.safetensors",
+ "model.layers.11.block_sparse_moe.experts.5.w2.weight": "model-00013-of-00059.safetensors",
+ "model.layers.11.block_sparse_moe.experts.5.w3.weight": "model-00013-of-00059.safetensors",
+ "model.layers.11.block_sparse_moe.experts.6.w1.weight": "model-00013-of-00059.safetensors",
+ "model.layers.11.block_sparse_moe.experts.6.w2.weight": "model-00013-of-00059.safetensors",
+ "model.layers.11.block_sparse_moe.experts.6.w3.weight": "model-00013-of-00059.safetensors",
+ "model.layers.11.block_sparse_moe.experts.7.w1.weight": "model-00013-of-00059.safetensors",
+ "model.layers.11.block_sparse_moe.experts.7.w2.weight": "model-00013-of-00059.safetensors",
+ "model.layers.11.block_sparse_moe.experts.7.w3.weight": "model-00013-of-00059.safetensors",
+ "model.layers.11.block_sparse_moe.gate.weight": "model-00012-of-00059.safetensors",
+ "model.layers.11.input_layernorm.weight": "model-00013-of-00059.safetensors",
+ "model.layers.11.post_attention_layernorm.weight": "model-00013-of-00059.safetensors",
+ "model.layers.11.self_attn.k_proj.weight": "model-00012-of-00059.safetensors",
+ "model.layers.11.self_attn.o_proj.weight": "model-00012-of-00059.safetensors",
+ "model.layers.11.self_attn.q_proj.weight": "model-00012-of-00059.safetensors",
+ "model.layers.11.self_attn.v_proj.weight": "model-00012-of-00059.safetensors",
+ "model.layers.12.block_sparse_moe.experts.0.w1.weight": "model-00013-of-00059.safetensors",
+ "model.layers.12.block_sparse_moe.experts.0.w2.weight": "model-00013-of-00059.safetensors",
+ "model.layers.12.block_sparse_moe.experts.0.w3.weight": "model-00013-of-00059.safetensors",
+ "model.layers.12.block_sparse_moe.experts.1.w1.weight": "model-00013-of-00059.safetensors",
+ "model.layers.12.block_sparse_moe.experts.1.w2.weight": "model-00013-of-00059.safetensors",
+ "model.layers.12.block_sparse_moe.experts.1.w3.weight": "model-00013-of-00059.safetensors",
+ "model.layers.12.block_sparse_moe.experts.2.w1.weight": "model-00013-of-00059.safetensors",
+ "model.layers.12.block_sparse_moe.experts.2.w2.weight": "model-00013-of-00059.safetensors",
+ "model.layers.12.block_sparse_moe.experts.2.w3.weight": "model-00013-of-00059.safetensors",
+ "model.layers.12.block_sparse_moe.experts.3.w1.weight": "model-00013-of-00059.safetensors",
+ "model.layers.12.block_sparse_moe.experts.3.w2.weight": "model-00014-of-00059.safetensors",
+ "model.layers.12.block_sparse_moe.experts.3.w3.weight": "model-00014-of-00059.safetensors",
+ "model.layers.12.block_sparse_moe.experts.4.w1.weight": "model-00014-of-00059.safetensors",
+ "model.layers.12.block_sparse_moe.experts.4.w2.weight": "model-00014-of-00059.safetensors",
+ "model.layers.12.block_sparse_moe.experts.4.w3.weight": "model-00014-of-00059.safetensors",
+ "model.layers.12.block_sparse_moe.experts.5.w1.weight": "model-00014-of-00059.safetensors",
+ "model.layers.12.block_sparse_moe.experts.5.w2.weight": "model-00014-of-00059.safetensors",
+ "model.layers.12.block_sparse_moe.experts.5.w3.weight": "model-00014-of-00059.safetensors",
+ "model.layers.12.block_sparse_moe.experts.6.w1.weight": "model-00014-of-00059.safetensors",
+ "model.layers.12.block_sparse_moe.experts.6.w2.weight": "model-00014-of-00059.safetensors",
+ "model.layers.12.block_sparse_moe.experts.6.w3.weight": "model-00014-of-00059.safetensors",
+ "model.layers.12.block_sparse_moe.experts.7.w1.weight": "model-00014-of-00059.safetensors",
+ "model.layers.12.block_sparse_moe.experts.7.w2.weight": "model-00014-of-00059.safetensors",
+ "model.layers.12.block_sparse_moe.experts.7.w3.weight": "model-00014-of-00059.safetensors",
+ "model.layers.12.block_sparse_moe.gate.weight": "model-00013-of-00059.safetensors",
+ "model.layers.12.input_layernorm.weight": "model-00014-of-00059.safetensors",
+ "model.layers.12.post_attention_layernorm.weight": "model-00014-of-00059.safetensors",
+ "model.layers.12.self_attn.k_proj.weight": "model-00013-of-00059.safetensors",
+ "model.layers.12.self_attn.o_proj.weight": "model-00013-of-00059.safetensors",
+ "model.layers.12.self_attn.q_proj.weight": "model-00013-of-00059.safetensors",
+ "model.layers.12.self_attn.v_proj.weight": "model-00013-of-00059.safetensors",
+ "model.layers.13.block_sparse_moe.experts.0.w1.weight": "model-00014-of-00059.safetensors",
+ "model.layers.13.block_sparse_moe.experts.0.w2.weight": "model-00014-of-00059.safetensors",
+ "model.layers.13.block_sparse_moe.experts.0.w3.weight": "model-00014-of-00059.safetensors",
+ "model.layers.13.block_sparse_moe.experts.1.w1.weight": "model-00014-of-00059.safetensors",
+ "model.layers.13.block_sparse_moe.experts.1.w2.weight": "model-00014-of-00059.safetensors",
+ "model.layers.13.block_sparse_moe.experts.1.w3.weight": "model-00014-of-00059.safetensors",
+ "model.layers.13.block_sparse_moe.experts.2.w1.weight": "model-00014-of-00059.safetensors",
+ "model.layers.13.block_sparse_moe.experts.2.w2.weight": "model-00014-of-00059.safetensors",
+ "model.layers.13.block_sparse_moe.experts.2.w3.weight": "model-00014-of-00059.safetensors",
+ "model.layers.13.block_sparse_moe.experts.3.w1.weight": "model-00015-of-00059.safetensors",
+ "model.layers.13.block_sparse_moe.experts.3.w2.weight": "model-00015-of-00059.safetensors",
+ "model.layers.13.block_sparse_moe.experts.3.w3.weight": "model-00015-of-00059.safetensors",
+ "model.layers.13.block_sparse_moe.experts.4.w1.weight": "model-00015-of-00059.safetensors",
+ "model.layers.13.block_sparse_moe.experts.4.w2.weight": "model-00015-of-00059.safetensors",
+ "model.layers.13.block_sparse_moe.experts.4.w3.weight": "model-00015-of-00059.safetensors",
+ "model.layers.13.block_sparse_moe.experts.5.w1.weight": "model-00015-of-00059.safetensors",
+ "model.layers.13.block_sparse_moe.experts.5.w2.weight": "model-00015-of-00059.safetensors",
+ "model.layers.13.block_sparse_moe.experts.5.w3.weight": "model-00015-of-00059.safetensors",
+ "model.layers.13.block_sparse_moe.experts.6.w1.weight": "model-00015-of-00059.safetensors",
+ "model.layers.13.block_sparse_moe.experts.6.w2.weight": "model-00015-of-00059.safetensors",
+ "model.layers.13.block_sparse_moe.experts.6.w3.weight": "model-00015-of-00059.safetensors",
+ "model.layers.13.block_sparse_moe.experts.7.w1.weight": "model-00015-of-00059.safetensors",
+ "model.layers.13.block_sparse_moe.experts.7.w2.weight": "model-00015-of-00059.safetensors",
+ "model.layers.13.block_sparse_moe.experts.7.w3.weight": "model-00015-of-00059.safetensors",
+ "model.layers.13.block_sparse_moe.gate.weight": "model-00014-of-00059.safetensors",
+ "model.layers.13.input_layernorm.weight": "model-00015-of-00059.safetensors",
+ "model.layers.13.post_attention_layernorm.weight": "model-00015-of-00059.safetensors",
+ "model.layers.13.self_attn.k_proj.weight": "model-00014-of-00059.safetensors",
+ "model.layers.13.self_attn.o_proj.weight": "model-00014-of-00059.safetensors",
+ "model.layers.13.self_attn.q_proj.weight": "model-00014-of-00059.safetensors",
+ "model.layers.13.self_attn.v_proj.weight": "model-00014-of-00059.safetensors",
+ "model.layers.14.block_sparse_moe.experts.0.w1.weight": "model-00015-of-00059.safetensors",
+ "model.layers.14.block_sparse_moe.experts.0.w2.weight": "model-00015-of-00059.safetensors",
+ "model.layers.14.block_sparse_moe.experts.0.w3.weight": "model-00015-of-00059.safetensors",
+ "model.layers.14.block_sparse_moe.experts.1.w1.weight": "model-00015-of-00059.safetensors",
+ "model.layers.14.block_sparse_moe.experts.1.w2.weight": "model-00015-of-00059.safetensors",
+ "model.layers.14.block_sparse_moe.experts.1.w3.weight": "model-00015-of-00059.safetensors",
+ "model.layers.14.block_sparse_moe.experts.2.w1.weight": "model-00015-of-00059.safetensors",
+ "model.layers.14.block_sparse_moe.experts.2.w2.weight": "model-00015-of-00059.safetensors",
+ "model.layers.14.block_sparse_moe.experts.2.w3.weight": "model-00016-of-00059.safetensors",
+ "model.layers.14.block_sparse_moe.experts.3.w1.weight": "model-00016-of-00059.safetensors",
+ "model.layers.14.block_sparse_moe.experts.3.w2.weight": "model-00016-of-00059.safetensors",
+ "model.layers.14.block_sparse_moe.experts.3.w3.weight": "model-00016-of-00059.safetensors",
+ "model.layers.14.block_sparse_moe.experts.4.w1.weight": "model-00016-of-00059.safetensors",
+ "model.layers.14.block_sparse_moe.experts.4.w2.weight": "model-00016-of-00059.safetensors",
+ "model.layers.14.block_sparse_moe.experts.4.w3.weight": "model-00016-of-00059.safetensors",
+ "model.layers.14.block_sparse_moe.experts.5.w1.weight": "model-00016-of-00059.safetensors",
+ "model.layers.14.block_sparse_moe.experts.5.w2.weight": "model-00016-of-00059.safetensors",
+ "model.layers.14.block_sparse_moe.experts.5.w3.weight": "model-00016-of-00059.safetensors",
+ "model.layers.14.block_sparse_moe.experts.6.w1.weight": "model-00016-of-00059.safetensors",
+ "model.layers.14.block_sparse_moe.experts.6.w2.weight": "model-00016-of-00059.safetensors",
+ "model.layers.14.block_sparse_moe.experts.6.w3.weight": "model-00016-of-00059.safetensors",
+ "model.layers.14.block_sparse_moe.experts.7.w1.weight": "model-00016-of-00059.safetensors",
+ "model.layers.14.block_sparse_moe.experts.7.w2.weight": "model-00016-of-00059.safetensors",
+ "model.layers.14.block_sparse_moe.experts.7.w3.weight": "model-00016-of-00059.safetensors",
+ "model.layers.14.block_sparse_moe.gate.weight": "model-00015-of-00059.safetensors",
+ "model.layers.14.input_layernorm.weight": "model-00016-of-00059.safetensors",
+ "model.layers.14.post_attention_layernorm.weight": "model-00016-of-00059.safetensors",
+ "model.layers.14.self_attn.k_proj.weight": "model-00015-of-00059.safetensors",
+ "model.layers.14.self_attn.o_proj.weight": "model-00015-of-00059.safetensors",
+ "model.layers.14.self_attn.q_proj.weight": "model-00015-of-00059.safetensors",
+ "model.layers.14.self_attn.v_proj.weight": "model-00015-of-00059.safetensors",
+ "model.layers.15.block_sparse_moe.experts.0.w1.weight": "model-00016-of-00059.safetensors",
+ "model.layers.15.block_sparse_moe.experts.0.w2.weight": "model-00016-of-00059.safetensors",
+ "model.layers.15.block_sparse_moe.experts.0.w3.weight": "model-00016-of-00059.safetensors",
+ "model.layers.15.block_sparse_moe.experts.1.w1.weight": "model-00016-of-00059.safetensors",
+ "model.layers.15.block_sparse_moe.experts.1.w2.weight": "model-00016-of-00059.safetensors",
+ "model.layers.15.block_sparse_moe.experts.1.w3.weight": "model-00016-of-00059.safetensors",
+ "model.layers.15.block_sparse_moe.experts.2.w1.weight": "model-00016-of-00059.safetensors",
+ "model.layers.15.block_sparse_moe.experts.2.w2.weight": "model-00017-of-00059.safetensors",
+ "model.layers.15.block_sparse_moe.experts.2.w3.weight": "model-00017-of-00059.safetensors",
+ "model.layers.15.block_sparse_moe.experts.3.w1.weight": "model-00017-of-00059.safetensors",
+ "model.layers.15.block_sparse_moe.experts.3.w2.weight": "model-00017-of-00059.safetensors",
+ "model.layers.15.block_sparse_moe.experts.3.w3.weight": "model-00017-of-00059.safetensors",
+ "model.layers.15.block_sparse_moe.experts.4.w1.weight": "model-00017-of-00059.safetensors",
+ "model.layers.15.block_sparse_moe.experts.4.w2.weight": "model-00017-of-00059.safetensors",
+ "model.layers.15.block_sparse_moe.experts.4.w3.weight": "model-00017-of-00059.safetensors",
+ "model.layers.15.block_sparse_moe.experts.5.w1.weight": "model-00017-of-00059.safetensors",
+ "model.layers.15.block_sparse_moe.experts.5.w2.weight": "model-00017-of-00059.safetensors",
+ "model.layers.15.block_sparse_moe.experts.5.w3.weight": "model-00017-of-00059.safetensors",
+ "model.layers.15.block_sparse_moe.experts.6.w1.weight": "model-00017-of-00059.safetensors",
+ "model.layers.15.block_sparse_moe.experts.6.w2.weight": "model-00017-of-00059.safetensors",
+ "model.layers.15.block_sparse_moe.experts.6.w3.weight": "model-00017-of-00059.safetensors",
+ "model.layers.15.block_sparse_moe.experts.7.w1.weight": "model-00017-of-00059.safetensors",
+ "model.layers.15.block_sparse_moe.experts.7.w2.weight": "model-00017-of-00059.safetensors",
+ "model.layers.15.block_sparse_moe.experts.7.w3.weight": "model-00017-of-00059.safetensors",
+ "model.layers.15.block_sparse_moe.gate.weight": "model-00016-of-00059.safetensors",
+ "model.layers.15.input_layernorm.weight": "model-00017-of-00059.safetensors",
+ "model.layers.15.post_attention_layernorm.weight": "model-00017-of-00059.safetensors",
+ "model.layers.15.self_attn.k_proj.weight": "model-00016-of-00059.safetensors",
+ "model.layers.15.self_attn.o_proj.weight": "model-00016-of-00059.safetensors",
+ "model.layers.15.self_attn.q_proj.weight": "model-00016-of-00059.safetensors",
+ "model.layers.15.self_attn.v_proj.weight": "model-00016-of-00059.safetensors",
+ "model.layers.16.block_sparse_moe.experts.0.w1.weight": "model-00017-of-00059.safetensors",
+ "model.layers.16.block_sparse_moe.experts.0.w2.weight": "model-00017-of-00059.safetensors",
+ "model.layers.16.block_sparse_moe.experts.0.w3.weight": "model-00017-of-00059.safetensors",
+ "model.layers.16.block_sparse_moe.experts.1.w1.weight": "model-00017-of-00059.safetensors",
+ "model.layers.16.block_sparse_moe.experts.1.w2.weight": "model-00017-of-00059.safetensors",
+ "model.layers.16.block_sparse_moe.experts.1.w3.weight": "model-00017-of-00059.safetensors",
+ "model.layers.16.block_sparse_moe.experts.2.w1.weight": "model-00018-of-00059.safetensors",
+ "model.layers.16.block_sparse_moe.experts.2.w2.weight": "model-00018-of-00059.safetensors",
+ "model.layers.16.block_sparse_moe.experts.2.w3.weight": "model-00018-of-00059.safetensors",
+ "model.layers.16.block_sparse_moe.experts.3.w1.weight": "model-00018-of-00059.safetensors",
+ "model.layers.16.block_sparse_moe.experts.3.w2.weight": "model-00018-of-00059.safetensors",
+ "model.layers.16.block_sparse_moe.experts.3.w3.weight": "model-00018-of-00059.safetensors",
+ "model.layers.16.block_sparse_moe.experts.4.w1.weight": "model-00018-of-00059.safetensors",
+ "model.layers.16.block_sparse_moe.experts.4.w2.weight": "model-00018-of-00059.safetensors",
+ "model.layers.16.block_sparse_moe.experts.4.w3.weight": "model-00018-of-00059.safetensors",
+ "model.layers.16.block_sparse_moe.experts.5.w1.weight": "model-00018-of-00059.safetensors",
+ "model.layers.16.block_sparse_moe.experts.5.w2.weight": "model-00018-of-00059.safetensors",
+ "model.layers.16.block_sparse_moe.experts.5.w3.weight": "model-00018-of-00059.safetensors",
+ "model.layers.16.block_sparse_moe.experts.6.w1.weight": "model-00018-of-00059.safetensors",
+ "model.layers.16.block_sparse_moe.experts.6.w2.weight": "model-00018-of-00059.safetensors",
+ "model.layers.16.block_sparse_moe.experts.6.w3.weight": "model-00018-of-00059.safetensors",
+ "model.layers.16.block_sparse_moe.experts.7.w1.weight": "model-00018-of-00059.safetensors",
+ "model.layers.16.block_sparse_moe.experts.7.w2.weight": "model-00018-of-00059.safetensors",
+ "model.layers.16.block_sparse_moe.experts.7.w3.weight": "model-00018-of-00059.safetensors",
+ "model.layers.16.block_sparse_moe.gate.weight": "model-00017-of-00059.safetensors",
+ "model.layers.16.input_layernorm.weight": "model-00018-of-00059.safetensors",
+ "model.layers.16.post_attention_layernorm.weight": "model-00018-of-00059.safetensors",
+ "model.layers.16.self_attn.k_proj.weight": "model-00017-of-00059.safetensors",
+ "model.layers.16.self_attn.o_proj.weight": "model-00017-of-00059.safetensors",
+ "model.layers.16.self_attn.q_proj.weight": "model-00017-of-00059.safetensors",
+ "model.layers.16.self_attn.v_proj.weight": "model-00017-of-00059.safetensors",
+ "model.layers.17.block_sparse_moe.experts.0.w1.weight": "model-00018-of-00059.safetensors",
+ "model.layers.17.block_sparse_moe.experts.0.w2.weight": "model-00018-of-00059.safetensors",
+ "model.layers.17.block_sparse_moe.experts.0.w3.weight": "model-00018-of-00059.safetensors",
+ "model.layers.17.block_sparse_moe.experts.1.w1.weight": "model-00018-of-00059.safetensors",
+ "model.layers.17.block_sparse_moe.experts.1.w2.weight": "model-00018-of-00059.safetensors",
+ "model.layers.17.block_sparse_moe.experts.1.w3.weight": "model-00019-of-00059.safetensors",
+ "model.layers.17.block_sparse_moe.experts.2.w1.weight": "model-00019-of-00059.safetensors",
+ "model.layers.17.block_sparse_moe.experts.2.w2.weight": "model-00019-of-00059.safetensors",
+ "model.layers.17.block_sparse_moe.experts.2.w3.weight": "model-00019-of-00059.safetensors",
+ "model.layers.17.block_sparse_moe.experts.3.w1.weight": "model-00019-of-00059.safetensors",
+ "model.layers.17.block_sparse_moe.experts.3.w2.weight": "model-00019-of-00059.safetensors",
+ "model.layers.17.block_sparse_moe.experts.3.w3.weight": "model-00019-of-00059.safetensors",
+ "model.layers.17.block_sparse_moe.experts.4.w1.weight": "model-00019-of-00059.safetensors",
+ "model.layers.17.block_sparse_moe.experts.4.w2.weight": "model-00019-of-00059.safetensors",
+ "model.layers.17.block_sparse_moe.experts.4.w3.weight": "model-00019-of-00059.safetensors",
+ "model.layers.17.block_sparse_moe.experts.5.w1.weight": "model-00019-of-00059.safetensors",
+ "model.layers.17.block_sparse_moe.experts.5.w2.weight": "model-00019-of-00059.safetensors",
+ "model.layers.17.block_sparse_moe.experts.5.w3.weight": "model-00019-of-00059.safetensors",
+ "model.layers.17.block_sparse_moe.experts.6.w1.weight": "model-00019-of-00059.safetensors",
+ "model.layers.17.block_sparse_moe.experts.6.w2.weight": "model-00019-of-00059.safetensors",
+ "model.layers.17.block_sparse_moe.experts.6.w3.weight": "model-00019-of-00059.safetensors",
+ "model.layers.17.block_sparse_moe.experts.7.w1.weight": "model-00019-of-00059.safetensors",
+ "model.layers.17.block_sparse_moe.experts.7.w2.weight": "model-00019-of-00059.safetensors",
+ "model.layers.17.block_sparse_moe.experts.7.w3.weight": "model-00019-of-00059.safetensors",
+ "model.layers.17.block_sparse_moe.gate.weight": "model-00018-of-00059.safetensors",
+ "model.layers.17.input_layernorm.weight": "model-00019-of-00059.safetensors",
+ "model.layers.17.post_attention_layernorm.weight": "model-00019-of-00059.safetensors",
+ "model.layers.17.self_attn.k_proj.weight": "model-00018-of-00059.safetensors",
+ "model.layers.17.self_attn.o_proj.weight": "model-00018-of-00059.safetensors",
+ "model.layers.17.self_attn.q_proj.weight": "model-00018-of-00059.safetensors",
+ "model.layers.17.self_attn.v_proj.weight": "model-00018-of-00059.safetensors",
+ "model.layers.18.block_sparse_moe.experts.0.w1.weight": "model-00019-of-00059.safetensors",
+ "model.layers.18.block_sparse_moe.experts.0.w2.weight": "model-00019-of-00059.safetensors",
+ "model.layers.18.block_sparse_moe.experts.0.w3.weight": "model-00019-of-00059.safetensors",
+ "model.layers.18.block_sparse_moe.experts.1.w1.weight": "model-00019-of-00059.safetensors",
+ "model.layers.18.block_sparse_moe.experts.1.w2.weight": "model-00020-of-00059.safetensors",
+ "model.layers.18.block_sparse_moe.experts.1.w3.weight": "model-00020-of-00059.safetensors",
+ "model.layers.18.block_sparse_moe.experts.2.w1.weight": "model-00020-of-00059.safetensors",
+ "model.layers.18.block_sparse_moe.experts.2.w2.weight": "model-00020-of-00059.safetensors",
+ "model.layers.18.block_sparse_moe.experts.2.w3.weight": "model-00020-of-00059.safetensors",
+ "model.layers.18.block_sparse_moe.experts.3.w1.weight": "model-00020-of-00059.safetensors",
+ "model.layers.18.block_sparse_moe.experts.3.w2.weight": "model-00020-of-00059.safetensors",
+ "model.layers.18.block_sparse_moe.experts.3.w3.weight": "model-00020-of-00059.safetensors",
+ "model.layers.18.block_sparse_moe.experts.4.w1.weight": "model-00020-of-00059.safetensors",
+ "model.layers.18.block_sparse_moe.experts.4.w2.weight": "model-00020-of-00059.safetensors",
+ "model.layers.18.block_sparse_moe.experts.4.w3.weight": "model-00020-of-00059.safetensors",
+ "model.layers.18.block_sparse_moe.experts.5.w1.weight": "model-00020-of-00059.safetensors",
+ "model.layers.18.block_sparse_moe.experts.5.w2.weight": "model-00020-of-00059.safetensors",
+ "model.layers.18.block_sparse_moe.experts.5.w3.weight": "model-00020-of-00059.safetensors",
+ "model.layers.18.block_sparse_moe.experts.6.w1.weight": "model-00020-of-00059.safetensors",
+ "model.layers.18.block_sparse_moe.experts.6.w2.weight": "model-00020-of-00059.safetensors",
+ "model.layers.18.block_sparse_moe.experts.6.w3.weight": "model-00020-of-00059.safetensors",
+ "model.layers.18.block_sparse_moe.experts.7.w1.weight": "model-00020-of-00059.safetensors",
+ "model.layers.18.block_sparse_moe.experts.7.w2.weight": "model-00020-of-00059.safetensors",
+ "model.layers.18.block_sparse_moe.experts.7.w3.weight": "model-00020-of-00059.safetensors",
+ "model.layers.18.block_sparse_moe.gate.weight": "model-00019-of-00059.safetensors",
+ "model.layers.18.input_layernorm.weight": "model-00020-of-00059.safetensors",
+ "model.layers.18.post_attention_layernorm.weight": "model-00020-of-00059.safetensors",
+ "model.layers.18.self_attn.k_proj.weight": "model-00019-of-00059.safetensors",
+ "model.layers.18.self_attn.o_proj.weight": "model-00019-of-00059.safetensors",
+ "model.layers.18.self_attn.q_proj.weight": "model-00019-of-00059.safetensors",
+ "model.layers.18.self_attn.v_proj.weight": "model-00019-of-00059.safetensors",
+ "model.layers.19.block_sparse_moe.experts.0.w1.weight": "model-00020-of-00059.safetensors",
+ "model.layers.19.block_sparse_moe.experts.0.w2.weight": "model-00020-of-00059.safetensors",
+ "model.layers.19.block_sparse_moe.experts.0.w3.weight": "model-00020-of-00059.safetensors",
+ "model.layers.19.block_sparse_moe.experts.1.w1.weight": "model-00021-of-00059.safetensors",
+ "model.layers.19.block_sparse_moe.experts.1.w2.weight": "model-00021-of-00059.safetensors",
+ "model.layers.19.block_sparse_moe.experts.1.w3.weight": "model-00021-of-00059.safetensors",
+ "model.layers.19.block_sparse_moe.experts.2.w1.weight": "model-00021-of-00059.safetensors",
+ "model.layers.19.block_sparse_moe.experts.2.w2.weight": "model-00021-of-00059.safetensors",
+ "model.layers.19.block_sparse_moe.experts.2.w3.weight": "model-00021-of-00059.safetensors",
+ "model.layers.19.block_sparse_moe.experts.3.w1.weight": "model-00021-of-00059.safetensors",
+ "model.layers.19.block_sparse_moe.experts.3.w2.weight": "model-00021-of-00059.safetensors",
+ "model.layers.19.block_sparse_moe.experts.3.w3.weight": "model-00021-of-00059.safetensors",
+ "model.layers.19.block_sparse_moe.experts.4.w1.weight": "model-00021-of-00059.safetensors",
+ "model.layers.19.block_sparse_moe.experts.4.w2.weight": "model-00021-of-00059.safetensors",
+ "model.layers.19.block_sparse_moe.experts.4.w3.weight": "model-00021-of-00059.safetensors",
+ "model.layers.19.block_sparse_moe.experts.5.w1.weight": "model-00021-of-00059.safetensors",
+ "model.layers.19.block_sparse_moe.experts.5.w2.weight": "model-00021-of-00059.safetensors",
+ "model.layers.19.block_sparse_moe.experts.5.w3.weight": "model-00021-of-00059.safetensors",
+ "model.layers.19.block_sparse_moe.experts.6.w1.weight": "model-00021-of-00059.safetensors",
+ "model.layers.19.block_sparse_moe.experts.6.w2.weight": "model-00021-of-00059.safetensors",
+ "model.layers.19.block_sparse_moe.experts.6.w3.weight": "model-00021-of-00059.safetensors",
+ "model.layers.19.block_sparse_moe.experts.7.w1.weight": "model-00021-of-00059.safetensors",
+ "model.layers.19.block_sparse_moe.experts.7.w2.weight": "model-00021-of-00059.safetensors",
+ "model.layers.19.block_sparse_moe.experts.7.w3.weight": "model-00021-of-00059.safetensors",
+ "model.layers.19.block_sparse_moe.gate.weight": "model-00020-of-00059.safetensors",
+ "model.layers.19.input_layernorm.weight": "model-00021-of-00059.safetensors",
+ "model.layers.19.post_attention_layernorm.weight": "model-00021-of-00059.safetensors",
+ "model.layers.19.self_attn.k_proj.weight": "model-00020-of-00059.safetensors",
+ "model.layers.19.self_attn.o_proj.weight": "model-00020-of-00059.safetensors",
+ "model.layers.19.self_attn.q_proj.weight": "model-00020-of-00059.safetensors",
+ "model.layers.19.self_attn.v_proj.weight": "model-00020-of-00059.safetensors",
+ "model.layers.2.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00059.safetensors",
+ "model.layers.2.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00059.safetensors",
+ "model.layers.2.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00059.safetensors",
+ "model.layers.2.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00059.safetensors",
+ "model.layers.2.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00059.safetensors",
+ "model.layers.2.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00059.safetensors",
+ "model.layers.2.block_sparse_moe.experts.2.w1.weight": "model-00003-of-00059.safetensors",
+ "model.layers.2.block_sparse_moe.experts.2.w2.weight": "model-00003-of-00059.safetensors",
+ "model.layers.2.block_sparse_moe.experts.2.w3.weight": "model-00003-of-00059.safetensors",
+ "model.layers.2.block_sparse_moe.experts.3.w1.weight": "model-00003-of-00059.safetensors",
+ "model.layers.2.block_sparse_moe.experts.3.w2.weight": "model-00003-of-00059.safetensors",
+ "model.layers.2.block_sparse_moe.experts.3.w3.weight": "model-00003-of-00059.safetensors",
+ "model.layers.2.block_sparse_moe.experts.4.w1.weight": "model-00003-of-00059.safetensors",
+ "model.layers.2.block_sparse_moe.experts.4.w2.weight": "model-00003-of-00059.safetensors",
+ "model.layers.2.block_sparse_moe.experts.4.w3.weight": "model-00003-of-00059.safetensors",
+ "model.layers.2.block_sparse_moe.experts.5.w1.weight": "model-00003-of-00059.safetensors",
+ "model.layers.2.block_sparse_moe.experts.5.w2.weight": "model-00003-of-00059.safetensors",
+ "model.layers.2.block_sparse_moe.experts.5.w3.weight": "model-00003-of-00059.safetensors",
+ "model.layers.2.block_sparse_moe.experts.6.w1.weight": "model-00003-of-00059.safetensors",
+ "model.layers.2.block_sparse_moe.experts.6.w2.weight": "model-00003-of-00059.safetensors",
+ "model.layers.2.block_sparse_moe.experts.6.w3.weight": "model-00004-of-00059.safetensors",
+ "model.layers.2.block_sparse_moe.experts.7.w1.weight": "model-00004-of-00059.safetensors",
+ "model.layers.2.block_sparse_moe.experts.7.w2.weight": "model-00004-of-00059.safetensors",
+ "model.layers.2.block_sparse_moe.experts.7.w3.weight": "model-00004-of-00059.safetensors",
+ "model.layers.2.block_sparse_moe.gate.weight": "model-00003-of-00059.safetensors",
+ "model.layers.2.input_layernorm.weight": "model-00004-of-00059.safetensors",
+ "model.layers.2.post_attention_layernorm.weight": "model-00004-of-00059.safetensors",
+ "model.layers.2.self_attn.k_proj.weight": "model-00003-of-00059.safetensors",
+ "model.layers.2.self_attn.o_proj.weight": "model-00003-of-00059.safetensors",
+ "model.layers.2.self_attn.q_proj.weight": "model-00003-of-00059.safetensors",
+ "model.layers.2.self_attn.v_proj.weight": "model-00003-of-00059.safetensors",
+ "model.layers.20.block_sparse_moe.experts.0.w1.weight": "model-00021-of-00059.safetensors",
+ "model.layers.20.block_sparse_moe.experts.0.w2.weight": "model-00021-of-00059.safetensors",
+ "model.layers.20.block_sparse_moe.experts.0.w3.weight": "model-00022-of-00059.safetensors",
+ "model.layers.20.block_sparse_moe.experts.1.w1.weight": "model-00022-of-00059.safetensors",
+ "model.layers.20.block_sparse_moe.experts.1.w2.weight": "model-00022-of-00059.safetensors",
+ "model.layers.20.block_sparse_moe.experts.1.w3.weight": "model-00022-of-00059.safetensors",
+ "model.layers.20.block_sparse_moe.experts.2.w1.weight": "model-00022-of-00059.safetensors",
+ "model.layers.20.block_sparse_moe.experts.2.w2.weight": "model-00022-of-00059.safetensors",
+ "model.layers.20.block_sparse_moe.experts.2.w3.weight": "model-00022-of-00059.safetensors",
+ "model.layers.20.block_sparse_moe.experts.3.w1.weight": "model-00022-of-00059.safetensors",
+ "model.layers.20.block_sparse_moe.experts.3.w2.weight": "model-00022-of-00059.safetensors",
+ "model.layers.20.block_sparse_moe.experts.3.w3.weight": "model-00022-of-00059.safetensors",
+ "model.layers.20.block_sparse_moe.experts.4.w1.weight": "model-00022-of-00059.safetensors",
+ "model.layers.20.block_sparse_moe.experts.4.w2.weight": "model-00022-of-00059.safetensors",
+ "model.layers.20.block_sparse_moe.experts.4.w3.weight": "model-00022-of-00059.safetensors",
+ "model.layers.20.block_sparse_moe.experts.5.w1.weight": "model-00022-of-00059.safetensors",
+ "model.layers.20.block_sparse_moe.experts.5.w2.weight": "model-00022-of-00059.safetensors",
+ "model.layers.20.block_sparse_moe.experts.5.w3.weight": "model-00022-of-00059.safetensors",
+ "model.layers.20.block_sparse_moe.experts.6.w1.weight": "model-00022-of-00059.safetensors",
+ "model.layers.20.block_sparse_moe.experts.6.w2.weight": "model-00022-of-00059.safetensors",
+ "model.layers.20.block_sparse_moe.experts.6.w3.weight": "model-00022-of-00059.safetensors",
+ "model.layers.20.block_sparse_moe.experts.7.w1.weight": "model-00022-of-00059.safetensors",
+ "model.layers.20.block_sparse_moe.experts.7.w2.weight": "model-00022-of-00059.safetensors",
+ "model.layers.20.block_sparse_moe.experts.7.w3.weight": "model-00022-of-00059.safetensors",
+ "model.layers.20.block_sparse_moe.gate.weight": "model-00021-of-00059.safetensors",
+ "model.layers.20.input_layernorm.weight": "model-00022-of-00059.safetensors",
+ "model.layers.20.post_attention_layernorm.weight": "model-00022-of-00059.safetensors",
+ "model.layers.20.self_attn.k_proj.weight": "model-00021-of-00059.safetensors",
+ "model.layers.20.self_attn.o_proj.weight": "model-00021-of-00059.safetensors",
+ "model.layers.20.self_attn.q_proj.weight": "model-00021-of-00059.safetensors",
+ "model.layers.20.self_attn.v_proj.weight": "model-00021-of-00059.safetensors",
+ "model.layers.21.block_sparse_moe.experts.0.w1.weight": "model-00022-of-00059.safetensors",
+ "model.layers.21.block_sparse_moe.experts.0.w2.weight": "model-00023-of-00059.safetensors",
+ "model.layers.21.block_sparse_moe.experts.0.w3.weight": "model-00023-of-00059.safetensors",
+ "model.layers.21.block_sparse_moe.experts.1.w1.weight": "model-00023-of-00059.safetensors",
+ "model.layers.21.block_sparse_moe.experts.1.w2.weight": "model-00023-of-00059.safetensors",
+ "model.layers.21.block_sparse_moe.experts.1.w3.weight": "model-00023-of-00059.safetensors",
+ "model.layers.21.block_sparse_moe.experts.2.w1.weight": "model-00023-of-00059.safetensors",
+ "model.layers.21.block_sparse_moe.experts.2.w2.weight": "model-00023-of-00059.safetensors",
+ "model.layers.21.block_sparse_moe.experts.2.w3.weight": "model-00023-of-00059.safetensors",
+ "model.layers.21.block_sparse_moe.experts.3.w1.weight": "model-00023-of-00059.safetensors",
+ "model.layers.21.block_sparse_moe.experts.3.w2.weight": "model-00023-of-00059.safetensors",
+ "model.layers.21.block_sparse_moe.experts.3.w3.weight": "model-00023-of-00059.safetensors",
+ "model.layers.21.block_sparse_moe.experts.4.w1.weight": "model-00023-of-00059.safetensors",
+ "model.layers.21.block_sparse_moe.experts.4.w2.weight": "model-00023-of-00059.safetensors",
+ "model.layers.21.block_sparse_moe.experts.4.w3.weight": "model-00023-of-00059.safetensors",
+ "model.layers.21.block_sparse_moe.experts.5.w1.weight": "model-00023-of-00059.safetensors",
+ "model.layers.21.block_sparse_moe.experts.5.w2.weight": "model-00023-of-00059.safetensors",
+ "model.layers.21.block_sparse_moe.experts.5.w3.weight": "model-00023-of-00059.safetensors",
+ "model.layers.21.block_sparse_moe.experts.6.w1.weight": "model-00023-of-00059.safetensors",
+ "model.layers.21.block_sparse_moe.experts.6.w2.weight": "model-00023-of-00059.safetensors",
+ "model.layers.21.block_sparse_moe.experts.6.w3.weight": "model-00023-of-00059.safetensors",
+ "model.layers.21.block_sparse_moe.experts.7.w1.weight": "model-00023-of-00059.safetensors",
+ "model.layers.21.block_sparse_moe.experts.7.w2.weight": "model-00023-of-00059.safetensors",
+ "model.layers.21.block_sparse_moe.experts.7.w3.weight": "model-00023-of-00059.safetensors",
+ "model.layers.21.block_sparse_moe.gate.weight": "model-00022-of-00059.safetensors",
+ "model.layers.21.input_layernorm.weight": "model-00023-of-00059.safetensors",
+ "model.layers.21.post_attention_layernorm.weight": "model-00023-of-00059.safetensors",
+ "model.layers.21.self_attn.k_proj.weight": "model-00022-of-00059.safetensors",
+ "model.layers.21.self_attn.o_proj.weight": "model-00022-of-00059.safetensors",
+ "model.layers.21.self_attn.q_proj.weight": "model-00022-of-00059.safetensors",
+ "model.layers.21.self_attn.v_proj.weight": "model-00022-of-00059.safetensors",
+ "model.layers.22.block_sparse_moe.experts.0.w1.weight": "model-00024-of-00059.safetensors",
+ "model.layers.22.block_sparse_moe.experts.0.w2.weight": "model-00024-of-00059.safetensors",
+ "model.layers.22.block_sparse_moe.experts.0.w3.weight": "model-00024-of-00059.safetensors",
+ "model.layers.22.block_sparse_moe.experts.1.w1.weight": "model-00024-of-00059.safetensors",
+ "model.layers.22.block_sparse_moe.experts.1.w2.weight": "model-00024-of-00059.safetensors",
+ "model.layers.22.block_sparse_moe.experts.1.w3.weight": "model-00024-of-00059.safetensors",
+ "model.layers.22.block_sparse_moe.experts.2.w1.weight": "model-00024-of-00059.safetensors",
+ "model.layers.22.block_sparse_moe.experts.2.w2.weight": "model-00024-of-00059.safetensors",
+ "model.layers.22.block_sparse_moe.experts.2.w3.weight": "model-00024-of-00059.safetensors",
+ "model.layers.22.block_sparse_moe.experts.3.w1.weight": "model-00024-of-00059.safetensors",
+ "model.layers.22.block_sparse_moe.experts.3.w2.weight": "model-00024-of-00059.safetensors",
+ "model.layers.22.block_sparse_moe.experts.3.w3.weight": "model-00024-of-00059.safetensors",
+ "model.layers.22.block_sparse_moe.experts.4.w1.weight": "model-00024-of-00059.safetensors",
+ "model.layers.22.block_sparse_moe.experts.4.w2.weight": "model-00024-of-00059.safetensors",
+ "model.layers.22.block_sparse_moe.experts.4.w3.weight": "model-00024-of-00059.safetensors",
+ "model.layers.22.block_sparse_moe.experts.5.w1.weight": "model-00024-of-00059.safetensors",
+ "model.layers.22.block_sparse_moe.experts.5.w2.weight": "model-00024-of-00059.safetensors",
+ "model.layers.22.block_sparse_moe.experts.5.w3.weight": "model-00024-of-00059.safetensors",
+ "model.layers.22.block_sparse_moe.experts.6.w1.weight": "model-00024-of-00059.safetensors",
+ "model.layers.22.block_sparse_moe.experts.6.w2.weight": "model-00024-of-00059.safetensors",
+ "model.layers.22.block_sparse_moe.experts.6.w3.weight": "model-00024-of-00059.safetensors",
+ "model.layers.22.block_sparse_moe.experts.7.w1.weight": "model-00024-of-00059.safetensors",
+ "model.layers.22.block_sparse_moe.experts.7.w2.weight": "model-00024-of-00059.safetensors",
+ "model.layers.22.block_sparse_moe.experts.7.w3.weight": "model-00024-of-00059.safetensors",
+ "model.layers.22.block_sparse_moe.gate.weight": "model-00023-of-00059.safetensors",
+ "model.layers.22.input_layernorm.weight": "model-00024-of-00059.safetensors",
+ "model.layers.22.post_attention_layernorm.weight": "model-00024-of-00059.safetensors",
+ "model.layers.22.self_attn.k_proj.weight": "model-00023-of-00059.safetensors",
+ "model.layers.22.self_attn.o_proj.weight": "model-00023-of-00059.safetensors",
+ "model.layers.22.self_attn.q_proj.weight": "model-00023-of-00059.safetensors",
+ "model.layers.22.self_attn.v_proj.weight": "model-00023-of-00059.safetensors",
+ "model.layers.23.block_sparse_moe.experts.0.w1.weight": "model-00025-of-00059.safetensors",
+ "model.layers.23.block_sparse_moe.experts.0.w2.weight": "model-00025-of-00059.safetensors",
+ "model.layers.23.block_sparse_moe.experts.0.w3.weight": "model-00025-of-00059.safetensors",
+ "model.layers.23.block_sparse_moe.experts.1.w1.weight": "model-00025-of-00059.safetensors",
+ "model.layers.23.block_sparse_moe.experts.1.w2.weight": "model-00025-of-00059.safetensors",
+ "model.layers.23.block_sparse_moe.experts.1.w3.weight": "model-00025-of-00059.safetensors",
+ "model.layers.23.block_sparse_moe.experts.2.w1.weight": "model-00025-of-00059.safetensors",
+ "model.layers.23.block_sparse_moe.experts.2.w2.weight": "model-00025-of-00059.safetensors",
+ "model.layers.23.block_sparse_moe.experts.2.w3.weight": "model-00025-of-00059.safetensors",
+ "model.layers.23.block_sparse_moe.experts.3.w1.weight": "model-00025-of-00059.safetensors",
+ "model.layers.23.block_sparse_moe.experts.3.w2.weight": "model-00025-of-00059.safetensors",
+ "model.layers.23.block_sparse_moe.experts.3.w3.weight": "model-00025-of-00059.safetensors",
+ "model.layers.23.block_sparse_moe.experts.4.w1.weight": "model-00025-of-00059.safetensors",
+ "model.layers.23.block_sparse_moe.experts.4.w2.weight": "model-00025-of-00059.safetensors",
+ "model.layers.23.block_sparse_moe.experts.4.w3.weight": "model-00025-of-00059.safetensors",
+ "model.layers.23.block_sparse_moe.experts.5.w1.weight": "model-00025-of-00059.safetensors",
+ "model.layers.23.block_sparse_moe.experts.5.w2.weight": "model-00025-of-00059.safetensors",
+ "model.layers.23.block_sparse_moe.experts.5.w3.weight": "model-00025-of-00059.safetensors",
+ "model.layers.23.block_sparse_moe.experts.6.w1.weight": "model-00025-of-00059.safetensors",
+ "model.layers.23.block_sparse_moe.experts.6.w2.weight": "model-00025-of-00059.safetensors",
+ "model.layers.23.block_sparse_moe.experts.6.w3.weight": "model-00025-of-00059.safetensors",
+ "model.layers.23.block_sparse_moe.experts.7.w1.weight": "model-00025-of-00059.safetensors",
+ "model.layers.23.block_sparse_moe.experts.7.w2.weight": "model-00025-of-00059.safetensors",
+ "model.layers.23.block_sparse_moe.experts.7.w3.weight": "model-00025-of-00059.safetensors",
+ "model.layers.23.block_sparse_moe.gate.weight": "model-00025-of-00059.safetensors",
+ "model.layers.23.input_layernorm.weight": "model-00025-of-00059.safetensors",
+ "model.layers.23.post_attention_layernorm.weight": "model-00025-of-00059.safetensors",
+ "model.layers.23.self_attn.k_proj.weight": "model-00024-of-00059.safetensors",
+ "model.layers.23.self_attn.o_proj.weight": "model-00025-of-00059.safetensors",
+ "model.layers.23.self_attn.q_proj.weight": "model-00024-of-00059.safetensors",
+ "model.layers.23.self_attn.v_proj.weight": "model-00024-of-00059.safetensors",
+ "model.layers.24.block_sparse_moe.experts.0.w1.weight": "model-00026-of-00059.safetensors",
+ "model.layers.24.block_sparse_moe.experts.0.w2.weight": "model-00026-of-00059.safetensors",
+ "model.layers.24.block_sparse_moe.experts.0.w3.weight": "model-00026-of-00059.safetensors",
+ "model.layers.24.block_sparse_moe.experts.1.w1.weight": "model-00026-of-00059.safetensors",
+ "model.layers.24.block_sparse_moe.experts.1.w2.weight": "model-00026-of-00059.safetensors",
+ "model.layers.24.block_sparse_moe.experts.1.w3.weight": "model-00026-of-00059.safetensors",
+ "model.layers.24.block_sparse_moe.experts.2.w1.weight": "model-00026-of-00059.safetensors",
+ "model.layers.24.block_sparse_moe.experts.2.w2.weight": "model-00026-of-00059.safetensors",
+ "model.layers.24.block_sparse_moe.experts.2.w3.weight": "model-00026-of-00059.safetensors",
+ "model.layers.24.block_sparse_moe.experts.3.w1.weight": "model-00026-of-00059.safetensors",
+ "model.layers.24.block_sparse_moe.experts.3.w2.weight": "model-00026-of-00059.safetensors",
+ "model.layers.24.block_sparse_moe.experts.3.w3.weight": "model-00026-of-00059.safetensors",
+ "model.layers.24.block_sparse_moe.experts.4.w1.weight": "model-00026-of-00059.safetensors",
+ "model.layers.24.block_sparse_moe.experts.4.w2.weight": "model-00026-of-00059.safetensors",
+ "model.layers.24.block_sparse_moe.experts.4.w3.weight": "model-00026-of-00059.safetensors",
+ "model.layers.24.block_sparse_moe.experts.5.w1.weight": "model-00026-of-00059.safetensors",
+ "model.layers.24.block_sparse_moe.experts.5.w2.weight": "model-00026-of-00059.safetensors",
+ "model.layers.24.block_sparse_moe.experts.5.w3.weight": "model-00026-of-00059.safetensors",
+ "model.layers.24.block_sparse_moe.experts.6.w1.weight": "model-00026-of-00059.safetensors",
+ "model.layers.24.block_sparse_moe.experts.6.w2.weight": "model-00026-of-00059.safetensors",
+ "model.layers.24.block_sparse_moe.experts.6.w3.weight": "model-00026-of-00059.safetensors",
+ "model.layers.24.block_sparse_moe.experts.7.w1.weight": "model-00026-of-00059.safetensors",
+ "model.layers.24.block_sparse_moe.experts.7.w2.weight": "model-00026-of-00059.safetensors",
+ "model.layers.24.block_sparse_moe.experts.7.w3.weight": "model-00026-of-00059.safetensors",
+ "model.layers.24.block_sparse_moe.gate.weight": "model-00026-of-00059.safetensors",
+ "model.layers.24.input_layernorm.weight": "model-00026-of-00059.safetensors",
+ "model.layers.24.post_attention_layernorm.weight": "model-00026-of-00059.safetensors",
+ "model.layers.24.self_attn.k_proj.weight": "model-00025-of-00059.safetensors",
+ "model.layers.24.self_attn.o_proj.weight": "model-00026-of-00059.safetensors",
+ "model.layers.24.self_attn.q_proj.weight": "model-00025-of-00059.safetensors",
+ "model.layers.24.self_attn.v_proj.weight": "model-00026-of-00059.safetensors",
+ "model.layers.25.block_sparse_moe.experts.0.w1.weight": "model-00027-of-00059.safetensors",
+ "model.layers.25.block_sparse_moe.experts.0.w2.weight": "model-00027-of-00059.safetensors",
+ "model.layers.25.block_sparse_moe.experts.0.w3.weight": "model-00027-of-00059.safetensors",
+ "model.layers.25.block_sparse_moe.experts.1.w1.weight": "model-00027-of-00059.safetensors",
+ "model.layers.25.block_sparse_moe.experts.1.w2.weight": "model-00027-of-00059.safetensors",
+ "model.layers.25.block_sparse_moe.experts.1.w3.weight": "model-00027-of-00059.safetensors",
+ "model.layers.25.block_sparse_moe.experts.2.w1.weight": "model-00027-of-00059.safetensors",
+ "model.layers.25.block_sparse_moe.experts.2.w2.weight": "model-00027-of-00059.safetensors",
+ "model.layers.25.block_sparse_moe.experts.2.w3.weight": "model-00027-of-00059.safetensors",
+ "model.layers.25.block_sparse_moe.experts.3.w1.weight": "model-00027-of-00059.safetensors",
+ "model.layers.25.block_sparse_moe.experts.3.w2.weight": "model-00027-of-00059.safetensors",
+ "model.layers.25.block_sparse_moe.experts.3.w3.weight": "model-00027-of-00059.safetensors",
+ "model.layers.25.block_sparse_moe.experts.4.w1.weight": "model-00027-of-00059.safetensors",
+ "model.layers.25.block_sparse_moe.experts.4.w2.weight": "model-00027-of-00059.safetensors",
+ "model.layers.25.block_sparse_moe.experts.4.w3.weight": "model-00027-of-00059.safetensors",
+ "model.layers.25.block_sparse_moe.experts.5.w1.weight": "model-00027-of-00059.safetensors",
+ "model.layers.25.block_sparse_moe.experts.5.w2.weight": "model-00027-of-00059.safetensors",
+ "model.layers.25.block_sparse_moe.experts.5.w3.weight": "model-00027-of-00059.safetensors",
+ "model.layers.25.block_sparse_moe.experts.6.w1.weight": "model-00027-of-00059.safetensors",
+ "model.layers.25.block_sparse_moe.experts.6.w2.weight": "model-00027-of-00059.safetensors",
+ "model.layers.25.block_sparse_moe.experts.6.w3.weight": "model-00027-of-00059.safetensors",
+ "model.layers.25.block_sparse_moe.experts.7.w1.weight": "model-00027-of-00059.safetensors",
+ "model.layers.25.block_sparse_moe.experts.7.w2.weight": "model-00027-of-00059.safetensors",
+ "model.layers.25.block_sparse_moe.experts.7.w3.weight": "model-00027-of-00059.safetensors",
+ "model.layers.25.block_sparse_moe.gate.weight": "model-00027-of-00059.safetensors",
+ "model.layers.25.input_layernorm.weight": "model-00027-of-00059.safetensors",
+ "model.layers.25.post_attention_layernorm.weight": "model-00027-of-00059.safetensors",
+ "model.layers.25.self_attn.k_proj.weight": "model-00027-of-00059.safetensors",
+ "model.layers.25.self_attn.o_proj.weight": "model-00027-of-00059.safetensors",
+ "model.layers.25.self_attn.q_proj.weight": "model-00026-of-00059.safetensors",
+ "model.layers.25.self_attn.v_proj.weight": "model-00027-of-00059.safetensors",
+ "model.layers.26.block_sparse_moe.experts.0.w1.weight": "model-00028-of-00059.safetensors",
+ "model.layers.26.block_sparse_moe.experts.0.w2.weight": "model-00028-of-00059.safetensors",
+ "model.layers.26.block_sparse_moe.experts.0.w3.weight": "model-00028-of-00059.safetensors",
+ "model.layers.26.block_sparse_moe.experts.1.w1.weight": "model-00028-of-00059.safetensors",
+ "model.layers.26.block_sparse_moe.experts.1.w2.weight": "model-00028-of-00059.safetensors",
+ "model.layers.26.block_sparse_moe.experts.1.w3.weight": "model-00028-of-00059.safetensors",
+ "model.layers.26.block_sparse_moe.experts.2.w1.weight": "model-00028-of-00059.safetensors",
+ "model.layers.26.block_sparse_moe.experts.2.w2.weight": "model-00028-of-00059.safetensors",
+ "model.layers.26.block_sparse_moe.experts.2.w3.weight": "model-00028-of-00059.safetensors",
+ "model.layers.26.block_sparse_moe.experts.3.w1.weight": "model-00028-of-00059.safetensors",
+ "model.layers.26.block_sparse_moe.experts.3.w2.weight": "model-00028-of-00059.safetensors",
+ "model.layers.26.block_sparse_moe.experts.3.w3.weight": "model-00028-of-00059.safetensors",
+ "model.layers.26.block_sparse_moe.experts.4.w1.weight": "model-00028-of-00059.safetensors",
+ "model.layers.26.block_sparse_moe.experts.4.w2.weight": "model-00028-of-00059.safetensors",
+ "model.layers.26.block_sparse_moe.experts.4.w3.weight": "model-00028-of-00059.safetensors",
+ "model.layers.26.block_sparse_moe.experts.5.w1.weight": "model-00028-of-00059.safetensors",
+ "model.layers.26.block_sparse_moe.experts.5.w2.weight": "model-00028-of-00059.safetensors",
+ "model.layers.26.block_sparse_moe.experts.5.w3.weight": "model-00028-of-00059.safetensors",
+ "model.layers.26.block_sparse_moe.experts.6.w1.weight": "model-00028-of-00059.safetensors",
+ "model.layers.26.block_sparse_moe.experts.6.w2.weight": "model-00028-of-00059.safetensors",
+ "model.layers.26.block_sparse_moe.experts.6.w3.weight": "model-00028-of-00059.safetensors",
+ "model.layers.26.block_sparse_moe.experts.7.w1.weight": "model-00028-of-00059.safetensors",
+ "model.layers.26.block_sparse_moe.experts.7.w2.weight": "model-00028-of-00059.safetensors",
+ "model.layers.26.block_sparse_moe.experts.7.w3.weight": "model-00029-of-00059.safetensors",
+ "model.layers.26.block_sparse_moe.gate.weight": "model-00028-of-00059.safetensors",
+ "model.layers.26.input_layernorm.weight": "model-00029-of-00059.safetensors",
+ "model.layers.26.post_attention_layernorm.weight": "model-00029-of-00059.safetensors",
+ "model.layers.26.self_attn.k_proj.weight": "model-00028-of-00059.safetensors",
+ "model.layers.26.self_attn.o_proj.weight": "model-00028-of-00059.safetensors",
+ "model.layers.26.self_attn.q_proj.weight": "model-00028-of-00059.safetensors",
+ "model.layers.26.self_attn.v_proj.weight": "model-00028-of-00059.safetensors",
+ "model.layers.27.block_sparse_moe.experts.0.w1.weight": "model-00029-of-00059.safetensors",
+ "model.layers.27.block_sparse_moe.experts.0.w2.weight": "model-00029-of-00059.safetensors",
+ "model.layers.27.block_sparse_moe.experts.0.w3.weight": "model-00029-of-00059.safetensors",
+ "model.layers.27.block_sparse_moe.experts.1.w1.weight": "model-00029-of-00059.safetensors",
+ "model.layers.27.block_sparse_moe.experts.1.w2.weight": "model-00029-of-00059.safetensors",
+ "model.layers.27.block_sparse_moe.experts.1.w3.weight": "model-00029-of-00059.safetensors",
+ "model.layers.27.block_sparse_moe.experts.2.w1.weight": "model-00029-of-00059.safetensors",
+ "model.layers.27.block_sparse_moe.experts.2.w2.weight": "model-00029-of-00059.safetensors",
+ "model.layers.27.block_sparse_moe.experts.2.w3.weight": "model-00029-of-00059.safetensors",
+ "model.layers.27.block_sparse_moe.experts.3.w1.weight": "model-00029-of-00059.safetensors",
+ "model.layers.27.block_sparse_moe.experts.3.w2.weight": "model-00029-of-00059.safetensors",
+ "model.layers.27.block_sparse_moe.experts.3.w3.weight": "model-00029-of-00059.safetensors",
+ "model.layers.27.block_sparse_moe.experts.4.w1.weight": "model-00029-of-00059.safetensors",
+ "model.layers.27.block_sparse_moe.experts.4.w2.weight": "model-00029-of-00059.safetensors",
+ "model.layers.27.block_sparse_moe.experts.4.w3.weight": "model-00029-of-00059.safetensors",
+ "model.layers.27.block_sparse_moe.experts.5.w1.weight": "model-00029-of-00059.safetensors",
+ "model.layers.27.block_sparse_moe.experts.5.w2.weight": "model-00029-of-00059.safetensors",
+ "model.layers.27.block_sparse_moe.experts.5.w3.weight": "model-00029-of-00059.safetensors",
+ "model.layers.27.block_sparse_moe.experts.6.w1.weight": "model-00029-of-00059.safetensors",
+ "model.layers.27.block_sparse_moe.experts.6.w2.weight": "model-00029-of-00059.safetensors",
+ "model.layers.27.block_sparse_moe.experts.6.w3.weight": "model-00029-of-00059.safetensors",
+ "model.layers.27.block_sparse_moe.experts.7.w1.weight": "model-00029-of-00059.safetensors",
+ "model.layers.27.block_sparse_moe.experts.7.w2.weight": "model-00030-of-00059.safetensors",
+ "model.layers.27.block_sparse_moe.experts.7.w3.weight": "model-00030-of-00059.safetensors",
+ "model.layers.27.block_sparse_moe.gate.weight": "model-00029-of-00059.safetensors",
+ "model.layers.27.input_layernorm.weight": "model-00030-of-00059.safetensors",
+ "model.layers.27.post_attention_layernorm.weight": "model-00030-of-00059.safetensors",
+ "model.layers.27.self_attn.k_proj.weight": "model-00029-of-00059.safetensors",
+ "model.layers.27.self_attn.o_proj.weight": "model-00029-of-00059.safetensors",
+ "model.layers.27.self_attn.q_proj.weight": "model-00029-of-00059.safetensors",
+ "model.layers.27.self_attn.v_proj.weight": "model-00029-of-00059.safetensors",
+ "model.layers.28.block_sparse_moe.experts.0.w1.weight": "model-00030-of-00059.safetensors",
+ "model.layers.28.block_sparse_moe.experts.0.w2.weight": "model-00030-of-00059.safetensors",
+ "model.layers.28.block_sparse_moe.experts.0.w3.weight": "model-00030-of-00059.safetensors",
+ "model.layers.28.block_sparse_moe.experts.1.w1.weight": "model-00030-of-00059.safetensors",
+ "model.layers.28.block_sparse_moe.experts.1.w2.weight": "model-00030-of-00059.safetensors",
+ "model.layers.28.block_sparse_moe.experts.1.w3.weight": "model-00030-of-00059.safetensors",
+ "model.layers.28.block_sparse_moe.experts.2.w1.weight": "model-00030-of-00059.safetensors",
+ "model.layers.28.block_sparse_moe.experts.2.w2.weight": "model-00030-of-00059.safetensors",
+ "model.layers.28.block_sparse_moe.experts.2.w3.weight": "model-00030-of-00059.safetensors",
+ "model.layers.28.block_sparse_moe.experts.3.w1.weight": "model-00030-of-00059.safetensors",
+ "model.layers.28.block_sparse_moe.experts.3.w2.weight": "model-00030-of-00059.safetensors",
+ "model.layers.28.block_sparse_moe.experts.3.w3.weight": "model-00030-of-00059.safetensors",
+ "model.layers.28.block_sparse_moe.experts.4.w1.weight": "model-00030-of-00059.safetensors",
+ "model.layers.28.block_sparse_moe.experts.4.w2.weight": "model-00030-of-00059.safetensors",
+ "model.layers.28.block_sparse_moe.experts.4.w3.weight": "model-00030-of-00059.safetensors",
+ "model.layers.28.block_sparse_moe.experts.5.w1.weight": "model-00030-of-00059.safetensors",
+ "model.layers.28.block_sparse_moe.experts.5.w2.weight": "model-00030-of-00059.safetensors",
+ "model.layers.28.block_sparse_moe.experts.5.w3.weight": "model-00030-of-00059.safetensors",
+ "model.layers.28.block_sparse_moe.experts.6.w1.weight": "model-00030-of-00059.safetensors",
+ "model.layers.28.block_sparse_moe.experts.6.w2.weight": "model-00030-of-00059.safetensors",
+ "model.layers.28.block_sparse_moe.experts.6.w3.weight": "model-00030-of-00059.safetensors",
+ "model.layers.28.block_sparse_moe.experts.7.w1.weight": "model-00031-of-00059.safetensors",
+ "model.layers.28.block_sparse_moe.experts.7.w2.weight": "model-00031-of-00059.safetensors",
+ "model.layers.28.block_sparse_moe.experts.7.w3.weight": "model-00031-of-00059.safetensors",
+ "model.layers.28.block_sparse_moe.gate.weight": "model-00030-of-00059.safetensors",
+ "model.layers.28.input_layernorm.weight": "model-00031-of-00059.safetensors",
+ "model.layers.28.post_attention_layernorm.weight": "model-00031-of-00059.safetensors",
+ "model.layers.28.self_attn.k_proj.weight": "model-00030-of-00059.safetensors",
+ "model.layers.28.self_attn.o_proj.weight": "model-00030-of-00059.safetensors",
+ "model.layers.28.self_attn.q_proj.weight": "model-00030-of-00059.safetensors",
+ "model.layers.28.self_attn.v_proj.weight": "model-00030-of-00059.safetensors",
+ "model.layers.29.block_sparse_moe.experts.0.w1.weight": "model-00031-of-00059.safetensors",
+ "model.layers.29.block_sparse_moe.experts.0.w2.weight": "model-00031-of-00059.safetensors",
+ "model.layers.29.block_sparse_moe.experts.0.w3.weight": "model-00031-of-00059.safetensors",
+ "model.layers.29.block_sparse_moe.experts.1.w1.weight": "model-00031-of-00059.safetensors",
+ "model.layers.29.block_sparse_moe.experts.1.w2.weight": "model-00031-of-00059.safetensors",
+ "model.layers.29.block_sparse_moe.experts.1.w3.weight": "model-00031-of-00059.safetensors",
+ "model.layers.29.block_sparse_moe.experts.2.w1.weight": "model-00031-of-00059.safetensors",
+ "model.layers.29.block_sparse_moe.experts.2.w2.weight": "model-00031-of-00059.safetensors",
+ "model.layers.29.block_sparse_moe.experts.2.w3.weight": "model-00031-of-00059.safetensors",
+ "model.layers.29.block_sparse_moe.experts.3.w1.weight": "model-00031-of-00059.safetensors",
+ "model.layers.29.block_sparse_moe.experts.3.w2.weight": "model-00031-of-00059.safetensors",
+ "model.layers.29.block_sparse_moe.experts.3.w3.weight": "model-00031-of-00059.safetensors",
+ "model.layers.29.block_sparse_moe.experts.4.w1.weight": "model-00031-of-00059.safetensors",
+ "model.layers.29.block_sparse_moe.experts.4.w2.weight": "model-00031-of-00059.safetensors",
+ "model.layers.29.block_sparse_moe.experts.4.w3.weight": "model-00031-of-00059.safetensors",
+ "model.layers.29.block_sparse_moe.experts.5.w1.weight": "model-00031-of-00059.safetensors",
+ "model.layers.29.block_sparse_moe.experts.5.w2.weight": "model-00031-of-00059.safetensors",
+ "model.layers.29.block_sparse_moe.experts.5.w3.weight": "model-00031-of-00059.safetensors",
+ "model.layers.29.block_sparse_moe.experts.6.w1.weight": "model-00031-of-00059.safetensors",
+ "model.layers.29.block_sparse_moe.experts.6.w2.weight": "model-00031-of-00059.safetensors",
+ "model.layers.29.block_sparse_moe.experts.6.w3.weight": "model-00032-of-00059.safetensors",
+ "model.layers.29.block_sparse_moe.experts.7.w1.weight": "model-00032-of-00059.safetensors",
+ "model.layers.29.block_sparse_moe.experts.7.w2.weight": "model-00032-of-00059.safetensors",
+ "model.layers.29.block_sparse_moe.experts.7.w3.weight": "model-00032-of-00059.safetensors",
+ "model.layers.29.block_sparse_moe.gate.weight": "model-00031-of-00059.safetensors",
+ "model.layers.29.input_layernorm.weight": "model-00032-of-00059.safetensors",
+ "model.layers.29.post_attention_layernorm.weight": "model-00032-of-00059.safetensors",
+ "model.layers.29.self_attn.k_proj.weight": "model-00031-of-00059.safetensors",
+ "model.layers.29.self_attn.o_proj.weight": "model-00031-of-00059.safetensors",
+ "model.layers.29.self_attn.q_proj.weight": "model-00031-of-00059.safetensors",
+ "model.layers.29.self_attn.v_proj.weight": "model-00031-of-00059.safetensors",
+ "model.layers.3.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00059.safetensors",
+ "model.layers.3.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00059.safetensors",
+ "model.layers.3.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00059.safetensors",
+ "model.layers.3.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00059.safetensors",
+ "model.layers.3.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00059.safetensors",
+ "model.layers.3.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00059.safetensors",
+ "model.layers.3.block_sparse_moe.experts.2.w1.weight": "model-00004-of-00059.safetensors",
+ "model.layers.3.block_sparse_moe.experts.2.w2.weight": "model-00004-of-00059.safetensors",
+ "model.layers.3.block_sparse_moe.experts.2.w3.weight": "model-00004-of-00059.safetensors",
+ "model.layers.3.block_sparse_moe.experts.3.w1.weight": "model-00004-of-00059.safetensors",
+ "model.layers.3.block_sparse_moe.experts.3.w2.weight": "model-00004-of-00059.safetensors",
+ "model.layers.3.block_sparse_moe.experts.3.w3.weight": "model-00004-of-00059.safetensors",
+ "model.layers.3.block_sparse_moe.experts.4.w1.weight": "model-00004-of-00059.safetensors",
+ "model.layers.3.block_sparse_moe.experts.4.w2.weight": "model-00004-of-00059.safetensors",
+ "model.layers.3.block_sparse_moe.experts.4.w3.weight": "model-00004-of-00059.safetensors",
+ "model.layers.3.block_sparse_moe.experts.5.w1.weight": "model-00004-of-00059.safetensors",
+ "model.layers.3.block_sparse_moe.experts.5.w2.weight": "model-00004-of-00059.safetensors",
+ "model.layers.3.block_sparse_moe.experts.5.w3.weight": "model-00004-of-00059.safetensors",
+ "model.layers.3.block_sparse_moe.experts.6.w1.weight": "model-00004-of-00059.safetensors",
+ "model.layers.3.block_sparse_moe.experts.6.w2.weight": "model-00005-of-00059.safetensors",
+ "model.layers.3.block_sparse_moe.experts.6.w3.weight": "model-00005-of-00059.safetensors",
+ "model.layers.3.block_sparse_moe.experts.7.w1.weight": "model-00005-of-00059.safetensors",
+ "model.layers.3.block_sparse_moe.experts.7.w2.weight": "model-00005-of-00059.safetensors",
+ "model.layers.3.block_sparse_moe.experts.7.w3.weight": "model-00005-of-00059.safetensors",
+ "model.layers.3.block_sparse_moe.gate.weight": "model-00004-of-00059.safetensors",
+ "model.layers.3.input_layernorm.weight": "model-00005-of-00059.safetensors",
+ "model.layers.3.post_attention_layernorm.weight": "model-00005-of-00059.safetensors",
+ "model.layers.3.self_attn.k_proj.weight": "model-00004-of-00059.safetensors",
+ "model.layers.3.self_attn.o_proj.weight": "model-00004-of-00059.safetensors",
+ "model.layers.3.self_attn.q_proj.weight": "model-00004-of-00059.safetensors",
+ "model.layers.3.self_attn.v_proj.weight": "model-00004-of-00059.safetensors",
+ "model.layers.30.block_sparse_moe.experts.0.w1.weight": "model-00032-of-00059.safetensors",
+ "model.layers.30.block_sparse_moe.experts.0.w2.weight": "model-00032-of-00059.safetensors",
+ "model.layers.30.block_sparse_moe.experts.0.w3.weight": "model-00032-of-00059.safetensors",
+ "model.layers.30.block_sparse_moe.experts.1.w1.weight": "model-00032-of-00059.safetensors",
+ "model.layers.30.block_sparse_moe.experts.1.w2.weight": "model-00032-of-00059.safetensors",
+ "model.layers.30.block_sparse_moe.experts.1.w3.weight": "model-00032-of-00059.safetensors",
+ "model.layers.30.block_sparse_moe.experts.2.w1.weight": "model-00032-of-00059.safetensors",
+ "model.layers.30.block_sparse_moe.experts.2.w2.weight": "model-00032-of-00059.safetensors",
+ "model.layers.30.block_sparse_moe.experts.2.w3.weight": "model-00032-of-00059.safetensors",
+ "model.layers.30.block_sparse_moe.experts.3.w1.weight": "model-00032-of-00059.safetensors",
+ "model.layers.30.block_sparse_moe.experts.3.w2.weight": "model-00032-of-00059.safetensors",
+ "model.layers.30.block_sparse_moe.experts.3.w3.weight": "model-00032-of-00059.safetensors",
+ "model.layers.30.block_sparse_moe.experts.4.w1.weight": "model-00032-of-00059.safetensors",
+ "model.layers.30.block_sparse_moe.experts.4.w2.weight": "model-00032-of-00059.safetensors",
+ "model.layers.30.block_sparse_moe.experts.4.w3.weight": "model-00032-of-00059.safetensors",
+ "model.layers.30.block_sparse_moe.experts.5.w1.weight": "model-00032-of-00059.safetensors",
+ "model.layers.30.block_sparse_moe.experts.5.w2.weight": "model-00032-of-00059.safetensors",
+ "model.layers.30.block_sparse_moe.experts.5.w3.weight": "model-00032-of-00059.safetensors",
+ "model.layers.30.block_sparse_moe.experts.6.w1.weight": "model-00032-of-00059.safetensors",
+ "model.layers.30.block_sparse_moe.experts.6.w2.weight": "model-00033-of-00059.safetensors",
+ "model.layers.30.block_sparse_moe.experts.6.w3.weight": "model-00033-of-00059.safetensors",
+ "model.layers.30.block_sparse_moe.experts.7.w1.weight": "model-00033-of-00059.safetensors",
+ "model.layers.30.block_sparse_moe.experts.7.w2.weight": "model-00033-of-00059.safetensors",
+ "model.layers.30.block_sparse_moe.experts.7.w3.weight": "model-00033-of-00059.safetensors",
+ "model.layers.30.block_sparse_moe.gate.weight": "model-00032-of-00059.safetensors",
+ "model.layers.30.input_layernorm.weight": "model-00033-of-00059.safetensors",
+ "model.layers.30.post_attention_layernorm.weight": "model-00033-of-00059.safetensors",
+ "model.layers.30.self_attn.k_proj.weight": "model-00032-of-00059.safetensors",
+ "model.layers.30.self_attn.o_proj.weight": "model-00032-of-00059.safetensors",
+ "model.layers.30.self_attn.q_proj.weight": "model-00032-of-00059.safetensors",
+ "model.layers.30.self_attn.v_proj.weight": "model-00032-of-00059.safetensors",
+ "model.layers.31.block_sparse_moe.experts.0.w1.weight": "model-00033-of-00059.safetensors",
+ "model.layers.31.block_sparse_moe.experts.0.w2.weight": "model-00033-of-00059.safetensors",
+ "model.layers.31.block_sparse_moe.experts.0.w3.weight": "model-00033-of-00059.safetensors",
+ "model.layers.31.block_sparse_moe.experts.1.w1.weight": "model-00033-of-00059.safetensors",
+ "model.layers.31.block_sparse_moe.experts.1.w2.weight": "model-00033-of-00059.safetensors",
+ "model.layers.31.block_sparse_moe.experts.1.w3.weight": "model-00033-of-00059.safetensors",
+ "model.layers.31.block_sparse_moe.experts.2.w1.weight": "model-00033-of-00059.safetensors",
+ "model.layers.31.block_sparse_moe.experts.2.w2.weight": "model-00033-of-00059.safetensors",
+ "model.layers.31.block_sparse_moe.experts.2.w3.weight": "model-00033-of-00059.safetensors",
+ "model.layers.31.block_sparse_moe.experts.3.w1.weight": "model-00033-of-00059.safetensors",
+ "model.layers.31.block_sparse_moe.experts.3.w2.weight": "model-00033-of-00059.safetensors",
+ "model.layers.31.block_sparse_moe.experts.3.w3.weight": "model-00033-of-00059.safetensors",
+ "model.layers.31.block_sparse_moe.experts.4.w1.weight": "model-00033-of-00059.safetensors",
+ "model.layers.31.block_sparse_moe.experts.4.w2.weight": "model-00033-of-00059.safetensors",
+ "model.layers.31.block_sparse_moe.experts.4.w3.weight": "model-00033-of-00059.safetensors",
+ "model.layers.31.block_sparse_moe.experts.5.w1.weight": "model-00033-of-00059.safetensors",
+ "model.layers.31.block_sparse_moe.experts.5.w2.weight": "model-00033-of-00059.safetensors",
+ "model.layers.31.block_sparse_moe.experts.5.w3.weight": "model-00033-of-00059.safetensors",
+ "model.layers.31.block_sparse_moe.experts.6.w1.weight": "model-00034-of-00059.safetensors",
+ "model.layers.31.block_sparse_moe.experts.6.w2.weight": "model-00034-of-00059.safetensors",
+ "model.layers.31.block_sparse_moe.experts.6.w3.weight": "model-00034-of-00059.safetensors",
+ "model.layers.31.block_sparse_moe.experts.7.w1.weight": "model-00034-of-00059.safetensors",
+ "model.layers.31.block_sparse_moe.experts.7.w2.weight": "model-00034-of-00059.safetensors",
+ "model.layers.31.block_sparse_moe.experts.7.w3.weight": "model-00034-of-00059.safetensors",
+ "model.layers.31.block_sparse_moe.gate.weight": "model-00033-of-00059.safetensors",
+ "model.layers.31.input_layernorm.weight": "model-00034-of-00059.safetensors",
+ "model.layers.31.post_attention_layernorm.weight": "model-00034-of-00059.safetensors",
+ "model.layers.31.self_attn.k_proj.weight": "model-00033-of-00059.safetensors",
+ "model.layers.31.self_attn.o_proj.weight": "model-00033-of-00059.safetensors",
+ "model.layers.31.self_attn.q_proj.weight": "model-00033-of-00059.safetensors",
+ "model.layers.31.self_attn.v_proj.weight": "model-00033-of-00059.safetensors",
+ "model.layers.32.block_sparse_moe.experts.0.w1.weight": "model-00034-of-00059.safetensors",
+ "model.layers.32.block_sparse_moe.experts.0.w2.weight": "model-00034-of-00059.safetensors",
+ "model.layers.32.block_sparse_moe.experts.0.w3.weight": "model-00034-of-00059.safetensors",
+ "model.layers.32.block_sparse_moe.experts.1.w1.weight": "model-00034-of-00059.safetensors",
+ "model.layers.32.block_sparse_moe.experts.1.w2.weight": "model-00034-of-00059.safetensors",
+ "model.layers.32.block_sparse_moe.experts.1.w3.weight": "model-00034-of-00059.safetensors",
+ "model.layers.32.block_sparse_moe.experts.2.w1.weight": "model-00034-of-00059.safetensors",
+ "model.layers.32.block_sparse_moe.experts.2.w2.weight": "model-00034-of-00059.safetensors",
+ "model.layers.32.block_sparse_moe.experts.2.w3.weight": "model-00034-of-00059.safetensors",
+ "model.layers.32.block_sparse_moe.experts.3.w1.weight": "model-00034-of-00059.safetensors",
+ "model.layers.32.block_sparse_moe.experts.3.w2.weight": "model-00034-of-00059.safetensors",
+ "model.layers.32.block_sparse_moe.experts.3.w3.weight": "model-00034-of-00059.safetensors",
+ "model.layers.32.block_sparse_moe.experts.4.w1.weight": "model-00034-of-00059.safetensors",
+ "model.layers.32.block_sparse_moe.experts.4.w2.weight": "model-00034-of-00059.safetensors",
+ "model.layers.32.block_sparse_moe.experts.4.w3.weight": "model-00034-of-00059.safetensors",
+ "model.layers.32.block_sparse_moe.experts.5.w1.weight": "model-00034-of-00059.safetensors",
+ "model.layers.32.block_sparse_moe.experts.5.w2.weight": "model-00034-of-00059.safetensors",
+ "model.layers.32.block_sparse_moe.experts.5.w3.weight": "model-00035-of-00059.safetensors",
+ "model.layers.32.block_sparse_moe.experts.6.w1.weight": "model-00035-of-00059.safetensors",
+ "model.layers.32.block_sparse_moe.experts.6.w2.weight": "model-00035-of-00059.safetensors",
+ "model.layers.32.block_sparse_moe.experts.6.w3.weight": "model-00035-of-00059.safetensors",
+ "model.layers.32.block_sparse_moe.experts.7.w1.weight": "model-00035-of-00059.safetensors",
+ "model.layers.32.block_sparse_moe.experts.7.w2.weight": "model-00035-of-00059.safetensors",
+ "model.layers.32.block_sparse_moe.experts.7.w3.weight": "model-00035-of-00059.safetensors",
+ "model.layers.32.block_sparse_moe.gate.weight": "model-00034-of-00059.safetensors",
+ "model.layers.32.input_layernorm.weight": "model-00035-of-00059.safetensors",
+ "model.layers.32.post_attention_layernorm.weight": "model-00035-of-00059.safetensors",
+ "model.layers.32.self_attn.k_proj.weight": "model-00034-of-00059.safetensors",
+ "model.layers.32.self_attn.o_proj.weight": "model-00034-of-00059.safetensors",
+ "model.layers.32.self_attn.q_proj.weight": "model-00034-of-00059.safetensors",
+ "model.layers.32.self_attn.v_proj.weight": "model-00034-of-00059.safetensors",
+ "model.layers.33.block_sparse_moe.experts.0.w1.weight": "model-00035-of-00059.safetensors",
+ "model.layers.33.block_sparse_moe.experts.0.w2.weight": "model-00035-of-00059.safetensors",
+ "model.layers.33.block_sparse_moe.experts.0.w3.weight": "model-00035-of-00059.safetensors",
+ "model.layers.33.block_sparse_moe.experts.1.w1.weight": "model-00035-of-00059.safetensors",
+ "model.layers.33.block_sparse_moe.experts.1.w2.weight": "model-00035-of-00059.safetensors",
+ "model.layers.33.block_sparse_moe.experts.1.w3.weight": "model-00035-of-00059.safetensors",
+ "model.layers.33.block_sparse_moe.experts.2.w1.weight": "model-00035-of-00059.safetensors",
+ "model.layers.33.block_sparse_moe.experts.2.w2.weight": "model-00035-of-00059.safetensors",
+ "model.layers.33.block_sparse_moe.experts.2.w3.weight": "model-00035-of-00059.safetensors",
+ "model.layers.33.block_sparse_moe.experts.3.w1.weight": "model-00035-of-00059.safetensors",
+ "model.layers.33.block_sparse_moe.experts.3.w2.weight": "model-00035-of-00059.safetensors",
+ "model.layers.33.block_sparse_moe.experts.3.w3.weight": "model-00035-of-00059.safetensors",
+ "model.layers.33.block_sparse_moe.experts.4.w1.weight": "model-00035-of-00059.safetensors",
+ "model.layers.33.block_sparse_moe.experts.4.w2.weight": "model-00035-of-00059.safetensors",
+ "model.layers.33.block_sparse_moe.experts.4.w3.weight": "model-00035-of-00059.safetensors",
+ "model.layers.33.block_sparse_moe.experts.5.w1.weight": "model-00035-of-00059.safetensors",
+ "model.layers.33.block_sparse_moe.experts.5.w2.weight": "model-00036-of-00059.safetensors",
+ "model.layers.33.block_sparse_moe.experts.5.w3.weight": "model-00036-of-00059.safetensors",
+ "model.layers.33.block_sparse_moe.experts.6.w1.weight": "model-00036-of-00059.safetensors",
+ "model.layers.33.block_sparse_moe.experts.6.w2.weight": "model-00036-of-00059.safetensors",
+ "model.layers.33.block_sparse_moe.experts.6.w3.weight": "model-00036-of-00059.safetensors",
+ "model.layers.33.block_sparse_moe.experts.7.w1.weight": "model-00036-of-00059.safetensors",
+ "model.layers.33.block_sparse_moe.experts.7.w2.weight": "model-00036-of-00059.safetensors",
+ "model.layers.33.block_sparse_moe.experts.7.w3.weight": "model-00036-of-00059.safetensors",
+ "model.layers.33.block_sparse_moe.gate.weight": "model-00035-of-00059.safetensors",
+ "model.layers.33.input_layernorm.weight": "model-00036-of-00059.safetensors",
+ "model.layers.33.post_attention_layernorm.weight": "model-00036-of-00059.safetensors",
+ "model.layers.33.self_attn.k_proj.weight": "model-00035-of-00059.safetensors",
+ "model.layers.33.self_attn.o_proj.weight": "model-00035-of-00059.safetensors",
+ "model.layers.33.self_attn.q_proj.weight": "model-00035-of-00059.safetensors",
+ "model.layers.33.self_attn.v_proj.weight": "model-00035-of-00059.safetensors",
+ "model.layers.34.block_sparse_moe.experts.0.w1.weight": "model-00036-of-00059.safetensors",
+ "model.layers.34.block_sparse_moe.experts.0.w2.weight": "model-00036-of-00059.safetensors",
+ "model.layers.34.block_sparse_moe.experts.0.w3.weight": "model-00036-of-00059.safetensors",
+ "model.layers.34.block_sparse_moe.experts.1.w1.weight": "model-00036-of-00059.safetensors",
+ "model.layers.34.block_sparse_moe.experts.1.w2.weight": "model-00036-of-00059.safetensors",
+ "model.layers.34.block_sparse_moe.experts.1.w3.weight": "model-00036-of-00059.safetensors",
+ "model.layers.34.block_sparse_moe.experts.2.w1.weight": "model-00036-of-00059.safetensors",
+ "model.layers.34.block_sparse_moe.experts.2.w2.weight": "model-00036-of-00059.safetensors",
+ "model.layers.34.block_sparse_moe.experts.2.w3.weight": "model-00036-of-00059.safetensors",
+ "model.layers.34.block_sparse_moe.experts.3.w1.weight": "model-00036-of-00059.safetensors",
+ "model.layers.34.block_sparse_moe.experts.3.w2.weight": "model-00036-of-00059.safetensors",
+ "model.layers.34.block_sparse_moe.experts.3.w3.weight": "model-00036-of-00059.safetensors",
+ "model.layers.34.block_sparse_moe.experts.4.w1.weight": "model-00036-of-00059.safetensors",
+ "model.layers.34.block_sparse_moe.experts.4.w2.weight": "model-00036-of-00059.safetensors",
+ "model.layers.34.block_sparse_moe.experts.4.w3.weight": "model-00036-of-00059.safetensors",
+ "model.layers.34.block_sparse_moe.experts.5.w1.weight": "model-00037-of-00059.safetensors",
+ "model.layers.34.block_sparse_moe.experts.5.w2.weight": "model-00037-of-00059.safetensors",
+ "model.layers.34.block_sparse_moe.experts.5.w3.weight": "model-00037-of-00059.safetensors",
+ "model.layers.34.block_sparse_moe.experts.6.w1.weight": "model-00037-of-00059.safetensors",
+ "model.layers.34.block_sparse_moe.experts.6.w2.weight": "model-00037-of-00059.safetensors",
+ "model.layers.34.block_sparse_moe.experts.6.w3.weight": "model-00037-of-00059.safetensors",
+ "model.layers.34.block_sparse_moe.experts.7.w1.weight": "model-00037-of-00059.safetensors",
+ "model.layers.34.block_sparse_moe.experts.7.w2.weight": "model-00037-of-00059.safetensors",
+ "model.layers.34.block_sparse_moe.experts.7.w3.weight": "model-00037-of-00059.safetensors",
+ "model.layers.34.block_sparse_moe.gate.weight": "model-00036-of-00059.safetensors",
+ "model.layers.34.input_layernorm.weight": "model-00037-of-00059.safetensors",
+ "model.layers.34.post_attention_layernorm.weight": "model-00037-of-00059.safetensors",
+ "model.layers.34.self_attn.k_proj.weight": "model-00036-of-00059.safetensors",
+ "model.layers.34.self_attn.o_proj.weight": "model-00036-of-00059.safetensors",
+ "model.layers.34.self_attn.q_proj.weight": "model-00036-of-00059.safetensors",
+ "model.layers.34.self_attn.v_proj.weight": "model-00036-of-00059.safetensors",
+ "model.layers.35.block_sparse_moe.experts.0.w1.weight": "model-00037-of-00059.safetensors",
+ "model.layers.35.block_sparse_moe.experts.0.w2.weight": "model-00037-of-00059.safetensors",
+ "model.layers.35.block_sparse_moe.experts.0.w3.weight": "model-00037-of-00059.safetensors",
+ "model.layers.35.block_sparse_moe.experts.1.w1.weight": "model-00037-of-00059.safetensors",
+ "model.layers.35.block_sparse_moe.experts.1.w2.weight": "model-00037-of-00059.safetensors",
+ "model.layers.35.block_sparse_moe.experts.1.w3.weight": "model-00037-of-00059.safetensors",
+ "model.layers.35.block_sparse_moe.experts.2.w1.weight": "model-00037-of-00059.safetensors",
+ "model.layers.35.block_sparse_moe.experts.2.w2.weight": "model-00037-of-00059.safetensors",
+ "model.layers.35.block_sparse_moe.experts.2.w3.weight": "model-00037-of-00059.safetensors",
+ "model.layers.35.block_sparse_moe.experts.3.w1.weight": "model-00037-of-00059.safetensors",
+ "model.layers.35.block_sparse_moe.experts.3.w2.weight": "model-00037-of-00059.safetensors",
+ "model.layers.35.block_sparse_moe.experts.3.w3.weight": "model-00037-of-00059.safetensors",
+ "model.layers.35.block_sparse_moe.experts.4.w1.weight": "model-00037-of-00059.safetensors",
+ "model.layers.35.block_sparse_moe.experts.4.w2.weight": "model-00037-of-00059.safetensors",
+ "model.layers.35.block_sparse_moe.experts.4.w3.weight": "model-00038-of-00059.safetensors",
+ "model.layers.35.block_sparse_moe.experts.5.w1.weight": "model-00038-of-00059.safetensors",
+ "model.layers.35.block_sparse_moe.experts.5.w2.weight": "model-00038-of-00059.safetensors",
+ "model.layers.35.block_sparse_moe.experts.5.w3.weight": "model-00038-of-00059.safetensors",
+ "model.layers.35.block_sparse_moe.experts.6.w1.weight": "model-00038-of-00059.safetensors",
+ "model.layers.35.block_sparse_moe.experts.6.w2.weight": "model-00038-of-00059.safetensors",
+ "model.layers.35.block_sparse_moe.experts.6.w3.weight": "model-00038-of-00059.safetensors",
+ "model.layers.35.block_sparse_moe.experts.7.w1.weight": "model-00038-of-00059.safetensors",
+ "model.layers.35.block_sparse_moe.experts.7.w2.weight": "model-00038-of-00059.safetensors",
+ "model.layers.35.block_sparse_moe.experts.7.w3.weight": "model-00038-of-00059.safetensors",
+ "model.layers.35.block_sparse_moe.gate.weight": "model-00037-of-00059.safetensors",
+ "model.layers.35.input_layernorm.weight": "model-00038-of-00059.safetensors",
+ "model.layers.35.post_attention_layernorm.weight": "model-00038-of-00059.safetensors",
+ "model.layers.35.self_attn.k_proj.weight": "model-00037-of-00059.safetensors",
+ "model.layers.35.self_attn.o_proj.weight": "model-00037-of-00059.safetensors",
+ "model.layers.35.self_attn.q_proj.weight": "model-00037-of-00059.safetensors",
+ "model.layers.35.self_attn.v_proj.weight": "model-00037-of-00059.safetensors",
+ "model.layers.36.block_sparse_moe.experts.0.w1.weight": "model-00038-of-00059.safetensors",
+ "model.layers.36.block_sparse_moe.experts.0.w2.weight": "model-00038-of-00059.safetensors",
+ "model.layers.36.block_sparse_moe.experts.0.w3.weight": "model-00038-of-00059.safetensors",
+ "model.layers.36.block_sparse_moe.experts.1.w1.weight": "model-00038-of-00059.safetensors",
+ "model.layers.36.block_sparse_moe.experts.1.w2.weight": "model-00038-of-00059.safetensors",
+ "model.layers.36.block_sparse_moe.experts.1.w3.weight": "model-00038-of-00059.safetensors",
+ "model.layers.36.block_sparse_moe.experts.2.w1.weight": "model-00038-of-00059.safetensors",
+ "model.layers.36.block_sparse_moe.experts.2.w2.weight": "model-00038-of-00059.safetensors",
+ "model.layers.36.block_sparse_moe.experts.2.w3.weight": "model-00038-of-00059.safetensors",
+ "model.layers.36.block_sparse_moe.experts.3.w1.weight": "model-00038-of-00059.safetensors",
+ "model.layers.36.block_sparse_moe.experts.3.w2.weight": "model-00038-of-00059.safetensors",
+ "model.layers.36.block_sparse_moe.experts.3.w3.weight": "model-00038-of-00059.safetensors",
+ "model.layers.36.block_sparse_moe.experts.4.w1.weight": "model-00038-of-00059.safetensors",
+ "model.layers.36.block_sparse_moe.experts.4.w2.weight": "model-00039-of-00059.safetensors",
+ "model.layers.36.block_sparse_moe.experts.4.w3.weight": "model-00039-of-00059.safetensors",
+ "model.layers.36.block_sparse_moe.experts.5.w1.weight": "model-00039-of-00059.safetensors",
+ "model.layers.36.block_sparse_moe.experts.5.w2.weight": "model-00039-of-00059.safetensors",
+ "model.layers.36.block_sparse_moe.experts.5.w3.weight": "model-00039-of-00059.safetensors",
+ "model.layers.36.block_sparse_moe.experts.6.w1.weight": "model-00039-of-00059.safetensors",
+ "model.layers.36.block_sparse_moe.experts.6.w2.weight": "model-00039-of-00059.safetensors",
+ "model.layers.36.block_sparse_moe.experts.6.w3.weight": "model-00039-of-00059.safetensors",
+ "model.layers.36.block_sparse_moe.experts.7.w1.weight": "model-00039-of-00059.safetensors",
+ "model.layers.36.block_sparse_moe.experts.7.w2.weight": "model-00039-of-00059.safetensors",
+ "model.layers.36.block_sparse_moe.experts.7.w3.weight": "model-00039-of-00059.safetensors",
+ "model.layers.36.block_sparse_moe.gate.weight": "model-00038-of-00059.safetensors",
+ "model.layers.36.input_layernorm.weight": "model-00039-of-00059.safetensors",
+ "model.layers.36.post_attention_layernorm.weight": "model-00039-of-00059.safetensors",
+ "model.layers.36.self_attn.k_proj.weight": "model-00038-of-00059.safetensors",
+ "model.layers.36.self_attn.o_proj.weight": "model-00038-of-00059.safetensors",
+ "model.layers.36.self_attn.q_proj.weight": "model-00038-of-00059.safetensors",
+ "model.layers.36.self_attn.v_proj.weight": "model-00038-of-00059.safetensors",
+ "model.layers.37.block_sparse_moe.experts.0.w1.weight": "model-00039-of-00059.safetensors",
+ "model.layers.37.block_sparse_moe.experts.0.w2.weight": "model-00039-of-00059.safetensors",
+ "model.layers.37.block_sparse_moe.experts.0.w3.weight": "model-00039-of-00059.safetensors",
+ "model.layers.37.block_sparse_moe.experts.1.w1.weight": "model-00039-of-00059.safetensors",
+ "model.layers.37.block_sparse_moe.experts.1.w2.weight": "model-00039-of-00059.safetensors",
+ "model.layers.37.block_sparse_moe.experts.1.w3.weight": "model-00039-of-00059.safetensors",
+ "model.layers.37.block_sparse_moe.experts.2.w1.weight": "model-00039-of-00059.safetensors",
+ "model.layers.37.block_sparse_moe.experts.2.w2.weight": "model-00039-of-00059.safetensors",
+ "model.layers.37.block_sparse_moe.experts.2.w3.weight": "model-00039-of-00059.safetensors",
+ "model.layers.37.block_sparse_moe.experts.3.w1.weight": "model-00039-of-00059.safetensors",
+ "model.layers.37.block_sparse_moe.experts.3.w2.weight": "model-00039-of-00059.safetensors",
+ "model.layers.37.block_sparse_moe.experts.3.w3.weight": "model-00039-of-00059.safetensors",
+ "model.layers.37.block_sparse_moe.experts.4.w1.weight": "model-00040-of-00059.safetensors",
+ "model.layers.37.block_sparse_moe.experts.4.w2.weight": "model-00040-of-00059.safetensors",
+ "model.layers.37.block_sparse_moe.experts.4.w3.weight": "model-00040-of-00059.safetensors",
+ "model.layers.37.block_sparse_moe.experts.5.w1.weight": "model-00040-of-00059.safetensors",
+ "model.layers.37.block_sparse_moe.experts.5.w2.weight": "model-00040-of-00059.safetensors",
+ "model.layers.37.block_sparse_moe.experts.5.w3.weight": "model-00040-of-00059.safetensors",
+ "model.layers.37.block_sparse_moe.experts.6.w1.weight": "model-00040-of-00059.safetensors",
+ "model.layers.37.block_sparse_moe.experts.6.w2.weight": "model-00040-of-00059.safetensors",
+ "model.layers.37.block_sparse_moe.experts.6.w3.weight": "model-00040-of-00059.safetensors",
+ "model.layers.37.block_sparse_moe.experts.7.w1.weight": "model-00040-of-00059.safetensors",
+ "model.layers.37.block_sparse_moe.experts.7.w2.weight": "model-00040-of-00059.safetensors",
+ "model.layers.37.block_sparse_moe.experts.7.w3.weight": "model-00040-of-00059.safetensors",
+ "model.layers.37.block_sparse_moe.gate.weight": "model-00039-of-00059.safetensors",
+ "model.layers.37.input_layernorm.weight": "model-00040-of-00059.safetensors",
+ "model.layers.37.post_attention_layernorm.weight": "model-00040-of-00059.safetensors",
+ "model.layers.37.self_attn.k_proj.weight": "model-00039-of-00059.safetensors",
+ "model.layers.37.self_attn.o_proj.weight": "model-00039-of-00059.safetensors",
+ "model.layers.37.self_attn.q_proj.weight": "model-00039-of-00059.safetensors",
+ "model.layers.37.self_attn.v_proj.weight": "model-00039-of-00059.safetensors",
+ "model.layers.38.block_sparse_moe.experts.0.w1.weight": "model-00040-of-00059.safetensors",
+ "model.layers.38.block_sparse_moe.experts.0.w2.weight": "model-00040-of-00059.safetensors",
+ "model.layers.38.block_sparse_moe.experts.0.w3.weight": "model-00040-of-00059.safetensors",
+ "model.layers.38.block_sparse_moe.experts.1.w1.weight": "model-00040-of-00059.safetensors",
+ "model.layers.38.block_sparse_moe.experts.1.w2.weight": "model-00040-of-00059.safetensors",
+ "model.layers.38.block_sparse_moe.experts.1.w3.weight": "model-00040-of-00059.safetensors",
+ "model.layers.38.block_sparse_moe.experts.2.w1.weight": "model-00040-of-00059.safetensors",
+ "model.layers.38.block_sparse_moe.experts.2.w2.weight": "model-00040-of-00059.safetensors",
+ "model.layers.38.block_sparse_moe.experts.2.w3.weight": "model-00040-of-00059.safetensors",
+ "model.layers.38.block_sparse_moe.experts.3.w1.weight": "model-00040-of-00059.safetensors",
+ "model.layers.38.block_sparse_moe.experts.3.w2.weight": "model-00040-of-00059.safetensors",
+ "model.layers.38.block_sparse_moe.experts.3.w3.weight": "model-00041-of-00059.safetensors",
+ "model.layers.38.block_sparse_moe.experts.4.w1.weight": "model-00041-of-00059.safetensors",
+ "model.layers.38.block_sparse_moe.experts.4.w2.weight": "model-00041-of-00059.safetensors",
+ "model.layers.38.block_sparse_moe.experts.4.w3.weight": "model-00041-of-00059.safetensors",
+ "model.layers.38.block_sparse_moe.experts.5.w1.weight": "model-00041-of-00059.safetensors",
+ "model.layers.38.block_sparse_moe.experts.5.w2.weight": "model-00041-of-00059.safetensors",
+ "model.layers.38.block_sparse_moe.experts.5.w3.weight": "model-00041-of-00059.safetensors",
+ "model.layers.38.block_sparse_moe.experts.6.w1.weight": "model-00041-of-00059.safetensors",
+ "model.layers.38.block_sparse_moe.experts.6.w2.weight": "model-00041-of-00059.safetensors",
+ "model.layers.38.block_sparse_moe.experts.6.w3.weight": "model-00041-of-00059.safetensors",
+ "model.layers.38.block_sparse_moe.experts.7.w1.weight": "model-00041-of-00059.safetensors",
+ "model.layers.38.block_sparse_moe.experts.7.w2.weight": "model-00041-of-00059.safetensors",
+ "model.layers.38.block_sparse_moe.experts.7.w3.weight": "model-00041-of-00059.safetensors",
+ "model.layers.38.block_sparse_moe.gate.weight": "model-00040-of-00059.safetensors",
+ "model.layers.38.input_layernorm.weight": "model-00041-of-00059.safetensors",
+ "model.layers.38.post_attention_layernorm.weight": "model-00041-of-00059.safetensors",
+ "model.layers.38.self_attn.k_proj.weight": "model-00040-of-00059.safetensors",
+ "model.layers.38.self_attn.o_proj.weight": "model-00040-of-00059.safetensors",
+ "model.layers.38.self_attn.q_proj.weight": "model-00040-of-00059.safetensors",
+ "model.layers.38.self_attn.v_proj.weight": "model-00040-of-00059.safetensors",
+ "model.layers.39.block_sparse_moe.experts.0.w1.weight": "model-00041-of-00059.safetensors",
+ "model.layers.39.block_sparse_moe.experts.0.w2.weight": "model-00041-of-00059.safetensors",
+ "model.layers.39.block_sparse_moe.experts.0.w3.weight": "model-00041-of-00059.safetensors",
+ "model.layers.39.block_sparse_moe.experts.1.w1.weight": "model-00041-of-00059.safetensors",
+ "model.layers.39.block_sparse_moe.experts.1.w2.weight": "model-00041-of-00059.safetensors",
+ "model.layers.39.block_sparse_moe.experts.1.w3.weight": "model-00041-of-00059.safetensors",
+ "model.layers.39.block_sparse_moe.experts.2.w1.weight": "model-00041-of-00059.safetensors",
+ "model.layers.39.block_sparse_moe.experts.2.w2.weight": "model-00041-of-00059.safetensors",
+ "model.layers.39.block_sparse_moe.experts.2.w3.weight": "model-00041-of-00059.safetensors",
+ "model.layers.39.block_sparse_moe.experts.3.w1.weight": "model-00041-of-00059.safetensors",
+ "model.layers.39.block_sparse_moe.experts.3.w2.weight": "model-00042-of-00059.safetensors",
+ "model.layers.39.block_sparse_moe.experts.3.w3.weight": "model-00042-of-00059.safetensors",
+ "model.layers.39.block_sparse_moe.experts.4.w1.weight": "model-00042-of-00059.safetensors",
+ "model.layers.39.block_sparse_moe.experts.4.w2.weight": "model-00042-of-00059.safetensors",
+ "model.layers.39.block_sparse_moe.experts.4.w3.weight": "model-00042-of-00059.safetensors",
+ "model.layers.39.block_sparse_moe.experts.5.w1.weight": "model-00042-of-00059.safetensors",
+ "model.layers.39.block_sparse_moe.experts.5.w2.weight": "model-00042-of-00059.safetensors",
+ "model.layers.39.block_sparse_moe.experts.5.w3.weight": "model-00042-of-00059.safetensors",
+ "model.layers.39.block_sparse_moe.experts.6.w1.weight": "model-00042-of-00059.safetensors",
+ "model.layers.39.block_sparse_moe.experts.6.w2.weight": "model-00042-of-00059.safetensors",
+ "model.layers.39.block_sparse_moe.experts.6.w3.weight": "model-00042-of-00059.safetensors",
+ "model.layers.39.block_sparse_moe.experts.7.w1.weight": "model-00042-of-00059.safetensors",
+ "model.layers.39.block_sparse_moe.experts.7.w2.weight": "model-00042-of-00059.safetensors",
+ "model.layers.39.block_sparse_moe.experts.7.w3.weight": "model-00042-of-00059.safetensors",
+ "model.layers.39.block_sparse_moe.gate.weight": "model-00041-of-00059.safetensors",
+ "model.layers.39.input_layernorm.weight": "model-00042-of-00059.safetensors",
+ "model.layers.39.post_attention_layernorm.weight": "model-00042-of-00059.safetensors",
+ "model.layers.39.self_attn.k_proj.weight": "model-00041-of-00059.safetensors",
+ "model.layers.39.self_attn.o_proj.weight": "model-00041-of-00059.safetensors",
+ "model.layers.39.self_attn.q_proj.weight": "model-00041-of-00059.safetensors",
+ "model.layers.39.self_attn.v_proj.weight": "model-00041-of-00059.safetensors",
+ "model.layers.4.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00059.safetensors",
+ "model.layers.4.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00059.safetensors",
+ "model.layers.4.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00059.safetensors",
+ "model.layers.4.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00059.safetensors",
+ "model.layers.4.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00059.safetensors",
+ "model.layers.4.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00059.safetensors",
+ "model.layers.4.block_sparse_moe.experts.2.w1.weight": "model-00005-of-00059.safetensors",
+ "model.layers.4.block_sparse_moe.experts.2.w2.weight": "model-00005-of-00059.safetensors",
+ "model.layers.4.block_sparse_moe.experts.2.w3.weight": "model-00005-of-00059.safetensors",
+ "model.layers.4.block_sparse_moe.experts.3.w1.weight": "model-00005-of-00059.safetensors",
+ "model.layers.4.block_sparse_moe.experts.3.w2.weight": "model-00005-of-00059.safetensors",
+ "model.layers.4.block_sparse_moe.experts.3.w3.weight": "model-00005-of-00059.safetensors",
+ "model.layers.4.block_sparse_moe.experts.4.w1.weight": "model-00005-of-00059.safetensors",
+ "model.layers.4.block_sparse_moe.experts.4.w2.weight": "model-00005-of-00059.safetensors",
+ "model.layers.4.block_sparse_moe.experts.4.w3.weight": "model-00005-of-00059.safetensors",
+ "model.layers.4.block_sparse_moe.experts.5.w1.weight": "model-00005-of-00059.safetensors",
+ "model.layers.4.block_sparse_moe.experts.5.w2.weight": "model-00005-of-00059.safetensors",
+ "model.layers.4.block_sparse_moe.experts.5.w3.weight": "model-00005-of-00059.safetensors",
+ "model.layers.4.block_sparse_moe.experts.6.w1.weight": "model-00006-of-00059.safetensors",
+ "model.layers.4.block_sparse_moe.experts.6.w2.weight": "model-00006-of-00059.safetensors",
+ "model.layers.4.block_sparse_moe.experts.6.w3.weight": "model-00006-of-00059.safetensors",
+ "model.layers.4.block_sparse_moe.experts.7.w1.weight": "model-00006-of-00059.safetensors",
+ "model.layers.4.block_sparse_moe.experts.7.w2.weight": "model-00006-of-00059.safetensors",
+ "model.layers.4.block_sparse_moe.experts.7.w3.weight": "model-00006-of-00059.safetensors",
+ "model.layers.4.block_sparse_moe.gate.weight": "model-00005-of-00059.safetensors",
+ "model.layers.4.input_layernorm.weight": "model-00006-of-00059.safetensors",
+ "model.layers.4.post_attention_layernorm.weight": "model-00006-of-00059.safetensors",
+ "model.layers.4.self_attn.k_proj.weight": "model-00005-of-00059.safetensors",
+ "model.layers.4.self_attn.o_proj.weight": "model-00005-of-00059.safetensors",
+ "model.layers.4.self_attn.q_proj.weight": "model-00005-of-00059.safetensors",
+ "model.layers.4.self_attn.v_proj.weight": "model-00005-of-00059.safetensors",
+ "model.layers.40.block_sparse_moe.experts.0.w1.weight": "model-00042-of-00059.safetensors",
+ "model.layers.40.block_sparse_moe.experts.0.w2.weight": "model-00042-of-00059.safetensors",
+ "model.layers.40.block_sparse_moe.experts.0.w3.weight": "model-00042-of-00059.safetensors",
+ "model.layers.40.block_sparse_moe.experts.1.w1.weight": "model-00042-of-00059.safetensors",
+ "model.layers.40.block_sparse_moe.experts.1.w2.weight": "model-00042-of-00059.safetensors",
+ "model.layers.40.block_sparse_moe.experts.1.w3.weight": "model-00042-of-00059.safetensors",
+ "model.layers.40.block_sparse_moe.experts.2.w1.weight": "model-00042-of-00059.safetensors",
+ "model.layers.40.block_sparse_moe.experts.2.w2.weight": "model-00042-of-00059.safetensors",
+ "model.layers.40.block_sparse_moe.experts.2.w3.weight": "model-00042-of-00059.safetensors",
+ "model.layers.40.block_sparse_moe.experts.3.w1.weight": "model-00043-of-00059.safetensors",
+ "model.layers.40.block_sparse_moe.experts.3.w2.weight": "model-00043-of-00059.safetensors",
+ "model.layers.40.block_sparse_moe.experts.3.w3.weight": "model-00043-of-00059.safetensors",
+ "model.layers.40.block_sparse_moe.experts.4.w1.weight": "model-00043-of-00059.safetensors",
+ "model.layers.40.block_sparse_moe.experts.4.w2.weight": "model-00043-of-00059.safetensors",
+ "model.layers.40.block_sparse_moe.experts.4.w3.weight": "model-00043-of-00059.safetensors",
+ "model.layers.40.block_sparse_moe.experts.5.w1.weight": "model-00043-of-00059.safetensors",
+ "model.layers.40.block_sparse_moe.experts.5.w2.weight": "model-00043-of-00059.safetensors",
+ "model.layers.40.block_sparse_moe.experts.5.w3.weight": "model-00043-of-00059.safetensors",
+ "model.layers.40.block_sparse_moe.experts.6.w1.weight": "model-00043-of-00059.safetensors",
+ "model.layers.40.block_sparse_moe.experts.6.w2.weight": "model-00043-of-00059.safetensors",
+ "model.layers.40.block_sparse_moe.experts.6.w3.weight": "model-00043-of-00059.safetensors",
+ "model.layers.40.block_sparse_moe.experts.7.w1.weight": "model-00043-of-00059.safetensors",
+ "model.layers.40.block_sparse_moe.experts.7.w2.weight": "model-00043-of-00059.safetensors",
+ "model.layers.40.block_sparse_moe.experts.7.w3.weight": "model-00043-of-00059.safetensors",
+ "model.layers.40.block_sparse_moe.gate.weight": "model-00042-of-00059.safetensors",
+ "model.layers.40.input_layernorm.weight": "model-00043-of-00059.safetensors",
+ "model.layers.40.post_attention_layernorm.weight": "model-00043-of-00059.safetensors",
+ "model.layers.40.self_attn.k_proj.weight": "model-00042-of-00059.safetensors",
+ "model.layers.40.self_attn.o_proj.weight": "model-00042-of-00059.safetensors",
+ "model.layers.40.self_attn.q_proj.weight": "model-00042-of-00059.safetensors",
+ "model.layers.40.self_attn.v_proj.weight": "model-00042-of-00059.safetensors",
+ "model.layers.41.block_sparse_moe.experts.0.w1.weight": "model-00043-of-00059.safetensors",
+ "model.layers.41.block_sparse_moe.experts.0.w2.weight": "model-00043-of-00059.safetensors",
+ "model.layers.41.block_sparse_moe.experts.0.w3.weight": "model-00043-of-00059.safetensors",
+ "model.layers.41.block_sparse_moe.experts.1.w1.weight": "model-00043-of-00059.safetensors",
+ "model.layers.41.block_sparse_moe.experts.1.w2.weight": "model-00043-of-00059.safetensors",
+ "model.layers.41.block_sparse_moe.experts.1.w3.weight": "model-00043-of-00059.safetensors",
+ "model.layers.41.block_sparse_moe.experts.2.w1.weight": "model-00043-of-00059.safetensors",
+ "model.layers.41.block_sparse_moe.experts.2.w2.weight": "model-00043-of-00059.safetensors",
+ "model.layers.41.block_sparse_moe.experts.2.w3.weight": "model-00044-of-00059.safetensors",
+ "model.layers.41.block_sparse_moe.experts.3.w1.weight": "model-00044-of-00059.safetensors",
+ "model.layers.41.block_sparse_moe.experts.3.w2.weight": "model-00044-of-00059.safetensors",
+ "model.layers.41.block_sparse_moe.experts.3.w3.weight": "model-00044-of-00059.safetensors",
+ "model.layers.41.block_sparse_moe.experts.4.w1.weight": "model-00044-of-00059.safetensors",
+ "model.layers.41.block_sparse_moe.experts.4.w2.weight": "model-00044-of-00059.safetensors",
+ "model.layers.41.block_sparse_moe.experts.4.w3.weight": "model-00044-of-00059.safetensors",
+ "model.layers.41.block_sparse_moe.experts.5.w1.weight": "model-00044-of-00059.safetensors",
+ "model.layers.41.block_sparse_moe.experts.5.w2.weight": "model-00044-of-00059.safetensors",
+ "model.layers.41.block_sparse_moe.experts.5.w3.weight": "model-00044-of-00059.safetensors",
+ "model.layers.41.block_sparse_moe.experts.6.w1.weight": "model-00044-of-00059.safetensors",
+ "model.layers.41.block_sparse_moe.experts.6.w2.weight": "model-00044-of-00059.safetensors",
+ "model.layers.41.block_sparse_moe.experts.6.w3.weight": "model-00044-of-00059.safetensors",
+ "model.layers.41.block_sparse_moe.experts.7.w1.weight": "model-00044-of-00059.safetensors",
+ "model.layers.41.block_sparse_moe.experts.7.w2.weight": "model-00044-of-00059.safetensors",
+ "model.layers.41.block_sparse_moe.experts.7.w3.weight": "model-00044-of-00059.safetensors",
+ "model.layers.41.block_sparse_moe.gate.weight": "model-00043-of-00059.safetensors",
+ "model.layers.41.input_layernorm.weight": "model-00044-of-00059.safetensors",
+ "model.layers.41.post_attention_layernorm.weight": "model-00044-of-00059.safetensors",
+ "model.layers.41.self_attn.k_proj.weight": "model-00043-of-00059.safetensors",
+ "model.layers.41.self_attn.o_proj.weight": "model-00043-of-00059.safetensors",
+ "model.layers.41.self_attn.q_proj.weight": "model-00043-of-00059.safetensors",
+ "model.layers.41.self_attn.v_proj.weight": "model-00043-of-00059.safetensors",
+ "model.layers.42.block_sparse_moe.experts.0.w1.weight": "model-00044-of-00059.safetensors",
+ "model.layers.42.block_sparse_moe.experts.0.w2.weight": "model-00044-of-00059.safetensors",
+ "model.layers.42.block_sparse_moe.experts.0.w3.weight": "model-00044-of-00059.safetensors",
+ "model.layers.42.block_sparse_moe.experts.1.w1.weight": "model-00044-of-00059.safetensors",
+ "model.layers.42.block_sparse_moe.experts.1.w2.weight": "model-00044-of-00059.safetensors",
+ "model.layers.42.block_sparse_moe.experts.1.w3.weight": "model-00044-of-00059.safetensors",
+ "model.layers.42.block_sparse_moe.experts.2.w1.weight": "model-00044-of-00059.safetensors",
+ "model.layers.42.block_sparse_moe.experts.2.w2.weight": "model-00045-of-00059.safetensors",
+ "model.layers.42.block_sparse_moe.experts.2.w3.weight": "model-00045-of-00059.safetensors",
+ "model.layers.42.block_sparse_moe.experts.3.w1.weight": "model-00045-of-00059.safetensors",
+ "model.layers.42.block_sparse_moe.experts.3.w2.weight": "model-00045-of-00059.safetensors",
+ "model.layers.42.block_sparse_moe.experts.3.w3.weight": "model-00045-of-00059.safetensors",
+ "model.layers.42.block_sparse_moe.experts.4.w1.weight": "model-00045-of-00059.safetensors",
+ "model.layers.42.block_sparse_moe.experts.4.w2.weight": "model-00045-of-00059.safetensors",
+ "model.layers.42.block_sparse_moe.experts.4.w3.weight": "model-00045-of-00059.safetensors",
+ "model.layers.42.block_sparse_moe.experts.5.w1.weight": "model-00045-of-00059.safetensors",
+ "model.layers.42.block_sparse_moe.experts.5.w2.weight": "model-00045-of-00059.safetensors",
+ "model.layers.42.block_sparse_moe.experts.5.w3.weight": "model-00045-of-00059.safetensors",
+ "model.layers.42.block_sparse_moe.experts.6.w1.weight": "model-00045-of-00059.safetensors",
+ "model.layers.42.block_sparse_moe.experts.6.w2.weight": "model-00045-of-00059.safetensors",
+ "model.layers.42.block_sparse_moe.experts.6.w3.weight": "model-00045-of-00059.safetensors",
+ "model.layers.42.block_sparse_moe.experts.7.w1.weight": "model-00045-of-00059.safetensors",
+ "model.layers.42.block_sparse_moe.experts.7.w2.weight": "model-00045-of-00059.safetensors",
+ "model.layers.42.block_sparse_moe.experts.7.w3.weight": "model-00045-of-00059.safetensors",
+ "model.layers.42.block_sparse_moe.gate.weight": "model-00044-of-00059.safetensors",
+ "model.layers.42.input_layernorm.weight": "model-00045-of-00059.safetensors",
+ "model.layers.42.post_attention_layernorm.weight": "model-00045-of-00059.safetensors",
+ "model.layers.42.self_attn.k_proj.weight": "model-00044-of-00059.safetensors",
+ "model.layers.42.self_attn.o_proj.weight": "model-00044-of-00059.safetensors",
+ "model.layers.42.self_attn.q_proj.weight": "model-00044-of-00059.safetensors",
+ "model.layers.42.self_attn.v_proj.weight": "model-00044-of-00059.safetensors",
+ "model.layers.43.block_sparse_moe.experts.0.w1.weight": "model-00045-of-00059.safetensors",
+ "model.layers.43.block_sparse_moe.experts.0.w2.weight": "model-00045-of-00059.safetensors",
+ "model.layers.43.block_sparse_moe.experts.0.w3.weight": "model-00045-of-00059.safetensors",
+ "model.layers.43.block_sparse_moe.experts.1.w1.weight": "model-00045-of-00059.safetensors",
+ "model.layers.43.block_sparse_moe.experts.1.w2.weight": "model-00045-of-00059.safetensors",
+ "model.layers.43.block_sparse_moe.experts.1.w3.weight": "model-00045-of-00059.safetensors",
+ "model.layers.43.block_sparse_moe.experts.2.w1.weight": "model-00046-of-00059.safetensors",
+ "model.layers.43.block_sparse_moe.experts.2.w2.weight": "model-00046-of-00059.safetensors",
+ "model.layers.43.block_sparse_moe.experts.2.w3.weight": "model-00046-of-00059.safetensors",
+ "model.layers.43.block_sparse_moe.experts.3.w1.weight": "model-00046-of-00059.safetensors",
+ "model.layers.43.block_sparse_moe.experts.3.w2.weight": "model-00046-of-00059.safetensors",
+ "model.layers.43.block_sparse_moe.experts.3.w3.weight": "model-00046-of-00059.safetensors",
+ "model.layers.43.block_sparse_moe.experts.4.w1.weight": "model-00046-of-00059.safetensors",
+ "model.layers.43.block_sparse_moe.experts.4.w2.weight": "model-00046-of-00059.safetensors",
+ "model.layers.43.block_sparse_moe.experts.4.w3.weight": "model-00046-of-00059.safetensors",
+ "model.layers.43.block_sparse_moe.experts.5.w1.weight": "model-00046-of-00059.safetensors",
+ "model.layers.43.block_sparse_moe.experts.5.w2.weight": "model-00046-of-00059.safetensors",
+ "model.layers.43.block_sparse_moe.experts.5.w3.weight": "model-00046-of-00059.safetensors",
+ "model.layers.43.block_sparse_moe.experts.6.w1.weight": "model-00046-of-00059.safetensors",
+ "model.layers.43.block_sparse_moe.experts.6.w2.weight": "model-00046-of-00059.safetensors",
+ "model.layers.43.block_sparse_moe.experts.6.w3.weight": "model-00046-of-00059.safetensors",
+ "model.layers.43.block_sparse_moe.experts.7.w1.weight": "model-00046-of-00059.safetensors",
+ "model.layers.43.block_sparse_moe.experts.7.w2.weight": "model-00046-of-00059.safetensors",
+ "model.layers.43.block_sparse_moe.experts.7.w3.weight": "model-00046-of-00059.safetensors",
+ "model.layers.43.block_sparse_moe.gate.weight": "model-00045-of-00059.safetensors",
+ "model.layers.43.input_layernorm.weight": "model-00046-of-00059.safetensors",
+ "model.layers.43.post_attention_layernorm.weight": "model-00046-of-00059.safetensors",
+ "model.layers.43.self_attn.k_proj.weight": "model-00045-of-00059.safetensors",
+ "model.layers.43.self_attn.o_proj.weight": "model-00045-of-00059.safetensors",
+ "model.layers.43.self_attn.q_proj.weight": "model-00045-of-00059.safetensors",
+ "model.layers.43.self_attn.v_proj.weight": "model-00045-of-00059.safetensors",
+ "model.layers.44.block_sparse_moe.experts.0.w1.weight": "model-00046-of-00059.safetensors",
+ "model.layers.44.block_sparse_moe.experts.0.w2.weight": "model-00046-of-00059.safetensors",
+ "model.layers.44.block_sparse_moe.experts.0.w3.weight": "model-00046-of-00059.safetensors",
+ "model.layers.44.block_sparse_moe.experts.1.w1.weight": "model-00046-of-00059.safetensors",
+ "model.layers.44.block_sparse_moe.experts.1.w2.weight": "model-00046-of-00059.safetensors",
+ "model.layers.44.block_sparse_moe.experts.1.w3.weight": "model-00047-of-00059.safetensors",
+ "model.layers.44.block_sparse_moe.experts.2.w1.weight": "model-00047-of-00059.safetensors",
+ "model.layers.44.block_sparse_moe.experts.2.w2.weight": "model-00047-of-00059.safetensors",
+ "model.layers.44.block_sparse_moe.experts.2.w3.weight": "model-00047-of-00059.safetensors",
+ "model.layers.44.block_sparse_moe.experts.3.w1.weight": "model-00047-of-00059.safetensors",
+ "model.layers.44.block_sparse_moe.experts.3.w2.weight": "model-00047-of-00059.safetensors",
+ "model.layers.44.block_sparse_moe.experts.3.w3.weight": "model-00047-of-00059.safetensors",
+ "model.layers.44.block_sparse_moe.experts.4.w1.weight": "model-00047-of-00059.safetensors",
+ "model.layers.44.block_sparse_moe.experts.4.w2.weight": "model-00047-of-00059.safetensors",
+ "model.layers.44.block_sparse_moe.experts.4.w3.weight": "model-00047-of-00059.safetensors",
+ "model.layers.44.block_sparse_moe.experts.5.w1.weight": "model-00047-of-00059.safetensors",
+ "model.layers.44.block_sparse_moe.experts.5.w2.weight": "model-00047-of-00059.safetensors",
+ "model.layers.44.block_sparse_moe.experts.5.w3.weight": "model-00047-of-00059.safetensors",
+ "model.layers.44.block_sparse_moe.experts.6.w1.weight": "model-00047-of-00059.safetensors",
+ "model.layers.44.block_sparse_moe.experts.6.w2.weight": "model-00047-of-00059.safetensors",
+ "model.layers.44.block_sparse_moe.experts.6.w3.weight": "model-00047-of-00059.safetensors",
+ "model.layers.44.block_sparse_moe.experts.7.w1.weight": "model-00047-of-00059.safetensors",
+ "model.layers.44.block_sparse_moe.experts.7.w2.weight": "model-00047-of-00059.safetensors",
+ "model.layers.44.block_sparse_moe.experts.7.w3.weight": "model-00047-of-00059.safetensors",
+ "model.layers.44.block_sparse_moe.gate.weight": "model-00046-of-00059.safetensors",
+ "model.layers.44.input_layernorm.weight": "model-00047-of-00059.safetensors",
+ "model.layers.44.post_attention_layernorm.weight": "model-00047-of-00059.safetensors",
+ "model.layers.44.self_attn.k_proj.weight": "model-00046-of-00059.safetensors",
+ "model.layers.44.self_attn.o_proj.weight": "model-00046-of-00059.safetensors",
+ "model.layers.44.self_attn.q_proj.weight": "model-00046-of-00059.safetensors",
+ "model.layers.44.self_attn.v_proj.weight": "model-00046-of-00059.safetensors",
+ "model.layers.45.block_sparse_moe.experts.0.w1.weight": "model-00047-of-00059.safetensors",
+ "model.layers.45.block_sparse_moe.experts.0.w2.weight": "model-00047-of-00059.safetensors",
+ "model.layers.45.block_sparse_moe.experts.0.w3.weight": "model-00047-of-00059.safetensors",
+ "model.layers.45.block_sparse_moe.experts.1.w1.weight": "model-00047-of-00059.safetensors",
+ "model.layers.45.block_sparse_moe.experts.1.w2.weight": "model-00048-of-00059.safetensors",
+ "model.layers.45.block_sparse_moe.experts.1.w3.weight": "model-00048-of-00059.safetensors",
+ "model.layers.45.block_sparse_moe.experts.2.w1.weight": "model-00048-of-00059.safetensors",
+ "model.layers.45.block_sparse_moe.experts.2.w2.weight": "model-00048-of-00059.safetensors",
+ "model.layers.45.block_sparse_moe.experts.2.w3.weight": "model-00048-of-00059.safetensors",
+ "model.layers.45.block_sparse_moe.experts.3.w1.weight": "model-00048-of-00059.safetensors",
+ "model.layers.45.block_sparse_moe.experts.3.w2.weight": "model-00048-of-00059.safetensors",
+ "model.layers.45.block_sparse_moe.experts.3.w3.weight": "model-00048-of-00059.safetensors",
+ "model.layers.45.block_sparse_moe.experts.4.w1.weight": "model-00048-of-00059.safetensors",
+ "model.layers.45.block_sparse_moe.experts.4.w2.weight": "model-00048-of-00059.safetensors",
+ "model.layers.45.block_sparse_moe.experts.4.w3.weight": "model-00048-of-00059.safetensors",
+ "model.layers.45.block_sparse_moe.experts.5.w1.weight": "model-00048-of-00059.safetensors",
+ "model.layers.45.block_sparse_moe.experts.5.w2.weight": "model-00048-of-00059.safetensors",
+ "model.layers.45.block_sparse_moe.experts.5.w3.weight": "model-00048-of-00059.safetensors",
+ "model.layers.45.block_sparse_moe.experts.6.w1.weight": "model-00048-of-00059.safetensors",
+ "model.layers.45.block_sparse_moe.experts.6.w2.weight": "model-00048-of-00059.safetensors",
+ "model.layers.45.block_sparse_moe.experts.6.w3.weight": "model-00048-of-00059.safetensors",
+ "model.layers.45.block_sparse_moe.experts.7.w1.weight": "model-00048-of-00059.safetensors",
+ "model.layers.45.block_sparse_moe.experts.7.w2.weight": "model-00048-of-00059.safetensors",
+ "model.layers.45.block_sparse_moe.experts.7.w3.weight": "model-00048-of-00059.safetensors",
+ "model.layers.45.block_sparse_moe.gate.weight": "model-00047-of-00059.safetensors",
+ "model.layers.45.input_layernorm.weight": "model-00048-of-00059.safetensors",
+ "model.layers.45.post_attention_layernorm.weight": "model-00048-of-00059.safetensors",
+ "model.layers.45.self_attn.k_proj.weight": "model-00047-of-00059.safetensors",
+ "model.layers.45.self_attn.o_proj.weight": "model-00047-of-00059.safetensors",
+ "model.layers.45.self_attn.q_proj.weight": "model-00047-of-00059.safetensors",
+ "model.layers.45.self_attn.v_proj.weight": "model-00047-of-00059.safetensors",
+ "model.layers.46.block_sparse_moe.experts.0.w1.weight": "model-00048-of-00059.safetensors",
+ "model.layers.46.block_sparse_moe.experts.0.w2.weight": "model-00048-of-00059.safetensors",
+ "model.layers.46.block_sparse_moe.experts.0.w3.weight": "model-00048-of-00059.safetensors",
+ "model.layers.46.block_sparse_moe.experts.1.w1.weight": "model-00049-of-00059.safetensors",
+ "model.layers.46.block_sparse_moe.experts.1.w2.weight": "model-00049-of-00059.safetensors",
+ "model.layers.46.block_sparse_moe.experts.1.w3.weight": "model-00049-of-00059.safetensors",
+ "model.layers.46.block_sparse_moe.experts.2.w1.weight": "model-00049-of-00059.safetensors",
+ "model.layers.46.block_sparse_moe.experts.2.w2.weight": "model-00049-of-00059.safetensors",
+ "model.layers.46.block_sparse_moe.experts.2.w3.weight": "model-00049-of-00059.safetensors",
+ "model.layers.46.block_sparse_moe.experts.3.w1.weight": "model-00049-of-00059.safetensors",
+ "model.layers.46.block_sparse_moe.experts.3.w2.weight": "model-00049-of-00059.safetensors",
+ "model.layers.46.block_sparse_moe.experts.3.w3.weight": "model-00049-of-00059.safetensors",
+ "model.layers.46.block_sparse_moe.experts.4.w1.weight": "model-00049-of-00059.safetensors",
+ "model.layers.46.block_sparse_moe.experts.4.w2.weight": "model-00049-of-00059.safetensors",
+ "model.layers.46.block_sparse_moe.experts.4.w3.weight": "model-00049-of-00059.safetensors",
+ "model.layers.46.block_sparse_moe.experts.5.w1.weight": "model-00049-of-00059.safetensors",
+ "model.layers.46.block_sparse_moe.experts.5.w2.weight": "model-00049-of-00059.safetensors",
+ "model.layers.46.block_sparse_moe.experts.5.w3.weight": "model-00049-of-00059.safetensors",
+ "model.layers.46.block_sparse_moe.experts.6.w1.weight": "model-00049-of-00059.safetensors",
+ "model.layers.46.block_sparse_moe.experts.6.w2.weight": "model-00049-of-00059.safetensors",
+ "model.layers.46.block_sparse_moe.experts.6.w3.weight": "model-00049-of-00059.safetensors",
+ "model.layers.46.block_sparse_moe.experts.7.w1.weight": "model-00049-of-00059.safetensors",
+ "model.layers.46.block_sparse_moe.experts.7.w2.weight": "model-00049-of-00059.safetensors",
+ "model.layers.46.block_sparse_moe.experts.7.w3.weight": "model-00049-of-00059.safetensors",
+ "model.layers.46.block_sparse_moe.gate.weight": "model-00048-of-00059.safetensors",
+ "model.layers.46.input_layernorm.weight": "model-00049-of-00059.safetensors",
+ "model.layers.46.post_attention_layernorm.weight": "model-00049-of-00059.safetensors",
+ "model.layers.46.self_attn.k_proj.weight": "model-00048-of-00059.safetensors",
+ "model.layers.46.self_attn.o_proj.weight": "model-00048-of-00059.safetensors",
+ "model.layers.46.self_attn.q_proj.weight": "model-00048-of-00059.safetensors",
+ "model.layers.46.self_attn.v_proj.weight": "model-00048-of-00059.safetensors",
+ "model.layers.47.block_sparse_moe.experts.0.w1.weight": "model-00049-of-00059.safetensors",
+ "model.layers.47.block_sparse_moe.experts.0.w2.weight": "model-00049-of-00059.safetensors",
+ "model.layers.47.block_sparse_moe.experts.0.w3.weight": "model-00050-of-00059.safetensors",
+ "model.layers.47.block_sparse_moe.experts.1.w1.weight": "model-00050-of-00059.safetensors",
+ "model.layers.47.block_sparse_moe.experts.1.w2.weight": "model-00050-of-00059.safetensors",
+ "model.layers.47.block_sparse_moe.experts.1.w3.weight": "model-00050-of-00059.safetensors",
+ "model.layers.47.block_sparse_moe.experts.2.w1.weight": "model-00050-of-00059.safetensors",
+ "model.layers.47.block_sparse_moe.experts.2.w2.weight": "model-00050-of-00059.safetensors",
+ "model.layers.47.block_sparse_moe.experts.2.w3.weight": "model-00050-of-00059.safetensors",
+ "model.layers.47.block_sparse_moe.experts.3.w1.weight": "model-00050-of-00059.safetensors",
+ "model.layers.47.block_sparse_moe.experts.3.w2.weight": "model-00050-of-00059.safetensors",
+ "model.layers.47.block_sparse_moe.experts.3.w3.weight": "model-00050-of-00059.safetensors",
+ "model.layers.47.block_sparse_moe.experts.4.w1.weight": "model-00050-of-00059.safetensors",
+ "model.layers.47.block_sparse_moe.experts.4.w2.weight": "model-00050-of-00059.safetensors",
+ "model.layers.47.block_sparse_moe.experts.4.w3.weight": "model-00050-of-00059.safetensors",
+ "model.layers.47.block_sparse_moe.experts.5.w1.weight": "model-00050-of-00059.safetensors",
+ "model.layers.47.block_sparse_moe.experts.5.w2.weight": "model-00050-of-00059.safetensors",
+ "model.layers.47.block_sparse_moe.experts.5.w3.weight": "model-00050-of-00059.safetensors",
+ "model.layers.47.block_sparse_moe.experts.6.w1.weight": "model-00050-of-00059.safetensors",
+ "model.layers.47.block_sparse_moe.experts.6.w2.weight": "model-00050-of-00059.safetensors",
+ "model.layers.47.block_sparse_moe.experts.6.w3.weight": "model-00050-of-00059.safetensors",
+ "model.layers.47.block_sparse_moe.experts.7.w1.weight": "model-00050-of-00059.safetensors",
+ "model.layers.47.block_sparse_moe.experts.7.w2.weight": "model-00050-of-00059.safetensors",
+ "model.layers.47.block_sparse_moe.experts.7.w3.weight": "model-00050-of-00059.safetensors",
+ "model.layers.47.block_sparse_moe.gate.weight": "model-00049-of-00059.safetensors",
+ "model.layers.47.input_layernorm.weight": "model-00050-of-00059.safetensors",
+ "model.layers.47.post_attention_layernorm.weight": "model-00050-of-00059.safetensors",
+ "model.layers.47.self_attn.k_proj.weight": "model-00049-of-00059.safetensors",
+ "model.layers.47.self_attn.o_proj.weight": "model-00049-of-00059.safetensors",
+ "model.layers.47.self_attn.q_proj.weight": "model-00049-of-00059.safetensors",
+ "model.layers.47.self_attn.v_proj.weight": "model-00049-of-00059.safetensors",
+ "model.layers.48.block_sparse_moe.experts.0.w1.weight": "model-00050-of-00059.safetensors",
+ "model.layers.48.block_sparse_moe.experts.0.w2.weight": "model-00051-of-00059.safetensors",
+ "model.layers.48.block_sparse_moe.experts.0.w3.weight": "model-00051-of-00059.safetensors",
+ "model.layers.48.block_sparse_moe.experts.1.w1.weight": "model-00051-of-00059.safetensors",
+ "model.layers.48.block_sparse_moe.experts.1.w2.weight": "model-00051-of-00059.safetensors",
+ "model.layers.48.block_sparse_moe.experts.1.w3.weight": "model-00051-of-00059.safetensors",
+ "model.layers.48.block_sparse_moe.experts.2.w1.weight": "model-00051-of-00059.safetensors",
+ "model.layers.48.block_sparse_moe.experts.2.w2.weight": "model-00051-of-00059.safetensors",
+ "model.layers.48.block_sparse_moe.experts.2.w3.weight": "model-00051-of-00059.safetensors",
+ "model.layers.48.block_sparse_moe.experts.3.w1.weight": "model-00051-of-00059.safetensors",
+ "model.layers.48.block_sparse_moe.experts.3.w2.weight": "model-00051-of-00059.safetensors",
+ "model.layers.48.block_sparse_moe.experts.3.w3.weight": "model-00051-of-00059.safetensors",
+ "model.layers.48.block_sparse_moe.experts.4.w1.weight": "model-00051-of-00059.safetensors",
+ "model.layers.48.block_sparse_moe.experts.4.w2.weight": "model-00051-of-00059.safetensors",
+ "model.layers.48.block_sparse_moe.experts.4.w3.weight": "model-00051-of-00059.safetensors",
+ "model.layers.48.block_sparse_moe.experts.5.w1.weight": "model-00051-of-00059.safetensors",
+ "model.layers.48.block_sparse_moe.experts.5.w2.weight": "model-00051-of-00059.safetensors",
+ "model.layers.48.block_sparse_moe.experts.5.w3.weight": "model-00051-of-00059.safetensors",
+ "model.layers.48.block_sparse_moe.experts.6.w1.weight": "model-00051-of-00059.safetensors",
+ "model.layers.48.block_sparse_moe.experts.6.w2.weight": "model-00051-of-00059.safetensors",
+ "model.layers.48.block_sparse_moe.experts.6.w3.weight": "model-00051-of-00059.safetensors",
+ "model.layers.48.block_sparse_moe.experts.7.w1.weight": "model-00051-of-00059.safetensors",
+ "model.layers.48.block_sparse_moe.experts.7.w2.weight": "model-00051-of-00059.safetensors",
+ "model.layers.48.block_sparse_moe.experts.7.w3.weight": "model-00051-of-00059.safetensors",
+ "model.layers.48.block_sparse_moe.gate.weight": "model-00050-of-00059.safetensors",
+ "model.layers.48.input_layernorm.weight": "model-00051-of-00059.safetensors",
+ "model.layers.48.post_attention_layernorm.weight": "model-00051-of-00059.safetensors",
+ "model.layers.48.self_attn.k_proj.weight": "model-00050-of-00059.safetensors",
+ "model.layers.48.self_attn.o_proj.weight": "model-00050-of-00059.safetensors",
+ "model.layers.48.self_attn.q_proj.weight": "model-00050-of-00059.safetensors",
+ "model.layers.48.self_attn.v_proj.weight": "model-00050-of-00059.safetensors",
+ "model.layers.49.block_sparse_moe.experts.0.w1.weight": "model-00052-of-00059.safetensors",
+ "model.layers.49.block_sparse_moe.experts.0.w2.weight": "model-00052-of-00059.safetensors",
+ "model.layers.49.block_sparse_moe.experts.0.w3.weight": "model-00052-of-00059.safetensors",
+ "model.layers.49.block_sparse_moe.experts.1.w1.weight": "model-00052-of-00059.safetensors",
+ "model.layers.49.block_sparse_moe.experts.1.w2.weight": "model-00052-of-00059.safetensors",
+ "model.layers.49.block_sparse_moe.experts.1.w3.weight": "model-00052-of-00059.safetensors",
+ "model.layers.49.block_sparse_moe.experts.2.w1.weight": "model-00052-of-00059.safetensors",
+ "model.layers.49.block_sparse_moe.experts.2.w2.weight": "model-00052-of-00059.safetensors",
+ "model.layers.49.block_sparse_moe.experts.2.w3.weight": "model-00052-of-00059.safetensors",
+ "model.layers.49.block_sparse_moe.experts.3.w1.weight": "model-00052-of-00059.safetensors",
+ "model.layers.49.block_sparse_moe.experts.3.w2.weight": "model-00052-of-00059.safetensors",
+ "model.layers.49.block_sparse_moe.experts.3.w3.weight": "model-00052-of-00059.safetensors",
+ "model.layers.49.block_sparse_moe.experts.4.w1.weight": "model-00052-of-00059.safetensors",
+ "model.layers.49.block_sparse_moe.experts.4.w2.weight": "model-00052-of-00059.safetensors",
+ "model.layers.49.block_sparse_moe.experts.4.w3.weight": "model-00052-of-00059.safetensors",
+ "model.layers.49.block_sparse_moe.experts.5.w1.weight": "model-00052-of-00059.safetensors",
+ "model.layers.49.block_sparse_moe.experts.5.w2.weight": "model-00052-of-00059.safetensors",
+ "model.layers.49.block_sparse_moe.experts.5.w3.weight": "model-00052-of-00059.safetensors",
+ "model.layers.49.block_sparse_moe.experts.6.w1.weight": "model-00052-of-00059.safetensors",
+ "model.layers.49.block_sparse_moe.experts.6.w2.weight": "model-00052-of-00059.safetensors",
+ "model.layers.49.block_sparse_moe.experts.6.w3.weight": "model-00052-of-00059.safetensors",
+ "model.layers.49.block_sparse_moe.experts.7.w1.weight": "model-00052-of-00059.safetensors",
+ "model.layers.49.block_sparse_moe.experts.7.w2.weight": "model-00052-of-00059.safetensors",
+ "model.layers.49.block_sparse_moe.experts.7.w3.weight": "model-00052-of-00059.safetensors",
+ "model.layers.49.block_sparse_moe.gate.weight": "model-00051-of-00059.safetensors",
+ "model.layers.49.input_layernorm.weight": "model-00052-of-00059.safetensors",
+ "model.layers.49.post_attention_layernorm.weight": "model-00052-of-00059.safetensors",
+ "model.layers.49.self_attn.k_proj.weight": "model-00051-of-00059.safetensors",
+ "model.layers.49.self_attn.o_proj.weight": "model-00051-of-00059.safetensors",
+ "model.layers.49.self_attn.q_proj.weight": "model-00051-of-00059.safetensors",
+ "model.layers.49.self_attn.v_proj.weight": "model-00051-of-00059.safetensors",
+ "model.layers.5.block_sparse_moe.experts.0.w1.weight": "model-00006-of-00059.safetensors",
+ "model.layers.5.block_sparse_moe.experts.0.w2.weight": "model-00006-of-00059.safetensors",
+ "model.layers.5.block_sparse_moe.experts.0.w3.weight": "model-00006-of-00059.safetensors",
+ "model.layers.5.block_sparse_moe.experts.1.w1.weight": "model-00006-of-00059.safetensors",
+ "model.layers.5.block_sparse_moe.experts.1.w2.weight": "model-00006-of-00059.safetensors",
+ "model.layers.5.block_sparse_moe.experts.1.w3.weight": "model-00006-of-00059.safetensors",
+ "model.layers.5.block_sparse_moe.experts.2.w1.weight": "model-00006-of-00059.safetensors",
+ "model.layers.5.block_sparse_moe.experts.2.w2.weight": "model-00006-of-00059.safetensors",
+ "model.layers.5.block_sparse_moe.experts.2.w3.weight": "model-00006-of-00059.safetensors",
+ "model.layers.5.block_sparse_moe.experts.3.w1.weight": "model-00006-of-00059.safetensors",
+ "model.layers.5.block_sparse_moe.experts.3.w2.weight": "model-00006-of-00059.safetensors",
+ "model.layers.5.block_sparse_moe.experts.3.w3.weight": "model-00006-of-00059.safetensors",
+ "model.layers.5.block_sparse_moe.experts.4.w1.weight": "model-00006-of-00059.safetensors",
+ "model.layers.5.block_sparse_moe.experts.4.w2.weight": "model-00006-of-00059.safetensors",
+ "model.layers.5.block_sparse_moe.experts.4.w3.weight": "model-00006-of-00059.safetensors",
+ "model.layers.5.block_sparse_moe.experts.5.w1.weight": "model-00006-of-00059.safetensors",
+ "model.layers.5.block_sparse_moe.experts.5.w2.weight": "model-00006-of-00059.safetensors",
+ "model.layers.5.block_sparse_moe.experts.5.w3.weight": "model-00007-of-00059.safetensors",
+ "model.layers.5.block_sparse_moe.experts.6.w1.weight": "model-00007-of-00059.safetensors",
+ "model.layers.5.block_sparse_moe.experts.6.w2.weight": "model-00007-of-00059.safetensors",
+ "model.layers.5.block_sparse_moe.experts.6.w3.weight": "model-00007-of-00059.safetensors",
+ "model.layers.5.block_sparse_moe.experts.7.w1.weight": "model-00007-of-00059.safetensors",
+ "model.layers.5.block_sparse_moe.experts.7.w2.weight": "model-00007-of-00059.safetensors",
+ "model.layers.5.block_sparse_moe.experts.7.w3.weight": "model-00007-of-00059.safetensors",
+ "model.layers.5.block_sparse_moe.gate.weight": "model-00006-of-00059.safetensors",
+ "model.layers.5.input_layernorm.weight": "model-00007-of-00059.safetensors",
+ "model.layers.5.post_attention_layernorm.weight": "model-00007-of-00059.safetensors",
+ "model.layers.5.self_attn.k_proj.weight": "model-00006-of-00059.safetensors",
+ "model.layers.5.self_attn.o_proj.weight": "model-00006-of-00059.safetensors",
+ "model.layers.5.self_attn.q_proj.weight": "model-00006-of-00059.safetensors",
+ "model.layers.5.self_attn.v_proj.weight": "model-00006-of-00059.safetensors",
+ "model.layers.50.block_sparse_moe.experts.0.w1.weight": "model-00053-of-00059.safetensors",
+ "model.layers.50.block_sparse_moe.experts.0.w2.weight": "model-00053-of-00059.safetensors",
+ "model.layers.50.block_sparse_moe.experts.0.w3.weight": "model-00053-of-00059.safetensors",
+ "model.layers.50.block_sparse_moe.experts.1.w1.weight": "model-00053-of-00059.safetensors",
+ "model.layers.50.block_sparse_moe.experts.1.w2.weight": "model-00053-of-00059.safetensors",
+ "model.layers.50.block_sparse_moe.experts.1.w3.weight": "model-00053-of-00059.safetensors",
+ "model.layers.50.block_sparse_moe.experts.2.w1.weight": "model-00053-of-00059.safetensors",
+ "model.layers.50.block_sparse_moe.experts.2.w2.weight": "model-00053-of-00059.safetensors",
+ "model.layers.50.block_sparse_moe.experts.2.w3.weight": "model-00053-of-00059.safetensors",
+ "model.layers.50.block_sparse_moe.experts.3.w1.weight": "model-00053-of-00059.safetensors",
+ "model.layers.50.block_sparse_moe.experts.3.w2.weight": "model-00053-of-00059.safetensors",
+ "model.layers.50.block_sparse_moe.experts.3.w3.weight": "model-00053-of-00059.safetensors",
+ "model.layers.50.block_sparse_moe.experts.4.w1.weight": "model-00053-of-00059.safetensors",
+ "model.layers.50.block_sparse_moe.experts.4.w2.weight": "model-00053-of-00059.safetensors",
+ "model.layers.50.block_sparse_moe.experts.4.w3.weight": "model-00053-of-00059.safetensors",
+ "model.layers.50.block_sparse_moe.experts.5.w1.weight": "model-00053-of-00059.safetensors",
+ "model.layers.50.block_sparse_moe.experts.5.w2.weight": "model-00053-of-00059.safetensors",
+ "model.layers.50.block_sparse_moe.experts.5.w3.weight": "model-00053-of-00059.safetensors",
+ "model.layers.50.block_sparse_moe.experts.6.w1.weight": "model-00053-of-00059.safetensors",
+ "model.layers.50.block_sparse_moe.experts.6.w2.weight": "model-00053-of-00059.safetensors",
+ "model.layers.50.block_sparse_moe.experts.6.w3.weight": "model-00053-of-00059.safetensors",
+ "model.layers.50.block_sparse_moe.experts.7.w1.weight": "model-00053-of-00059.safetensors",
+ "model.layers.50.block_sparse_moe.experts.7.w2.weight": "model-00053-of-00059.safetensors",
+ "model.layers.50.block_sparse_moe.experts.7.w3.weight": "model-00053-of-00059.safetensors",
+ "model.layers.50.block_sparse_moe.gate.weight": "model-00053-of-00059.safetensors",
+ "model.layers.50.input_layernorm.weight": "model-00053-of-00059.safetensors",
+ "model.layers.50.post_attention_layernorm.weight": "model-00053-of-00059.safetensors",
+ "model.layers.50.self_attn.k_proj.weight": "model-00052-of-00059.safetensors",
+ "model.layers.50.self_attn.o_proj.weight": "model-00053-of-00059.safetensors",
+ "model.layers.50.self_attn.q_proj.weight": "model-00052-of-00059.safetensors",
+ "model.layers.50.self_attn.v_proj.weight": "model-00052-of-00059.safetensors",
+ "model.layers.51.block_sparse_moe.experts.0.w1.weight": "model-00054-of-00059.safetensors",
+ "model.layers.51.block_sparse_moe.experts.0.w2.weight": "model-00054-of-00059.safetensors",
+ "model.layers.51.block_sparse_moe.experts.0.w3.weight": "model-00054-of-00059.safetensors",
+ "model.layers.51.block_sparse_moe.experts.1.w1.weight": "model-00054-of-00059.safetensors",
+ "model.layers.51.block_sparse_moe.experts.1.w2.weight": "model-00054-of-00059.safetensors",
+ "model.layers.51.block_sparse_moe.experts.1.w3.weight": "model-00054-of-00059.safetensors",
+ "model.layers.51.block_sparse_moe.experts.2.w1.weight": "model-00054-of-00059.safetensors",
+ "model.layers.51.block_sparse_moe.experts.2.w2.weight": "model-00054-of-00059.safetensors",
+ "model.layers.51.block_sparse_moe.experts.2.w3.weight": "model-00054-of-00059.safetensors",
+ "model.layers.51.block_sparse_moe.experts.3.w1.weight": "model-00054-of-00059.safetensors",
+ "model.layers.51.block_sparse_moe.experts.3.w2.weight": "model-00054-of-00059.safetensors",
+ "model.layers.51.block_sparse_moe.experts.3.w3.weight": "model-00054-of-00059.safetensors",
+ "model.layers.51.block_sparse_moe.experts.4.w1.weight": "model-00054-of-00059.safetensors",
+ "model.layers.51.block_sparse_moe.experts.4.w2.weight": "model-00054-of-00059.safetensors",
+ "model.layers.51.block_sparse_moe.experts.4.w3.weight": "model-00054-of-00059.safetensors",
+ "model.layers.51.block_sparse_moe.experts.5.w1.weight": "model-00054-of-00059.safetensors",
+ "model.layers.51.block_sparse_moe.experts.5.w2.weight": "model-00054-of-00059.safetensors",
+ "model.layers.51.block_sparse_moe.experts.5.w3.weight": "model-00054-of-00059.safetensors",
+ "model.layers.51.block_sparse_moe.experts.6.w1.weight": "model-00054-of-00059.safetensors",
+ "model.layers.51.block_sparse_moe.experts.6.w2.weight": "model-00054-of-00059.safetensors",
+ "model.layers.51.block_sparse_moe.experts.6.w3.weight": "model-00054-of-00059.safetensors",
+ "model.layers.51.block_sparse_moe.experts.7.w1.weight": "model-00054-of-00059.safetensors",
+ "model.layers.51.block_sparse_moe.experts.7.w2.weight": "model-00054-of-00059.safetensors",
+ "model.layers.51.block_sparse_moe.experts.7.w3.weight": "model-00054-of-00059.safetensors",
+ "model.layers.51.block_sparse_moe.gate.weight": "model-00054-of-00059.safetensors",
+ "model.layers.51.input_layernorm.weight": "model-00054-of-00059.safetensors",
+ "model.layers.51.post_attention_layernorm.weight": "model-00054-of-00059.safetensors",
+ "model.layers.51.self_attn.k_proj.weight": "model-00053-of-00059.safetensors",
+ "model.layers.51.self_attn.o_proj.weight": "model-00054-of-00059.safetensors",
+ "model.layers.51.self_attn.q_proj.weight": "model-00053-of-00059.safetensors",
+ "model.layers.51.self_attn.v_proj.weight": "model-00054-of-00059.safetensors",
+ "model.layers.52.block_sparse_moe.experts.0.w1.weight": "model-00055-of-00059.safetensors",
+ "model.layers.52.block_sparse_moe.experts.0.w2.weight": "model-00055-of-00059.safetensors",
+ "model.layers.52.block_sparse_moe.experts.0.w3.weight": "model-00055-of-00059.safetensors",
+ "model.layers.52.block_sparse_moe.experts.1.w1.weight": "model-00055-of-00059.safetensors",
+ "model.layers.52.block_sparse_moe.experts.1.w2.weight": "model-00055-of-00059.safetensors",
+ "model.layers.52.block_sparse_moe.experts.1.w3.weight": "model-00055-of-00059.safetensors",
+ "model.layers.52.block_sparse_moe.experts.2.w1.weight": "model-00055-of-00059.safetensors",
+ "model.layers.52.block_sparse_moe.experts.2.w2.weight": "model-00055-of-00059.safetensors",
+ "model.layers.52.block_sparse_moe.experts.2.w3.weight": "model-00055-of-00059.safetensors",
+ "model.layers.52.block_sparse_moe.experts.3.w1.weight": "model-00055-of-00059.safetensors",
+ "model.layers.52.block_sparse_moe.experts.3.w2.weight": "model-00055-of-00059.safetensors",
+ "model.layers.52.block_sparse_moe.experts.3.w3.weight": "model-00055-of-00059.safetensors",
+ "model.layers.52.block_sparse_moe.experts.4.w1.weight": "model-00055-of-00059.safetensors",
+ "model.layers.52.block_sparse_moe.experts.4.w2.weight": "model-00055-of-00059.safetensors",
+ "model.layers.52.block_sparse_moe.experts.4.w3.weight": "model-00055-of-00059.safetensors",
+ "model.layers.52.block_sparse_moe.experts.5.w1.weight": "model-00055-of-00059.safetensors",
+ "model.layers.52.block_sparse_moe.experts.5.w2.weight": "model-00055-of-00059.safetensors",
+ "model.layers.52.block_sparse_moe.experts.5.w3.weight": "model-00055-of-00059.safetensors",
+ "model.layers.52.block_sparse_moe.experts.6.w1.weight": "model-00055-of-00059.safetensors",
+ "model.layers.52.block_sparse_moe.experts.6.w2.weight": "model-00055-of-00059.safetensors",
+ "model.layers.52.block_sparse_moe.experts.6.w3.weight": "model-00055-of-00059.safetensors",
+ "model.layers.52.block_sparse_moe.experts.7.w1.weight": "model-00055-of-00059.safetensors",
+ "model.layers.52.block_sparse_moe.experts.7.w2.weight": "model-00055-of-00059.safetensors",
+ "model.layers.52.block_sparse_moe.experts.7.w3.weight": "model-00055-of-00059.safetensors",
+ "model.layers.52.block_sparse_moe.gate.weight": "model-00055-of-00059.safetensors",
+ "model.layers.52.input_layernorm.weight": "model-00055-of-00059.safetensors",
+ "model.layers.52.post_attention_layernorm.weight": "model-00055-of-00059.safetensors",
+ "model.layers.52.self_attn.k_proj.weight": "model-00055-of-00059.safetensors",
+ "model.layers.52.self_attn.o_proj.weight": "model-00055-of-00059.safetensors",
+ "model.layers.52.self_attn.q_proj.weight": "model-00054-of-00059.safetensors",
+ "model.layers.52.self_attn.v_proj.weight": "model-00055-of-00059.safetensors",
+ "model.layers.53.block_sparse_moe.experts.0.w1.weight": "model-00056-of-00059.safetensors",
+ "model.layers.53.block_sparse_moe.experts.0.w2.weight": "model-00056-of-00059.safetensors",
+ "model.layers.53.block_sparse_moe.experts.0.w3.weight": "model-00056-of-00059.safetensors",
+ "model.layers.53.block_sparse_moe.experts.1.w1.weight": "model-00056-of-00059.safetensors",
+ "model.layers.53.block_sparse_moe.experts.1.w2.weight": "model-00056-of-00059.safetensors",
+ "model.layers.53.block_sparse_moe.experts.1.w3.weight": "model-00056-of-00059.safetensors",
+ "model.layers.53.block_sparse_moe.experts.2.w1.weight": "model-00056-of-00059.safetensors",
+ "model.layers.53.block_sparse_moe.experts.2.w2.weight": "model-00056-of-00059.safetensors",
+ "model.layers.53.block_sparse_moe.experts.2.w3.weight": "model-00056-of-00059.safetensors",
+ "model.layers.53.block_sparse_moe.experts.3.w1.weight": "model-00056-of-00059.safetensors",
+ "model.layers.53.block_sparse_moe.experts.3.w2.weight": "model-00056-of-00059.safetensors",
+ "model.layers.53.block_sparse_moe.experts.3.w3.weight": "model-00056-of-00059.safetensors",
+ "model.layers.53.block_sparse_moe.experts.4.w1.weight": "model-00056-of-00059.safetensors",
+ "model.layers.53.block_sparse_moe.experts.4.w2.weight": "model-00056-of-00059.safetensors",
+ "model.layers.53.block_sparse_moe.experts.4.w3.weight": "model-00056-of-00059.safetensors",
+ "model.layers.53.block_sparse_moe.experts.5.w1.weight": "model-00056-of-00059.safetensors",
+ "model.layers.53.block_sparse_moe.experts.5.w2.weight": "model-00056-of-00059.safetensors",
+ "model.layers.53.block_sparse_moe.experts.5.w3.weight": "model-00056-of-00059.safetensors",
+ "model.layers.53.block_sparse_moe.experts.6.w1.weight": "model-00056-of-00059.safetensors",
+ "model.layers.53.block_sparse_moe.experts.6.w2.weight": "model-00056-of-00059.safetensors",
+ "model.layers.53.block_sparse_moe.experts.6.w3.weight": "model-00056-of-00059.safetensors",
+ "model.layers.53.block_sparse_moe.experts.7.w1.weight": "model-00056-of-00059.safetensors",
+ "model.layers.53.block_sparse_moe.experts.7.w2.weight": "model-00056-of-00059.safetensors",
+ "model.layers.53.block_sparse_moe.experts.7.w3.weight": "model-00057-of-00059.safetensors",
+ "model.layers.53.block_sparse_moe.gate.weight": "model-00056-of-00059.safetensors",
+ "model.layers.53.input_layernorm.weight": "model-00057-of-00059.safetensors",
+ "model.layers.53.post_attention_layernorm.weight": "model-00057-of-00059.safetensors",
+ "model.layers.53.self_attn.k_proj.weight": "model-00056-of-00059.safetensors",
+ "model.layers.53.self_attn.o_proj.weight": "model-00056-of-00059.safetensors",
+ "model.layers.53.self_attn.q_proj.weight": "model-00056-of-00059.safetensors",
+ "model.layers.53.self_attn.v_proj.weight": "model-00056-of-00059.safetensors",
+ "model.layers.54.block_sparse_moe.experts.0.w1.weight": "model-00057-of-00059.safetensors",
+ "model.layers.54.block_sparse_moe.experts.0.w2.weight": "model-00057-of-00059.safetensors",
+ "model.layers.54.block_sparse_moe.experts.0.w3.weight": "model-00057-of-00059.safetensors",
+ "model.layers.54.block_sparse_moe.experts.1.w1.weight": "model-00057-of-00059.safetensors",
+ "model.layers.54.block_sparse_moe.experts.1.w2.weight": "model-00057-of-00059.safetensors",
+ "model.layers.54.block_sparse_moe.experts.1.w3.weight": "model-00057-of-00059.safetensors",
+ "model.layers.54.block_sparse_moe.experts.2.w1.weight": "model-00057-of-00059.safetensors",
+ "model.layers.54.block_sparse_moe.experts.2.w2.weight": "model-00057-of-00059.safetensors",
+ "model.layers.54.block_sparse_moe.experts.2.w3.weight": "model-00057-of-00059.safetensors",
+ "model.layers.54.block_sparse_moe.experts.3.w1.weight": "model-00057-of-00059.safetensors",
+ "model.layers.54.block_sparse_moe.experts.3.w2.weight": "model-00057-of-00059.safetensors",
+ "model.layers.54.block_sparse_moe.experts.3.w3.weight": "model-00057-of-00059.safetensors",
+ "model.layers.54.block_sparse_moe.experts.4.w1.weight": "model-00057-of-00059.safetensors",
+ "model.layers.54.block_sparse_moe.experts.4.w2.weight": "model-00057-of-00059.safetensors",
+ "model.layers.54.block_sparse_moe.experts.4.w3.weight": "model-00057-of-00059.safetensors",
+ "model.layers.54.block_sparse_moe.experts.5.w1.weight": "model-00057-of-00059.safetensors",
+ "model.layers.54.block_sparse_moe.experts.5.w2.weight": "model-00057-of-00059.safetensors",
+ "model.layers.54.block_sparse_moe.experts.5.w3.weight": "model-00057-of-00059.safetensors",
+ "model.layers.54.block_sparse_moe.experts.6.w1.weight": "model-00057-of-00059.safetensors",
+ "model.layers.54.block_sparse_moe.experts.6.w2.weight": "model-00057-of-00059.safetensors",
+ "model.layers.54.block_sparse_moe.experts.6.w3.weight": "model-00057-of-00059.safetensors",
+ "model.layers.54.block_sparse_moe.experts.7.w1.weight": "model-00057-of-00059.safetensors",
+ "model.layers.54.block_sparse_moe.experts.7.w2.weight": "model-00058-of-00059.safetensors",
+ "model.layers.54.block_sparse_moe.experts.7.w3.weight": "model-00058-of-00059.safetensors",
+ "model.layers.54.block_sparse_moe.gate.weight": "model-00057-of-00059.safetensors",
+ "model.layers.54.input_layernorm.weight": "model-00058-of-00059.safetensors",
+ "model.layers.54.post_attention_layernorm.weight": "model-00058-of-00059.safetensors",
+ "model.layers.54.self_attn.k_proj.weight": "model-00057-of-00059.safetensors",
+ "model.layers.54.self_attn.o_proj.weight": "model-00057-of-00059.safetensors",
+ "model.layers.54.self_attn.q_proj.weight": "model-00057-of-00059.safetensors",
+ "model.layers.54.self_attn.v_proj.weight": "model-00057-of-00059.safetensors",
+ "model.layers.55.block_sparse_moe.experts.0.w1.weight": "model-00058-of-00059.safetensors",
+ "model.layers.55.block_sparse_moe.experts.0.w2.weight": "model-00058-of-00059.safetensors",
+ "model.layers.55.block_sparse_moe.experts.0.w3.weight": "model-00058-of-00059.safetensors",
+ "model.layers.55.block_sparse_moe.experts.1.w1.weight": "model-00058-of-00059.safetensors",
+ "model.layers.55.block_sparse_moe.experts.1.w2.weight": "model-00058-of-00059.safetensors",
+ "model.layers.55.block_sparse_moe.experts.1.w3.weight": "model-00058-of-00059.safetensors",
+ "model.layers.55.block_sparse_moe.experts.2.w1.weight": "model-00058-of-00059.safetensors",
+ "model.layers.55.block_sparse_moe.experts.2.w2.weight": "model-00058-of-00059.safetensors",
+ "model.layers.55.block_sparse_moe.experts.2.w3.weight": "model-00058-of-00059.safetensors",
+ "model.layers.55.block_sparse_moe.experts.3.w1.weight": "model-00058-of-00059.safetensors",
+ "model.layers.55.block_sparse_moe.experts.3.w2.weight": "model-00058-of-00059.safetensors",
+ "model.layers.55.block_sparse_moe.experts.3.w3.weight": "model-00058-of-00059.safetensors",
+ "model.layers.55.block_sparse_moe.experts.4.w1.weight": "model-00058-of-00059.safetensors",
+ "model.layers.55.block_sparse_moe.experts.4.w2.weight": "model-00058-of-00059.safetensors",
+ "model.layers.55.block_sparse_moe.experts.4.w3.weight": "model-00058-of-00059.safetensors",
+ "model.layers.55.block_sparse_moe.experts.5.w1.weight": "model-00058-of-00059.safetensors",
+ "model.layers.55.block_sparse_moe.experts.5.w2.weight": "model-00058-of-00059.safetensors",
+ "model.layers.55.block_sparse_moe.experts.5.w3.weight": "model-00058-of-00059.safetensors",
+ "model.layers.55.block_sparse_moe.experts.6.w1.weight": "model-00058-of-00059.safetensors",
+ "model.layers.55.block_sparse_moe.experts.6.w2.weight": "model-00058-of-00059.safetensors",
+ "model.layers.55.block_sparse_moe.experts.6.w3.weight": "model-00058-of-00059.safetensors",
+ "model.layers.55.block_sparse_moe.experts.7.w1.weight": "model-00059-of-00059.safetensors",
+ "model.layers.55.block_sparse_moe.experts.7.w2.weight": "model-00059-of-00059.safetensors",
+ "model.layers.55.block_sparse_moe.experts.7.w3.weight": "model-00059-of-00059.safetensors",
+ "model.layers.55.block_sparse_moe.gate.weight": "model-00058-of-00059.safetensors",
+ "model.layers.55.input_layernorm.weight": "model-00059-of-00059.safetensors",
+ "model.layers.55.post_attention_layernorm.weight": "model-00059-of-00059.safetensors",
+ "model.layers.55.self_attn.k_proj.weight": "model-00058-of-00059.safetensors",
+ "model.layers.55.self_attn.o_proj.weight": "model-00058-of-00059.safetensors",
+ "model.layers.55.self_attn.q_proj.weight": "model-00058-of-00059.safetensors",
+ "model.layers.55.self_attn.v_proj.weight": "model-00058-of-00059.safetensors",
+ "model.layers.6.block_sparse_moe.experts.0.w1.weight": "model-00007-of-00059.safetensors",
+ "model.layers.6.block_sparse_moe.experts.0.w2.weight": "model-00007-of-00059.safetensors",
+ "model.layers.6.block_sparse_moe.experts.0.w3.weight": "model-00007-of-00059.safetensors",
+ "model.layers.6.block_sparse_moe.experts.1.w1.weight": "model-00007-of-00059.safetensors",
+ "model.layers.6.block_sparse_moe.experts.1.w2.weight": "model-00007-of-00059.safetensors",
+ "model.layers.6.block_sparse_moe.experts.1.w3.weight": "model-00007-of-00059.safetensors",
+ "model.layers.6.block_sparse_moe.experts.2.w1.weight": "model-00007-of-00059.safetensors",
+ "model.layers.6.block_sparse_moe.experts.2.w2.weight": "model-00007-of-00059.safetensors",
+ "model.layers.6.block_sparse_moe.experts.2.w3.weight": "model-00007-of-00059.safetensors",
+ "model.layers.6.block_sparse_moe.experts.3.w1.weight": "model-00007-of-00059.safetensors",
+ "model.layers.6.block_sparse_moe.experts.3.w2.weight": "model-00007-of-00059.safetensors",
+ "model.layers.6.block_sparse_moe.experts.3.w3.weight": "model-00007-of-00059.safetensors",
+ "model.layers.6.block_sparse_moe.experts.4.w1.weight": "model-00007-of-00059.safetensors",
+ "model.layers.6.block_sparse_moe.experts.4.w2.weight": "model-00007-of-00059.safetensors",
+ "model.layers.6.block_sparse_moe.experts.4.w3.weight": "model-00007-of-00059.safetensors",
+ "model.layers.6.block_sparse_moe.experts.5.w1.weight": "model-00007-of-00059.safetensors",
+ "model.layers.6.block_sparse_moe.experts.5.w2.weight": "model-00008-of-00059.safetensors",
+ "model.layers.6.block_sparse_moe.experts.5.w3.weight": "model-00008-of-00059.safetensors",
+ "model.layers.6.block_sparse_moe.experts.6.w1.weight": "model-00008-of-00059.safetensors",
+ "model.layers.6.block_sparse_moe.experts.6.w2.weight": "model-00008-of-00059.safetensors",
+ "model.layers.6.block_sparse_moe.experts.6.w3.weight": "model-00008-of-00059.safetensors",
+ "model.layers.6.block_sparse_moe.experts.7.w1.weight": "model-00008-of-00059.safetensors",
+ "model.layers.6.block_sparse_moe.experts.7.w2.weight": "model-00008-of-00059.safetensors",
+ "model.layers.6.block_sparse_moe.experts.7.w3.weight": "model-00008-of-00059.safetensors",
+ "model.layers.6.block_sparse_moe.gate.weight": "model-00007-of-00059.safetensors",
+ "model.layers.6.input_layernorm.weight": "model-00008-of-00059.safetensors",
+ "model.layers.6.post_attention_layernorm.weight": "model-00008-of-00059.safetensors",
+ "model.layers.6.self_attn.k_proj.weight": "model-00007-of-00059.safetensors",
+ "model.layers.6.self_attn.o_proj.weight": "model-00007-of-00059.safetensors",
+ "model.layers.6.self_attn.q_proj.weight": "model-00007-of-00059.safetensors",
+ "model.layers.6.self_attn.v_proj.weight": "model-00007-of-00059.safetensors",
+ "model.layers.7.block_sparse_moe.experts.0.w1.weight": "model-00008-of-00059.safetensors",
+ "model.layers.7.block_sparse_moe.experts.0.w2.weight": "model-00008-of-00059.safetensors",
+ "model.layers.7.block_sparse_moe.experts.0.w3.weight": "model-00008-of-00059.safetensors",
+ "model.layers.7.block_sparse_moe.experts.1.w1.weight": "model-00008-of-00059.safetensors",
+ "model.layers.7.block_sparse_moe.experts.1.w2.weight": "model-00008-of-00059.safetensors",
+ "model.layers.7.block_sparse_moe.experts.1.w3.weight": "model-00008-of-00059.safetensors",
+ "model.layers.7.block_sparse_moe.experts.2.w1.weight": "model-00008-of-00059.safetensors",
+ "model.layers.7.block_sparse_moe.experts.2.w2.weight": "model-00008-of-00059.safetensors",
+ "model.layers.7.block_sparse_moe.experts.2.w3.weight": "model-00008-of-00059.safetensors",
+ "model.layers.7.block_sparse_moe.experts.3.w1.weight": "model-00008-of-00059.safetensors",
+ "model.layers.7.block_sparse_moe.experts.3.w2.weight": "model-00008-of-00059.safetensors",
+ "model.layers.7.block_sparse_moe.experts.3.w3.weight": "model-00008-of-00059.safetensors",
+ "model.layers.7.block_sparse_moe.experts.4.w1.weight": "model-00008-of-00059.safetensors",
+ "model.layers.7.block_sparse_moe.experts.4.w2.weight": "model-00008-of-00059.safetensors",
+ "model.layers.7.block_sparse_moe.experts.4.w3.weight": "model-00008-of-00059.safetensors",
+ "model.layers.7.block_sparse_moe.experts.5.w1.weight": "model-00009-of-00059.safetensors",
+ "model.layers.7.block_sparse_moe.experts.5.w2.weight": "model-00009-of-00059.safetensors",
+ "model.layers.7.block_sparse_moe.experts.5.w3.weight": "model-00009-of-00059.safetensors",
+ "model.layers.7.block_sparse_moe.experts.6.w1.weight": "model-00009-of-00059.safetensors",
+ "model.layers.7.block_sparse_moe.experts.6.w2.weight": "model-00009-of-00059.safetensors",
+ "model.layers.7.block_sparse_moe.experts.6.w3.weight": "model-00009-of-00059.safetensors",
+ "model.layers.7.block_sparse_moe.experts.7.w1.weight": "model-00009-of-00059.safetensors",
+ "model.layers.7.block_sparse_moe.experts.7.w2.weight": "model-00009-of-00059.safetensors",
+ "model.layers.7.block_sparse_moe.experts.7.w3.weight": "model-00009-of-00059.safetensors",
+ "model.layers.7.block_sparse_moe.gate.weight": "model-00008-of-00059.safetensors",
+ "model.layers.7.input_layernorm.weight": "model-00009-of-00059.safetensors",
+ "model.layers.7.post_attention_layernorm.weight": "model-00009-of-00059.safetensors",
+ "model.layers.7.self_attn.k_proj.weight": "model-00008-of-00059.safetensors",
+ "model.layers.7.self_attn.o_proj.weight": "model-00008-of-00059.safetensors",
+ "model.layers.7.self_attn.q_proj.weight": "model-00008-of-00059.safetensors",
+ "model.layers.7.self_attn.v_proj.weight": "model-00008-of-00059.safetensors",
+ "model.layers.8.block_sparse_moe.experts.0.w1.weight": "model-00009-of-00059.safetensors",
+ "model.layers.8.block_sparse_moe.experts.0.w2.weight": "model-00009-of-00059.safetensors",
+ "model.layers.8.block_sparse_moe.experts.0.w3.weight": "model-00009-of-00059.safetensors",
+ "model.layers.8.block_sparse_moe.experts.1.w1.weight": "model-00009-of-00059.safetensors",
+ "model.layers.8.block_sparse_moe.experts.1.w2.weight": "model-00009-of-00059.safetensors",
+ "model.layers.8.block_sparse_moe.experts.1.w3.weight": "model-00009-of-00059.safetensors",
+ "model.layers.8.block_sparse_moe.experts.2.w1.weight": "model-00009-of-00059.safetensors",
+ "model.layers.8.block_sparse_moe.experts.2.w2.weight": "model-00009-of-00059.safetensors",
+ "model.layers.8.block_sparse_moe.experts.2.w3.weight": "model-00009-of-00059.safetensors",
+ "model.layers.8.block_sparse_moe.experts.3.w1.weight": "model-00009-of-00059.safetensors",
+ "model.layers.8.block_sparse_moe.experts.3.w2.weight": "model-00009-of-00059.safetensors",
+ "model.layers.8.block_sparse_moe.experts.3.w3.weight": "model-00009-of-00059.safetensors",
+ "model.layers.8.block_sparse_moe.experts.4.w1.weight": "model-00009-of-00059.safetensors",
+ "model.layers.8.block_sparse_moe.experts.4.w2.weight": "model-00009-of-00059.safetensors",
+ "model.layers.8.block_sparse_moe.experts.4.w3.weight": "model-00010-of-00059.safetensors",
+ "model.layers.8.block_sparse_moe.experts.5.w1.weight": "model-00010-of-00059.safetensors",
+ "model.layers.8.block_sparse_moe.experts.5.w2.weight": "model-00010-of-00059.safetensors",
+ "model.layers.8.block_sparse_moe.experts.5.w3.weight": "model-00010-of-00059.safetensors",
+ "model.layers.8.block_sparse_moe.experts.6.w1.weight": "model-00010-of-00059.safetensors",
+ "model.layers.8.block_sparse_moe.experts.6.w2.weight": "model-00010-of-00059.safetensors",
+ "model.layers.8.block_sparse_moe.experts.6.w3.weight": "model-00010-of-00059.safetensors",
+ "model.layers.8.block_sparse_moe.experts.7.w1.weight": "model-00010-of-00059.safetensors",
+ "model.layers.8.block_sparse_moe.experts.7.w2.weight": "model-00010-of-00059.safetensors",
+ "model.layers.8.block_sparse_moe.experts.7.w3.weight": "model-00010-of-00059.safetensors",
+ "model.layers.8.block_sparse_moe.gate.weight": "model-00009-of-00059.safetensors",
+ "model.layers.8.input_layernorm.weight": "model-00010-of-00059.safetensors",
+ "model.layers.8.post_attention_layernorm.weight": "model-00010-of-00059.safetensors",
+ "model.layers.8.self_attn.k_proj.weight": "model-00009-of-00059.safetensors",
+ "model.layers.8.self_attn.o_proj.weight": "model-00009-of-00059.safetensors",
+ "model.layers.8.self_attn.q_proj.weight": "model-00009-of-00059.safetensors",
+ "model.layers.8.self_attn.v_proj.weight": "model-00009-of-00059.safetensors",
+ "model.layers.9.block_sparse_moe.experts.0.w1.weight": "model-00010-of-00059.safetensors",
+ "model.layers.9.block_sparse_moe.experts.0.w2.weight": "model-00010-of-00059.safetensors",
+ "model.layers.9.block_sparse_moe.experts.0.w3.weight": "model-00010-of-00059.safetensors",
+ "model.layers.9.block_sparse_moe.experts.1.w1.weight": "model-00010-of-00059.safetensors",
+ "model.layers.9.block_sparse_moe.experts.1.w2.weight": "model-00010-of-00059.safetensors",
+ "model.layers.9.block_sparse_moe.experts.1.w3.weight": "model-00010-of-00059.safetensors",
+ "model.layers.9.block_sparse_moe.experts.2.w1.weight": "model-00010-of-00059.safetensors",
+ "model.layers.9.block_sparse_moe.experts.2.w2.weight": "model-00010-of-00059.safetensors",
+ "model.layers.9.block_sparse_moe.experts.2.w3.weight": "model-00010-of-00059.safetensors",
+ "model.layers.9.block_sparse_moe.experts.3.w1.weight": "model-00010-of-00059.safetensors",
+ "model.layers.9.block_sparse_moe.experts.3.w2.weight": "model-00010-of-00059.safetensors",
+ "model.layers.9.block_sparse_moe.experts.3.w3.weight": "model-00010-of-00059.safetensors",
+ "model.layers.9.block_sparse_moe.experts.4.w1.weight": "model-00010-of-00059.safetensors",
+ "model.layers.9.block_sparse_moe.experts.4.w2.weight": "model-00011-of-00059.safetensors",
+ "model.layers.9.block_sparse_moe.experts.4.w3.weight": "model-00011-of-00059.safetensors",
+ "model.layers.9.block_sparse_moe.experts.5.w1.weight": "model-00011-of-00059.safetensors",
+ "model.layers.9.block_sparse_moe.experts.5.w2.weight": "model-00011-of-00059.safetensors",
+ "model.layers.9.block_sparse_moe.experts.5.w3.weight": "model-00011-of-00059.safetensors",
+ "model.layers.9.block_sparse_moe.experts.6.w1.weight": "model-00011-of-00059.safetensors",
+ "model.layers.9.block_sparse_moe.experts.6.w2.weight": "model-00011-of-00059.safetensors",
+ "model.layers.9.block_sparse_moe.experts.6.w3.weight": "model-00011-of-00059.safetensors",
+ "model.layers.9.block_sparse_moe.experts.7.w1.weight": "model-00011-of-00059.safetensors",
+ "model.layers.9.block_sparse_moe.experts.7.w2.weight": "model-00011-of-00059.safetensors",
+ "model.layers.9.block_sparse_moe.experts.7.w3.weight": "model-00011-of-00059.safetensors",
+ "model.layers.9.block_sparse_moe.gate.weight": "model-00010-of-00059.safetensors",
+ "model.layers.9.input_layernorm.weight": "model-00011-of-00059.safetensors",
+ "model.layers.9.post_attention_layernorm.weight": "model-00011-of-00059.safetensors",
+ "model.layers.9.self_attn.k_proj.weight": "model-00010-of-00059.safetensors",
+ "model.layers.9.self_attn.o_proj.weight": "model-00010-of-00059.safetensors",
+ "model.layers.9.self_attn.q_proj.weight": "model-00010-of-00059.safetensors",
+ "model.layers.9.self_attn.v_proj.weight": "model-00010-of-00059.safetensors",
+ "model.norm.weight": "model-00059-of-00059.safetensors"
+ }
+}
diff --git a/special_tokens_map.json b/special_tokens_map.json
new file mode 100644
index 0000000000000000000000000000000000000000..14761dcf1466dc232bd41de9c21d4c617b15755e
--- /dev/null
+++ b/special_tokens_map.json
@@ -0,0 +1,24 @@
+{
+ "bos_token": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false
+ },
+ "eos_token": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false
+ },
+ "pad_token": "",
+ "unk_token": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false
+ }
+}
diff --git a/tokenizer.model b/tokenizer.model
new file mode 100644
index 0000000000000000000000000000000000000000..8b443ef19c2a19acc3ac64fb9c3db4a72921dff6
--- /dev/null
+++ b/tokenizer.model
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443
diff --git a/tokenizer_config.json b/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..123b0918cd371d90ed74371b83058aa842ef74d2
--- /dev/null
+++ b/tokenizer_config.json
@@ -0,0 +1,43 @@
+{
+ "add_bos_token": true,
+ "add_eos_token": false,
+ "added_tokens_decoder": {
+ "0": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "1": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "2": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ }
+ },
+ "additional_special_tokens": [],
+ "bos_token": "",
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "",
+ "legacy": true,
+ "model_max_length": 1000000000000000019884624838656,
+ "pad_token": "",
+ "padding_side": "right",
+ "sp_model_kwargs": {},
+ "spaces_between_special_tokens": false,
+ "tokenizer_class": "LlamaTokenizer",
+ "unk_token": "",
+ "use_default_system_prompt": false
+}