Update config

README.md

@@ -1,8 +1,8 @@
 ---
 base_model:
-- rombodawg/Rombos-Coder-V2.5-Qwen-32b
 - FuseAI/FuseO1-DeepSeekR1-Qwen2.5-Coder-32B-Preview
 - Qwen/Qwen2.5-Coder-32B
+- rombodawg/Rombos-Coder-V2.5-Qwen-32b
 library_name: transformers
 tags:
 - mergekit
@@ -21,23 +21,25 @@ This model was merged using the sce merge method using [Qwen/Qwen2.5-Coder-32B](
 ### Models Merged
 
 The following models were included in the merge:
-* [rombodawg/Rombos-Coder-V2.5-Qwen-32b](https://huggingface.co/rombodawg/Rombos-Coder-V2.5-Qwen-32b)
 * [FuseAI/FuseO1-DeepSeekR1-Qwen2.5-Coder-32B-Preview](https://huggingface.co/FuseAI/FuseO1-DeepSeekR1-Qwen2.5-Coder-32B-Preview)
+* [rombodawg/Rombos-Coder-V2.5-Qwen-32b](https://huggingface.co/rombodawg/Rombos-Coder-V2.5-Qwen-32b)
 
 ### Configuration
 
 The following YAML configuration was used to produce this model:
 
 ```yaml
-models:
-  - model: rombodawg/Rombos-Coder-V2.5-Qwen-32b
-  - model: FuseAI/FuseO1-DeepSeekR1-Qwen2.5-Coder-32B-Preview
-
-merge_method: sce
 base_model: Qwen/Qwen2.5-Coder-32B
-
-parameters:
-  select_topk: 1.0       # or whatever fraction/threshold you prefer
 dtype: bfloat16
-
+merge_method: sce
+parameters:
+  select_topk: 1.0
+slices:
+- sources:
+  - layer_range: [0, 64]
+    model: rombodawg/Rombos-Coder-V2.5-Qwen-32b
+  - layer_range: [0, 64]
+    model: FuseAI/FuseO1-DeepSeekR1-Qwen2.5-Coder-32B-Preview
+  - layer_range: [0, 64]
+    model: Qwen/Qwen2.5-Coder-32B
 ```

config.json

@@ -1,5 +1,4 @@
 {
-  "_name_or_path": "Qwen/Qwen2.5-Coder-32B",
   "architectures": [
     "Qwen2ForCausalLM"
   ],
@@ -10,19 +9,18 @@
   "hidden_size": 5120,
   "initializer_range": 0.02,
   "intermediate_size": 27648,
-  "max_position_embeddings": 32768,
+  "max_position_embeddings": 131072,
   "max_window_layers": 64,
   "model_type": "qwen2",
   "num_attention_heads": 40,
   "num_hidden_layers": 64,
   "num_key_value_heads": 8,
-  "rms_norm_eps": 1e-06,
-  "rope_scaling": null,
+  "rms_norm_eps": 1e-05,
   "rope_theta": 1000000.0,
-  "sliding_window": null,
+  "sliding_window": 131072,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.48.2",
+  "transformers_version": "4.43.1",
   "use_cache": true,
   "use_sliding_window": false,
   "vocab_size": 152064

fusechat-sce.yml

@@ -1,10 +1,13 @@
-models:
-  - model: rombodawg/Rombos-Coder-V2.5-Qwen-32b
-  - model: FuseAI/FuseO1-DeepSeekR1-Qwen2.5-Coder-32B-Preview
-
-merge_method: sce
 base_model: Qwen/Qwen2.5-Coder-32B
-
-parameters:
-  select_topk: 1.0       # or whatever fraction/threshold you prefer
 dtype: bfloat16
+merge_method: sce
+parameters:
+  select_topk: 1.0
+slices:
+- sources:
+  - layer_range: [0, 64]
+    model: rombodawg/Rombos-Coder-V2.5-Qwen-32b
+  - layer_range: [0, 64]
+    model: FuseAI/FuseO1-DeepSeekR1-Qwen2.5-Coder-32B-Preview
+  - layer_range: [0, 64]
+    model: Qwen/Qwen2.5-Coder-32B

generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 151646,
+  "eos_token_id": 151643,
+  "do_sample": true,
+  "temperature": 0.6,
+  "top_p": 0.95,
+  "transformers_version": "4.39.3"
+}

model-00001-of-00014.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5ba1c23eb69a3f3b3487b2bc6b87e4427dc160bcd4ee61fe3c114a5e4010de17
+oid sha256:881abc282596b0d4ed41c9b68fc30474d8cee6d1ebcb9545b38633d946b3fe82
 size 4949338472

model-00002-of-00014.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:80f5bd38e7a52453bac081b7fa50b250d9d1b8a524028e48725015f681347550
+oid sha256:399ef3f46a0eb39ee8964eb2ec89132b15a5ac0a3fbd1af66a09ed911bf0d41f
 size 4991425976

model-00003-of-00014.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fbba1dfc5c20e5ee9e2a34f23da9964823142f9dcb6f15a73d410a1ca389c6da
+oid sha256:daa6fcba2548a945e10b52baff8b6ec5b6e1fba8ad5fe0acab2a7d1276c26c2c
 size 4876059408

model-00004-of-00014.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:22ddefaf972d558135caf9c317d3e6b4a3a302f0dbcd5c56bd1ffa7b6f3ee665
+oid sha256:c922a2395c91941d46c84ce228e71b91bbb8044bcd65ffd0bdceeb191541e904
 size 4876059400

model-00005-of-00014.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7bb27d554988b382131979fe3c2f05eccfa56525b0bda1a5451f2273ae079797
+oid sha256:fde69d45e305fa8b02d9b28b1a131e1e02e8ce832d8c3d23f5c5b1e562aa7b83
 size 4876059408

model-00006-of-00014.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3b2e19e490b9fe03807de588351668fa89196c15d56dda5fe2288fb77a4317e8
+oid sha256:ebf633d81549a16c363ea1515a0f763232e35bdb665013cd441686ac447be788
 size 4876059400

model-00007-of-00014.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:75fdff87f86d1ccf2e9e17a3b8af002692403800ff4da99b596e8d91758e3e91
+oid sha256:56a06b6b218e1be01cae4c63d4cd0103ae9f7ee78b3a580407f6a2192e6d988b
 size 4876059408

model-00008-of-00014.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c9c78d51ce56fe6386dbbcfd838a4049c6bcef6e767ebf6d503b93889682dc6e
+oid sha256:92204531cbc720f14e6391a871e90f41c508bfd4479cde3971224e2677310729
 size 4876059400

model-00009-of-00014.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a2b2f9380c2249c19a79ccbbb77d993baa3142bcfc5159461346674b8ad8270d
+oid sha256:a16a134bed3ea2ad8b43872f030a586f16a7a7d2f902cb28c850668bf1e2bc70
 size 4876059408

model-00010-of-00014.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:403a5493e6c018ded33fe8812707be69bdc91da6fa69cffd5f60050f66dd406b
+oid sha256:4bc50c47244c7c74bc35ceed793673fceb1533c8e22dcf80e743ec913c95771f
 size 4876059400

model-00011-of-00014.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:86da186ef5263051a0af2fcd9458b40b96a22248b788c56ad20e4d5f526e5d19
+oid sha256:09c7b3fdd1dceeb950b4406a17d48308490d0c5e794dc62dab9a0bf5f45dd3d5
 size 4876059408

model-00012-of-00014.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ce3ce5ce33cc9fad3b5ce1b9a4a21d28bb1f3775b4818705620f62260813a205
+oid sha256:7916d74ffd4f082bfc21bef028db82838e6f75cc44ca10bc729158346365f2a0
 size 4876059400

model-00013-of-00014.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e4831189f6c08af1625184d09e7f9e885b1248ba5dfd2b6d1940bea2f4328ebf
+oid sha256:ae2b8a869964fcf6281f68f98dc7820924645fbc369a5ebdf1101218b7d8a575
 size 4876059400

model-00014-of-00014.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8cac904ae88e9958969d285bca1967b5e5fc408160aef401c5aa8e11bcddeaf4
+oid sha256:0290ac48f7f0be2b9365ba34efd21d2e6ae160b604ad197421b8ace8650695de
 size 1950423728

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
-size 11421896
+oid sha256:88145e3c3249adc2546ede277e9819d6e405e19072456e4b521cbc724bd60773
+size 7031660

tokenizer_config.json

@@ -1,208 +1,35 @@
 {
   "add_bos_token": false,
-  "add_prefix_space": false,
-  "added_tokens_decoder": {
-    "151643": {
-      "content": "<|endoftext|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151644": {
-      "content": "<|im_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151645": {
-      "content": "<|im_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151646": {
-      "content": "<|object_ref_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151647": {
-      "content": "<|object_ref_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151648": {
-      "content": "<|box_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151649": {
-      "content": "<|box_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151650": {
-      "content": "<|quad_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151651": {
-      "content": "<|quad_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151652": {
-      "content": "<|vision_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151653": {
-      "content": "<|vision_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151654": {
-      "content": "<|vision_pad|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151655": {
-      "content": "<|image_pad|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151656": {
-      "content": "<|video_pad|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151657": {
-      "content": "<tool_call>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "151658": {
-      "content": "</tool_call>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "151659": {
-      "content": "<|fim_prefix|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "151660": {
-      "content": "<|fim_middle|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "151661": {
-      "content": "<|fim_suffix|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "151662": {
-      "content": "<|fim_pad|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "151663": {
-      "content": "<|repo_name|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "151664": {
-      "content": "<|file_sep|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    }
+  "add_eos_token": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<｜begin▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
   },
-  "additional_special_tokens": [
-    "<|im_start|>",
-    "<|im_end|>",
-    "<|object_ref_start|>",
-    "<|object_ref_end|>",
-    "<|box_start|>",
-    "<|box_end|>",
-    "<|quad_start|>",
-    "<|quad_end|>",
-    "<|vision_start|>",
-    "<|vision_end|>",
-    "<|vision_pad|>",
-    "<|image_pad|>",
-    "<|video_pad|>"
-  ],
-  "bos_token": null,
-  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
   "clean_up_tokenization_spaces": false,
-  "eos_token": "<|endoftext|>",
-  "errors": "replace",
-  "extra_special_tokens": {},
-  "model_max_length": 32768,
-  "pad_token": "<|endoftext|>",
-  "split_special_tokens": false,
-  "tokenizer_class": "Qwen2Tokenizer",
-  "unk_token": null
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "<｜end▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "legacy": true,
+  "model_max_length": 16384,
+  "pad_token": {
+    "__type": "AddedToken",
+    "content": "<｜end▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sp_model_kwargs": {},
+  "unk_token": null,
+  "tokenizer_class": "LlamaTokenizerFast",
+  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='') %}{%- for message in messages %}{%- if message['role'] == 'system' %}{% set ns.system_prompt = message['content'] %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is none %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls']%}{%- if not ns.is_first %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{%- set ns.is_first = true -%}{%- else %}{{'\\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- endfor %}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{% if '</think>' in content %}{% set content = content.split('</think>')[-1] %}{% endif %}{{'<｜Assistant｜>' + content + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'\\n<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜>'}}{% endif %}"
 }