Mixtral - branch gptq-4bit-32g-actorder_True

config.json

@@ -0,0 +1,55 @@
+{
+  "_name_or_path": "TheBloke/Mixtral-8x7B-Instruct-v0.1-GPTQ",
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mixtral",
+  "num_attention_heads": 32,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "num_local_experts": 8,
+  "output_router_logits": false,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "quantization_config": {
+    "batch_size": 1,
+    "bits": 4,
+    "block_name_to_quantize": null,
+    "cache_block_outputs": true,
+    "damp_percent": 0.1,
+    "dataset": null,
+    "desc_act": true,
+    "exllama_config": {
+      "version": 1
+    },
+    "group_size": 32,
+    "max_input_length": null,
+    "model_seqlen": null,
+    "module_name_preceding_first_block": null,
+    "pad_token_id": null,
+    "quant_method": "gptq",
+    "sym": true,
+    "tokenizer": null,
+    "true_sequential": true,
+    "use_cuda_fp16": false,
+    "use_exllama": true
+  },
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000.0,
+  "router_aux_loss_coef": 0.02,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.36.2",
+  "use_cache": true,
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.36.2"
+}

model-00001-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74c41bdc04cd199fbf435b91d3c4a2bc704645a17e23c138aa68c015ff39dd7f
+size 796482960

model-00002-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b0679092928de50266c21540283e46106ec0d2b7eb973b2b1a6ca5b6f028706
+size 772385112

model-00003-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f6e4f5604b11c154b54aa812a07a0c9ce1e49691a4918066443d45ec0d43416
+size 772405472

model-00004-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31dbc81c32f81e637cc97bbce70e73bd63ca637bd102e93228e508d72c8edda4
+size 772384992

model-00005-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6eb43fd7bfafd8767fa3bb2f312ac32202b78778e84fcf8df190f79e5a04397
+size 772384992

model-00006-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8cf895d407c7b79d57cd259449694b6c527e66c4999dd8bf38ff1f782af915c2
+size 772405472

model-00007-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b97511661a1b8510418ce55e263f0c919e3782d07356b2994ba34893359382b0
+size 772384992

model-00008-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dde535592fb511435172e6644fdbb3c7c4699e7017c4a03af817cbfed72a9d57
+size 772384992

model-00009-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46ffab1278b42c047208459b0fdf6c2aefe7c551ccf2e9728fca435bbdd9c953
+size 794230528

model-00010-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1f4520e191551d6e8a79f83eee982dfc955e04074fd433a0204b8af6cdc42ef
+size 794178632

model-00011-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd93c715ab02b6429726354f6e840933cd8cb576e6344a3fb088a1a1ad819403
+size 772384992

model-00012-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89362b5e06a50690332eefe471177cf3e542f471dfef50afc9a312254004dc37
+size 772385112

model-00013-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2cd538781db218dea37c567c0a725a28eac8bbaef7a1c096d4d09ec06a088bf3
+size 772405608

model-00014-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a9dbb4ad1a1082da436a98e79d6f5fad763132c23ad3d76e9f52eaf221d0153
+size 772385128

model-00015-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04fa52c37f9680803c6a83b8961cd3958d01ccded8766d2b9413cfba2bf846a0
+size 772385128

model-00016-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a48c0c54d73b16b61f98eeafc59095e1710bc277d5378d6475cf916063bb2231
+size 772405608

model-00017-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a6642a61c6bb5e4c8dfac1908ffa771336307437d25063de277a0a25a5b30bd
+size 772385128

model-00018-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38e04ab6bdf2706963db5cc150e15cd4924526b217dfc9242d00697ac02c0c2e
+size 772385128

model-00019-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f2fe12652acb2a89e1497ce8c1e0f14e7c43a1301eebe170b8809a99a9a1ba7
+size 772405608

model-00020-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f402e8e381fb28624d75278736137bd3c7baf761c76c06a796edb0d884b7ee3
+size 772385128

model-00021-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4803ddbab527d83a7e814de42825d8aa474907c81bebabb43fe997ec9ce1ead0
+size 772385128

model-00022-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:246b765a3fc2017b305347dcd011d0a28a60a66353ed6613aaf95f66a8be3e5b
+size 794230656

model-00023-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5cfcbe8afd1dd9b734def6313f26782934cc2f5f3f770c377c20b57e8e8d29f
+size 794178752

model-00024-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec92d73e337223a42970798d3c70dc529256c964d8b75173342421f77734a061
+size 772385128

model-00025-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a62e0851f46134626d379595d531a1938279bf56cd641f28566f26dfce4450e4
+size 772385128

model-00026-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9796b576490c20144cd0d6029a57da1b54d377ae34f113eeccc4fb0ea5f57cd
+size 772405608

model-00027-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23ba69560fb6a76631846cfd402c55afbef1b934eedbcc7cc853a21878934e5f
+size 772385128

model-00028-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e19d3784e340c111fee7dd7aa24101da753c7f069718973e150d49c9effff5a7
+size 772385128

model-00029-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7fb225896bfccbe9687724435f7bc7f544f2cb396c4d62146ec0f3d3a335d83d
+size 772405608

model-00030-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad5d5209ce5d6bfee4dff7ef9f59cbdcfe067ad362449312c8f651e0e50f16f1
+size 772385128

model-00031-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62f52ee97b5bfceacdf16a23a7570f15dd4d179ec947603b40bd1618c9882654
+size 772385128

model-00032-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:563652ca12dff1933d8c3be31cffc9bd16511915457a9e206bde09e8231c771f
+size 772405608

model-00033-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a35690e08ff2beb2b64ff3c5f0a5784202c4632e2e42086a34494783f549bfed
+size 772385128

model-00034-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e568341142afe7e9f69dd74ab596a7d13a81fe1e3d828809764269a2910b16a
+size 772385128

model-00035-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11162a62cb1dfa562dd64cd90d4c3321ae74ac1860d19458b749d286cd213672
+size 782070648

model-00036-of-00036.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7d0c42f0bc6af0aa7f77bccf3b1c818c97e6a898386bddea1eea2ffdb70c8b6
+size 262144128

special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": null,
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}