Hugging Face's logo

aws-neuron
/
optimum-neuron-cache

Model card Files
xet
Community
584
dacorvo HF Staff commited on
Commit
7342c16
·
verified ·
1 Parent(s): b41e94c

Update inference-cache-config/mixtral.json

Browse files
Files changed (1) hide show
  1. inference-cache-config/mixtral.json +17 -3
inference-cache-config/mixtral.json CHANGED
@@ -1,16 +1,30 @@
1
  {
2
- "mistralai/Mixtral-8x7B-Instruct-v0.1": [
3
  {
4
  "batch_size": 1,
5
  "sequence_length": 4096,
6
  "num_cores": 24,
7
- "auto_cast_type": "fp16"
8
  },
9
  {
10
  "batch_size": 4,
11
  "sequence_length": 4096,
12
  "num_cores": 24,
13
- "auto_cast_type": "fp16"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
14
  }
15
  ]
16
  }
 
1
  {
2
+ "mistralai/Mixtral-8x7B-Instruct-v0.1": [
3
  {
4
  "batch_size": 1,
5
  "sequence_length": 4096,
6
  "num_cores": 24,
7
+ "auto_cast_type": "bf16"
8
  },
9
  {
10
  "batch_size": 4,
11
  "sequence_length": 4096,
12
  "num_cores": 24,
13
+ "auto_cast_type": "bf16"
14
+ }
15
+ ],
16
+ "mistralai/Mixtral-8x22B-Instruct-v0.1": [
17
+ {
18
+ "batch_size": 1,
19
+ "sequence_length": 4096,
20
+ "num_cores": 24,
21
+ "auto_cast_type": "bf16"
22
+ },
23
+ {
24
+ "batch_size": 4,
25
+ "sequence_length": 4096,
26
+ "num_cores": 24,
27
+ "auto_cast_type": "bf16"
28
  }
29
  ]
30
  }