abhinavkulkarni
/

mosaicml-mpt-30b-instruct-w4-g128-awq

Text Generation

text-generation-inference

Model card Files Files and versions Community

Abhinav Kulkarni commited on Jul 14, 2023

Commit

846fb01

·

1 Parent(s): 5f1fe45

Updated README

Files changed (1) hide show

README.md +2 -2

README.md CHANGED Viewed

@@ -45,7 +45,7 @@ from transformers import AutoModelForCausalLM, AutoConfig, AutoTokenizer
 from accelerate import init_empty_weights, load_checkpoint_and_dispatch
 from huggingface_hub import snapshot_download
-model_name = "mosaicml/mpt-30b-instruct"
 # Config
 config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)
@@ -60,7 +60,7 @@ q_config = {
     "q_group_size": 128,
 }
-load_quant = snapshot_download('abhinavkulkarni/mosaicml-mpt-30b-instruct-w4-g128-awq')
 with init_empty_weights():
     model = AutoModelForCausalLM.from_config(config=config,

 from accelerate import init_empty_weights, load_checkpoint_and_dispatch
 from huggingface_hub import snapshot_download
+model_name = "abhinavkulkarni/mosaicml-mpt-30b-instruct-w4-g128-awq"
 # Config
 config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)
     "q_group_size": 128,
 }
+load_quant = snapshot_download(model_name)
 with init_empty_weights():
     model = AutoModelForCausalLM.from_config(config=config,