add app.py int4 cpu model

.idea/chatglm-6b-int4.iml

@@ -2,7 +2,7 @@
 <module type="PYTHON_MODULE" version="4">
  <component name="NewModuleRootManager">
  <content url="file://$MODULE_DIR$" />
- <orderEntry type="inheritedJdk" />
+ <orderEntry type="jdk" jdkName="Python 3.8" jdkType="Python SDK" />
  <orderEntry type="sourceFolder" forTests="false" />
  </component>
  <component name="PyDocumentationSettings">

app.py

@@ -1,16 +1,20 @@
 from transformers import AutoTokenizer, AutoModel
 import gradio as gr
 
-tokenizer = AutoTokenizer.from_pretrained(".\\models\\chatglm-6b-int4", trust_remote_code=True, revision="")
-model = AutoModel.from_pretrained(".\\models\\chatglm-6b-int4", trust_remote_code=True, revision="").half().cuda()
+# tokenizer = AutoTokenizer.from_pretrained(".\\models\\chatglm-6b-int4", trust_remote_code=True, revision="")
+# model = AutoModel.from_pretrained(".\\models\\chatglm-6b-int4", trust_remote_code=True, revision="").half().cuda()
+# chatglm-6b-int4 cuda，本地可以运行成功
+# tokenizer = AutoTokenizer.from_pretrained(".\\models\\chatglm-6b-int4", trust_remote_code=True, revision="")
+# model = AutoModel.from_pretrained(".\\models\\chatglm-6b-int4", trust_remote_code=True, revision="").half().cuda()
+
 
+# chatglm-6b-int4 CPU，
+tokenizer = AutoTokenizer.from_pretrained(".\\models\\chatglm-6b-int4", trust_remote_code=True, revision="")
+model = AutoModel.from_pretrained(".\\models\\chatglm-6b-int4", trust_remote_code=True, revision="").float()
 
-# from transformers import AutoTokenizer, AutoModel
-# tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True)
-# model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True)
-# model = model.eval()
 
 
+# chatglm-6b
 # kernel_file = "./models/chatglm-6b-int4/quantization_kernels.so"
 # tokenizer = AutoTokenizer.from_pretrained("./models/chatglm-6b-int4", trust_remote_code=True, revision="")
 # model = AutoModel.from_pretrained("./models/chatglm-6b-int4", trust_remote_code=True, revision="").half().cuda()

app_local.py

@@ -2,7 +2,8 @@ from transformers import AutoTokenizer, AutoModel
 import gradio as gr
 
 tokenizer = AutoTokenizer.from_pretrained(".\\models\\chatglm-6b-int4", trust_remote_code=True, revision="")
-model = AutoModel.from_pretrained(".\\models\\chatglm-6b-int4", trust_remote_code=True, revision="").half().cuda()
+model = AutoModel.from_pretrained(".\\models\\chatglm-6b-int4", trust_remote_code=True, revision="").half()
+# model = AutoModel.from_pretrained(".\\models\\chatglm-6b-int4", trust_remote_code=True, revision="").half().cuda()
 
 
 # from transformers import AutoTokenizer, AutoModel

demo_single_chat.py

@@ -1,8 +1,15 @@
 from transformers import AutoTokenizer, AutoModel
 tokenizer = AutoTokenizer.from_pretrained(".\\models\\chatglm-6b-int4", trust_remote_code=True, revision="")
-model = AutoModel.from_pretrained(".\\models\\chatglm-6b-int4", trust_remote_code=True, revision="").half().cuda()
-kernel_file =
-model = model.quantize(bits=4, kernel_file=kernel)
+# model = AutoModel.from_pretrained(".\\models\\chatglm-6b-int4", trust_remote_code=True, revision="").half().cuda()
+model = AutoModel.from_pretrained(".\\models\\chatglm-6b-int4", trust_remote_code=True, revision="").float()
+
+# tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True)
+# model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True)
+
+
+
+kernel_file = "./models/chatglm-6b-int4/quantization_kernels.so"
+model = model.quantize(bits=4, kernel_file=kernel_file)
 model = model.eval()
 
 def parse_text(text):