Liang-Su commited on
Commit
5c4969a
1 Parent(s): 9701742

Upload 27 files

Browse files
Baichuan2-13B-Chat/.idea/.gitignore ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ # Default ignored files
2
+ /shelf/
3
+ /workspace.xml
4
+ # Datasource local storage ignored files
5
+ /../../../../../../:\复旦\金融风控LLM+KG\baichuan\Baichuan2-13B-Chat\.idea/dataSources/
6
+ /dataSources.local.xml
7
+ # Editor-based HTTP Client requests
8
+ /httpRequests/
Baichuan2-13B-Chat/.idea/Baichuan2-13B-Chat.iml ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <module type="PYTHON_MODULE" version="4">
3
+ <component name="NewModuleRootManager">
4
+ <content url="file://$MODULE_DIR$" />
5
+ <orderEntry type="jdk" jdkName="Python 3.10 (icl) (3)" jdkType="Python SDK" />
6
+ <orderEntry type="sourceFolder" forTests="false" />
7
+ </component>
8
+ <component name="PyDocumentationSettings">
9
+ <option name="format" value="GOOGLE" />
10
+ <option name="myDocStringFormat" value="Google" />
11
+ </component>
12
+ </module>
Baichuan2-13B-Chat/.idea/inspectionProfiles/Project_Default.xml ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <component name="InspectionProjectProfileManager">
2
+ <profile version="1.0">
3
+ <option name="myName" value="Project Default" />
4
+ <inspection_tool class="DuplicatedCode" enabled="true" level="WEAK WARNING" enabled_by_default="true">
5
+ <Languages>
6
+ <language minSize="2390" name="Python" />
7
+ </Languages>
8
+ </inspection_tool>
9
+ <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
10
+ <option name="ignoredPackages">
11
+ <value>
12
+ <list size="8">
13
+ <item index="0" class="java.lang.String" itemvalue="scipy" />
14
+ <item index="1" class="java.lang.String" itemvalue="powerlaw" />
15
+ <item index="2" class="java.lang.String" itemvalue="networkx" />
16
+ <item index="3" class="java.lang.String" itemvalue="Faker" />
17
+ <item index="4" class="java.lang.String" itemvalue="matplotlib" />
18
+ <item index="5" class="java.lang.String" itemvalue="python-dateutil" />
19
+ <item index="6" class="java.lang.String" itemvalue="pygraphviz" />
20
+ <item index="7" class="java.lang.String" itemvalue="numpy" />
21
+ </list>
22
+ </value>
23
+ </option>
24
+ </inspection_tool>
25
+ <inspection_tool class="PyPep8Inspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
26
+ <option name="ignoredErrors">
27
+ <list>
28
+ <option value="W605" />
29
+ <option value="E501" />
30
+ </list>
31
+ </option>
32
+ </inspection_tool>
33
+ <inspection_tool class="PyPep8NamingInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
34
+ <option name="ignoredErrors">
35
+ <list>
36
+ <option value="N801" />
37
+ <option value="N802" />
38
+ <option value="N806" />
39
+ <option value="N803" />
40
+ </list>
41
+ </option>
42
+ </inspection_tool>
43
+ <inspection_tool class="PyShadowingBuiltinsInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
44
+ <option name="ignoredNames">
45
+ <list>
46
+ <option value="object" />
47
+ </list>
48
+ </option>
49
+ </inspection_tool>
50
+ <inspection_tool class="PyStubPackagesAdvertiser" enabled="true" level="WARNING" enabled_by_default="true">
51
+ <option name="ignoredPackages">
52
+ <list>
53
+ <option value="pyspark-stubs==3.0.0.post1" />
54
+ </list>
55
+ </option>
56
+ </inspection_tool>
57
+ <inspection_tool class="PyUnresolvedReferencesInspection" enabled="true" level="WARNING" enabled_by_default="true">
58
+ <option name="ignoredIdentifiers">
59
+ <list>
60
+ <option value="bool.sum" />
61
+ </list>
62
+ </option>
63
+ </inspection_tool>
64
+ </profile>
65
+ </component>
Baichuan2-13B-Chat/.idea/inspectionProfiles/profiles_settings.xml ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ <component name="InspectionProjectProfileManager">
2
+ <settings>
3
+ <option name="USE_PROJECT_PROFILE" value="false" />
4
+ <version value="1.0" />
5
+ </settings>
6
+ </component>
Baichuan2-13B-Chat/.idea/misc.xml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <project version="4">
3
+ <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (icl) (3)" project-jdk-type="Python SDK" />
4
+ </project>
Baichuan2-13B-Chat/.idea/modules.xml ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <project version="4">
3
+ <component name="ProjectModuleManager">
4
+ <modules>
5
+ <module fileurl="file://$PROJECT_DIR$/.idea/Baichuan2-13B-Chat.iml" filepath="$PROJECT_DIR$/.idea/Baichuan2-13B-Chat.iml" />
6
+ </modules>
7
+ </component>
8
+ </project>
Baichuan2-13B-Chat/.idea/workspace.xml ADDED
@@ -0,0 +1,72 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <project version="4">
3
+ <component name="ChangeListManager">
4
+ <list default="true" id="d79fba24-fc9f-4360-be3f-063ce6bd8b75" name="Default Changelist" comment="" />
5
+ <option name="SHOW_DIALOG" value="false" />
6
+ <option name="HIGHLIGHT_CONFLICTS" value="true" />
7
+ <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
8
+ <option name="LAST_RESOLUTION" value="IGNORE" />
9
+ </component>
10
+ <component name="FileTemplateManagerImpl">
11
+ <option name="RECENT_TEMPLATES">
12
+ <list>
13
+ <option value="Python Script" />
14
+ </list>
15
+ </option>
16
+ </component>
17
+ <component name="ProjectId" id="2eClVhKVO1dOyoA1xuy1DLOrNBj" />
18
+ <component name="ProjectViewState">
19
+ <option name="hideEmptyMiddlePackages" value="true" />
20
+ <option name="showLibraryContents" value="true" />
21
+ </component>
22
+ <component name="PropertiesComponent">
23
+ <property name="RunOnceActivity.OpenProjectViewOnStart" value="true" />
24
+ <property name="RunOnceActivity.ShowReadmeOnStart" value="true" />
25
+ <property name="WebServerToolWindowFactoryState" value="false" />
26
+ <property name="last_opened_file_path" value="$PROJECT_DIR$/upload.py" />
27
+ </component>
28
+ <component name="RunManager">
29
+ <configuration name="upload" type="PythonConfigurationType" factoryName="Python" nameIsGenerated="true">
30
+ <module name="Baichuan2-13B-Chat" />
31
+ <option name="INTERPRETER_OPTIONS" value="" />
32
+ <option name="PARENT_ENVS" value="true" />
33
+ <envs>
34
+ <env name="PYTHONUNBUFFERED" value="1" />
35
+ </envs>
36
+ <option name="SDK_HOME" value="D:\Anaconda\envs\icl\python.exe" />
37
+ <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
38
+ <option name="IS_MODULE_SDK" value="false" />
39
+ <option name="ADD_CONTENT_ROOTS" value="true" />
40
+ <option name="ADD_SOURCE_ROOTS" value="true" />
41
+ <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
42
+ <option name="SCRIPT_NAME" value="$PROJECT_DIR$/upload.py" />
43
+ <option name="PARAMETERS" value="" />
44
+ <option name="SHOW_COMMAND_LINE" value="false" />
45
+ <option name="EMULATE_TERMINAL" value="false" />
46
+ <option name="MODULE_MODE" value="false" />
47
+ <option name="REDIRECT_INPUT" value="false" />
48
+ <option name="INPUT_FILE" value="" />
49
+ <method v="2" />
50
+ </configuration>
51
+ </component>
52
+ <component name="SpellCheckerSettings" RuntimeDictionaries="0" Folders="0" CustomDictionaries="0" DefaultDictionary="application-level" UseSingleDictionary="true" transferred="true" />
53
+ <component name="TaskManager">
54
+ <task active="true" id="Default" summary="Default task">
55
+ <changelist id="d79fba24-fc9f-4360-be3f-063ce6bd8b75" name="Default Changelist" comment="" />
56
+ <created>1711416470636</created>
57
+ <option name="number" value="Default" />
58
+ <option name="presentableId" value="Default" />
59
+ <updated>1711416470636</updated>
60
+ <workItem from="1711416473833" duration="414000" />
61
+ <workItem from="1711416906329" duration="227000" />
62
+ <workItem from="1711417155540" duration="506000" />
63
+ </task>
64
+ <servers />
65
+ </component>
66
+ <component name="TypeScriptGeneratedFilesManager">
67
+ <option name="version" value="3" />
68
+ </component>
69
+ <component name="com.intellij.coverage.CoverageDataManagerImpl">
70
+ <SUITE FILE_PATH="coverage/Baichuan2_13B_Chat$upload.coverage" NAME="upload Coverage Results" MODIFIED="1711417610802" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
71
+ </component>
72
+ </project>
Baichuan2-13B-Chat/.mdl ADDED
Binary file (54 Bytes). View file
 
Baichuan2-13B-Chat/.msc ADDED
Binary file (1.3 kB). View file
 
Baichuan2-13B-Chat/README.md ADDED
@@ -0,0 +1,182 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ language:
3
+ - en
4
+ - zh
5
+ license: other
6
+ tasks:
7
+ - text-generation
8
+ studios:
9
+ - baichuan-inc/Baichuan-13B-Chatdemo
10
+ ---
11
+
12
+ <!-- markdownlint-disable first-line-h1 -->
13
+ <!-- markdownlint-disable html -->
14
+ <div align="center">
15
+ <h1>
16
+ Baichuan 2
17
+ </h1>
18
+ </div>
19
+
20
+ <div align="center">
21
+ <a href="https://github.com/baichuan-inc/Baichuan2" target="_blank">🦉GitHub</a> |
22
+ <a href="https://modelscope.cn/models/baichuan-inc/Baichuan-13B-Base/file/view/master/wechat.jpeg" target="_blank">💬WeChat</a> |
23
+ <a href="https://modelscope.cn/studios/baichuan-inc/Baichuan-13B-Chatdemo/summary" target="_blank">🤖Demo</a>
24
+ </div>
25
+ <div align="center">
26
+ 🚀 <a href="https://www.baichuan-ai.com/" target="_blank">百川大模型在线对话平台</a> 已正式向公众开放 🎉
27
+ </div>
28
+
29
+ # 目录
30
+
31
+ - [📖 模型介绍](#模型介绍)
32
+ - [⚙️ 快速开始](#快速开始)
33
+ - [📊 Benchmark评估](#评估)
34
+ - [📜 声明与协议](#声明与协议)
35
+
36
+ # 模型介绍
37
+
38
+ - Baichuan 2 是[百川智能]推出的**新一代开源大语言模型**,采用 **2.6 万亿** Tokens 的高质量语料训练。
39
+ - Baichuan 2 在多个权威的中文、英文和多语言的通用、领域 benchmark 上取得同尺寸**最佳**的效果。
40
+ - 本次发布包含有 **7B**、**13B** 的 **Base** 和 **Chat** 版本,并提供了 Chat 版本的 **4bits 量化**。
41
+ - 所有版本对学术研究完全开放。同时,开发者通过邮件申请并获得官方商用许可后,即可**免费商用**,请参考[协议](#协议)章节。
42
+ - 欢迎阅读我们的技术报告 [Baichuan 2: Open Large-scale Language Models] 获取更多信息。
43
+
44
+ 本次发布版本和下载链接见下表:
45
+
46
+ | | 基座模型 | 对齐模型 | 对齐模型 4bits 量化 |
47
+ |:---:|:--------------------:|:--------------------:|:--------------------------:|
48
+ | 7B | [Baichuan2-7B-Base] | [Baichuan2-7B-Chat] | [Baichuan2-7B-Chat-4bits] |
49
+ | 13B | [Baichuan2-13B-Base] | [Baichuan2-13B-Chat] | [Baichuan2-13B-Chat-4bits] |
50
+
51
+ # 快速开始
52
+
53
+ ```python
54
+ import torch
55
+ from modelscope import snapshot_download, AutoModelForCausalLM, AutoTokenizer,GenerationConfig
56
+ model_dir = snapshot_download("baichuan-inc/Baichuan2-13B-Chat", revision='v1.0.1')
57
+ tokenizer = AutoTokenizer.from_pretrained(model_dir, device_map="auto",
58
+ trust_remote_code=True, torch_dtype=torch.float16)
59
+ model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="auto",
60
+ trust_remote_code=True, torch_dtype=torch.float16)
61
+ model.generation_config = GenerationConfig.from_pretrained(model_dir)
62
+ messages = []
63
+ messages.append({"role": "user", "content": "讲解一下“温故而知新”"})
64
+ response = model.chat(tokenizer, messages)
65
+ print(response)
66
+ messages.append({'role': 'assistant', 'content': response})
67
+ messages.append({"role": "user", "content": "背诵一下将进酒"})
68
+ response = model.chat(tokenizer, messages)
69
+ print(response)
70
+ ```
71
+ 在魔搭社区的免费算力上,也可以通过量化的方式使用13B对话模型
72
+ ```python
73
+ import torch
74
+ from modelscope import snapshot_download, AutoModelForCausalLM, AutoTokenizer,GenerationConfig
75
+ from transformers import BitsAndBytesConfig
76
+
77
+ quantization_config = BitsAndBytesConfig(
78
+ False,
79
+ True,
80
+ bnb_4bit_compute_dtype=torch.bfloat16,
81
+ bnb_4bit_quant_type='nf4',
82
+ bnb_4bit_use_double_quant=True)
83
+ model_dir = snapshot_download("baichuan-inc/Baichuan2-13B-Chat", revision='v1.0.1')
84
+ tokenizer = AutoTokenizer.from_pretrained(model_dir, device_map="auto",
85
+ trust_remote_code=True, torch_dtype=torch.float16)
86
+ model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="auto",
87
+ trust_remote_code=True, torch_dtype=torch.float16,
88
+ quantization_config=quantization_config)
89
+ model.generation_config = GenerationConfig.from_pretrained(model_dir)
90
+ messages = []
91
+ messages.append({"role": "user", "content": "讲解一下“温故而知新”"})
92
+ response = model.chat(tokenizer, messages)
93
+ print(response)
94
+ messages.append({'role': 'assistant', 'content': response})
95
+ messages.append({"role": "user", "content": "背诵一下将进酒"})
96
+ response = model.chat(tokenizer, messages)
97
+ print(response)
98
+ ```
99
+ # Benchmark 结果
100
+
101
+ 我们在[通用]、[法律]、[医疗]、[数学]、[代码]和[多语言翻译]六个领域的中英文权威数据集上对模型进行了广泛测试,更多详细测评结果可查看[GitHub]。
102
+
103
+ ### 7B 模型结果
104
+
105
+ | | **C-Eval** | **MMLU** | **CMMLU** | **Gaokao** | **AGIEval** | **BBH** |
106
+ |:-----------------------:|:----------:|:--------:|:---------:|:----------:|:-----------:|:-------:|
107
+ | | 5-shot | 5-shot | 5-shot | 5-shot | 5-shot | 3-shot |
108
+ | **GPT-4** | 68.40 | 83.93 | 70.33 | 66.15 | 63.27 | 75.12 |
109
+ | **GPT-3.5 Turbo** | 51.10 | 68.54 | 54.06 | 47.07 | 46.13 | 61.59 |
110
+ | **LLaMA-7B** | 27.10 | 35.10 | 26.75 | 27.81 | 28.17 | 32.38 |
111
+ | **LLaMA2-7B** | 28.90 | 45.73 | 31.38 | 25.97 | 26.53 | 39.16 |
112
+ | **MPT-7B** | 27.15 | 27.93 | 26.00 | 26.54 | 24.83 | 35.20 |
113
+ | **Falcon-7B** | 24.23 | 26.03 | 25.66 | 24.24 | 24.10 | 28.77 |
114
+ | **ChatGLM2-6B** | 50.20 | 45.90 | 49.00 | 49.44 | 45.28 | 31.65 |
115
+ | **[Baichuan-7B]** | 42.80 | 42.30 | 44.02 | 36.34 | 34.44 | 32.48 |
116
+ | **[Baichuan2-7B-Base]** | 54.00 | 54.16 | 57.07 | 47.47 | 42.73 | 41.56 |
117
+
118
+ ### 13B 模型结果
119
+
120
+ | | **C-Eval** | **MMLU** | **CMMLU** | **Gaokao** | **AGIEval** | **BBH** |
121
+ |:---------------------------:|:----------:|:--------:|:---------:|:----------:|:-----------:|:-------:|
122
+ | | 5-shot | 5-shot | 5-shot | 5-shot | 5-shot | 3-shot |
123
+ | **GPT-4** | 68.40 | 83.93 | 70.33 | 66.15 | 63.27 | 75.12 |
124
+ | **GPT-3.5 Turbo** | 51.10 | 68.54 | 54.06 | 47.07 | 46.13 | 61.59 |
125
+ | **LLaMA-13B** | 28.50 | 46.30 | 31.15 | 28.23 | 28.22 | 37.89 |
126
+ | **LLaMA2-13B** | 35.80 | 55.09 | 37.99 | 30.83 | 32.29 | 46.98 |
127
+ | **Vicuna-13B** | 32.80 | 52.00 | 36.28 | 30.11 | 31.55 | 43.04 |
128
+ | **Chinese-Alpaca-Plus-13B** | 38.80 | 43.90 | 33.43 | 34.78 | 35.46 | 28.94 |
129
+ | **XVERSE-13B** | 53.70 | 55.21 | 58.44 | 44.69 | 42.54 | 38.06 |
130
+ | **[Baichuan-13B-Base]** | 52.40 | 51.60 | 55.30 | 49.69 | 43.20 | 43.01 |
131
+ | **[Baichuan2-13B-Base]** | 58.10 | 59.17 | 61.97 | 54.33 | 48.17 | 48.78 |
132
+
133
+
134
+ ## 训练过程模型
135
+
136
+ 除了训练了 2.6 万亿 Tokens 的 [Baichuan2-7B-Base] 模型,我们还提供了在此之前的另外 11 个中间过程的模型(分别对应训练了约 0.2 ~ 2.4 万亿 Tokens)供社区研究使用([训练过程heckpoint下载])。下图给出了这些 checkpoints 在 C-Eval、MMLU、CMMLU 三个 benchmark 上的效果变化:
137
+
138
+ ![checkpoint](https://modelscope.cn/api/v1/models/baichuan-inc/Baichuan2-7B-Base/repo?Revision=master&FilePath=media/checkpoints.jpeg&View=true)
139
+
140
+ # 声明与协议
141
+
142
+ ## 声明
143
+
144
+ 我们在此声明,我们的开发团队并未基于 Baichuan 2 模型开发任何应用,无论是在 iOS、Android、网页或任何其他平台。我们强烈呼吁所有使用者,不要利用
145
+ Baichuan 2 模型进行任何危害国家社会安全或违法的活动。另外,我们也要求使用者不要将 Baichuan 2
146
+ 模型用于未经适当安全审查和备案的互联网服务。我们希望所有的使用者都能遵守这个原则,确保科技的发展能在规范和合法的环境下进行。
147
+
148
+ 我们已经尽我们所能,来确保模型训练过程中使用的数据的合规性。然而,尽管我们已经做出了巨大的努力,但由于模型和数据的复杂性,仍有可能存在一些无法预见的问题。因此,如果由于使用
149
+ Baichuan 2 开源模型而导致的任何问题,包括但不限于数据安全问题、公共舆论风险,或模型被误导、滥用、传播或不当利用所带来的任何风险和问题,我们将不承担任何责任。
150
+
151
+ ## 协议
152
+
153
+ * Baichuan 2 模型的社区使用需遵循[《Baichuan 2 模型社区许可协议》]。
154
+ * Baichuan 2 支持商用,如果将 Baichuan 2 模型或其衍生品用作商业用途,请您按照如下方式联系许可方,以进行登记并向许可方申请书面授权:联系邮箱 [[email protected]]。
155
+
156
+ [GitHub]:https://github.com/baichuan-inc/Baichuan2
157
+ [Baichuan2]:https://github.com/baichuan-inc/Baichuan2
158
+
159
+ [Baichuan-7B]:https://modelscope.cn/models/baichuan-inc/baichuan-7B/summary
160
+ [Baichuan2-7B-Base]:https://modelscope.cn/models/baichuan-inc/Baichuan2-7B-Base/summary
161
+ [Baichuan2-7B-Chat]:https://modelscope.cn/models/baichuan-inc/Baichuan2-7B-Chat/summary
162
+ [Baichuan2-7B-Chat-4bits]:https://modelscope.cn/models/baichuan-inc/Baichuan2-7B-Chat-4bits/summary
163
+ [Baichuan-13B-Base]:https://modelscope.cn/models/baichuan-inc/Baichuan-13B-Base/summary
164
+ [Baichuan2-13B-Base]:https://modelscope.cn/models/baichuan-inc/Baichuan2-13B-Base/summary
165
+ [Baichuan2-13B-Chat]:https://modelscope.cn/models/baichuan-inc/Baichuan2-13B-Chat/summary
166
+ [Baichuan2-13B-Chat-4bits]:https://modelscope.cn/models/baichuan-inc/Baichuan2-13B-Chat-4bits/summary
167
+
168
+ [通用]:https://github.com/baichuan-inc/Baichuan2#%E9%80%9A%E7%94%A8%E9%A2%86%E5%9F%9F
169
+ [法律]:https://github.com/baichuan-inc/Baichuan2#%E6%B3%95%E5%BE%8B%E5%8C%BB%E7%96%97
170
+ [医疗]:https://github.com/baichuan-inc/Baichuan2#%E6%B3%95%E5%BE%8B%E5%8C%BB%E7%96%97
171
+ [数学]:https://github.com/baichuan-inc/Baichuan2#%E6%95%B0%E5%AD%A6%E4%BB%A3%E7%A0%81
172
+ [代码]:https://github.com/baichuan-inc/Baichuan2#%E6%95%B0%E5%AD%A6%E4%BB%A3%E7%A0%81
173
+ [多语言翻译]:https://github.com/baichuan-inc/Baichuan2#%E5%A4%9A%E8%AF%AD%E8%A8%80%E7%BF%BB%E8%AF%91
174
+
175
+ [《Baichuan 2 模型社区许可协议》]:https://huggingface.co/baichuan-inc/Baichuan2-7B-Base/blob/main/Baichuan2%20%E6%A8%A1%E5%9E%8B%E7%A4%BE%E5%8C%BA%E8%AE%B8%E5%8F%AF%E5%8D%8F%E8%AE%AE.pdf
176
+
177
+ [邮件申请]: mailto:[email protected]
178
+ [Email]: mailto:[email protected]
179
180
+ [训练过程heckpoint下载]: https://huggingface.co/baichuan-inc/Baichuan2-7B-Intermediate-Checkpoints
181
+ [百川智能]: https://www.baichuan-ai.com
182
+ [Baichuan 2: Open Large-scale Language Models]:https://cdn.baichuan-ai.com/paper/Baichuan2-technical-report.pdf
Baichuan2-13B-Chat/config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "architectures": [
4
+ "BaichuanForCausalLM"
5
+ ],
6
+ "auto_map": {
7
+ "AutoConfig": "configuration_baichuan.BaichuanConfig",
8
+ "AutoModelForCausalLM": "modeling_baichuan.BaichuanForCausalLM"
9
+ },
10
+ "bos_token_id": 1,
11
+ "eos_token_id": 2,
12
+ "gradient_checkpointing": false,
13
+ "hidden_act": "silu",
14
+ "hidden_size": 5120,
15
+ "initializer_range": 0.02,
16
+ "intermediate_size": 13696,
17
+ "model_max_length": 4096,
18
+ "model_type": "baichuan",
19
+ "num_attention_heads": 40,
20
+ "num_hidden_layers": 40,
21
+ "pad_token_id": 0,
22
+ "rms_norm_eps": 1e-06,
23
+ "tie_word_embeddings": false,
24
+ "torch_dtype": "bfloat16",
25
+ "transformers_version": "4.29.2",
26
+ "use_cache": true,
27
+ "vocab_size": 125696
28
+ }
Baichuan2-13B-Chat/configuration.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "framework": "pytorch",
3
+ "task": "text-generation",
4
+ "model": {
5
+ "type": "Baichuan2-13B-Chat"
6
+ },
7
+ "pipeline": {
8
+ "type": "Baichuan2-13B-chatbot-pipe"
9
+ },
10
+ "allow_remote": true
11
+ }
Baichuan2-13B-Chat/configuration_baichuan.py ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) 2023, Baichuan Intelligent Technology. All rights reserved.
2
+
3
+ from transformers.configuration_utils import PretrainedConfig
4
+
5
+
6
+ class BaichuanConfig(PretrainedConfig):
7
+ model_type = "baichuan"
8
+ keys_to_ignore_at_inference = ["past_key_values"]
9
+
10
+ def __init__(
11
+ self,
12
+ vocab_size=64000,
13
+ hidden_size=5120,
14
+ intermediate_size=13696,
15
+ num_hidden_layers=40,
16
+ num_attention_heads=40,
17
+ hidden_act="silu",
18
+ model_max_length=4096,
19
+ initializer_range=0.02,
20
+ rms_norm_eps=1e-6,
21
+ use_cache=True,
22
+ pad_token_id=0,
23
+ bos_token_id=1,
24
+ eos_token_id=2,
25
+ tie_word_embeddings=False,
26
+ gradient_checkpointing=False,
27
+ z_loss_weight=0,
28
+ **kwargs,
29
+ ):
30
+ self.vocab_size = vocab_size
31
+ self.model_max_length = model_max_length
32
+ self.hidden_size = hidden_size
33
+ self.intermediate_size = intermediate_size
34
+ self.num_hidden_layers = num_hidden_layers
35
+ self.num_attention_heads = num_attention_heads
36
+ self.hidden_act = hidden_act
37
+ self.initializer_range = initializer_range
38
+ self.rms_norm_eps = rms_norm_eps
39
+ self.use_cache = use_cache
40
+ self.z_loss_weight = z_loss_weight
41
+ self.gradient_checkpointing = (gradient_checkpointing,)
42
+ super().__init__(
43
+ pad_token_id=pad_token_id,
44
+ bos_token_id=bos_token_id,
45
+ eos_token_id=eos_token_id,
46
+ tie_word_embeddings=tie_word_embeddings,
47
+ **kwargs,
48
+ )
Baichuan2-13B-Chat/generation_config.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "pad_token_id": 0,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": 2,
5
+ "user_token_id": 195,
6
+ "assistant_token_id": 196,
7
+ "max_new_tokens": 2048,
8
+ "temperature": 0.3,
9
+ "top_k": 5,
10
+ "top_p": 0.85,
11
+ "repetition_penalty": 1.05,
12
+ "do_sample": true,
13
+ "transformers_version": "4.29.2"
14
+ }
Baichuan2-13B-Chat/generation_utils.py ADDED
@@ -0,0 +1,83 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import List
2
+ from queue import Queue
3
+
4
+ import torch
5
+
6
+
7
+ def build_chat_input(model, tokenizer, messages: List[dict], max_new_tokens: int=0):
8
+ def _parse_messages(messages, split_role="user"):
9
+ system, rounds = "", []
10
+ round = []
11
+ for i, message in enumerate(messages):
12
+ if message["role"] == "system":
13
+ assert i == 0
14
+ system = message["content"]
15
+ continue
16
+ if message["role"] == split_role and round:
17
+ rounds.append(round)
18
+ round = []
19
+ round.append(message)
20
+ if round:
21
+ rounds.append(round)
22
+ return system, rounds
23
+
24
+ max_new_tokens = max_new_tokens or model.generation_config.max_new_tokens
25
+ max_input_tokens = model.config.model_max_length - max_new_tokens
26
+ system, rounds = _parse_messages(messages, split_role="user")
27
+ system_tokens = tokenizer.encode(system)
28
+ max_history_tokens = max_input_tokens - len(system_tokens)
29
+
30
+ history_tokens = []
31
+ for round in rounds[::-1]:
32
+ round_tokens = []
33
+ for message in round:
34
+ if message["role"] == "user":
35
+ round_tokens.append(model.generation_config.user_token_id)
36
+ else:
37
+ round_tokens.append(model.generation_config.assistant_token_id)
38
+ round_tokens.extend(tokenizer.encode(message["content"]))
39
+ if len(history_tokens) == 0 or len(history_tokens) + len(round_tokens) <= max_history_tokens:
40
+ history_tokens = round_tokens + history_tokens # concat left
41
+ if len(history_tokens) < max_history_tokens:
42
+ continue
43
+ break
44
+
45
+ input_tokens = system_tokens + history_tokens
46
+ if messages[-1]["role"] != "assistant":
47
+ input_tokens.append(model.generation_config.assistant_token_id)
48
+ input_tokens = input_tokens[-max_input_tokens:] # truncate left
49
+ return torch.LongTensor([input_tokens]).to(model.device)
50
+
51
+
52
+ class TextIterStreamer:
53
+ def __init__(self, tokenizer, skip_prompt=False, skip_special_tokens=False):
54
+ self.tokenizer = tokenizer
55
+ self.skip_prompt = skip_prompt
56
+ self.skip_special_tokens = skip_special_tokens
57
+ self.tokens = []
58
+ self.text_queue = Queue()
59
+ self.next_tokens_are_prompt = True
60
+
61
+ def put(self, value):
62
+ if self.skip_prompt and self.next_tokens_are_prompt:
63
+ self.next_tokens_are_prompt = False
64
+ else:
65
+ if len(value.shape) > 1:
66
+ value = value[0]
67
+ self.tokens.extend(value.tolist())
68
+ self.text_queue.put(
69
+ self.tokenizer.decode(self.tokens, skip_special_tokens=self.skip_special_tokens))
70
+
71
+ def end(self):
72
+ self.text_queue.put(None)
73
+
74
+ def __iter__(self):
75
+ return self
76
+
77
+ def __next__(self):
78
+ value = self.text_queue.get()
79
+ if value is None:
80
+ raise StopIteration()
81
+ else:
82
+ return value
83
+
Baichuan2-13B-Chat/modeling_baichuan.py ADDED
@@ -0,0 +1,827 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) 2023, Baichuan Intelligent Technology. All rights reserved.
2
+
3
+ from .configuration_baichuan import BaichuanConfig
4
+ from .generation_utils import build_chat_input, TextIterStreamer
5
+
6
+ import math
7
+ from threading import Thread
8
+ from typing import List, Optional, Tuple, Union
9
+
10
+ import torch
11
+ from torch import nn
12
+ from torch.nn import CrossEntropyLoss
13
+ from torch.nn import functional as F
14
+ from transformers import PreTrainedModel, PretrainedConfig
15
+ from transformers.activations import ACT2FN
16
+ from transformers.generation.utils import GenerationConfig
17
+ from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
18
+ from transformers.utils import logging, ContextManagers
19
+
20
+ import os
21
+ from contextlib import contextmanager
22
+ from accelerate import init_empty_weights
23
+
24
+ logger = logging.get_logger(__name__)
25
+
26
+ try:
27
+ from xformers import ops as xops
28
+ except ImportError:
29
+ xops = None
30
+ logger.warning(
31
+ "Xformers is not installed correctly. If you want to use memory_efficient_attention to accelerate training use the following command to install Xformers\npip install xformers."
32
+ )
33
+
34
+
35
+ def _get_interleave(n):
36
+ def _get_interleave_power_of_2(n):
37
+ start = 2 ** (-(2 ** -(math.log2(n) - 3)))
38
+ ratio = start
39
+ return [start * ratio**i for i in range(n)]
40
+
41
+ if math.log2(n).is_integer():
42
+ return _get_interleave_power_of_2(n)
43
+ else:
44
+ closest_power_of_2 = 2 ** math.floor(math.log2(n))
45
+ return (
46
+ _get_interleave_power_of_2(closest_power_of_2)
47
+ + _get_interleave(2 * closest_power_of_2)[0::2][: n - closest_power_of_2]
48
+ )
49
+
50
+
51
+ def _fill_with_neg_inf(t):
52
+ """FP16-compatible function that fills a tensor with -inf."""
53
+ return t.float().fill_(float("-inf")).type_as(t)
54
+
55
+
56
+ def _buffered_future_mask(tensor, maxpos, alibi, attn_heads):
57
+ _future_mask = torch.triu(_fill_with_neg_inf(torch.zeros([maxpos, maxpos])), 1)
58
+ _future_mask = _future_mask.unsqueeze(0) + alibi
59
+ new_future_mask = _future_mask.to(tensor)
60
+ return new_future_mask[: tensor.shape[0] * attn_heads, :maxpos, :maxpos]
61
+
62
+
63
+ def _gen_alibi_mask(tensor, n_head, max_pos):
64
+ slopes = torch.Tensor(_get_interleave(n_head))
65
+ position_point = torch.arange(max_pos) - max_pos + 1
66
+ position_point = position_point.unsqueeze(0).unsqueeze(0).expand(n_head, -1, -1)
67
+ diag = torch.diag(position_point[0])
68
+ position_point = position_point - diag.unsqueeze(0).unsqueeze(0).transpose(-1, -2)
69
+ alibi = slopes.unsqueeze(1).unsqueeze(1) * position_point
70
+ alibi = alibi.view(n_head, 1, max_pos)
71
+ alibi_mask = torch.triu(_fill_with_neg_inf(torch.zeros([max_pos, max_pos])), 1)
72
+ alibi_mask = alibi_mask.unsqueeze(0) + alibi
73
+ return alibi_mask
74
+
75
+
76
+ class RMSNorm(torch.nn.Module):
77
+ def __init__(self, hidden_size, epsilon=1e-6):
78
+ super().__init__()
79
+ self.weight = torch.nn.Parameter(torch.empty(hidden_size))
80
+ self.epsilon = epsilon
81
+
82
+ def forward(self, hidden_states):
83
+ variance = hidden_states.to(torch.float32).pow(2).mean(-1, keepdim=True)
84
+ hidden_states = hidden_states * torch.rsqrt(variance + self.epsilon)
85
+
86
+ # convert into half-precision
87
+ if self.weight.dtype in [torch.float16, torch.bfloat16]:
88
+ hidden_states = hidden_states.to(self.weight.dtype)
89
+
90
+ return self.weight * hidden_states
91
+
92
+
93
+ class MLP(torch.nn.Module):
94
+ def __init__(
95
+ self,
96
+ hidden_size: int,
97
+ intermediate_size: int,
98
+ hidden_act: str,
99
+ ):
100
+ super().__init__()
101
+ self.gate_proj = torch.nn.Linear(hidden_size, intermediate_size, bias=False)
102
+ self.down_proj = torch.nn.Linear(intermediate_size, hidden_size, bias=False)
103
+ self.up_proj = torch.nn.Linear(hidden_size, intermediate_size, bias=False)
104
+ self.act_fn = ACT2FN[hidden_act]
105
+
106
+ def forward(self, x):
107
+ return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
108
+
109
+
110
+ class BaichuanAttention(torch.nn.Module):
111
+ def __init__(self, config: BaichuanConfig):
112
+ super().__init__()
113
+ self.config = config
114
+ self.hidden_size = config.hidden_size
115
+ self.num_heads = config.num_attention_heads
116
+ self.head_dim = self.hidden_size // self.num_heads
117
+ self.max_position_embeddings = config.model_max_length
118
+
119
+ if (self.head_dim * self.num_heads) != self.hidden_size:
120
+ raise ValueError(
121
+ f"hidden_size {self.hidden_size} is not divisible by num_heads {self.num_heads}"
122
+ )
123
+ self.W_pack = torch.nn.Linear(
124
+ self.hidden_size, 3 * self.hidden_size, bias=False
125
+ )
126
+ self.o_proj = torch.nn.Linear(
127
+ self.num_heads * self.head_dim, self.hidden_size, bias=False
128
+ )
129
+
130
+ def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
131
+ return (
132
+ tensor.view(bsz, seq_len, self.num_heads, self.head_dim)
133
+ .transpose(1, 2)
134
+ .contiguous()
135
+ )
136
+
137
+ def forward(
138
+ self,
139
+ hidden_states: torch.Tensor,
140
+ attention_mask: Optional[torch.Tensor] = None,
141
+ past_key_value: Optional[Tuple[torch.Tensor]] = None,
142
+ output_attentions: bool = False,
143
+ use_cache: bool = False,
144
+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
145
+ bsz, q_len, _ = hidden_states.size()
146
+
147
+ proj = self.W_pack(hidden_states)
148
+ proj = (
149
+ proj.unflatten(-1, (3, self.hidden_size))
150
+ .unsqueeze(0)
151
+ .transpose(0, -2)
152
+ .squeeze(-2)
153
+ )
154
+ query_states = (
155
+ proj[0].view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
156
+ )
157
+ key_states = (
158
+ proj[1].view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
159
+ )
160
+ value_states = (
161
+ proj[2].view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
162
+ )
163
+
164
+ kv_seq_len = key_states.shape[-2]
165
+ if past_key_value is not None:
166
+ kv_seq_len += past_key_value[0].shape[-2]
167
+
168
+ if past_key_value is not None:
169
+ # reuse k, v, self_attention
170
+ key_states = torch.cat([past_key_value[0], key_states], dim=2)
171
+ value_states = torch.cat([past_key_value[1], value_states], dim=2)
172
+
173
+ past_key_value = (key_states, value_states) if use_cache else None
174
+ if xops is not None and self.training:
175
+ attn_weights = None
176
+ # query_states = query_states.transpose(1, 2)
177
+ # key_states = key_states.transpose(1, 2)
178
+ # value_states = value_states.transpose(1, 2)
179
+ # attn_output = xops.memory_efficient_attention(
180
+ # query_states, key_states, value_states, attn_bias=attention_mask
181
+ # )
182
+ with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=True, enable_mem_efficient=True):
183
+ attn_output = F.scaled_dot_product_attention(query_states, key_states, value_states, attn_mask = attention_mask)
184
+ attn_output = attn_output.transpose(1, 2)
185
+ else:
186
+ attn_weights = torch.matmul(
187
+ query_states, key_states.transpose(2, 3)
188
+ ) / math.sqrt(self.head_dim)
189
+
190
+ if attention_mask is not None:
191
+ if q_len == 1: # inference with cache
192
+ if len(attention_mask.size()) == 4:
193
+ attention_mask = attention_mask[:, :, -1:, :]
194
+ else:
195
+ attention_mask = attention_mask[:, -1:, :]
196
+ attn_weights = attn_weights + attention_mask
197
+ attn_weights = torch.max(
198
+ attn_weights, torch.tensor(torch.finfo(attn_weights.dtype).min)
199
+ )
200
+
201
+ attn_weights = torch.nn.functional.softmax(attn_weights, dim=-1)
202
+ attn_output = torch.matmul(attn_weights, value_states)
203
+
204
+ attn_output = attn_output.transpose(1, 2)
205
+ attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)
206
+ attn_output = self.o_proj(attn_output)
207
+
208
+ if not output_attentions:
209
+ attn_weights = None
210
+
211
+ return attn_output, attn_weights, past_key_value
212
+
213
+
214
+ class BaichuanLayer(torch.nn.Module):
215
+ def __init__(self, config: BaichuanConfig):
216
+ super().__init__()
217
+ self.hidden_size = config.hidden_size
218
+ self.self_attn = BaichuanAttention(config=config)
219
+ self.mlp = MLP(
220
+ hidden_size=self.hidden_size,
221
+ intermediate_size=config.intermediate_size,
222
+ hidden_act=config.hidden_act,
223
+ )
224
+ self.input_layernorm = RMSNorm(config.hidden_size, epsilon=config.rms_norm_eps)
225
+ self.post_attention_layernorm = RMSNorm(
226
+ config.hidden_size, epsilon=config.rms_norm_eps
227
+ )
228
+
229
+ def forward(
230
+ self,
231
+ hidden_states: torch.Tensor,
232
+ attention_mask: Optional[torch.Tensor] = None,
233
+ past_key_value: Optional[Tuple[torch.Tensor]] = None,
234
+ output_attentions: Optional[bool] = False,
235
+ use_cache: Optional[bool] = False,
236
+ ) -> Tuple[
237
+ torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]
238
+ ]:
239
+ residual = hidden_states
240
+
241
+ hidden_states = self.input_layernorm(hidden_states)
242
+
243
+ # Self Attention
244
+ hidden_states, self_attn_weights, present_key_value = self.self_attn(
245
+ hidden_states=hidden_states,
246
+ attention_mask=attention_mask,
247
+ past_key_value=past_key_value,
248
+ output_attentions=output_attentions,
249
+ use_cache=use_cache,
250
+ )
251
+ hidden_states = residual + hidden_states
252
+
253
+ # Fully Connected
254
+ residual = hidden_states
255
+ hidden_states = self.post_attention_layernorm(hidden_states)
256
+ hidden_states = self.mlp(hidden_states)
257
+ hidden_states = residual + hidden_states
258
+
259
+ outputs = (hidden_states,)
260
+
261
+ if use_cache:
262
+ outputs += (present_key_value,)
263
+
264
+ return outputs
265
+
266
+
267
+ class BaichuanPreTrainedModel(PreTrainedModel):
268
+ config_class = BaichuanConfig
269
+ base_model_prefix = "model"
270
+ supports_gradient_checkpointing = True
271
+ _no_split_modules = ["BaichuanLayer"]
272
+ _keys_to_ignore_on_load_unexpected = [r"decoder\.version"]
273
+
274
+ def _init_weights(self, module):
275
+ std = self.config.initializer_range
276
+ if isinstance(module, torch.nn.Linear):
277
+ module.weight.data.normal_(mean=0.0, std=std)
278
+ if module.bias is not None:
279
+ module.bias.data.zero_()
280
+ elif isinstance(module, torch.nn.Embedding):
281
+ module.weight.data.normal_(mean=0.0, std=std)
282
+ if module.padding_idx is not None:
283
+ module.weight.data[module.padding_idx].zero_()
284
+
285
+ def _set_gradient_checkpointing(self, module, value=False):
286
+ if isinstance(module, BaichuanModel):
287
+ module.gradient_checkpointing = value
288
+
289
+
290
+ class BaichuanModel(BaichuanPreTrainedModel):
291
+ def __init__(self, config: BaichuanConfig):
292
+ super().__init__(config)
293
+ self.padding_idx = config.pad_token_id
294
+ self.vocab_size = config.vocab_size
295
+ self.n_head = config.num_attention_heads
296
+ self.embed_tokens = torch.nn.Embedding(
297
+ config.vocab_size, config.hidden_size, self.padding_idx
298
+ )
299
+ self.layers = torch.nn.ModuleList(
300
+ [BaichuanLayer(config) for _ in range(config.num_hidden_layers)]
301
+ )
302
+ self.norm = RMSNorm(config.hidden_size, epsilon=config.rms_norm_eps)
303
+
304
+ self.gradient_checkpointing = config.gradient_checkpointing
305
+ self.post_init()
306
+ self.max_cache_pos = config.model_max_length
307
+ self.first_run = True
308
+ self.alibi_mask = None
309
+
310
+ def get_input_embeddings(self):
311
+ return self.embed_tokens
312
+
313
+ def set_input_embeddings(self, value):
314
+ self.embed_tokens = value
315
+
316
+ def get_alibi_mask(self, tensor, seq_length_with_past):
317
+ if self.training:
318
+ slopes = torch.Tensor(_get_interleave(self.n_head))
319
+ position_point = (
320
+ torch.arange(seq_length_with_past) - seq_length_with_past + 1
321
+ )
322
+ position_point = (
323
+ position_point.unsqueeze(0)
324
+ .unsqueeze(0)
325
+ .expand(self.n_head, seq_length_with_past, -1)
326
+ )
327
+ diag = torch.diag(position_point[0])
328
+ position_point = position_point - diag.unsqueeze(0).unsqueeze(0).transpose(
329
+ -1, -2
330
+ )
331
+ alibi = slopes.unsqueeze(1).unsqueeze(1) * position_point
332
+ mask = _buffered_future_mask(
333
+ tensor, seq_length_with_past, alibi, self.n_head
334
+ )
335
+ else:
336
+ if self.first_run:
337
+ self.first_run = False
338
+ self.register_buffer(
339
+ "future_mask",
340
+ _gen_alibi_mask(tensor, self.n_head, self.max_cache_pos).to(
341
+ tensor
342
+ ),
343
+ persistent=False,
344
+ )
345
+ if seq_length_with_past > self.max_cache_pos:
346
+ self.max_cache_pos = seq_length_with_past
347
+ self.register_buffer(
348
+ "future_mask",
349
+ _gen_alibi_mask(tensor, self.n_head, self.max_cache_pos).to(
350
+ tensor
351
+ ),
352
+ persistent=False,
353
+ )
354
+ mask = self.future_mask[
355
+ : self.n_head, :seq_length_with_past, :seq_length_with_past
356
+ ]
357
+ return mask
358
+
359
+ def forward(
360
+ self,
361
+ input_ids: torch.LongTensor = None,
362
+ attention_mask: Optional[torch.Tensor] = None,
363
+ past_key_values: Optional[List[torch.FloatTensor]] = None,
364
+ inputs_embeds: Optional[torch.FloatTensor] = None,
365
+ use_cache: Optional[bool] = False,
366
+ output_attentions: Optional[bool] = False,
367
+ output_hidden_states: Optional[bool] = False,
368
+ return_dict: Optional[bool] = True,
369
+ ) -> Union[Tuple, BaseModelOutputWithPast]:
370
+ if input_ids is not None and inputs_embeds is not None:
371
+ raise ValueError(
372
+ "You cannot provide both input_ids and inputs_embeds simultaneously"
373
+ )
374
+ elif input_ids is not None:
375
+ batch_size, seq_length = input_ids.shape
376
+ elif inputs_embeds is not None:
377
+ batch_size, seq_length, _ = inputs_embeds.shape
378
+ else:
379
+ raise ValueError("You need to provide input_ids or inputs_embeds")
380
+
381
+ return_dict = (
382
+ return_dict if return_dict is not None else self.config.use_return_dict
383
+ )
384
+
385
+ seq_length_with_past = seq_length
386
+
387
+ if past_key_values is not None:
388
+ past_key_values_length = past_key_values[0][0].shape[2]
389
+ seq_length_with_past = seq_length_with_past + past_key_values_length
390
+
391
+ if inputs_embeds is None:
392
+ inputs_embeds = self.embed_tokens(input_ids)
393
+
394
+ if self.training:
395
+ if (
396
+ self.alibi_mask is None
397
+ or self.alibi_mask.shape[-1] != seq_length_with_past
398
+ ):
399
+ self.alibi_mask = self.get_alibi_mask(
400
+ inputs_embeds, seq_length_with_past
401
+ )
402
+ alibi_mask = self.alibi_mask
403
+ else:
404
+ alibi_mask = self.get_alibi_mask(inputs_embeds, seq_length_with_past)
405
+
406
+ if attention_mask is not None:
407
+ if len(attention_mask.shape) == 2:
408
+ expanded_mask = attention_mask.to(alibi_mask.dtype)
409
+ expanded_mask = torch.tril(
410
+ torch.gt(expanded_mask[:, :, None] * expanded_mask[:, None, :], 0)
411
+ ) * torch.eq(expanded_mask[:, :, None] - expanded_mask[:, None, :], 0)
412
+ else:
413
+ expanded_mask = attention_mask
414
+ bsz = inputs_embeds.size(0)
415
+ src_len, tgt_len = alibi_mask.size()[-2:]
416
+ expanded_mask = (
417
+ expanded_mask.unsqueeze(1)
418
+ .expand(bsz, 1, src_len, tgt_len)
419
+ .to(alibi_mask.dtype)
420
+ )
421
+ inverted_mask = 1.0 - expanded_mask
422
+ inverted_mask = inverted_mask.masked_fill(
423
+ inverted_mask.to(torch.bool), torch.finfo(alibi_mask.dtype).min
424
+ )
425
+ attention_mask = inverted_mask + alibi_mask.unsqueeze(0)
426
+ else:
427
+ attention_mask = alibi_mask
428
+
429
+ hidden_states = inputs_embeds
430
+
431
+ if self.gradient_checkpointing and self.training:
432
+ if use_cache:
433
+ logger.warning_once(
434
+ "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
435
+ )
436
+ use_cache = False
437
+
438
+ # decoder layers
439
+ all_hidden_states = () if output_hidden_states else None
440
+ all_self_attns = () if output_attentions else None
441
+ next_decoder_cache = () if use_cache else None
442
+
443
+ for idx, decoder_layer in enumerate(self.layers):
444
+ if output_hidden_states:
445
+ all_hidden_states += (hidden_states,)
446
+
447
+ past_key_value = (
448
+ past_key_values[idx] if past_key_values is not None else None
449
+ )
450
+
451
+ if self.gradient_checkpointing and self.training:
452
+
453
+ def create_custom_forward(module):
454
+ def custom_forward(*inputs):
455
+ # None for past_key_value
456
+ return module(*inputs, output_attentions, None)
457
+
458
+ return custom_forward
459
+
460
+ layer_outputs = torch.utils.checkpoint.checkpoint(
461
+ create_custom_forward(decoder_layer),
462
+ hidden_states,
463
+ attention_mask,
464
+ None,
465
+ )
466
+ else:
467
+ layer_outputs = decoder_layer(
468
+ hidden_states,
469
+ attention_mask=attention_mask,
470
+ past_key_value=past_key_value,
471
+ output_attentions=output_attentions,
472
+ use_cache=use_cache,
473
+ )
474
+
475
+ hidden_states = layer_outputs[0]
476
+
477
+ if use_cache:
478
+ next_decoder_cache += (layer_outputs[2 if output_attentions else 1],)
479
+
480
+ if output_attentions:
481
+ all_self_attns += (layer_outputs[1],)
482
+
483
+ hidden_states = self.norm(hidden_states)
484
+
485
+ # add hidden states from the last decoder layer
486
+ if output_hidden_states:
487
+ all_hidden_states += (hidden_states,)
488
+
489
+ next_cache = next_decoder_cache if use_cache else None
490
+ if not return_dict:
491
+ return tuple(
492
+ v
493
+ for v in [hidden_states, next_cache, all_hidden_states, all_self_attns]
494
+ if v is not None
495
+ )
496
+ return BaseModelOutputWithPast(
497
+ last_hidden_state=hidden_states,
498
+ past_key_values=next_cache,
499
+ hidden_states=all_hidden_states,
500
+ attentions=all_self_attns,
501
+ )
502
+
503
+
504
+ class NormHead(nn.Module):
505
+ def __init__(self, hidden_size, vocab_size, bias=False):
506
+ super().__init__()
507
+ self.weight = nn.Parameter(torch.empty((vocab_size, hidden_size)))
508
+ nn.init.kaiming_uniform_(self.weight, a=math.sqrt(5))
509
+ self.first_flag = True
510
+
511
+ def forward(self, hidden_states):
512
+ if self.training:
513
+ norm_weight = nn.functional.normalize(self.weight)
514
+ self.first_flag = True
515
+ elif self.first_flag:
516
+ self.first_flag = False
517
+ self.weight.data = nn.functional.normalize(self.weight)
518
+ norm_weight = self.weight
519
+ else:
520
+ norm_weight = self.weight
521
+ return nn.functional.linear(hidden_states, norm_weight)
522
+
523
+ _init_weights = True
524
+ @contextmanager
525
+ def no_init_weights(_enable=True):
526
+ global _init_weights
527
+ old_init_weights = _init_weights
528
+ if _enable:
529
+ _init_weights = False
530
+ try:
531
+ yield
532
+ finally:
533
+ _init_weights = old_init_weights
534
+
535
+
536
+ class BaichuanForCausalLM(BaichuanPreTrainedModel):
537
+ def __init__(self, config, *model_args, **model_kwargs):
538
+ super().__init__(config, *model_args, **model_kwargs)
539
+ self.model = BaichuanModel(config)
540
+ self.lm_head = NormHead(config.hidden_size, config.vocab_size, bias=False)
541
+ #if hasattr(config, "quantization_config") and config.quantization_config['load_in_4bit']:
542
+ if hasattr(config, "quantization_config") and isinstance(config.quantization_config, dict) and config.quantization_config.get('load_in_4bit', False):
543
+ try:
544
+ from .quantizer import quantize_offline, init_model_weight_int4
545
+ except ImportError:
546
+ raise ImportError(f"Needs quantize_offline to run quantize.")
547
+ quantize_offline(self, 4)
548
+ # Initialize weights and apply final processing
549
+ self.post_init()
550
+
551
+ def get_input_embeddings(self):
552
+ return self.model.embed_tokens
553
+
554
+ def set_input_embeddings(self, value):
555
+ self.model.embed_tokens = value
556
+
557
+ def get_output_embeddings(self):
558
+ return self.lm_head
559
+
560
+ def set_output_embeddings(self, new_embeddings):
561
+ self.lm_head = new_embeddings
562
+
563
+ def set_decoder(self, decoder):
564
+ self.model = decoder
565
+
566
+ def get_decoder(self):
567
+ return self.model
568
+
569
+ @classmethod
570
+ def from_pretrained(
571
+ cls,
572
+ pretrained_model_name_or_path: Optional[Union[str, os.PathLike]],
573
+ *model_args,
574
+ config: Optional[Union[PretrainedConfig, str, os.PathLike]] = None,
575
+ cache_dir: Optional[Union[str, os.PathLike]] = None,
576
+ ignore_mismatched_sizes: bool = False,
577
+ force_download: bool = False,
578
+ local_files_only: bool = False,
579
+ token: Optional[Union[str, bool]] = None,
580
+ revision: str = "main",
581
+ use_safetensors: bool = None,
582
+ **kwargs,
583
+ ):
584
+
585
+ # Load config if we don't provide a configuration
586
+ if not isinstance(config, PretrainedConfig):
587
+ config_path = config if config is not None else pretrained_model_name_or_path
588
+ config, model_kwargs = cls.config_class.from_pretrained(
589
+ config_path,
590
+ cache_dir=cache_dir,
591
+ return_unused_kwargs=True,
592
+ force_download=force_download,
593
+ resume_download=False,
594
+ proxies=None,
595
+ local_files_only=local_files_only,
596
+ token=token,
597
+ revision=revision,
598
+ subfolder="",
599
+ _from_auto=False,
600
+ _from_pipeline=None,
601
+ **kwargs,
602
+ )
603
+ else:
604
+ model_kwargs = kwargs
605
+
606
+ if hasattr(config, "quantization_config") and config.quantization_config['load_in_4bit']:
607
+ try:
608
+ from .quantizer import init_model_weight_int4
609
+ from accelerate import init_empty_weights, dispatch_model, infer_auto_device_map
610
+ from accelerate.utils import CustomDtype
611
+ from accelerate.utils import get_balanced_memory
612
+ except ImportError:
613
+ raise ImportError(f"Needs import model weight init func to run quantize.")
614
+ # Instantiate model.
615
+ init_contexts = [no_init_weights(_enable=True)]
616
+ init_contexts.append(init_empty_weights())
617
+ with ContextManagers(init_contexts):
618
+ model = cls(config)
619
+
620
+ model_file = os.path.join(pretrained_model_name_or_path, 'pytorch_model.bin')
621
+ state_dict = torch.load(model_file, map_location="cpu")
622
+ model.is_quantized = True
623
+
624
+ device_map = kwargs.pop("device_map", None)
625
+ torch_dtype = kwargs.pop("torch_dtype", None)
626
+ if device_map is not None:
627
+ kwargs = {"no_split_module_classes": model._no_split_modules}
628
+ target_dtype = CustomDtype.INT4
629
+ max_memory = get_balanced_memory(
630
+ model,
631
+ dtype=target_dtype,
632
+ low_zero=(device_map == "balanced_low_0"),
633
+ max_memory=None,
634
+ **kwargs,
635
+ )
636
+ kwargs["max_memory"] = max_memory
637
+ device_map = infer_auto_device_map(model, dtype=target_dtype, **kwargs)
638
+ model = init_model_weight_int4(config, model, state_dict)
639
+
640
+ # Set model in evaluation mode to deactivate DropOut modules by default
641
+ model.eval()
642
+ # If it is a model with generation capabilities, attempt to load the generation config
643
+ if model.can_generate():
644
+ try:
645
+ model.generation_config = GenerationConfig.from_pretrained(
646
+ pretrained_model_name_or_path,
647
+ cache_dir=cache_dir,
648
+ force_download=force_download,
649
+ resume_download=False,
650
+ proxies=None,
651
+ local_files_only=local_files_only,
652
+ token=token,
653
+ revision=revision,
654
+ subfolder="",
655
+ _from_auto=False,
656
+ _from_pipeline=None,
657
+ **kwargs,
658
+ )
659
+ except (OSError, TypeError):
660
+ logger.info(
661
+ "Generation config file not found, using a generation config created from the model config."
662
+ )
663
+ pass
664
+
665
+ if device_map is not None:
666
+ dispatch_model(model, device_map=device_map)
667
+
668
+ return model
669
+
670
+ return super(BaichuanForCausalLM, cls).from_pretrained(pretrained_model_name_or_path, *model_args,
671
+ config=config, cache_dir=cache_dir, ignore_mismatched_sizes=ignore_mismatched_sizes,
672
+ force_download=force_download, local_files_only=local_files_only, token=token, revision=revision,
673
+ use_safetensors=use_safetensors, **kwargs)
674
+
675
+ def forward(
676
+ self,
677
+ input_ids: torch.LongTensor = None,
678
+ attention_mask: Optional[torch.Tensor] = None,
679
+ past_key_values: Optional[List[torch.FloatTensor]] = None,
680
+ inputs_embeds: Optional[torch.FloatTensor] = None,
681
+ labels: Optional[torch.LongTensor] = None,
682
+ use_cache: Optional[bool] = None,
683
+ output_attentions: Optional[bool] = False,
684
+ output_hidden_states: Optional[bool] = False,
685
+ return_dict: Optional[bool] = True,
686
+ **kwargs,
687
+ ) -> Union[Tuple, CausalLMOutputWithPast]:
688
+ return_dict = (
689
+ return_dict if return_dict is not None else self.config.use_return_dict
690
+ )
691
+
692
+ # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
693
+ outputs = self.model(
694
+ input_ids=input_ids,
695
+ attention_mask=attention_mask,
696
+ past_key_values=past_key_values,
697
+ inputs_embeds=inputs_embeds,
698
+ use_cache=use_cache,
699
+ output_attentions=output_attentions,
700
+ output_hidden_states=output_hidden_states,
701
+ return_dict=return_dict,
702
+ )
703
+
704
+ hidden_states = outputs[0]
705
+ logits = self.lm_head(hidden_states)
706
+ loss = None
707
+ if labels is not None:
708
+ # Shift so that tokens < n predict n
709
+ shift_logits = logits[..., :-1, :].contiguous()
710
+ shift_labels = labels[..., 1:].contiguous()
711
+ # Flatten the tokens
712
+ loss_fct = CrossEntropyLoss()
713
+ shift_logits = shift_logits.view(-1, self.config.vocab_size)
714
+ shift_labels = shift_labels.view(-1)
715
+ softmax_normalizer = shift_logits.max(-1).values ** 2
716
+ z_loss = self.config.z_loss_weight * softmax_normalizer.mean()
717
+ # Enable model parallelism
718
+ shift_labels = shift_labels.to(shift_logits.device)
719
+ loss = loss_fct(shift_logits, shift_labels) + z_loss
720
+
721
+ if not return_dict:
722
+ output = (logits,) + outputs[1:]
723
+ return (loss,) + output if loss is not None else output
724
+
725
+ return CausalLMOutputWithPast(
726
+ loss=loss,
727
+ logits=logits,
728
+ past_key_values=outputs.past_key_values,
729
+ hidden_states=outputs.hidden_states,
730
+ attentions=outputs.attentions,
731
+ )
732
+
733
+ def quantize(self, bits: int):
734
+ try:
735
+ from .quantizer import quantize_online
736
+ except ImportError:
737
+ raise ImportError(f"Needs QLinear to run quantize.")
738
+ return quantize_online(self, bits)
739
+
740
+ def prepare_inputs_for_generation(
741
+ self,
742
+ input_ids: torch.LongTensor,
743
+ past_key_values: Optional[torch.Tensor] = None,
744
+ attention_mask: Optional[torch.Tensor] = None,
745
+ inputs_embeds: Optional[torch.Tensor] = None,
746
+ **kwargs,
747
+ ):
748
+ if past_key_values:
749
+ input_ids = input_ids[:, -1:]
750
+
751
+ # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
752
+ if inputs_embeds is not None and past_key_values is None:
753
+ model_inputs = {"inputs_embeds": inputs_embeds}
754
+ else:
755
+ model_inputs = {"input_ids": input_ids}
756
+
757
+ model_inputs.update(
758
+ {
759
+ "past_key_values": past_key_values,
760
+ "use_cache": kwargs.get("use_cache"),
761
+ "attention_mask": attention_mask,
762
+ }
763
+ )
764
+ return model_inputs
765
+
766
+ @staticmethod
767
+ def _reorder_cache(past_key_values, beam_idx):
768
+ return tuple(
769
+ tuple(past_state.index_select(0, beam_idx) for past_state in layer_past)
770
+ for layer_past in past_key_values
771
+ )
772
+
773
+ def _build_chat_input(
774
+ self, tokenizer, messages: List[dict], max_new_tokens: int = 0
775
+ ):
776
+ max_new_tokens = max_new_tokens or self.generation_config.max_new_tokens
777
+ max_input_tokens = self.config.model_max_length - max_new_tokens
778
+ max_input_tokens = max(self.config.model_max_length // 2, max_input_tokens)
779
+ total_input, round_input = [], []
780
+ for i, message in enumerate(messages[::-1]):
781
+ content_tokens = tokenizer.encode(message["content"])
782
+ if message["role"] == "user":
783
+ round_input = (
784
+ [self.generation_config.user_token_id]
785
+ + content_tokens
786
+ + round_input
787
+ )
788
+ if (
789
+ total_input
790
+ and len(total_input) + len(round_input) > max_input_tokens
791
+ ):
792
+ break
793
+ else:
794
+ total_input = round_input + total_input
795
+ if len(total_input) >= max_input_tokens:
796
+ break
797
+ else:
798
+ round_input = []
799
+ elif message["role"] == "assistant":
800
+ round_input = (
801
+ [self.generation_config.assistant_token_id]
802
+ + content_tokens
803
+ + [self.generation_config.eos_token_id]
804
+ + round_input
805
+ )
806
+ else:
807
+ raise ValueError(f"message role not supported yet: {message['role']}")
808
+ total_input = total_input[-max_input_tokens:] # truncate left
809
+ total_input.append(self.generation_config.assistant_token_id)
810
+ total_input = torch.LongTensor([total_input]).to(self.device)
811
+ return total_input
812
+
813
+ def chat(self, tokenizer, messages: List[dict], stream=False,
814
+ generation_config: Optional[GenerationConfig]=None):
815
+ generation_config = generation_config or self.generation_config
816
+ input_ids = build_chat_input(self, tokenizer, messages, generation_config.max_new_tokens)
817
+ if stream:
818
+ streamer = TextIterStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
819
+ Thread(target=self.generate, kwargs=dict(
820
+ inputs=input_ids, streamer=streamer,
821
+ generation_config=generation_config,
822
+ )).start()
823
+ return streamer
824
+ else:
825
+ outputs = self.generate(input_ids, generation_config=generation_config)
826
+ response = tokenizer.decode(outputs[0][len(input_ids[0]):], skip_special_tokens=True)
827
+ return response
Baichuan2-13B-Chat/ms_wrapper.py ADDED
@@ -0,0 +1,74 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import torch
3
+ from typing import Union, Dict, Any
4
+ from modelscope.pipelines.builder import PIPELINES
5
+ from modelscope.models.builder import MODELS
6
+ from modelscope.utils.constant import Tasks
7
+ from modelscope.pipelines.base import Pipeline
8
+ from modelscope.outputs import OutputKeys
9
+ from modelscope.pipelines.nlp.text_generation_pipeline import TextGenerationPipeline
10
+ from modelscope.models.base import Model, TorchModel
11
+ from modelscope.utils.logger import get_logger
12
+ from transformers import AutoModelForCausalLM, AutoTokenizer, GenerationConfig
13
+ from transformers.generation.utils import GenerationConfig
14
+
15
+ @PIPELINES.register_module(Tasks.text_generation, module_name='Baichuan2-13B-chatbot-pipe')
16
+ class Baichuan13BChatTextGenerationPipeline(TextGenerationPipeline):
17
+ def __init__(
18
+ self,
19
+ model: Union[Model, str],
20
+ *args,
21
+ **kwargs):
22
+ self.model = Baichuan13BChatTextGeneration(model) if isinstance(model, str) else model
23
+ super().__init__(model=model, **kwargs)
24
+
25
+ def preprocess(self, inputs, **preprocess_params) -> Dict[str, Any]:
26
+ return inputs
27
+
28
+ def _sanitize_parameters(self, **pipeline_parameters):
29
+ return {},pipeline_parameters,{}
30
+
31
+ # define the forward pass
32
+ def forward(self, inputs: Dict, **forward_params) -> Dict[str, Any]:
33
+ output = {}
34
+ device = self.model.model.device
35
+ input_ids = self.model.tokenizer(inputs, return_tensors="pt").input_ids.to(device)
36
+ pred = self.model.model.generate(input_ids,**forward_params)
37
+ out = self.model.tokenizer.decode(pred.cpu()[0], skip_special_tokens=True)
38
+ output['text'] = out
39
+ return output
40
+
41
+ # format the outputs from pipeline
42
+ def postprocess(self, input, **kwargs) -> Dict[str, Any]:
43
+ return input
44
+
45
+
46
+ @MODELS.register_module(Tasks.text_generation, module_name='Baichuan2-13B-Chat')
47
+ class Baichuan13BChatTextGeneration(TorchModel):
48
+ def __init__(self, model_dir=None, *args, **kwargs):
49
+ super().__init__(model_dir, *args, **kwargs)
50
+ self.logger = get_logger()
51
+ # loading tokenizer
52
+ self.tokenizer = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True)
53
+ self.model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="auto", torch_dtype=torch.float16, trust_remote_code=True)
54
+ # self.model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="auto",trust_remote_code=True)
55
+ self.model.generation_config = GenerationConfig.from_pretrained(model_dir)
56
+ self.model = self.model.eval()
57
+
58
+ def forward(self,input: Dict, *args, **kwargs) -> Dict[str, Any]:
59
+ output = {}
60
+ response = self.model.chat(self.tokenizer, input, *args, **kwargs)
61
+ history = input.copy()
62
+ history.append({'role': 'assistant', 'content': response})
63
+ return {OutputKeys.RESPONSE:response, OutputKeys.HISTORY: history}
64
+
65
+ def quantize(self, bits: int):
66
+ self.model = self.model.quantize(bits)
67
+ return self
68
+
69
+ def infer(self, input, **kwargs):
70
+ device = self.model.device
71
+ input_ids = self.tokenizer(input, return_tensors="pt").input_ids.to(device)
72
+ pred = self.model.generate(input_ids,**kwargs)
73
+ out = self.tokenizer.decode(pred.cpu()[0], skip_special_tokens=True)
74
+ return out
Baichuan2-13B-Chat/pytorch_model-00001-of-00003.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ed5f863097bc541ff1f47f5d5ac875cbac2c1269c384bc77f540ffca4cc2cb5d
3
+ size 9973568983
Baichuan2-13B-Chat/pytorch_model-00002-of-00003.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:13ac578202dce2688d81fcee55e332d898ad980cec5019a7de386ea9760302c0
3
+ size 9947421360
Baichuan2-13B-Chat/pytorch_model-00003-of-00003.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:95a78916eaeb9891bde335461ba8a474b039e6c5f17a12f987dc03a93372a812
3
+ size 7872446643
Baichuan2-13B-Chat/pytorch_model.bin.index.json ADDED
@@ -0,0 +1,290 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "metadata": {
3
+ "total_size": 27793336320
4
+ },
5
+ "weight_map": {
6
+ "lm_head.weight": "pytorch_model-00003-of-00003.bin",
7
+ "model.embed_tokens.weight": "pytorch_model-00001-of-00003.bin",
8
+ "model.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
9
+ "model.layers.0.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
10
+ "model.layers.0.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
11
+ "model.layers.0.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
12
+ "model.layers.0.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
13
+ "model.layers.0.self_attn.W_pack.weight": "pytorch_model-00001-of-00003.bin",
14
+ "model.layers.0.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
15
+ "model.layers.1.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
16
+ "model.layers.1.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
17
+ "model.layers.1.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
18
+ "model.layers.1.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
19
+ "model.layers.1.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
20
+ "model.layers.1.self_attn.W_pack.weight": "pytorch_model-00001-of-00003.bin",
21
+ "model.layers.1.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
22
+ "model.layers.10.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
23
+ "model.layers.10.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
24
+ "model.layers.10.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
25
+ "model.layers.10.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
26
+ "model.layers.10.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
27
+ "model.layers.10.self_attn.W_pack.weight": "pytorch_model-00001-of-00003.bin",
28
+ "model.layers.10.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
29
+ "model.layers.11.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
30
+ "model.layers.11.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
31
+ "model.layers.11.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
32
+ "model.layers.11.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
33
+ "model.layers.11.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
34
+ "model.layers.11.self_attn.W_pack.weight": "pytorch_model-00001-of-00003.bin",
35
+ "model.layers.11.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
36
+ "model.layers.12.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
37
+ "model.layers.12.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
38
+ "model.layers.12.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
39
+ "model.layers.12.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
40
+ "model.layers.12.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
41
+ "model.layers.12.self_attn.W_pack.weight": "pytorch_model-00001-of-00003.bin",
42
+ "model.layers.12.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
43
+ "model.layers.13.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
44
+ "model.layers.13.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
45
+ "model.layers.13.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
46
+ "model.layers.13.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
47
+ "model.layers.13.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
48
+ "model.layers.13.self_attn.W_pack.weight": "pytorch_model-00001-of-00003.bin",
49
+ "model.layers.13.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
50
+ "model.layers.14.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
51
+ "model.layers.14.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
52
+ "model.layers.14.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
53
+ "model.layers.14.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
54
+ "model.layers.14.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
55
+ "model.layers.14.self_attn.W_pack.weight": "pytorch_model-00002-of-00003.bin",
56
+ "model.layers.14.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
57
+ "model.layers.15.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
58
+ "model.layers.15.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
59
+ "model.layers.15.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
60
+ "model.layers.15.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
61
+ "model.layers.15.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
62
+ "model.layers.15.self_attn.W_pack.weight": "pytorch_model-00002-of-00003.bin",
63
+ "model.layers.15.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
64
+ "model.layers.16.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
65
+ "model.layers.16.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
66
+ "model.layers.16.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
67
+ "model.layers.16.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
68
+ "model.layers.16.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
69
+ "model.layers.16.self_attn.W_pack.weight": "pytorch_model-00002-of-00003.bin",
70
+ "model.layers.16.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
71
+ "model.layers.17.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
72
+ "model.layers.17.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
73
+ "model.layers.17.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
74
+ "model.layers.17.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
75
+ "model.layers.17.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
76
+ "model.layers.17.self_attn.W_pack.weight": "pytorch_model-00002-of-00003.bin",
77
+ "model.layers.17.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
78
+ "model.layers.18.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
79
+ "model.layers.18.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
80
+ "model.layers.18.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
81
+ "model.layers.18.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
82
+ "model.layers.18.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
83
+ "model.layers.18.self_attn.W_pack.weight": "pytorch_model-00002-of-00003.bin",
84
+ "model.layers.18.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
85
+ "model.layers.19.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
86
+ "model.layers.19.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
87
+ "model.layers.19.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
88
+ "model.layers.19.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
89
+ "model.layers.19.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
90
+ "model.layers.19.self_attn.W_pack.weight": "pytorch_model-00002-of-00003.bin",
91
+ "model.layers.19.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
92
+ "model.layers.2.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
93
+ "model.layers.2.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
94
+ "model.layers.2.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
95
+ "model.layers.2.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
96
+ "model.layers.2.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
97
+ "model.layers.2.self_attn.W_pack.weight": "pytorch_model-00001-of-00003.bin",
98
+ "model.layers.2.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
99
+ "model.layers.20.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
100
+ "model.layers.20.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
101
+ "model.layers.20.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
102
+ "model.layers.20.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
103
+ "model.layers.20.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
104
+ "model.layers.20.self_attn.W_pack.weight": "pytorch_model-00002-of-00003.bin",
105
+ "model.layers.20.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
106
+ "model.layers.21.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
107
+ "model.layers.21.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
108
+ "model.layers.21.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
109
+ "model.layers.21.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
110
+ "model.layers.21.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
111
+ "model.layers.21.self_attn.W_pack.weight": "pytorch_model-00002-of-00003.bin",
112
+ "model.layers.21.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
113
+ "model.layers.22.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
114
+ "model.layers.22.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
115
+ "model.layers.22.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
116
+ "model.layers.22.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
117
+ "model.layers.22.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
118
+ "model.layers.22.self_attn.W_pack.weight": "pytorch_model-00002-of-00003.bin",
119
+ "model.layers.22.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
120
+ "model.layers.23.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
121
+ "model.layers.23.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
122
+ "model.layers.23.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
123
+ "model.layers.23.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
124
+ "model.layers.23.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
125
+ "model.layers.23.self_attn.W_pack.weight": "pytorch_model-00002-of-00003.bin",
126
+ "model.layers.23.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
127
+ "model.layers.24.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
128
+ "model.layers.24.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
129
+ "model.layers.24.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
130
+ "model.layers.24.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
131
+ "model.layers.24.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
132
+ "model.layers.24.self_attn.W_pack.weight": "pytorch_model-00002-of-00003.bin",
133
+ "model.layers.24.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
134
+ "model.layers.25.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
135
+ "model.layers.25.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
136
+ "model.layers.25.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
137
+ "model.layers.25.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
138
+ "model.layers.25.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
139
+ "model.layers.25.self_attn.W_pack.weight": "pytorch_model-00002-of-00003.bin",
140
+ "model.layers.25.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
141
+ "model.layers.26.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
142
+ "model.layers.26.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
143
+ "model.layers.26.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
144
+ "model.layers.26.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
145
+ "model.layers.26.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
146
+ "model.layers.26.self_attn.W_pack.weight": "pytorch_model-00002-of-00003.bin",
147
+ "model.layers.26.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
148
+ "model.layers.27.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
149
+ "model.layers.27.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
150
+ "model.layers.27.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
151
+ "model.layers.27.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
152
+ "model.layers.27.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
153
+ "model.layers.27.self_attn.W_pack.weight": "pytorch_model-00002-of-00003.bin",
154
+ "model.layers.27.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
155
+ "model.layers.28.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
156
+ "model.layers.28.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
157
+ "model.layers.28.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
158
+ "model.layers.28.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
159
+ "model.layers.28.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
160
+ "model.layers.28.self_attn.W_pack.weight": "pytorch_model-00002-of-00003.bin",
161
+ "model.layers.28.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
162
+ "model.layers.29.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
163
+ "model.layers.29.mlp.down_proj.weight": "pytorch_model-00003-of-00003.bin",
164
+ "model.layers.29.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
165
+ "model.layers.29.mlp.up_proj.weight": "pytorch_model-00003-of-00003.bin",
166
+ "model.layers.29.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
167
+ "model.layers.29.self_attn.W_pack.weight": "pytorch_model-00002-of-00003.bin",
168
+ "model.layers.29.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
169
+ "model.layers.3.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
170
+ "model.layers.3.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
171
+ "model.layers.3.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
172
+ "model.layers.3.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
173
+ "model.layers.3.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
174
+ "model.layers.3.self_attn.W_pack.weight": "pytorch_model-00001-of-00003.bin",
175
+ "model.layers.3.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
176
+ "model.layers.30.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
177
+ "model.layers.30.mlp.down_proj.weight": "pytorch_model-00003-of-00003.bin",
178
+ "model.layers.30.mlp.gate_proj.weight": "pytorch_model-00003-of-00003.bin",
179
+ "model.layers.30.mlp.up_proj.weight": "pytorch_model-00003-of-00003.bin",
180
+ "model.layers.30.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
181
+ "model.layers.30.self_attn.W_pack.weight": "pytorch_model-00003-of-00003.bin",
182
+ "model.layers.30.self_attn.o_proj.weight": "pytorch_model-00003-of-00003.bin",
183
+ "model.layers.31.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
184
+ "model.layers.31.mlp.down_proj.weight": "pytorch_model-00003-of-00003.bin",
185
+ "model.layers.31.mlp.gate_proj.weight": "pytorch_model-00003-of-00003.bin",
186
+ "model.layers.31.mlp.up_proj.weight": "pytorch_model-00003-of-00003.bin",
187
+ "model.layers.31.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
188
+ "model.layers.31.self_attn.W_pack.weight": "pytorch_model-00003-of-00003.bin",
189
+ "model.layers.31.self_attn.o_proj.weight": "pytorch_model-00003-of-00003.bin",
190
+ "model.layers.32.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
191
+ "model.layers.32.mlp.down_proj.weight": "pytorch_model-00003-of-00003.bin",
192
+ "model.layers.32.mlp.gate_proj.weight": "pytorch_model-00003-of-00003.bin",
193
+ "model.layers.32.mlp.up_proj.weight": "pytorch_model-00003-of-00003.bin",
194
+ "model.layers.32.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
195
+ "model.layers.32.self_attn.W_pack.weight": "pytorch_model-00003-of-00003.bin",
196
+ "model.layers.32.self_attn.o_proj.weight": "pytorch_model-00003-of-00003.bin",
197
+ "model.layers.33.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
198
+ "model.layers.33.mlp.down_proj.weight": "pytorch_model-00003-of-00003.bin",
199
+ "model.layers.33.mlp.gate_proj.weight": "pytorch_model-00003-of-00003.bin",
200
+ "model.layers.33.mlp.up_proj.weight": "pytorch_model-00003-of-00003.bin",
201
+ "model.layers.33.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
202
+ "model.layers.33.self_attn.W_pack.weight": "pytorch_model-00003-of-00003.bin",
203
+ "model.layers.33.self_attn.o_proj.weight": "pytorch_model-00003-of-00003.bin",
204
+ "model.layers.34.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
205
+ "model.layers.34.mlp.down_proj.weight": "pytorch_model-00003-of-00003.bin",
206
+ "model.layers.34.mlp.gate_proj.weight": "pytorch_model-00003-of-00003.bin",
207
+ "model.layers.34.mlp.up_proj.weight": "pytorch_model-00003-of-00003.bin",
208
+ "model.layers.34.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
209
+ "model.layers.34.self_attn.W_pack.weight": "pytorch_model-00003-of-00003.bin",
210
+ "model.layers.34.self_attn.o_proj.weight": "pytorch_model-00003-of-00003.bin",
211
+ "model.layers.35.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
212
+ "model.layers.35.mlp.down_proj.weight": "pytorch_model-00003-of-00003.bin",
213
+ "model.layers.35.mlp.gate_proj.weight": "pytorch_model-00003-of-00003.bin",
214
+ "model.layers.35.mlp.up_proj.weight": "pytorch_model-00003-of-00003.bin",
215
+ "model.layers.35.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
216
+ "model.layers.35.self_attn.W_pack.weight": "pytorch_model-00003-of-00003.bin",
217
+ "model.layers.35.self_attn.o_proj.weight": "pytorch_model-00003-of-00003.bin",
218
+ "model.layers.36.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
219
+ "model.layers.36.mlp.down_proj.weight": "pytorch_model-00003-of-00003.bin",
220
+ "model.layers.36.mlp.gate_proj.weight": "pytorch_model-00003-of-00003.bin",
221
+ "model.layers.36.mlp.up_proj.weight": "pytorch_model-00003-of-00003.bin",
222
+ "model.layers.36.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
223
+ "model.layers.36.self_attn.W_pack.weight": "pytorch_model-00003-of-00003.bin",
224
+ "model.layers.36.self_attn.o_proj.weight": "pytorch_model-00003-of-00003.bin",
225
+ "model.layers.37.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
226
+ "model.layers.37.mlp.down_proj.weight": "pytorch_model-00003-of-00003.bin",
227
+ "model.layers.37.mlp.gate_proj.weight": "pytorch_model-00003-of-00003.bin",
228
+ "model.layers.37.mlp.up_proj.weight": "pytorch_model-00003-of-00003.bin",
229
+ "model.layers.37.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
230
+ "model.layers.37.self_attn.W_pack.weight": "pytorch_model-00003-of-00003.bin",
231
+ "model.layers.37.self_attn.o_proj.weight": "pytorch_model-00003-of-00003.bin",
232
+ "model.layers.38.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
233
+ "model.layers.38.mlp.down_proj.weight": "pytorch_model-00003-of-00003.bin",
234
+ "model.layers.38.mlp.gate_proj.weight": "pytorch_model-00003-of-00003.bin",
235
+ "model.layers.38.mlp.up_proj.weight": "pytorch_model-00003-of-00003.bin",
236
+ "model.layers.38.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
237
+ "model.layers.38.self_attn.W_pack.weight": "pytorch_model-00003-of-00003.bin",
238
+ "model.layers.38.self_attn.o_proj.weight": "pytorch_model-00003-of-00003.bin",
239
+ "model.layers.39.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
240
+ "model.layers.39.mlp.down_proj.weight": "pytorch_model-00003-of-00003.bin",
241
+ "model.layers.39.mlp.gate_proj.weight": "pytorch_model-00003-of-00003.bin",
242
+ "model.layers.39.mlp.up_proj.weight": "pytorch_model-00003-of-00003.bin",
243
+ "model.layers.39.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
244
+ "model.layers.39.self_attn.W_pack.weight": "pytorch_model-00003-of-00003.bin",
245
+ "model.layers.39.self_attn.o_proj.weight": "pytorch_model-00003-of-00003.bin",
246
+ "model.layers.4.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
247
+ "model.layers.4.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
248
+ "model.layers.4.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
249
+ "model.layers.4.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
250
+ "model.layers.4.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
251
+ "model.layers.4.self_attn.W_pack.weight": "pytorch_model-00001-of-00003.bin",
252
+ "model.layers.4.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
253
+ "model.layers.5.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
254
+ "model.layers.5.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
255
+ "model.layers.5.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
256
+ "model.layers.5.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
257
+ "model.layers.5.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
258
+ "model.layers.5.self_attn.W_pack.weight": "pytorch_model-00001-of-00003.bin",
259
+ "model.layers.5.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
260
+ "model.layers.6.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
261
+ "model.layers.6.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
262
+ "model.layers.6.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
263
+ "model.layers.6.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
264
+ "model.layers.6.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
265
+ "model.layers.6.self_attn.W_pack.weight": "pytorch_model-00001-of-00003.bin",
266
+ "model.layers.6.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
267
+ "model.layers.7.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
268
+ "model.layers.7.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
269
+ "model.layers.7.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
270
+ "model.layers.7.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
271
+ "model.layers.7.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
272
+ "model.layers.7.self_attn.W_pack.weight": "pytorch_model-00001-of-00003.bin",
273
+ "model.layers.7.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
274
+ "model.layers.8.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
275
+ "model.layers.8.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
276
+ "model.layers.8.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
277
+ "model.layers.8.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
278
+ "model.layers.8.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
279
+ "model.layers.8.self_attn.W_pack.weight": "pytorch_model-00001-of-00003.bin",
280
+ "model.layers.8.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
281
+ "model.layers.9.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
282
+ "model.layers.9.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
283
+ "model.layers.9.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
284
+ "model.layers.9.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
285
+ "model.layers.9.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
286
+ "model.layers.9.self_attn.W_pack.weight": "pytorch_model-00001-of-00003.bin",
287
+ "model.layers.9.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
288
+ "model.norm.weight": "pytorch_model-00003-of-00003.bin"
289
+ }
290
+ }
Baichuan2-13B-Chat/quantizer.py ADDED
@@ -0,0 +1,215 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ try:
2
+ import bitsandbytes as bnb
3
+ from bitsandbytes.nn.modules import Params4bit, Int8Params
4
+ except ImportError:
5
+ print('import bitsandbytes Error')
6
+
7
+ from accelerate import init_empty_weights
8
+ import torch
9
+
10
+ def Params4bitCuda(self, device):
11
+ self.data = self.data.cuda(device)
12
+ self.quant_state[0] = self.quant_state[0].cuda(device)
13
+ self.quant_state[4][0] = self.quant_state[4][0].cuda(device)
14
+ self.quant_state[4][1][0] = self.quant_state[4][1][0].cuda(device)
15
+ self.quant_state[4][1][1] = self.quant_state[4][1][1].cuda(device)
16
+
17
+ self.quant_state[6] = self.quant_state[6].cuda(device)
18
+ return self
19
+
20
+ class Linear4bitOnline(torch.nn.Module):
21
+ def __init__(self, weight, bias, quant_type):
22
+ super().__init__()
23
+ self.weight = Params4bit(
24
+ weight.data, requires_grad=False, compress_statistics=True, quant_type=quant_type
25
+ )
26
+ self.compute_dtype = None
27
+ #self.weight.cuda(weight.device)
28
+ self.bias = bias
29
+
30
+ def forward(self, x: torch.Tensor):
31
+ # weights are cast automatically as Int8Params, but the bias has to be cast manually
32
+ if self.bias is not None and self.bias.dtype != x.dtype:
33
+ self.bias.data = self.bias.data.to(x.dtype)
34
+
35
+ if getattr(self.weight, "quant_state", None) is None:
36
+ print(
37
+ "FP4 quantization state not initialized. Please call .cuda() or .to(device) on the LinearFP4 layer first."
38
+ )
39
+ inp_dtype = x.dtype
40
+ if self.compute_dtype is not None:
41
+ x = x.to(self.compute_dtype)
42
+
43
+ bias = None if self.bias is None else self.bias.to(self.compute_dtype)
44
+ out = bnb.matmul_4bit(
45
+ x, self.weight.t(), bias=bias, quant_state=self.weight.quant_state
46
+ )
47
+
48
+ out = out.to(inp_dtype)
49
+
50
+ return out
51
+
52
+ class Linear8bitLtOnline(torch.nn.Module):
53
+ def __init__(
54
+ self,
55
+ weight,
56
+ bias,
57
+ has_fp16_weights=True,
58
+ memory_efficient_backward=False,
59
+ threshold=0.0,
60
+ index=None,
61
+ ):
62
+ super().__init__()
63
+ assert (
64
+ not memory_efficient_backward
65
+ ), "memory_efficient_backward is no longer required and the argument is deprecated in 0.37.0 and will be removed in 0.39.0"
66
+ self.state = bnb.MatmulLtState()
67
+ self.index = index
68
+
69
+ # Necessary for stacked layers
70
+ self.state.threshold = threshold
71
+ self.state.has_fp16_weights = has_fp16_weights
72
+ self.state.memory_efficient_backward = memory_efficient_backward
73
+ if threshold > 0.0 and not has_fp16_weights:
74
+ self.state.use_pool = True
75
+
76
+ self.weight = Int8Params(
77
+ weight.data,
78
+ has_fp16_weights=has_fp16_weights,
79
+ requires_grad=has_fp16_weights,
80
+ )
81
+ self.bias = bias
82
+
83
+ def init_8bit_state(self):
84
+ self.state.CB = self.weight.CB
85
+ self.state.SCB = self.weight.SCB
86
+ self.weight.CB = None
87
+ self.weight.SCB = None
88
+
89
+ def forward(self, x: torch.Tensor):
90
+ self.state.is_training = self.training
91
+ if self.weight.CB is not None:
92
+ self.init_8bit_state()
93
+
94
+ # weights are cast automatically as Int8Params, but the bias has to be cast manually
95
+ if self.bias is not None and self.bias.dtype != x.dtype:
96
+ self.bias.data = self.bias.data.to(x.dtype)
97
+
98
+ out = bnb.matmul(x, self.weight, bias=self.bias, state=self.state)
99
+
100
+ if not self.state.has_fp16_weights:
101
+ if self.state.CB is not None and self.state.CxB is not None:
102
+ # we converted 8-bit row major to turing/ampere format in the first inference pass
103
+ # we no longer need the row-major weight
104
+ del self.state.CB
105
+ self.weight.data = self.state.CxB
106
+ return out
107
+
108
+ def quantize_offline(model, bits: int):
109
+ assert (bits == 4), f'bits: {bits} is not supported'
110
+
111
+ for i, layer in enumerate(model.model.layers):
112
+ layer.self_attn.W_pack = bnb.nn.Linear4bit(
113
+ layer.self_attn.W_pack.weight.shape[1],
114
+ layer.self_attn.W_pack.weight.shape[0],
115
+ False,
116
+ torch.float16,
117
+ compress_statistics=True,
118
+ quant_type="nf4",
119
+ )
120
+ layer.self_attn.o_proj = bnb.nn.Linear4bit(
121
+ layer.self_attn.o_proj.weight.shape[1],
122
+ layer.self_attn.o_proj.weight.shape[0],
123
+ False,
124
+ torch.float16,
125
+ compress_statistics=True,
126
+ quant_type="nf4",
127
+ )
128
+
129
+ layer.mlp.gate_proj = bnb.nn.Linear4bit(
130
+ layer.mlp.gate_proj.weight.shape[1],
131
+ layer.mlp.gate_proj.weight.shape[0],
132
+ False,
133
+ torch.float16,
134
+ compress_statistics=True,
135
+ quant_type="nf4",
136
+ )
137
+ layer.mlp.down_proj = bnb.nn.Linear4bit(
138
+ layer.mlp.down_proj.weight.shape[1],
139
+ layer.mlp.down_proj.weight.shape[0],
140
+ False,
141
+ torch.float16,
142
+ compress_statistics=True,
143
+ quant_type="nf4",
144
+ )
145
+ layer.mlp.up_proj = bnb.nn.Linear4bit(
146
+ layer.mlp.up_proj.weight.shape[1],
147
+ layer.mlp.up_proj.weight.shape[0],
148
+ False,
149
+ torch.float16,
150
+ compress_statistics=True,
151
+ quant_type="nf4",
152
+ )
153
+ return model
154
+
155
+ def quantize_online(model, bits: int):
156
+ def quant(weight, bias=None):
157
+ if bits == 8:
158
+ linear = Linear8bitLtOnline(
159
+ weight,
160
+ bias,
161
+ has_fp16_weights=False,
162
+ threshold=6.0,
163
+ )
164
+ if bias is not None:
165
+ linear.bias = torch.nn.Parameter(bias)
166
+ elif bits == 4:
167
+ linear = Linear4bitOnline(
168
+ weight,
169
+ bias,
170
+ quant_type="nf4", #fp4/nf4
171
+ )
172
+ else:
173
+ raise ValueError("quantize only support 4/8 bit")
174
+ return linear
175
+
176
+ for i, layer in enumerate(model.model.layers):
177
+ layer.self_attn.W_pack = quant(layer.self_attn.W_pack.weight)
178
+ layer.self_attn.o_proj = quant(layer.self_attn.o_proj.weight)
179
+ layer.mlp.gate_proj = quant(layer.mlp.gate_proj.weight)
180
+ layer.mlp.down_proj = quant(layer.mlp.down_proj.weight)
181
+ layer.mlp.up_proj = quant(layer.mlp.up_proj.weight)
182
+ return model
183
+
184
+ def init_model_weight_int4(config, model, state_dict):
185
+ #replace Params4bit.cuda with Params4bitCuda
186
+ Params4bit.cuda = Params4bitCuda
187
+
188
+ for i in range(config.num_hidden_layers):
189
+ weight_data = state_dict[f'model.layers.{i}.self_attn.W_pack.weight.data']
190
+ weight_quant_state = state_dict[f'model.layers.{i}.self_attn.W_pack.weight.quant_state']
191
+ model.model.layers[i].self_attn.W_pack.weight = Params4bit(weight_data, requires_grad=False, quant_state=weight_quant_state)
192
+
193
+ weight_data = state_dict[f'model.layers.{i}.self_attn.o_proj.weight.data']
194
+ weight_quant_state = state_dict[f'model.layers.{i}.self_attn.o_proj.weight.quant_state']
195
+ model.model.layers[i].self_attn.o_proj.weight = Params4bit(weight_data, requires_grad=False, quant_state=weight_quant_state)
196
+
197
+ weight_data = state_dict[f'model.layers.{i}.mlp.gate_proj.weight.data']
198
+ weight_quant_state = state_dict[f'model.layers.{i}.mlp.gate_proj.weight.quant_state']
199
+ model.model.layers[i].mlp.gate_proj.weight = Params4bit(weight_data, requires_grad=False, quant_state=weight_quant_state)
200
+
201
+ weight_data = state_dict[f'model.layers.{i}.mlp.up_proj.weight.data']
202
+ weight_quant_state = state_dict[f'model.layers.{i}.mlp.up_proj.weight.quant_state']
203
+ model.model.layers[i].mlp.up_proj.weight = Params4bit(weight_data, requires_grad=False, quant_state=weight_quant_state)
204
+
205
+ weight_data = state_dict[f'model.layers.{i}.mlp.down_proj.weight.data']
206
+ weight_quant_state = state_dict[f'model.layers.{i}.mlp.down_proj.weight.quant_state']
207
+ model.model.layers[i].mlp.down_proj.weight = Params4bit(weight_data, requires_grad=False, quant_state=weight_quant_state)
208
+
209
+ model.model.layers[i].input_layernorm.weight = state_dict[f'model.layers.{i}.input_layernorm.weight']
210
+ model.model.layers[i].post_attention_layernorm.weight = state_dict[f'model.layers.{i}.post_attention_layernorm.weight']
211
+
212
+ model.model.embed_tokens.weight = state_dict['model.embed_tokens.weight']
213
+ model.model.norm.weight = state_dict['model.norm.weight']
214
+ model.lm_head.weight = state_dict['lm_head.weight']
215
+ return model
Baichuan2-13B-Chat/special_tokens_map.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": true,
6
+ "rstrip": false,
7
+ "single_word": true
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": true,
13
+ "rstrip": false,
14
+ "single_word": true
15
+ },
16
+ "unk_token": {
17
+ "content": "<unk>",
18
+ "lstrip": false,
19
+ "normalized": true,
20
+ "rstrip": false,
21
+ "single_word": true
22
+ },
23
+ "pad_token": {
24
+ "content": "<unk>",
25
+ "lstrip": false,
26
+ "normalized": true,
27
+ "rstrip": false,
28
+ "single_word": true
29
+ }
30
+ }
Baichuan2-13B-Chat/tokenization_baichuan.py ADDED
@@ -0,0 +1,258 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) 2023, Baichuan Intelligent Technology. All rights reserved.
2
+
3
+ import os
4
+ from shutil import copyfile
5
+ from typing import Any, Dict, List, Optional, Tuple
6
+
7
+ import sentencepiece as spm
8
+ from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
9
+ from transformers.utils import logging
10
+
11
+
12
+ logger = logging.get_logger(__name__)
13
+
14
+ VOCAB_FILES_NAMES = {"vocab_file": "tokenizer.model"}
15
+
16
+ PRETRAINED_VOCAB_FILES_MAP = {
17
+ "vocab_file": {},
18
+ "tokenizer_file": {},
19
+ }
20
+ PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {}
21
+
22
+
23
+ class BaichuanTokenizer(PreTrainedTokenizer):
24
+ """
25
+ Construct a Baichuan tokenizer. Based on byte-level Byte-Pair-Encoding.
26
+
27
+ Args:
28
+ vocab_file (`str`):
29
+ Path to the vocabulary file.
30
+ """
31
+
32
+ vocab_files_names = VOCAB_FILES_NAMES
33
+ pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
34
+ max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
35
+ model_input_names = ["input_ids", "attention_mask"]
36
+
37
+ def __init__(
38
+ self,
39
+ vocab_file,
40
+ unk_token="<unk>",
41
+ bos_token="<s>",
42
+ eos_token="</s>",
43
+ pad_token=None,
44
+ sp_model_kwargs: Optional[Dict[str, Any]] = None,
45
+ add_bos_token=True,
46
+ add_eos_token=False,
47
+ clean_up_tokenization_spaces=False,
48
+ **kwargs,
49
+ ):
50
+ self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
51
+ bos_token = (
52
+ AddedToken(bos_token, lstrip=False, rstrip=False)
53
+ if isinstance(bos_token, str)
54
+ else bos_token
55
+ )
56
+ eos_token = (
57
+ AddedToken(eos_token, lstrip=False, rstrip=False)
58
+ if isinstance(eos_token, str)
59
+ else eos_token
60
+ )
61
+ unk_token = (
62
+ AddedToken(unk_token, lstrip=False, rstrip=False)
63
+ if isinstance(unk_token, str)
64
+ else unk_token
65
+ )
66
+ pad_token = (
67
+ AddedToken(pad_token, lstrip=False, rstrip=False)
68
+ if isinstance(pad_token, str)
69
+ else pad_token
70
+ )
71
+ self.vocab_file = vocab_file
72
+ self.add_bos_token = add_bos_token
73
+ self.add_eos_token = add_eos_token
74
+ self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
75
+ self.sp_model.Load(vocab_file)
76
+ super().__init__(
77
+ bos_token=bos_token,
78
+ eos_token=eos_token,
79
+ unk_token=unk_token,
80
+ pad_token=pad_token,
81
+ add_bos_token=add_bos_token,
82
+ add_eos_token=add_eos_token,
83
+ sp_model_kwargs=self.sp_model_kwargs,
84
+ clean_up_tokenization_spaces=clean_up_tokenization_spaces,
85
+ **kwargs,
86
+ )
87
+
88
+ def __getstate__(self):
89
+ state = self.__dict__.copy()
90
+ state["sp_model"] = None
91
+ return state
92
+
93
+ def __setstate__(self, d):
94
+ self.__dict__ = d
95
+ self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
96
+ self.sp_model.Load(self.vocab_file)
97
+
98
+ @property
99
+ def vocab_size(self):
100
+ """Returns vocab size"""
101
+ return self.sp_model.get_piece_size()
102
+
103
+ def get_vocab(self):
104
+ """Returns vocab as a dict"""
105
+ vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
106
+ vocab.update(self.added_tokens_encoder)
107
+ return vocab
108
+
109
+ def _tokenize(self, text):
110
+ """Returns a tokenized string."""
111
+ return self.sp_model.encode(text, out_type=str)
112
+
113
+ def _convert_token_to_id(self, token):
114
+ """Converts a token (str) in an id using the vocab."""
115
+ return self.sp_model.piece_to_id(token)
116
+
117
+ def _convert_id_to_token(self, index):
118
+ """Converts an index (integer) in a token (str) using the vocab."""
119
+ token = self.sp_model.IdToPiece(index)
120
+ return token
121
+
122
+ def convert_tokens_to_string(self, tokens):
123
+ """Converts a sequence of tokens (string) in a single string."""
124
+ current_sub_tokens = []
125
+ out_string = ""
126
+ prev_is_special = False
127
+ for i, token in enumerate(tokens):
128
+ # make sure that special tokens are not decoded using sentencepiece model
129
+ if token in self.all_special_tokens:
130
+ if not prev_is_special and i != 0:
131
+ out_string += " "
132
+ out_string += self.sp_model.decode(current_sub_tokens) + token
133
+ prev_is_special = True
134
+ current_sub_tokens = []
135
+ else:
136
+ current_sub_tokens.append(token)
137
+ prev_is_special = False
138
+ out_string += self.sp_model.decode(current_sub_tokens)
139
+ return out_string
140
+
141
+ def save_vocabulary(
142
+ self, save_directory, filename_prefix: Optional[str] = None
143
+ ) -> Tuple[str]:
144
+ """
145
+ Save the vocabulary and special tokens file to a directory.
146
+
147
+ Args:
148
+ save_directory (`str`):
149
+ The directory in which to save the vocabulary.
150
+
151
+ Returns:
152
+ `Tuple(str)`: Paths to the files saved.
153
+ """
154
+ if not os.path.isdir(save_directory):
155
+ logger.error(f"Vocabulary path ({save_directory}) should be a directory")
156
+ return
157
+ out_vocab_file = os.path.join(
158
+ save_directory,
159
+ (filename_prefix + "-" if filename_prefix else "")
160
+ + VOCAB_FILES_NAMES["vocab_file"],
161
+ )
162
+
163
+ if os.path.abspath(self.vocab_file) != os.path.abspath(
164
+ out_vocab_file
165
+ ) and os.path.isfile(self.vocab_file):
166
+ copyfile(self.vocab_file, out_vocab_file)
167
+ elif not os.path.isfile(self.vocab_file):
168
+ with open(out_vocab_file, "wb") as fi:
169
+ content_spiece_model = self.sp_model.serialized_model_proto()
170
+ fi.write(content_spiece_model)
171
+
172
+ return (out_vocab_file,)
173
+
174
+ def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
175
+ bos_token_id = [self.bos_token_id] if self.add_bos_token else []
176
+ eos_token_id = [self.eos_token_id] if self.add_eos_token else []
177
+
178
+ output = bos_token_id + token_ids_0 + eos_token_id
179
+
180
+ if token_ids_1 is not None:
181
+ output = output + bos_token_id + token_ids_1 + eos_token_id
182
+
183
+ return output
184
+
185
+ def get_special_tokens_mask(
186
+ self,
187
+ token_ids_0: List[int],
188
+ token_ids_1: Optional[List[int]] = None,
189
+ already_has_special_tokens: bool = False,
190
+ ) -> List[int]:
191
+ """
192
+ Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
193
+ special tokens using the tokenizer `prepare_for_model` method.
194
+
195
+ Args:
196
+ token_ids_0 (`List[int]`):
197
+ List of IDs.
198
+ token_ids_1 (`List[int]`, *optional*):
199
+ Optional second list of IDs for sequence pairs.
200
+ already_has_special_tokens (`bool`, *optional*, defaults to `False`):
201
+ Whether or not the token list is already formatted with special tokens for the model.
202
+
203
+ Returns:
204
+ `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
205
+ """
206
+ if already_has_special_tokens:
207
+ return super().get_special_tokens_mask(
208
+ token_ids_0=token_ids_0,
209
+ token_ids_1=token_ids_1,
210
+ already_has_special_tokens=True,
211
+ )
212
+
213
+ bos_token_id = [1] if self.add_bos_token else []
214
+ eos_token_id = [1] if self.add_eos_token else []
215
+
216
+ if token_ids_1 is None:
217
+ return bos_token_id + ([0] * len(token_ids_0)) + eos_token_id
218
+ return (
219
+ bos_token_id
220
+ + ([0] * len(token_ids_0))
221
+ + eos_token_id
222
+ + bos_token_id
223
+ + ([0] * len(token_ids_1))
224
+ + eos_token_id
225
+ )
226
+
227
+ def create_token_type_ids_from_sequences(
228
+ self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
229
+ ) -> List[int]:
230
+ """
231
+ Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT
232
+ sequence pair mask has the following format:
233
+
234
+ ```
235
+ 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
236
+ | first sequence | second sequence |
237
+ ```
238
+
239
+ if token_ids_1 is None, only returns the first portion of the mask (0s).
240
+
241
+ Args:
242
+ token_ids_0 (`List[int]`):
243
+ List of ids.
244
+ token_ids_1 (`List[int]`, *optional*):
245
+ Optional second list of IDs for sequence pairs.
246
+
247
+ Returns:
248
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
249
+ """
250
+ bos_token_id = [self.bos_token_id] if self.add_bos_token else []
251
+ eos_token_id = [self.eos_token_id] if self.add_eos_token else []
252
+
253
+ output = [0] * len(bos_token_id + token_ids_0 + eos_token_id)
254
+
255
+ if token_ids_1 is not None:
256
+ output += [1] * len(bos_token_id + token_ids_1 + eos_token_id)
257
+
258
+ return output
Baichuan2-13B-Chat/tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:79452955be6b419a65984273a9f08af86042e1c2a75ee3ba989cbf620a133cc2
3
+ size 2001107
Baichuan2-13B-Chat/tokenizer_config.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": false,
3
+ "add_eos_token": false,
4
+ "auto_map": {
5
+ "AutoTokenizer": [
6
+ "tokenization_baichuan.BaichuanTokenizer",
7
+ null
8
+ ]
9
+ },
10
+ "bos_token": {
11
+ "__type": "AddedToken",
12
+ "content": "<s>",
13
+ "lstrip": false,
14
+ "normalized": true,
15
+ "rstrip": false,
16
+ "single_word": true
17
+ },
18
+ "clean_up_tokenization_spaces": false,
19
+ "eos_token": {
20
+ "__type": "AddedToken",
21
+ "content": "</s>",
22
+ "lstrip": false,
23
+ "normalized": true,
24
+ "rstrip": false,
25
+ "single_word": true
26
+ },
27
+ "model_max_length": 4096,
28
+ "pad_token": {
29
+ "__type": "AddedToken",
30
+ "content": "<unk>",
31
+ "lstrip": false,
32
+ "normalized": true,
33
+ "rstrip": false,
34
+ "single_word": true
35
+ },
36
+ "sp_model_kwargs": {},
37
+ "tokenizer_class": "BaichuanTokenizer",
38
+ "unk_token": {
39
+ "__type": "AddedToken",
40
+ "content": "<unk>",
41
+ "lstrip": false,
42
+ "normalized": true,
43
+ "rstrip": false,
44
+ "single_word": true
45
+ }
46
+ }
Baichuan2-13B-Chat/upload.py ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ from modelscope.hub.api import HubApi
2
+
3
+ YOUR_ACCESS_TOKEN = 'f1775e28-a401-4ad3-9442-9903fbfb80dd'
4
+
5
+ api = HubApi()
6
+ api.login(YOUR_ACCESS_TOKEN)
7
+ api.push_model(
8
+ model_id="V1ncentS/RC-GPT",
9
+ model_dir="D:\复旦\金融风控LLM+KG\Baichuan\Baichuan2-13B-Chat" # 本地模型目录,要求目录中必须包含configuration.json
10
+ )