thomas-yanxin commited on
Commit
02d13dc
1 Parent(s): 95d444a
Files changed (46) hide show
  1. app.py +259 -0
  2. chatllm.py +159 -0
  3. chinese_text_splitter.py +24 -0
  4. nltk_data/taggers/averaged_perceptron_tagger/averaged_perceptron_tagger.pickle +3 -0
  5. nltk_data/tokenizers/punkt/.DS_Store +0 -0
  6. nltk_data/tokenizers/punkt/PY3/README +98 -0
  7. nltk_data/tokenizers/punkt/PY3/czech.pickle +3 -0
  8. nltk_data/tokenizers/punkt/PY3/danish.pickle +3 -0
  9. nltk_data/tokenizers/punkt/PY3/dutch.pickle +3 -0
  10. nltk_data/tokenizers/punkt/PY3/english.pickle +3 -0
  11. nltk_data/tokenizers/punkt/PY3/estonian.pickle +3 -0
  12. nltk_data/tokenizers/punkt/PY3/finnish.pickle +3 -0
  13. nltk_data/tokenizers/punkt/PY3/french.pickle +3 -0
  14. nltk_data/tokenizers/punkt/PY3/german.pickle +3 -0
  15. nltk_data/tokenizers/punkt/PY3/greek.pickle +3 -0
  16. nltk_data/tokenizers/punkt/PY3/italian.pickle +3 -0
  17. nltk_data/tokenizers/punkt/PY3/malayalam.pickle +3 -0
  18. nltk_data/tokenizers/punkt/PY3/norwegian.pickle +3 -0
  19. nltk_data/tokenizers/punkt/PY3/polish.pickle +3 -0
  20. nltk_data/tokenizers/punkt/PY3/portuguese.pickle +3 -0
  21. nltk_data/tokenizers/punkt/PY3/russian.pickle +3 -0
  22. nltk_data/tokenizers/punkt/PY3/slovene.pickle +3 -0
  23. nltk_data/tokenizers/punkt/PY3/spanish.pickle +3 -0
  24. nltk_data/tokenizers/punkt/PY3/swedish.pickle +3 -0
  25. nltk_data/tokenizers/punkt/PY3/turkish.pickle +3 -0
  26. nltk_data/tokenizers/punkt/README +98 -0
  27. nltk_data/tokenizers/punkt/czech.pickle +3 -0
  28. nltk_data/tokenizers/punkt/danish.pickle +3 -0
  29. nltk_data/tokenizers/punkt/dutch.pickle +3 -0
  30. nltk_data/tokenizers/punkt/english.pickle +3 -0
  31. nltk_data/tokenizers/punkt/estonian.pickle +3 -0
  32. nltk_data/tokenizers/punkt/finnish.pickle +3 -0
  33. nltk_data/tokenizers/punkt/french.pickle +3 -0
  34. nltk_data/tokenizers/punkt/german.pickle +3 -0
  35. nltk_data/tokenizers/punkt/greek.pickle +3 -0
  36. nltk_data/tokenizers/punkt/italian.pickle +3 -0
  37. nltk_data/tokenizers/punkt/malayalam.pickle +3 -0
  38. nltk_data/tokenizers/punkt/norwegian.pickle +3 -0
  39. nltk_data/tokenizers/punkt/polish.pickle +3 -0
  40. nltk_data/tokenizers/punkt/portuguese.pickle +3 -0
  41. nltk_data/tokenizers/punkt/russian.pickle +3 -0
  42. nltk_data/tokenizers/punkt/slovene.pickle +3 -0
  43. nltk_data/tokenizers/punkt/spanish.pickle +3 -0
  44. nltk_data/tokenizers/punkt/swedish.pickle +3 -0
  45. nltk_data/tokenizers/punkt/turkish.pickle +3 -0
  46. requirements.txt +16 -0
app.py ADDED
@@ -0,0 +1,259 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+
3
+ import gradio as gr
4
+ import nltk
5
+ import sentence_transformers
6
+ import torch
7
+ from duckduckgo_search import ddg
8
+ from duckduckgo_search.utils import SESSION
9
+ from langchain.chains import RetrievalQA
10
+ from langchain.document_loaders import UnstructuredFileLoader
11
+ from langchain.embeddings.huggingface import HuggingFaceEmbeddings
12
+ from langchain.prompts import PromptTemplate
13
+ from langchain.prompts.prompt import PromptTemplate
14
+ from langchain.vectorstores import FAISS
15
+
16
+ from chatllm import ChatLLM
17
+ from chinese_text_splitter import ChineseTextSplitter
18
+
19
+ nltk.data.path.append('./nltk_data')
20
+
21
+ embedding_model_dict = {
22
+ "ernie-tiny": "nghuyong/ernie-3.0-nano-zh",
23
+ "ernie-base": "nghuyong/ernie-3.0-base-zh",
24
+ "text2vec-base": "GanymedeNil/text2vec-base-chinese"
25
+ }
26
+
27
+ llm_model_dict = {
28
+ "ChatGLM-6B-int4": "THUDM/chatglm-6b-int4",
29
+ "ChatGLM-6B-int8": "THUDM/chatglm-6b-int8",
30
+ "ChatGLM-6b-int4-qe": "THUDM/chatglm-6b-int4-qe",
31
+ "Minimax": "Minimax"
32
+ }
33
+
34
+
35
+ DEVICE = "cuda" if torch.cuda.is_available(
36
+ ) else "mps" if torch.backends.mps.is_available() else "cpu"
37
+
38
+ def search_web(query):
39
+
40
+ SESSION.proxies = {
41
+ "http": f"socks5h://localhost:7890",
42
+ "https": f"socks5h://localhost:7890"
43
+ }
44
+ results = ddg(query)
45
+ web_content = ''
46
+ if results:
47
+ for result in results:
48
+ web_content += result['body']
49
+ return web_content
50
+
51
+ def load_file(filepath):
52
+ if filepath.lower().endswith(".pdf"):
53
+ loader = UnstructuredFileLoader(filepath)
54
+ textsplitter = ChineseTextSplitter(pdf=True)
55
+ docs = loader.load_and_split(textsplitter)
56
+ else:
57
+ loader = UnstructuredFileLoader(filepath, mode="elements")
58
+ textsplitter = ChineseTextSplitter(pdf=False)
59
+ docs = loader.load_and_split(text_splitter=textsplitter)
60
+ return docs
61
+
62
+
63
+
64
+ def init_knowledge_vector_store(embedding_model, filepath):
65
+ embeddings = HuggingFaceEmbeddings(
66
+ model_name=embedding_model_dict[embedding_model], )
67
+ embeddings.client = sentence_transformers.SentenceTransformer(
68
+ embeddings.model_name, device=DEVICE)
69
+
70
+ docs = load_file(filepath)
71
+
72
+ vector_store = FAISS.from_documents(docs, embeddings)
73
+ return vector_store
74
+
75
+
76
+ def get_knowledge_based_answer(query,
77
+ large_language_model,
78
+ vector_store,
79
+ VECTOR_SEARCH_TOP_K,
80
+ web_content,
81
+ history_len,
82
+ temperature,
83
+ top_p,
84
+ chat_history=[]):
85
+ if web_content:
86
+ prompt_template = f"""基于以下已知信息,简洁和专业的来回答用户的问题。
87
+ 如果无法从中得到答案,请说 "根据已知信息无法回答该问题" 或 "没有提供足够的相关信息",不允许在答案中添加编造成分,答案请使用中文。
88
+ 已知网络检索内容:{web_content}""" + """
89
+ 已知内容:
90
+ {context}
91
+ 问题:
92
+ {question}"""
93
+ else:
94
+ prompt_template = """基于以下已知信息,请简洁并专业地回答用户的问题。
95
+ 如果无法从中得到答案,请说 "根据已知信息无法回答该问题" 或 "没有提供足够的相关信息"。不允许在答案中添加编造成分。另外,答案请使用中文。
96
+
97
+ 已知内容:
98
+ {context}
99
+
100
+ 问题:
101
+ {question}"""
102
+ prompt = PromptTemplate(template=prompt_template,
103
+ input_variables=["context", "question"])
104
+ chatLLM = ChatLLM()
105
+ chatLLM.load_model(model_name_or_path=llm_model_dict[large_language_model])
106
+ chatLLM.history = chat_history[-history_len:] if history_len > 0 else []
107
+
108
+ chatLLM.temperature = temperature
109
+ chatLLM.top_p = top_p
110
+
111
+ knowledge_chain = RetrievalQA.from_llm(
112
+ llm=chatLLM,
113
+ retriever=vector_store.as_retriever(
114
+ search_kwargs={"k": VECTOR_SEARCH_TOP_K}),
115
+ prompt=prompt)
116
+ knowledge_chain.combine_documents_chain.document_prompt = PromptTemplate(
117
+ input_variables=["page_content"], template="{page_content}")
118
+
119
+ knowledge_chain.return_source_documents = True
120
+
121
+ result = knowledge_chain({"query": query})
122
+ return result
123
+
124
+
125
+ def clear_session():
126
+ return '', None
127
+
128
+
129
+ def predict(input,
130
+ large_language_model,
131
+ embedding_model,
132
+ file_obj,
133
+ VECTOR_SEARCH_TOP_K,
134
+ history_len,
135
+ temperature,
136
+ top_p,
137
+ use_web,
138
+ history=None):
139
+ if history == None:
140
+ history = []
141
+ print(file_obj.name)
142
+ vector_store = init_knowledge_vector_store(embedding_model, file_obj.name)
143
+ if use_web == 'True':
144
+ web_content = search_web(query=input)
145
+ else:
146
+ web_content = ''
147
+ resp = get_knowledge_based_answer(
148
+ query=input,
149
+ large_language_model=large_language_model,
150
+ vector_store=vector_store,
151
+ VECTOR_SEARCH_TOP_K=VECTOR_SEARCH_TOP_K,
152
+ web_content=web_content,
153
+ chat_history=history,
154
+ history_len=history_len,
155
+ temperature=temperature,
156
+ top_p=top_p,
157
+ )
158
+ print(resp)
159
+ history.append((input, resp['result']))
160
+ return '', history, history
161
+
162
+
163
+ if __name__ == "__main__":
164
+ block = gr.Blocks()
165
+ with block as demo:
166
+ gr.Markdown("""<h1><center>LangChain-ChatLLM-Webui</center></h1>
167
+ <center><font size=3>
168
+ 本项目基于LangChain和大型语言模型系列模型, 提供基于本地知识的自动问答应用. <br>
169
+ 目前项目提供基于<a href='https://github.com/THUDM/ChatGLM-6B' target="_blank">ChatGLM-6B </a>的LLM和包括GanymedeNil/text2vec-large-chinese、nghuyong/ernie-3.0-base-zh、nghuyong/ernie-3.0-nano-zh在内的多个Embedding模型, 支持上传 txt、docx、md 等文本格式文件. <br>
170
+ 后续将提供更加多样化的LLM、Embedding和参数选项供用户尝试, 欢迎关注<a href='https://github.com/thomas-yanxin/LangChain-ChatGLM-Webui' target="_blank">Github地址</a>.
171
+ </center></font>
172
+ """)
173
+ with gr.Row():
174
+ with gr.Column(scale=1):
175
+ model_choose = gr.Accordion("模型选择")
176
+ with model_choose:
177
+ large_language_model = gr.Dropdown(
178
+ list(llm_model_dict.keys()),
179
+ label="large language model",
180
+ value="ChatGLM-6B-int8")
181
+
182
+ embedding_model = gr.Dropdown(list(embedding_model_dict.keys()),
183
+ label="Embedding model",
184
+ value="text2vec-base")
185
+
186
+ file = gr.File(label='请上传知识库文件',
187
+ file_types=['.txt', '.md', '.docx', '.pdf'])
188
+
189
+ use_web = gr.Radio(["True", "False"], label="Web Search",
190
+ value="False"
191
+ )
192
+ model_argument = gr.Accordion("模型参数配置")
193
+
194
+ with model_argument:
195
+
196
+ VECTOR_SEARCH_TOP_K = gr.Slider(1,
197
+ 20,
198
+ value=6,
199
+ step=1,
200
+ label="vector search top k",
201
+ interactive=True)
202
+
203
+ HISTORY_LEN = gr.Slider(0,
204
+ 3,
205
+ value=0,
206
+ step=1,
207
+ label="history len",
208
+ interactive=True)
209
+
210
+ temperature = gr.Slider(0,
211
+ 1,
212
+ value=0.01,
213
+ step=0.01,
214
+ label="temperature",
215
+ interactive=True)
216
+ top_p = gr.Slider(0,
217
+ 1,
218
+ value=0.9,
219
+ step=0.1,
220
+ label="top_p",
221
+ interactive=True)
222
+
223
+
224
+ with gr.Column(scale=4):
225
+ chatbot = gr.Chatbot(label='ChatLLM').style(height=400)
226
+ message = gr.Textbox(label='请输入问题')
227
+ state = gr.State()
228
+
229
+ with gr.Row():
230
+ clear_history = gr.Button("🧹 清除历史对话")
231
+ send = gr.Button("🚀 发送")
232
+
233
+ send.click(predict,
234
+ inputs=[
235
+ message, large_language_model,
236
+ embedding_model, file, VECTOR_SEARCH_TOP_K,
237
+ HISTORY_LEN, temperature, top_p, use_web,state
238
+ ],
239
+ outputs=[message, chatbot, state])
240
+ clear_history.click(fn=clear_session,
241
+ inputs=[],
242
+ outputs=[chatbot, state],
243
+ queue=False)
244
+
245
+ message.submit(predict,
246
+ inputs=[
247
+ message, large_language_model,
248
+ embedding_model, file,
249
+ VECTOR_SEARCH_TOP_K, HISTORY_LEN,
250
+ temperature, top_p, use_web,state
251
+ ],
252
+ outputs=[message, chatbot, state])
253
+ gr.Markdown("""提醒:<br>
254
+ 1. 更改LLM模型前请先刷新页面,否则将返回error(后续将完善此部分). <br>
255
+ 2. 使用时请先上传自己的知识文件,并且文件中不含某些特殊字符,否则将返回error. <br>
256
+ 3. 请勿上传或输入敏感内容,否则输出内容将被平台拦截返回error.<br>
257
+ 4. 有任何使用问题,请通过[问题交流区](https://modelscope.cn/studios/thomas/ChatYuan-test/comment)或[Github Issue区](https://github.com/thomas-yanxin/LangChain-ChatGLM-Webui/issues)进行反馈. <br>
258
+ """)
259
+ demo.queue().launch(server_name='0.0.0.0', share=False)
chatllm.py ADDED
@@ -0,0 +1,159 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ import os
3
+ from typing import Dict, List, Optional, Tuple, Union
4
+
5
+ import torch
6
+ from langchain.llms.base import LLM
7
+ from langchain.llms.utils import enforce_stop_tokens
8
+ from transformers import AutoModel, AutoTokenizer
9
+
10
+ os.environ["TOKENIZERS_PARALLELISM"] = "false"
11
+
12
+ DEVICE = "cpu"
13
+ DEVICE_ID = "0"
14
+ CUDA_DEVICE = f"{DEVICE}:{DEVICE_ID}" if DEVICE_ID else DEVICE
15
+
16
+
17
+ def torch_gc():
18
+ if torch.cuda.is_available():
19
+ with torch.cuda.device(CUDA_DEVICE):
20
+ torch.cuda.empty_cache()
21
+ torch.cuda.ipc_collect()
22
+
23
+ def auto_configure_device_map(num_gpus: int) -> Dict[str, int]:
24
+ # transformer.word_embeddings 占用1层
25
+ # transformer.final_layernorm 和 lm_head 占用1层
26
+ # transformer.layers 占用 28 层
27
+ # 总共30层分配到num_gpus张卡上
28
+ num_trans_layers = 28
29
+ per_gpu_layers = 30 / num_gpus
30
+
31
+ # bugfix: 在linux中调用torch.embedding传入的weight,input不在同一device上,导致RuntimeError
32
+ # windows下 model.device 会被设置成 transformer.word_embeddings.device
33
+ # linux下 model.device 会被设置成 lm_head.device
34
+ # 在调用chat或者stream_chat时,input_ids会被放到model.device上
35
+ # 如果transformer.word_embeddings.device和model.device不同,则会导致RuntimeError
36
+ # 因此这里将transformer.word_embeddings,transformer.final_layernorm,lm_head都放到第一张卡上
37
+ device_map = {'transformer.word_embeddings': 0,
38
+ 'transformer.final_layernorm': 0, 'lm_head': 0}
39
+
40
+ used = 2
41
+ gpu_target = 0
42
+ for i in range(num_trans_layers):
43
+ if used >= per_gpu_layers:
44
+ gpu_target += 1
45
+ used = 0
46
+ assert gpu_target < num_gpus
47
+ device_map[f'transformer.layers.{i}'] = gpu_target
48
+ used += 1
49
+
50
+ return device_map
51
+
52
+
53
+
54
+ class ChatLLM(LLM):
55
+ max_token: int = 10000
56
+ temperature: float = 0.1
57
+ top_p = 0.9
58
+ history = []
59
+ tokenizer: object = None
60
+ model: object = None
61
+
62
+ def __init__(self):
63
+ super().__init__()
64
+
65
+ @property
66
+ def _llm_type(self) -> str:
67
+ return "ChatLLM"
68
+
69
+ def _call(self,
70
+ prompt: str,
71
+ stop: Optional[List[str]] = None) -> str:
72
+
73
+ if self.model_name == 'Minimax':
74
+ import requests
75
+
76
+ group_id = os.getenv('group_id')
77
+ api_key = os.getenv('api_key')
78
+
79
+ url = f'https://api.minimax.chat/v1/text/chatcompletion?GroupId={group_id}'
80
+ headers = {
81
+ "Authorization": f"Bearer {api_key}",
82
+ "Content-Type": "application/json"
83
+ }
84
+ request_body = {
85
+ "model": "abab5-chat",
86
+ "tokens_to_generate": 512,
87
+ 'messages': []
88
+ }
89
+
90
+ for i in self.history:
91
+ h_input = i[0]
92
+ h_reply = i[1]
93
+ request_body['messages'].append({
94
+ "sender_type": "USER",
95
+ "text": h_input
96
+ })
97
+ request_body['messages'].append({"sender_type": "BOT", "text": h_reply})
98
+
99
+ request_body['messages'].append({"sender_type": "USER", "text": prompt})
100
+ resp = requests.post(url, headers=headers, json=request_body)
101
+ response = resp.json()['reply']
102
+ # 将当次的ai回复内容加入messages
103
+ request_body['messages'].append({"sender_type": "BOT", "text": response})
104
+ self.history.append((prompt, response))
105
+
106
+ else:
107
+
108
+ response, _ = self.model.chat(
109
+ self.tokenizer,
110
+ prompt,
111
+ history=self.history,
112
+ max_length=self.max_token,
113
+ temperature=self.temperature,
114
+ )
115
+ torch_gc()
116
+ if stop is not None:
117
+ response = enforce_stop_tokens(response, stop)
118
+ self.history = self.history+[[None, response]]
119
+ return response
120
+
121
+ def load_model(self,
122
+ model_name_or_path: str = "THUDM/chatglm-6b-int4",
123
+ llm_device=DEVICE,
124
+ device_map: Optional[Dict[str, int]] = None,
125
+ **kwargs):
126
+ self.tokenizer = AutoTokenizer.from_pretrained(
127
+ model_name_or_path,
128
+ trust_remote_code=True
129
+ )
130
+ if torch.cuda.is_available() and llm_device.lower().startswith("cuda"):
131
+ # 根据当前设备GPU数量决定是否进行多卡部署
132
+ num_gpus = torch.cuda.device_count()
133
+ if num_gpus < 2 and device_map is None:
134
+ self.model = (
135
+ AutoModel.from_pretrained(
136
+ model_name_or_path,
137
+ trust_remote_code=True,
138
+ **kwargs)
139
+ .half()
140
+ .cuda()
141
+ )
142
+ else:
143
+ from accelerate import dispatch_model
144
+
145
+ model = AutoModel.from_pretrained(model_name_or_path, trust_remote_code=True, **kwargs).half()
146
+ # 可传入device_map自定义每张卡的部署情况
147
+ if device_map is None:
148
+ device_map = auto_configure_device_map(num_gpus)
149
+
150
+ self.model = dispatch_model(model, device_map=device_map)
151
+ else:
152
+ self.model = (
153
+ AutoModel.from_pretrained(
154
+ model_name_or_path,
155
+ trust_remote_code=True)
156
+ .float()
157
+ .to(llm_device)
158
+ )
159
+ self.model = self.model.eval()
chinese_text_splitter.py ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import re
2
+ from typing import List
3
+
4
+ from langchain.text_splitter import CharacterTextSplitter
5
+
6
+
7
+ class ChineseTextSplitter(CharacterTextSplitter):
8
+ def __init__(self, pdf: bool = False, **kwargs):
9
+ super().__init__(**kwargs)
10
+ self.pdf = pdf
11
+
12
+ def split_text(self, text: str) -> List[str]:
13
+ if self.pdf:
14
+ text = re.sub(r"\n{3,}", "\n", text)
15
+ text = re.sub('\s', ' ', text)
16
+ text = text.replace("\n\n", "")
17
+ sent_sep_pattern = re.compile('([﹒﹔﹖﹗.。!?]["’”」』]{0,2}|(?=["‘“「『]{1,2}|$))') # del :;
18
+ sent_list = []
19
+ for ele in sent_sep_pattern.split(text):
20
+ if sent_sep_pattern.match(ele) and sent_list:
21
+ sent_list[-1] += ele
22
+ elif ele:
23
+ sent_list.append(ele)
24
+ return sent_list
nltk_data/taggers/averaged_perceptron_tagger/averaged_perceptron_tagger.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:25a5a19c7ced7b2bac3831da5bc0afcc2c34e5dd01cd4f361bb799949a696238
3
+ size 6138625
nltk_data/tokenizers/punkt/.DS_Store ADDED
Binary file (6.15 kB). View file
 
nltk_data/tokenizers/punkt/PY3/README ADDED
@@ -0,0 +1,98 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Pretrained Punkt Models -- Jan Strunk (New version trained after issues 313 and 514 had been corrected)
2
+
3
+ Most models were prepared using the test corpora from Kiss and Strunk (2006). Additional models have
4
+ been contributed by various people using NLTK for sentence boundary detection.
5
+
6
+ For information about how to use these models, please confer the tokenization HOWTO:
7
+ http://nltk.googlecode.com/svn/trunk/doc/howto/tokenize.html
8
+ and chapter 3.8 of the NLTK book:
9
+ http://nltk.googlecode.com/svn/trunk/doc/book/ch03.html#sec-segmentation
10
+
11
+ There are pretrained tokenizers for the following languages:
12
+
13
+ File Language Source Contents Size of training corpus(in tokens) Model contributed by
14
+ =======================================================================================================================================================================
15
+ czech.pickle Czech Multilingual Corpus 1 (ECI) Lidove Noviny ~345,000 Jan Strunk / Tibor Kiss
16
+ Literarni Noviny
17
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
18
+ danish.pickle Danish Avisdata CD-Rom Ver. 1.1. 1995 Berlingske Tidende ~550,000 Jan Strunk / Tibor Kiss
19
+ (Berlingske Avisdata, Copenhagen) Weekend Avisen
20
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
21
+ dutch.pickle Dutch Multilingual Corpus 1 (ECI) De Limburger ~340,000 Jan Strunk / Tibor Kiss
22
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
23
+ english.pickle English Penn Treebank (LDC) Wall Street Journal ~469,000 Jan Strunk / Tibor Kiss
24
+ (American)
25
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
26
+ estonian.pickle Estonian University of Tartu, Estonia Eesti Ekspress ~359,000 Jan Strunk / Tibor Kiss
27
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
28
+ finnish.pickle Finnish Finnish Parole Corpus, Finnish Books and major national ~364,000 Jan Strunk / Tibor Kiss
29
+ Text Bank (Suomen Kielen newspapers
30
+ Tekstipankki)
31
+ Finnish Center for IT Science
32
+ (CSC)
33
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
34
+ french.pickle French Multilingual Corpus 1 (ECI) Le Monde ~370,000 Jan Strunk / Tibor Kiss
35
+ (European)
36
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
37
+ german.pickle German Neue Zürcher Zeitung AG Neue Zürcher Zeitung ~847,000 Jan Strunk / Tibor Kiss
38
+ (Switzerland) CD-ROM
39
+ (Uses "ss"
40
+ instead of "ß")
41
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
42
+ greek.pickle Greek Efstathios Stamatatos To Vima (TO BHMA) ~227,000 Jan Strunk / Tibor Kiss
43
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
44
+ italian.pickle Italian Multilingual Corpus 1 (ECI) La Stampa, Il Mattino ~312,000 Jan Strunk / Tibor Kiss
45
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
46
+ norwegian.pickle Norwegian Centre for Humanities Bergens Tidende ~479,000 Jan Strunk / Tibor Kiss
47
+ (Bokmål and Information Technologies,
48
+ Nynorsk) Bergen
49
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
50
+ polish.pickle Polish Polish National Corpus Literature, newspapers, etc. ~1,000,000 Krzysztof Langner
51
+ (http://www.nkjp.pl/)
52
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
53
+ portuguese.pickle Portuguese CETENFolha Corpus Folha de São Paulo ~321,000 Jan Strunk / Tibor Kiss
54
+ (Brazilian) (Linguateca)
55
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
56
+ slovene.pickle Slovene TRACTOR Delo ~354,000 Jan Strunk / Tibor Kiss
57
+ Slovene Academy for Arts
58
+ and Sciences
59
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
60
+ spanish.pickle Spanish Multilingual Corpus 1 (ECI) Sur ~353,000 Jan Strunk / Tibor Kiss
61
+ (European)
62
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
63
+ swedish.pickle Swedish Multilingual Corpus 1 (ECI) Dagens Nyheter ~339,000 Jan Strunk / Tibor Kiss
64
+ (and some other texts)
65
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
66
+ turkish.pickle Turkish METU Turkish Corpus Milliyet ~333,000 Jan Strunk / Tibor Kiss
67
+ (Türkçe Derlem Projesi)
68
+ University of Ankara
69
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
70
+
71
+ The corpora contained about 400,000 tokens on average and mostly consisted of newspaper text converted to
72
+ Unicode using the codecs module.
73
+
74
+ Kiss, Tibor and Strunk, Jan (2006): Unsupervised Multilingual Sentence Boundary Detection.
75
+ Computational Linguistics 32: 485-525.
76
+
77
+ ---- Training Code ----
78
+
79
+ # import punkt
80
+ import nltk.tokenize.punkt
81
+
82
+ # Make a new Tokenizer
83
+ tokenizer = nltk.tokenize.punkt.PunktSentenceTokenizer()
84
+
85
+ # Read in training corpus (one example: Slovene)
86
+ import codecs
87
+ text = codecs.open("slovene.plain","Ur","iso-8859-2").read()
88
+
89
+ # Train tokenizer
90
+ tokenizer.train(text)
91
+
92
+ # Dump pickled tokenizer
93
+ import pickle
94
+ out = open("slovene.pickle","wb")
95
+ pickle.dump(tokenizer, out)
96
+ out.close()
97
+
98
+ ---------
nltk_data/tokenizers/punkt/PY3/czech.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:64b0734b6fbe8e8d7cac79f48d1dd9f853824e57c4e3594dadd74ba2c1d97f50
3
+ size 1119050
nltk_data/tokenizers/punkt/PY3/danish.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6189c7dd254e29e2bd406a7f6a4336297c8953214792466a790ea4444223ceb3
3
+ size 1191710
nltk_data/tokenizers/punkt/PY3/dutch.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fda0d6a13f02e8898daec7fe923da88e25abe081bcfa755c0e015075c215fe4c
3
+ size 693759
nltk_data/tokenizers/punkt/PY3/english.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5cad3758596392364e3be9803dbd7ebeda384b68937b488a01365f5551bb942c
3
+ size 406697
nltk_data/tokenizers/punkt/PY3/estonian.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b364f72538d17b146a98009ad239a8096ce6c0a8b02958c0bc776ecd0c58a25f
3
+ size 1499502
nltk_data/tokenizers/punkt/PY3/finnish.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6a4b5ff5500ee851c456f9dd40d5fc0d8c1859c88eb3178de1317d26b7d22833
3
+ size 1852226
nltk_data/tokenizers/punkt/PY3/french.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:28e3a4cd2971989b3cb9fd3433a6f15d17981e464db2be039364313b5de94f29
3
+ size 553575
nltk_data/tokenizers/punkt/PY3/german.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ddcbbe85e2042a019b1a6e37fd8c153286c38ba201fae0f5bfd9a3f74abae25c
3
+ size 1463575
nltk_data/tokenizers/punkt/PY3/greek.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:85dabc44ab90a5f208ef37ff6b4892ebe7e740f71fb4da47cfd95417ca3e22fd
3
+ size 876006
nltk_data/tokenizers/punkt/PY3/italian.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:68a94007b1e4ffdc4d1a190185ca5442c3dafeb17ab39d30329e84cd74a43947
3
+ size 615089
nltk_data/tokenizers/punkt/PY3/malayalam.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1f8cf58acbdb7f472ac40affc13663be42dafb47c15030c11ade0444c9e0e53d
3
+ size 221207
nltk_data/tokenizers/punkt/PY3/norwegian.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4ff7a46d1438b311457d15d7763060b8d3270852c1850fd788c5cee194dc4a1d
3
+ size 1181271
nltk_data/tokenizers/punkt/PY3/polish.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:624900ae3ddfb4854a98c5d3b8b1c9bb719975f33fee61ce1441dab9f8a00718
3
+ size 1738386
nltk_data/tokenizers/punkt/PY3/portuguese.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:02a0b7b25c3c7471e1791b66a31bbb530afbb0160aee4fcecf0107652067b4a1
3
+ size 611919
nltk_data/tokenizers/punkt/PY3/russian.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:549762f8190024d89b511472df21a3a135eee5d9233e63ac244db737c2c61d7e
3
+ size 33020
nltk_data/tokenizers/punkt/PY3/slovene.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:52ef2cc0ed27d79b3aa635cbbc40ad811883a75a4b8a8be1ae406972870fd864
3
+ size 734444
nltk_data/tokenizers/punkt/PY3/spanish.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:164a50fadc5a49f8ec7426eae11d3111ee752b48a3ef373d47745011192a5984
3
+ size 562337
nltk_data/tokenizers/punkt/PY3/swedish.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b0f7d538bfd5266633b09e842cd92e9e0ac10f1d923bf211e1497972ddc47318
3
+ size 979681
nltk_data/tokenizers/punkt/PY3/turkish.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ae68ef5863728ac5332e87eb1f6bae772ff32a13a4caa2b01a5c68103e853c5b
3
+ size 1017038
nltk_data/tokenizers/punkt/README ADDED
@@ -0,0 +1,98 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Pretrained Punkt Models -- Jan Strunk (New version trained after issues 313 and 514 had been corrected)
2
+
3
+ Most models were prepared using the test corpora from Kiss and Strunk (2006). Additional models have
4
+ been contributed by various people using NLTK for sentence boundary detection.
5
+
6
+ For information about how to use these models, please confer the tokenization HOWTO:
7
+ http://nltk.googlecode.com/svn/trunk/doc/howto/tokenize.html
8
+ and chapter 3.8 of the NLTK book:
9
+ http://nltk.googlecode.com/svn/trunk/doc/book/ch03.html#sec-segmentation
10
+
11
+ There are pretrained tokenizers for the following languages:
12
+
13
+ File Language Source Contents Size of training corpus(in tokens) Model contributed by
14
+ =======================================================================================================================================================================
15
+ czech.pickle Czech Multilingual Corpus 1 (ECI) Lidove Noviny ~345,000 Jan Strunk / Tibor Kiss
16
+ Literarni Noviny
17
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
18
+ danish.pickle Danish Avisdata CD-Rom Ver. 1.1. 1995 Berlingske Tidende ~550,000 Jan Strunk / Tibor Kiss
19
+ (Berlingske Avisdata, Copenhagen) Weekend Avisen
20
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
21
+ dutch.pickle Dutch Multilingual Corpus 1 (ECI) De Limburger ~340,000 Jan Strunk / Tibor Kiss
22
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
23
+ english.pickle English Penn Treebank (LDC) Wall Street Journal ~469,000 Jan Strunk / Tibor Kiss
24
+ (American)
25
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
26
+ estonian.pickle Estonian University of Tartu, Estonia Eesti Ekspress ~359,000 Jan Strunk / Tibor Kiss
27
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
28
+ finnish.pickle Finnish Finnish Parole Corpus, Finnish Books and major national ~364,000 Jan Strunk / Tibor Kiss
29
+ Text Bank (Suomen Kielen newspapers
30
+ Tekstipankki)
31
+ Finnish Center for IT Science
32
+ (CSC)
33
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
34
+ french.pickle French Multilingual Corpus 1 (ECI) Le Monde ~370,000 Jan Strunk / Tibor Kiss
35
+ (European)
36
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
37
+ german.pickle German Neue Zürcher Zeitung AG Neue Zürcher Zeitung ~847,000 Jan Strunk / Tibor Kiss
38
+ (Switzerland) CD-ROM
39
+ (Uses "ss"
40
+ instead of "ß")
41
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
42
+ greek.pickle Greek Efstathios Stamatatos To Vima (TO BHMA) ~227,000 Jan Strunk / Tibor Kiss
43
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
44
+ italian.pickle Italian Multilingual Corpus 1 (ECI) La Stampa, Il Mattino ~312,000 Jan Strunk / Tibor Kiss
45
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
46
+ norwegian.pickle Norwegian Centre for Humanities Bergens Tidende ~479,000 Jan Strunk / Tibor Kiss
47
+ (Bokmål and Information Technologies,
48
+ Nynorsk) Bergen
49
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
50
+ polish.pickle Polish Polish National Corpus Literature, newspapers, etc. ~1,000,000 Krzysztof Langner
51
+ (http://www.nkjp.pl/)
52
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
53
+ portuguese.pickle Portuguese CETENFolha Corpus Folha de São Paulo ~321,000 Jan Strunk / Tibor Kiss
54
+ (Brazilian) (Linguateca)
55
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
56
+ slovene.pickle Slovene TRACTOR Delo ~354,000 Jan Strunk / Tibor Kiss
57
+ Slovene Academy for Arts
58
+ and Sciences
59
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
60
+ spanish.pickle Spanish Multilingual Corpus 1 (ECI) Sur ~353,000 Jan Strunk / Tibor Kiss
61
+ (European)
62
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
63
+ swedish.pickle Swedish Multilingual Corpus 1 (ECI) Dagens Nyheter ~339,000 Jan Strunk / Tibor Kiss
64
+ (and some other texts)
65
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
66
+ turkish.pickle Turkish METU Turkish Corpus Milliyet ~333,000 Jan Strunk / Tibor Kiss
67
+ (Türkçe Derlem Projesi)
68
+ University of Ankara
69
+ -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
70
+
71
+ The corpora contained about 400,000 tokens on average and mostly consisted of newspaper text converted to
72
+ Unicode using the codecs module.
73
+
74
+ Kiss, Tibor and Strunk, Jan (2006): Unsupervised Multilingual Sentence Boundary Detection.
75
+ Computational Linguistics 32: 485-525.
76
+
77
+ ---- Training Code ----
78
+
79
+ # import punkt
80
+ import nltk.tokenize.punkt
81
+
82
+ # Make a new Tokenizer
83
+ tokenizer = nltk.tokenize.punkt.PunktSentenceTokenizer()
84
+
85
+ # Read in training corpus (one example: Slovene)
86
+ import codecs
87
+ text = codecs.open("slovene.plain","Ur","iso-8859-2").read()
88
+
89
+ # Train tokenizer
90
+ tokenizer.train(text)
91
+
92
+ # Dump pickled tokenizer
93
+ import pickle
94
+ out = open("slovene.pickle","wb")
95
+ pickle.dump(tokenizer, out)
96
+ out.close()
97
+
98
+ ---------
nltk_data/tokenizers/punkt/czech.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1c085f6283bed0f1390d36a55d126ccc29c9b4dfcd2705e862b1711b7c6bb5ab
3
+ size 1424691
nltk_data/tokenizers/punkt/danish.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:df8366ad67db22b1f838cd63fcc589a6006faf66d7a46be5312d9c487ce2c811
3
+ size 1427491
nltk_data/tokenizers/punkt/dutch.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:12f46024d3c840529b56ac2a3118b80b8dc77705734bcdd71ff7c46f5808395e
3
+ size 839761
nltk_data/tokenizers/punkt/english.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5e2d25d5adc3ee51ac192ce611bdc5378acae7136af5d3c52c2903c669f9aff0
3
+ size 495006
nltk_data/tokenizers/punkt/estonian.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9083ef6ef3d5b9992a8a4ea09e889a87be75e2122ad25648307178960634cd8d
3
+ size 1803082
nltk_data/tokenizers/punkt/finnish.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ce1b4dbe72e400e902220061457f9bd5f491ec37f7af468bc4694980c9623817
3
+ size 2192034
nltk_data/tokenizers/punkt/french.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7e0be48e38a54232ea88c817cf34c1f1f8f44954e21f118c65af9f2d6a43cdbd
3
+ size 664010
nltk_data/tokenizers/punkt/german.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:251c2f4bde61ab3fc1cabc2158c62e6ab285fddd16267d2d3885f71e3ed61c7f
3
+ size 1708012
nltk_data/tokenizers/punkt/greek.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8b3a6da093ed2df084ded6dc49c88f101d47a0c69398f19ae50af6785d93b1c5
3
+ size 2042362
nltk_data/tokenizers/punkt/italian.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:41e6aaf554e696703b3d41890973368b9b2f17c342745c07369742928d363731
3
+ size 748532
nltk_data/tokenizers/punkt/malayalam.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1f8cf58acbdb7f472ac40affc13663be42dafb47c15030c11ade0444c9e0e53d
3
+ size 221207
nltk_data/tokenizers/punkt/norwegian.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:45828b0d57da9a66f107ea277752f6c1cbde51b9f9feba173b2c6e2edb28af21
3
+ size 1422756
nltk_data/tokenizers/punkt/polish.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:79d09a9406f90dbf20f8cbb0a04a7aa0bdb4b71604eda31e97c3df2de5cd2837
3
+ size 2287622
nltk_data/tokenizers/punkt/portuguese.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c09561e770b6f17e3d85112f83007ff1397dec66c23acb15b9fe046eaefd2e86
3
+ size 739845
nltk_data/tokenizers/punkt/russian.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bc984432fbe31f7000014f8047502476889169c60f09be5413ca09276b16c909
3
+ size 33027
nltk_data/tokenizers/punkt/slovene.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2dc83b900e347c16ed0123868369107cd19d1a6125d099e26889580c4dbba277
3
+ size 939791
nltk_data/tokenizers/punkt/spanish.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:61afae663cb2968148e0e27d5a3fcd4a5f19648688800caf8e7f998eaa75f4a7
3
+ size 680466
nltk_data/tokenizers/punkt/swedish.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e5794208b223b2a54bd4ed565045172f9c6ef80b5bead94f71a5499455cda955
3
+ size 1168214
nltk_data/tokenizers/punkt/turkish.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d2abb5d7ec4e80aeeb994407254a2e1a0928520727cc25f7bd3fc9ce0b5a78c1
3
+ size 1363199
requirements.txt ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ langchain
2
+ accelerate
3
+ duckduckgo_search
4
+ transformers==4.27.1
5
+ unstructured[local-inference]
6
+ layoutparser[layoutmodels,tesseract]
7
+ nltk
8
+ sentence-transformers
9
+ beautifulsoup4
10
+ icetk
11
+ cpm_kernels
12
+ faiss-cpu
13
+ gradio
14
+ nltk
15
+ accelerate
16
+ detectron2@git+https://github.com/facebookresearch/[email protected]#egg=detectron2