Update tokenization_chatglm.py
#72
by
JeremyHibiki
- opened
- tokenization_chatglm.py +1 -2
tokenization_chatglm.py
CHANGED
|
@@ -3,10 +3,9 @@ import base64
|
|
| 3 |
import os
|
| 4 |
import json
|
| 5 |
import tiktoken
|
| 6 |
-
from torch import TensorType
|
| 7 |
from typing import List, Optional, Union, Dict, Any
|
| 8 |
from transformers import PreTrainedTokenizer
|
| 9 |
-
from transformers.utils import logging, PaddingStrategy
|
| 10 |
from transformers.tokenization_utils_base import EncodedInput, BatchEncoding
|
| 11 |
|
| 12 |
|
|
|
|
| 3 |
import os
|
| 4 |
import json
|
| 5 |
import tiktoken
|
|
|
|
| 6 |
from typing import List, Optional, Union, Dict, Any
|
| 7 |
from transformers import PreTrainedTokenizer
|
| 8 |
+
from transformers.utils import logging, PaddingStrategy, TensorType
|
| 9 |
from transformers.tokenization_utils_base import EncodedInput, BatchEncoding
|
| 10 |
|
| 11 |
|