Commit
•
fed08d6
1
Parent(s):
69a4f29
Upload tokenizer
Browse files- added_tokens.json +3 -0
- merges.txt +202 -0
- special_tokens_map.json +24 -0
- tokenizer.json +725 -0
- tokenizer_config.json +33 -0
- vocab.json +1 -0
added_tokens.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"<|endoftext|>": 462
|
| 3 |
+
}
|
merges.txt
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version: 0.2
|
| 2 |
+
Ġ +
|
| 3 |
+
Ġ s
|
| 4 |
+
) .
|
| 5 |
+
( -
|
| 6 |
+
Ġ r
|
| 7 |
+
Ġ Ċ
|
| 8 |
+
_ _
|
| 9 |
+
a n
|
| 10 |
+
i an
|
| 11 |
+
p ian
|
| 12 |
+
pian o
|
| 13 |
+
m p
|
| 14 |
+
p p
|
| 15 |
+
Ġ h
|
| 16 |
+
Ġ l
|
| 17 |
+
) (
|
| 18 |
+
Ġ %
|
| 19 |
+
) +
|
| 20 |
+
I I
|
| 21 |
+
e d
|
| 22 |
+
' ]
|
| 23 |
+
[ '
|
| 24 |
+
q d
|
| 25 |
+
Ġ V
|
| 26 |
+
s d
|
| 27 |
+
a c
|
| 28 |
+
a u
|
| 29 |
+
e n
|
| 30 |
+
f r
|
| 31 |
+
g m
|
| 32 |
+
au gm
|
| 33 |
+
en t
|
| 34 |
+
fr ac
|
| 35 |
+
augm ent
|
| 36 |
+
Ġ II
|
| 37 |
+
Ġ I
|
| 38 |
+
b r
|
| 39 |
+
g h
|
| 40 |
+
i gh
|
| 41 |
+
br igh
|
| 42 |
+
brigh t
|
| 43 |
+
1 1
|
| 44 |
+
1 0
|
| 45 |
+
) ,
|
| 46 |
+
Ġ 2
|
| 47 |
+
ĠV I
|
| 48 |
+
ĠV II
|
| 49 |
+
) )
|
| 50 |
+
ĠII I
|
| 51 |
+
h d
|
| 52 |
+
) ).
|
| 53 |
+
ĠI V
|
| 54 |
+
Ġ2 4
|
| 55 |
+
V II
|
| 56 |
+
6 4
|
| 57 |
+
Ġ2 1
|
| 58 |
+
I V
|
| 59 |
+
pp p
|
| 60 |
+
6 5
|
| 61 |
+
) )+
|
| 62 |
+
w d
|
| 63 |
+
4 3
|
| 64 |
+
f f
|
| 65 |
+
V I
|
| 66 |
+
Ġ 4
|
| 67 |
+
5 6
|
| 68 |
+
Ġ 56
|
| 69 |
+
Ġ4 0
|
| 70 |
+
1 2
|
| 71 |
+
II I
|
| 72 |
+
Ġ 3
|
| 73 |
+
1 3
|
| 74 |
+
Ġ3 5
|
| 75 |
+
1 5
|
| 76 |
+
Ġ 15
|
| 77 |
+
1 4
|
| 78 |
+
Ġ4 2
|
| 79 |
+
) ),
|
| 80 |
+
Ġ3 0
|
| 81 |
+
) ']
|
| 82 |
+
1 7
|
| 83 |
+
( +
|
| 84 |
+
(+ )']
|
| 85 |
+
{ -
|
| 86 |
+
} ']
|
| 87 |
+
ff f
|
| 88 |
+
1 9
|
| 89 |
+
m m
|
| 90 |
+
1 6
|
| 91 |
+
2 3
|
| 92 |
+
)) )+
|
| 93 |
+
[' (+)']
|
| 94 |
+
Ġ 8
|
| 95 |
+
2 5
|
| 96 |
+
Ġ 16
|
| 97 |
+
Ġ8 4
|
| 98 |
+
2 0
|
| 99 |
+
3 1
|
| 100 |
+
2 1
|
| 101 |
+
2 9
|
| 102 |
+
1 8
|
| 103 |
+
3 7
|
| 104 |
+
Ġ16 8
|
| 105 |
+
6 0
|
| 106 |
+
Ġ 60
|
| 107 |
+
t d
|
| 108 |
+
5 3
|
| 109 |
+
5 9
|
| 110 |
+
3 5
|
| 111 |
+
4 5
|
| 112 |
+
Ġ 12
|
| 113 |
+
Ġ12 0
|
| 114 |
+
4 7
|
| 115 |
+
7 3
|
| 116 |
+
7 0
|
| 117 |
+
4 1
|
| 118 |
+
Ġ 70
|
| 119 |
+
5 5
|
| 120 |
+
3 9
|
| 121 |
+
Ġ8 0
|
| 122 |
+
3 3
|
| 123 |
+
6 1
|
| 124 |
+
8 3
|
| 125 |
+
8 7
|
| 126 |
+
2 6
|
| 127 |
+
8 1
|
| 128 |
+
6 7
|
| 129 |
+
8 9
|
| 130 |
+
7 7
|
| 131 |
+
Ġ4 8
|
| 132 |
+
5 7
|
| 133 |
+
Ġ 11
|
| 134 |
+
Ġ11 2
|
| 135 |
+
2 7
|
| 136 |
+
10 1
|
| 137 |
+
6 2
|
| 138 |
+
7 9
|
| 139 |
+
10 9
|
| 140 |
+
2 2
|
| 141 |
+
Ġ 14
|
| 142 |
+
Ġ14 0
|
| 143 |
+
4 9
|
| 144 |
+
7 1
|
| 145 |
+
8 0
|
| 146 |
+
9 5
|
| 147 |
+
11 3
|
| 148 |
+
10 3
|
| 149 |
+
11 5
|
| 150 |
+
Ġ2 80
|
| 151 |
+
15 1
|
| 152 |
+
1 43
|
| 153 |
+
3 2
|
| 154 |
+
6 9
|
| 155 |
+
8 5
|
| 156 |
+
Ġ24 0
|
| 157 |
+
2 41
|
| 158 |
+
4 0
|
| 159 |
+
9 1
|
| 160 |
+
9 7
|
| 161 |
+
12 5
|
| 162 |
+
13 7
|
| 163 |
+
2 23
|
| 164 |
+
2 29
|
| 165 |
+
3 4
|
| 166 |
+
4 8
|
| 167 |
+
6 3
|
| 168 |
+
9 2
|
| 169 |
+
9 3
|
| 170 |
+
Ġ 10
|
| 171 |
+
11 9
|
| 172 |
+
12 1
|
| 173 |
+
12 3
|
| 174 |
+
12 7
|
| 175 |
+
13 1
|
| 176 |
+
17 3
|
| 177 |
+
17 9
|
| 178 |
+
Ġ84 0
|
| 179 |
+
29 3
|
| 180 |
+
Ġ10 5
|
| 181 |
+
2 21
|
| 182 |
+
2 89
|
| 183 |
+
3 6
|
| 184 |
+
3 65
|
| 185 |
+
4 4
|
| 186 |
+
4 6
|
| 187 |
+
11 7
|
| 188 |
+
Ġ2 10
|
| 189 |
+
)) )
|
| 190 |
+
12 9
|
| 191 |
+
Ġ3 36
|
| 192 |
+
13 3
|
| 193 |
+
13 9
|
| 194 |
+
15 7
|
| 195 |
+
19 7
|
| 196 |
+
19 9
|
| 197 |
+
16 7
|
| 198 |
+
16 9
|
| 199 |
+
20 7
|
| 200 |
+
18 1
|
| 201 |
+
18 5
|
| 202 |
+
18 7
|
special_tokens_map.json
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token": {
|
| 3 |
+
"content": "<|endoftext|>",
|
| 4 |
+
"lstrip": false,
|
| 5 |
+
"normalized": true,
|
| 6 |
+
"rstrip": false,
|
| 7 |
+
"single_word": false
|
| 8 |
+
},
|
| 9 |
+
"eos_token": {
|
| 10 |
+
"content": "<|endoftext|>",
|
| 11 |
+
"lstrip": false,
|
| 12 |
+
"normalized": true,
|
| 13 |
+
"rstrip": false,
|
| 14 |
+
"single_word": false
|
| 15 |
+
},
|
| 16 |
+
"pad_token": "<pad>",
|
| 17 |
+
"unk_token": {
|
| 18 |
+
"content": "<|endoftext|>",
|
| 19 |
+
"lstrip": false,
|
| 20 |
+
"normalized": true,
|
| 21 |
+
"rstrip": false,
|
| 22 |
+
"single_word": false
|
| 23 |
+
}
|
| 24 |
+
}
|
tokenizer.json
ADDED
|
@@ -0,0 +1,725 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"version": "1.0",
|
| 3 |
+
"truncation": {
|
| 4 |
+
"direction": "Right",
|
| 5 |
+
"max_length": 128,
|
| 6 |
+
"strategy": "LongestFirst",
|
| 7 |
+
"stride": 0
|
| 8 |
+
},
|
| 9 |
+
"padding": null,
|
| 10 |
+
"added_tokens": [
|
| 11 |
+
{
|
| 12 |
+
"id": 1,
|
| 13 |
+
"content": "<pad>",
|
| 14 |
+
"single_word": false,
|
| 15 |
+
"lstrip": false,
|
| 16 |
+
"rstrip": false,
|
| 17 |
+
"normalized": false,
|
| 18 |
+
"special": true
|
| 19 |
+
},
|
| 20 |
+
{
|
| 21 |
+
"id": 462,
|
| 22 |
+
"content": "<|endoftext|>",
|
| 23 |
+
"single_word": false,
|
| 24 |
+
"lstrip": false,
|
| 25 |
+
"rstrip": false,
|
| 26 |
+
"normalized": true,
|
| 27 |
+
"special": true
|
| 28 |
+
}
|
| 29 |
+
],
|
| 30 |
+
"normalizer": null,
|
| 31 |
+
"pre_tokenizer": {
|
| 32 |
+
"type": "ByteLevel",
|
| 33 |
+
"add_prefix_space": false,
|
| 34 |
+
"trim_offsets": true,
|
| 35 |
+
"use_regex": true
|
| 36 |
+
},
|
| 37 |
+
"post_processor": {
|
| 38 |
+
"type": "ByteLevel",
|
| 39 |
+
"add_prefix_space": true,
|
| 40 |
+
"trim_offsets": false,
|
| 41 |
+
"use_regex": true
|
| 42 |
+
},
|
| 43 |
+
"decoder": {
|
| 44 |
+
"type": "ByteLevel",
|
| 45 |
+
"add_prefix_space": true,
|
| 46 |
+
"trim_offsets": true,
|
| 47 |
+
"use_regex": true
|
| 48 |
+
},
|
| 49 |
+
"model": {
|
| 50 |
+
"type": "BPE",
|
| 51 |
+
"dropout": null,
|
| 52 |
+
"unk_token": null,
|
| 53 |
+
"continuing_subword_prefix": "",
|
| 54 |
+
"end_of_word_suffix": "",
|
| 55 |
+
"fuse_unk": false,
|
| 56 |
+
"byte_fallback": false,
|
| 57 |
+
"vocab": {
|
| 58 |
+
"<s>": 0,
|
| 59 |
+
"<pad>": 1,
|
| 60 |
+
"</s>": 2,
|
| 61 |
+
"<unk>": 3,
|
| 62 |
+
"<mask>": 4,
|
| 63 |
+
"!": 5,
|
| 64 |
+
"\"": 6,
|
| 65 |
+
"#": 7,
|
| 66 |
+
"$": 8,
|
| 67 |
+
"%": 9,
|
| 68 |
+
"&": 10,
|
| 69 |
+
"'": 11,
|
| 70 |
+
"(": 12,
|
| 71 |
+
")": 13,
|
| 72 |
+
"*": 14,
|
| 73 |
+
"+": 15,
|
| 74 |
+
",": 16,
|
| 75 |
+
"-": 17,
|
| 76 |
+
".": 18,
|
| 77 |
+
"/": 19,
|
| 78 |
+
"0": 20,
|
| 79 |
+
"1": 21,
|
| 80 |
+
"2": 22,
|
| 81 |
+
"3": 23,
|
| 82 |
+
"4": 24,
|
| 83 |
+
"5": 25,
|
| 84 |
+
"6": 26,
|
| 85 |
+
"7": 27,
|
| 86 |
+
"8": 28,
|
| 87 |
+
"9": 29,
|
| 88 |
+
":": 30,
|
| 89 |
+
";": 31,
|
| 90 |
+
"<": 32,
|
| 91 |
+
"=": 33,
|
| 92 |
+
">": 34,
|
| 93 |
+
"?": 35,
|
| 94 |
+
"@": 36,
|
| 95 |
+
"A": 37,
|
| 96 |
+
"B": 38,
|
| 97 |
+
"C": 39,
|
| 98 |
+
"D": 40,
|
| 99 |
+
"E": 41,
|
| 100 |
+
"F": 42,
|
| 101 |
+
"G": 43,
|
| 102 |
+
"H": 44,
|
| 103 |
+
"I": 45,
|
| 104 |
+
"J": 46,
|
| 105 |
+
"K": 47,
|
| 106 |
+
"L": 48,
|
| 107 |
+
"M": 49,
|
| 108 |
+
"N": 50,
|
| 109 |
+
"O": 51,
|
| 110 |
+
"P": 52,
|
| 111 |
+
"Q": 53,
|
| 112 |
+
"R": 54,
|
| 113 |
+
"S": 55,
|
| 114 |
+
"T": 56,
|
| 115 |
+
"U": 57,
|
| 116 |
+
"V": 58,
|
| 117 |
+
"W": 59,
|
| 118 |
+
"X": 60,
|
| 119 |
+
"Y": 61,
|
| 120 |
+
"Z": 62,
|
| 121 |
+
"[": 63,
|
| 122 |
+
"\\": 64,
|
| 123 |
+
"]": 65,
|
| 124 |
+
"^": 66,
|
| 125 |
+
"_": 67,
|
| 126 |
+
"`": 68,
|
| 127 |
+
"a": 69,
|
| 128 |
+
"b": 70,
|
| 129 |
+
"c": 71,
|
| 130 |
+
"d": 72,
|
| 131 |
+
"e": 73,
|
| 132 |
+
"f": 74,
|
| 133 |
+
"g": 75,
|
| 134 |
+
"h": 76,
|
| 135 |
+
"i": 77,
|
| 136 |
+
"j": 78,
|
| 137 |
+
"k": 79,
|
| 138 |
+
"l": 80,
|
| 139 |
+
"m": 81,
|
| 140 |
+
"n": 82,
|
| 141 |
+
"o": 83,
|
| 142 |
+
"p": 84,
|
| 143 |
+
"q": 85,
|
| 144 |
+
"r": 86,
|
| 145 |
+
"s": 87,
|
| 146 |
+
"t": 88,
|
| 147 |
+
"u": 89,
|
| 148 |
+
"v": 90,
|
| 149 |
+
"w": 91,
|
| 150 |
+
"x": 92,
|
| 151 |
+
"y": 93,
|
| 152 |
+
"z": 94,
|
| 153 |
+
"{": 95,
|
| 154 |
+
"|": 96,
|
| 155 |
+
"}": 97,
|
| 156 |
+
"~": 98,
|
| 157 |
+
"¡": 99,
|
| 158 |
+
"¢": 100,
|
| 159 |
+
"£": 101,
|
| 160 |
+
"¤": 102,
|
| 161 |
+
"¥": 103,
|
| 162 |
+
"¦": 104,
|
| 163 |
+
"§": 105,
|
| 164 |
+
"¨": 106,
|
| 165 |
+
"©": 107,
|
| 166 |
+
"ª": 108,
|
| 167 |
+
"«": 109,
|
| 168 |
+
"¬": 110,
|
| 169 |
+
"®": 111,
|
| 170 |
+
"¯": 112,
|
| 171 |
+
"°": 113,
|
| 172 |
+
"±": 114,
|
| 173 |
+
"²": 115,
|
| 174 |
+
"³": 116,
|
| 175 |
+
"´": 117,
|
| 176 |
+
"µ": 118,
|
| 177 |
+
"¶": 119,
|
| 178 |
+
"·": 120,
|
| 179 |
+
"¸": 121,
|
| 180 |
+
"¹": 122,
|
| 181 |
+
"º": 123,
|
| 182 |
+
"»": 124,
|
| 183 |
+
"¼": 125,
|
| 184 |
+
"½": 126,
|
| 185 |
+
"¾": 127,
|
| 186 |
+
"¿": 128,
|
| 187 |
+
"À": 129,
|
| 188 |
+
"Á": 130,
|
| 189 |
+
"Â": 131,
|
| 190 |
+
"Ã": 132,
|
| 191 |
+
"Ä": 133,
|
| 192 |
+
"Å": 134,
|
| 193 |
+
"Æ": 135,
|
| 194 |
+
"Ç": 136,
|
| 195 |
+
"È": 137,
|
| 196 |
+
"É": 138,
|
| 197 |
+
"Ê": 139,
|
| 198 |
+
"Ë": 140,
|
| 199 |
+
"Ì": 141,
|
| 200 |
+
"Í": 142,
|
| 201 |
+
"Î": 143,
|
| 202 |
+
"Ï": 144,
|
| 203 |
+
"Ð": 145,
|
| 204 |
+
"Ñ": 146,
|
| 205 |
+
"Ò": 147,
|
| 206 |
+
"Ó": 148,
|
| 207 |
+
"Ô": 149,
|
| 208 |
+
"Õ": 150,
|
| 209 |
+
"Ö": 151,
|
| 210 |
+
"×": 152,
|
| 211 |
+
"Ø": 153,
|
| 212 |
+
"Ù": 154,
|
| 213 |
+
"Ú": 155,
|
| 214 |
+
"Û": 156,
|
| 215 |
+
"Ü": 157,
|
| 216 |
+
"Ý": 158,
|
| 217 |
+
"Þ": 159,
|
| 218 |
+
"ß": 160,
|
| 219 |
+
"à": 161,
|
| 220 |
+
"á": 162,
|
| 221 |
+
"â": 163,
|
| 222 |
+
"ã": 164,
|
| 223 |
+
"ä": 165,
|
| 224 |
+
"å": 166,
|
| 225 |
+
"æ": 167,
|
| 226 |
+
"ç": 168,
|
| 227 |
+
"è": 169,
|
| 228 |
+
"é": 170,
|
| 229 |
+
"ê": 171,
|
| 230 |
+
"ë": 172,
|
| 231 |
+
"ì": 173,
|
| 232 |
+
"í": 174,
|
| 233 |
+
"î": 175,
|
| 234 |
+
"ï": 176,
|
| 235 |
+
"ð": 177,
|
| 236 |
+
"ñ": 178,
|
| 237 |
+
"ò": 179,
|
| 238 |
+
"ó": 180,
|
| 239 |
+
"ô": 181,
|
| 240 |
+
"õ": 182,
|
| 241 |
+
"ö": 183,
|
| 242 |
+
"÷": 184,
|
| 243 |
+
"ø": 185,
|
| 244 |
+
"ù": 186,
|
| 245 |
+
"ú": 187,
|
| 246 |
+
"û": 188,
|
| 247 |
+
"ü": 189,
|
| 248 |
+
"ý": 190,
|
| 249 |
+
"þ": 191,
|
| 250 |
+
"ÿ": 192,
|
| 251 |
+
"Ā": 193,
|
| 252 |
+
"ā": 194,
|
| 253 |
+
"Ă": 195,
|
| 254 |
+
"ă": 196,
|
| 255 |
+
"Ą": 197,
|
| 256 |
+
"ą": 198,
|
| 257 |
+
"Ć": 199,
|
| 258 |
+
"ć": 200,
|
| 259 |
+
"Ĉ": 201,
|
| 260 |
+
"ĉ": 202,
|
| 261 |
+
"Ċ": 203,
|
| 262 |
+
"ċ": 204,
|
| 263 |
+
"Č": 205,
|
| 264 |
+
"č": 206,
|
| 265 |
+
"Ď": 207,
|
| 266 |
+
"ď": 208,
|
| 267 |
+
"Đ": 209,
|
| 268 |
+
"đ": 210,
|
| 269 |
+
"Ē": 211,
|
| 270 |
+
"ē": 212,
|
| 271 |
+
"Ĕ": 213,
|
| 272 |
+
"ĕ": 214,
|
| 273 |
+
"Ė": 215,
|
| 274 |
+
"ė": 216,
|
| 275 |
+
"Ę": 217,
|
| 276 |
+
"ę": 218,
|
| 277 |
+
"Ě": 219,
|
| 278 |
+
"ě": 220,
|
| 279 |
+
"Ĝ": 221,
|
| 280 |
+
"ĝ": 222,
|
| 281 |
+
"Ğ": 223,
|
| 282 |
+
"ğ": 224,
|
| 283 |
+
"Ġ": 225,
|
| 284 |
+
"ġ": 226,
|
| 285 |
+
"Ģ": 227,
|
| 286 |
+
"ģ": 228,
|
| 287 |
+
"Ĥ": 229,
|
| 288 |
+
"ĥ": 230,
|
| 289 |
+
"Ħ": 231,
|
| 290 |
+
"ħ": 232,
|
| 291 |
+
"Ĩ": 233,
|
| 292 |
+
"ĩ": 234,
|
| 293 |
+
"Ī": 235,
|
| 294 |
+
"ī": 236,
|
| 295 |
+
"Ĭ": 237,
|
| 296 |
+
"ĭ": 238,
|
| 297 |
+
"Į": 239,
|
| 298 |
+
"į": 240,
|
| 299 |
+
"İ": 241,
|
| 300 |
+
"ı": 242,
|
| 301 |
+
"IJ": 243,
|
| 302 |
+
"ij": 244,
|
| 303 |
+
"Ĵ": 245,
|
| 304 |
+
"ĵ": 246,
|
| 305 |
+
"Ķ": 247,
|
| 306 |
+
"ķ": 248,
|
| 307 |
+
"ĸ": 249,
|
| 308 |
+
"Ĺ": 250,
|
| 309 |
+
"ĺ": 251,
|
| 310 |
+
"Ļ": 252,
|
| 311 |
+
"ļ": 253,
|
| 312 |
+
"Ľ": 254,
|
| 313 |
+
"ľ": 255,
|
| 314 |
+
"Ŀ": 256,
|
| 315 |
+
"ŀ": 257,
|
| 316 |
+
"Ł": 258,
|
| 317 |
+
"ł": 259,
|
| 318 |
+
"Ń": 260,
|
| 319 |
+
"Ġ+": 261,
|
| 320 |
+
"Ġs": 262,
|
| 321 |
+
").": 263,
|
| 322 |
+
"(-": 264,
|
| 323 |
+
"Ġr": 265,
|
| 324 |
+
"ĠĊ": 266,
|
| 325 |
+
"__": 267,
|
| 326 |
+
"an": 268,
|
| 327 |
+
"ian": 269,
|
| 328 |
+
"pian": 270,
|
| 329 |
+
"piano": 271,
|
| 330 |
+
"mp": 272,
|
| 331 |
+
"pp": 273,
|
| 332 |
+
"Ġh": 274,
|
| 333 |
+
"Ġl": 275,
|
| 334 |
+
")(": 276,
|
| 335 |
+
"Ġ%": 277,
|
| 336 |
+
")+": 278,
|
| 337 |
+
"II": 279,
|
| 338 |
+
"ed": 280,
|
| 339 |
+
"']": 281,
|
| 340 |
+
"['": 282,
|
| 341 |
+
"qd": 283,
|
| 342 |
+
"ĠV": 284,
|
| 343 |
+
"sd": 285,
|
| 344 |
+
"ac": 286,
|
| 345 |
+
"au": 287,
|
| 346 |
+
"en": 288,
|
| 347 |
+
"fr": 289,
|
| 348 |
+
"gm": 290,
|
| 349 |
+
"augm": 291,
|
| 350 |
+
"ent": 292,
|
| 351 |
+
"frac": 293,
|
| 352 |
+
"augment": 294,
|
| 353 |
+
"ĠII": 295,
|
| 354 |
+
"ĠI": 296,
|
| 355 |
+
"br": 297,
|
| 356 |
+
"gh": 298,
|
| 357 |
+
"igh": 299,
|
| 358 |
+
"brigh": 300,
|
| 359 |
+
"bright": 301,
|
| 360 |
+
"11": 302,
|
| 361 |
+
"10": 303,
|
| 362 |
+
"),": 304,
|
| 363 |
+
"Ġ2": 305,
|
| 364 |
+
"ĠVI": 306,
|
| 365 |
+
"ĠVII": 307,
|
| 366 |
+
"))": 308,
|
| 367 |
+
"ĠIII": 309,
|
| 368 |
+
"hd": 310,
|
| 369 |
+
")).": 311,
|
| 370 |
+
"ĠIV": 312,
|
| 371 |
+
"Ġ24": 313,
|
| 372 |
+
"VII": 314,
|
| 373 |
+
"64": 315,
|
| 374 |
+
"Ġ21": 316,
|
| 375 |
+
"IV": 317,
|
| 376 |
+
"ppp": 318,
|
| 377 |
+
"65": 319,
|
| 378 |
+
"))+": 320,
|
| 379 |
+
"wd": 321,
|
| 380 |
+
"43": 322,
|
| 381 |
+
"ff": 323,
|
| 382 |
+
"VI": 324,
|
| 383 |
+
"Ġ4": 325,
|
| 384 |
+
"56": 326,
|
| 385 |
+
"Ġ56": 327,
|
| 386 |
+
"Ġ40": 328,
|
| 387 |
+
"12": 329,
|
| 388 |
+
"III": 330,
|
| 389 |
+
"Ġ3": 331,
|
| 390 |
+
"13": 332,
|
| 391 |
+
"Ġ35": 333,
|
| 392 |
+
"15": 334,
|
| 393 |
+
"Ġ15": 335,
|
| 394 |
+
"14": 336,
|
| 395 |
+
"Ġ42": 337,
|
| 396 |
+
")),": 338,
|
| 397 |
+
"Ġ30": 339,
|
| 398 |
+
")']": 340,
|
| 399 |
+
"17": 341,
|
| 400 |
+
"(+": 342,
|
| 401 |
+
"(+)']": 343,
|
| 402 |
+
"{-": 344,
|
| 403 |
+
"}']": 345,
|
| 404 |
+
"fff": 346,
|
| 405 |
+
"19": 347,
|
| 406 |
+
"mm": 348,
|
| 407 |
+
"16": 349,
|
| 408 |
+
"23": 350,
|
| 409 |
+
")))+": 351,
|
| 410 |
+
"['(+)']": 352,
|
| 411 |
+
"Ġ8": 353,
|
| 412 |
+
"25": 354,
|
| 413 |
+
"Ġ16": 355,
|
| 414 |
+
"Ġ84": 356,
|
| 415 |
+
"20": 357,
|
| 416 |
+
"31": 358,
|
| 417 |
+
"21": 359,
|
| 418 |
+
"29": 360,
|
| 419 |
+
"18": 361,
|
| 420 |
+
"37": 362,
|
| 421 |
+
"Ġ168": 363,
|
| 422 |
+
"60": 364,
|
| 423 |
+
"Ġ60": 365,
|
| 424 |
+
"td": 366,
|
| 425 |
+
"53": 367,
|
| 426 |
+
"59": 368,
|
| 427 |
+
"35": 369,
|
| 428 |
+
"45": 370,
|
| 429 |
+
"Ġ12": 371,
|
| 430 |
+
"Ġ120": 372,
|
| 431 |
+
"47": 373,
|
| 432 |
+
"73": 374,
|
| 433 |
+
"70": 375,
|
| 434 |
+
"41": 376,
|
| 435 |
+
"Ġ70": 377,
|
| 436 |
+
"55": 378,
|
| 437 |
+
"39": 379,
|
| 438 |
+
"Ġ80": 380,
|
| 439 |
+
"33": 381,
|
| 440 |
+
"61": 382,
|
| 441 |
+
"83": 383,
|
| 442 |
+
"87": 384,
|
| 443 |
+
"26": 385,
|
| 444 |
+
"81": 386,
|
| 445 |
+
"67": 387,
|
| 446 |
+
"89": 388,
|
| 447 |
+
"77": 389,
|
| 448 |
+
"Ġ48": 390,
|
| 449 |
+
"57": 391,
|
| 450 |
+
"Ġ11": 392,
|
| 451 |
+
"Ġ112": 393,
|
| 452 |
+
"27": 394,
|
| 453 |
+
"101": 395,
|
| 454 |
+
"62": 396,
|
| 455 |
+
"79": 397,
|
| 456 |
+
"109": 398,
|
| 457 |
+
"22": 399,
|
| 458 |
+
"Ġ14": 400,
|
| 459 |
+
"Ġ140": 401,
|
| 460 |
+
"49": 402,
|
| 461 |
+
"71": 403,
|
| 462 |
+
"80": 404,
|
| 463 |
+
"95": 405,
|
| 464 |
+
"113": 406,
|
| 465 |
+
"103": 407,
|
| 466 |
+
"115": 408,
|
| 467 |
+
"Ġ280": 409,
|
| 468 |
+
"151": 410,
|
| 469 |
+
"143": 411,
|
| 470 |
+
"32": 412,
|
| 471 |
+
"69": 413,
|
| 472 |
+
"85": 414,
|
| 473 |
+
"Ġ240": 415,
|
| 474 |
+
"241": 416,
|
| 475 |
+
"40": 417,
|
| 476 |
+
"91": 418,
|
| 477 |
+
"97": 419,
|
| 478 |
+
"125": 420,
|
| 479 |
+
"137": 421,
|
| 480 |
+
"223": 422,
|
| 481 |
+
"229": 423,
|
| 482 |
+
"34": 424,
|
| 483 |
+
"48": 425,
|
| 484 |
+
"63": 426,
|
| 485 |
+
"92": 427,
|
| 486 |
+
"93": 428,
|
| 487 |
+
"Ġ10": 429,
|
| 488 |
+
"119": 430,
|
| 489 |
+
"121": 431,
|
| 490 |
+
"123": 432,
|
| 491 |
+
"127": 433,
|
| 492 |
+
"131": 434,
|
| 493 |
+
"173": 435,
|
| 494 |
+
"179": 436,
|
| 495 |
+
"Ġ840": 437,
|
| 496 |
+
"293": 438,
|
| 497 |
+
"Ġ105": 439,
|
| 498 |
+
"221": 440,
|
| 499 |
+
"289": 441,
|
| 500 |
+
"36": 442,
|
| 501 |
+
"365": 443,
|
| 502 |
+
"44": 444,
|
| 503 |
+
"46": 445,
|
| 504 |
+
"117": 446,
|
| 505 |
+
"Ġ210": 447,
|
| 506 |
+
")))": 448,
|
| 507 |
+
"129": 449,
|
| 508 |
+
"Ġ336": 450,
|
| 509 |
+
"133": 451,
|
| 510 |
+
"139": 452,
|
| 511 |
+
"157": 453,
|
| 512 |
+
"197": 454,
|
| 513 |
+
"199": 455,
|
| 514 |
+
"167": 456,
|
| 515 |
+
"169": 457,
|
| 516 |
+
"207": 458,
|
| 517 |
+
"181": 459,
|
| 518 |
+
"185": 460,
|
| 519 |
+
"187": 461
|
| 520 |
+
},
|
| 521 |
+
"merges": [
|
| 522 |
+
"Ġ +",
|
| 523 |
+
"Ġ s",
|
| 524 |
+
") .",
|
| 525 |
+
"( -",
|
| 526 |
+
"Ġ r",
|
| 527 |
+
"Ġ Ċ",
|
| 528 |
+
"_ _",
|
| 529 |
+
"a n",
|
| 530 |
+
"i an",
|
| 531 |
+
"p ian",
|
| 532 |
+
"pian o",
|
| 533 |
+
"m p",
|
| 534 |
+
"p p",
|
| 535 |
+
"Ġ h",
|
| 536 |
+
"Ġ l",
|
| 537 |
+
") (",
|
| 538 |
+
"Ġ %",
|
| 539 |
+
") +",
|
| 540 |
+
"I I",
|
| 541 |
+
"e d",
|
| 542 |
+
"' ]",
|
| 543 |
+
"[ '",
|
| 544 |
+
"q d",
|
| 545 |
+
"Ġ V",
|
| 546 |
+
"s d",
|
| 547 |
+
"a c",
|
| 548 |
+
"a u",
|
| 549 |
+
"e n",
|
| 550 |
+
"f r",
|
| 551 |
+
"g m",
|
| 552 |
+
"au gm",
|
| 553 |
+
"en t",
|
| 554 |
+
"fr ac",
|
| 555 |
+
"augm ent",
|
| 556 |
+
"Ġ II",
|
| 557 |
+
"Ġ I",
|
| 558 |
+
"b r",
|
| 559 |
+
"g h",
|
| 560 |
+
"i gh",
|
| 561 |
+
"br igh",
|
| 562 |
+
"brigh t",
|
| 563 |
+
"1 1",
|
| 564 |
+
"1 0",
|
| 565 |
+
") ,",
|
| 566 |
+
"Ġ 2",
|
| 567 |
+
"ĠV I",
|
| 568 |
+
"ĠV II",
|
| 569 |
+
") )",
|
| 570 |
+
"ĠII I",
|
| 571 |
+
"h d",
|
| 572 |
+
") ).",
|
| 573 |
+
"ĠI V",
|
| 574 |
+
"Ġ2 4",
|
| 575 |
+
"V II",
|
| 576 |
+
"6 4",
|
| 577 |
+
"Ġ2 1",
|
| 578 |
+
"I V",
|
| 579 |
+
"pp p",
|
| 580 |
+
"6 5",
|
| 581 |
+
") )+",
|
| 582 |
+
"w d",
|
| 583 |
+
"4 3",
|
| 584 |
+
"f f",
|
| 585 |
+
"V I",
|
| 586 |
+
"Ġ 4",
|
| 587 |
+
"5 6",
|
| 588 |
+
"Ġ 56",
|
| 589 |
+
"Ġ4 0",
|
| 590 |
+
"1 2",
|
| 591 |
+
"II I",
|
| 592 |
+
"Ġ 3",
|
| 593 |
+
"1 3",
|
| 594 |
+
"Ġ3 5",
|
| 595 |
+
"1 5",
|
| 596 |
+
"Ġ 15",
|
| 597 |
+
"1 4",
|
| 598 |
+
"Ġ4 2",
|
| 599 |
+
") ),",
|
| 600 |
+
"Ġ3 0",
|
| 601 |
+
") ']",
|
| 602 |
+
"1 7",
|
| 603 |
+
"( +",
|
| 604 |
+
"(+ )']",
|
| 605 |
+
"{ -",
|
| 606 |
+
"} ']",
|
| 607 |
+
"ff f",
|
| 608 |
+
"1 9",
|
| 609 |
+
"m m",
|
| 610 |
+
"1 6",
|
| 611 |
+
"2 3",
|
| 612 |
+
")) )+",
|
| 613 |
+
"[' (+)']",
|
| 614 |
+
"Ġ 8",
|
| 615 |
+
"2 5",
|
| 616 |
+
"Ġ 16",
|
| 617 |
+
"Ġ8 4",
|
| 618 |
+
"2 0",
|
| 619 |
+
"3 1",
|
| 620 |
+
"2 1",
|
| 621 |
+
"2 9",
|
| 622 |
+
"1 8",
|
| 623 |
+
"3 7",
|
| 624 |
+
"Ġ16 8",
|
| 625 |
+
"6 0",
|
| 626 |
+
"Ġ 60",
|
| 627 |
+
"t d",
|
| 628 |
+
"5 3",
|
| 629 |
+
"5 9",
|
| 630 |
+
"3 5",
|
| 631 |
+
"4 5",
|
| 632 |
+
"Ġ 12",
|
| 633 |
+
"Ġ12 0",
|
| 634 |
+
"4 7",
|
| 635 |
+
"7 3",
|
| 636 |
+
"7 0",
|
| 637 |
+
"4 1",
|
| 638 |
+
"Ġ 70",
|
| 639 |
+
"5 5",
|
| 640 |
+
"3 9",
|
| 641 |
+
"Ġ8 0",
|
| 642 |
+
"3 3",
|
| 643 |
+
"6 1",
|
| 644 |
+
"8 3",
|
| 645 |
+
"8 7",
|
| 646 |
+
"2 6",
|
| 647 |
+
"8 1",
|
| 648 |
+
"6 7",
|
| 649 |
+
"8 9",
|
| 650 |
+
"7 7",
|
| 651 |
+
"Ġ4 8",
|
| 652 |
+
"5 7",
|
| 653 |
+
"Ġ 11",
|
| 654 |
+
"Ġ11 2",
|
| 655 |
+
"2 7",
|
| 656 |
+
"10 1",
|
| 657 |
+
"6 2",
|
| 658 |
+
"7 9",
|
| 659 |
+
"10 9",
|
| 660 |
+
"2 2",
|
| 661 |
+
"Ġ 14",
|
| 662 |
+
"Ġ14 0",
|
| 663 |
+
"4 9",
|
| 664 |
+
"7 1",
|
| 665 |
+
"8 0",
|
| 666 |
+
"9 5",
|
| 667 |
+
"11 3",
|
| 668 |
+
"10 3",
|
| 669 |
+
"11 5",
|
| 670 |
+
"Ġ2 80",
|
| 671 |
+
"15 1",
|
| 672 |
+
"1 43",
|
| 673 |
+
"3 2",
|
| 674 |
+
"6 9",
|
| 675 |
+
"8 5",
|
| 676 |
+
"Ġ24 0",
|
| 677 |
+
"2 41",
|
| 678 |
+
"4 0",
|
| 679 |
+
"9 1",
|
| 680 |
+
"9 7",
|
| 681 |
+
"12 5",
|
| 682 |
+
"13 7",
|
| 683 |
+
"2 23",
|
| 684 |
+
"2 29",
|
| 685 |
+
"3 4",
|
| 686 |
+
"4 8",
|
| 687 |
+
"6 3",
|
| 688 |
+
"9 2",
|
| 689 |
+
"9 3",
|
| 690 |
+
"Ġ 10",
|
| 691 |
+
"11 9",
|
| 692 |
+
"12 1",
|
| 693 |
+
"12 3",
|
| 694 |
+
"12 7",
|
| 695 |
+
"13 1",
|
| 696 |
+
"17 3",
|
| 697 |
+
"17 9",
|
| 698 |
+
"Ġ84 0",
|
| 699 |
+
"29 3",
|
| 700 |
+
"Ġ10 5",
|
| 701 |
+
"2 21",
|
| 702 |
+
"2 89",
|
| 703 |
+
"3 6",
|
| 704 |
+
"3 65",
|
| 705 |
+
"4 4",
|
| 706 |
+
"4 6",
|
| 707 |
+
"11 7",
|
| 708 |
+
"Ġ2 10",
|
| 709 |
+
")) )",
|
| 710 |
+
"12 9",
|
| 711 |
+
"Ġ3 36",
|
| 712 |
+
"13 3",
|
| 713 |
+
"13 9",
|
| 714 |
+
"15 7",
|
| 715 |
+
"19 7",
|
| 716 |
+
"19 9",
|
| 717 |
+
"16 7",
|
| 718 |
+
"16 9",
|
| 719 |
+
"20 7",
|
| 720 |
+
"18 1",
|
| 721 |
+
"18 5",
|
| 722 |
+
"18 7"
|
| 723 |
+
]
|
| 724 |
+
}
|
| 725 |
+
}
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_bos_token": false,
|
| 3 |
+
"add_prefix_space": false,
|
| 4 |
+
"bos_token": {
|
| 5 |
+
"__type": "AddedToken",
|
| 6 |
+
"content": "<|endoftext|>",
|
| 7 |
+
"lstrip": false,
|
| 8 |
+
"normalized": true,
|
| 9 |
+
"rstrip": false,
|
| 10 |
+
"single_word": false
|
| 11 |
+
},
|
| 12 |
+
"eos_token": {
|
| 13 |
+
"__type": "AddedToken",
|
| 14 |
+
"content": "<|endoftext|>",
|
| 15 |
+
"lstrip": false,
|
| 16 |
+
"normalized": true,
|
| 17 |
+
"rstrip": false,
|
| 18 |
+
"single_word": false
|
| 19 |
+
},
|
| 20 |
+
"errors": "replace",
|
| 21 |
+
"model_max_length": 1000000000000000019884624838656,
|
| 22 |
+
"pad_token": null,
|
| 23 |
+
"special_tokens_map_file": null,
|
| 24 |
+
"tokenizer_class": "GPT2Tokenizer",
|
| 25 |
+
"unk_token": {
|
| 26 |
+
"__type": "AddedToken",
|
| 27 |
+
"content": "<|endoftext|>",
|
| 28 |
+
"lstrip": false,
|
| 29 |
+
"normalized": true,
|
| 30 |
+
"rstrip": false,
|
| 31 |
+
"single_word": false
|
| 32 |
+
}
|
| 33 |
+
}
|
vocab.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"<s>":0,"<pad>":1,"</s>":2,"<unk>":3,"<mask>":4,"!":5,"\"":6,"#":7,"$":8,"%":9,"&":10,"'":11,"(":12,")":13,"*":14,"+":15,",":16,"-":17,".":18,"/":19,"0":20,"1":21,"2":22,"3":23,"4":24,"5":25,"6":26,"7":27,"8":28,"9":29,":":30,";":31,"<":32,"=":33,">":34,"?":35,"@":36,"A":37,"B":38,"C":39,"D":40,"E":41,"F":42,"G":43,"H":44,"I":45,"J":46,"K":47,"L":48,"M":49,"N":50,"O":51,"P":52,"Q":53,"R":54,"S":55,"T":56,"U":57,"V":58,"W":59,"X":60,"Y":61,"Z":62,"[":63,"\\":64,"]":65,"^":66,"_":67,"`":68,"a":69,"b":70,"c":71,"d":72,"e":73,"f":74,"g":75,"h":76,"i":77,"j":78,"k":79,"l":80,"m":81,"n":82,"o":83,"p":84,"q":85,"r":86,"s":87,"t":88,"u":89,"v":90,"w":91,"x":92,"y":93,"z":94,"{":95,"|":96,"}":97,"~":98,"¡":99,"¢":100,"£":101,"¤":102,"¥":103,"¦":104,"§":105,"¨":106,"©":107,"ª":108,"«":109,"¬":110,"®":111,"¯":112,"°":113,"±":114,"²":115,"³":116,"´":117,"µ":118,"¶":119,"·":120,"¸":121,"¹":122,"º":123,"»":124,"¼":125,"½":126,"¾":127,"¿":128,"À":129,"Á":130,"Â":131,"Ã":132,"Ä":133,"Å":134,"Æ":135,"Ç":136,"È":137,"É":138,"Ê":139,"Ë":140,"Ì":141,"Í":142,"Î":143,"Ï":144,"Ð":145,"Ñ":146,"Ò":147,"Ó":148,"Ô":149,"Õ":150,"Ö":151,"×":152,"Ø":153,"Ù":154,"Ú":155,"Û":156,"Ü":157,"Ý":158,"Þ":159,"ß":160,"à":161,"á":162,"â":163,"ã":164,"ä":165,"å":166,"æ":167,"ç":168,"è":169,"é":170,"ê":171,"ë":172,"ì":173,"í":174,"î":175,"ï":176,"ð":177,"ñ":178,"ò":179,"ó":180,"ô":181,"õ":182,"ö":183,"÷":184,"ø":185,"ù":186,"ú":187,"û":188,"ü":189,"ý":190,"þ":191,"ÿ":192,"Ā":193,"ā":194,"Ă":195,"ă":196,"Ą":197,"ą":198,"Ć":199,"ć":200,"Ĉ":201,"ĉ":202,"Ċ":203,"ċ":204,"Č":205,"č":206,"Ď":207,"ď":208,"Đ":209,"đ":210,"Ē":211,"ē":212,"Ĕ":213,"ĕ":214,"Ė":215,"ė":216,"Ę":217,"ę":218,"Ě":219,"ě":220,"Ĝ":221,"ĝ":222,"Ğ":223,"ğ":224,"Ġ":225,"ġ":226,"Ģ":227,"ģ":228,"Ĥ":229,"ĥ":230,"Ħ":231,"ħ":232,"Ĩ":233,"ĩ":234,"Ī":235,"ī":236,"Ĭ":237,"ĭ":238,"Į":239,"į":240,"İ":241,"ı":242,"IJ":243,"ij":244,"Ĵ":245,"ĵ":246,"Ķ":247,"ķ":248,"ĸ":249,"Ĺ":250,"ĺ":251,"Ļ":252,"ļ":253,"Ľ":254,"ľ":255,"Ŀ":256,"ŀ":257,"Ł":258,"ł":259,"Ń":260,"Ġ+":261,"Ġs":262,").":263,"(-":264,"Ġr":265,"ĠĊ":266,"__":267,"an":268,"ian":269,"pian":270,"piano":271,"mp":272,"pp":273,"Ġh":274,"Ġl":275,")(":276,"Ġ%":277,")+":278,"II":279,"ed":280,"']":281,"['":282,"qd":283,"ĠV":284,"sd":285,"ac":286,"au":287,"en":288,"fr":289,"gm":290,"augm":291,"ent":292,"frac":293,"augment":294,"ĠII":295,"ĠI":296,"br":297,"gh":298,"igh":299,"brigh":300,"bright":301,"11":302,"10":303,"),":304,"Ġ2":305,"ĠVI":306,"ĠVII":307,"))":308,"ĠIII":309,"hd":310,")).":311,"ĠIV":312,"Ġ24":313,"VII":314,"64":315,"Ġ21":316,"IV":317,"ppp":318,"65":319,"))+":320,"wd":321,"43":322,"ff":323,"VI":324,"Ġ4":325,"56":326,"Ġ56":327,"Ġ40":328,"12":329,"III":330,"Ġ3":331,"13":332,"Ġ35":333,"15":334,"Ġ15":335,"14":336,"Ġ42":337,")),":338,"Ġ30":339,")']":340,"17":341,"(+":342,"(+)']":343,"{-":344,"}']":345,"fff":346,"19":347,"mm":348,"16":349,"23":350,")))+":351,"['(+)']":352,"Ġ8":353,"25":354,"Ġ16":355,"Ġ84":356,"20":357,"31":358,"21":359,"29":360,"18":361,"37":362,"Ġ168":363,"60":364,"Ġ60":365,"td":366,"53":367,"59":368,"35":369,"45":370,"Ġ12":371,"Ġ120":372,"47":373,"73":374,"70":375,"41":376,"Ġ70":377,"55":378,"39":379,"Ġ80":380,"33":381,"61":382,"83":383,"87":384,"26":385,"81":386,"67":387,"89":388,"77":389,"Ġ48":390,"57":391,"Ġ11":392,"Ġ112":393,"27":394,"101":395,"62":396,"79":397,"109":398,"22":399,"Ġ14":400,"Ġ140":401,"49":402,"71":403,"80":404,"95":405,"113":406,"103":407,"115":408,"Ġ280":409,"151":410,"143":411,"32":412,"69":413,"85":414,"Ġ240":415,"241":416,"40":417,"91":418,"97":419,"125":420,"137":421,"223":422,"229":423,"34":424,"48":425,"63":426,"92":427,"93":428,"Ġ10":429,"119":430,"121":431,"123":432,"127":433,"131":434,"173":435,"179":436,"Ġ840":437,"293":438,"Ġ105":439,"221":440,"289":441,"36":442,"365":443,"44":444,"46":445,"117":446,"Ġ210":447,")))":448,"129":449,"Ġ336":450,"133":451,"139":452,"157":453,"197":454,"199":455,"167":456,"169":457,"207":458,"181":459,"185":460,"187":461}
|