Hugging Face's logo

xmutly
/
robustvlm-object-centric

Model card Files Community
xmutly commited on
Commit
e1aaaac
·
verified ·
1 Parent(s): d6f788a

Upload 294 files

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +1 -0
  2. .idea/.gitignore +8 -0
  3. .idea/RobustVLM.iml +12 -0
  4. .idea/deployment.xml +309 -0
  5. .idea/inspectionProfiles/Project_Default.xml +105 -0
  6. .idea/inspectionProfiles/profiles_settings.xml +6 -0
  7. .idea/misc.xml +4 -0
  8. .idea/modules.xml +8 -0
  9. .idea/workspace.xml +162 -0
  10. CLIP_benchmark/LICENSE +22 -0
  11. CLIP_benchmark/README.md +2 -0
  12. CLIP_benchmark/bash/build.sh +12 -0
  13. CLIP_benchmark/bash/run_benchmark_adv.sh +19 -0
  14. CLIP_benchmark/bash/run_benchmark_clean.sh +21 -0
  15. CLIP_benchmark/benchmark/README.md +0 -0
  16. CLIP_benchmark/benchmark/benchmark.csv +508 -0
  17. CLIP_benchmark/benchmark/dataset_type.csv +42 -0
  18. CLIP_benchmark/benchmark/datasets.txt +13 -0
  19. CLIP_benchmark/benchmark/datasets_multilingual.txt +13 -0
  20. CLIP_benchmark/benchmark/models.txt +2 -0
  21. CLIP_benchmark/benchmark/webdatasets.txt +41 -0
  22. CLIP_benchmark/clip_benchmark/__init__.py +5 -0
  23. CLIP_benchmark/clip_benchmark/cli.py +404 -0
  24. CLIP_benchmark/clip_benchmark/datasets/__init__.py +0 -0
  25. CLIP_benchmark/clip_benchmark/datasets/ar_classnames.json +1004 -0
  26. CLIP_benchmark/clip_benchmark/datasets/ar_zeroshot_classification_templates.json +59 -0
  27. CLIP_benchmark/clip_benchmark/datasets/babel_imagenet.json +0 -0
  28. CLIP_benchmark/clip_benchmark/datasets/babel_imagenet.py +20 -0
  29. CLIP_benchmark/clip_benchmark/datasets/builder.py +817 -0
  30. CLIP_benchmark/clip_benchmark/datasets/caltech101.py +243 -0
  31. CLIP_benchmark/clip_benchmark/datasets/cn_classnames.json +1004 -0
  32. CLIP_benchmark/clip_benchmark/datasets/cn_zeroshot_classification_templates.json +84 -0
  33. CLIP_benchmark/clip_benchmark/datasets/cupl_prompts.json +0 -0
  34. CLIP_benchmark/clip_benchmark/datasets/en_classnames.json +1701 -0
  35. CLIP_benchmark/clip_benchmark/datasets/en_zeroshot_classification_templates.json +295 -0
  36. CLIP_benchmark/clip_benchmark/datasets/flickr.py +62 -0
  37. CLIP_benchmark/clip_benchmark/datasets/imagenetv2.py +98 -0
  38. CLIP_benchmark/clip_benchmark/datasets/it_classnames.json +1004 -0
  39. CLIP_benchmark/clip_benchmark/datasets/it_zeroshot_classification_templates.json +53 -0
  40. CLIP_benchmark/clip_benchmark/datasets/jp_classnames.json +1004 -0
  41. CLIP_benchmark/clip_benchmark/datasets/jp_zeroshot_classification_templates.json +41 -0
  42. CLIP_benchmark/clip_benchmark/datasets/kitti.py +209 -0
  43. CLIP_benchmark/clip_benchmark/datasets/multilingual_mscoco.py +91 -0
  44. CLIP_benchmark/clip_benchmark/datasets/nllb_dist13b_prompts.json +0 -0
  45. CLIP_benchmark/clip_benchmark/datasets/objectnet.py +76 -0
  46. CLIP_benchmark/clip_benchmark/datasets/sugar_crepe.py +22 -0
  47. CLIP_benchmark/clip_benchmark/datasets/tfds.py +48 -0
  48. CLIP_benchmark/clip_benchmark/datasets/voc2007.py +248 -0
  49. CLIP_benchmark/clip_benchmark/metrics/__init__.py +0 -0
  50. CLIP_benchmark/clip_benchmark/metrics/captioning.py +99 -0
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ scienceqa_eval/problems.json filter=lfs diff=lfs merge=lfs -text
.idea/.gitignore ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ # Default ignored files
2
+ /shelf/
3
+ /workspace.xml
4
+ # Editor-based HTTP Client requests
5
+ /httpRequests/
6
+ # Datasource local storage ignored files
7
+ /dataSources/
8
+ /dataSources.local.xml
.idea/RobustVLM.iml ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <module type="PYTHON_MODULE" version="4">
3
+ <component name="NewModuleRootManager">
4
+ <content url="file://$MODULE_DIR$" />
5
+ <orderEntry type="jdk" jdkName="Remote Python 3.11.0 (sftp://[email protected]:2122/home/tly/anaconda3/envs/robustclip/bin/python)" jdkType="Python SDK" />
6
+ <orderEntry type="sourceFolder" forTests="false" />
7
+ </component>
8
+ <component name="PyDocumentationSettings">
9
+ <option name="format" value="PLAIN" />
10
+ <option name="myDocStringFormat" value="Plain" />
11
+ </component>
12
+ </module>
.idea/deployment.xml ADDED
@@ -0,0 +1,309 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <project version="4">
3
+ <component name="PublishConfigData" autoUpload="Always" serverName="[email protected]:2122 key (7)" remoteFilesAllowedToDisappearOnAutoupload="false">
4
+ <serverData>
5
+ <paths name="[email protected]:22 password">
6
+ <serverdata>
7
+ <mappings>
8
+ <mapping local="$PROJECT_DIR$" web="/" />
9
+ </mappings>
10
+ </serverdata>
11
+ </paths>
12
+ <paths name="[email protected]:22 password (2)">
13
+ <serverdata>
14
+ <mappings>
15
+ <mapping local="$PROJECT_DIR$" web="/" />
16
+ </mappings>
17
+ </serverdata>
18
+ </paths>
19
+ <paths name="[email protected]:1122 key">
20
+ <serverdata>
21
+ <mappings>
22
+ <mapping local="$PROJECT_DIR$" web="/" />
23
+ </mappings>
24
+ </serverdata>
25
+ </paths>
26
+ <paths name="[email protected]:8001 password">
27
+ <serverdata>
28
+ <mappings>
29
+ <mapping local="$PROJECT_DIR$" web="/" />
30
+ </mappings>
31
+ </serverdata>
32
+ </paths>
33
+ <paths name="[email protected]:8001 password (2)">
34
+ <serverdata>
35
+ <mappings>
36
+ <mapping local="$PROJECT_DIR$" web="/" />
37
+ </mappings>
38
+ </serverdata>
39
+ </paths>
40
+ <paths name="[email protected]:8001 password (3)">
41
+ <serverdata>
42
+ <mappings>
43
+ <mapping local="$PROJECT_DIR$" web="/" />
44
+ </mappings>
45
+ </serverdata>
46
+ </paths>
47
+ <paths name="[email protected]:8001 password (4)">
48
+ <serverdata>
49
+ <mappings>
50
+ <mapping local="$PROJECT_DIR$" web="/" />
51
+ </mappings>
52
+ </serverdata>
53
+ </paths>
54
+ <paths name="[email protected]:8001 password (5)">
55
+ <serverdata>
56
+ <mappings>
57
+ <mapping local="$PROJECT_DIR$" web="/" />
58
+ </mappings>
59
+ </serverdata>
60
+ </paths>
61
+ <paths name="[email protected]:8001 password (6)">
62
+ <serverdata>
63
+ <mappings>
64
+ <mapping local="$PROJECT_DIR$" web="/" />
65
+ </mappings>
66
+ </serverdata>
67
+ </paths>
68
+ <paths name="[email protected]:8001 password">
69
+ <serverdata>
70
+ <mappings>
71
+ <mapping local="$PROJECT_DIR$" web="/" />
72
+ </mappings>
73
+ </serverdata>
74
+ </paths>
75
+ <paths name="[email protected]:22 password">
76
+ <serverdata>
77
+ <mappings>
78
+ <mapping local="$PROJECT_DIR$" web="/" />
79
+ </mappings>
80
+ </serverdata>
81
+ </paths>
82
+ <paths name="[email protected]:1622 key">
83
+ <serverdata>
84
+ <mappings>
85
+ <mapping local="$PROJECT_DIR$" web="/" />
86
+ </mappings>
87
+ </serverdata>
88
+ </paths>
89
+ <paths name="[email protected]:1622 key (2)">
90
+ <serverdata>
91
+ <mappings>
92
+ <mapping local="$PROJECT_DIR$" web="/" />
93
+ </mappings>
94
+ </serverdata>
95
+ </paths>
96
+ <paths name="[email protected]:1622 key (3)">
97
+ <serverdata>
98
+ <mappings>
99
+ <mapping local="$PROJECT_DIR$" web="/" />
100
+ </mappings>
101
+ </serverdata>
102
+ </paths>
103
+ <paths name="[email protected]:1622 key (4)">
104
+ <serverdata>
105
+ <mappings>
106
+ <mapping local="$PROJECT_DIR$" web="/" />
107
+ </mappings>
108
+ </serverdata>
109
+ </paths>
110
+ <paths name="[email protected]:22 password">
111
+ <serverdata>
112
+ <mappings>
113
+ <mapping local="$PROJECT_DIR$" web="/" />
114
+ </mappings>
115
+ </serverdata>
116
+ </paths>
117
+ <paths name="[email protected]:22 password (2)">
118
+ <serverdata>
119
+ <mappings>
120
+ <mapping local="$PROJECT_DIR$" web="/" />
121
+ </mappings>
122
+ </serverdata>
123
+ </paths>
124
+ <paths name="[email protected]:22 password (3)">
125
+ <serverdata>
126
+ <mappings>
127
+ <mapping local="$PROJECT_DIR$" web="/" />
128
+ </mappings>
129
+ </serverdata>
130
+ </paths>
131
+ <paths name="[email protected]:22 password (4)">
132
+ <serverdata>
133
+ <mappings>
134
+ <mapping local="$PROJECT_DIR$" web="/" />
135
+ </mappings>
136
+ </serverdata>
137
+ </paths>
138
+ <paths name="[email protected]:22 password (5)">
139
+ <serverdata>
140
+ <mappings>
141
+ <mapping local="$PROJECT_DIR$" web="/" />
142
+ </mappings>
143
+ </serverdata>
144
+ </paths>
145
+ <paths name="[email protected]:22 password (6)">
146
+ <serverdata>
147
+ <mappings>
148
+ <mapping local="$PROJECT_DIR$" web="/" />
149
+ </mappings>
150
+ </serverdata>
151
+ </paths>
152
+ <paths name="[email protected]:22 password (7)">
153
+ <serverdata>
154
+ <mappings>
155
+ <mapping local="$PROJECT_DIR$" web="/" />
156
+ </mappings>
157
+ </serverdata>
158
+ </paths>
159
+ <paths name="[email protected]:22 password (8)">
160
+ <serverdata>
161
+ <mappings>
162
+ <mapping local="$PROJECT_DIR$" web="/" />
163
+ </mappings>
164
+ </serverdata>
165
+ </paths>
166
+ <paths name="[email protected]:10003 key">
167
+ <serverdata>
168
+ <mappings>
169
+ <mapping local="$PROJECT_DIR$" web="/" />
170
+ </mappings>
171
+ </serverdata>
172
+ </paths>
173
+ <paths name="[email protected]:77 key">
174
+ <serverdata>
175
+ <mappings>
176
+ <mapping local="$PROJECT_DIR$" web="/" />
177
+ </mappings>
178
+ </serverdata>
179
+ </paths>
180
+ <paths name="[email protected]:22 key">
181
+ <serverdata>
182
+ <mappings>
183
+ <mapping local="$PROJECT_DIR$" web="/" />
184
+ </mappings>
185
+ </serverdata>
186
+ </paths>
187
+ <paths name="[email protected]:22 key (2)">
188
+ <serverdata>
189
+ <mappings>
190
+ <mapping local="$PROJECT_DIR$" web="/" />
191
+ </mappings>
192
+ </serverdata>
193
+ </paths>
194
+ <paths name="[email protected]:1322 key">
195
+ <serverdata>
196
+ <mappings>
197
+ <mapping local="$PROJECT_DIR$" web="/" />
198
+ </mappings>
199
+ </serverdata>
200
+ </paths>
201
+ <paths name="[email protected]:1822 key">
202
+ <serverdata>
203
+ <mappings>
204
+ <mapping local="$PROJECT_DIR$" web="/" />
205
+ </mappings>
206
+ </serverdata>
207
+ </paths>
208
+ <paths name="[email protected]:1822 key (2)">
209
+ <serverdata>
210
+ <mappings>
211
+ <mapping local="$PROJECT_DIR$" web="/" />
212
+ </mappings>
213
+ </serverdata>
214
+ </paths>
215
+ <paths name="[email protected]:1822 key (3)">
216
+ <serverdata>
217
+ <mappings>
218
+ <mapping local="$PROJECT_DIR$" web="/" />
219
+ </mappings>
220
+ </serverdata>
221
+ </paths>
222
+ <paths name="[email protected]:2122 key">
223
+ <serverdata>
224
+ <mappings>
225
+ <mapping local="$PROJECT_DIR$" web="/" />
226
+ </mappings>
227
+ </serverdata>
228
+ </paths>
229
+ <paths name="[email protected]:2122 key (2)">
230
+ <serverdata>
231
+ <mappings>
232
+ <mapping local="$PROJECT_DIR$" web="/" />
233
+ </mappings>
234
+ </serverdata>
235
+ </paths>
236
+ <paths name="[email protected]:2122 key (3)">
237
+ <serverdata>
238
+ <mappings>
239
+ <mapping local="$PROJECT_DIR$" web="/" />
240
+ </mappings>
241
+ </serverdata>
242
+ </paths>
243
+ <paths name="[email protected]:2122 key (4)">
244
+ <serverdata>
245
+ <mappings>
246
+ <mapping local="$PROJECT_DIR$" web="/" />
247
+ </mappings>
248
+ </serverdata>
249
+ </paths>
250
+ <paths name="[email protected]:2122 key (5)">
251
+ <serverdata>
252
+ <mappings>
253
+ <mapping local="$PROJECT_DIR$" web="/" />
254
+ </mappings>
255
+ </serverdata>
256
+ </paths>
257
+ <paths name="[email protected]:2122 key (6)">
258
+ <serverdata>
259
+ <mappings>
260
+ <mapping local="$PROJECT_DIR$" web="/" />
261
+ </mappings>
262
+ </serverdata>
263
+ </paths>
264
+ <paths name="[email protected]:2122 key (7)">
265
+ <serverdata>
266
+ <mappings>
267
+ <mapping deploy="/RobustVLM" local="$PROJECT_DIR$" />
268
+ </mappings>
269
+ </serverdata>
270
+ </paths>
271
+ <paths name="tongkang">
272
+ <serverdata>
273
+ <mappings>
274
+ <mapping local="$PROJECT_DIR$" web="/" />
275
+ </mappings>
276
+ </serverdata>
277
+ </paths>
278
+ <paths name="[email protected]:44152 password">
279
+ <serverdata>
280
+ <mappings>
281
+ <mapping local="$PROJECT_DIR$" web="/" />
282
+ </mappings>
283
+ </serverdata>
284
+ </paths>
285
+ <paths name="[email protected]:8001 password">
286
+ <serverdata>
287
+ <mappings>
288
+ <mapping local="$PROJECT_DIR$" web="/" />
289
+ </mappings>
290
+ </serverdata>
291
+ </paths>
292
+ <paths name="[email protected]:8001 password">
293
+ <serverdata>
294
+ <mappings>
295
+ <mapping local="$PROJECT_DIR$" web="/" />
296
+ </mappings>
297
+ </serverdata>
298
+ </paths>
299
+ <paths name="[email protected]:1922 key">
300
+ <serverdata>
301
+ <mappings>
302
+ <mapping local="$PROJECT_DIR$" web="/" />
303
+ </mappings>
304
+ </serverdata>
305
+ </paths>
306
+ </serverData>
307
+ <option name="myAutoUpload" value="ALWAYS" />
308
+ </component>
309
+ </project>
.idea/inspectionProfiles/Project_Default.xml ADDED
@@ -0,0 +1,105 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <component name="InspectionProjectProfileManager">
2
+ <profile version="1.0">
3
+ <option name="myName" value="Project Default" />
4
+ <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
5
+ <option name="ignoredPackages">
6
+ <value>
7
+ <list size="92">
8
+ <item index="0" class="java.lang.String" itemvalue="pandas" />
9
+ <item index="1" class="java.lang.String" itemvalue="numba" />
10
+ <item index="2" class="java.lang.String" itemvalue="timm" />
11
+ <item index="3" class="java.lang.String" itemvalue="matplotlib" />
12
+ <item index="4" class="java.lang.String" itemvalue="einops" />
13
+ <item index="5" class="java.lang.String" itemvalue="protobuf" />
14
+ <item index="6" class="java.lang.String" itemvalue="scipy" />
15
+ <item index="7" class="java.lang.String" itemvalue="thop" />
16
+ <item index="8" class="java.lang.String" itemvalue="PyYAML" />
17
+ <item index="9" class="java.lang.String" itemvalue="setuptools" />
18
+ <item index="10" class="java.lang.String" itemvalue="torch" />
19
+ <item index="11" class="java.lang.String" itemvalue="numpy" />
20
+ <item index="12" class="java.lang.String" itemvalue="requests" />
21
+ <item index="13" class="java.lang.String" itemvalue="torchvision" />
22
+ <item index="14" class="java.lang.String" itemvalue="tqdm" />
23
+ <item index="15" class="java.lang.String" itemvalue="seaborn" />
24
+ <item index="16" class="java.lang.String" itemvalue="Pillow" />
25
+ <item index="17" class="java.lang.String" itemvalue="yacs" />
26
+ <item index="18" class="java.lang.String" itemvalue="gdown" />
27
+ <item index="19" class="java.lang.String" itemvalue="tensorboard" />
28
+ <item index="20" class="java.lang.String" itemvalue="yapf" />
29
+ <item index="21" class="java.lang.String" itemvalue="future" />
30
+ <item index="22" class="java.lang.String" itemvalue="isort" />
31
+ <item index="23" class="java.lang.String" itemvalue="flake8" />
32
+ <item index="24" class="java.lang.String" itemvalue="tb-nightly" />
33
+ <item index="25" class="java.lang.String" itemvalue="easydict" />
34
+ <item index="26" class="java.lang.String" itemvalue="scikit_learn" />
35
+ <item index="27" class="java.lang.String" itemvalue="tensorboardX" />
36
+ <item index="28" class="java.lang.String" itemvalue="mxnet" />
37
+ <item index="29" class="java.lang.String" itemvalue="mxnet_cu90" />
38
+ <item index="30" class="java.lang.String" itemvalue="bcolz" />
39
+ <item index="31" class="java.lang.String" itemvalue="opencv_python" />
40
+ <item index="32" class="java.lang.String" itemvalue="opencv-python" />
41
+ <item index="33" class="java.lang.String" itemvalue="pretty_midi" />
42
+ <item index="34" class="java.lang.String" itemvalue="pytest-runner" />
43
+ <item index="35" class="java.lang.String" itemvalue="interrogate" />
44
+ <item index="36" class="java.lang.String" itemvalue="parameterized" />
45
+ <item index="37" class="java.lang.String" itemvalue="pytest-cov" />
46
+ <item index="38" class="java.lang.String" itemvalue="waymo-open-dataset-tf-2-6-0" />
47
+ <item index="39" class="java.lang.String" itemvalue="black" />
48
+ <item index="40" class="java.lang.String" itemvalue="kwarray" />
49
+ <item index="41" class="java.lang.String" itemvalue="xdoctest" />
50
+ <item index="42" class="java.lang.String" itemvalue="codecov" />
51
+ <item index="43" class="java.lang.String" itemvalue="ubelt" />
52
+ <item index="44" class="java.lang.String" itemvalue="scikit-image" />
53
+ <item index="45" class="java.lang.String" itemvalue="sklearn" />
54
+ <item index="46" class="java.lang.String" itemvalue="sktime" />
55
+ <item index="47" class="java.lang.String" itemvalue="patool" />
56
+ <item index="48" class="java.lang.String" itemvalue="scikit-learn" />
57
+ <item index="49" class="java.lang.String" itemvalue="reformer-pytorch" />
58
+ <item index="50" class="java.lang.String" itemvalue="sympy" />
59
+ <item index="51" class="java.lang.String" itemvalue="absl-py" />
60
+ <item index="52" class="java.lang.String" itemvalue="joblib" />
61
+ <item index="53" class="java.lang.String" itemvalue="threadpoolctl" />
62
+ <item index="54" class="java.lang.String" itemvalue="smmap" />
63
+ <item index="55" class="java.lang.String" itemvalue="python-dateutil" />
64
+ <item index="56" class="java.lang.String" itemvalue="cycler" />
65
+ <item index="57" class="java.lang.String" itemvalue="MarkupSafe" />
66
+ <item index="58" class="java.lang.String" itemvalue="mkl-random" />
67
+ <item index="59" class="java.lang.String" itemvalue="gitdb" />
68
+ <item index="60" class="java.lang.String" itemvalue="mkl-fft" />
69
+ <item index="61" class="java.lang.String" itemvalue="appdirs" />
70
+ <item index="62" class="java.lang.String" itemvalue="sentry-sdk" />
71
+ <item index="63" class="java.lang.String" itemvalue="pip" />
72
+ <item index="64" class="java.lang.String" itemvalue="certifi" />
73
+ <item index="65" class="java.lang.String" itemvalue="wandb" />
74
+ <item index="66" class="java.lang.String" itemvalue="urllib3" />
75
+ <item index="67" class="java.lang.String" itemvalue="pyparsing" />
76
+ <item index="68" class="java.lang.String" itemvalue="Markdown" />
77
+ <item index="69" class="java.lang.String" itemvalue="GitPython" />
78
+ <item index="70" class="java.lang.String" itemvalue="six" />
79
+ <item index="71" class="java.lang.String" itemvalue="Werkzeug" />
80
+ <item index="72" class="java.lang.String" itemvalue="wheel" />
81
+ <item index="73" class="java.lang.String" itemvalue="tensorboard-data-server" />
82
+ <item index="74" class="java.lang.String" itemvalue="tzdata" />
83
+ <item index="75" class="java.lang.String" itemvalue="kiwisolver" />
84
+ <item index="76" class="java.lang.String" itemvalue="packaging" />
85
+ <item index="77" class="java.lang.String" itemvalue="click" />
86
+ <item index="78" class="java.lang.String" itemvalue="contourpy" />
87
+ <item index="79" class="java.lang.String" itemvalue="docker-pycreds" />
88
+ <item index="80" class="java.lang.String" itemvalue="psutil" />
89
+ <item index="81" class="java.lang.String" itemvalue="chardet" />
90
+ <item index="82" class="java.lang.String" itemvalue="fonttools" />
91
+ <item index="83" class="java.lang.String" itemvalue="typing_extensions" />
92
+ <item index="84" class="java.lang.String" itemvalue="charset-normalizer" />
93
+ <item index="85" class="java.lang.String" itemvalue="pillow" />
94
+ <item index="86" class="java.lang.String" itemvalue="grpcio" />
95
+ <item index="87" class="java.lang.String" itemvalue="pytz" />
96
+ <item index="88" class="java.lang.String" itemvalue="setproctitle" />
97
+ <item index="89" class="java.lang.String" itemvalue="idna" />
98
+ <item index="90" class="java.lang.String" itemvalue="mkl-service" />
99
+ <item index="91" class="java.lang.String" itemvalue="torchaudio" />
100
+ </list>
101
+ </value>
102
+ </option>
103
+ </inspection_tool>
104
+ </profile>
105
+ </component>
.idea/inspectionProfiles/profiles_settings.xml ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ <component name="InspectionProjectProfileManager">
2
+ <settings>
3
+ <option name="USE_PROJECT_PROFILE" value="false" />
4
+ <version value="1.0" />
5
+ </settings>
6
+ </component>
.idea/misc.xml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <project version="4">
3
+ <component name="ProjectRootManager" version="2" project-jdk-name="Remote Python 3.11.0 (sftp://[email protected]:2122/home/tly/anaconda3/envs/robustclip/bin/python)" project-jdk-type="Python SDK" />
4
+ </project>
.idea/modules.xml ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <project version="4">
3
+ <component name="ProjectModuleManager">
4
+ <modules>
5
+ <module fileurl="file://$PROJECT_DIR$/.idea/RobustVLM.iml" filepath="$PROJECT_DIR$/.idea/RobustVLM.iml" />
6
+ </modules>
7
+ </component>
8
+ </project>
.idea/workspace.xml ADDED
@@ -0,0 +1,162 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <project version="4">
3
+ <component name="AutoImportSettings">
4
+ <option name="autoReloadType" value="SELECTIVE" />
5
+ </component>
6
+ <component name="ChangeListManager">
7
+ <list default="true" id="ba57d845-ced9-46fc-a8d3-426558963497" name="Changes" comment="" />
8
+ <option name="SHOW_DIALOG" value="false" />
9
+ <option name="HIGHLIGHT_CONFLICTS" value="true" />
10
+ <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
11
+ <option name="LAST_RESOLUTION" value="IGNORE" />
12
+ </component>
13
+ <component name="MarkdownSettingsMigration">
14
+ <option name="stateVersion" value="1" />
15
+ </component>
16
+ <component name="ProjectId" id="2nSLSQiWX77cv8fcU5XA3n96QkM" />
17
+ <component name="ProjectViewState">
18
+ <option name="hideEmptyMiddlePackages" value="true" />
19
+ <option name="showLibraryContents" value="true" />
20
+ </component>
21
+ <component name="PropertiesComponent">{
22
+ &quot;keyToString&quot;: {
23
+ &quot;RunOnceActivity.OpenProjectViewOnStart&quot;: &quot;true&quot;,
24
+ &quot;RunOnceActivity.ShowReadmeOnStart&quot;: &quot;true&quot;,
25
+ &quot;WebServerToolWindowFactoryState&quot;: &quot;true&quot;,
26
+ &quot;last_opened_file_path&quot;: &quot;D:/AI/Mine/Codes/RobustVLM&quot;,
27
+ &quot;node.js.detected.package.eslint&quot;: &quot;true&quot;,
28
+ &quot;node.js.detected.package.tslint&quot;: &quot;true&quot;,
29
+ &quot;node.js.selected.package.eslint&quot;: &quot;(autodetect)&quot;,
30
+ &quot;node.js.selected.package.tslint&quot;: &quot;(autodetect)&quot;,
31
+ &quot;nodejs_package_manager_path&quot;: &quot;npm&quot;,
32
+ &quot;settings.editor.selected.configurable&quot;: &quot;reference.idesettings.debugger.python&quot;,
33
+ &quot;vue.rearranger.settings.migration&quot;: &quot;true&quot;
34
+ }
35
+ }</component>
36
+ <component name="PyDebuggerOptionsProvider">
37
+ <option name="mySupportGeventDebugging" value="true" />
38
+ </component>
39
+ <component name="RecentsManager">
40
+ <key name="CopyFile.RECENT_KEYS">
41
+ <recent name="D:\AI\Mine\Codes\RobustVLM" />
42
+ <recent name="D:\AI\Mine\Codes\RobustVLM\train" />
43
+ <recent name="D:\AI\Mine\Codes\RobustVLM\slots" />
44
+ </key>
45
+ </component>
46
+ <component name="RunManager" selected="Python.training_clip_slots">
47
+ <configuration name="adversarial_training_clip" type="PythonConfigurationType" factoryName="Python" nameIsGenerated="true">
48
+ <module name="RobustVLM" />
49
+ <option name="INTERPRETER_OPTIONS" value="" />
50
+ <option name="PARENT_ENVS" value="true" />
51
+ <envs>
52
+ <env name="PYTHONUNBUFFERED" value="1" />
53
+ <env name="CUDA_VISIBLE_DEVICES" value="2" />
54
+ </envs>
55
+ <option name="SDK_HOME" value="/home/tly/anaconda3/envs/robustclip/bin/python" />
56
+ <option name="SDK_NAME" value="Remote Python 3.11.0 (sftp://[email protected]:2122/home/tly/anaconda3/envs/robustclip/bin/python)" />
57
+ <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
58
+ <option name="IS_MODULE_SDK" value="false" />
59
+ <option name="ADD_CONTENT_ROOTS" value="true" />
60
+ <option name="ADD_SOURCE_ROOTS" value="true" />
61
+ <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
62
+ <option name="SCRIPT_NAME" value="$PROJECT_DIR$/train/adversarial_training_clip.py" />
63
+ <option name="PARAMETERS" value="--clip_model_name ViT-L-14 --pretrained openai --dataset imagenet --imagenet_root /home/hkz/Datasets/ForGCD/imagenet --template std --output_normalize False --steps 160000 --warmup 11200 --batch_size 12 --loss l2 --opt adamw --lr 1e-6 --wd 1e-4 --attack pgd --inner_loss l2 --norm linf --eps 4 --iterations_adv 10 --stepsize_adv 1 --wandb False --output_dir ./output --experiment_name FARE4 --log_freq 1000 --eval_freq 1000" />
64
+ <option name="SHOW_COMMAND_LINE" value="false" />
65
+ <option name="EMULATE_TERMINAL" value="false" />
66
+ <option name="MODULE_MODE" value="false" />
67
+ <option name="REDIRECT_INPUT" value="false" />
68
+ <option name="INPUT_FILE" value="" />
69
+ <method v="2" />
70
+ </configuration>
71
+ <configuration name="training_clip_get_features" type="PythonConfigurationType" factoryName="Python" nameIsGenerated="true">
72
+ <module name="RobustVLM" />
73
+ <option name="INTERPRETER_OPTIONS" value="" />
74
+ <option name="PARENT_ENVS" value="true" />
75
+ <envs>
76
+ <env name="PYTHONUNBUFFERED" value="1" />
77
+ <env name="CUDA_VISIBLE_DEVICES" value="2" />
78
+ </envs>
79
+ <option name="SDK_HOME" value="/home/tly/anaconda3/envs/robustclip/bin/python" />
80
+ <option name="SDK_NAME" value="Remote Python 3.11.0 (sftp://[email protected]:2122/home/tly/anaconda3/envs/robustclip/bin/python)" />
81
+ <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
82
+ <option name="IS_MODULE_SDK" value="false" />
83
+ <option name="ADD_CONTENT_ROOTS" value="true" />
84
+ <option name="ADD_SOURCE_ROOTS" value="true" />
85
+ <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
86
+ <option name="SCRIPT_NAME" value="$PROJECT_DIR$/train/training_clip_get_features.py" />
87
+ <option name="PARAMETERS" value="--clip_model_name ViT-L-14 --pretrained openai --dataset imagenet --imagenet_root /home/hkz/Datasets/ForGCD/imagenet --template std --output_normalize False --steps 150000 --warmup 1500 --batch_size 64 --loss l2 --opt adamw --lr 1e-5 --wd 1e-4 --attack pgd --inner_loss l2 --norm linf --eps 4 --iterations_adv 10 --stepsize_adv 1 --wandb False --output_dir ./output_slots --experiment_name SLOTS --log_freq 1000 --eval_freq 1000" />
88
+ <option name="SHOW_COMMAND_LINE" value="false" />
89
+ <option name="EMULATE_TERMINAL" value="false" />
90
+ <option name="MODULE_MODE" value="false" />
91
+ <option name="REDIRECT_INPUT" value="false" />
92
+ <option name="INPUT_FILE" value="" />
93
+ <method v="2" />
94
+ </configuration>
95
+ <configuration name="training_clip_slots" type="PythonConfigurationType" factoryName="Python" nameIsGenerated="true">
96
+ <module name="RobustVLM" />
97
+ <option name="INTERPRETER_OPTIONS" value="" />
98
+ <option name="PARENT_ENVS" value="true" />
99
+ <envs>
100
+ <env name="PYTHONUNBUFFERED" value="1" />
101
+ <env name="CUDA_VISIBLE_DEVICES" value="0,1" />
102
+ </envs>
103
+ <option name="SDK_HOME" value="/home/tly/anaconda3/envs/robustclip/bin/python" />
104
+ <option name="SDK_NAME" value="Remote Python 3.11.0 (sftp://[email protected]:2122/home/tly/anaconda3/envs/robustclip/bin/python)" />
105
+ <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
106
+ <option name="IS_MODULE_SDK" value="false" />
107
+ <option name="ADD_CONTENT_ROOTS" value="true" />
108
+ <option name="ADD_SOURCE_ROOTS" value="true" />
109
+ <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
110
+ <option name="SCRIPT_NAME" value="$PROJECT_DIR$/train/training_clip_slots.py" />
111
+ <option name="PARAMETERS" value="--clip_model_name ViT-L-14 --pretrained openai --dataset imagenet --imagenet_root /home/hkz/Datasets/ForGCD/imagenet --template std --output_normalize False --steps 1500000 --warmup 15000 --batch_size 64 --loss l2 --opt adamw --lr 5e-5 --wd 1e-4 --attack pgd --inner_loss l2 --norm linf --eps 4 --iterations_adv 10 --stepsize_adv 1 --wandb False --output_dir ./output_slots --experiment_name SLOTS --log_freq 1000 --eval_freq 1000" />
112
+ <option name="SHOW_COMMAND_LINE" value="false" />
113
+ <option name="EMULATE_TERMINAL" value="false" />
114
+ <option name="MODULE_MODE" value="false" />
115
+ <option name="REDIRECT_INPUT" value="false" />
116
+ <option name="INPUT_FILE" value="" />
117
+ <method v="2" />
118
+ </configuration>
119
+ <list>
120
+ <item itemvalue="Python.adversarial_training_clip" />
121
+ <item itemvalue="Python.training_clip_slots" />
122
+ <item itemvalue="Python.training_clip_get_features" />
123
+ </list>
124
+ </component>
125
+ <component name="SpellCheckerSettings" RuntimeDictionaries="0" Folders="0" CustomDictionaries="0" DefaultDictionary="application-level" UseSingleDictionary="true" transferred="true" />
126
+ <component name="TaskManager">
127
+ <task active="true" id="Default" summary="Default task">
128
+ <changelist id="ba57d845-ced9-46fc-a8d3-426558963497" name="Changes" comment="" />
129
+ <created>1728961530244</created>
130
+ <option name="number" value="Default" />
131
+ <option name="presentableId" value="Default" />
132
+ <updated>1728961530244</updated>
133
+ <workItem from="1728961535817" duration="22428000" />
134
+ <workItem from="1729131617455" duration="11209000" />
135
+ </task>
136
+ <servers />
137
+ </component>
138
+ <component name="TypeScriptGeneratedFilesManager">
139
+ <option name="version" value="3" />
140
+ </component>
141
+ <component name="XDebuggerManager">
142
+ <breakpoint-manager>
143
+ <breakpoints>
144
+ <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
145
+ <url>file://$PROJECT_DIR$/train/training_clip_slots.py</url>
146
+ <line>306</line>
147
+ <option name="timeStamp" value="19" />
148
+ </line-breakpoint>
149
+ <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
150
+ <url>file://$PROJECT_DIR$/train/training_clip_get_features.py</url>
151
+ <line>299</line>
152
+ <option name="timeStamp" value="20" />
153
+ </line-breakpoint>
154
+ </breakpoints>
155
+ </breakpoint-manager>
156
+ </component>
157
+ <component name="com.intellij.coverage.CoverageDataManagerImpl">
158
+ <SUITE FILE_PATH="coverage/RobustVLM$training_clip_slots.coverage" NAME="training_clip_slots Coverage Results" MODIFIED="1729131753552" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
159
+ <SUITE FILE_PATH="coverage/RobustVLM$adversarial_training_clip.coverage" NAME="adversarial_training_clip Coverage Results" MODIFIED="1729061917005" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
160
+ <SUITE FILE_PATH="coverage/RobustVLM$training_clip_get_features.coverage" NAME="training_clip_get_features Coverage Results" MODIFIED="1729077166280" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
161
+ </component>
162
+ </project>
CLIP_benchmark/LICENSE ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ MIT License
2
+
3
+ Copyright (c) 2022, Mehdi Cherti
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
22
+
CLIP_benchmark/README.md ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ # CLIP Benchmark
2
+ - Forked from [CLIP Benchmark](https://github.com/LAION-AI/CLIP_benchmark)
CLIP_benchmark/bash/build.sh ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ # gathers results from a results directory and builds a csv
3
+ # enter results dir in format /path/to/results/*
4
+ export PYTHONPATH="../":"${PYTHONPATH}"
5
+ set -e
6
+ echo "Enter path to results directory: "
7
+ read RES_DIR
8
+ echo "building results csv... ${RES_DIR}"
9
+ RND=${RANDOM}${RANDOM}
10
+ python -m clip_benchmark.cli build ${RES_DIR} --output "res${RND}.csv"
11
+ echo "reformatting csv..."
12
+ python reformat_csv.py res${RND}.csv
CLIP_benchmark/bash/run_benchmark_adv.sh ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ set -e # stop on error
3
+ SECONDS=0
4
+ # add parent dir to python path
5
+ export PYTHONPATH="../":"${PYTHONPATH}"
6
+ EPS=2 # TODO
7
+
8
+ SAMPLES=1000
9
+ SAVE_DIR=/path/to/out/dir # TODO
10
+ mkdir -p "$SAVE_DIR"
11
+ python -m clip_benchmark.cli eval --dataset_root "https://huggingface.co/datasets/clip-benchmark/wds_{dataset_cleaned}/tree/main" --dataset benchmark/datasets.txt \
12
+ --pretrained_model benchmark/models.txt \
13
+ --output "${SAVE_DIR}/adv_{model}_{pretrained}_{dataset}_{n_samples}_bs{bs}_{attack}_{eps}_{iterations}.json" \
14
+ --attack aa --eps $EPS \
15
+ --batch_size 50 --n_samples $SAMPLES
16
+
17
+ hours=$((SECONDS / 3600))
18
+ minutes=$(( (SECONDS % 3600) / 60 ))
19
+ echo "[Runtime] $hours h $minutes min"
CLIP_benchmark/bash/run_benchmark_clean.sh ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ set -e # stop on error
3
+ # add parent to python path
4
+ export PYTHONPATH="../":"${PYTHONPATH}"
5
+
6
+ SECONDS=0
7
+ SAMPLES=-1
8
+ BS=1000
9
+
10
+ SAVE_DIR=/path/to/out/dir # TODO
11
+ mkdir -p "$SAVE_DIR"
12
+ python -m clip_benchmark.cli eval --dataset_root "https://huggingface.co/datasets/clip-benchmark/wds_{dataset_cleaned}/tree/main" --dataset benchmark/datasets.txt \
13
+ --pretrained_model benchmark/models.txt \
14
+ --output "${SAVE_DIR}/clean_{model}_{pretrained}_beta{beta}_{dataset}_{n_samples}_bs{bs}_{attack}_{eps}_{iterations}.json" \
15
+ --attack none --eps 1 \
16
+ --batch_size $BS --n_samples $SAMPLES \
17
+
18
+
19
+ hours=$((SECONDS / 3600))
20
+ minutes=$(( (SECONDS % 3600) / 60 ))
21
+ echo "[Runtime] $hours h $minutes min"
CLIP_benchmark/benchmark/README.md ADDED
File without changes
CLIP_benchmark/benchmark/benchmark.csv ADDED
@@ -0,0 +1,508 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ acc1,acc5,mean_per_class_recall,dataset,model,pretrained,task,mean_average_precision,image_retrieval_recall@5,text_retrieval_recall@5,model_fullname
2
+ 0.0232340494791666,0.1152615017361111,0.0242046402834269,vtab/dsprites_label_orientation,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
3
+ 0.4460852605182502,0.9469211479520758,0.3940612716631316,fer2013,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
4
+ 0.7738666666666667,0.9362666666666668,0.7345750490593081,imagenet-a,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
5
+ 0.4457446808510638,0.7585106382978724,0.449468085106383,vtab/dtd,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
6
+ 0.7521562425290105,0.9608106368501388,0.7512820500659019,sun397,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
7
+ ,,,voc2007_multilabel,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,0.796263575553894,,,ViT-B-32 laion2b_s34b_b79k
8
+ 0.7557,0.9386,0.7554000000000001,vtab/cifar100,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
9
+ 0.665,0.89844,0.66506,imagenet1k,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
10
+ 0.1889597536837475,0.819793270288102,0.1681683759314615,vtab/dmlab,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
11
+ 0.4955,0.8497,0.5259367109048434,mnist,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
12
+ 0.5175536361103371,0.9651713569239344,0.5055924943171644,fer2013,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
13
+ 0.6246666789267521,0.91313422954558,0.6346910684418603,sun397,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
14
+ 0.9070591441809758,0.995366584900518,0.9070894162634328,vtab/pets,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
15
+ 0.5609659540775931,0.8069675376088677,0.5169427663206733,gtsrb,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
16
+ 0.0313720703125,0.1469997829861111,0.0320098582855241,vtab/dsprites_label_x_position,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
17
+ 0.3628226797787339,0.6565765212046711,0.4033487053539641,vtab/svhn,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
18
+ 0.4362628661916072,0.7091844813935075,0.370358797280688,gtsrb,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
19
+ 0.5535494775660725,0.9136447449293178,0.5591738787984386,vtab/svhn,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
20
+ 0.4417465274038979,0.7008721869279638,0.4272611771980346,objectnet,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
21
+ 0.7174074074074074,0.9561111111111112,0.7202760268647987,vtab/eurosat,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
22
+ 0.501821060965954,0.7586698337292161,0.4394378810250903,gtsrb,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
23
+ 0.9321340964840557,0.9978195693649496,0.9313974108097984,vtab/pets,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
24
+ 0.4934283452098179,0.7430720506730008,0.4353727539920664,gtsrb,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
25
+ 0.8522353714661407,0.963346482577252,0.944284654839904,vtab/caltech101,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
26
+ ,,,flickr30k,ViT-B-32-quickgelu,laion400m_e32,zeroshot_retrieval,,0.8546000123023987,0.9409999847412108,ViT-B-32-quickgelu laion400m_e32
27
+ 0.1178600823045267,0.5817283950617284,0.1208963734895024,vtab/smallnorb_label_elevation,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
28
+ 0.76664,0.9485,0.76656,imagenet1k,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
29
+ 0.7679,0.9386,0.7581398074696393,mnist,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
30
+ 0.2883263009845288,,0.3645688070267072,vtab/kitti_closest_vehicle_distance,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
31
+ 0.1854185418541854,0.4452445244524452,0.1875846702317291,fgvc_aircraft,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
32
+ 0.0197618272569444,0.1133355034722222,0.0176459217575104,vtab/dsprites_label_orientation,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
33
+ ,,,flickr8k,ViT-B-16-plus-240,laion400m_e32,zeroshot_retrieval,,0.873199999332428,0.9549999833106995,ViT-B-16-plus-240 laion400m_e32
34
+ 0.551063829787234,0.8356382978723405,0.5499999999999999,vtab/dtd,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
35
+ 0.8391,0.9729,0.8388,vtab/cifar100,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
36
+ 0.482037037037037,0.935,0.493913656654034,vtab/eurosat,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
37
+ 0.6828752642706131,0.8578630671653927,0.6628139602370955,vtab/flowers,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
38
+ 0.5844813935075218,0.820744259699129,0.5442606899522975,gtsrb,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
39
+ 0.50628662109375,,0.5062329426609509,vtab/pcam,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
40
+ 0.6932,0.8887,0.6785851251044699,imagenet-r,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
41
+ 0.2304230423042304,0.5295529552955296,0.2319696969696969,fgvc_aircraft,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
42
+ 0.605010986328125,,0.605165824864527,vtab/pcam,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
43
+ ,,,flickr8k,ViT-B-16,laion400m_e32,zeroshot_retrieval,,0.8575999736785889,0.9409999847412109,ViT-B-16 laion400m_e32
44
+ 0.7835420393559929,0.9242153195641568,0.7862863094545812,vtab/flowers,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
45
+ 0.9467,0.999,0.9466,vtab/cifar10,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
46
+ 0.6961,0.9086,0.6957000000000001,imagenetv2,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
47
+ 0.4345209817893903,0.707680126682502,0.400638686800972,gtsrb,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
48
+ 0.0317789713541666,0.147705078125,0.0324921668671336,vtab/dsprites_label_x_position,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
49
+ 0.7871,0.9505,0.7775130720394369,mnist,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
50
+ 0.63334,0.88778,0.63284,imagenet1k,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
51
+ 0.6031914893617021,0.8867021276595745,0.6042553191489363,vtab/dtd,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
52
+ 0.985875,0.99975,0.9864999999999998,stl10,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
53
+ 0.2636018957345971,0.5149289099526067,0.2629857819905213,country211,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
54
+ 0.5132978723404256,0.7787234042553192,0.5095744680851063,vtab/dtd,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
55
+ 0.4342160768689946,1.0,0.2167811161586062,vtab/diabetic_retinopathy,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
56
+ 0.5512,0.8156,0.5509,imagenetv2,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
57
+ 0.7105220361034315,0.8590014636526264,0.6857234783638442,vtab/flowers,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
58
+ 0.6867793368519779,0.9372344925244128,0.6845985471139586,sun397,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
59
+ 0.5016666666666667,0.959074074074074,0.511474800858698,vtab/eurosat,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
60
+ 0.8468,0.9733,0.8471000000000001,vtab/cifar100,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
61
+ ,,,voc2007_multilabel,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,0.801436722278595,,,ViT-H-14 laion2b_s32b_b79k
62
+ 0.9172,0.9975,0.9172,vtab/cifar10,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
63
+ 0.6981352410026298,0.9398183055335896,0.6849982147927691,sun397,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
64
+ 0.77972,0.95216,0.77952,imagenet1k,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
65
+ 0.029541015625,0.1558973524305555,0.0292117961574083,vtab/dsprites_label_x_position,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
66
+ 0.5,0.7502771179730799,0.4499584478173962,gtsrb,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
67
+ 0.7590333333333333,0.9128666666666668,0.7444515544684158,imagenet-r,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
68
+ 0.4690721649484536,0.931178601281694,0.4334946917742944,fer2013,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
69
+ 0.485626220703125,,0.4856418903784925,vtab/pcam,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
70
+ 0.9438539111474517,0.9986372308530936,0.9434557685576204,vtab/pets,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
71
+ ,,,flickr8k,ViT-L-14,laion400m_e32,zeroshot_retrieval,,0.8984000086784363,0.9649999737739563,ViT-L-14 laion400m_e32
72
+ 0.0972839506172839,0.5397530864197531,0.0973286727349915,vtab/smallnorb_label_elevation,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
73
+ 0.5159912109375,,0.5157975788193991,vtab/pcam,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
74
+ 0.7824519230769231,0.9688835470085472,0.8629106820310023,voc2007,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
75
+ 0.0293918185763888,0.150390625,0.0306791200330755,vtab/dsprites_label_x_position,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
76
+ 0.5710666666666666,0.8348,0.5639196371233688,imagenet-a,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
77
+ 0.6408566721581549,,0.6410094956864107,renderedsst2,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
78
+ 0.1134156378600823,0.5579423868312757,0.1146512139135114,vtab/smallnorb_label_elevation,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
79
+ 0.1590499230261711,0.8403782713877281,0.1701282125753662,vtab/dmlab,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
80
+ ,,,voc2007_multilabel,ViT-B-32,laion2b_e16,zeroshot_classification,0.7927550077438354,,,ViT-B-32 laion2b_e16
81
+ ,,,mscoco_captions,ViT-B-16,laion400m_e32,zeroshot_retrieval,,0.6364254355430603,0.7961999773979187,ViT-B-16 laion400m_e32
82
+ 0.758985200845666,0.8894129126687266,0.7455642498757189,vtab/flowers,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
83
+ ,,,voc2007_multilabel,ViT-B-32,openai,zeroshot_classification,0.7601363658905029,,,ViT-B-32 openai
84
+ 0.6968,0.9081,0.6974,imagenetv2,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
85
+ 0.8893431452711911,0.9940038157536112,0.884512216368383,vtab/pets,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
86
+ 0.7021,0.9244,0.703,vtab/cifar100,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
87
+ 0.5072222222222222,0.9255555555555556,0.489609055911575,vtab/eurosat,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
88
+ 0.7067545304777595,,0.7068315384169996,renderedsst2,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
89
+ 0.665743087897188,0.8816640138340309,0.6655023529411764,imagenet_sketch,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
90
+ 0.5155555555555555,0.9201851851851852,0.526225901185735,vtab/eurosat,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
91
+ ,,,flickr30k,ViT-B-32,openai,zeroshot_retrieval,,0.8338000178337097,0.9490000009536744,ViT-B-32 openai
92
+ 0.1593333333333333,0.9299333333333332,0.1673057808855792,vtab/clevr_closest_object_distance,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
93
+ 0.7464,0.9285,0.7471,vtab/cifar100,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
94
+ ,,,mscoco_captions,ViT-L-14,laion400m_e32,zeroshot_retrieval,,0.6805678009986877,0.8216000199317932,ViT-L-14 laion400m_e32
95
+ 0.9307713273371492,0.9980921231943308,0.9330900923082088,vtab/pets,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
96
+ 0.6188,0.8745,0.6202000000000001,imagenetv2,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
97
+ 0.4230383776454636,0.7030792509186661,0.4230898039215686,imagenet_sketch,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
98
+ 0.7910657051282052,0.9600026709401708,0.8052125971178338,voc2007,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
99
+ 0.1674950516824279,0.8040906091928745,0.1782001238774596,vtab/dmlab,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
100
+ 0.8310322156476002,0.9529914529914528,0.903296243135675,vtab/caltech101,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
101
+ 0.1094650205761317,0.5571193415637861,0.1098932195729559,vtab/smallnorb_label_elevation,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
102
+ 0.5716,0.8386,0.5721,imagenetv2,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
103
+ 0.1080658436213991,0.5204115226337449,0.108510287776451,vtab/smallnorb_label_elevation,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
104
+ 0.8982,0.9963,0.8995000000000001,vtab/cifar10,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
105
+ 0.9381302807304442,0.9986372308530936,0.9372081115891412,vtab/pets,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
106
+ 0.5819,0.8386,0.5815,imagenetv2,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
107
+ 0.6666666666666666,0.941904761904762,0.6759805529181834,vtab/resisc45,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
108
+ ,,,voc2007_multilabel,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,0.7846916913986206,,,ViT-B-16-plus-240 laion400m_e32
109
+ 0.5363831083338246,0.7926467409459804,0.53684,imagenet_sketch,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
110
+ 0.7930605646063923,0.986693197363512,0.79277195221544,cars,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
111
+ 0.6457990115321252,,0.6459400874297956,renderedsst2,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
112
+ 0.6445715707001617,0.9434149981345604,0.6469166001999892,cars,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
113
+ 0.503753662109375,,0.5035515136049098,vtab/pcam,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
114
+ 0.5706822372464659,0.9138752304855562,0.5886888789913961,vtab/svhn,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
115
+ 0.7761084401709402,0.9418402777777778,0.8508423918048074,voc2007,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
116
+ 0.1884360189573459,0.4163507109004739,0.1883412322274882,country211,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
117
+ 0.97075,0.999375,0.971875,stl10,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
118
+ 0.1508247195953375,0.8088849791071036,0.1720986035113953,vtab/dmlab,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
119
+ 0.2629716428404031,1.0,0.2194476579174435,vtab/diabetic_retinopathy,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
120
+ 0.4630454824830977,0.817340196681008,0.4869931863892911,vtab/svhn,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
121
+ ,,,mscoco_captions,ViT-B-32,laion2b_e16,zeroshot_retrieval,,0.6467413306236267,0.7950000166893005,ViT-B-32 laion2b_e16
122
+ 0.270042194092827,,0.3517916468296155,vtab/kitti_closest_vehicle_distance,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
123
+ 0.7082,0.9169,0.709,imagenetv2,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
124
+ 0.6352380952380953,0.9225396825396824,0.6419889996880412,vtab/resisc45,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
125
+ 0.5404255319148936,0.8398936170212766,0.5367021276595745,vtab/dtd,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
126
+ 0.2236286919831223,,0.3717030717825018,vtab/kitti_closest_vehicle_distance,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
127
+ 0.5850793650793651,0.910952380952381,0.5919202546199539,vtab/resisc45,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
128
+ 0.7567441239316239,0.9461805555555556,0.7914514618991711,voc2007,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
129
+ 0.3319333333333333,0.9534,0.3193231509666999,vtab/clevr_count_all,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
130
+ 0.5661724327292696,,0.5658672775172254,renderedsst2,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
131
+ 0.2686357243319268,,0.3735376764204429,vtab/kitti_closest_vehicle_distance,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
132
+ 0.1856168902573125,0.82608313173521,0.1925396140565279,vtab/dmlab,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
133
+ 0.8504273504273504,0.9681130834976988,0.9394300669046936,vtab/caltech101,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
134
+ 0.7768934212162666,0.983957219251337,0.777448930592225,cars,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
135
+ 0.3048171481253841,0.7613706207744315,0.3503741918499782,vtab/svhn,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
136
+ 0.96575,0.999375,0.966625,stl10,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
137
+ 0.6219482421875,,0.6220625731388706,vtab/pcam,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
138
+ ,,,flickr30k,ViT-L-14,laion400m_e32,zeroshot_retrieval,,0.9082000255584716,0.977999985218048,ViT-L-14 laion400m_e32
139
+ 0.72734,0.9293,0.72694,imagenet1k,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
140
+ 0.7766666666666666,0.9304333333333332,0.7605432098970494,imagenet-r,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
141
+ 0.6137,0.8644,0.6146999999999999,imagenetv2,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
142
+ 0.2877333333333333,0.9075333333333332,0.2821869879006831,vtab/clevr_count_all,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
143
+ 0.6190740740740741,0.962037037037037,0.6309344676440423,vtab/eurosat,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
144
+ 0.0313313802083333,0.1560872395833333,0.031335128135504,vtab/dsprites_label_x_position,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
145
+ 0.1149794238683127,0.5761316872427984,0.115904554765943,vtab/smallnorb_label_elevation,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
146
+ 0.7760611481541714,0.9046999512115792,0.7813275676765017,vtab/flowers,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
147
+ 0.8865,0.9695666666666668,0.8748158307459671,imagenet-r,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
148
+ ,,,mscoco_captions,ViT-H-14,laion2b_s32b_b79k,zeroshot_retrieval,,0.734306275844574,0.8604000210762024,ViT-H-14 laion2b_s32b_b79k
149
+ 0.7524,0.9418,0.7528999999999999,vtab/cifar100,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
150
+ 0.2618666666666667,0.5721333333333334,0.2839019082932269,imagenet-a,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
151
+ 0.9345852505907224,0.9988807362268376,0.9351484667320789,cars,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
152
+ 0.4286709389802173,0.898300362217888,0.392124222029496,fer2013,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
153
+ 0.68352,0.91864,0.68396,imagenet1k,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
154
+ 0.1123456790123456,0.5467489711934156,0.1133381228564946,vtab/smallnorb_label_elevation,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
155
+ 0.4932893159621922,0.7566861207726621,0.4940521568627451,imagenet_sketch,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
156
+ 0.5009752020061299,0.932153803287824,0.449919123283142,fer2013,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
157
+ 0.7549,0.946,0.7545,imagenet1k,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
158
+ 0.0346001519097222,0.1698269314236111,0.0339338982697889,vtab/dsprites_label_x_position,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
159
+ 0.71777753849467,0.9007214385700442,0.7007187288769688,objectnet,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
160
+ 0.7294,0.9415,0.7332910901261492,mnist,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
161
+ 0.0342746310763888,0.1416965060763889,0.034365141983162,vtab/dsprites_label_orientation,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
162
+ 0.210639793766112,1.0,0.2335698910327324,vtab/diabetic_retinopathy,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
163
+ 0.8392504930966469,0.9510190664036818,0.9090841082001052,vtab/caltech101,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
164
+ 0.2841068917018284,,0.4076831334000694,vtab/kitti_closest_vehicle_distance,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
165
+ 0.6813829787234043,0.925531914893617,0.6829787234042553,vtab/dtd,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
166
+ 0.6909,0.91432,0.69156,imagenet1k,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
167
+ 0.2424666666666666,0.7876,0.2312503165591237,vtab/clevr_count_all,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
168
+ 0.67002,0.90424,0.67026,imagenet1k,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
169
+ ,,,flickr30k,ViT-B-16,laion400m_e32,zeroshot_retrieval,,0.881600022315979,0.9679999947547911,ViT-B-16 laion400m_e32
170
+ 0.8787,0.9709333333333332,0.8651131734542029,imagenet-r,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
171
+ 0.62918,0.87652,0.6289,imagenet1k,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
172
+ 0.5469522240527183,,0.5464163192635053,renderedsst2,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
173
+ 0.4425531914893617,0.7638297872340426,0.4430851063829787,vtab/dtd,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
174
+ 0.5149133196941962,0.7512652094325402,0.5017005288059357,objectnet,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
175
+ 0.4855,0.8418,0.4575381785680641,mnist,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
176
+ 0.6550016151609777,0.8503284160654678,0.6433184742666794,objectnet,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
177
+ 0.7744,0.9471,0.7737999999999998,vtab/cifar100,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
178
+ 0.7920656634746922,0.9787339883099117,0.7926165075935756,cars,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
179
+ 0.2081575246132208,,0.1791674645508319,vtab/kitti_closest_vehicle_distance,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
180
+ 0.0260687934027777,0.1259223090277778,0.0268337475785376,vtab/dsprites_label_orientation,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
181
+ 0.6922,0.9403,0.6883700135057857,mnist,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
182
+ 0.1143209876543209,0.5373662551440329,0.1138628810502107,vtab/smallnorb_label_elevation,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
183
+ 0.6769,0.9012,0.6781,imagenetv2,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
184
+ 0.7683627136752137,0.952590811965812,0.8035754023986389,voc2007,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
185
+ 0.8504273504273504,0.953155818540434,0.9440706929933655,vtab/caltech101,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
186
+ 0.6434889751181566,0.9243614027989776,0.6527406670624641,sun397,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
187
+ 0.3311163895486936,0.7250989707046714,0.3196447660118034,gtsrb,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
188
+ 0.6521841655367565,0.8748649020416986,0.6524090196078433,imagenet_sketch,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
189
+ 0.1469194312796208,0.3500473933649289,0.1470142180094787,country211,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
190
+ 0.96875,0.99975,0.9695,stl10,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
191
+ 0.6908043501669,0.8805319263486594,0.6736647184602601,objectnet,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
192
+ 0.984375,0.999875,0.9849999999999998,stl10,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
193
+ 0.5544309249488533,0.7926671691611931,0.5363732822578842,objectnet,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
194
+ 0.7555422008547008,0.9489182692307692,0.830992972154603,voc2007,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
195
+ 0.5293285385839769,0.78944369117098,0.5286741176470588,imagenet_sketch,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
196
+ 0.7889957264957265,0.9573317307692308,0.8054447759101763,voc2007,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
197
+ 0.675285506739982,0.9369310554094562,0.6824513086557495,sun397,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
198
+ 0.111275720164609,0.5548971193415638,0.1102182875784799,vtab/smallnorb_label_elevation,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
199
+ ,,,flickr8k,ViT-B-32,laion2b_e16,zeroshot_retrieval,,0.8574000000953674,0.9319999814033508,ViT-B-32 laion2b_e16
200
+ 0.5989555292344136,0.8103262625174976,0.586320244179702,objectnet,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
201
+ ,,,mscoco_captions,ViT-g-14,laion2b_s12b_b42k,zeroshot_retrieval,,0.7239903807640076,0.853600025177002,ViT-g-14 laion2b_s12b_b42k
202
+ 0.7101,0.9209,0.7106999999999999,vtab/cifar100,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
203
+ 0.189873417721519,,0.2568338943834677,vtab/kitti_closest_vehicle_distance,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
204
+ ,,,mscoco_captions,ViT-B-32,laion2b_s34b_b79k,zeroshot_retrieval,,0.654218316078186,0.7982000112533569,ViT-B-32 laion2b_s34b_b79k
205
+ 0.1186008230452674,0.560082304526749,0.1176434679315522,vtab/smallnorb_label_elevation,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
206
+ 0.5634266886326195,,0.5635892536622082,renderedsst2,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
207
+ ,,,flickr8k,ViT-L-14,laion2b_s32b_b82k,zeroshot_retrieval,,0.9147999882698059,0.9670000076293945,ViT-L-14 laion2b_s32b_b82k
208
+ 0.0237358940972222,0.1186116536458333,0.0218059467808842,vtab/dsprites_label_orientation,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
209
+ 0.1316147176001874,1.0,0.230780955782727,vtab/diabetic_retinopathy,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
210
+ 0.8599676657132197,0.9912946150976246,0.8615494787047302,cars,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
211
+ 0.5569148936170213,0.8361702127659575,0.5563829787234043,vtab/dtd,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
212
+ ,,,flickr30k,ViT-B-32,laion2b_s34b_b79k,zeroshot_retrieval,,0.8835999965667725,0.9629999995231628,ViT-B-32 laion2b_s34b_b79k
213
+ 0.0715256620576517,1.0,0.2196341124982623,vtab/diabetic_retinopathy,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
214
+ 0.5699,0.8279,0.5675693520579019,mnist,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
215
+ 0.5940803382663847,0.917174480785972,0.5967107342155968,cars,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
216
+ 0.98275,1.0,0.982875,stl10,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
217
+ 0.4199524940617577,0.6838479809976247,0.393417229364651,gtsrb,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
218
+ 0.0633231778767283,1.0,0.2107285863733837,vtab/diabetic_retinopathy,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
219
+ 0.9497,0.9963,0.9497,vtab/cifar10,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
220
+ 0.76548,0.95168,0.7656000000000001,imagenet1k,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
221
+ 0.2742616033755274,,0.405770386325608,vtab/kitti_closest_vehicle_distance,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
222
+ 0.2380126552613077,1.0,0.233320813717688,vtab/diabetic_retinopathy,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
223
+ 0.2307582938388625,0.4708056872037914,0.2308530805687204,country211,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
224
+ 0.2633333333333333,0.574,0.2790514196577626,imagenet-a,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
225
+ ,,,voc2007_multilabel,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,0.7621888518333435,,,ViT-B-32-quickgelu laion400m_e32
226
+ 0.199,0.7285333333333334,0.1948019376127608,vtab/clevr_count_all,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
227
+ 0.5376190476190477,0.8671428571428571,0.5417275836816031,vtab/resisc45,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
228
+ 0.0302191840277777,0.1466335720486111,0.0300817149220147,vtab/dsprites_label_x_position,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
229
+ ,,,flickr8k,ViT-g-14,laion2b_s12b_b42k,zeroshot_retrieval,,0.9175999760627747,0.9739999771118164,ViT-g-14 laion2b_s12b_b42k
230
+ 0.157994281944139,0.8222564328128437,0.1661260800820199,vtab/dmlab,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
231
+ 0.1812322274881516,0.4011374407582938,0.181563981042654,country211,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
232
+ 0.9272,0.9988,0.9272000000000002,vtab/cifar10,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
233
+ 0.9401,0.9992,0.9405,vtab/cifar10,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
234
+ 0.7331380360909304,1.0,0.206266837915639,vtab/diabetic_retinopathy,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
235
+ 0.2587904360056259,,0.3397102822066769,vtab/kitti_closest_vehicle_distance,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
236
+ 0.2166666666666666,0.5310666666666667,0.2348332752565157,imagenet-a,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
237
+ 0.99375,1.0,0.993625,stl10,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
238
+ 0.087368174361378,1.0,0.2520359622114083,vtab/diabetic_retinopathy,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
239
+ 0.1236939151813153,0.6480485556238476,0.1332639799678708,vtab/svhn,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
240
+ 0.5782537067545305,,0.5785203520352036,renderedsst2,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
241
+ 0.0588477366255144,0.268395061728395,0.0601925439678357,vtab/smallnorb_label_azimuth,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
242
+ 0.6367,0.9218,0.6276012948452819,mnist,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
243
+ 0.0313856336805555,0.1553955078125,0.0307666564982354,vtab/dsprites_label_x_position,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
244
+ 0.9569,0.9963,0.9572,vtab/cifar10,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
245
+ 0.4973871733966746,0.7704671417260491,0.4655936453259506,gtsrb,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
246
+ 0.5853968253968254,0.896984126984127,0.5932699170625959,vtab/resisc45,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
247
+ 0.1419837255333186,0.8292060699362217,0.16573619863836,vtab/dmlab,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
248
+ 0.612762451171875,,0.6128028129314551,vtab/pcam,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
249
+ 0.1945898394545854,0.8513305476138113,0.1635130198328585,vtab/dmlab,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
250
+ 0.7259778950659286,0.9500616069294004,0.7127784763377728,sun397,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
251
+ 0.4661333333333333,0.7688,0.4728184257301568,imagenet-a,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
252
+ 0.6371428571428571,0.9277777777777778,0.6463008933552572,vtab/resisc45,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
253
+ 0.1111111111111111,,0.2722929936305732,vtab/kitti_closest_vehicle_distance,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
254
+ 0.6105248678496336,0.8503998899565721,0.6103113725490197,imagenet_sketch,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
255
+ 0.8732624693376942,0.9926410466067048,0.8695864391489154,vtab/pets,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
256
+ 0.1486,0.9095333333333332,0.1443124682607226,vtab/clevr_closest_object_distance,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
257
+ 0.0243462456597222,0.1128879123263889,0.0249444752915047,vtab/dsprites_label_orientation,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
258
+ 0.596238872840889,0.8439741397944546,0.5964254901960785,imagenet_sketch,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
259
+ 0.6276595744680851,0.9069148936170212,0.6324468085106383,vtab/dtd,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
260
+ ,,,voc2007_multilabel,ViT-B-16,openai,zeroshot_classification,0.788827121257782,,,ViT-B-16 openai
261
+ ,,,flickr30k,ViT-L-14,openai,zeroshot_retrieval,,0.8715999722480774,0.9739999771118164,ViT-L-14 openai
262
+ 0.2505250525052505,0.6012601260126013,0.2483244206773618,fgvc_aircraft,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
263
+ 0.5559259259259259,0.8868518518518519,0.5469811732579133,vtab/eurosat,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
264
+ 0.9427636958299264,0.9983646770237122,0.9434000102313552,vtab/pets,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
265
+ 0.1746174617461746,0.45004500450045,0.1753386809269162,fgvc_aircraft,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
266
+ ,,,flickr30k,ViT-B-16,openai,zeroshot_retrieval,,0.855400025844574,0.9629999995231628,ViT-B-16 openai
267
+ 0.8934314527119106,0.9956391387298992,0.8906060208128682,vtab/pets,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
268
+ 0.6266666666666667,0.9611111111111112,0.6380077170682305,vtab/eurosat,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
269
+ 0.5596,0.8341,0.5602,imagenetv2,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
270
+ 0.481439996855902,0.763229774607479,0.4822572549019607,imagenet_sketch,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
271
+ 0.2450666666666666,0.8254,0.1666666666666666,vtab/clevr_closest_object_distance,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
272
+ ,,,flickr30k,ViT-H-14,laion2b_s32b_b79k,zeroshot_retrieval,,0.9409999847412108,0.9929999709129332,ViT-H-14 laion2b_s32b_b79k
273
+ ,,,voc2007_multilabel,ViT-L-14,openai,zeroshot_classification,0.7903817892074585,,,ViT-L-14 openai
274
+ 0.4267205349679576,0.9361939258846476,0.3989364402674789,fer2013,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
275
+ 0.6960663515824705,0.9390183349577946,0.6804128851625355,sun397,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
276
+ 0.3318666666666666,0.6664,0.3409181030776702,imagenet-a,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
277
+ 0.9083,0.9944,0.9082,vtab/cifar10,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
278
+ ,,,voc2007_multilabel,ViT-L-14-336,openai,zeroshot_classification,0.8035513162612915,,,ViT-L-14-336 openai
279
+ 0.5922666666666667,0.8565333333333334,0.5810468077571583,imagenet-a,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
280
+ 0.6730158730158731,0.938095238095238,0.6781338184038964,vtab/resisc45,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
281
+ 0.7354112959925005,1.0,0.1999235670840696,vtab/diabetic_retinopathy,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
282
+ ,,,flickr30k,ViT-B-16-plus-240,laion400m_e32,zeroshot_retrieval,,0.8894000053405762,0.9710000157356262,ViT-B-16-plus-240 laion400m_e32
283
+ ,,,flickr8k,ViT-B-32,openai,zeroshot_retrieval,,0.805400013923645,0.9139999747276306,ViT-B-32 openai
284
+ 0.5897858319604613,,0.5895015488390944,renderedsst2,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
285
+ 0.5388,0.8212,0.5362716338708938,imagenet-a,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
286
+ ,,,mscoco_captions,ViT-B-16,openai,zeroshot_retrieval,,0.5836865305900574,0.7681999802589417,ViT-B-16 openai
287
+ ,,,flickr8k,ViT-B-32,laion2b_s34b_b79k,zeroshot_retrieval,,0.8629999756813049,0.9409999847412109,ViT-B-32 laion2b_s34b_b79k
288
+ 0.99425,0.999875,0.9945,stl10,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
289
+ 0.5612707437000615,0.8849492931776275,0.5565312569182044,vtab/svhn,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
290
+ 0.1097119341563786,0.5409053497942387,0.1081320376350696,vtab/smallnorb_label_elevation,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
291
+ 0.805221688034188,0.9551282051282052,0.8491537874687232,voc2007,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
292
+ ,,,flickr8k,ViT-L-14-336,openai,zeroshot_retrieval,,0.8795999884605408,0.9390000104904175,ViT-L-14-336 openai
293
+ 0.1716,0.9095333333333332,0.1619443104834767,vtab/clevr_closest_object_distance,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
294
+ 0.1573333333333333,0.6964666666666667,0.1495026928557915,vtab/clevr_count_all,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
295
+ ,,,mscoco_captions,ViT-B-16-plus-240,laion400m_e32,zeroshot_retrieval,,0.6620951890945435,0.8101999759674072,ViT-B-16-plus-240 laion400m_e32
296
+ 0.8336,0.9666,0.8325000000000001,vtab/cifar100,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
297
+ 0.0984362139917695,0.5293827160493827,0.0977694426163014,vtab/smallnorb_label_elevation,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
298
+ ,,,voc2007_multilabel,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,0.8066232800483704,,,ViT-g-14 laion2b_s12b_b42k
299
+ 0.6087301587301587,0.9147619047619048,0.6152225643499313,vtab/resisc45,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
300
+ 0.0550617283950617,0.2645267489711934,0.055379474051494,vtab/smallnorb_label_azimuth,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
301
+ 0.1676666666666666,0.8819333333333333,0.1954414937508546,vtab/clevr_closest_object_distance,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
302
+ 0.0200330946180555,0.1053195529513889,0.0222572574909185,vtab/dsprites_label_orientation,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
303
+ 0.5526123046875,,0.5526478181769814,vtab/pcam,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
304
+ ,,,mscoco_captions,ViT-L-14,openai,zeroshot_retrieval,,0.6108356714248657,0.7918000221252441,ViT-L-14 openai
305
+ ,,,mscoco_captions,ViT-B-32-quickgelu,laion400m_e32,zeroshot_retrieval,,0.6084766387939453,0.7675999999046326,ViT-B-32-quickgelu laion400m_e32
306
+ 0.7537811026183119,0.8923402179216132,0.7256696912813558,vtab/flowers,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
307
+ 0.1659634317862166,,0.3247233185334074,vtab/kitti_closest_vehicle_distance,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
308
+ 0.0193684895833333,0.1180826822916666,0.0197744129948227,vtab/dsprites_label_orientation,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
309
+ 0.9037884982284,0.9970019078768056,0.9039815014388496,vtab/pets,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
310
+ 0.3742,0.7294,0.3706020613065869,mnist,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
311
+ 0.6746527403897922,0.8668030580381177,0.6650572756513145,objectnet,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
312
+ 0.765625,0.959869123931624,0.8071517771314477,voc2007,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
313
+ 0.546031746031746,0.902063492063492,0.5542849348347576,vtab/resisc45,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
314
+ 0.0427924262152777,0.1601019965277778,0.0430371102717507,vtab/dsprites_label_x_position,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
315
+ 0.6957142857142857,0.9571428571428572,0.706242238089474,vtab/resisc45,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
316
+ 0.6514814814814814,0.9551851851851852,0.6638062361650154,vtab/eurosat,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
317
+ 0.8475333333333334,0.9550666666666666,0.8331685531673508,imagenet-r,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
318
+ 0.1902,0.9085333333333332,0.1387289271305892,vtab/clevr_closest_object_distance,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
319
+ 0.3852181929932391,0.7822295636140135,0.379296565517112,vtab/svhn,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
320
+ 0.3182464454976303,0.5937914691943128,0.3175829383886256,country211,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
321
+ 0.6659,0.9224,0.6665327286676481,mnist,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
322
+ 0.6327693607655879,0.8639588123170037,0.6325447058823529,imagenet_sketch,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
323
+ 0.2279620853080568,0.486303317535545,0.2282938388625592,country211,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
324
+ 0.764,0.9231,0.7589335620721703,mnist,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
325
+ 0.0268283420138888,0.1092258029513889,0.025387675404758,vtab/dsprites_label_orientation,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
326
+ 0.3693369336933693,0.744974497449745,0.3649286987522281,fgvc_aircraft,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
327
+ 0.4102258758451137,0.8022818070067609,0.4216815643098484,vtab/svhn,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
328
+ 0.743319785938908,0.9583279695459478,0.7348385903018446,sun397,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
329
+ ,,,flickr8k,ViT-H-14,laion2b_s32b_b79k,zeroshot_retrieval,,0.9277999997138977,0.9729999899864197,ViT-H-14 laion2b_s32b_b79k
330
+ 0.5487,0.8411,0.5430718178404617,mnist,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
331
+ 0.7171428571428572,0.958095238095238,0.7258469461953507,vtab/resisc45,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
332
+ 0.3454976303317535,0.6221800947867299,0.3445971563981042,country211,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
333
+ 0.9645,0.999375,0.965125,stl10,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
334
+ 0.779,0.9289333333333334,0.7643246651538985,imagenet-r,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
335
+ 0.1633333333333333,0.7125333333333334,0.1575975877364315,vtab/clevr_count_all,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
336
+ 0.8683565004088307,0.9934587080948488,0.8661667839491306,vtab/pets,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
337
+ 0.647962962962963,0.9805555555555556,0.6445566610022502,vtab/eurosat,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
338
+ 0.6929582045861116,0.8668076109936576,0.6668176645957112,vtab/flowers,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
339
+ 0.6630346397788258,0.8573751829565783,0.6645264657992297,vtab/flowers,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
340
+ 0.1772666666666666,0.7803333333333333,0.2270014558851883,vtab/clevr_closest_object_distance,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
341
+ 0.2784,0.8929333333333334,0.2563239722391655,vtab/clevr_count_all,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
342
+ 0.8328402366863905,0.94543063773833,0.9085289082247568,vtab/caltech101,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
343
+ 0.2786954517516902,0.7308312845728334,0.2795999671407248,vtab/svhn,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
344
+ 0.035400390625,0.1624348958333333,0.0364155761076967,vtab/dsprites_label_x_position,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
345
+ 0.2699554722287321,1.0,0.2211405088000341,vtab/diabetic_retinopathy,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
346
+ 0.0259874131944444,0.1214463975694444,0.0262399607702056,vtab/dsprites_label_orientation,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
347
+ 0.819197896120973,0.9465811965811964,0.8786521640800292,vtab/caltech101,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
348
+ 0.4896907216494845,0.9721370855391472,0.4887152232577444,fer2013,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
349
+ ,,,voc2007_multilabel,ViT-L-14,laion400m_e32,zeroshot_classification,0.7847012877464294,,,ViT-L-14 laion400m_e32
350
+ 0.8020816392909416,0.9253537160513904,0.7985485908748879,vtab/flowers,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
351
+ 0.5163895486935867,0.7605700712589074,0.4472134621454642,gtsrb,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
352
+ 0.5255354200988468,,0.5258471570841294,renderedsst2,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
353
+ 0.2242797448867385,0.8738508906971629,0.1816019549723685,vtab/dmlab,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
354
+ 0.5442040519562185,0.8052231327005837,0.5451807843137254,imagenet_sketch,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
355
+ 0.545867919921875,,0.5459467330999297,vtab/pcam,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
356
+ 0.697049639280715,0.877516959190266,0.6846202591899297,objectnet,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
357
+ 0.0558847736625514,0.2832098765432099,0.0522661133926831,vtab/smallnorb_label_azimuth,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
358
+ 0.2352,0.7871333333333334,0.2199570643185355,vtab/clevr_count_all,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
359
+ 0.8039,0.9392333333333334,0.7907448605291261,imagenet-r,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
360
+ 0.2292545710267229,,0.3081761474014508,vtab/kitti_closest_vehicle_distance,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
361
+ 0.04559670781893,0.274320987654321,0.0456436809466583,vtab/smallnorb_label_azimuth,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
362
+ 0.585723876953125,,0.5857677281000415,vtab/pcam,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
363
+ 0.8932,0.9735333333333334,0.8804663010091829,imagenet-r,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
364
+ 0.3677333333333333,0.7024,0.3814206048810433,imagenet-a,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
365
+ 0.378037803780378,0.8028802880288028,0.3781639928698752,fgvc_aircraft,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
366
+ 0.5430851063829787,0.8356382978723405,0.5473404255319149,vtab/dtd,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
367
+ 0.9261285909712722,0.9976371098122124,0.9261518670531818,cars,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
368
+ 0.6904,0.9518,0.6833555055021581,mnist,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
369
+ 0.6174603174603175,0.921904761904762,0.624279367877562,vtab/resisc45,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
370
+ 0.537057676232934,0.9492894956812482,0.5338791359180816,fer2013,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
371
+ ,,,voc2007_multilabel,ViT-B-16,laion400m_e32,zeroshot_classification,0.7843208312988281,,,ViT-B-16 laion400m_e32
372
+ 0.4582056283087211,0.9531903037057676,0.4167779537443768,fer2013,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
373
+ 0.1594666666666666,0.8705333333333334,0.1702024394180443,vtab/clevr_closest_object_distance,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
374
+ 0.955,0.9995,0.955375,stl10,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
375
+ 0.7640666666666667,0.9159,0.7521727338627011,imagenet-r,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
376
+ 0.8328402366863905,0.952827087442472,0.9177765781998192,vtab/caltech101,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
377
+ 0.53631591796875,,0.53614377703907,vtab/pcam,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
378
+ 0.0517695473251028,0.2697942386831276,0.0537673960190341,vtab/smallnorb_label_azimuth,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
379
+ 0.6453,0.888,0.6451,vtab/cifar100,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
380
+ ,,,flickr30k,ViT-B-32,laion2b_e16,zeroshot_retrieval,,0.8812000155448914,0.9639999866485596,ViT-B-32 laion2b_e16
381
+ 0.4393776246365888,0.684612899752342,0.4268760394558317,objectnet,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
382
+ ,,,flickr30k,ViT-g-14,laion2b_s12b_b42k,zeroshot_retrieval,,0.9348000288009644,0.99099999666214,ViT-g-14 laion2b_s12b_b42k
383
+ 0.381760909649662,0.7632913337430854,0.4057750407393451,vtab/svhn,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
384
+ 0.5006666666666667,0.8033333333333333,0.4832835476168289,imagenet-a,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
385
+ 0.9705,0.9994,0.9711,vtab/cifar10,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
386
+ 0.4800780161604904,0.9711618835330176,0.4909158276198425,fer2013,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
387
+ ,,,mscoco_captions,ViT-L-14-336,openai,zeroshot_retrieval,,0.615513801574707,0.8101999759674072,ViT-L-14-336 openai
388
+ 0.7068,0.9062666666666668,0.6753602288814418,imagenet-a,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
389
+ 0.9078,0.9977,0.9083,vtab/cifar10,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
390
+ 0.3860633066994468,0.8061232329440688,0.3685311302620919,vtab/svhn,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
391
+ 0.7160513904699951,0.8744511302650837,0.6998995164388323,vtab/flowers,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
392
+ 0.1971197119711971,0.5022502250225023,0.197344028520499,fgvc_aircraft,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
393
+ 0.6033343577135832,0.8973570989551322,0.5683458085959752,vtab/svhn,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
394
+ 0.4899860019381932,0.7341983417680629,0.4820376550831692,objectnet,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
395
+ 0.7626201923076923,0.9526575854700856,0.815718996924191,voc2007,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
396
+ 0.1462728551336146,,0.1818997858588953,vtab/kitti_closest_vehicle_distance,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
397
+ 0.6787234042553192,0.923936170212766,0.6813829787234043,vtab/dtd,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
398
+ 0.1721574664614031,0.8585001099626127,0.1577427840925754,vtab/dmlab,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
399
+ 0.7916734428362335,0.9193364774760124,0.7931691849985836,vtab/flowers,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
400
+ 0.0549794238683127,0.2670781893004115,0.0559609415916413,vtab/smallnorb_label_azimuth,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
401
+ 0.7843883547008547,0.9570646367521368,0.835061321772101,voc2007,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
402
+ 0.6177777777777778,0.957037037037037,0.6299267597724122,vtab/eurosat,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
403
+ 0.543701171875,,0.5435553179465629,vtab/pcam,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
404
+ 0.0315348307291666,0.1596544053819444,0.0323590170008084,vtab/dsprites_label_x_position,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
405
+ 0.5796296296296296,0.9525925925925924,0.5888803943202612,vtab/eurosat,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
406
+ 0.9185064050149904,0.9956391387298992,0.9161666963118203,vtab/pets,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
407
+ 0.5381177990739744,0.7715085603531818,0.5274232792623416,objectnet,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
408
+ ,,,flickr30k,ViT-L-14,laion2b_s32b_b82k,zeroshot_retrieval,,0.929199993610382,0.9869999885559082,ViT-L-14 laion2b_s32b_b82k
409
+ 0.8368362144011939,0.987439373212287,0.8375394945435981,cars,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
410
+ ,,,flickr8k,ViT-B-32-quickgelu,laion400m_e32,zeroshot_retrieval,,0.8303999900817871,0.9169999957084656,ViT-B-32-quickgelu laion400m_e32
411
+ 0.9742,0.9994,0.9742999999999998,vtab/cifar10,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
412
+ 0.613015873015873,0.91,0.6150226494017114,vtab/resisc45,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
413
+ 0.1901913349461183,0.8472839234660215,0.1733383864929211,vtab/dmlab,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
414
+ 0.1979333333333333,0.8076,0.1820160989597282,vtab/clevr_count_all,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
415
+ ,,,voc2007_multilabel,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,0.8199208974838257,,,ViT-L-14 laion2b_s32b_b82k
416
+ 0.1674,0.8695333333333334,0.1827530663763387,vtab/clevr_closest_object_distance,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
417
+ 0.1715165876777251,0.4024170616113744,0.1707582938388625,country211,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
418
+ 0.75202,0.94252,0.7526400000000001,imagenet1k,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
419
+ 0.8923412373943854,0.9967293540474244,0.8919759072052595,vtab/pets,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
420
+ 0.487670937870141,0.7249919241951115,0.4750858280106649,objectnet,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
421
+ 0.8959084690958836,0.9960203954732,0.8961635505798664,cars,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
422
+ 0.7832532051282052,0.9692174145299144,0.864352156405908,voc2007,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
423
+ 0.5031481481481481,0.92,0.5110567650187513,vtab/eurosat,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
424
+ 0.161,0.9128666666666668,0.1739135544326629,vtab/clevr_closest_object_distance,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
425
+ 0.8441736102474816,0.9912946150976246,0.8456794155511405,cars,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
426
+ 0.7372,0.9338,0.7372000000000001,vtab/cifar100,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
427
+ 0.7136119694259229,0.8591640917222313,0.691284904068223,vtab/flowers,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
428
+ 0.1485814822960193,0.8129316032548933,0.1482313831309264,vtab/dmlab,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
429
+ 0.6112026359143328,,0.6113264286955011,renderedsst2,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
430
+ 0.5370675453047776,,0.5373060332349024,renderedsst2,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
431
+ ,,,mscoco_captions,ViT-B-32,openai,zeroshot_retrieval,,0.5584565997123718,0.748199999332428,ViT-B-32 openai
432
+ ,,,flickr8k,ViT-B-16,openai,zeroshot_retrieval,,0.8285999894142151,0.9139999747276306,ViT-B-16 openai
433
+ 0.8379355687047995,0.9518408941485864,0.9328934325841473,vtab/caltech101,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
434
+ 0.0307752821180555,0.1437852647569444,0.0304426618193977,vtab/dsprites_label_orientation,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
435
+ 0.841715976331361,0.940828402366864,0.9341112975275198,vtab/caltech101,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
436
+ 0.0311008029513888,0.1568060980902778,0.0321637232587243,vtab/dsprites_label_x_position,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
437
+ ,,,flickr30k,ViT-L-14-336,openai,zeroshot_retrieval,,0.8889999985694885,0.9810000061988832,ViT-L-14-336 openai
438
+ 0.5961406197803062,0.8388060288077973,0.5956784313725489,imagenet_sketch,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
439
+ 0.4657286152131513,0.9445528002229032,0.4812190866355833,fer2013,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
440
+ 0.0562962962962962,0.274156378600823,0.0567608380809354,vtab/smallnorb_label_azimuth,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
441
+ 0.1583333333333333,0.8006,0.1676244908434004,vtab/clevr_closest_object_distance,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
442
+ 0.7326224513709867,1.0,0.2068672210509669,vtab/diabetic_retinopathy,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
443
+ 0.8911666666666667,0.9757333333333332,0.8776131879598171,imagenet-r,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
444
+ 0.988625,0.999875,0.988625,stl10,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
445
+ 0.6865218750574692,0.9403608143148756,0.6921311150610732,sun397,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
446
+ 0.5509033203125,,0.5508520491113902,vtab/pcam,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
447
+ 0.1606666666666666,0.9079333333333334,0.1772310296867651,vtab/clevr_closest_object_distance,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
448
+ 0.5574468085106383,0.8648936170212767,0.5622340425531915,vtab/dtd,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
449
+ 0.2315333333333333,0.805,0.2332901934437162,vtab/clevr_count_all,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
450
+ 0.0298394097222222,0.1414794921875,0.0308127750442299,vtab/dsprites_label_x_position,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
451
+ 0.0623868312757201,0.2708641975308642,0.0631906837062103,vtab/smallnorb_label_azimuth,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
452
+ 0.4774310392867094,0.9402340484814712,0.4649079545536754,fer2013,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
453
+ 0.5230010414824422,0.7913694511584036,0.5233670588235293,imagenet_sketch,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
454
+ 0.7341666666666666,0.9035,0.7214778957504825,imagenet-r,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
455
+ 0.8353057199211046,0.9546351084812624,0.9005042720791782,vtab/caltech101,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
456
+ 0.3109333333333333,0.8006,0.3066391557930237,vtab/clevr_count_all,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
457
+ 0.6985172981878089,,0.6986603265589717,renderedsst2,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
458
+ 0.1087242798353909,0.540164609053498,0.1086253991970707,vtab/smallnorb_label_elevation,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
459
+ 0.9351,0.998,0.936,vtab/cifar10,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
460
+ ,,,flickr8k,ViT-L-14,openai,zeroshot_retrieval,,0.8633999824523926,0.9409999847412109,ViT-L-14 openai
461
+ 0.65528,0.894,0.6563199999999999,imagenet1k,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
462
+ 0.5930807248764415,,0.5925413265010712,renderedsst2,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
463
+ 0.1650710900473933,0.3788625592417061,0.164218009478673,country211,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
464
+ 0.168016801680168,0.4119411941194119,0.1658110516934046,fgvc_aircraft,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
465
+ 0.0303819444444444,0.1311170789930555,0.0332405586865836,vtab/dsprites_label_orientation,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
466
+ 0.3171317131713171,0.7827782778277828,0.3170053475935828,fgvc_aircraft,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
467
+ 0.6691,0.8925,0.6685000000000001,vtab/cifar100,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
468
+ 0.0516872427983539,0.2725925925925926,0.0526600212836742,vtab/smallnorb_label_azimuth,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
469
+ 0.6559,0.8854,0.6541,imagenetv2,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
470
+ 0.5569148936170213,0.8558510638297873,0.5542553191489361,vtab/dtd,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
471
+ 0.1663981042654028,0.3825118483412322,0.1670142180094786,country211,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
472
+ 0.7074,0.9179,0.7075,imagenetv2,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
473
+ 0.3137333333333333,0.6410666666666667,0.3236449813371542,imagenet-a,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
474
+ 0.0454320987654321,0.2672427983539094,0.0450240211431134,vtab/smallnorb_label_azimuth,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
475
+ 0.8741,0.9655666666666668,0.8599962924086103,imagenet-r,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
476
+ 0.4275427542754275,0.8361836183618362,0.4260962566844919,fgvc_aircraft,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
477
+ ,,,mscoco_captions,ViT-L-14,laion2b_s32b_b82k,zeroshot_retrieval,,0.7107957005500793,0.8399999737739563,ViT-L-14 laion2b_s32b_b82k
478
+ 0.5958,0.8547,0.5955999999999999,imagenetv2,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
479
+ 0.7540228405391985,0.9617025580668296,0.7523924485563404,sun397,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
480
+ 0.2418241824182418,0.6054605460546054,0.2405525846702317,fgvc_aircraft,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
481
+ 0.0590123456790123,0.279835390946502,0.0603763349553147,vtab/smallnorb_label_azimuth,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
482
+ 0.7613,0.9289,0.7611,vtab/cifar100,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
483
+ 0.9277453053102848,0.9983832856609874,0.9288577913034778,cars,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
484
+ 0.0257297092013888,0.1208224826388889,0.0259727501505128,vtab/dsprites_label_orientation,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
485
+ 0.8254437869822485,0.954963839579224,0.908884143116176,vtab/caltech101,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
486
+ 0.980125,0.999875,0.980875,stl10,ViT-L-14,laion400m_e32,zeroshot_classification,,,,ViT-L-14 laion400m_e32
487
+ 0.2463246324632463,0.5724572457245725,0.2460249554367201,fgvc_aircraft,ViT-B-32,laion2b_s34b_b79k,zeroshot_classification,,,,ViT-B-32 laion2b_s34b_b79k
488
+ 0.8102964743589743,0.9655448717948718,0.8579252085748035,voc2007,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
489
+ 0.2044,0.7866666666666666,0.2151124081246672,vtab/clevr_count_all,ViT-B-16,openai,zeroshot_classification,,,,ViT-B-16 openai
490
+ 0.2872511848341232,0.542085308056872,0.2880094786729857,country211,ViT-g-14,laion2b_s12b_b42k,zeroshot_classification,,,,ViT-g-14 laion2b_s12b_b42k
491
+ 0.06,0.288312757201646,0.0630119225348046,vtab/smallnorb_label_azimuth,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
492
+ 0.9664,0.9987,0.9665,vtab/cifar10,ViT-L-14,laion2b_s32b_b82k,zeroshot_classification,,,,ViT-L-14 laion2b_s32b_b82k
493
+ 0.1895333333333333,0.7248666666666667,0.1870254885776544,vtab/clevr_count_all,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
494
+ 0.8415619947767691,0.9900509886829996,0.8435641961196442,cars,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
495
+ 0.337965783923131,1.0,0.2591163084325771,vtab/diabetic_retinopathy,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
496
+ 0.1582,0.8814666666666666,0.1812267071156741,vtab/clevr_closest_object_distance,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
497
+ 0.0475720164609053,0.2691358024691358,0.0457953163680852,vtab/smallnorb_label_azimuth,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
498
+ 0.3651623119556611,0.7007917656373713,0.3512103003164681,gtsrb,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
499
+ 0.1592258632065097,0.8009676709918627,0.1713416703583154,vtab/dmlab,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
500
+ 0.4948535233570863,0.7578780680918448,0.4319824074083427,gtsrb,ViT-B-16-plus-240,laion400m_e32,zeroshot_classification,,,,ViT-B-16-plus-240 laion400m_e32
501
+ 0.3294329432943294,0.7836783678367837,0.3317290552584671,fgvc_aircraft,ViT-L-14-336,openai,zeroshot_classification,,,,ViT-L-14-336 openai
502
+ 0.684701252367729,0.9328576420177648,0.6783238400960281,sun397,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
503
+ 0.6885672467067816,0.8466417303626605,0.6732279033655394,vtab/flowers,ViT-B-32,laion2b_e16,zeroshot_classification,,,,ViT-B-32 laion2b_e16
504
+ 0.8385930309007232,0.9539776462853384,0.9334530557615972,vtab/caltech101,ViT-L-14,openai,zeroshot_classification,,,,ViT-L-14 openai
505
+ 0.6696489324530592,0.9273222134358275,0.6609448269493161,sun397,ViT-B-32-quickgelu,laion400m_e32,zeroshot_classification,,,,ViT-B-32-quickgelu laion400m_e32
506
+ 0.3000947867298578,0.556872037914692,0.2994312796208531,country211,ViT-H-14,laion2b_s32b_b79k,zeroshot_classification,,,,ViT-H-14 laion2b_s32b_b79k
507
+ 0.409445528002229,0.9413485650599052,0.3587300457745208,fer2013,ViT-B-32,openai,zeroshot_classification,,,,ViT-B-32 openai
508
+ 0.96975,0.999875,0.96975,stl10,ViT-B-16,laion400m_e32,zeroshot_classification,,,,ViT-B-16 laion400m_e32
CLIP_benchmark/benchmark/dataset_type.csv ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ dataset,type
2
+ imagenet1k,natural
3
+ imagenetv2,natural
4
+ imagenet-r,natural
5
+ imagenet_sketch,specialized
6
+ objectnet,natural
7
+ imagenet-a,natural
8
+ imagenet-o,natural
9
+ vtab/cifar10,natural
10
+ vtab/cifar100,natural
11
+ mnist,specialized
12
+ vtab/flowers,natural
13
+ cars,natural
14
+ vtab/svhn,natural
15
+ fer2013,natural
16
+ renderedsst2,specialized
17
+ vtab/pets,natural
18
+ vtab/caltech101,natural
19
+ voc2007_multilabel,natural
20
+ voc2007,natural
21
+ sun397,natural
22
+ fgvc_aircraft,natural
23
+ country211,natural
24
+ vtab/dtd,natural
25
+ gtsrb,natural
26
+ stl10,natural
27
+ vtab/diabetic_retinopathy,specialized
28
+ vtab/eurosat,specialized
29
+ vtab/resisc45,specialized
30
+ vtab/pcam,specialized
31
+ vtab/clevr_count_all,structured
32
+ vtab/clevr_closest_object_distance,structured
33
+ vtab/dsprites_label_orientation,structured
34
+ vtab/dsprites_label_x_position,structured
35
+ vtab/dsprites_label_y_position,structured
36
+ vtab/smallnorb_label_elevation,structured
37
+ vtab/smallnorb_label_azimuth,structured
38
+ vtab/dmlab,structured
39
+ vtab/kitti_closest_vehicle_distance,structured
40
+ mscoco_captions,retrieval
41
+ flickr8k,retrieval
42
+ flickr30k,retrieval
CLIP_benchmark/benchmark/datasets.txt ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ wds/vtab/cifar10
2
+ wds/stl10
3
+ wds/vtab/cifar100
4
+ wds/cars
5
+ wds/vtab/caltech101
6
+ wds/vtab/pets
7
+ wds/vtab/flowers
8
+ wds/vtab/dtd
9
+ wds/vtab/eurosat
10
+ wds/fgvc_aircraft
11
+ wds/vtab/pcam
12
+ wds/imagenet-r
13
+ wds/imagenet_sketch
CLIP_benchmark/benchmark/datasets_multilingual.txt ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ multilingual_mscoco_captions,es
2
+ multilingual_mscoco_captions,it
3
+ multilingual_mscoco_captions,ko
4
+ multilingual_mscoco_captions,pl
5
+ multilingual_mscoco_captions,ru
6
+ multilingual_mscoco_captions,tr
7
+ multilingual_mscoco_captions,zh
8
+ multilingual_mscoco_captions,en
9
+ imagenet1k,zh
10
+ imagenet1k,it
11
+ imagenet1k,jp
12
+ imagenet1k,en
13
+ imagenet1k,ar
CLIP_benchmark/benchmark/models.txt ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ ViT-L-14,openai
2
+ ViT-L-14,/path/to/fare_eps_4.pt
CLIP_benchmark/benchmark/webdatasets.txt ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ wds/mscoco_captions
2
+ wds/flickr8k
3
+ wds/flickr30k
4
+ wds/imagenet1k
5
+ wds/imagenetv2
6
+ wds/imagenet_sketch
7
+ wds/imagenet-a
8
+ wds/imagenet-r
9
+ wds/imagenet-o
10
+ wds/objectnet
11
+ wds/fer2013
12
+ wds/voc2007
13
+ wds/voc2007_multilabel
14
+ wds/sun397
15
+ wds/cars
16
+ wds/fgvc_aircraft
17
+ wds/mnist
18
+ wds/stl10
19
+ wds/gtsrb
20
+ wds/country211
21
+ wds/renderedsst2
22
+ wds/vtab/caltech101
23
+ wds/vtab/cifar10
24
+ wds/vtab/cifar100
25
+ wds/vtab/clevr_count_all
26
+ wds/vtab/clevr_closest_object_distance
27
+ wds/vtab/diabetic_retinopathy
28
+ wds/vtab/dmlab
29
+ wds/vtab/dsprites_label_orientation
30
+ wds/vtab/dsprites_label_x_position
31
+ wds/vtab/dsprites_label_y_position
32
+ wds/vtab/dtd
33
+ wds/vtab/eurosat
34
+ wds/vtab/kitti_closest_vehicle_distance
35
+ wds/vtab/flowers
36
+ wds/vtab/pets
37
+ wds/vtab/pcam
38
+ wds/vtab/resisc45
39
+ wds/vtab/smallnorb_label_azimuth
40
+ wds/vtab/smallnorb_label_elevation
41
+ wds/vtab/svhn
CLIP_benchmark/clip_benchmark/__init__.py ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ """Top-level package for CLIP Benchmark."""
2
+
3
+ __author__ = """Mehdi Cherti"""
4
+ __email__ = '[email protected]'
5
+ __version__ = '0.1.0'
CLIP_benchmark/clip_benchmark/cli.py ADDED
@@ -0,0 +1,404 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Console script for clip_benchmark."""
2
+ import argparse
3
+ import random
4
+ import sys
5
+ import json
6
+ from collections import defaultdict
7
+
8
+ import numpy as np
9
+ import torch
10
+ import csv
11
+ from copy import copy
12
+ import os
13
+ from torchvision import transforms
14
+ from torchvision.transforms.transforms import Compose, Resize
15
+ from clip_benchmark.datasets.builder import build_dataset, get_dataset_collate_fn, get_dataset_default_task, dataset_collection, get_dataset_collection_from_file
16
+ from clip_benchmark.metrics import image_caption_selection, zeroshot_classification, zeroshot_retrieval, linear_probe, captioning
17
+ from clip_benchmark.model_collection import get_model_collection_from_file, model_collection
18
+ from clip_benchmark.models import load_clip, MODEL_TYPES
19
+
20
+ def get_parser_args():
21
+ parser = argparse.ArgumentParser()
22
+ subparsers = parser.add_subparsers()
23
+
24
+ parser_eval = subparsers.add_parser('eval', help='Evaluate')
25
+ parser_eval.add_argument('--dataset', type=str, default="cifar10", nargs="+", help="Dataset(s) to use for the benchmark. Can be the name of a dataset, or a collection name ('vtab', 'vtab+', 'imagenet_robustness', 'retrieval') or path of a text file where each line is a dataset name")
26
+ parser_eval.add_argument('--dataset_root', default="root", type=str, help="dataset root folder where the datasets are downloaded. Can be in the form of a template depending on dataset name, e.g., --dataset_root='datasets/{dataset}'. This is useful if you evaluate on multiple datasets.")
27
+ parser_eval.add_argument('--split', type=str, default="test", help="Dataset split to use")
28
+ parser_eval.add_argument('--model', type=str, default="ViT-B-32-quickgelu", help="Model architecture to use from OpenCLIP")
29
+ parser_eval.add_argument('--pretrained', type=str, default="laion400m_e32", help="Model checkpoint name to use from OpenCLIP")
30
+ parser_eval.add_argument('--pretrained_model', type=str, default="", nargs="+", help="Pre-trained model(s) to use. Can be the full model name where `model` and `pretrained` are comma separated (e.g., --pretrained_model='ViT-B-32-quickgelu,laion400m_e32'), a model collection name ('openai' or 'openclip_base' or 'openclip_multilingual' or 'openclip_all'), or path of a text file where each line is a model fullname where model and pretrained are comma separated (e.g., ViT-B-32-quickgelu,laion400m_e32). --model and --pretrained are ignored if --pretrained_model is used.")
31
+ parser_eval.add_argument('--task', type=str, default="auto", choices=["zeroshot_classification", "zeroshot_retrieval", "linear_probe", "captioning", "image_caption_selection", "auto"], help="Task to evaluate on. With --task=auto, the task is automatically inferred from the dataset.")
32
+ parser_eval.add_argument('--no_amp', action="store_false", dest="amp", default=False, help="whether to use mixed precision") # we set default to False, as we don't want amp for attacks
33
+ parser_eval.add_argument('--num_workers', default=4, type=int)
34
+ parser_eval.add_argument('--recall_k', default=[5], type=int, help="for retrieval, select the k for Recall@K metric. ", nargs="+",)
35
+ parser_eval.add_argument('--fewshot_k', default=-1, type=int, help="for linear probe, how many shots. -1 = whole dataset.")
36
+ parser_eval.add_argument('--fewshot_epochs', default=10, type=int, help="for linear probe, how many epochs.")
37
+ parser_eval.add_argument('--fewshot_lr', default=0.1, type=float, help="for linear probe, what is the learning rate.")
38
+ parser_eval.add_argument("--skip_load", action="store_true", help="for linear probes, when everything is cached, no need to load model.")
39
+ parser_eval.add_argument('--seed', default=0, type=int, help="random seed.")
40
+ parser_eval.add_argument('--batch_size', default=64, type=int)
41
+ parser_eval.add_argument('--model_cache_dir', default=None, type=str, help="directory to where downloaded models are cached")
42
+ parser_eval.add_argument('--feature_root', default="features", type=str, help="feature root folder where the features are stored.")
43
+ parser_eval.add_argument('--annotation_file', default="", type=str, help="text annotation file for retrieval datasets. Only needed for when `--task` is `zeroshot_retrieval`.")
44
+ parser_eval.add_argument('--custom_classname_file', default=None, type=str, help="use custom json file with classnames for each dataset, where keys are dataset names and values are list of classnames.")
45
+ parser_eval.add_argument('--custom_template_file', default=None, type=str, help="use custom json file with prompts for each dataset, where keys are dataset names and values are list of prompts. For instance, to use CuPL prompts, use --custom_template_file='cupl_prompts.json'")
46
+
47
+ parser_eval.add_argument('--language', default="en", type=str, nargs="+", help="language(s) of classname and prompts to use for zeroshot classification.")
48
+ parser_eval.add_argument('--output', default="result.json", type=str, help="output file where to dump the metrics. Can be in form of a template, e.g., --output='{dataset}_{pretrained}_{model}_{language}_{task}.json'")
49
+ parser_eval.add_argument('--quiet', dest='verbose', action="store_false", help="suppress verbose messages")
50
+ parser_eval.add_argument('--save_clf', default=None, type=str, help="optionally save the classification layer output by the text tower")
51
+ parser_eval.add_argument('--load_clfs', nargs='+', default=[], type=str, help="optionally load and average mutliple layers output by text towers.")
52
+ parser_eval.add_argument('--skip_existing', default=False, action="store_true", help="whether to skip an evaluation if the output file exists.")
53
+ parser_eval.add_argument('--model_type', default="open_clip", type=str, choices=MODEL_TYPES, help="clip model type")
54
+ parser_eval.add_argument('--wds_cache_dir', default=None, type=str, help="optional cache directory for webdataset only")
55
+ parser_eval.add_argument('--n_samples', default=-1, type=int, help="number of samples to evaluate on. -1 = whole dataset.", choices=[-1, 11, 1000])
56
+
57
+ parser_eval.add_argument('--interpolate', default=False, action="store_true", help="interpolate with clean model")
58
+ parser_eval.add_argument('--beta', default=0.5, type=float, help="interpolate with clean model, 0=clean")
59
+ parser_eval.add_argument('--attack', default='none', type=str, help="attack to use", choices=['none', 'aa'])
60
+ parser_eval.add_argument('--norm', default='Linf', type=str, help="norm to use")
61
+ parser_eval.add_argument('--eps', default=1., type=float, help="epsilon to use")
62
+ parser_eval.add_argument('--iterations_adv', default=100, type=int, help="number of attack iterations to use")
63
+
64
+ parser_eval.set_defaults(which='eval')
65
+ parser_build = subparsers.add_parser('build', help='Build CSV from evaluations')
66
+ parser_build.add_argument('files', type=str, nargs="+", help="path(s) of JSON result files")
67
+ parser_build.add_argument('--output', type=str, default="benchmark.csv", help="CSV output file")
68
+ parser_build.set_defaults(which='build')
69
+
70
+ args = parser.parse_args()
71
+ return args
72
+
73
+ def main():
74
+ base = get_parser_args()
75
+ if base.which == "eval":
76
+ main_eval(base)
77
+ elif base.which == "build":
78
+ main_build(base)
79
+
80
+ def main_build(base):
81
+ # Build a benchmark single CSV file from a set of evaluations (JSON files)
82
+ rows = []
83
+ fieldnames = set()
84
+ for path in base.files:
85
+ data = json.load(open(path))
86
+ row = {}
87
+ row.update(data["metrics"])
88
+ row.update(data)
89
+ del row["metrics"]
90
+ row['model_fullname'] = row['model'] + ' ' + row['pretrained']
91
+ for field in row.keys():
92
+ fieldnames.add(field)
93
+ rows.append(row)
94
+ with open(base.output, 'w') as csvfile:
95
+ writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
96
+ writer.writeheader()
97
+ for row in rows:
98
+ writer.writerow(row)
99
+
100
+ def main_eval(base):
101
+ # Get list of pre-trained models to evaluate
102
+ pretrained_model = _as_list(base.pretrained_model)
103
+ if pretrained_model:
104
+ models = []
105
+ for name in pretrained_model:
106
+ if os.path.isfile(name):
107
+ # if path, read file, each line is a pre-trained model
108
+ models.extend(get_model_collection_from_file(name))
109
+ elif name in model_collection:
110
+ # if part of `model_collection`, retrieve from it
111
+ models.extend(model_collection[name])
112
+ else:
113
+ # if not, assume it is in the form of `model,pretrained`
114
+ model, pretrained = name.split(',')
115
+ models.append((model, pretrained))
116
+ else:
117
+ models = [(base.model, base.pretrained)]
118
+
119
+ # Ge list of datasets to evaluate on
120
+ datasets = []
121
+ for name in _as_list(base.dataset):
122
+ if os.path.isfile(name):
123
+ # If path, read file, each line is a dataset name
124
+ datasets.extend(get_dataset_collection_from_file(name))
125
+ elif name in dataset_collection:
126
+ # if part of `dataset_collection`, retrieve from it
127
+ datasets.extend(dataset_collection[name])
128
+ else:
129
+ # if not, assume it is simply the name of the dataset
130
+ datasets.append(name)
131
+
132
+ # Get list of languages to evaluate on
133
+ languages = _as_list(base.language)
134
+
135
+ if base.verbose:
136
+ print(f"[Models] {models}")
137
+ print(f"[Datasets] {datasets}")
138
+ print(f"[Languages] {languages}")
139
+
140
+ for model, pretrained in models:
141
+ for i, dataset in enumerate(datasets):
142
+ print(f"\n{i+1} / {len(datasets)}")
143
+ for language in languages:
144
+ # We iterative over all possible model/dataset/languages
145
+ # TODO: possibility to parallelize evaluation here
146
+ args = copy(base)
147
+ args.model = model
148
+ args.pretrained = pretrained
149
+ args.dataset = dataset
150
+ args.language = language
151
+ run(args)
152
+
153
+ def _as_list(l):
154
+ if not l:
155
+ return []
156
+ return [l] if type(l) != list else l
157
+
158
+ def interpolate_state_dict(m1, beta):
159
+ m = {}
160
+
161
+ m2 = torch.load("/path/to/ckpt.pt", map_location='cpu')
162
+ for k in m1.keys():
163
+ # print(m1[k].shape, m2[k].shape)
164
+ m[k] = beta * m1[k] + (1 - beta) * m2[k]
165
+ return m
166
+
167
+ def run(args):
168
+ print("[args]", args, "\n")
169
+ """Console script for clip_benchmark."""
170
+ args.device = "cuda" if torch.cuda.is_available() else "cpu"
171
+ # set seed.
172
+ torch.manual_seed(args.seed)
173
+ random.seed(args.seed)
174
+ np.random.seed(args.seed)
175
+
176
+ task = args.task
177
+ if args.dataset.startswith("wds/"):
178
+ dataset_name = args.dataset.replace("wds/", "", 1)
179
+ elif args.dataset.startswith("#"):
180
+ print(f"Skip commented dataset {args.dataset}")
181
+ return
182
+ else:
183
+ dataset_name = args.dataset
184
+ if task == "auto":
185
+ task = get_dataset_default_task(dataset_name)
186
+ pretrained_slug = (
187
+ args.pretrained.split('/')[-1] if os.path.isfile(args.pretrained) else args.pretrained
188
+ )
189
+ if len(pretrained_slug) > 180:
190
+ pretrained_slug = pretrained_slug[140:]
191
+ pretrained_slug_full_path = args.pretrained.replace('/', '_') if os.path.isfile(args.pretrained) else args.pretrained
192
+ dataset_slug = dataset_name.replace('/', '_')
193
+ output = args.output.format(
194
+ model=args.model,
195
+ attack=args.attack,
196
+ eps=str(int(args.eps)),
197
+ iterations=args.iterations_adv,
198
+ pretrained=pretrained_slug,
199
+ pretrained_full_path=pretrained_slug_full_path,
200
+ task=task,
201
+ dataset=dataset_slug,
202
+ n_samples=args.n_samples,
203
+ language=args.language,
204
+ bs=args.batch_size,
205
+ beta=args.beta if args.interpolate else None,
206
+ )
207
+ if os.path.exists(output) and args.skip_existing:
208
+ if args.verbose:
209
+ print(f"Skip {output}, exists already.")
210
+ return
211
+ if args.verbose:
212
+ print(f"[Dataset] {args.dataset}")
213
+ print(f"[Task] {task} [model] {args.pretrained} [language] {args.language}")
214
+ print(f"[Output] {output}")
215
+ os.makedirs(os.path.dirname(output), exist_ok=True)
216
+ dataset_root = args.dataset_root.format(dataset=dataset_name, dataset_cleaned=dataset_name.replace("/", "-"))
217
+ if args.skip_load:
218
+ model, transform, collate_fn, dataloader = None, None, None, None
219
+ else:
220
+ if args.interpolate:
221
+ inter_dict = torch.load(args.pretrained, map_location=torch.device('cpu'))
222
+ inter_dict = interpolate_state_dict(inter_dict, args.beta)
223
+
224
+ model, transform, tokenizer = load_clip(
225
+ model_type=args.model_type,
226
+ model_name=args.model,
227
+ pretrained=args.pretrained if not args.interpolate else inter_dict,
228
+ cache_dir=args.model_cache_dir,
229
+ device=args.device
230
+ )
231
+ if ("cifar10" in args.dataset) or ("cifar100" in args.dataset) or ("stl10" in args.dataset):
232
+ # compute robustness wrt. original resolution
233
+ transform_unnorm = transforms.transforms.ToTensor()
234
+ resize = Resize(size=224, interpolation=transforms.InterpolationMode.BICUBIC, max_size=None, antialias=None)
235
+ else:
236
+ transform_unnorm = Compose(transform.transforms[:-1]) # remove normalize
237
+ resize = None
238
+ normalize = transform.transforms[-1]
239
+ del transform # make sure we don't use it by accident
240
+ print(f"[Transform unnorm] {transform_unnorm}")
241
+ print(f"[Normalize] {normalize}")
242
+
243
+ model.eval()
244
+ dataset = build_dataset(
245
+ dataset_name=args.dataset,
246
+ root=dataset_root,
247
+ transform=transform_unnorm,
248
+ split=args.split,
249
+ annotation_file=args.annotation_file,
250
+ download=True,
251
+ language=args.language,
252
+ task=task,
253
+ custom_template_file=args.custom_template_file,
254
+ custom_classname_file=args.custom_classname_file,
255
+ wds_cache_dir=args.wds_cache_dir,
256
+ )
257
+ if args.n_samples > 0:
258
+ dataset = dataset.shuffle(10000, initial=10000, rng=random.Random(args.seed))
259
+ collate_fn = get_dataset_collate_fn(args.dataset)
260
+ if args.verbose:
261
+ try:
262
+ print(f"Dataset size: {len(dataset)}")
263
+ except TypeError:
264
+ print("IterableDataset has no len()")
265
+ print(f"Dataset split: {args.split}")
266
+ if hasattr(dataset, "classes") and dataset.classes:
267
+ try:
268
+ print(f"Dataset classes: {dataset.classes[:20]}...")
269
+ print(f"Dataset number of classes: {len(dataset.classes)}")
270
+ except AttributeError:
271
+ print("Dataset has no classes.")
272
+
273
+ if args.dataset.startswith("wds/"):
274
+ if args.n_samples > 0:
275
+ assert args.batch_size == 50, "Otherwise we get different samples"
276
+ dataloader = torch.utils.data.DataLoader(
277
+ dataset.batched(args.batch_size), batch_size=None,
278
+ shuffle=False, num_workers=args.num_workers,
279
+ )
280
+ else:
281
+ dataloader = torch.utils.data.DataLoader(
282
+ dataset, batch_size=args.batch_size,
283
+ shuffle=False, num_workers=args.num_workers,
284
+ collate_fn=collate_fn
285
+ )
286
+ if task == "zeroshot_classification":
287
+ zeroshot_templates = dataset.templates if hasattr(dataset, "templates") else None
288
+ if args.verbose:
289
+ print(f"Zero-shot templates: {zeroshot_templates}")
290
+ classnames = dataset.classes if hasattr(dataset, "classes") else None
291
+ assert (zeroshot_templates is not None and classnames is not None), "Dataset does not support classification"
292
+ if args.attack is None:
293
+ attack_config = {
294
+ "attack": "none",
295
+ "bs": args.batch_size,
296
+ "n_samples": args.n_samples,
297
+ }
298
+ else:
299
+ attack_config = {
300
+ "attack": args.attack,
301
+ "norm": args.norm,
302
+ "eps": args.eps,
303
+ "iterations": args.iterations_adv,
304
+ "bs": args.batch_size,
305
+ "n_samples": args.n_samples,
306
+ }
307
+ print(f"Attack config: {attack_config}")
308
+ metrics = zeroshot_classification.evaluate(
309
+ model,
310
+ dataloader,
311
+ tokenizer,
312
+ classnames, zeroshot_templates,
313
+ normalize=normalize,
314
+ resize=resize,
315
+ device=args.device,
316
+ amp=args.amp,
317
+ verbose=args.verbose,
318
+ save_clf=args.save_clf,
319
+ load_clfs=args.load_clfs,
320
+ attack_config=attack_config,
321
+ )
322
+ elif task == "zeroshot_retrieval":
323
+ metrics = zeroshot_retrieval.evaluate(
324
+ model,
325
+ dataloader,
326
+ tokenizer,
327
+ recall_k_list=args.recall_k,
328
+ device=args.device,
329
+ amp=args.amp
330
+ )
331
+ elif task == "image_caption_selection":
332
+ metrics = image_caption_selection.evaluate(
333
+ model,
334
+ dataloader,
335
+ tokenizer,
336
+ device=args.device,
337
+ amp=args.amp,
338
+ )
339
+ elif task == "linear_probe":
340
+ # we also need the train split for linear probing.
341
+ train_dataset = build_dataset(
342
+ dataset_name=args.dataset,
343
+ root=dataset_root,
344
+ transform=transform,
345
+ split='train',
346
+ annotation_file=args.annotation_file,
347
+ download=True,
348
+ )
349
+ train_dataloader = torch.utils.data.DataLoader(
350
+ train_dataset, batch_size=args.batch_size,
351
+ shuffle=False, num_workers=args.num_workers,
352
+ collate_fn=collate_fn, pin_memory=True,
353
+ )
354
+ metrics = linear_probe.evaluate(
355
+ model,
356
+ train_dataloader,
357
+ dataloader,
358
+ args.fewshot_k,
359
+ args.batch_size,
360
+ args.num_workers,
361
+ args.fewshot_lr,
362
+ args.fewshot_epochs,
363
+ (args.model + '-' + args.pretrained + '-' + args.dataset).replace('/', '_'),
364
+ args.seed,
365
+ args.feature_root,
366
+ device=args.device,
367
+ amp=args.amp,
368
+ verbose=args.verbose,
369
+ )
370
+ elif task == "captioning":
371
+ metrics = captioning.evaluate(
372
+ model=model,
373
+ dataloader=dataloader,
374
+ batch_size=args.batch_size,
375
+ num_workers=args.num_workers,
376
+ device=args.device,
377
+ amp=args.amp,
378
+ verbose=args.verbose,
379
+ transform=transform
380
+ )
381
+ else:
382
+ raise ValueError("Unsupported task: {}. task should be `zeroshot_classification`, `zeroshot_retrieval`, `linear_probe`, or `captioning`".format(task))
383
+ dump = {
384
+ "dataset": args.dataset,
385
+ "model": args.model,
386
+ "pretrained": args.pretrained,
387
+ "beta": args.beta if args.interpolate else None,
388
+ "task": task,
389
+ "metrics": metrics,
390
+ "language": args.language,
391
+ "attack": args.attack,
392
+ "iterations_adv": args.iterations_adv,
393
+ "eps": args.eps,
394
+ "norm": args.norm,
395
+ }
396
+ if args.verbose:
397
+ print(f"Dump results to: {output}")
398
+ with open(output, "w") as f:
399
+ json.dump(dump, f)
400
+ return 0
401
+
402
+
403
+ if __name__ == "__main__":
404
+ sys.exit(main()) # pragma: no cover
CLIP_benchmark/clip_benchmark/datasets/__init__.py ADDED
File without changes
CLIP_benchmark/clip_benchmark/datasets/ar_classnames.json ADDED
@@ -0,0 +1,1004 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "imagenet1k": [
3
+ "\u0633\u0645\u0643 \u0627\u0644\u062a\u0646\u0634",
4
+ "\u0627\u0644\u0633\u0645\u0643\u0629 \u0627\u0644\u0630\u0647\u0628\u064a\u0629",
5
+ "\u0627\u0644\u0642\u0631\u0634 \u0627\u0644\u0623\u0628\u064a\u0636 \u0627\u0644\u0643\u0628\u064a\u0631",
6
+ "\u0627\u0644\u0642\u0631\u0634 \u0627\u0644\u0628\u0628\u0631\u064a",
7
+ "\u0627\u0644\u0642\u0631\u0634 \u0627\u0644\u0645\u0637\u0631\u0642\u0629",
8
+ "\u0633\u0645\u0643 \u0627\u0644\u0631\u0639\u0627\u062f",
9
+ "\u0633\u0645\u0643 \u0627\u0644\u0631\u0642\u064a\u0637\u0629",
10
+ "\u062f\u064a\u0643",
11
+ "\u062f\u062c\u0627\u062c\u0629",
12
+ "\u0646\u0639\u0627\u0645\u0629",
13
+ "\u0627\u0644\u0634\u0631\u0634\u0648\u0631 \u0627\u0644\u062c\u0628\u0644\u064a",
14
+ "\u0637\u0627\u0626\u0631 \u0627\u0644\u062d\u0633\u0648\u0646",
15
+ "\u0637\u0627\u0626\u0631 \u0627\u0644\u062a\u0641\u0627\u062d\u064a \u0627\u0644\u0627\u0648\u0631\u0648\u0628\u064a",
16
+ "\u0637\u0627\u0626\u0631 \u0627\u0644\u062c\u0646\u0643 \u062f\u0627\u0643\u0646 \u0627\u0644\u0639\u064a\u0648\u0646",
17
+ "\u0637\u0627\u0626\u0631 \u0627\u0644\u062f\u0631\u0633\u0629 \u0627\u0644\u0633\u0645\u0627\u0648\u064a",
18
+ "\u0637\u0627\u0626\u0631 \u0627\u0628\u0648 \u0627\u0644\u062d\u0646\u0627\u0621",
19
+ "\u0628\u0644\u0628\u0644",
20
+ "\u0637\u0627\u0626\u0631 \u0627\u0644\u0642\u064a\u0642",
21
+ "\u0639\u0642\u0639\u0642 \u0637\u0627\u0626\u0631 \u0627\u0644\u0630\u064a\u0644 \u0627\u0644\u0637\u0648\u064a\u0644",
22
+ "\u0637\u0627\u0626\u0631 \u0627\u0644\u0642\u0631\u0642\u0641",
23
+ "\u0637\u0627\u0626\u0631 \u0627\u0644\u063a\u0637\u0627\u0633",
24
+ "\u0637\u0627\u0626\u0631 \u0627\u0644\u062d\u062f\u0623\u0629",
25
+ "\u0637\u0627\u0626\u0631 \u0627\u0644\u0639\u0642\u0627\u0628 \u0627\u0644\u0631\u062e\u0645\u0629",
26
+ "\u0646\u0633\u0631",
27
+ "\u0627\u0644\u0628\u0648\u0645\u0629 \u0627\u0644\u0631\u0645\u0627\u062f\u064a\u0629",
28
+ "\u0627\u0644\u0633\u0645\u0646\u062f\u0631 \u0627\u0644\u0646\u0627\u0631\u064a",
29
+ "\u0644\u064a\u0633\u0648\u062a\u0631\u064a\u062a\u0648\u0646 \u0641\u0648\u0644\u062c\u0627\u0631\u064a\u0633",
30
+ "\u0627\u0644\u0633\u0645\u0646\u062f\u0631 \u0627\u0644\u0645\u0627\u0626\u064a",
31
+ "\u0627\u0644\u0633\u0645\u0646\u062f\u0631 \u0627\u0644\u0645\u0631\u0642\u0637",
32
+ "\u0627\u0644\u0633\u0645\u0646\u062f\u0631 \u0627\u0644\u0645\u0643\u0633\u064a\u0643\u064a",
33
+ "\u0636\u0641\u062f\u0639 \u0627\u0644\u062b\u0648\u0631 \u0627\u0644\u0627\u0645\u0631\u064a\u0643\u064a",
34
+ "\u0636\u0641\u062f\u0639 \u0627\u0644\u0634\u062c\u0631",
35
+ "\u0627\u0644\u0636\u0641\u0627\u062f\u0639 \u0630\u0627\u062a \u0627\u0644\u0630\u064a\u0644",
36
+ "\u0627\u0644\u0633\u0644\u062d\u0641\u0627\u0629 \u0627\u0644\u0628\u062d\u0631\u064a\u0629 \u0636\u062e\u0645\u0629 \u0627\u0644\u0631\u0623\u0633",
37
+ "\u0633\u0644\u062d\u0641\u0627\u0629 \u0627\u0644\u0645\u062d\u064a\u0637 \u062c\u0644\u062f\u064a\u0629 \u0627\u0644\u0638\u0647\u0631",
38
+ "\u0633\u0644\u062d\u0641\u0627\u0629 \u0627\u0644\u0637\u064a\u0646",
39
+ "\u0627\u0644\u0633\u0644\u062d\u0641\u0627\u0629 \u0630\u0627\u062a \u0638\u0647\u0631 \u0627\u0644\u0645\u0639\u064a\u0646",
40
+ "\u0633\u0644\u062d\u0641\u0627\u0629 \u0635\u0646\u062f\u0648\u0642\u064a\u0629",
41
+ "\u0627\u0644\u0648\u0632\u063a",
42
+ "\u0627\u0644\u0625\u063a\u0648\u0627\u0646\u0629",
43
+ "\u0627\u0644\u062d\u0631\u0628\u0627\u0621 \u0627\u0644\u062e\u0636\u0631\u0627\u0621",
44
+ "\u0627\u0644\u0633\u062d\u0627\u0644\u064a \u0627\u0644\u0635\u062d\u0631\u0627\u0648\u064a\u0629",
45
+ "\u0627\u0644\u0639\u064e\u0636\u0652\u0631\u064e\u0641\u064f\u0648\u0637",
46
+ " \u0633\u062d\u0644\u064a\u0629 \u0647\u062f\u0628 \u0627\u0644\u0639\u0646\u0642",
47
+ "\u0633\u062d\u0644\u064a\u0629 \u0627\u0644\u062a\u0645\u0633\u0627\u062d",
48
+ "\u0648\u062d\u0634 \u062c\u064a\u0644\u0627",
49
+ "\u0633\u062d\u0644\u064a\u0629 \u062e\u0636\u0631\u0627\u0621",
50
+ "\u062d\u0631\u0628\u0627\u0621 \u0627\u0641\u0631\u064a\u0642\u064a\u0629",
51
+ "\u062a\u0646\u064a\u0646 \u0643\u0648\u0645\u0648\u062f\u0648",
52
+ "\u0627\u0644\u062a\u0645\u0633\u0627\u062d \u0627\u0644\u0623\u0641\u0631\u064a\u0642\u064a",
53
+ "\u062a\u0645\u0633\u0627\u062d \u0627\u0644\u0642\u0627\u0637\u0648\u0631 \u0627\u0644\u0623\u0645\u0631\u064a\u0643\u064a",
54
+ "\u0627\u0644\u062f\u064a\u0646\u0627\u0635\u0648\u0631 \u062b\u064f\u0644\u0627\u062b\u064a\u064f\u0651 \u0627\u0644\u0642\u064f\u0631\u0648\u0646",
55
+ "\u0627\u0644\u062b\u0639\u0627\u0628\u064a\u0646 \u0627\u0644\u062f\u0648\u062f\u064a\u0629",
56
+ "\u0623\u0641\u0639\u0649 \u0627\u0644\u0637\u0648\u0642",
57
+ "\u0627\u0644\u0623\u0641\u0639\u0649 \u0627\u0644\u0646\u0641\u0627\u062b\u0629 ",
58
+ "\u0627\u0644\u0623\u0641\u0639\u0649 \u0627\u0644\u062e\u0636\u0631\u0627\u0621",
59
+ "\u0627\u0644\u062b\u0639\u0628\u0627\u0646 \u0627\u0644\u0645\u0644\u0643",
60
+ "\u0623\u0641\u0639\u0649 \u0627\u0644\u0631\u0628\u0627\u0637",
61
+ "\u0627\u0641\u0639\u0649 \u0627\u0644\u0645\u0627\u0621",
62
+ "\u062b\u0639\u0628\u0627\u0646 \u0646\u0628\u0627\u062a \u0643\u0631\u0645\u0629",
63
+ "\u0627\u0644\u062b\u0639\u0628\u0627\u0646 \u0627\u0644\u0644\u064a\u0644\u064a ",
64
+ "\u062b\u0639\u0628\u0627\u0646 \u0627\u0644\u0623\u0635\u0644\u0629 \u0627\u0644\u0639\u0627\u0635\u0631\u0629",
65
+ "\u0627\u0644\u062b\u0639\u0628\u0627\u0646 \u0627\u0644\u0635\u062e\u0631\u064a",
66
+ "\u0643\u0648\u0628\u0631\u0627 \u0647\u0646\u062f\u064a\u0629",
67
+ "\u0645\u0627\u0645\u0628\u0627 \u062e\u0636\u0631\u0627\u0621",
68
+ "\u0627\u0644\u063a\u064a\u062f\u0642\u0627\u0648\u0627\u062a",
69
+ "\u0627\u0644\u0623\u0641\u0639\u0649 \u0627\u0644\u0645\u0642\u0631\u0646\u0629",
70
+ "\u0627\u0644\u0623\u0641\u0639\u0649 \u0627\u0644\u062c\u0631\u0633\u064a\u0629 \u0630\u0627\u062a \u0627\u0644\u0635\u062f\u0631 \u0627\u0644\u0645\u0627\u0633\u064a",
71
+ "\u0627\u0641\u0639\u0649 \u0644\u0627\u0641\u0651\u0629 \u0627\u0644\u062c\u0646\u0628",
72
+ "\u0645\u0641\u0635\u0644\u064a\u0627\u062a \u062b\u0644\u0627\u062b\u064a\u0629 \u0627\u0644\u0641\u0635\u0648\u0635",
73
+ "\u062d\u0634\u0631\u0629 \u0627\u0644\u0639\u0646\u0643\u0628\u0648\u062a \u0630\u0627\u062a \u0627\u0644\u0642\u0648\u0627\u0626\u0645 \u0627\u0644\u0637\u0648\u064a\u0644\u0629",
74
+ "\u0639\u0642\u0631\u0628",
75
+ "\u0639\u0646\u0643\u0628\u0648\u062a \u0627\u0644\u062d\u062f\u064a\u0642\u0629 \u0630\u0627 \u0627\u0644\u0644\u0648\u0646 \u0627\u0644\u0623\u0633\u0648\u062f \u0648\u0627\u0644\u0623\u0635\u0641\u0631 ",
76
+ "\u0639\u0646\u0643\u0628\u0648\u062a \u0627\u0644\u062d\u0638\u064a\u0631\u0629",
77
+ "\u0639\u0646\u0643\u0628\u0648\u062a \u0627\u0644\u062d\u062f\u064a\u0642\u0629 \u0627\u0644\u0623\u0648\u0631\u0628\u064a",
78
+ "\u0639\u0646\u0643\u0628\u0648\u062a \u0627\u0644\u0623\u0631\u0645\u0644\u0629 \u0627\u0644\u0633\u0648\u062f\u0627\u0621",
79
+ "\u0639\u0646\u0643\u0628\u0648\u062a \u0631\u062a\u064a\u0644\u0627\u0621 \u0630\u0627\u062a \u0623\u0631\u062c\u0644 \u062d\u0645\u0631\u0627\u0621",
80
+ "\u0627\u0644\u0639\u0646\u0643\u0628\u0648\u062a \u0627\u0644\u0630\u0626\u0628\u064a",
81
+ "\u0639\u0646\u0643\u0628\u0648\u062a \u0627\u0644\u0642\u064f\u0631\u064e\u0627\u062f",
82
+ "\u062d\u0634\u0631\u0629 \u0623\u0645 \u0623\u0631\u0628\u0639\u0629 \u0648\u0623\u0631\u0628\u0639\u064a\u0646",
83
+ "\u062f\u062c\u0627\u062c\u0629 \u0627\u0644\u0637\u0647\u064a\u0648\u062c \u0627\u0644\u0623\u0633\u0648\u062f",
84
+ "\u062f\u062c\u0627\u062c\u0629 \u062a\u0631\u0645\u062c\u0627\u0646",
85
+ "\u062f\u062c\u0627\u062c\u0629 \u0637\u064a\u0647\u0648\u062c \u0645\u0637\u0648\u0642",
86
+ "\u062f\u062c\u0627\u062c\u0629 \u0627\u0644\u0637\u0647\u0628\u0648\u062c",
87
+ "\u0627\u0644\u0637\u0627\u0648\u0648\u0633",
88
+ "\u0637\u0627\u0626\u0631 \u0627\u0644\u0633\u0645\u0627\u0646",
89
+ "\u0637\u0627\u0626\u0631 \u0627\u0644\u062d\u062c\u0644",
90
+ "\u0627\u0644\u0628\u0628\u063a\u0627\u0621 \u0627\u0644\u0631\u0645\u0627\u062f\u064a",
91
+ "\u0628\u0628\u063a\u0627\u0621 \u0645\u0643\u0627\u0648",
92
+ "\u0628\u0628\u063a\u0627\u0621 \u0643\u0648\u0643\u0627\u062a\u0648 \u0643\u0628\u0631\u064a\u062a\u064a \u0627\u0644\u0639\u0631\u0641",
93
+ "\u0628\u0628\u063a\u0627\u0621 \u0642\u0648\u0633 \u0642\u0632\u062d",
94
+ "\u0637\u064a\u0631 \u0627\u0644\u0648\u0642\u0648\u0627\u0642 \u0643\u0648\u0643\u0627\u0644",
95
+ "\u0637\u064a\u0631 \u0648\u0631\u0648\u0627\u0631 ",
96
+ "\u0637\u064a\u0631 \u0623\u0628\u0648 \u0642\u0631\u0646",
97
+ "\u0627\u0644\u0637\u0627\u0626\u0631 \u0627\u0644\u0637\u0646\u0627\u0646",
98
+ "\u0637\u064a\u0648\u0631 \u0627\u0644\u064a\u0642\u0645\u064e\u0631",
99
+ "\u0637\u0627\u0626\u0631 \u0645\u0637\u0648\u0642",
100
+ "\u0630\u0643\u0631 \u0627\u0644\u0628\u0637",
101
+ "\u0627\u0644\u0628\u0637\u0629 \u0627\u0644\u063a\u0648\u0627\u0635\u0629 \u062d\u0645\u0631\u0627\u0621 \u0627\u0644\u0635\u062f\u0631",
102
+ "\u0625\u0648\u0632\u0629",
103
+ "\u0627\u0644\u0625\u0648\u0632\u0629 \u0633\u0648\u062f\u0627\u0621",
104
+ "\u0627\u0644\u0641\u064a\u0644",
105
+ " \u0622\u0643\u0644 \u0627\u0644\u0646\u0645\u0644 \u0627\u0644\u0634\u0648\u0643\u064a",
106
+ "\u062e\u0644\u062f \u0627\u0644\u0645\u0627\u0621",
107
+ "\u0648\u0644\u0628",
108
+ "\u0643\u0648\u0627\u0644\u0627",
109
+ "\u0627\u0644\u0648\u0645\u0628\u062a\u064a\u0627\u062a",
110
+ "\u0642\u0646\u062f\u064a\u0644 \u0628\u062d\u0631",
111
+ "\u0634\u0642\u0627\u0626\u0642 \u0646\u0639\u0645\u0627\u0646 \u0627\u0644\u0628\u062d\u0631",
112
+ "\u0645\u0631\u062c\u0627\u0646 \u0627\u0644\u062f\u0645\u0627\u063a",
113
+ "\u062f\u064a\u062f\u0627\u0646 \u0645\u0633\u0637\u062d\u0629",
114
+ "\u062f\u064a\u062f\u0627\u0646 \u0623\u0633\u0637\u0648\u0627\u0646\u064a\u0629",
115
+ "\u0645\u062d\u0627\u0631\u0629",
116
+ "\u062d\u0644\u0632\u0648\u0646",
117
+ "\u0628\u0632\u0627\u0642",
118
+ "\u062d\u0644\u0632\u0648\u0646 \u0645\u0627\u0626\u064a",
119
+ "\u0631\u062e\u0648\u064a\u0627\u062a \u0628\u062d\u0631\u064a\u0647 \u062a\u0633\u0645\u0649 \u0627\u0644\u062e\u064a\u062a\u0648\u0646",
120
+ "\u0646\u0648\u062a\u064a\u0644\u0648\u0633 \u0627\u0644\u062d\u062c\u0631\u064a",
121
+ "\u0633\u0631\u0637\u0627\u0646 \u0627\u0644\u0628\u062d\u0631",
122
+ "\u0633\u0631\u0637\u0627\u0646 \u0627\u0644\u0628\u062d\u0631 \u0627\u0644\u0623\u0637\u0644\u0633\u064a",
123
+ "\u0627\u0644\u0633\u0631\u0637\u0627\u0646 \u0627\u0644\u0639\u0627\u0632\u0641",
124
+ "\u0645\u0644\u0643 \u0627\u0644\u0633\u0644\u0637\u0639\u0648\u0646",
125
+ "\u062c\u0631\u0627\u062f \u0627\u0644\u0628\u062d\u0631 \u0627\u0644\u0623\u0645\u0631\u064a\u0643\u064a",
126
+ "\u0643\u0631\u0643\u0646\u062f \u0634\u0627\u0626\u0643",
127
+ "\u062c\u0631\u0627\u062f \u0627\u0644\u0645\u064a\u0627\u0647 \u0627\u0644\u0639\u0630\u0628\u0629",
128
+ "\u0627\u0644\u0633\u0631\u0637\u0627\u0646 \u0627\u0644\u0646\u0627\u0633\u0643",
129
+ "\u0645\u062a\u0633\u0627\u0648\u064a\u0627\u062a \u0627\u0644\u0623\u0642\u062f\u0627\u0645",
130
+ "\u0644\u0642\u0644\u0642 \u0623\u0628\u064a\u0636",
131
+ "\u0644\u0642\u0644\u0642 \u0623\u0633\u0648\u062f",
132
+ "\u0637\u0627\u0626\u0631 \u0627\u0644\u0628\u062c\u0639 \u0627\u0644\u0645\u0633\u0645\u0649 \u0623\u0628\u0648 \u0645\u0644\u0639\u0642\u0629",
133
+ "\u0637\u0627\u0626\u0631 \u0627\u0644\u0641\u0644\u0627\u0645\u064a\u0646\u063a\u0648",
134
+ "\u0637\u0627\u0626\u0631 \u0628\u0644\u0634\u0648\u0646 \u0627\u0644\u0623\u0632\u0631\u0642 \u0627\u0644\u0635\u063a\u064a\u0631",
135
+ "\u0637\u0627\u0626\u0631 \u0627\u0644\u0628\u0644\u0634\u0648\u0646 \u0627\u0644\u0623\u0628\u064a\u0636 \u0627\u0644\u0643\u0628\u064a\u0631",
136
+ "\u0637\u0627\u0626\u0631 \u0627\u0644\u0639\u062c\u0627\u062c",
137
+ "\u0637\u0627\u0626\u0631 \u0627\u0644\u0643\u0631\u0643\u064a\u0629",
138
+ "\u0637\u0627\u0626\u0631 \u0627\u0644\u0631\u0637\u0627\u0633",
139
+ "\u062f\u062c\u0627\u062c\u0629 \u0627\u0644\u0645\u0627\u0621 \u0627\u0644\u0623\u0631\u062c\u0648\u0627\u0646\u064a\u0629",
140
+ "\u0637\u0627\u0626\u0631 \u0627\u0644\u063a\u0631 \u0627\u0644\u0623\u0645\u0631\u064a\u0643\u064a",
141
+ "\u0637\u0627\u0626\u0631 \u0627\u0644\u062d\u0628\u0627\u0631",
142
+ "\u0637\u0627\u0626\u0631 \u0642\u0646\u0628\u0631\u0629 \u0627\u0644\u0645\u0627\u0621 \u0627\u0644\u0645\u062a\u0648\u0631\u062f",
143
+ "\u0637\u0627\u0626\u0631 \u062f\u0631\u064a\u062c\u0629 \u0623\u0644\u0628\u064a\u0629",
144
+ "\u0637\u0627\u0626\u0631 \u0627\u0644\u0637\u064a\u0637\u0648\u064a \u0623\u062d\u0645\u0631 \u0627\u0644\u0633\u0627\u0642",
145
+ "\u0639\u064e\u062f\u0651\u0627\u0621 \u0627\u0644\u0645\u0633\u062a\u0646\u0642\u0639\u0627\u062a \u0623\u0648 \u0627\u0644\u062f\u0651\u062a\u0634\u0631",
146
+ "\u0635\u0627\u0626\u062f \u0627\u0644\u0645\u062d\u0627\u0631",
147
+ "\u0628\u062c\u0639\u0629",
148
+ "\u0627\u0644\u0628\u0637\u0631\u064a\u0642 \u0627\u0644\u0645\u0644\u0643",
149
+ "\u0637\u0627\u0626\u0631 \u0627\u0644\u0642\u0637\u0631\u0633",
150
+ "\u0627\u0644\u062d\u0648\u062a \u0627\u0644\u0631\u0645\u0627\u062f\u064a \u0627\u0644\u0635\u0644\u0628",
151
+ "\u0627\u0644\u062d\u064f\u0648\u062a\u064f \u0627\u0644\u0642\u0627\u062a\u0650\u0644\u064f",
152
+ "\u0628\u0642\u0631\u0629 \u0627\u0644\u0628\u062d\u0631",
153
+ "\u0623\u0633\u062f \u0627\u0644\u0628\u062d\u0631",
154
+ "\u0643\u0644\u0628 \u0634\u064a\u0648\u0627\u0648\u0627",
155
+ "\u0643\u0644\u0628 \u0627\u0644\u0630\u0642\u0646 \u0627\u0644\u064a\u0627\u0628\u0627\u0646\u064a",
156
+ "\u0643\u0644\u0628 \u0645\u0627\u0644\u0637\u064a",
157
+ "\u0643\u0644\u0628 \u0628\u0643\u064a\u0646\u064a",
158
+ "\u0643\u0644\u0628 \u062a\u0634\u064a\u0647 \u062a\u0632\u0648 \u0627\u0644\u0635\u064a\u0646\u064a",
159
+ "\u0643\u0644\u0628 \u0627\u0644\u0645\u0644\u0643 \u062a\u0634\u0627\u0631\u0644\u0632",
160
+ "\u0643\u0644\u0628 \u0628\u0627\u0628\u064a\u0644\u0648\u0646",
161
+ "\u0643\u0644\u0628 \u0627\u0644\u062a\u0631\u064a\u0631 \u0627\u0644\u0639\u0631\u0636",
162
+ "\u0643\u0644\u0627\u0628 \u0631\u064a\u062f\u062c \u0628\u0627\u0643",
163
+ "\u0643\u0644\u0628 \u0627\u0644\u0635\u064a\u062f \u0627\u0644\u0623\u0641\u063a\u0627\u0646\u064a",
164
+ "\u0643\u0644\u0628 \u0627\u0644\u0628\u0627\u0633\u0637",
165
+ "\u0643\u0644\u0628 \u0628\u064a\u063a\u0644",
166
+ "\u0643\u0644\u0628 \u0627\u0644\u062f\u0645\u0648\u0645",
167
+ "\u0643\u0644\u0628 \u0628\u0644\u0648\u064a\u062a\u064a\u0643 \u0643\u0648\u0646\u0647\u0648\u0646\u062f",
168
+ "\u0643\u0644\u0628 \u0643\u0648\u0646\u0647\u0648\u0646\u062f \u0627\u0644\u0628\u0646\u064a \u0648\u0627\u0644\u0623\u0633\u0648\u062f",
169
+ "\u0643\u0644\u0628 \u0627\u0644\u0648\u0648\u0643\u0631",
170
+ "\u0643\u0644\u0628 \u0635\u064a\u062f \u0627\u0644\u062b\u0639\u0627\u0644\u0628 \u0627\u0644\u0625\u0646\u062c\u0644\u064a\u0632\u064a",
171
+ "\u0643\u0644\u0628 \u0631\u064a\u062f\u0628\u0648\u0646 \u0643\u0648\u0646\u0647\u0648\u0646\u062f",
172
+ "\u0643\u0644\u0628 \u0628\u0648\u0631\u0632\u0648\u064a",
173
+ "\u0643\u0644\u0628 \u0627\u0644\u0630\u0626\u0628 \u0627\u0644\u0627\u064a\u0631\u0644\u0646\u062f\u064a",
174
+ "\u0643\u0644\u0628 \u0627\u0644\u0633\u0644\u0648\u0642\u064a \u0627\u0644\u0627\u064a\u0637\u0627\u0644\u064a",
175
+ "\u0643\u0644\u0628 \u0627\u0644\u0648\u064a\u0628\u062a",
176
+ "\u0643\u0644\u0628 \u0627\u064a\u0628\u064a\u0632\u0627\u0646 \u0647\u0648\u0646\u062f",
177
+ "\u0627\u0644\u0643\u0644\u0628 \u0627\u0644\u0646\u0631\u0648\u064a\u062c\u064a",
178
+ "\u0643\u0644\u0628 \u0623\u0648\u062a\u064a\u0631 \u0647\u0627\u0648\u0646\u062f",
179
+ "\u0643\u0644\u0628 \u0633\u0644\u0648\u0642\u064a",
180
+ "\u0643\u0644\u0628 \u062f\u064a\u0631 \u0647\u0627\u0648\u0646\u062f \u0627\u0644\u0627\u0633\u0643\u062a\u0644\u0646\u062f\u064a",
181
+ "\u0643\u0644\u0628 \u0648\u0627\u064a\u0645\u0631\u064a",
182
+ "\u0643\u0644\u0628 \u0633\u062a\u0627\u0641\u0648\u0631\u062f\u0634\u0627\u064a\u0631 \u0628\u0648\u0644 \u062a\u0631\u064a\u0631",
183
+ "\u0643\u0644\u0628 \u0633\u062a\u0627\u0641\u0648\u0631\u062f\u0634\u0627\u064a\u0631 \u0627\u0644\u0623\u0645\u0631\u064a\u0643\u064a",
184
+ "\u0643\u0644\u0628 \u0628\u064a\u062f\u0644\u064a\u0646\u062c\u062a\u0648\u0646 \u062a\u0631\u064a\u0631",
185
+ "\u0643\u0644\u0628 \u0627\u0644\u0628\u0648\u0631\u062f\u0631 \u062a\u064a\u0631\u064a\u0631",
186
+ "\u0643\u0644\u0628 \u0643\u064a\u0631\u064a \u0628\u0644\u0648 \u062a\u0631\u064a\u0631",
187
+ "\u0643\u0644\u0628 \u0627\u0644\u062a\u0631\u064a\u0631 \u0627\u0644\u0625\u064a\u0631\u0644\u0646\u062f\u064a",
188
+ "\u0643\u0644\u0628 \u0646\u0648\u0631\u0641\u0648\u0644\u0643 \u062a\u064a\u0631\u064a\u0631",
189
+ "\u0643\u0644\u0628 \u0646\u0648\u0631\u064a\u062a\u0634 \u062a\u0631\u064a\u0631",
190
+ "\u0643\u0644\u0628 \u064a\u0648\u0631\u0643 \u0634\u0627\u064a\u0631",
191
+ "\u0643\u0644\u0628 \u0648\u064a\u0631 \u0641\u0648\u0643\u0633 \u062a\u0631\u064a\u0631",
192
+ "\u0643\u0644\u0628 \u0644\u064a\u0643\u0644\u0627\u0646\u062f \u062a\u064a\u0631\u064a\u0631",
193
+ "\u0643\u0644\u0628 \u0633\u064a\u0627\u0644\u064a\u0647\u0627\u0645 \u062a\u064a\u0631\u064a\u0631",
194
+ "\u0643\u0644\u0627\u0628 \u0627\u0644\u0623\u0631\u062f\u064a\u0644",
195
+ "\u0643\u0644\u0628 \u0643\u064a\u0631\u0646 \u062a\u0631\u064a\u0631",
196
+ "\u0643\u0644\u0628 \u062a\u0631\u064a\u0631 \u0627\u0633\u062a\u0631\u0627\u0644\u064a",
197
+ "\u0643\u0644\u0628 \u062f\u0627\u0646\u062f\u064a \u062f\u064a\u0646\u0645\u0648\u0646\u062a",
198
+ "\u0643\u0644\u0628 \u0628\u0648\u0633\u0637\u0646 \u062a\u064a\u0631\u064a\u0631",
199
+ "\u0643\u0644\u0628 \u0634\u0646\u0627\u0648\u062a\u0633\u0631 \u0645\u0646\u0645\u0646\u0645",
200
+ "\u0643\u0644\u0628 \u0634\u0646\u0627\u0648\u062a\u0633\u0631 \u0627\u0644\u0639\u0645\u0644\u0627\u0642",
201
+ "\u0643\u0644\u0628 \u0634\u0646\u0627\u0648\u062a\u0633\u0631 \u0627\u0644\u0639\u0627\u062f\u064a",
202
+ "\u0627\u0644\u0643\u0644\u0628 \u0627\u0644\u0627\u0633\u0643\u062a\u0644\u0646\u062f\u064a",
203
+ "\u0643\u0644\u0628 \u062a\u0631\u064a\u0631 \u0627\u0644\u062a\u0628\u062a",
204
+ "\u0643\u0644\u0628 \u0633\u064a\u0644\u0643\u064a \u062a\u064a\u0631\u064a\u0631",
205
+ "\u0643\u0644\u0628 \u0627\u0644\u062a\u064a\u0631\u064a\u0631 \u0627\u0644\u0642\u0645\u062d\u064a \u0627\u0644\u0646\u0627\u0639\u0645",
206
+ "\u0643\u0644\u0628 \u0648\u064a\u0633\u062a\u064a",
207
+ "\u0643\u0644\u0628 \u0644\u0627\u0633\u0627 \u0623\u0628\u0633\u0648",
208
+ "\u0643\u0644\u0628 \u0627\u0644\u0645\u0633\u062a\u0631\u062f \u0627\u0644\u0630\u0647\u0628\u064a",
209
+ "\u0643\u0644\u0628 \u0627\u0644\u0645\u0633\u062a\u0631\u062f \u0645\u062c\u0639\u062f \u0627\u0644\u0634\u0639\u0631",
210
+ "\u0643\u0644\u0628 \u0627\u0644\u0645\u0633\u062a\u0631\u062f \u0627\u0644\u0630\u0647\u0628\u064a",
211
+ "\u0643\u0644\u0628 \u0644\u0627\u0628\u0631\u0627\u062f\u0648\u0631 \u0631\u064a\u062a\u0631\u064a\u0641\u0631",
212
+ "\u0643\u0644\u0628 \u0634\u064a\u0633\u0628\u064a\u0643\u0627",
213
+ "\u0643\u0644\u0628 \u0628\u0648\u064a\u0646\u062a\u0631 \u0627\u0644\u0623\u0644\u0645\u0627\u0646\u064a \u0642\u0635\u064a\u0631 \u0627\u0644\u0634\u0639\u0631",
214
+ "\u0643\u0644\u0628 \u0641\u064a\u0632\u0644\u0627",
215
+ "\u0643\u0644\u0628 \u0633\u064a\u062a\u0631 \u0627\u0644\u0625\u0646\u062c\u0644\u064a\u0632\u064a",
216
+ "\u0643\u0644\u0628 \u0633\u064a\u062a\u0631 \u0627\u0644\u0625\u064a\u0631\u0644\u0646\u062f\u064a",
217
+ "\u0643\u0644\u0628 \u0633\u064a\u062a\u0631 \u0627\u0644\u062c\u0648\u0631\u062f\u0648\u0646\u064a",
218
+ "\u0643\u0644\u0628 \u0628\u0631\u064a\u062a\u0627\u0646\u064a \u0633\u0628\u0627\u0646\u064a\u0644",
219
+ "\u0643\u0644\u0628 \u0642\u0644\u0645\u0628\u0631",
220
+ "\u0643\u0644\u0628 \u0633\u0628\u0631\u064a\u0646\u063a\u0631 \u0633\u0628\u0627\u0646\u064a\u0644 \u0627\u0644\u0625\u0646\u062c\u0644\u064a\u0632\u064a",
221
+ "\u0643\u0644\u0628 \u0627\u0644\u0633\u0628\u0631\u064a\u0646\u063a\u0631 \u0627\u0644\u0648\u064a\u0644\u0632\u064a",
222
+ "\u0643\u0644\u0628 \u062f\u0644\u0644 \u0627\u0644\u0630\u0644\u064a\u0644",
223
+ "\u0643\u0644\u0628 \u0627\u0644\u0633\u0627\u0643\u0633 \u0627\u0644\u0625\u0633\u0628\u0627\u0646\u064a",
224
+ "\u0643\u0644\u0628 \u0633\u0628\u0627\u064a\u0646\u0644 \u0627\u0644\u0645\u0627\u0621 \u0627\u0644\u0625\u064a\u0631\u0644\u0646\u062f\u064a",
225
+ "\u0643\u0644\u0628 \u0643\u0648\u0641\u0627\u0632",
226
+ "\u0643\u0644\u0628 \u0634\u064a\u0628\u0631\u0643",
227
+ "\u0643\u0644\u0628 \u062c\u0631\u0648\u0646\u064a\u0646\u062f\u064a\u0644",
228
+ "\u0643\u0644\u0628 \u0645\u0627\u0644\u064a\u0646\u0648",
229
+ "\u0643\u0644\u0628 \u0628\u0631\u064a\u0627\u0631",
230
+ "\u0643\u0644\u0628 \u0627\u0644\u0643\u0644\u064a\u0628\u064a \u0627\u0644\u0625\u0633\u062a\u0631\u0627\u0644\u064a",
231
+ "\u0643\u0644\u0628 \u0643\u0648\u0645\u0648\u0646\u062f\u0648\u0631",
232
+ "\u0643\u0644\u0628 \u0627\u0644\u0631\u0627\u0639\u064a \u0627\u0644\u0625\u0646\u062c\u0644\u064a\u0632\u064a \u0627\u0644\u0642\u062f\u064a\u0645",
233
+ "\u0643\u0644\u0628 \u0627\u0644\u0631\u0627\u0639\u064a \u0627\u0644\u0634\u062a\u0644\u0646\u062f\u0649",
234
+ "\u0643\u0644\u0628 \u0627\u0644\u0643\u0648\u0644\u064a",
235
+ "\u0643\u0644\u0628 \u0628\u0648\u0631\u062f\u0631 \u0643\u0648\u0644\u064a",
236
+ "\u0643\u0644\u0628 \u0628\u0648\u0641\u064a \u062f\u064a \u0641\u0644\u0627\u0646\u062f\u0631",
237
+ "\u0643\u0644\u0628 \u0631\u0648\u062a \u0648\u0627\u064a\u0644\u0631",
238
+ "\u0643\u0644\u0628 \u0627\u0644\u0631\u0627\u0639\u064a \u0627\u0644\u0623\u0644\u0645\u0627\u0646\u064a",
239
+ "\u0643\u0644\u0628 \u0627\u0644\u062f\u0648\u0628\u064a\u0631\u0645\u0627\u0646",
240
+ "\u0643\u0644\u0628 \u0628\u064a\u0646\u0634\u0631 \u0627\u0644\u0645\u0635\u063a\u0631",
241
+ "\u0643\u0644\u0628 \u0627\u0644\u062c\u0628\u0644 \u0627\u0644\u0633\u0648\u064a\u0633\u0631\u064a",
242
+ "\u0643\u0644\u0628 \u062c\u0628\u0644 \u0627\u0644\u0628\u0631\u0646\u064a\u0632",
243
+ "\u0643\u0644\u0628 \u0627\u067e\u064a\u0646\u0632\u064a\u0644\u064a\u0631 \u0633\u064a\u0646\u064a\u0646\u0647\u0648\u0646\u062f",
244
+ "\u0643\u0644\u0628 \u0627\u0646\u062a\u0644\u0628\u062a\u0634\u0631",
245
+ "\u0643\u0644\u0628 \u0627\u0644\u0628\u0648\u0643\u0633\u0631",
246
+ "\u0643\u0644\u0628 \u0628\u0648\u0644 \u0645\u0627\u0633\u062a\u064a\u0641",
247
+ "\u0643\u0644\u0628 \u0627\u0644\u0645\u0627\u0633\u062a\u064a\u0641 \u0627\u0644\u062a\u064a\u0628\u062a\u064a",
248
+ "\u0643\u0644\u0628 \u0627\u0644\u0628\u0648\u0644\u062f\u0648\u063a \u0627\u0644\u0641\u0631\u0646\u0633\u064a",
249
+ "\u0627\u0644\u0643\u0644\u0628 \u0627\u0644\u062f\u0627\u0646\u0645\u0627\u0631\u0643\u064a \u0627\u0644\u0636\u062e\u0645",
250
+ "\u0643\u0644\u0628 \u0633\u0627\u0646\u062a \u0628\u0631\u0646\u0627\u0631\u062f",
251
+ "\u0643\u0644\u0628 \u0627\u0644\u0647\u0627\u0633\u0643\u064a",
252
+ "\u0643\u0644\u0628 \u0627\u0644\u0645\u0644\u0645\u0648\u062a \u0627\u0644\u0623\u0644\u0627\u0633\u0643\u064a",
253
+ "\u0643\u0644\u0628 \u0627\u0644\u0647\u0633\u0643\u064a \u0627\u0644\u0633\u064a\u0628\u064a\u0631\u064a",
254
+ "\u0643\u0644\u0628 \u062f\u0644\u0645\u0627\u0633\u064a \u0627\u0644\u0645\u0631\u0642\u0637",
255
+ "\u0643\u0644\u0628 \u0623\u0641\u064a\u0646\u0628\u064a\u0646\u0634\u0631",
256
+ "\u0643\u0644\u0628 \u0628\u0627\u0633\u0646\u062c\u064a",
257
+ "\u0643\u0644\u0628 \u0627\u0644\u0628\u062c",
258
+ "\u0643\u0644\u0628 \u0644\u064a\u0648\u0646 \u0628\u064a\u0631\u062c\u0631",
259
+ "\u0643\u0644\u0628 \u0646\u064a\u0648\u0641\u0627\u0648\u0646\u062f\u0644\u0627\u0646\u062f",
260
+ "\u0643\u0644\u0640\u0628 \u062c\u0628\u0627\u0644 \u0627\u0644\u0628\u0631\u0627\u0646\u0633",
261
+ "\u0643\u0644\u0628 \u0633\u0627\u0645\u0648\u062f\u064a",
262
+ "\u0643\u0644\u0628 \u0628\u0648\u0645\u064a\u0631\u0627\u0646\u064a\u0627\u0646",
263
+ "\u0643\u0644\u0628 \u0627\u0644\u062a\u0634\u0627\u0648 \u062a\u0634\u0627\u0648",
264
+ "\u0643\u0644\u0628 \u0627\u0644\u0643\u064a\u0634\u0648\u0646\u062f",
265
+ "\u0643\u0644\u0628 \u0627\u0644\u062c\u0631\u064a\u0641\u0648\u0646",
266
+ "\u0643\u0644\u0628\u0628 \u0628\u064a\u0645\u0628\u0631\u0648\u0643 \u0648\u064a\u0644\u0634 \u0643\u0648\u0631\u062c\u0649",
267
+ "\u0643\u0644\u0628 \u0627\u0644\u0643\u0627\u0631\u062f\u064a\u062c\u0627\u0646",
268
+ "\u0643\u0644\u0628 \u0627\u0644\u062a\u0648\u064a \u0627\u0644\u0628\u0648\u062f\u0644",
269
+ "\u0643\u0644\u0628 \u0627\u0644\u0628\u0648\u062f\u0644 \u0627\u0644\u0635\u063a\u064a\u0631",
270
+ "\u0643\u0644\u0628 \u0627\u0644\u0628\u0648\u062f\u0644 \u0627\u0644\u0642\u064a\u0627\u0633\u064a",
271
+ "\u0627\u0644\u0643\u0644\u0628 \u0627\u0644\u0645\u0643\u0633\u064a\u0643\u064a \u0628\u0644\u0627 \u0634\u0639\u0631",
272
+ "\u0627\u0644\u0630\u0626\u0628 \u0627\u0644\u0631\u0645\u0627\u062f\u064a",
273
+ "\u0627\u0644\u0630\u0626\u0628 \u0627\u0644\u0623\u0628\u064a\u0636",
274
+ "\u0627\u0644\u0630\u0626\u0628 \u0627\u0644\u0623\u062d\u0645\u0631",
275
+ "\u0627\u0644\u0642\u064a\u0648\u0637",
276
+ "\u0627\u0644\u0643\u0644\u0628 \u0627\u0644\u0625\u0633\u062a\u0631\u0627\u0644\u064a",
277
+ "\u0643\u0644\u0628 \u0627\u0644\u062f\u0648\u0644",
278
+ "\u0627\u0644\u0643\u0644\u0628 \u0627\u0644\u0628\u0631\u064a \u0627\u0644\u0625\u0641\u0631\u064a\u0642\u064a",
279
+ "\u0627\u0644\u0636\u0628\u0639",
280
+ "\u0627\u0644\u062b\u0639\u0644\u0628 \u0627\u0644\u0623\u062d\u0645\u0631",
281
+ "\u0627\u0644\u062b\u0639\u0644\u0628 \u0627\u0644\u0642\u0632\u0645",
282
+ "\u0627\u0644\u062b\u0639\u0644\u0628 \u0627\u0644\u0642\u0637\u0628\u064a",
283
+ "\u0627\u0644\u062b\u0639\u0644\u0628 \u0627\u0644\u0631\u0645\u0627\u062f\u064a",
284
+ "\u0627\u0644\u0642\u0637\u0637 \u0627\u0644\u0646\u0645\u0631\u064a\u0629",
285
+ "\u0627\u0644\u0633\u0646\u0648\u0631 \u0627\u0644\u0645\u064f\u0631\u0642\u0637 \u0627\u0644\u0635\u063a\u064a\u0631",
286
+ "\u0642\u0637 \u0634\u064a\u0631\u0627\u0632\u064a",
287
+ "\u0627\u0644\u0642\u0637 \u0627\u0644\u0633\u064a\u0627\u0645\u064a",
288
+ "\u0642\u0637 \u0644\u064a\u0628\u064a",
289
+ "\u0623\u0633\u062f \u0627\u0644\u062c\u0628\u0627\u0644",
290
+ "\u0627\u0644\u0648\u064e\u0634\u064e\u0642",
291
+ "\u0646\u0645\u0631 ",
292
+ "\u0646\u0645\u0631 \u0627\u0644\u062b\u0644\u0648\u062c",
293
+ "\u064a\u063a\u0648\u0631",
294
+ "\u0623\u0633\u062f",
295
+ "\u0627\u0644\u0628\u0628\u0631",
296
+ "\u0627\u0644\u0641\u0647\u062f",
297
+ "\u062f\u0628 \u0628\u0646\u064a",
298
+ "\u062f\u0628 \u0623\u0633\u0648\u062f \u0623\u0645\u0631\u064a\u0643\u064a",
299
+ "\u062f\u0628 \u0642\u0637\u0628\u064a",
300
+ "\u0627\u0644\u062f\u0628 \u0627\u0644\u0643\u0633\u0644\u0627\u0646 ",
301
+ "\u0627\u0644\u0633\u0645\u0648\u0631\u064a\u0627\u062a",
302
+ "\u0633\u0631\u0642\u0627\u0637",
303
+ "\u062e\u0646\u0627\u0641\u0633 \u0646\u0645\u0631\u064a\u0629",
304
+ "\u062f\u0639\u0633\u0648\u0642\u0629",
305
+ "\u062e\u0646\u0641\u0633\u0627\u0621 \u0623\u0631\u0636\u064a\u0629",
306
+ "\u062e\u0646\u0627\u0641\u0633 \u0637\u0648\u064a\u0644\u0629 \u0627\u0644\u0642\u0631\u0648\u0646 ",
307
+ "\u062e\u0646\u0641\u0633\u0629 \u0627\u0644\u0623\u0648\u0631\u0627\u0642",
308
+ "\u062e\u0646\u0627\u0641\u0633 \u0627\u0644\u0631\u0648\u062b",
309
+ "\u062e\u0646\u0641\u0633\u0627\u0621 \u0648\u062d\u064a\u062f \u0627\u0644\u0642\u0631\u0646",
310
+ "\u0627\u0644\u0633\u064f\u0648\u0633\u064a\u0646\u0627\u062a",
311
+ "\u062d\u0634\u0631\u0629 \u0630\u0648\u0627\u062a \u0627\u0644\u062c\u0646\u0627\u062d\u064a\u0646",
312
+ "\u0646\u062d\u0644",
313
+ "\u0627\u0644\u0646\u0645\u0644",
314
+ "\u062c\u0646\u062f\u0628",
315
+ "\u062d\u0634\u0631\u0629 \u0627\u0644\u0643\u0631\u064a\u0643\u064a\u062a",
316
+ "\u0627\u0644\u062d\u0634\u0631\u0629 \u0627\u0644\u0639\u0635\u0648\u064a\u0629",
317
+ "\u0635\u0631\u0635\u0648\u0631",
318
+ "\u0641\u0631\u0633 \u0627\u0644\u0646\u0628\u064a",
319
+ "\u062d\u0634\u0631\u0629 \u0632\u064a\u0632\u064a\u0627\u062a",
320
+ "\u0642\u0627\u0641\u0632\u0627\u062a \u0627\u0644\u0623\u0648\u0631\u0627\u0642",
321
+ "\u0639\u0631\u0642\u064a\u0627\u062a \u0627\u0644\u0623\u062c\u0646\u062d\u0629",
322
+ "\u0627\u0644\u064a\u0639\u0633\u0648\u0628",
323
+ "\u0645\u0642\u062a\u0631\u0646\u0627\u062a \u0627\u0644\u0623\u062c\u0646\u062d\u0629",
324
+ "\u0641\u0631\u0627\u0634\u0629 \u0628\u0634\u0648\u0631\u0629 \u0627\u0644\u0635\u064a\u0641",
325
+ "\u0641\u0631\u0627\u0634\u0629 \u062d\u0644\u064a\u0642\u0629",
326
+ "\u0641\u064e\u0631\u064e\u0627\u0634\u0629 \u0627\u0644\u0645\u064e\u0644\u0643",
327
+ "\u0641\u0631\u0627\u0634\u0629 \u0627\u0644\u0628\u064a\u0636\u0627\u0621 \u0627\u0644\u0635\u063a\u064a\u0631\u0629",
328
+ "\u0641\u0631\u0627\u0634\u0629 \u0627\u0644\u0643\u0628\u0631\u064a\u062a",
329
+ "\u0627\u0644\u0641\u0631\u0627\u0634\u0629 \u0627\u0644\u0646\u062d\u0627\u0633\u064a\u0629",
330
+ "\u0646\u062c\u0645 \u0627\u0644\u0628\u062d\u0631",
331
+ "\u0642\u0646\u0641\u0630 \u0627\u0644\u0628\u062d\u0631",
332
+ "\u062e\u064a\u0627\u0631 \u0627\u0644\u0628\u062d\u0631",
333
+ "\u0623\u0631\u0627\u0646\u0628 \u0642\u0637\u0646\u064a\u0629 \u0627\u0644\u0630\u064a\u0644",
334
+ "\u0623\u0631\u0646\u0628 \u0628\u0631\u064a",
335
+ "\u0627\u0644\u0623\u0646\u062c\u0648\u0631\u0627",
336
+ "\u0623\u0642\u062f\u0627\u062f",
337
+ "\u0627\u0644\u0646\u064a\u0635",
338
+ "\u0633\u0646\u062c\u0627\u0628 \u062b\u0639\u0644\u0628\u064a",
339
+ "\u0627\u0644\u0645\u0631\u0645\u0648\u0637",
340
+ "\u0627\u0644\u0642\u0646\u062f\u0633",
341
+ "\u0643\u0627\u0628\u064a\u0627\u0621 \u062e\u0646\u0632\u064a\u0631\u064a\u0629",
342
+ "\u0627\u0644\u062d\u0635\u0627\u0646 \u0627\u0644\u062d\u0645\u064a\u0636",
343
+ "\u0627\u0644\u062d\u0645\u0627\u0631 \u0627\u0644\u0645\u062e\u0637\u0637",
344
+ "\u0627\u0644\u062e\u0646\u0632\u064a\u0631 \u0627\u0644\u0623\u0644\u064a\u0641 \u0623\u0648 \u0627\u0644\u062e\u0646\u0632\u064a\u0631 \u0627\u0644\u0645\u0633\u062a\u0623\u0646\u0633",
345
+ "\u0627\u0644\u062e\u0646\u0632\u064a\u0631 \u0627\u0644\u0628\u0631\u064a ",
346
+ "\u062e\u0646\u0627\u0632\u064a\u0631 \u062a\u0624\u0644\u0648\u0644\u064a\u0629",
347
+ "\u0641\u0631\u0633 \u0627\u0644\u0646\u0647\u0631",
348
+ "\u0627\u0644\u062b\u0648\u0631",
349
+ "\u062c\u0627\u0645\u0648\u0633 \u0627\u0644\u0645\u0627\u0621",
350
+ "\u0627\u0644\u0628\u064a\u0633\u0648\u0646",
351
+ "\u0630\u0643\u0631 \u0627\u0644\u062e\u0631\u0648\u0641",
352
+ "\u0643\u0628\u0634 \u0627\u0644\u062c\u0628\u0627\u0644 \u0627\u0644\u0635\u062e\u0631\u064a\u0629",
353
+ "\u0627\u0644\u0648\u0639\u0644",
354
+ "\u062b\u064a\u062a\u0644 \u0627\u0644\u0647\u0631\u062a\u0628\u064a\u0633",
355
+ "\u0625\u0645\u0628\u0627\u0644\u0629",
356
+ "\u063a\u0632\u0627\u0644",
357
+ "\u0627\u0644\u062c\u0645\u0644 \u0627\u0644\u0639\u0631\u0628\u064a",
358
+ "\u0627\u0644\u0644\u0627\u0651\u0645\u0629",
359
+ "\u0627\u0628\u0646 \u0639\u0631\u0633",
360
+ "\u0627\u0644\u0645\u0646\u0643",
361
+ "\u0627\u0644\u0633\u0641\u0634\u0629",
362
+ "\u0627\u0628\u0646 \u0645\u0642\u0631\u0636 \u0623\u0633\u0648\u062f \u0627\u0644\u0623\u0642\u062f\u0627\u0645",
363
+ "\u0642\u0636\u0627\u0639\u0629",
364
+ "\u0638\u0631\u0628\u0627\u0646",
365
+ "\u0627\u0644\u063a\u0631\u064a\u0631",
366
+ "\u0627\u0644\u0645\u064f\u062f\u064e\u0631\u064e\u0651\u0639 \u0623\u0648 \u0627\u0644\u0623\u0631\u0645\u0627\u062f\u064a\u0644\u0644\u0648",
367
+ "\u0643\u0633\u0644\u0627\u0646 \u062b\u0644\u0627\u062b\u064a \u0627\u0644\u0623\u0635\u0627\u0628\u0639",
368
+ "\u0642\u0631\u062f \u0627\u0644\u0627\u0648\u0631\u0627\u0646\u063a\u0648\u062a\u0627\u0646",
369
+ "\u0627\u0644\u063a\u0648\u0631\u064a\u0644\u0627 \u0623\u0648 \u0627\u0644\u0642\u064f\u0631\u062f\u0648\u062d",
370
+ "\u0627\u0644\u0634\u0645\u0628\u0627\u0646\u0632\u064a \u0627\u0644\u0634\u0627\u0626\u0639 \u0623\u0648 \u0627\u0644\u0628\u064e\u0639\u0627\u0645",
371
+ "\u0642\u0631\u062f \u0627\u0644\u062c\u0628\u0648\u0646",
372
+ "\u0642\u0631\u062f \u0627\u0644\u0633\u064a\u0627\u0645\u0646\u062c",
373
+ "\u0633\u0639\u062f\u0627\u0646 \u0627\u0644\u063a\u064a\u0646\u0648\u0646",
374
+ "\u0633\u0639\u062f\u0627\u0646 \u0627\u0644\u0628\u0627\u062a\u0627\u0633",
375
+ "\u0627\u0644\u0631\u064f\u0628\u064e\u0651\u0627\u062d",
376
+ "\u0642\u0631\u062f \u0627\u0644\u0645\u0643\u0627\u0643",
377
+ "\u0642\u0631\u062f \u0627\u0644\u0643\u0648\u0644\u0628\u0633\u0627\u0648\u0627\u062a",
378
+ "\u0642\u0631\u062f \u0627\u0644\u0643\u0648\u0644\u0628\u0633",
379
+ "\u0642\u0631\u062f \u0627\u0644\u0645\u0644\u0645\u0644\u0629",
380
+ "\u0642\u0631\u0648\u062f \u0627\u0644\u0642\u0634\u0629",
381
+ "\u0642\u0631\u062f \u0627\u0644\u0643\u0628\u0648\u0634\u0629 \u0623\u0628\u064a\u0636 \u0627\u0644\u0648\u062c\u0647",
382
+ "\u0633\u0639\u062f\u0627\u0646 \u0627\u0644\u0639\u0648\u0627\u0621",
383
+ "\u0642\u0631\u062f \u0633\u0639\u062f\u0627\u0646 \u0627\u0644\u062a\u064a\u062a\u064a",
384
+ "\u0627\u0644\u0633\u064e\u0651\u0639\u062f\u0627\u0646 \u0627\u0644\u0639\u0646\u0643\u0628\u0648\u062a\u064a",
385
+ "\u0627\u0644\u0633\u0639\u062f\u0627\u0646 \u0627\u0644\u0633\u0646\u062c\u0627\u0628\u064a",
386
+ "\u0644\u064a\u0645\u0648\u0631 \u062d\u0644\u0642\u064a \u0627\u0644\u0630\u064a\u0644",
387
+ "\u062d\u064a\u0648\u0627\u0646 \u0627\u0644\u0627\u0646\u062f\u0631\u064a",
388
+ "\u0641\u064a\u0644 \u0647\u0646\u062f\u064a",
389
+ "\u0641\u064a\u0644 \u0623\u0641\u0631\u064a\u0642\u064a",
390
+ "\u0627\u0644\u0628\u0627\u0646\u062f\u0627 \u0627\u0644\u0623\u062d\u0645\u0631",
391
+ "\u0627\u0644\u0628\u0627\u0646\u062f\u0627 \u0627\u0644\u0639\u0645\u0644\u0627\u0642\u0629",
392
+ "\u062b\u064a\u0631\u0633\u064a\u062a\u064a\u0627\u062a",
393
+ "\u0633\u0645\u0643 \u0627\u0644\u0627\u0646\u0642\u0644\u064a\u0633",
394
+ "\u0633\u0645\u0643 \u0627\u0644\u0643\u0648\u0647\u0648 \u0627\u0644\u0633\u064a\u0644\u0645\u0648\u0646",
395
+ "\u0633\u0645\u0643 \u0627\u0644\u062c\u0645\u0627\u0644 \u0627\u0644\u0635\u062e\u0631\u064a",
396
+ "\u0633\u0645\u0643\u0629 \u0627\u0644\u0645\u0647\u0631\u062c",
397
+ "\u0633\u0645\u0643\u0629 \u0627\u0644\u062d\u0641\u0634\u064a\u0629",
398
+ "\u0633\u0645\u0643 \u0627\u0644\u0631\u0645\u062d",
399
+ "\u0633\u0645\u0643\u0629 \u0627\u0644\u062a\u0646\u064a\u0646",
400
+ "\u0633\u0645\u0643\u0629 \u0627\u0644\u064a\u0646\u0641\u0648\u062e\u064a\u0629",
401
+ "\u0627\u0644\u0645\u0650\u0639\u0652\u062f\u064e\u0627\u062f",
402
+ "\u0627\u0644\u0639\u0628\u0627\u0621\u0629",
403
+ "\u0644\u0628\u0627\u0633 \u062a\u062e\u0631\u062c",
404
+ "\u0623\u0643\u0648\u0631\u062f\u064a\u0648\u0646",
405
+ "\u0627\u0644\u0642\u064a\u062b\u0627\u0631\u0629 \u0627\u0644\u0635\u0648\u062a\u064a\u0629",
406
+ "\u062d\u0627\u0645\u0644\u0629 \u0637\u0627\u0626\u0631\u0627\u062a",
407
+ "\u0637\u0627\u0626\u0631\u0629 \u0631\u062d\u0644\u0627\u062a",
408
+ "\u0633\u0641\u064a\u0646\u0629 \u0647\u0648\u0627\u0626\u064a\u0629",
409
+ "\u0645\u0630\u0628\u062d",
410
+ "\u0633\u064a\u0627\u0631\u0629 \u0625\u0633\u0639\u0627\u0641",
411
+ "\u0627\u0644\u0645\u0631\u0643\u0628\u0629 \u0627\u0644\u0628\u0631\u0645\u0627\u0626\u064a\u0629",
412
+ "\u0627\u0644\u0633\u0627\u0639\u0629 \u0627\u0644\u0645\u062a\u0646\u0627\u0638\u0631\u0629",
413
+ "\u0627\u0644\u0645\u0646\u062d\u0644 \u0623\u0648 \u0627\u0644\u0645\u064e\u0646\u062d\u064e\u0644\u064e\u0629",
414
+ "\u0645\u0626\u0632\u0631",
415
+ "\u062d\u0627\u0648\u064a\u0629 \u0627\u0644\u0646\u0641\u0627\u064a\u0627\u062a",
416
+ "\u0628\u0646\u062f\u0642\u064a\u0629 \u0627\u0642\u062a\u062d\u0627\u0645",
417
+ "\u062d\u0642\u064a\u0628\u0629 \u0638\u0647\u0631",
418
+ "\u0627\u0644\u0645\u062e\u0628\u0632",
419
+ "\u0639\u0627\u0631\u0636\u0629 \u0627\u0644\u062a\u0648\u0627\u0632\u0646",
420
+ "\u0627\u0644\u0628\u0627\u0644\u0648\u0646",
421
+ "\u0642\u0644\u0645 \u062d\u0628\u0631 \u062c\u0627\u0641",
422
+ "\u0636\u0645\u0627\u062f\u0629 \u0637\u0628\u064a\u0629 \u0644\u0627\u0635\u0642\u0629",
423
+ "\u0627\u0644\u0628\u0627\u0646\u062c\u0648",
424
+ "\u062f\u0631\u0627\u0628\u0632\u064a\u0646",
425
+ "\u062d\u062f\u064a\u062f\u0629 (\u0631\u0641\u0639 \u0623\u062b\u0642\u0627\u0644)",
426
+ "\u0643\u0631\u0633\u064a \u0627\u0644\u062d\u0644\u0627\u0642\u0629",
427
+ "\u0645\u062d\u0644 \u0635\u0627\u0644\u0648\u0646 \u0627\u0644\u062d\u0644\u0627\u0642\u0629",
428
+ "\u062d\u0638\u064a\u0631\u0629",
429
+ "\u0627\u0644\u0628\u0627\u0631\u0648\u0645\u062a\u0631",
430
+ "\u0627\u0644\u0628\u0631\u0645\u064a\u0644",
431
+ "\u0639\u062c\u0644\u0629 \u0627\u0644\u064a\u062f",
432
+ "\u0643\u0631\u0629 \u0627\u0644\u0642\u0627\u0639\u062f\u0629 \u0623\u0648 \u0627\u0644\u0628\u064a\u0633\u0628\u0648\u0644",
433
+ "\u0643\u0631\u0629 \u0633\u0644\u0629",
434
+ "\u0633\u0631\u064a\u0631 \u0627\u0644\u0623\u0637\u0641\u0627\u0644",
435
+ "\u0645\u0632\u0645\u0627\u0631",
436
+ "\u0642\u0628\u0639\u0629 \u0633\u0628\u0627\u062d\u0629",
437
+ "\u0645\u0646\u0634\u0641\u0629",
438
+ "\u062d\u0648\u0636 \u0627\u0644\u0627\u0633\u062a\u062d\u0645\u0627\u0645",
439
+ "\u0633\u064a\u0627\u0629 \u0648\u0627\u063a\u0646",
440
+ "\u0627\u0644\u0645\u0646\u0627\u0631\u0629 \u0623\u0648 \u0627\u0644\u0641\u0646\u0627\u0631",
441
+ "\u0643\u0648\u0628 \u0632\u062c\u0627\u062c\u064a",
442
+ "\u0642\u0628\u0639\u0629 \u0627\u0644\u062f\u0628",
443
+ "\u0632\u062c\u0627\u062c\u0629 \u0627\u0644\u0628\u064a\u0631\u0629",
444
+ "\u0643\u0623\u0633 \u062c\u0639\u0629",
445
+ "\u0628\u0631\u062c \u0627\u0644\u0646\u0627\u0642\u0648\u0633",
446
+ "\u0645\u0631\u0648\u0644\u0629",
447
+ "\u0627\u0644\u062f\u0631\u0627\u062c\u0629 \u0627\u0644\u062a\u0631\u0627\u062f\u0641\u064a\u0629",
448
+ "\u0628\u0643\u064a\u0646\u064a",
449
+ "\u0627\u0644\u0645\u062c\u0644\u062f\u0627\u062a \u0627\u0644\u062d\u0644\u0642\u064a\u0629",
450
+ "\u0627\u0644\u0645\u0650\u0646\u0652\u0638\u0627\u0631",
451
+ "\u0635\u0646\u062f\u0648\u0642 \u0627\u0644\u0639\u0634",
452
+ "\u0627\u0644\u0645\u0631\u0641\u0623",
453
+ "\u0627\u0644\u0632\u0644\u0627\u062c\u0629 \u0627\u0644\u062c\u0645\u0627\u0639\u064a\u0629",
454
+ "\u0631\u0628\u0637\u0629 \u0639\u0646\u0642 \u0628\u0648\u0644\u0648",
455
+ "\u0627\u0644\u0643\u0632\u0629",
456
+ "\u0631\u0641 \u0627\u0644\u0643\u062a\u0628",
457
+ "\u0645\u0643\u062a\u0628\u0629",
458
+ "\u063a\u0637\u0627\u0621 \u0642\u0627\u0631\u0648\u0631\u0629",
459
+ "\u0627\u0644\u0642\u0648\u0633",
460
+ "\u0623\u0631\u0628\u0629 \u0641\u0631\u0627\u0634\u064a\u0629",
461
+ "\u0644\u0627\u0641\u062a\u0629 \u062a\u0627\u0631\u064a\u062e\u064a\u0629 \u0645\u0646 \u0627\u0644\u0646\u062d\u0627\u0633",
462
+ "\u062d\u0645\u0627\u0644\u0629 \u0627\u0644\u0635\u062f\u0631",
463
+ "\u062d\u0627\u062c\u0632 \u0627\u0644\u0623\u0645\u0648\u0627\u062c",
464
+ "\u062f\u0631\u0639 \u0627\u0644\u0635\u062f\u0631",
465
+ "\u0627\u0644\u0645\u0643\u0646\u0633\u0629",
466
+ "\u0627\u0644\u062f\u0644\u0648",
467
+ "\u0645\u0631\u0628\u0637 \u0627\u0644\u062d\u0632\u0627\u0645",
468
+ "\u0627\u0644\u0633\u062a\u0631\u0629 \u0627\u0644\u0648\u0627\u0642\u064a\u0629 \u0645\u0646 \u0627\u0644\u0631\u0635\u0627\u0635",
469
+ "\u0642\u0637\u0627\u0631 \u0627\u0644\u0637\u0644\u0642\u0629",
470
+ "\u0627\u0644\u0645\u062c\u0632\u0631\u0629",
471
+ "\u0633\u064a\u0627\u0631\u0629 \u0623\u062c\u0631\u0629",
472
+ "\u062d\u0644\u0629 (\u0622\u0646\u064a\u0629)",
473
+ "\u0634\u0645\u0639\u0629",
474
+ "\u0645\u062f\u0641\u0639",
475
+ "\u0642\u0627\u0631\u0628 \u0627\u0644\u0643\u0627\u0646\u0648",
476
+ "\u0641\u0627\u062a\u062d\u0629 \u0639\u0644\u0628",
477
+ "\u0633\u062a\u0631\u0629 \u0645\u062d\u0628\u0648\u0643\u0629",
478
+ "\u0627\u0644\u0645\u0631\u0622\u0629 \u0627\u0644\u062c\u0627\u0646\u0628\u064a\u0629",
479
+ "\u062f\u0648\u0627\u0645\u0629 \u0627\u0644\u062e\u064a\u0644",
480
+ " \u0623\u062f\u0648\u0627\u062a \u0627\u0644\u0635\u064a\u0627\u0646\u0629",
481
+ "\u0635\u0646\u062f\u0648\u0642 \u0643\u0631\u062a\u0648\u0646",
482
+ "\u0627\u0644\u0625\u0637\u0627\u0631 \u0627\u0644\u0645\u0637\u0627\u0637",
483
+ "\u0627\u0644\u0635\u0631\u0627\u0641 \u0627\u0644\u0622\u0644\u064a",
484
+ "\u0627\u0644\u0634\u0631\u064a\u0637 \u0627\u0644\u0645\u062f\u0645\u062c",
485
+ "\u0627\u0644\u0645\u0633\u062c\u0644",
486
+ "\u0627\u0644\u0642\u064e\u0644\u0652\u0639\u064e\u0629",
487
+ "\u0627\u0644\u0642\u0637\u0645\u0631\u0627\u0646",
488
+ "\u062c\u0647\u0627\u0632 \u0627\u0644\u0642\u0631\u0635 \u0627\u0644\u0645\u0636\u063a\u0648\u0637",
489
+ "\u062a\u0634\u064a\u0644\u0648",
490
+ "\u0647\u0627\u062a\u0641 \u0645\u062d\u0645\u0648\u0644",
491
+ "\u0633\u0644\u0633\u0644\u0629",
492
+ "\u0633\u064a\u0627\u062c \u0645\u0634\u0628\u0643",
493
+ "\u0627\u0644\u0632\u0631\u062f",
494
+ "\u0645\u0646\u0634\u0627\u0631 \u062c\u0646\u0632\u064a\u0631\u064a",
495
+ " \u0635\u0646\u062f\u0648\u0642 \u0627\u0644\u062a\u062e\u0632\u064a\u0646",
496
+ "\u062e\u0632\u0627\u0646\u0629 \u0627\u0644\u0623\u062b\u0627\u062b",
497
+ "\u0627\u0644\u0622\u0644\u0629 \u0627\u0644\u0625\u064a\u0642\u0627\u0639\u064a\u0629 ",
498
+ "\u0627\u0644\u062e\u0632\u0627\u0646\u0629 \u0627\u0644\u0635\u064a\u0646\u064a\u0629",
499
+ "\u062c\u0648\u0631\u0628 \u0639\u064a\u062f \u0627\u0644\u0645\u064a\u0644\u0627\u062f",
500
+ "\u0643\u0646\u064a\u0633\u0629",
501
+ "\u0645\u0633\u0631\u062d \u0623\u0641\u0644\u0627\u0645",
502
+ "\u0627\u0644\u0633\u0627\u0637\u0648\u0631",
503
+ "\u0645\u0633\u0627\u0643\u0646 \u0627\u0644\u062c\u0631\u0641",
504
+ "\u0627\u0644\u0645\u0639\u0637\u0641 \u0627\u0644\u0641\u0636\u0641\u0627\u0636",
505
+ "\u0627\u0644\u0642\u0628\u0642\u0627\u0628",
506
+ "\u062e\u0627\u0644\u0637 \u0627\u0644\u0645\u0634\u0631\u0648\u0628\u0627\u062a \u0627\u0644\u0643\u062d\u0648\u0644\u064a\u0629",
507
+ "\u0643\u0648\u0632 (\u0622\u0646\u064a\u0629)",
508
+ "\u0622\u0644\u0629 \u062a\u062d\u0636\u064a\u0631 \u0627\u0644\u0642\u0647\u0648\u0629",
509
+ "\u0627\u0644\u0634\u0643\u0644 \u0627\u0644\u062d\u0644\u0632\u0648\u0646\u064a",
510
+ "\u0627\u0644\u0642\u0641\u0644 \u0627\u0644\u0631\u0645\u0632\u064a",
511
+ "\u0644\u0648\u062d\u0629 \u0627\u0644\u0645\u0641\u0627\u062a\u064a\u062d",
512
+ "\u0645\u062a\u062c\u0631 \u0627\u0644\u062d\u0644\u0648\u064a\u0627\u062a",
513
+ "\u0633\u0641\u064a\u0646\u0629 \u062d\u0627\u0648\u064a\u0627\u062a",
514
+ "\u0633\u064a\u0627\u0631\u0629 \u0645\u0643\u0634\u0648\u0641\u0629",
515
+ "\u0628\u0631\u0627\u0645\u0629",
516
+ "\u0627\u0644\u0634\u064a\u0627\u0639",
517
+ "\u062c\u0632\u0645\u0629 \u0631\u0627\u0639\u064a \u0627\u0644\u0628\u0642\u0631",
518
+ "\u0642\u0628\u0639\u0629 \u0631\u0627\u0639\u064a \u0627\u0644\u0628\u0642\u0631",
519
+ "\u0627\u0644\u0645\u0647\u062f",
520
+ "\u0631\u0627\u0641\u0639\u0629",
521
+ "\u0627\u0644\u062e\u0648\u0630\u0629",
522
+ "\u062d\u0627\u0648\u064a\u0629 \u0634\u062d\u0646 \u0643\u0628\u064a\u0631\u0629",
523
+ "\u0633\u0631\u064a\u0631 \u0627\u0644\u0631\u0636\u064a\u0639",
524
+ "\u0642\u062f\u0631 \u0627\u0644\u0637\u0628\u062e \u0627\u0644\u0643\u0647\u0631\u0628\u0627\u0626\u064a",
525
+ "\u0643\u0631\u0648\u0643\u064a\u062a",
526
+ "\u0627\u0644\u0639\u0643\u0627\u0632",
527
+ "\u0643\u0648\u064a\u0631\u0633",
528
+ "\u0627\u0644\u0633\u062f",
529
+ "\u0627\u0644\u0645\u0643\u062a\u0628",
530
+ "\u062d\u0627\u0633\u0648\u0628 \u0645\u0643\u062a\u0628\u064a",
531
+ "\u0627\u0644\u0647\u0627\u062a\u0641 \u0627\u0644\u062f\u0648\u0627\u0631",
532
+ "\u0627\u0644\u062d\u0641\u0627\u0638\u0629",
533
+ "\u0627\u0644\u0633\u0627\u0639\u0629 \u0627\u0644\u0631\u0642\u0645\u064a\u0629",
534
+ "\u0633\u0627\u0639\u0627\u062a \u0627\u0644\u064a\u062f \u0627\u0644\u0631\u0642\u0645\u064a\u0629",
535
+ "\u0627\u0644\u0645\u0646\u0636\u062f\u0629",
536
+ "\u0642\u0645\u0627\u0634 \u0627\u0644\u0623\u0637\u0628\u0627\u0642",
537
+ "\u063a\u0633\u0627\u0644\u0629 \u0635\u062d\u0648\u0646",
538
+ "\u0645\u0643\u0628\u062d \u0642\u0631\u0635\u064a",
539
+ "\u0645\u064a\u0646\u0627\u0621",
540
+ "\u0627\u0644\u0632\u0644\u0627\u062c\u0629 \u0627\u0644\u062a\u064a \u062a\u062c\u0631\u0647\u0627 \u0627\u0644\u0643\u0644\u0627\u0628",
541
+ "\u0642\u0628\u0629",
542
+ "\u0627\u0644\u062d\u0635\u064a\u0631\u0629",
543
+ "\u0645\u0646\u0635\u0629 \u062d\u0641\u0631",
544
+ "\u0627\u0644\u0637\u0628\u0644",
545
+ "\u0639\u0635\u0627 \u0627\u0644\u0637\u0628\u0644",
546
+ "\u062b\u0642\u0627\u0644\u0627\u062a \u062d\u062f\u064a\u062f",
547
+ "\u0641\u0631\u0646 \u0647\u0648\u0644\u0646\u062f\u064a",
548
+ "\u0645\u0631\u0648\u062d\u0629",
549
+ "\u0627\u0644\u062c\u064a\u062a\u0627\u0631 \u0627\u0644\u0643\u0647\u0631\u0628\u0627\u0626\u064a",
550
+ "\u0627\u0644\u0642\u0627\u0637\u0631\u0629 \u0627\u0644\u0643\u0647\u0631\u0628\u0627\u0626\u064a\u0629",
551
+ "\u0645\u0631\u0643\u0632 \u0627\u0644\u062a\u0631\u0641\u064a\u0647",
552
+ "\u0638\u0631\u0641 \u0628\u0631\u064a\u062f\u064a",
553
+ "\u0622\u0644\u0629 \u0627\u0644\u0625\u0633\u0628\u0631\u064a\u0633\u0648",
554
+ "\u0628\u0648\u062f\u0631\u0629 \u0627\u0644\u0648\u062c\u0647",
555
+ "\u0623\u0635\u0644\u0629 \u0631\u064a\u0634\u064a\u0629",
556
+ "\u062e\u0632\u0627\u0646\u0629 \u0627\u0644\u0645\u0644\u0641\u0627\u062a",
557
+ "\u0632\u0648\u0631\u0642 \u0627\u0644\u0625\u0637\u0641\u0627\u0621",
558
+ "\u0633\u064a\u0627\u0631\u0629 \u0627\u0644\u0625\u0637\u0641\u0627\u0621",
559
+ "\u0648\u0627\u0642\u064a \u0627\u0644\u0646\u0627\u0631",
560
+ "\u0633\u0627\u0631\u064a\u0629 \u0627\u0644\u0639\u0644\u0645",
561
+ "\u0627\u0644\u0641\u0644\u0648\u062a",
562
+ "\u0643\u0631\u0633\u064a \u0642\u0627\u0628\u0644 \u0644\u0644\u0637\u064a",
563
+ "\u062e\u0648\u0630\u0629 \u0643\u0631\u0629 \u0627\u0644\u0642\u062f\u0645",
564
+ "\u0631\u0627\u0641\u0639\u0629 \u0627\u0644\u062d\u0645\u0648\u0644\u0629",
565
+ "\u0627\u0644\u0646\u0627\u0641\u0648\u0631\u0629",
566
+ "\u0642\u0644\u0645 \u062d\u0628\u0631 \u0633\u0627\u0626\u0644",
567
+ "\u0627\u0644\u0633\u0631\u064a\u0631 \u0628\u0623\u0631\u0628\u0639\u0629 \u0623\u0639\u0645\u062f\u0629",
568
+ "\u0633\u064a\u0627\u0631\u0629 \u0634\u062d\u0646",
569
+ "\u0627\u0644\u0628\u0648\u0642 \u0627\u0644\u0641\u0631\u0646\u0633\u064a",
570
+ "\u0645\u0642\u0644\u0627\u0629",
571
+ "\u0627\u0644\u0645\u0644\u0627\u0628\u0633 \u0627\u0644\u0645\u0635\u0646\u0648\u0639\u0629 \u0645\u0646 \u0627\u0644\u0641\u0631\u0648\u0629",
572
+ "\u0634\u0627\u062d\u0646\u0629 \u0642\u0645\u0627\u0645\u0629",
573
+ "\u0642\u0646\u0627\u0639 \u0627\u0644\u063a\u0627\u0632",
574
+ "\u0645\u0636\u062e\u0629 \u0627\u0644\u0648\u0642\u0648\u062f",
575
+ "\u0643\u0623\u0633 \u0627\u0644\u0646\u0628\u064a\u0630",
576
+ "\u0633\u064a\u0627\u0631\u0629 \u062c\u0648 \u0643\u0627\u0631\u062a",
577
+ "\u0643\u0631\u0629 \u0627\u0644\u062c\u0648\u0644\u0641",
578
+ "\u0639\u0631\u0628\u0629 \u0627\u0644\u062c\u0648\u0644\u0641",
579
+ "\u0627\u0644\u0642\u0627\u0631\u0628 \u0627\u0644\u0637\u0648\u064a\u0644 \u0623\u0648 \u0627\u0644\u063a\u0646\u062f\u0648\u0644",
580
+ "\u0627\u0644\u0635\u0646\u062c\u0629",
581
+ "\u0627\u0644\u062b\u0648\u0628 \u0627\u0644\u0646\u0633\u0627\u0626\u064a \u0623\u0648 \u0627\u0644\u0641\u0633\u062a\u0627\u0646",
582
+ "\u0627\u0644\u0628\u064a\u0627\u0646\u0648 \u0627\u0644\u0643\u0628\u064a\u0631",
583
+ "\u0627\u0644\u062f\u0641\u064a\u0626\u0629 \u0627\u0644\u0632\u0631\u0627\u0639\u064a\u0629",
584
+ "\u0634\u0628\u0643 \u0627\u0644\u0633\u064a\u0627\u0631\u0629",
585
+ "\u0627\u0644\u0628\u0642\u0627\u0644\u0629",
586
+ "\u0627\u0644\u0645\u0642\u0635\u0644\u0629",
587
+ "\u0645\u0634\u0628\u0643 \u0644\u0644\u0634\u0639\u0631",
588
+ "\u0628\u062e\u0627\u062e \u0645\u062b\u0628\u062a \u0627\u0644\u0634\u0639\u0631",
589
+ "\u0627\u0644\u0639\u0631\u0628\u0629 \u0646\u0635\u0641 \u0627\u0644\u0645\u062c\u0646\u0632\u0631\u0629",
590
+ "\u0645\u0637\u0631\u0642\u0629",
591
+ "\u0627\u0644\u0633\u0644\u0629",
592
+ "\u0645\u062c\u0641\u0641 \u0627\u0644\u0634\u0639\u0631",
593
+ "\u062c\u0647\u0627\u0632 \u0645\u062d\u0645\u0648\u0644 \u0628\u0627\u0644\u064a\u062f",
594
+ "\u0627\u0644\u0645\u0646\u062f\u064a\u0644 \u0623\u0648 \u0627\u0644\u0645\u062d\u0631\u0645\u0629",
595
+ "\u0642\u0631\u0635 \u0635\u0644\u0628",
596
+ "\u0627\u0644\u0634\u064e\u0651\u0641\u064e\u0648\u0650\u064a\u064e\u0651\u0629",
597
+ "\u0627\u0644\u0642\u064a\u062b\u0627\u0631\u0629",
598
+ "\u0627\u0644\u062d\u0635\u0651\u0627\u062f\u0629",
599
+ "\u062e\u0635\u064a\u0646",
600
+ "\u062d\u0627\u0641\u0638\u0629 \u0627\u0644\u0645\u0633\u062f\u0633",
601
+ "\u0627\u0644\u0645\u0633\u0631\u062d \u0627\u0644\u0645\u0646\u0632\u0644\u064a",
602
+ "\u0642\u0631\u0635 \u0639\u0633\u0644",
603
+ "\u0627\u0644\u062e\u0637\u0627\u0641",
604
+ "\u0627\u0644\u062a\u0646\u0648\u0631\u0629 \u0627\u0644\u0645\u064f\u0637\u064e\u0648\u064e\u0651\u0642\u0629",
605
+ "\u0639\u0642\u0644\u0629 (\u062c\u0645\u0628\u0627\u0632)",
606
+ "\u0639\u0631\u0628\u0629 \u0627\u0644\u062e\u064a\u0648\u0644",
607
+ "\u0633\u0627\u0639\u0629 \u0631\u0645\u0644\u064a\u0629",
608
+ "\u0622\u064a \u0628\u0648\u062f",
609
+ "\u0627\u0644\u0645\u0643\u0648\u0627\u0629",
610
+ "\u0627\u0644\u0642\u0631\u0639\u0629 \u0627\u0644\u0645\u0636\u064a\u0626\u0629",
611
+ "\u0627\u0644\u062c\u064a\u0646\u0632",
612
+ "\u062c\u064a\u0628 (\u0633\u064a\u0627\u0631\u0629)",
613
+ "\u0642\u0645\u064a\u0635 \u0642\u0635\u064a\u0631 \u0627\u0644\u0643\u0645\u064a\u0646",
614
+ "\u0623\u062d\u062c\u064a\u0629 \u0627\u0644\u0635\u0648\u0631 \u0627\u0644\u0645\u0642\u0637\u0648\u0639\u0629",
615
+ "\u0631\u064a\u0643\u0634\u0627",
616
+ "\u0630\u0631\u0627\u0639 \u0627\u0644\u062a\u0648\u062c\u064a\u0647",
617
+ "\u0627\u0644\u0643\u064a\u0645\u0648\u0646\u0648",
618
+ "\u0648\u0633\u0627\u062f\u0627\u062a \u0627\u0644\u0631\u0643\u0628\u0629",
619
+ "\u0627\u0644\u0639\u0642\u062f\u0629",
620
+ "\u0645\u0639\u0637\u0641 \u0627\u0644\u0645\u062e\u062a\u0628\u0631",
621
+ "\u0627\u0644\u0645\u063a\u0631\u0641\u0629",
622
+ "\u0639\u0627\u0643\u0633 \u0627\u0644\u0636\u0648\u0621",
623
+ "\u062d\u0627\u0633\u0648\u0628 \u0645\u062d\u0645\u0648\u0644",
624
+ "\u062c\u0632\u0627\u0632\u0629 \u0627\u0644\u0639\u0634\u0628",
625
+ "\u063a\u0637\u0627\u0621 \u0627\u0644\u0639\u062f\u0633\u0629",
626
+ "\u0641\u062a\u0627\u062d\u0629 \u0627\u0644\u0631\u0633\u0627\u0626\u0644",
627
+ "\u0645\u0643\u062a\u0628\u0629",
628
+ "\u0642\u0627\u0631\u0628 \u0627\u0644\u0646\u062c\u0627\u0629",
629
+ "\u0627\u0644\u0642\u064e\u062f\u064e\u0651\u0627\u062d\u064e\u0629",
630
+ "\u0627\u0644\u0644\u064a\u0645\u0648\u0632\u064a\u0646",
631
+ "\u0639\u0627\u0628\u0631\u0629 \u0645\u062d\u064a\u0637 \u0645\u0646\u062a\u0638\u0645\u0629",
632
+ "\u0623\u062d\u0645\u0631 \u0634\u0641\u0627\u0647",
633
+ "\u0627\u0644\u062d\u0630\u0627\u0621 \u0633\u0647\u0644 \u0627\u0644\u0627\u0631\u062a\u062f\u0627\u0621",
634
+ "\u063a\u0633\u0648\u0644",
635
+ "\u0645\u0643\u0628\u0631 \u0627\u0644\u0635\u0648\u062a",
636
+ "\u0639\u062f\u0633\u0629",
637
+ "\u0627\u0644\u0645\u0646\u0634\u0631\u0629",
638
+ "\u0627\u0644\u0628\u0648\u0635\u0644\u0629",
639
+ "\u062d\u0642\u064a\u0628\u0629 \u0633\u0627\u0639\u064a \u0627\u0644\u0628\u0631\u064a\u062f",
640
+ "\u0635\u0646\u062f\u0648\u0642 \u0628\u0631\u064a\u062f",
641
+ "\u0627\u0644\u062c\u0648\u0627\u0631\u0628 \u0627\u0644\u0637\u0648\u064a\u0644\u0629",
642
+ "\u0627\u0644\u0645\u0627\u064a\u0648\u0647",
643
+ "\u063a\u0637\u0627\u0621 \u0627\u0644\u0645\u0637\u0628\u0642",
644
+ "\u0622\u0644\u0629 \u0645\u0627\u0631\u0627\u0643\u0633",
645
+ "\u0627\u0644\u0645\u0627\u0631\u064a\u0645\u0628\u0627",
646
+ "\u0642\u0646\u0627\u0639",
647
+ "\u0623\u0639\u0648\u0627\u062f \u0627\u0644\u062b\u0642\u0627\u0628",
648
+ "\u0633\u0627\u0631\u064a\u0629 \u0645\u0627\u064a\u0648",
649
+ "\u0627\u0644\u0645\u062a\u0627\u0647\u0629",
650
+ "\u0643\u0648\u0628 \u0627\u0644\u0642\u064a\u0627\u0633",
651
+ "\u062e\u0632\u0627\u0646\u0629 \u0627\u0644\u0623\u062f\u0648\u064a\u0629",
652
+ "\u0627\u0644\u0622\u062b\u0627\u0631 \u0627\u0644\u0635\u062e\u0631\u064a\u0629",
653
+ "\u0627\u0644\u0644\u0627\u0642\u0637 \u0627\u0644\u0635\u0648\u062a\u064a",
654
+ "\u0641\u0631\u0646 \u0627\u0644\u0645\u064a\u0643\u0631\u0648\u064a\u0641",
655
+ "\u0627\u0644\u0632\u064a \u0627\u0644\u0639\u0633\u0643\u0631\u064a",
656
+ "\u0645\u062f\u0644\u062c\u0629",
657
+ "\u0627\u0644\u062d\u0627\u0641\u0644\u0629 \u0627\u0644\u0635\u063a\u064a\u0631\u0629",
658
+ "\u0627\u0644\u062a\u0646\u0648\u0631\u0629 \u0627\u0644\u0642\u0635\u064a\u0631\u0629",
659
+ "\u0633\u064a\u0627\u0631\u0629 \u0627\u0644\u0645\u064a\u0646\u064a \u0641\u0627\u0646 \u0627\u0644\u0639\u0627\u0626\u0644\u064a\u0629",
660
+ "\u0627\u0644\u0642\u0630\u064a\u0641\u0629 \u0627\u0644\u0645\u0648\u062c\u0647\u0629",
661
+ "\u0627\u0644\u0642\u0641\u0627\u0632 \u0645\u0644\u062a\u0635\u0642 \u0627\u0644\u0623\u0635\u0627\u0628\u0639",
662
+ "\u0637\u0628\u0642 \u062e\u0644\u0637",
663
+ "\u0627\u0644\u0645\u0646\u0632\u0644 \u0627\u0644\u0645\u062a\u0646\u0642\u0644",
664
+ "\u0641\u0648\u0631\u062f \u0645\u0648\u062f\u064a\u0644 \u062a\u064a",
665
+ "\u0627\u0644\u0645\u0648\u062f\u0645",
666
+ "\u0627\u0644\u062f\u064a\u0631",
667
+ "\u0634\u0627\u0634\u0629 \u062d\u0627\u0633\u0648\u0628",
668
+ "\u0627\u0644\u062f\u0631\u0627\u062c\u0629 \u0627\u0644\u0646\u0627\u0631\u064a\u0629 \u0627\u0644\u0635\u063a\u064a\u0631\u0629",
669
+ "\u0627\u0644\u0647\u0627\u0648\u0646 \u0648\u0627\u0644\u0645\u062f\u0642\u0629",
670
+ "\u0627\u0644\u0642\u0628\u0639\u0629 \u0627\u0644\u062c\u0627\u0645\u0639\u064a\u0629 \u0627\u0644\u0645\u0631\u0628\u0639\u0629",
671
+ "\u0645\u0633\u062c\u062f",
672
+ "\u0627\u0644\u0646\u0627\u0645\u0648\u0633\u064a\u0651\u0629",
673
+ "\u0627\u0644\u062f\u0639\u0631\u0648\u0645\u0629",
674
+ "\u0627\u0644\u062f\u0631\u0627\u062c\u0629 \u0627\u0644\u0647\u0648\u0627\u0626\u064a\u0629 \u0627\u0644\u062c\u0628\u0644\u064a\u0629",
675
+ "\u062e\u064a\u0645\u0629",
676
+ "\u0627\u0644\u0641\u0623\u0631\u0629",
677
+ "\u0645\u0635\u064a\u062f\u0629 \u0627\u0644\u0641\u0626\u0631\u0627\u0646",
678
+ "\u0633\u064a\u0627\u0631\u0627\u062a \u0634\u0631\u0643\u0629 \u0627\u0644\u0646\u0642\u0644",
679
+ "\u0643\u0645\u0627\u0645 \u0627\u0644\u0641\u0645",
680
+ "\u0645\u0633\u0645\u0627\u0631",
681
+ "\u0637\u0648\u0642 \u0627\u0644\u0639\u0646\u0642",
682
+ "\u0627\u0644\u0642\u0644\u0627\u062f\u0629",
683
+ "\u0627\u0644\u0631\u0636\u0627\u0639\u0629",
684
+ "\u062d\u0627\u0633\u0628 \u0627\u0644\u0645\u0641\u0643\u0631\u0629",
685
+ "\u0627\u0644\u0645\u0633\u0644\u0629",
686
+ "\u0622\u0644\u0629 \u0627\u0644\u0623\u0648\u0628\u0648\u0627",
687
+ "\u0623\u0643\u0631\u064a\u0646\u0629",
688
+ "\u0639\u062f\u0627\u062f \u0627\u0644\u0645\u0633\u0627\u0641\u0627\u062a",
689
+ "\u0641\u0644\u062a\u0631 \u0627\u0644\u0632\u064a\u062a",
690
+ "\u0627\u0644\u0623\u0631\u063a\u0646 \u0630\u0648 \u0627\u0644\u0623\u0646\u0627\u0628\u064a\u0628",
691
+ "\u062c\u0647\u0627\u0632 \u0631\u0627\u0633\u0645 \u0627\u0644\u0625\u0634\u0627\u0631\u0629",
692
+ "\u0627\u0644\u062a\u0646\u0648\u0631\u0629 \u0627\u0644\u062e\u0627\u0631\u062c\u064a\u0629",
693
+ "\u0639\u0631\u0628\u0629 \u064a\u062c\u0631\u0647\u0627 \u0627\u0644\u062b\u0648\u0631",
694
+ "\u0642\u0646\u0627\u0639 \u0623\u0643\u0633\u062c\u064a\u0646",
695
+ "\u0627\u0644\u062a\u063a\u0644\u064a\u0641",
696
+ "\u0627\u0644\u0645\u0650\u063a\u0652\u062f\u0627\u0641",
697
+ "\u0639\u064e\u062c\u064e\u0644\u0629 \u0627\u0644\u062a\u064e\u063a\u062f\u064a\u0641",
698
+ "\u0642\u0641\u0644 \u062d\u0644\u0642\u064a",
699
+ "\u0641\u0631\u0634\u0627\u0629 \u0627\u0644\u0631\u0633\u0645",
700
+ "\u0627\u0644\u0645\u0650\u0646\u064e\u0627\u0645\u064e\u0629\u064f",
701
+ "\u0642\u0635\u0631",
702
+ "\u0627\u0644\u0645\u0650\u0635\u0641\u0627\u0631",
703
+ "\u0645\u0646\u0634\u0641\u0629 \u0648\u0631\u0642\u064a\u0629",
704
+ "\u0645\u0650\u0638\u064e\u0644\u064e\u0651\u0629 \u0627\u0644\u0647\u0628\u064f\u0648\u0637",
705
+ "\u062c\u0647\u0627\u0632 \u0627\u0644\u0639\u0642\u0644\u0629",
706
+ "\u0645\u0642\u0639\u062f \u0639\u0627\u0645",
707
+ "\u0639\u062f\u0627\u062f \u0627\u0646\u062a\u0638\u0627\u0631 \u0627\u0644\u0633\u064a\u0627\u0631\u0627\u062a",
708
+ "\u0639\u0631\u0628\u0629 \u0627\u0644\u0642\u0637\u0627\u0631",
709
+ "\u0627\u0644\u0641\u0646\u0627\u0621",
710
+ "\u0627\u0644\u0647\u0627\u062a\u0641 \u0627\u0644\u0639\u0645\u0648\u0645\u064a",
711
+ "\u0627\u0644\u0631\u0643\u064a\u0632\u0629",
712
+ "\u0627\u0644\u0645\u064e\u0642\u0652\u0644\u064e\u0645\u064e\u0629",
713
+ "\u0627\u0644\u0645\u0628\u0631\u0627\u0629",
714
+ "\u0627\u0644\u0639\u0637\u0631",
715
+ "\u0637\u0628\u0642 \u0628\u062a\u0631\u064a",
716
+ "\u0627\u0644\u0622\u0644\u0629 \u0627\u0644\u0646\u0627\u0633\u062e\u0629",
717
+ "\u0627\u0644\u0631\u064a\u0634\u0629 \u0627\u0644\u0645\u0648\u0633\u064a\u0642\u064a\u0629",
718
+ "\u062e\u0648\u0630\u0629 \u0628\u064a\u0643\u0644\u0647\u0627\u0648\u0628\u0647",
719
+ "\u0627\u0644\u0633\u064a\u0627\u062c \u0627\u0644\u0648\u062a\u062f\u064a",
720
+ "\u0627\u0644\u0633\u064a\u0627\u0631\u0629 \u0627\u0644\u0646\u0635\u0641-\u0646\u0642\u0644",
721
+ "\u0627\u0644\u0631\u0635\u064a\u0641 \u0627\u0644\u0628\u062d\u0631\u064a",
722
+ "\u062d\u0635\u0627\u0644\u0629",
723
+ "\u0627\u0644\u062a\u063a\u0644\u064a\u0641 \u0627\u0644\u0635\u064a\u062f\u0644\u0627\u0646\u064a",
724
+ "\u0648\u0633\u0627\u062f\u0629",
725
+ "\u0643\u0631\u0629 \u0627\u0644\u0637\u0627\u0648\u0644\u0629",
726
+ "\u0644\u0639\u0628\u0629 \u0637\u0627\u062d\u0648\u0646\u0629 \u0647\u0648\u0627\u0621",
727
+ "\u0633\u0641\u064a\u0646\u0629 \u0627\u0644\u0642\u0631\u0627\u0635\u0646\u0629",
728
+ "\u0627\u0644\u0625\u0628\u0631\u064a\u0642",
729
+ "\u0627\u0644\u0645\u0650\u0633\u0652\u062d\u064e\u062c",
730
+ "\u0627\u0644\u0642\u0628\u0629 \u0627\u0644\u0641\u0644\u0643\u064a\u0629",
731
+ "\u0643\u064a\u0633 \u0646\u0627\u064a\u0644\u0648\u0646",
732
+ "\u0631\u0641 \u062a\u0646\u0634\u064a\u0641 \u0627\u0644\u0623\u0637\u0628\u0627\u0642",
733
+ "\u0627\u0644\u0645\u062d\u0627\u0631\u064a\u062b \u0627\u0644\u062d\u0641\u0627\u0631\u0629",
734
+ "\u0627\u0644\u0645\u0643\u0628\u0633 \u0627\u0644\u063a\u0637\u064e\u0651\u0627\u0633",
735
+ "\u0627\u0644\u0643\u0627\u0645\u064a\u0631\u0627 \u0627\u0644\u0641\u0648\u0631\u064a\u0629",
736
+ "\u0627\u0644\u0639\u0645\u0648\u062f",
737
+ "\u0639\u0631\u0628\u0629 \u0627\u0644\u0634\u0631\u0637\u0629",
738
+ "\u0644\u0628\u0627\u0633 \u0627\u0644\u0628\u0646\u0634",
739
+ "\u0637\u0627\u0648\u0644\u0629 \u0627\u0644\u0628\u0644\u064a\u0627\u0631\u062f\u0648",
740
+ "\u0639\u0628\u0648\u0629 \u0627\u0644\u0645\u0634\u0631\u0648\u0628\u0627\u062a",
741
+ "\u0627\u0644\u0623\u0635\u064a\u0635",
742
+ "\u0639\u062c\u0644\u0629 \u0641\u062e\u0627\u0631",
743
+ "\u0627\u0644\u0645\u0650\u062b\u0652\u0642\u064e\u0628",
744
+ "\u0633\u062c\u0627\u062f\u0629 \u0627\u0644\u0635\u0644\u0627\u0629",
745
+ "\u0627\u0644\u0637\u0627\u0628\u0639\u0629 \u0627\u0644\u062d\u0627\u0633\u0648\u0628\u064a\u0629",
746
+ "\u0627\u0644\u0633\u062c\u0646",
747
+ "\u0627\u0644\u0642\u0630\u064a\u0641\u0629",
748
+ "\u0627\u0644\u0628\u0631\u0648\u062c\u0643\u062a\u0631",
749
+ "\u0642\u0631\u0635 \u0627\u0644\u0647\u0648\u0643\u064a",
750
+ "\u0627\u0644\u0645\u0644\u0643\u0645\u0629",
751
+ "\u062d\u0642\u064a\u0628\u0629 \u064a\u062f",
752
+ "\u0627\u0644\u0631\u064a\u0634\u0629 ",
753
+ "\u0627\u0644\u0644\u0650\u062d\u064e\u0627\u0641",
754
+ "\u0633\u064a\u0627\u0631\u0629 \u0627\u0644\u0633\u0628\u0627\u0642",
755
+ "\u0645\u0636\u0631\u0628 \u0627\u0644\u062a\u0646\u0633",
756
+ "\u0645\u0634\u0639\u0627\u0639",
757
+ "\u0627\u0644\u0645\u0630\u064a\u0627\u0639",
758
+ "\u0627\u0644\u0645\u0642\u0631\u0627\u0628 \u0627\u0644\u0627\u0630\u0627\u0639\u064a",
759
+ "\u062e\u0632\u0627\u0646 \u0645\u064a\u0627\u0647 \u0627\u0644\u0623\u0645\u0637\u0627\u0631",
760
+ "\u0627\u0644\u0645\u0631\u0643\u0628\u0627\u062a \u0627\u0644\u062a\u0631\u0641\u064a\u0647\u064a\u0629",
761
+ "\u0628\u0643\u0631\u0629 \u0635\u064a\u062f",
762
+ "\u0627\u0644\u0643\u0627\u0645\u064a\u0631\u0627 \u0627\u0644\u0627\u0646\u0639\u0643\u0627\u0633\u064a\u0629",
763
+ "\u0627\u0644\u062b\u0644\u0627\u064e\u0651\u062c\u0629",
764
+ "\u062c\u0647\u0627\u0632 \u062a\u062d\u0643\u0645 \u0639\u0646 \u0628\u0639\u062f",
765
+ "\u0645\u0637\u0639\u0645",
766
+ "\u0627\u0644\u0645\u0633\u062f\u0633 \u0627\u0644\u062f\u0648\u0627\u0631",
767
+ "\u0628\u0646\u062f\u0642\u064a\u0629",
768
+ "\u0627\u0644\u0643\u0631\u0633\u064a \u0627\u0644\u0647\u0632\u0627\u0632",
769
+ "\u0627\u0644\u0645\u0634\u0648\u0627\u0629",
770
+ "\u0627\u0644\u0645\u0645\u062d\u0627\u0629",
771
+ "\u0643\u0631\u0629 \u0627\u0644\u0631\u063a\u0628\u064a",
772
+ "\u0627\u0644\u0645\u0633\u0637\u0631\u0629",
773
+ "\u0627\u0644\u062d\u0630\u0627\u0621 \u0627\u0644\u0631\u064a\u0627\u0636\u064a",
774
+ "\u0627\u0644\u062e\u0632\u0627\u0646\u0629",
775
+ "\u062f\u0628\u0648\u0633 \u0645\u0634\u0628\u0643",
776
+ "\u0639\u0644\u0628 \u0627\u0644\u0645\u0644\u062d \u0648\u0627\u0644\u0641\u0644\u0641\u0644",
777
+ "\u0627\u0644\u0635\u0646\u062f\u0644",
778
+ "\u0627\u0644\u0633\u0627\u0631\u0648\u0646\u062c",
779
+ "\u0633\u0627\u0643\u0633\u0641\u0648\u0646 \u0627\u0644\u0629 \u0646\u0641\u062e \u0645\u0648\u0633\u064a\u0642\u064a\u0629",
780
+ "\u0627\u0644\u063a\u0650\u0645\u0652\u062f \u0623\u0648 \u063a\u0650\u0645\u0652\u062f \u0627\u0644\u0633\u064a\u0641 \u0623\u0648 \u063a\u0650\u0645\u0652\u062f \u0627\u0644\u062e\u0646\u062c\u0631",
781
+ "\u0627\u0644\u0645\u064a\u0632\u0627\u0646",
782
+ "\u062d\u0627\u0641\u0644\u0629 \u0645\u062f\u0631\u0633\u064a\u0629",
783
+ "\u0627\u0644\u0645\u0631\u0643\u0628 \u0627\u0644\u0634\u0631\u0627\u0639\u064a ",
784
+ "\u0644\u0648\u062d\u0629 \u0627\u0644\u0646\u062a\u0627\u0626\u062c",
785
+ "\u0634\u0627\u0634\u0629 \u0627\u0644\u0633\u064a \u0623\u0631 \u062a\u064a",
786
+ "\u0628\u0631\u063a\u064a",
787
+ "\u0645\u0641\u0643 \u0627\u0644\u0628\u0631\u0627\u063a\u064a",
788
+ "\u062d\u0632\u0627\u0645 \u0627\u0644\u0623\u0645\u0627\u0646",
789
+ "\u0622\u0644\u0629 \u0627\u0644\u062e\u064a\u0627\u0637\u0629",
790
+ "\u0627\u0644\u062f\u0631\u0639",
791
+ "\u0645\u062a\u062c\u0631 \u0627\u0644\u0623\u062d\u0630\u064a\u0629",
792
+ "\u0645\u0642\u0633\u0645 \u0627\u0644\u063a\u0631\u0641\u0629",
793
+ "\u0633\u0644\u0629 \u0627\u0644\u062a\u0633\u0648\u0642",
794
+ "\u0639\u0631\u0628\u0629 \u0627\u0644\u062a\u0633\u0648\u0642",
795
+ "\u0627\u0644\u0645\u0650\u062c\u0652\u0631\u064e\u0641\u064e\u0629",
796
+ "\u0642\u0628\u0639\u0629 \u0627\u0644\u0627\u0633\u062a\u062d\u0645\u0627\u0645",
797
+ "\u0633\u062a\u0627\u0626\u0631 \u0627\u0644\u0627\u0633\u062a\u062d\u0645\u0627\u0645",
798
+ "\u062a\u0632\u062d\u0644\u0642 \u0639\u0644\u0649 \u0627\u0644\u062b\u0644\u062c",
799
+ "\u0642\u0646\u0627\u0639 \u0627\u0644\u062a\u0632\u0644\u062c",
800
+ "\u0643\u064a\u0633 \u0627\u0644\u0646\u0648\u0645",
801
+ "\u0627\u0644\u0645\u0633\u0637\u0631\u0629 \u0627\u0644\u062d\u0627\u0633\u0628\u0629",
802
+ "\u0627\u0644\u0628\u0627\u0628 \u0627\u0644\u0645\u0646\u0632\u0644\u0642",
803
+ "\u0645\u0627\u0643\u064a\u0646\u0629 \u0627\u0644\u062d\u0638",
804
+ "\u0627\u0644\u063a\u0637\u0633 \u062a\u062d\u062a \u0627\u0644\u0645\u0627\u0621",
805
+ "\u0639\u0631\u0628\u0629 \u0627\u0644\u062c\u0644\u064a\u062f \u0627\u0644\u0622\u0644\u064a\u0629",
806
+ "\u0643\u0627\u0633\u062d\u0629 \u062b\u0644\u0648\u062c",
807
+ "\u0645\u0648\u0632\u0639 \u0627\u0644\u0635\u0627\u0628\u0648\u0646",
808
+ "\u0643\u0631\u0629 (\u0643\u0631\u0629 \u0627\u0644\u0642\u062f\u0645)",
809
+ "\u062c\u0648\u0631\u0628",
810
+ "\u0645\u062c\u0645\u0639 \u0627\u0644\u0637\u0627\u0642\u0629 \u0627\u0644\u0634\u0645\u0633\u064a\u0629 \u0627\u0644\u062d\u0631\u0627\u0631\u064a\u0629",
811
+ "\u0627\u0644\u0635\u064e\u0645\u0652\u0628\u0631\u0650\u064a\u0631\u0629",
812
+ "\u0648\u0639\u0627\u0621 \u0627\u0644\u0634\u0648\u0631\u0628\u0629",
813
+ "\u0645\u0641\u062a\u0627\u062d \u0627\u0644\u0645\u0633\u0627\u0641\u0629",
814
+ "\u0627\u0644\u0645\u062f\u0641\u0623\u0629",
815
+ "\u0645\u0643\u0648\u0643 \u0627\u0644\u0641\u0636\u0627\u0621",
816
+ "\u0627\u0644\u0645\u0650\u0644\u0648\u064e\u0642",
817
+ "\u0627\u0644\u0642\u0627\u0631\u0628 \u0627\u0644\u0633\u0631\u064a\u0639",
818
+ "\u0634\u0628\u0643\u0629 \u0627\u0644\u0639\u0646\u0643\u0628\u0648\u062a",
819
+ "\u062e\u0634\u0628\u0629 \u0627\u0644\u0645\u063a\u0632\u0644",
820
+ "\u0627\u0644\u0633\u064a\u0627\u0631\u0629 \u0627\u0644\u0631\u064a\u0627\u0636\u064a\u0629",
821
+ "\u0628\u0642\u0639\u0629 \u0636\u0648\u0621",
822
+ "\u0633\u0637\u062d \u0627\u0644\u0645\u0633\u0631\u062d",
823
+ "\u0627\u0644\u0642\u0627\u0637\u0631\u0629 \u0627\u0644\u0628\u062e\u0627\u0631\u064a\u0629",
824
+ "\u062c\u0633\u0631 \u0645\u0642\u0648\u0633 \u0646\u0641\u0642\u064a",
825
+ "\u0627\u0644\u0637\u0628\u0644 \u0627\u0644\u0646\u062d\u0627\u0633\u064a",
826
+ "\u0627\u0644\u0633\u0645\u0627\u0639\u0629 \u0627\u0644\u0637\u0628\u064a\u0629",
827
+ "\u0627\u0644\u0644\u0650\u0641\u0627\u0639",
828
+ "\u0627\u0644\u062c\u062f\u0627\u0631 \u0627\u0644\u062c\u0627\u0641",
829
+ "\u0645\u0624\u0642\u062a",
830
+ "\u0627\u0644\u0645\u0648\u0642\u062f",
831
+ "\u0627\u0644\u063a\u0631\u0628\u0627\u0644",
832
+ "\u0627\u0644\u062a\u0631\u0627\u0645",
833
+ "\u0627\u0644\u0646\u0642\u0627\u0644\u0629",
834
+ "\u0627\u0644\u0623\u0631\u064a\u0643\u0629",
835
+ "\u0645\u0628\u0646\u0649 \u0633\u062a\u0648\u064a\u0627",
836
+ "\u0627\u0644\u063a\u0648\u0627\u0635\u0629",
837
+ "\u0627\u0644\u0628\u0630\u0644\u0629",
838
+ "\u0627\u0644\u0645\u0632\u0648\u0644\u0629",
839
+ "\u0627\u0644\u0646\u0638\u0627\u0631\u0629 \u0627\u0644\u0634\u0645\u0633\u064a\u0629",
840
+ "\u0627\u0644\u0646\u0638\u0627\u0631\u0629 \u0627\u0644\u0634\u0645\u0633\u064a\u0629",
841
+ "\u0627\u0644\u0648\u0627\u0642\u064a \u0627\u0644\u0634\u0645\u0633\u064a",
842
+ "\u0627\u0644\u062c\u0633\u0631 \u0627\u0644\u0645\u0639\u0644\u0642",
843
+ "\u0627\u0644\u0645\u0645\u0633\u062d\u0629",
844
+ "\u0627\u0644\u0642\u0645\u064a\u0635 \u0627\u0644\u062b\u0642\u064a\u0644",
845
+ "\u0627\u0644\u062a\u064f\u0628\u0651\u0627\u0646 \u0623\u0648 \u0627\u0644\u0628\u0646\u0637\u0627\u0644 \u0627\u0644\u0642\u0635\u064a\u0631\\",
846
+ "\u0627\u0644\u0623\u0631\u062c\u0648\u062d\u0629",
847
+ "\u0627\u0644\u0645\u0641\u062a\u0627\u062d \u0627\u0644\u0643\u0647\u0631\u0628\u0627\u0626\u064a",
848
+ "\u0645\u062d\u0642\u0646\u0629 \u0623\u0648 \u0627\u0644\u0625\u0628\u0631\u0629",
849
+ "\u0627\u0644\u0623\u0628\u0627\u062c\u0648\u0631\u0629",
850
+ "\u0627\u0644\u062f\u0628\u0627\u0628\u0629",
851
+ "\u0645\u0633\u062c\u0644 \u0627\u0644\u0634\u0631\u064a\u0637 \u0627\u0644\u0635\u0648\u062a\u064a",
852
+ "\u0625\u0628\u0631\u064a\u0642 \u0627\u0644\u0634\u0627\u064a",
853
+ "\u0627\u0644\u062f\u0628\u062f\u0648\u0628",
854
+ "\u0627\u0644\u0631\u0627\u0626\u064a",
855
+ "\u0643\u0631\u0629 \u062a\u0646\u0633",
856
+ "\u0627\u0644\u062a\u0633\u0642\u064a\u0641 \u0628\u0627\u0644\u0642\u0634",
857
+ "\u0627\u0644\u0633\u062a\u0627\u0631\u0629 \u0627\u0644\u0645\u0633\u0631\u062d\u064a\u0629",
858
+ "\u0627\u0644\u0643\u064f\u0634\u0652\u062a\u0650\u0628\u064e\u0627\u0646",
859
+ "\u0627\u0644\u062f\u0631\u064e\u0651\u0627\u0633\u0629",
860
+ "\u0639\u0631\u0634",
861
+ "\u0628\u0644\u0627\u0637 \u0627\u0644\u0633\u0642\u0641",
862
+ "\u0622\u0644\u0629 \u062a\u062d\u0645\u064a\u0635 \u0627\u0644\u062e\u0628\u0632",
863
+ "\u0645\u062d\u0644\u0627\u062a \u0628\u064a\u0639 \u0644\u0648\u0627\u0632\u0645 \u0627\u0644\u062a\u062f\u062e\u064a\u0646",
864
+ "\u0645\u0642\u0639\u062f \u0627\u0644\u0645\u0631\u062d\u0627\u0636",
865
+ "\u0645\u0634\u0639\u0644\u0629",
866
+ "\u0627\u0644\u0623\u064e\u0639\u0652\u0645\u0650\u062f\u064e\u0629\u064f \u0627\u0644\u0637\u064e\u0651\u0648\u0652\u0637\u064e\u0645\u0650\u064a\u064e\u0651\u0629\u0650",
867
+ "\u0634\u0627\u062d\u0646\u0629 \u0627\u0644\u0642\u0637\u0631",
868
+ "\u0645\u062a\u062c\u0631 \u0627\u0644\u0623\u0644\u0639\u0627\u0628",
869
+ "\u0633\u064a\u0627\u0631\u0629 \u0627\u0644\u062c\u0631\u0627\u0631",
870
+ "\u0634\u0627\u062d\u0646\u0629 \u0646\u0635\u0641 \u0645\u0642\u0637\u0648\u0631\u0629",
871
+ "\u0627\u0644\u0635\u064a\u0646\u064a\u0629",
872
+ "\u0645\u0639\u0637\u0641 \u0627\u0644\u062e\u0646\u062f\u0642",
873
+ "\u0627\u0644\u062f\u0631\u0627\u062c\u0629 \u062b\u0644\u0627\u062b\u064a\u0629 \u0627\u0644\u0639\u062c\u0644\u0627\u062a",
874
+ "\u0642\u0627\u0631\u0628 \u0627\u0644\u062f\u0639\u0627\u0645\u0629 \u0627\u0644\u0645\u0632\u062f\u0648\u062c\u0629",
875
+ "\u062d\u0627\u0645\u0644 \u062b\u0644\u0627\u062b\u064a",
876
+ "\u0642\u0648\u0633 \u0627\u0644\u0646\u0635\u0631",
877
+ "\u0627\u0644\u062d\u0627\u0641\u0644\u0629 \u0633\u0637\u062d\u064a\u0629 \u0627\u0644\u062a\u0645\u062f\u064a\u062f \u0627\u0644\u0643\u0647\u0631\u0628\u0627\u0626\u064a",
878
+ "\u0627\u0644\u062a\u0631\u0648\u0645\u0628\u0648\u0646",
879
+ "\u062d\u0648\u0636 \u0627\u0644\u0627\u0633\u062a\u062d\u0645\u0627\u0645",
880
+ "\u0627\u0644\u0628\u0648\u0627\u0628\u0629 \u0627\u0644\u062f\u0648\u0627\u0631\u0629",
881
+ "\u0627\u0644\u0622\u0644\u0629 \u0627\u0644\u0643\u0627\u062a\u0628\u0629",
882
+ "\u0627\u0644\u0645\u0638\u0644\u0629",
883
+ "\u0627\u0644\u062f\u0631\u0627\u062c\u0629 \u0627\u0644\u0623\u062d\u0627\u062f\u064a\u0629",
884
+ "\u0627\u0644\u0628\u064a\u0627\u0646\u0648 \u0627\u0644\u0642\u0627\u0626\u0645",
885
+ "\u0627\u0644\u0645\u0643\u0646\u0633\u0629 \u0627\u0644\u0643\u0647\u0631\u0628\u0627\u0626\u064a\u0629",
886
+ "\u0627\u0644\u0645\u0632\u0647\u0631\u064a\u0629",
887
+ "\u0627\u0644\u0642\u0646\u0637\u0631\u0629",
888
+ "\u0627\u0644\u0642\u0645\u0627\u0634 \u0627\u0644\u0645\u062e\u0645\u0644\u064a",
889
+ "\u0622\u0644\u0629 \u0627\u0644\u0628\u064a\u0639 \u0627\u0644\u0630\u0627\u062a\u064a",
890
+ "\u0627\u0644\u0635\u062f\u0627\u0631\u064a",
891
+ "\u0642\u0646\u0637\u0631\u0629 \u0645\u062a\u0639\u062f\u062f\u0629 \u0627\u0644\u0631\u0643\u0627\u0626\u0632",
892
+ "\u0627\u0644\u0643\u0645\u0627\u0646",
893
+ "\u0627\u0644\u0643\u0631\u0629 \u0627\u0644\u0637\u0627\u0626\u0631\u0629",
894
+ "\u0635\u0627\u0646\u0639\u0629 \u0627\u0644\u0648\u0627\u0641\u0644",
895
+ "\u0633\u0627\u0639\u0629 \u0627\u0644\u062d\u0627\u0626\u0637",
896
+ "\u0645\u062d\u0641\u0638\u0629",
897
+ "\u062e\u0632\u0627\u0646\u0629 \u0627\u0644\u0635\u0648\u0627\u0646",
898
+ "\u0637\u0627\u0626\u0631\u0629 \u0639\u0633\u0643\u0631\u064a\u0629",
899
+ "\u0627\u0644\u0645\u062c\u0644\u0649",
900
+ "\u0627\u0644\u063a\u0633\u0627\u0644\u0629",
901
+ "\u0642\u0627\u0631\u0648\u0631\u0629 \u0645\u0627\u0621",
902
+ "\u0625\u0628\u0631\u064a\u0642 \u0627\u0644\u0645\u0627\u0621",
903
+ "\u0628\u0631\u062c \u0627\u0644\u0645\u064a\u0627\u0647",
904
+ "\u0625\u0628\u0631\u064a\u0642 \u0627\u0644\u0643\u062d\u0648\u0644\u064a\u0627\u062a",
905
+ "\u0627\u0644\u0635\u0627\u0641\u0631\u0629",
906
+ "\u0634\u0639\u0631 \u0645\u0633\u062a\u0639\u0627\u0631",
907
+ "\u0627\u0644\u0646\u0627\u0641\u0630\u0629 \u0627\u0644\u0648\u0627\u0642\u064a\u0629",
908
+ "\u0627\u0644\u0633\u062a\u0627\u0631 \u0627\u0644\u0644\u0641\u064e\u0651\u0627\u0641",
909
+ "\u0631\u0628\u0637\u0629 \u0639\u0646\u0642 \u0648\u0646\u062f\u0633\u0648\u0631 ",
910
+ "\u0632\u062c\u0627\u062c\u0629 \u0627\u0644\u0646\u0628\u064a\u0630",
911
+ "\u062c\u0646\u0627\u062d \u0627\u0644\u0637\u0627\u0626\u0631\u0629",
912
+ "\u0645\u0642\u0644\u0627\u0629 \u0635\u064a\u0646\u064a\u0629",
913
+ "\u0627\u0644\u0645\u0644\u0639\u0642\u0629 \u0627\u0644\u062e\u0634\u0628\u064a\u0629",
914
+ "\u0627\u0644\u0635\u0648\u0641",
915
+ "\u0627\u0644\u0633\u064a\u0627\u062c \u0627\u0644\u0645\u0646\u0642\u0633\u0645",
916
+ "\u062d\u0637\u0627\u0645 \u0627\u0644\u0633\u0641\u064a\u0646\u0629",
917
+ "\u0627\u0644\u0632\u0648\u0631\u0642 \u0627\u0644\u0634\u0631\u0627\u0639\u064a",
918
+ "\u0645\u0646\u0632\u0644 \u0627\u0644\u064a\u0648\u0631\u062a",
919
+ "\u0645\u0648\u0627\u0642\u0639 \u0627\u0644\u0648\u064a\u0628",
920
+ "\u0643\u062a\u0627\u0628 \u0631\u0633\u0648\u0645 \u0647\u0632\u0644\u064a\u0629",
921
+ "\u0627\u0644\u0643\u0644\u0645\u0627\u062a \u0627\u0644\u0645\u062a\u0642\u0627\u0637\u0639\u0629",
922
+ "\u0644\u0627\u0641\u062a\u0629 \u0645\u0631\u0648\u0631\u064a\u0629",
923
+ "\u0625\u0634\u0627\u0631\u0629 \u0627\u0644\u0645\u0631\u0648\u0631 \u0627\u0644\u0636\u0648\u0626\u064a\u0629",
924
+ "\u063a\u0644\u0627\u0641 \u0627\u0644\u062d\u0645\u0627\u064a\u0629 \u0644\u0644\u0643\u062a\u0627\u0628",
925
+ "\u0642\u0627\u0626\u0645\u0629 \u0637\u0639\u0627\u0645",
926
+ "\u0635\u062d\u0646",
927
+ "\u0642\u0646\u0628\u064a\u0637 \u0623\u062e\u0636\u0631",
928
+ "\u062d\u0633\u0627\u0621 \u0627\u0644\u0643\u0648\u0646\u0633\u0648\u0645\u064a\u0629",
929
+ "\u0627\u0644\u0648\u0639\u0627\u0621 \u0627\u0644\u0633\u0627\u062e\u0646",
930
+ "\u062a\u0631\u0627\u064a\u0641\u0644",
931
+ "\u0627\u0644\u0645\u062b\u0644\u062c\u0627\u062a",
932
+ "\u0627\u0644\u0645\u0635\u0627\u0635\u0629",
933
+ "\u0627\u0644\u062e\u0628\u0632 \u0627\u0644\u0641\u0631\u0646\u0633\u064a",
934
+ "\u062e\u0628\u0632 \u0627\u0644\u0628\u064a\u063a\u0644",
935
+ "\u0627\u0644\u0645\u062e\u0628\u0648\u0632\u0627\u062a \u0627\u0644\u0639\u064f\u0642\u0652\u062f\u0650\u064a\u064e\u0651\u0629",
936
+ "\u062a\u0634\u064a\u0632 \u0628\u0631\u062c\u0631",
937
+ "\u0627\u0644\u0646\u0642\u0627\u0646\u0642",
938
+ "\u0627\u0644\u0628\u0637\u0627\u0637\u0627 \u0627\u0644\u0645\u0647\u0631\u0648\u0633\u0629",
939
+ "\u0645\u0644\u0641\u0648\u0641",
940
+ "\u0627\u0644\u0642\u0631\u0646\u0628\u064a\u0637 \u0627\u0644\u0623\u062e\u0636\u0631",
941
+ "\u0627\u0644\u0642\u0631\u0646\u0628\u064a\u0637",
942
+ "\u0627\u0644\u0643\u0648\u0633\u0627",
943
+ "\u0645\u0639\u0643\u0631\u0648\u0646\u0629 \u0627\u0644\u0627\u0633\u0643\u0648\u0627\u0634",
944
+ "\u0642\u0631\u0639 \u0627\u0644\u0628\u0644\u0648\u0637",
945
+ "\u0642\u0631\u0639 \u0627\u0644\u062c\u0648\u0632",
946
+ "\u062e\u064a\u0627\u0631",
947
+ "\u0627\u0644\u062e\u0631\u0634\u0648\u0641 \u0627\u0644\u0634\u0648\u0643\u064a",
948
+ "\u0641\u0644\u0641\u0644 \u062d\u0644\u0648",
949
+ "\u0627\u0644\u062e\u0631\u0634\u0648\u0641 \u0627\u0644\u0633\u0643\u0648\u0644\u064a\u0645\u064a",
950
+ "\u0639\u064a\u0634 \u0627\u0644\u063a\u0631\u0627\u0628",
951
+ "\u062a\u0641\u0627\u062d \u0623\u062e\u0636\u0631",
952
+ "\u0627\u0644\u0641\u0631\u0627\u0648\u0644\u0629",
953
+ "\u0627\u0644\u0628\u0631\u062a\u0642\u0627\u0644",
954
+ "\u0627\u0644\u0644\u064a\u0645\u0648\u0646",
955
+ "\u0627\u0644\u062a\u064a\u0646",
956
+ "\u0627\u0644\u0623\u0646\u0627\u0646\u0627\u0633",
957
+ "\u0627\u0644\u0645\u0648\u0632",
958
+ "\u062c\u0627\u0643 \u0641\u0631\u0648\u062a",
959
+ "\u0642\u0634\u0637\u0629 \u0634\u0631\u064a\u0645\u0648\u0644\u064a\u0627",
960
+ "\u0627\u0644\u0631\u0645\u0627\u0646",
961
+ "\u0627\u0644\u062f\u0631\u064a\u0633",
962
+ "\u0643\u0631\u0628\u0646\u0627\u0631\u0629",
963
+ "\u0635\u0644\u0635\u0629 \u0627\u0644\u0634\u0648\u0643\u0648\u0644\u0627",
964
+ "\u0639\u062c\u064a\u0646\u0629 \u0627\u0644\u062e\u0628\u0632",
965
+ "\u0631\u063a\u064a\u0641 \u0627\u0644\u0644\u062d\u0645",
966
+ "\u0628\u064a\u062a\u0632\u0627",
967
+ "\u0641\u0637\u064a\u0631\u0629 \u0627\u0644\u0642\u062f\u0631",
968
+ "\u0628\u0648\u0631\u064a\u062a\u0648",
969
+ "\u0627\u0644\u0646\u0628\u064a\u0630 \u0627\u0644\u0623\u062d\u0645\u0631",
970
+ "\u0642\u0647\u0648\u0629 \u0625\u0633\u0628\u0631\u064a\u0633\u0648",
971
+ "\u0641\u0646\u062c\u0627\u0646 \u0627\u0644\u0634\u0627\u064a",
972
+ "\u062d\u0644\u064a\u0628 \u0627\u0644\u0628\u064a\u0636",
973
+ "\u0627\u0644\u062c\u0628\u0644",
974
+ "\u0641\u0642\u0627\u0639\u0629",
975
+ "\u0627\u0644\u062c\u0631\u0641",
976
+ "\u0627\u0644\u0634\u0639\u0627\u0628 \u0627\u0644\u0645\u0631\u062c\u0627\u0646\u064a\u0629",
977
+ "\u0627\u0644\u0641\u0648\u0627\u0631\u0629 \u0627\u0644\u062d\u0627\u0631\u0629",
978
+ "\u0627\u0644\u0636\u0641\u0629",
979
+ "\u0627\u0644\u0634\u0646\u062e\u0629",
980
+ "\u0627\u0644\u0645\u064a\u0627\u0647 \u0627\u0644\u0636\u062d\u0644\u0629",
981
+ "\u0627\u0644\u0634\u0627\u0637\u0626",
982
+ "\u0627\u0644\u0648\u0627\u062f\u064a",
983
+ "\u0628\u0631\u0643\u0627\u0646",
984
+ "\u0644\u0627\u0639\u0628 \u0643\u0631\u0629 \u0627\u0644\u0642\u0627\u0639\u062f\u0629",
985
+ "\u0627\u0644\u0639\u0631\u064a\u0633",
986
+ "\u0627\u0644\u063a\u0648\u0635 \u0628\u062c\u0647\u0627\u0632 \u0627\u0644\u062a\u0646\u0641\u0633 ",
987
+ "\u0627\u0644\u0633\u0644\u062c\u0645",
988
+ "\u0632\u0647\u0631\u0629 \u0627\u0644\u0644\u0624\u0644\u0624 ",
989
+ "\u062e\u0641 \u0627\u0644\u0633\u064a\u062f\u0629 \u0627\u0644\u0623\u0635\u0641\u0631",
990
+ "\u0627\u0644\u0630\u0631\u0629",
991
+ "\u0634\u062c\u0631\u0629 \u062b\u0645\u0631\u0629 \u0627\u0644\u0628\u0644\u0648\u0637",
992
+ "\u062b\u0645\u0631 \u0627\u0644\u0648\u0631\u062f \u0627\u0644\u0628\u0631\u064a",
993
+ "\u0628\u0630\u0648\u0631 \u0643\u0633\u062a\u0646\u0627\u0621 \u0627\u0644\u062d\u0635\u0627\u0646",
994
+ "\u0627\u0644\u0641\u0637\u0631\u064a\u0627\u062a \u0627\u0644\u0645\u0631\u062c\u0627\u0646\u064a\u0629",
995
+ "\u0641\u0637\u0631 \u063a\u0627\u0631\u064a\u0642\u0648\u0646",
996
+ "\u0641\u0637\u0631 \u062c\u0627\u0631\u0648\u0645\u064a\u062a\u0631\u0627 \u0627\u064a\u0633\u0643\u0644\u0646\u062a\u0627",
997
+ "\u0627\u0644\u0642\u0631\u0646 \u0627\u0644\u0646\u062a\u0646",
998
+ "\u0641\u0637\u0631 \u0646\u062c\u0645 \u0627\u0644\u0623\u0631\u0636",
999
+ "\u0641\u0637\u0631 \u0631\u0641 \u0627\u0644\u0643\u0628\u0631\u064a\u062a",
1000
+ "\u0641\u0637\u0631 \u0627\u0644\u0628\u0648\u0644\u064a\u0637",
1001
+ "\u0627\u0644\u0639\u0631\u0646\u0627\u0633",
1002
+ "\u0648\u0631\u0642 \u0627\u0644\u0645\u0631\u062d\u0627\u0636"
1003
+ ]
1004
+ }
CLIP_benchmark/clip_benchmark/datasets/ar_zeroshot_classification_templates.json ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "imagenet1k": [
3
+ "{c}",
4
+ "\u0635\u0648\u0631\u0629 \u0633\u064a\u0626\u0629 \u0644\u0640 {c}",
5
+ "\u0635\u0648\u0631\u0629 \u0633\u064a\u0626\u0629 \u062a\u062d\u062a\u0648\u064a \u0639\u0644\u0649 {c}",
6
+ "\u0646\u062d\u062a \u0644\u0634\u0643\u0644 {c}",
7
+ "\u0646\u062d\u062a \u0644\u0640 {c}",
8
+ "\u0635\u0648\u0631\u0629 \u0630\u0627\u062a \u062c\u0648\u0648\u062f\u0629 \u0645\u0646\u062e\u0641\u0636\u0629 \u0644\u0640 {c}",
9
+ "\u0635\u0648\u0631\u0629 \u0630\u0627\u062a \u062c\u0648\u0648\u062f\u0629 \u0645\u0646\u062e\u0641\u0636\u0629 \u062a\u062d\u062a\u0648\u064a {c}",
10
+ "\u0631\u0633\u0648\u0645\u0627\u062a \u062c\u062f\u0627\u0631\u064a\u0629 \u062a\u062d\u062a\u0648\u064a {c}",
11
+ "\u0631\u0633\u0648\u0645\u0627\u062a \u062c\u062f\u0627\u0631\u064a\u0629 \u0644\u0640 {c}",
12
+ "\u0635\u0648\u0631\u0629 \u0645\u0642\u062a\u0637\u0639\u0629 \u062a\u062d\u062a\u0648\u064a \u0639\u0644\u0649 {c}",
13
+ "\u0635\u0648\u0631\u0629 \u0645\u0642\u062a\u0637\u0639\u0629 \u0644\u0640 {c}",
14
+ "\u062a\u0637\u0631\u064a\u0632 {c} ",
15
+ " \u0635\u0648\u0631\u0629 \u064a\u0635\u0639\u0628 \u0641\u064a\u0647\u0627 \u0631\u0624\u064a\u0629 {c} ",
16
+ "\u0635\u0648\u0631\u0629 \u0633\u0627\u0637\u0639\u0629 \u0644\u0640 {c}",
17
+ "\u0635\u0648\u0631\u0629 \u0648\u0627\u0636\u062d\u0629 \u0644\u0640 {c}",
18
+ "\u0635\u0648\u0631\u0629 \u0645\u062a\u0633\u062e\u0629 \u0644\u0640 {c}",
19
+ "\u0635\u0648\u0631\u0629 \u0645\u0638\u0644\u0645\u0629 \u0644\u0640 {c}",
20
+ "\u0635\u0648\u0631\u0629 \u0623\u0628\u064a\u0636 \u0648\u0623\u0633\u0648\u062f {c}",
21
+ "{c} \u0641\u064a \u0644\u0642\u0637\u0629 \u0642\u0631\u064a\u0628\u0629",
22
+ "\u0635\u0648\u0631\u0629 \u0631\u0627\u0626\u0639\u0629 \u0644\u0640 {c}",
23
+ "\u0644\u0642\u0637\u0629 \u0642\u0631\u064a\u0628\u0629 \u0644\u0640 {c}",
24
+ "\u0631\u0633\u0645 \u062d\u0627\u0633\u0648\u0628\u064a \u064a\u062d\u062a\u0648\u064a {c}",
25
+ "\u0635\u0648\u0631\u0629 \u0645\u0631\u0633\u0648\u0645\u0629 \u062a\u062d\u062a\u0648\u064a {c}",
26
+ "\u0631\u0633\u0645\u0629 \u0644\u0640 {c}",
27
+ "\u0631\u0633\u0645\u0629 {c}",
28
+ "\u0631\u0633\u0645 \u064a\u062d\u062a\u0648\u064a {c} ",
29
+ "\u0635\u0648\u0631\u0629 \u0628\u0646\u0645\u0637 \u0627\u0644\u0628\u0643\u0633\u0644 \u0644\u0640 {c}",
30
+ " \u0635\u0648\u0631\u0629 \u0633\u0627\u0637\u0639\u0629 {c}",
31
+ "\u0648\u0634\u0645 {c}",
32
+ "{c} \u0641\u064a \u0627\u0644\u0635\u0648\u0631\u0629",
33
+ "\u0635\u0648\u0631\u0629 \u0645\u062a\u0633\u062e\u0629 \u062a\u062d\u062a\u0648\u064a {c}",
34
+ "\u0635\u0648\u0631\u0629 \u062a\u0627\u0644\u0641\u0629 {c}",
35
+ "\u0635\u0648\u0631\u0629 \u0636\u0628\u0627\u0628\u064a\u0629 \u0644\u0640 {c}",
36
+ "\u0635\u0648\u0631\u0629 {c}",
37
+ "\u0635\u0648\u0631\u0629 \u062c\u064a\u062f\u0629 \u0644\u0640 {c}",
38
+ "\u0635\u0648\u0631\u0629 \u0644\u0640 {c}",
39
+ "\u062a\u0635\u064a\u064a\u0631 \u0644\u0640 {c}",
40
+ "{c} \u0639\u0644\u0649 \u0634\u0643\u0644 \u0631\u0633\u0645 \u062d\u0627\u0633\u0648\u0628\u064a \u062b\u0646\u0627\u0626\u064a \u0623\u0648 \u062b\u0644\u0627\u062b\u064a \u0627\u0644\u0623\u0628\u0639\u0627\u062f",
41
+ "\u064a\u0648\u062c\u062f {c} \u0648\u0627\u062d\u062f \u0641\u064a \u0627\u0644\u0635\u0648\u0631\u0629",
42
+ "\u0631\u0633\u0645 \u062d\u0627\u0633\u0648\u0628\u064a \u0644\u0640 {c}",
43
+ "\u0627\u0648\u0631\u064a\u063a\u0627\u0645\u064a \u0644\u0640 {c}",
44
+ "{c} \u0645\u0635\u0646\u0648\u0639 \u0639\u0646 \u0637\u0631\u064a\u0642 \u0641\u0646 \u0637\u064a \u0627\u0644\u0648\u0631\u0642",
45
+ "{c} \u0641\u064a \u0644\u0639\u0628\u0629 \u0641\u064a\u062f\u064a\u0648",
46
+ "{c} \u0645\u0648\u062c\u0648\u062f \u0641\u064a \u0644\u0639\u0628\u0629 \u0627\u0644\u0641\u064a\u062f\u064a\u0648",
47
+ "\u0631\u0633\u0645 \u062a\u0642\u0631\u064a\u0628\u064a \u0644\u0640 {c}",
48
+ "{c} \u0645\u0631\u0633\u0648\u0645 \u0628\u0627\u0644\u062e\u0631\u0627\u0628\u064a\u0634",
49
+ "\u0635\u0648\u0631\u0629 \u0628\u0641\u0646 \u0627\u0644\u062e\u0631\u0627\u0628\u064a\u0634 \u0644\u0640 {c}",
50
+ "\u0644\u0639\u0628\u0629 {c}",
51
+ "\u0635\u0648\u0631\u0629 \u064a\u0648\u062c\u062f \u0641\u064a\u0647\u0627 {c}",
52
+ "\u0631\u0633\u0648\u0645 \u0645\u062a\u062d\u0631\u0643\u0629 \u0644\u0640 {c} ",
53
+ "\u0635\u0648\u0631\u0629 \u0644\u0639\u062f\u062f \u0645\u0646 {c}",
54
+ "\u0635\u0648\u0631\u0629 \u064a\u0638\u0647\u0631 \u0641\u064a\u0647\u0627 {c}",
55
+ "\u0635\u0648\u0631\u0629 {c} \u0635\u063a\u064a\u0631 ",
56
+ "\u0635\u0648\u0631\u0629 {c} \u0643\u0628\u064a\u0631",
57
+ "{c} \u064a\u0638\u0647\u0631 \u0641\u064a \u0627\u0644\u0635\u0648\u0631\u0629"
58
+ ]
59
+ }
CLIP_benchmark/clip_benchmark/datasets/babel_imagenet.json ADDED
The diff for this file is too large to render. See raw diff
 
CLIP_benchmark/clip_benchmark/datasets/babel_imagenet.py ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torchvision
2
+
3
+ """
4
+ BabelImageNet from https://arxiv.org/pdf/2306.08658.pdf
5
+ Adapted from https://github.com/gregor-ge/Babel-ImageNet, thanks to the authors
6
+ """
7
+ class BabelImageNet(torchvision.datasets.ImageNet):
8
+ def __init__(self, root: str, idxs, split: str = "val", download=None, **kwargs) -> None:
9
+ super().__init__(root, split, **kwargs)
10
+ examples_per_class = len(self.targets) // 1000
11
+ select_idxs = [idx*examples_per_class + i for idx in idxs for i in range(examples_per_class)]
12
+ self.targets = [i for i in range(len(idxs)) for _ in range(examples_per_class)]
13
+ self.imgs = [self.imgs[i] for i in select_idxs]
14
+ self.samples = [self.samples[i] for i in select_idxs]
15
+ self.idxs = idxs
16
+
17
+ def __getitem__(self, i):
18
+ img, target = super().__getitem__(i)
19
+ target = self.idxs.index(target)
20
+ return img, target
CLIP_benchmark/clip_benchmark/datasets/builder.py ADDED
@@ -0,0 +1,817 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import warnings
3
+ import sys
4
+ import json
5
+ from subprocess import call
6
+ from collections import defaultdict
7
+ import torch
8
+ from torchvision.datasets import (
9
+ VisionDataset, ImageFolder,
10
+ CIFAR10, CIFAR100, ImageNet, CocoCaptions, Flickr8k, Flickr30k, Food101, SUN397,
11
+ StanfordCars, FGVCAircraft, DTD, OxfordIIITPet, Caltech101, Flowers102,
12
+ MNIST, STL10, EuroSAT, GTSRB, Kitti, Country211, PCAM, RenderedSST2
13
+ )
14
+
15
+ from . import voc2007, flickr, caltech101, imagenetv2, objectnet, babel_imagenet, sugar_crepe
16
+ from torch.utils.data import default_collate
17
+ from PIL import Image
18
+
19
+
20
+ def build_dataset(dataset_name, root="root", transform=None, split="test", download=True, annotation_file=None, language="en", task="zeroshot_classification", wds_cache_dir=None, custom_classname_file=None, custom_template_file=None, **kwargs):
21
+ """
22
+ Main function to use in order to build a dataset instance,
23
+
24
+ dataset_name: str
25
+ name of the dataset
26
+
27
+ root: str
28
+ root folder where the dataset is downloaded and stored. can be shared among datasets.
29
+
30
+ transform: torchvision transform applied to images
31
+
32
+ split: str
33
+ split to use, depending on the dataset can have different options.
34
+ In general, `train` and `test` are available.
35
+ For specific splits, please look at the corresponding dataset.
36
+
37
+ annotation_file: str or None
38
+ only for datasets with captions (used for retrieval) such as COCO
39
+ and Flickr.
40
+
41
+ custom_classname_file: str or None
42
+ Custom classname file where keys are dataset names and values are list of classnames.
43
+
44
+ custom_template_file: str or None
45
+ Custom template file where keys are dataset names and values are list of prompts, or dicts
46
+ where keys are classnames and values are class-specific prompts.
47
+
48
+ """
49
+ if task in ('zeroshot_classification', 'linear_probe'): # Only load templates and classnames if we have to
50
+ current_folder = os.path.dirname(__file__)
51
+
52
+ if dataset_name == "babel_imagenet":
53
+ classnames = json.load(open(os.path.join(current_folder, "babel_imagenet.json")))
54
+ assert language.upper() in classnames, f"Language '{language}' not supported for Babel-ImageNet"
55
+ classnames = classnames[language.upper()]
56
+ templates = json.load(open(os.path.join(current_folder, "nllb_dist13b_prompts.json")))
57
+ templates = templates[language.upper()]
58
+ templates = [t.replace('{}', '{c}') for t in templates]
59
+ else:
60
+ if custom_classname_file and not os.path.exists(custom_classname_file):
61
+ # look at current_folder
62
+ custom_classname_file_attempt = os.path.join(current_folder, custom_classname_file)
63
+ assert os.path.exists(custom_classname_file_attempt), f"Custom classname file '{custom_classname_file}' does not exist"
64
+ custom_classname_file = custom_classname_file_attempt
65
+ else:
66
+ custom_classname_file = os.path.join(current_folder, language + "_classnames.json")
67
+
68
+ if custom_template_file and not os.path.exists(custom_template_file):
69
+ # look at current_folder
70
+ custom_template_file_attempt = os.path.join(current_folder, custom_template_file)
71
+ assert os.path.exists(custom_template_file_attempt), f"Custom template file '{custom_template_file}' does not exist"
72
+ custom_template_file = custom_template_file_attempt
73
+ else:
74
+ custom_template_file = os.path.join(current_folder, language + "_zeroshot_classification_templates.json")
75
+
76
+ with open(custom_classname_file, "r") as f:
77
+ classnames = json.load(f)
78
+
79
+ with open(custom_template_file, "r") as f:
80
+ templates = json.load(f)
81
+
82
+ default_template = templates["imagenet1k"] if "imagenet1k" in templates else None
83
+
84
+ if dataset_name.startswith("tfds/") or dataset_name.startswith("vtab/") or dataset_name.startswith("wds/"):
85
+ name = dataset_name.split("/")[-1]
86
+ else:
87
+ name = dataset_name
88
+ templates = templates.get(name, default_template)
89
+ assert templates is not None, f"Templates for dataset '{dataset_name}' not found in '{custom_template_file}'"
90
+ else:
91
+ classnames, templates = None, None
92
+
93
+ def download_imagenet(r):
94
+ os.makedirs(r, exist_ok=True)
95
+ call(f"wget https://image-net.org/data/ILSVRC/2012/ILSVRC2012_devkit_t12.tar.gz --output-document={r}/ILSVRC2012_devkit_t12.tar.gz", shell=True)
96
+ call(f"wget https://image-net.org/data/ILSVRC/2012/ILSVRC2012_img_val.tar --output-document={r}/ILSVRC2012_img_val.tar", shell=True)
97
+
98
+ train = (split == "train")
99
+ if dataset_name == "cifar10":
100
+ ds = CIFAR10(root=root, train=train, transform=transform, download=download, **kwargs)
101
+ elif dataset_name == "cifar100":
102
+ ds = CIFAR100(root=root, train=train, transform=transform, download=download, **kwargs)
103
+ elif dataset_name == "imagenet1k":
104
+ if not os.path.exists(root):
105
+ download_imagenet(root)
106
+ ds = ImageNet(root=root, split="train" if train else "val", transform=transform, **kwargs)
107
+ ds.classes = classnames["imagenet1k"]
108
+ elif dataset_name == "imagenet-w":
109
+ from imagenet_w import AddWatermark
110
+ from torchvision.transforms import Normalize, CenterCrop
111
+ if not os.path.exists(root):
112
+ download_imagenet(root)
113
+ index_normalize = None
114
+ crop_size = None
115
+ for i, t in enumerate(transform.transforms):
116
+ if isinstance(t, Normalize):
117
+ index_normalize = i
118
+ elif isinstance(t, CenterCrop):
119
+ crop_size = min(t.size)
120
+ assert crop_size is not None, "CenterCrop not found in transform"
121
+ assert index_normalize is not None, "Normalize not found in transform"
122
+ transform.transforms.insert(index_normalize, AddWatermark(crop_size))
123
+ ds = ImageNet(root=root, split="train" if train else "val", transform=transform, **kwargs)
124
+ ds.classes = classnames["imagenet1k"]
125
+ elif dataset_name == "babel_imagenet":
126
+ # babel ImageNet from https://github.com/gregor-ge/Babel-ImageNet
127
+ if not os.path.exists(root):
128
+ download_imagenet(root)
129
+ idxs, classnames = classnames
130
+ ds = babel_imagenet.BabelImageNet(root=root, idxs=idxs, split="train" if train else "val", transform=transform, **kwargs)
131
+ ds.classes = classnames
132
+ elif dataset_name == "imagenet1k-unverified":
133
+ split = "train" if train else "val"
134
+ ds = ImageFolder(root=os.path.join(root, split), transform=transform, **kwargs)
135
+ # use classnames from OpenAI
136
+ ds.classes = classnames["imagenet1k"]
137
+ elif dataset_name == "imagenetv2":
138
+ assert split == "test", f"Only test split available for {dataset_name}"
139
+ os.makedirs(root, exist_ok=True)
140
+ ds = imagenetv2.ImageNetV2Dataset(variant="matched-frequency", transform=transform, location=root)
141
+ ds.classes = classnames["imagenet1k"]
142
+ elif dataset_name == "imagenet_sketch":
143
+ assert split == "test", f"Only test split available for {dataset_name}"
144
+ # Downloadable from https://drive.google.com/open?id=1Mj0i5HBthqH1p_yeXzsg22gZduvgoNeA
145
+ if not os.path.exists(root):
146
+ # Automatic download
147
+ print("Downloading imagenet_sketch...")
148
+ if not has_gdown():
149
+ print("GDown is needed to download the dataset. Please install it via `pip install gdown`")
150
+ sys.exit(1)
151
+ # Download ImageNet-Sketch.zip
152
+ call("gdown --id 1Mj0i5HBthqH1p_yeXzsg22gZduvgoNeA", shell=True)
153
+ assert os.path.exists("ImageNet-Sketch.zip")
154
+ # Unzip and move to `root`
155
+ call("unzip ImageNet-Sketch.zip", shell=True)
156
+ call(f"mv sketch {root}", shell=True)
157
+ ds = ImageFolder(root=root, transform=transform, **kwargs)
158
+ ds.classes = classnames["imagenet1k"]
159
+ elif dataset_name == "imagenet-a":
160
+ assert split == "test", f"Only test split available for {dataset_name}"
161
+ # Downloadable from https://people.eecs.berkeley.edu/~hendrycks/imagenet-a.tar
162
+ if not os.path.exists(root):
163
+ print("Downloading imagenet-a...")
164
+ call("wget https://people.eecs.berkeley.edu/~hendrycks/imagenet-a.tar", shell=True)
165
+ # Untar and move to `root`
166
+ call("tar xvf imagenet-a.tar", shell=True)
167
+ call(f"mv imagenet-a {root}", shell=True)
168
+ ds = ImageFolder(root=root, transform=transform, **kwargs)
169
+ ds.classes = classnames["imagenet1k"]
170
+ imagenet_a_wnids = ['n01498041', 'n01531178', 'n01534433', 'n01558993', 'n01580077', 'n01614925', 'n01616318', 'n01631663', 'n01641577', 'n01669191', 'n01677366', 'n01687978', 'n01694178', 'n01698640', 'n01735189', 'n01770081', 'n01770393', 'n01774750', 'n01784675', 'n01819313', 'n01820546', 'n01833805', 'n01843383', 'n01847000', 'n01855672', 'n01882714', 'n01910747', 'n01914609', 'n01924916', 'n01944390', 'n01985128', 'n01986214', 'n02007558', 'n02009912', 'n02037110', 'n02051845', 'n02077923', 'n02085620', 'n02099601', 'n02106550', 'n02106662', 'n02110958', 'n02119022', 'n02123394', 'n02127052', 'n02129165', 'n02133161', 'n02137549', 'n02165456', 'n02174001', 'n02177972', 'n02190166', 'n02206856', 'n02219486', 'n02226429', 'n02231487', 'n02233338', 'n02236044', 'n02259212', 'n02268443', 'n02279972', 'n02280649', 'n02281787', 'n02317335', 'n02325366', 'n02346627', 'n02356798', 'n02361337', 'n02410509', 'n02445715', 'n02454379', 'n02486410', 'n02492035', 'n02504458', 'n02655020', 'n02669723', 'n02672831', 'n02676566', 'n02690373', 'n02701002', 'n02730930', 'n02777292', 'n02782093', 'n02787622', 'n02793495', 'n02797295', 'n02802426', 'n02814860', 'n02815834', 'n02837789', 'n02879718', 'n02883205', 'n02895154', 'n02906734', 'n02948072', 'n02951358', 'n02980441', 'n02992211', 'n02999410', 'n03014705', 'n03026506', 'n03124043', 'n03125729', 'n03187595', 'n03196217', 'n03223299', 'n03250847', 'n03255030', 'n03291819', 'n03325584', 'n03355925', 'n03384352', 'n03388043', 'n03417042', 'n03443371', 'n03444034', 'n03445924', 'n03452741', 'n03483316', 'n03584829', 'n03590841', 'n03594945', 'n03617480', 'n03666591', 'n03670208', 'n03717622', 'n03720891', 'n03721384', 'n03724870', 'n03775071', 'n03788195', 'n03804744', 'n03837869', 'n03840681', 'n03854065', 'n03888257', 'n03891332', 'n03935335', 'n03982430', 'n04019541', 'n04033901', 'n04039381', 'n04067472', 'n04086273', 'n04099969', 'n04118538', 'n04131690', 'n04133789', 'n04141076', 'n04146614', 'n04147183', 'n04179913', 'n04208210', 'n04235860', 'n04252077', 'n04252225', 'n04254120', 'n04270147', 'n04275548', 'n04310018', 'n04317175', 'n04344873', 'n04347754', 'n04355338', 'n04366367', 'n04376876', 'n04389033', 'n04399382', 'n04442312', 'n04456115', 'n04482393', 'n04507155', 'n04509417', 'n04532670', 'n04540053', 'n04554684', 'n04562935', 'n04591713', 'n04606251', 'n07583066', 'n07695742', 'n07697313', 'n07697537', 'n07714990', 'n07718472', 'n07720875', 'n07734744', 'n07749582', 'n07753592', 'n07760859', 'n07768694', 'n07831146', 'n09229709', 'n09246464', 'n09472597', 'n09835506', 'n11879895', 'n12057211', 'n12144580', 'n12267677']
171
+ imagenet_a_mask = [wnid in set(imagenet_a_wnids) for wnid in all_imagenet_wordnet_ids]
172
+ ds.classes = [cl for cl, mask in zip(ds.classes, imagenet_a_mask) if mask]
173
+ elif dataset_name == "imagenet-r":
174
+ assert split == "test", f"Only test split available for {dataset_name}"
175
+ # downloadable from https://people.eecs.berkeley.edu/~hendrycks/imagenet-r.tar
176
+ if not os.path.exists(root):
177
+ print("Downloading imagenet-r...")
178
+ call("wget https://people.eecs.berkeley.edu/~hendrycks/imagenet-r.tar", shell=True)
179
+ # Untar and move to `root`
180
+ call("tar xvf imagenet-r.tar", shell=True)
181
+ call(f"mv imagenet-r {root}", shell=True)
182
+ imagenet_r_wnids = {'n01443537', 'n01484850', 'n01494475', 'n01498041', 'n01514859', 'n01518878', 'n01531178', 'n01534433', 'n01614925', 'n01616318', 'n01630670', 'n01632777', 'n01644373', 'n01677366', 'n01694178', 'n01748264', 'n01770393', 'n01774750', 'n01784675', 'n01806143', 'n01820546', 'n01833805', 'n01843383', 'n01847000', 'n01855672', 'n01860187', 'n01882714', 'n01910747', 'n01944390', 'n01983481', 'n01986214', 'n02007558', 'n02009912', 'n02051845', 'n02056570', 'n02066245', 'n02071294', 'n02077923', 'n02085620', 'n02086240', 'n02088094', 'n02088238', 'n02088364', 'n02088466', 'n02091032', 'n02091134', 'n02092339', 'n02094433', 'n02096585', 'n02097298', 'n02098286', 'n02099601', 'n02099712', 'n02102318', 'n02106030', 'n02106166', 'n02106550', 'n02106662', 'n02108089', 'n02108915', 'n02109525', 'n02110185', 'n02110341', 'n02110958', 'n02112018', 'n02112137', 'n02113023', 'n02113624', 'n02113799', 'n02114367', 'n02117135', 'n02119022', 'n02123045', 'n02128385', 'n02128757', 'n02129165', 'n02129604', 'n02130308', 'n02134084', 'n02138441', 'n02165456', 'n02190166', 'n02206856', 'n02219486', 'n02226429', 'n02233338', 'n02236044', 'n02268443', 'n02279972', 'n02317335', 'n02325366', 'n02346627', 'n02356798', 'n02363005', 'n02364673', 'n02391049', 'n02395406', 'n02398521', 'n02410509', 'n02423022', 'n02437616', 'n02445715', 'n02447366', 'n02480495', 'n02480855', 'n02481823', 'n02483362', 'n02486410', 'n02510455', 'n02526121', 'n02607072', 'n02655020', 'n02672831', 'n02701002', 'n02749479', 'n02769748', 'n02793495', 'n02797295', 'n02802426', 'n02808440', 'n02814860', 'n02823750', 'n02841315', 'n02843684', 'n02883205', 'n02906734', 'n02909870', 'n02939185', 'n02948072', 'n02950826', 'n02951358', 'n02966193', 'n02980441', 'n02992529', 'n03124170', 'n03272010', 'n03345487', 'n03372029', 'n03424325', 'n03452741', 'n03467068', 'n03481172', 'n03494278', 'n03495258', 'n03498962', 'n03594945', 'n03602883', 'n03630383', 'n03649909', 'n03676483', 'n03710193', 'n03773504', 'n03775071', 'n03888257', 'n03930630', 'n03947888', 'n04086273', 'n04118538', 'n04133789', 'n04141076', 'n04146614', 'n04147183', 'n04192698', 'n04254680', 'n04266014', 'n04275548', 'n04310018', 'n04325704', 'n04347754', 'n04389033', 'n04409515', 'n04465501', 'n04487394', 'n04522168', 'n04536866', 'n04552348', 'n04591713', 'n07614500', 'n07693725', 'n07695742', 'n07697313', 'n07697537', 'n07714571', 'n07714990', 'n07718472', 'n07720875', 'n07734744', 'n07742313', 'n07745940', 'n07749582', 'n07753275', 'n07753592', 'n07768694', 'n07873807', 'n07880968', 'n07920052', 'n09472597', 'n09835506', 'n10565667', 'n12267677'}
183
+ imagenet_r_mask = [wnid in imagenet_r_wnids for wnid in all_imagenet_wordnet_ids]
184
+ ds = ImageFolder(root=root, transform=transform, **kwargs)
185
+ ds.classes = classnames["imagenet1k"]
186
+ ds.classes = [cl for cl, mask in zip(ds.classes, imagenet_r_mask) if mask]
187
+ elif dataset_name == "imagenet-o":
188
+ assert split == "test", f"Only test split available for {dataset_name}"
189
+ # downloadable from https://people.eecs.berkeley.edu/~hendrycks/imagenet-o.tar
190
+ if not os.path.exists(root):
191
+ print("Downloading imagenet-o...")
192
+ call("wget https://people.eecs.berkeley.edu/~hendrycks/imagenet-o.tar", shell=True)
193
+ # Untar and move to `root`
194
+ call("tar xvf imagenet-o.tar", shell=True)
195
+ call(f"mv imagenet-o {root}", shell=True)
196
+ ds = ImageFolder(root=root, transform=transform, **kwargs)
197
+ ds.classes = classnames["imagenet1k"]
198
+ imagenet_o_wnids = ['n01443537', 'n01704323', 'n01770081', 'n01784675', 'n01819313', 'n01820546', 'n01910747', 'n01917289', 'n01968897', 'n02074367', 'n02317335', 'n02319095', 'n02395406', 'n02454379', 'n02606052', 'n02655020', 'n02666196', 'n02672831', 'n02730930', 'n02777292', 'n02783161', 'n02786058', 'n02787622', 'n02791270', 'n02808304', 'n02817516', 'n02841315', 'n02865351', 'n02877765', 'n02892767', 'n02906734', 'n02910353', 'n02916936', 'n02948072', 'n02965783', 'n03000134', 'n03000684', 'n03017168', 'n03026506', 'n03032252', 'n03075370', 'n03109150', 'n03126707', 'n03134739', 'n03160309', 'n03196217', 'n03207743', 'n03218198', 'n03223299', 'n03240683', 'n03271574', 'n03291819', 'n03297495', 'n03314780', 'n03325584', 'n03344393', 'n03347037', 'n03372029', 'n03376595', 'n03388043', 'n03388183', 'n03400231', 'n03445777', 'n03457902', 'n03467068', 'n03482405', 'n03483316', 'n03494278', 'n03530642', 'n03544143', 'n03584829', 'n03590841', 'n03598930', 'n03602883', 'n03649909', 'n03661043', 'n03666591', 'n03676483', 'n03692522', 'n03706229', 'n03717622', 'n03720891', 'n03721384', 'n03724870', 'n03729826', 'n03733131', 'n03733281', 'n03742115', 'n03786901', 'n03788365', 'n03794056', 'n03804744', 'n03814639', 'n03814906', 'n03825788', 'n03840681', 'n03843555', 'n03854065', 'n03857828', 'n03868863', 'n03874293', 'n03884397', 'n03891251', 'n03908714', 'n03920288', 'n03929660', 'n03930313', 'n03937543', 'n03942813', 'n03944341', 'n03961711', 'n03970156', 'n03982430', 'n03991062', 'n03995372', 'n03998194', 'n04005630', 'n04023962', 'n04033901', 'n04040759', 'n04067472', 'n04074963', 'n04116512', 'n04118776', 'n04125021', 'n04127249', 'n04131690', 'n04141975', 'n04153751', 'n04154565', 'n04201297', 'n04204347', 'n04209133', 'n04209239', 'n04228054', 'n04235860', 'n04243546', 'n04252077', 'n04254120', 'n04258138', 'n04265275', 'n04270147', 'n04275548', 'n04330267', 'n04332243', 'n04336792', 'n04347754', 'n04371430', 'n04371774', 'n04372370', 'n04376876', 'n04409515', 'n04417672', 'n04418357', 'n04423845', 'n04429376', 'n04435653', 'n04442312', 'n04482393', 'n04501370', 'n04507155', 'n04525305', 'n04542943', 'n04554684', 'n04557648', 'n04562935', 'n04579432', 'n04591157', 'n04597913', 'n04599235', 'n06785654', 'n06874185', 'n07615774', 'n07693725', 'n07695742', 'n07697537', 'n07711569', 'n07714990', 'n07715103', 'n07716358', 'n07717410', 'n07718472', 'n07720875', 'n07742313', 'n07745940', 'n07747607', 'n07749582', 'n07753275', 'n07753592', 'n07754684', 'n07768694', 'n07836838', 'n07871810', 'n07873807', 'n07880968', 'n09229709', 'n09472597', 'n12144580', 'n12267677', 'n13052670']
199
+ imagenet_o_mask = [wnid in set(imagenet_o_wnids) for wnid in all_imagenet_wordnet_ids]
200
+ ds.classes = [cl for cl, mask in zip(ds.classes, imagenet_o_mask) if mask]
201
+ elif dataset_name == "objectnet":
202
+ assert split == "test", f"Only test split available for {dataset_name}"
203
+ # downloadable from https://objectnet.dev/downloads/objectnet-1.0.zip or https://www.dropbox.com/s/raw/cxeztdtm16nzvuw/objectnet-1.0.zip
204
+ if not os.path.exists(root):
205
+ print("Downloading objectnet...")
206
+ call("wget https://objectnet.dev/downloads/objectnet-1.0.zip", shell=True)
207
+ # Untar and move to `root`
208
+ call("UNZIP_DISABLE_ZIPBOMB_DETECTION=TRUE unzip -P objectnetisatestset objectnet-1.0.zip", shell=True)
209
+ os.makedirs(root)
210
+ call(f"mv objectnet-1.0 {root}", shell=True)
211
+ call(f"cp {root}/objectnet-1.0/mappings/* {root}", shell=True)
212
+ ds = objectnet.ObjectNetDataset(root=root, transform=transform)
213
+ elif dataset_name == "voc2007":
214
+ ds = voc2007.PASCALVoc2007Cropped(root=root, set="train" if train else "test", transform=transform, download=download, **kwargs)
215
+ elif dataset_name == "voc2007_multilabel":
216
+ ds = voc2007.PASCALVoc2007(root=root, set="train" if train else "test", transform=transform, download=download, **kwargs)
217
+ elif dataset_name.startswith("sugar_crepe"):
218
+ # https://github.com/RAIVNLab/sugar-crepe/tree/main
219
+ _, task = dataset_name.split("/")
220
+ assert task in ("add_att", "add_obj", "replace_att", "replace_obj", "replace_rel", "swap_att", "swap_obj"), f"Unknown task {task} for {dataset_name}"
221
+ assert split == "test", f"Only test split available for {dataset_name}"
222
+ archive_name = "val2017.zip"
223
+ root_split = os.path.join(root, archive_name.replace(".zip", ""))
224
+ if not os.path.exists(root_split):
225
+ print(f"Downloading coco captions {archive_name}...")
226
+ if not os.path.exists(os.path.join(root, archive_name)):
227
+ call(f"wget http://images.cocodataset.org/zips/{archive_name} --output-document={root}/{archive_name}", shell=True)
228
+ call(f"unzip {root}/{archive_name} -d {root}", shell=True)
229
+ ann = f"{root}/{task}.json"
230
+ if not os.path.exists(ann):
231
+ url = f"https://raw.githubusercontent.com/RAIVNLab/sugar-crepe/main/data/{task}.json"
232
+ call(f"wget {url} --output-document={ann}", shell=True)
233
+ ds = sugar_crepe.SugarCrepe(root=os.path.join(root, "val2017"), ann_file=ann, transform=transform, **kwargs)
234
+ elif dataset_name == "mscoco_captions":
235
+ # https://github.com/mehdidc/retrieval_annotations/releases/tag/1.0.0(annotations)
236
+ if split == "train":
237
+ archive_name = "train2014.zip"
238
+ elif split in ("val", "test"):
239
+ archive_name = "val2014.zip"
240
+ else:
241
+ raise ValueError(f"split should be train or val or test for `{dataset_name}`")
242
+ root_split = os.path.join(root, archive_name.replace(".zip", ""))
243
+ if not os.path.exists(root_split):
244
+ print(f"Downloading mscoco_captions {archive_name}...")
245
+ if not os.path.exists(os.path.join(root, archive_name)):
246
+ call(f"wget http://images.cocodataset.org/zips/{archive_name} --output-document={root}/{archive_name}", shell=True)
247
+ call(f"unzip {root}/{archive_name} -d {root}", shell=True)
248
+ if not annotation_file:
249
+ annotation_file = f"{root}/coco_{split}_karpathy.json"
250
+ if not os.path.exists(annotation_file):
251
+ call(f"wget https://github.com/mehdidc/retrieval_annotations/releases/download/1.0.0/coco_{split}_karpathy.json --output-document={annotation_file}", shell=True)
252
+ ds = CocoCaptions(root=root_split, annFile=annotation_file, transform=transform, **kwargs)
253
+ elif dataset_name == 'multilingual_mscoco_captions':
254
+ from clip_benchmark.datasets import multilingual_mscoco
255
+ if(language not in multilingual_mscoco.SUPPORTED_LANGUAGES):
256
+ raise ValueError("Unsupported language for multilingual_ms_coco:", language)
257
+
258
+ def get_archive_name(target_split):
259
+ if target_split == "train":
260
+ return "train2014.zip"
261
+ elif target_split in ("val", "test"):
262
+ return "val2014.zip"
263
+ else:
264
+ raise ValueError(f"split should be train or val or test for `{dataset_name}`")
265
+
266
+ def download_mscoco_split(target_split):
267
+ archive_name = get_archive_name(target_split)
268
+ root_split = os.path.join(root, archive_name.replace(".zip", ""))
269
+ if not os.path.exists(root_split):
270
+ print(f"Downloading mscoco_captions {archive_name}...")
271
+ if not os.path.exists(os.path.join(root, archive_name)):
272
+ call(f"wget http://images.cocodataset.org/zips/{archive_name} --output-document={root}/{archive_name}", shell=True)
273
+ call(f"unzip {root}/{archive_name} -d {root}", shell=True)
274
+
275
+ # The multilingual MS-COCO uses images from various splits
276
+ for target_split in ['train', 'val', 'test']:
277
+ download_mscoco_split(target_split)
278
+
279
+ annotation_file = os.path.join(root, multilingual_mscoco.CAPTIONS_FILE_NAME.format(language))
280
+ if (os.path.exists(annotation_file) == False):
281
+ multilingual_mscoco.create_annotation_file(root, language)
282
+
283
+ ds = multilingual_mscoco.Multilingual_MSCOCO(root=root, ann_file=annotation_file, transform=transform, **kwargs)
284
+ elif dataset_name == "flickr30k":
285
+ # downloadable from https://www.kaggle.com/datasets/adityajn105/flickr30k
286
+ # https://github.com/mehdidc/retrieval_annotations/releases/tag/1.0.0(annotations)
287
+ # `kaggle datasets download -d adityajn105/flickr30k`
288
+ if not os.path.exists(root):
289
+ # Automatic download
290
+ print("Downloading flickr30k...")
291
+ if not has_kaggle():
292
+ print("Kaggle is needed to download the dataset. Please install it via `pip install kaggle`")
293
+ sys.exit(1)
294
+ call("kaggle datasets download -d hsankesara/flickr-image-dataset", shell=True)
295
+ call(f"unzip flickr-image-dataset.zip", shell=True)
296
+ call(f"mv flickr30k_images/flickr30k_images {root} && rm -rf flickr30k_images", shell=True)
297
+ if not annotation_file:
298
+ if language == "en":
299
+ annotation_file = f"{root}/flickr30k_{split}_karpathy.txt"
300
+ elif language == "zh":
301
+ annotation_file = f"{root}/flickr30k_{split}_zh.txt"
302
+ else:
303
+ raise ValueError(f"Unsupported language {language} for `{dataset_name}`")
304
+ if not os.path.exists(annotation_file):
305
+ # Download Flickr30K Karpathy test set
306
+ if language== "en":
307
+ call(f"wget https://github.com/mehdidc/retrieval_annotations/releases/download/1.0.0/flickr30k_{split}_karpathy.txt --output-document={annotation_file}", shell=True)
308
+ elif language =="zh":
309
+ call(f"wget https://github.com/mehdidc/retrieval_annotations/releases/download/1.0.0/flickr30k_{split}_zh.txt --output-document={annotation_file}", shell=True)
310
+ else:
311
+ raise ValueError(f"Unsupported language {language} for `{dataset_name}`")
312
+ ds = flickr.Flickr(root=root, ann_file=annotation_file, transform=transform, **kwargs)
313
+ elif dataset_name == "flickr8k":
314
+ # downloadable from https://www.kaggle.com/datasets/adityajn105/flickr8k
315
+ # `kaggle datasets download -d adityajn105/flickr8k`
316
+ # https://github.com/mehdidc/retrieval_annotations/releases/tag/1.0.0(annotations)
317
+ if not os.path.exists(root):
318
+ # Automatic download
319
+ print("Downloading flickr8k...")
320
+ if not has_kaggle():
321
+ print("Kaggle is needed to download the dataset. Please install it via `pip install kaggle`")
322
+ sys.exit(1)
323
+ call("kaggle datasets download -d adityajn105/flickr8k", shell=True)
324
+ call(f"unzip flickr8k.zip", shell=True)
325
+ call(f"mv Images {root}", shell=True)
326
+ call(f"mv captions.txt {root}", shell=True)
327
+ if not annotation_file:
328
+ if language == "en":
329
+ annotation_file = f"{root}/flickr8k_{split}_karpathy.txt"
330
+ elif language == "zh":
331
+ annotation_file = f"{root}/flickr8k_{split}_zh.txt"
332
+ else:
333
+ raise ValueError(f"Unsupported language {language} for `{dataset_name}`")
334
+ if not os.path.exists(annotation_file):
335
+ # Download Flickr8K Karpathy test set
336
+ if language == "en":
337
+ call(f"wget https://github.com/mehdidc/retrieval_annotations/releases/download/1.0.0/flickr8k_{split}_karpathy.txt --output-document={annotation_file}", shell=True)
338
+ elif language == "zh":
339
+ call(f"wget https://github.com/mehdidc/retrieval_annotations/releases/download/1.0.0/flickr8k_{split}_zh.txt --output-document={annotation_file}", shell=True)
340
+ else:
341
+ raise ValueError(f"Unsupported language {language} for `{dataset_name}`")
342
+ ds = flickr.Flickr(root=root, ann_file=annotation_file, transform=transform, **kwargs)
343
+ elif dataset_name == "food101":
344
+ ds = Food101(root=root, split="train" if train else "test", transform=transform, download=download, **kwargs)
345
+ # we use the default class names, we just replace "_" by spaces
346
+ # to delimit words
347
+ ds.classes = [cl.replace("_", " ") for cl in ds.classes]
348
+ elif dataset_name == "sun397":
349
+ warnings.warn(f"split argument ignored for `{dataset_name}`, there are no pre-defined train/test splits for this dataset")
350
+ # we use the default class names, we just replace "_" and "/" by spaces
351
+ # to delimit words
352
+ ds = SUN397(root=root, transform=transform, download=download, **kwargs)
353
+ ds.classes = [cl.replace("_", " ").replace("/", " ") for cl in ds.classes]
354
+ elif dataset_name == "cars":
355
+ ds = StanfordCars(root=root, split="train" if train else "test", transform=transform, download=download, **kwargs)
356
+ elif dataset_name == "fgvc_aircraft":
357
+ ds = FGVCAircraft(root=root, annotation_level="variant", split="train" if train else "test", transform=transform, download=download, **kwargs)
358
+ elif dataset_name == "dtd":
359
+ ds = DTD(root=root, split="train" if train else "test", transform=transform, download=download, **kwargs)
360
+ elif dataset_name == "pets":
361
+ ds = OxfordIIITPet(root=root, split="train" if train else "test", target_types="category", transform=transform, download=download, **kwargs)
362
+ elif dataset_name == "caltech101":
363
+ warnings.warn(f"split argument ignored for `{dataset_name}`, there are no pre-defined train/test splits for this dataset")
364
+ # broken download link (can't download google drive), fixed by this PR https://github.com/pytorch/vision/pull/5645
365
+ # also available in "vtab/caltech101" using VTAB splits, we advice to use VTAB version rather than this one
366
+ # since in this one (torchvision) there are no pre-defined test splits
367
+ ds = caltech101.Caltech101(root=root, target_type="category", transform=transform, download=download, **kwargs)
368
+ ds.classes = classnames["caltech101"]
369
+ elif dataset_name == "flowers":
370
+ ds = Flowers102(root=root, split="train" if train else "test", transform=transform, download=download, **kwargs)
371
+ # class indices started by 1 until it was fixed in a PR (#TODO link of the PR)
372
+ # if older torchvision version, fix it using a target transform that decrements label index
373
+ # TODO figure out minimal torchvision version needed instead of decrementing
374
+ if ds[0][1] == 1:
375
+ ds.target_transform = lambda y:y-1
376
+ ds.classes = classnames["flowers"]
377
+ elif dataset_name == "mnist":
378
+ ds = MNIST(root=root, train=train, transform=transform, download=download, **kwargs)
379
+ ds.classes = classnames["mnist"]
380
+ elif dataset_name == "stl10":
381
+ ds = STL10(root=root, split="train" if train else "test", transform=transform, download=download, **kwargs)
382
+ elif dataset_name == "eurosat":
383
+ warnings.warn(f"split argument ignored for `{dataset_name}`, there are no pre-defined train/test splits for this dataset")
384
+ ds = EuroSAT(root=root, transform=transform, download=download, **kwargs)
385
+ ds.classes = classnames["eurosat"]
386
+ elif dataset_name == "gtsrb":
387
+ ds = GTSRB(root=root, split="train" if train else "test", transform=transform, download=download, **kwargs)
388
+ ds.classes = classnames["gtsrb"]
389
+ elif dataset_name == "country211":
390
+ ds = Country211(root=root, split="train" if train else "test", transform=transform, download=download, **kwargs)
391
+ ds.classes = classnames["country211"]
392
+ elif dataset_name == "pcam":
393
+ # Dead link. Fixed by this PR on torchvision https://github.com/pytorch/vision/pull/5645
394
+ # TODO figure out minimal torchvision version needed
395
+ ds = PCAM(root=root, split="train" if train else "test", transform=transform, download=download, **kwargs)
396
+ ds.classes = classnames["pcam"]
397
+ elif dataset_name == "renderedsst2":
398
+ ds = RenderedSST2(root=root, split="train" if train else "test", transform=transform, download=download, **kwargs)
399
+ elif dataset_name == "fer2013":
400
+ # Downloadable from https://www.kaggle.com/datasets/msambare/fer2013
401
+ # `kaggle datasets download -d msambare/fer2013`
402
+ if not os.path.exists(root):
403
+ # Automatic download
404
+ print("Downloading fer2013...")
405
+ if not has_kaggle():
406
+ print("Kaggle is needed to download the dataset. Please install it via `pip install kaggle`")
407
+ sys.exit(1)
408
+ call("kaggle datasets download -d msambare/fer2013", shell=True)
409
+ call(f"unzip fer2013.zip -d {root}", shell=True)
410
+ root = os.path.join(root, "train" if train else "test")
411
+ ds = ImageFolder(root=root, transform=transform)
412
+ ds.classes = classnames["fer2013"]
413
+ elif dataset_name.startswith("tfds/"):
414
+ # TFDS datasets support using `timm` and `tensorflow_datasets`
415
+ prefix, *name_list = dataset_name.split("/")
416
+ name = "/".join(name_list)
417
+ ds = build_tfds_dataset(name, download=download, split=split, data_dir=root, transform=transform)
418
+ elif dataset_name.startswith("vtab/"):
419
+ # VTAB datasets support using `tensorflow_datasets` and `task_adaptation`
420
+ prefix, *name_list = dataset_name.split("/")
421
+ name = "/".join(name_list)
422
+ ds = build_vtab_dataset(name, download=download, split=split, data_dir=root, transform=transform, classnames=classnames)
423
+ elif dataset_name.startswith("wds/"):
424
+ # WebDataset support using `webdataset` library
425
+ name = dataset_name.split("/", 1)[1]
426
+ ds = build_wds_dataset(name, transform=transform, split=split, data_dir=root, cache_dir=wds_cache_dir)
427
+ elif dataset_name == "dummy":
428
+ ds = Dummy()
429
+ else:
430
+ raise ValueError(f"Unsupported dataset: {dataset_name}.")
431
+ ds.templates = templates
432
+ return ds
433
+
434
+ class Dummy():
435
+
436
+ def __init__(self):
437
+ self.classes = ["blank image", "noisy image"]
438
+
439
+ def __getitem__(self, i):
440
+ return torch.zeros(3,224,224), 0
441
+
442
+ def __len__(self):
443
+ return 1
444
+
445
+ def get_dataset_default_task(dataset):
446
+ if dataset in ("flickr30k", "flickr8k", "mscoco_captions", "multilingual_mscoco_captions"):
447
+ return "zeroshot_retrieval"
448
+ elif dataset.startswith("sugar_crepe"):
449
+ return "image_caption_selection"
450
+ else:
451
+ return "zeroshot_classification"
452
+
453
+ def get_dataset_collate_fn(dataset_name):
454
+ if dataset_name in ("mscoco_captions", "multilingual_mscoco_captions", "flickr30k", "flickr8k") or dataset_name.startswith("sugar_crepe"):
455
+ return image_captions_collate_fn
456
+ else:
457
+ return default_collate
458
+
459
+ def has_gdown():
460
+ return call("which gdown", shell=True) == 0
461
+
462
+ def has_kaggle():
463
+ return call("which kaggle", shell=True) == 0
464
+
465
+
466
+ def build_vtab_dataset(dataset_name, transform, download=True, split="test", data_dir="root", classnames=[]):
467
+ # Using VTAB splits instead of default TFDS splits
468
+ from .tfds import VTABIterableDataset, disable_gpus_on_tensorflow, download_tfds_dataset
469
+
470
+ # avoid Tensorflow owning GPUs to not clash with PyTorch
471
+ disable_gpus_on_tensorflow()
472
+
473
+ # by default we take classes from TFDS (default behavior if `classes` stays None),
474
+ # except for the datasets that will override `classes` (e.g., clevr_*)
475
+ classes = None
476
+ if dataset_name == "caltech101":
477
+ from task_adaptation.data.caltech import Caltech101
478
+ tfds_dataset = Caltech101(data_dir=data_dir)
479
+ classes = classnames["caltech101_vtab"]
480
+ elif dataset_name == "cars":
481
+ from task_adaptation.data.cars import CarsData
482
+ tfds_dataset = CarsData(data_dir=data_dir)
483
+ elif dataset_name in ("cifar10", "cifar100"):
484
+ from task_adaptation.data.cifar import CifarData
485
+ tfds_dataset = CifarData(data_dir=data_dir, num_classes=10 if dataset_name == "cifar10" else 100)
486
+ elif dataset_name.startswith("clevr_"):
487
+ from task_adaptation.data.clevr import CLEVRData
488
+ task = _extract_task(dataset_name)
489
+ assert task in ("count_all", "closest_object_distance")
490
+ tfds_dataset = CLEVRData(task=task, data_dir=data_dir)
491
+ if task == "count_all":
492
+ classes = classnames["clevr_count_all"]
493
+ elif task == "closest_object_distance":
494
+ classes = classnames["clevr_closest_object_distance"]
495
+ else:
496
+ raise ValueError(f"non supported: {task}")
497
+ elif dataset_name == "cub":
498
+ from task_adaptation.data.cub import CUB2011Data
499
+ tfds_dataset = CUB2011Data(data_dir=data_dir)
500
+ elif dataset_name == "diabetic_retinopathy":
501
+ # Needs manual download from Kaggle
502
+ # 1) `kaggle competitions download -c diabetic-retinopathy-detection` on $ROOT/downloads/manual
503
+ # 2) extract archives on $ROOT/downloads/manual
504
+ if not os.path.exists(data_dir):
505
+ # Automatic download
506
+ print("Downloading diabetic_retinopathy...")
507
+ if not has_kaggle():
508
+ print("Kaggle is needed to download the dataset. Please install it via `pip install kaggle`")
509
+ sys.exit(1)
510
+ os.makedirs(os.path.join(data_dir, "downloads", "manual"))
511
+ call(f"kaggle competitions download -c diabetic-retinopathy-detection -p {data_dir}/downloads/manual", shell=True)
512
+ call(f"cd {data_dir}/downloads/manual;unzip diabetic-retinopathy-detection.zip;cat train.zip*>train.zip;cat test.zip*>test.zip;unzip train.zip; unzip test.zip;unzip sample.zip;unzip trainLabels.csv.zip", shell=True)
513
+ from task_adaptation.data.diabetic_retinopathy import RetinopathyData
514
+ tfds_dataset = RetinopathyData(config="btgraham-300", data_dir=data_dir)
515
+ classes = classnames["diabetic_retinopathy"]
516
+ elif dataset_name == "dmlab":
517
+ from task_adaptation.data.dmlab import DmlabData
518
+ download_tfds_dataset("dmlab", data_dir=data_dir) # it's not called in the original VTAB code, so we do it explictly
519
+ tfds_dataset = DmlabData(data_dir=data_dir)
520
+ classes = classnames["dmlab"]
521
+ elif dataset_name.startswith("dsprites_"):
522
+ from task_adaptation.data.dsprites import DSpritesData
523
+ task = _extract_task(dataset_name)
524
+ assert task in ("label_shape", "label_scale", "label_orientation", "label_x_position", "label_y_position")
525
+ tfds_dataset = DSpritesData(task, data_dir=data_dir)
526
+ classes = tfds_dataset._dataset_builder.info.features[task].names
527
+ elif dataset_name == "dtd":
528
+ from task_adaptation.data.dtd import DTDData
529
+ tfds_dataset = DTDData(data_dir=data_dir)
530
+ elif dataset_name == "eurosat":
531
+ from task_adaptation.data.eurosat import EurosatData
532
+ tfds_dataset = EurosatData(subset="rgb", data_key="image", data_dir=data_dir)
533
+ classes = classnames["eurosat"]
534
+ elif dataset_name == "food101":
535
+ from task_adaptation.data.food101 import Food101Data
536
+ tfds_dataset = Food101Data(data_dir=data_dir)
537
+ elif dataset_name == "inaturalist":
538
+ from task_adaptation.data.inaturalist import INaturalistData
539
+ tfds_dataset = INaturalistData(data_dir=data_dir, year=2017)
540
+ elif dataset_name.startswith("kitti_"):
541
+ from .kitti import KittiData
542
+ task = _extract_task(dataset_name)
543
+ assert task in (
544
+ "count_all", "count_left", "count_far", "count_near",
545
+ "closest_object_distance", "closest_object_x_location",
546
+ "count_vehicles", "closest_vehicle_distance",
547
+ )
548
+ tfds_dataset = KittiData(task=task, data_dir=data_dir)
549
+ if task == "closest_vehicle_distance":
550
+ classes = classnames["kitti_closest_vehicle_distance"]
551
+ else:
552
+ raise ValueError(f"Unsupported task: {task}")
553
+ elif dataset_name == "flowers":
554
+ from task_adaptation.data.oxford_flowers102 import OxfordFlowers102Data
555
+ tfds_dataset = OxfordFlowers102Data(data_dir=data_dir)
556
+ elif dataset_name == "pets":
557
+ from task_adaptation.data.oxford_iiit_pet import OxfordIIITPetData
558
+ tfds_dataset = OxfordIIITPetData(data_dir=data_dir)
559
+ classes = classnames["pets"]
560
+ elif dataset_name == "pcam":
561
+ from task_adaptation.data.patch_camelyon import PatchCamelyonData
562
+ tfds_dataset = PatchCamelyonData(data_dir=data_dir)
563
+ classes = classnames["pcam"]
564
+ elif dataset_name == "resisc45":
565
+ # Needs download from OneDrive: https://1drv.ms/u/s!AmgKYzARBl5ca3HNaHIlzp_IXjs
566
+ # The archive needs to to be put at <DATASET_ROOT>/downloads/manual then extracted
567
+ if not os.path.exists(data_dir):
568
+ os.makedirs(os.path.join(data_dir, "downloads", "manual"))
569
+ call(f"wget 'https://onedrive.live.com/download?resid=5C5E061130630A68!107&authkey=!AHHNaHIlzp_IXjs' --output-document={data_dir}/downloads/manual/resisc45.rar", shell=True)
570
+ call(f"cd {data_dir}/downloads/manual;unrar x resisc45.rar", shell=True)
571
+ from task_adaptation.data.resisc45 import Resisc45Data
572
+ tfds_dataset = Resisc45Data(data_dir=data_dir)
573
+ elif dataset_name.startswith("smallnorb_"):
574
+ from task_adaptation.data.smallnorb import SmallNORBData
575
+ task = _extract_task(dataset_name)
576
+ assert task in ("label_category", "label_elevation", "label_azimuth", "label_lighting")
577
+ tfds_dataset = SmallNORBData(predicted_attribute=task, data_dir=data_dir)
578
+ classes = tfds_dataset._dataset_builder.info.features[task].names
579
+ elif dataset_name == "sun397":
580
+ from task_adaptation.data.sun397 import Sun397Data
581
+ #FIXME There is a problem in `sun397`, when TFDS tries download it
582
+ # there is an image that cannot be decoded. For the time being
583
+ # we will use torchvision's SUN397 instead.
584
+ tfds_dataset = Sun397Data(config="tfds", data_dir=data_dir)
585
+ elif dataset_name == "svhn":
586
+ from task_adaptation.data.svhn import SvhnData
587
+ tfds_dataset = SvhnData(data_dir=data_dir)
588
+ classes = classnames["svhn"]
589
+ else:
590
+ raise ValueError(f"Unsupported dataset: {dataset_name}")
591
+ ds = VTABIterableDataset(
592
+ tfds_dataset,
593
+ input_name="image", label_name="label",
594
+ transform=transform,
595
+ target_transform=int,
596
+ split=split,
597
+ classes=classes,
598
+ )
599
+ return ds
600
+
601
+ def build_tfds_dataset(name, transform, download=True, split="test", data_dir="root", classes=None):
602
+ from .tfds import disable_gpus_on_tensorflow
603
+ disable_gpus_on_tensorflow()
604
+ import tensorflow_datasets as tfds
605
+ import timm
606
+ builder = tfds.builder(name, data_dir=data_dir)
607
+ if download:
608
+ builder.download_and_prepare()
609
+ splits = list(builder.info.splits.keys())
610
+ assert split in splits, (split, splits)
611
+ ds = timm.data.create_dataset(f"tfds/{name}", data_dir, split=split, transform=transform, target_transform=int)
612
+ ds.classes = builder.info.features['label'].names if classes is None else classes
613
+ return ds
614
+
615
+
616
+ def build_wds_dataset(dataset_name, transform, split="test", data_dir="root", cache_dir=None):
617
+ """
618
+ Load a dataset in WebDataset format. Either local paths or HTTP URLs can be specified.
619
+ Expected file structure is:
620
+ ```
621
+ data_dir/
622
+ train/
623
+ nshards.txt
624
+ 0.tar
625
+ 1.tar
626
+ ...
627
+ test/
628
+ nshards.txt
629
+ 0.tar
630
+ 1.tar
631
+ ...
632
+ classnames.txt
633
+ zeroshot_classification_templates.txt
634
+ dataset_type.txt
635
+ ```
636
+ Classnames and templates are required for zeroshot classification, while dataset type
637
+ (equal to "retrieval") is required for zeroshot retrieval datasets.
638
+
639
+ You can use the `clip_benchmark_export_wds` or corresponding API
640
+ (`clip_benchmark.webdataset_builder.convert_dataset`) to convert datasets to this format.
641
+
642
+ Set `cache_dir` to a path to cache the dataset, otherwise, no caching will occur.
643
+ """
644
+ print(dataset_name, data_dir, split)
645
+ import webdataset as wds
646
+
647
+ def read_txt(fname):
648
+ if "://" in fname:
649
+ stream = os.popen("curl -L -s --fail '%s'" % fname, "r")
650
+ value = stream.read()
651
+ if stream.close():
652
+ raise FileNotFoundError("Failed to retreive data")
653
+ else:
654
+ with open(fname, "r") as file:
655
+ value = file.read()
656
+ return value
657
+ # Special handling for Huggingface datasets
658
+ # Git LFS files have a different file path to access the raw data than other files
659
+ if data_dir.startswith("https://huggingface.co/datasets"):
660
+ # Format: https://huggingface.co/datasets/<USERNAME>/<REPO>/tree/<BRANCH>
661
+ *split_url_head, _, url_path = data_dir.split("/", 7)
662
+ url_head = "/".join(split_url_head)
663
+ metadata_dir = "/".join([url_head, "raw", url_path])
664
+ tardata_dir = "/".join([url_head, "resolve", url_path])
665
+ else:
666
+ metadata_dir = tardata_dir = data_dir
667
+ # Get number of shards
668
+ nshards_fname = os.path.join(metadata_dir, split, "nshards.txt")
669
+ nshards = int(read_txt(nshards_fname)) # Do not catch FileNotFound, nshards.txt should be mandatory
670
+ # Get dataset type (classification or retrieval)
671
+ type_fname = os.path.join(metadata_dir, "dataset_type.txt")
672
+ try:
673
+ dataset_type = read_txt(type_fname).strip().lower()
674
+ except FileNotFoundError:
675
+ # print("WARNING: dataset_type.txt not found, assuming type=classification")
676
+ dataset_type = "classification"
677
+ #
678
+ filepattern = os.path.join(tardata_dir, split, "{0..%d}.tar" % (nshards - 1))
679
+ # Load webdataset (support WEBP, PNG, and JPG for now)
680
+ if not cache_dir or not isinstance(cache_dir, str):
681
+ cache_dir = None
682
+ dataset = (
683
+ wds.WebDataset(filepattern, cache_dir=cache_dir)
684
+ .decode(wds.autodecode.ImageHandler("pil", extensions=["webp", "png", "jpg", "jpeg"]))
685
+ )
686
+ # Load based on classification or retrieval task
687
+ if dataset_type == "retrieval":
688
+ dataset = (dataset
689
+ .to_tuple(["webp", "png", "jpg", "jpeg"], "txt")
690
+ .map_tuple(transform, str.splitlines)
691
+ )
692
+ dataset.classes = dataset.templates = None
693
+ else:
694
+ label_type = "npy" if dataset_type == "multilabel" else "cls" # Special case for multilabel
695
+ dataset = (dataset
696
+ .to_tuple(["webp", "png", "jpg", "jpeg"], label_type)
697
+ .map_tuple(transform, None)
698
+ )
699
+ # Get class names if present
700
+ classnames_fname = os.path.join(metadata_dir, "classnames.txt")
701
+ try:
702
+ dataset.classes = [line.strip() for line in read_txt(classnames_fname).splitlines() if line.strip()]
703
+ except FileNotFoundError:
704
+ print("WARNING: classnames.txt not found")
705
+ dataset.classes = None
706
+ # Get zeroshot classification templates if present
707
+ templates_fname = os.path.join(metadata_dir, "zeroshot_classification_templates.txt")
708
+ try:
709
+ dataset.templates = [line.strip() for line in read_txt(templates_fname).splitlines() if line.strip()]
710
+ except FileNotFoundError:
711
+ print("WARNING: zeroshot_classification_templates.txt not found")
712
+ dataset.templates = None
713
+
714
+ return dataset
715
+
716
+
717
+ def _extract_task(dataset_name):
718
+ prefix, *task_name_list = dataset_name.split("_")
719
+ task = "_".join(task_name_list)
720
+ return task
721
+
722
+
723
+ def image_captions_collate_fn(batch):
724
+ transposed = list(zip(*batch))
725
+ imgs = default_collate(transposed[0])
726
+ texts = transposed[1]
727
+ return imgs, texts
728
+
729
+ def get_dataset_collection_from_file(path):
730
+ return [l.strip() for l in open(path).readlines()]
731
+
732
+ dataset_collection = {
733
+ "vtab": [
734
+ "vtab/caltech101",
735
+ "vtab/cifar100",
736
+ "vtab/clevr_count_all",
737
+ "vtab/clevr_closest_object_distance",
738
+ "vtab/diabetic_retinopathy",
739
+ "vtab/dmlab",
740
+ "vtab/dsprites_label_orientation",
741
+ "vtab/dsprites_label_x_position",
742
+ "vtab/dtd",
743
+ "vtab/eurosat",
744
+ "vtab/kitti_closest_vehicle_distance",
745
+ "vtab/flowers",
746
+ "vtab/pets",
747
+ "vtab/pcam",
748
+ "vtab/resisc45",
749
+ "vtab/smallnorb_label_azimuth",
750
+ "vtab/smallnorb_label_elevation",
751
+ "sun397",
752
+ "vtab/svhn",
753
+ ],
754
+ "vtab+":[
755
+ "imagenet1k",
756
+ "imagenetv2",
757
+ "imagenet_sketch",
758
+ "imagenet-a",
759
+ "imagenet-r",
760
+ "objectnet",
761
+ "fer2013",
762
+ "voc2007",
763
+ "voc2007_multilabel",
764
+ "sun397",
765
+ "cars",
766
+ "fgvc_aircraft",
767
+ "mnist",
768
+ "stl10",
769
+ "gtsrb",
770
+ "country211",
771
+ "renderedsst2",
772
+ "vtab/caltech101",
773
+ "vtab/cifar10",
774
+ "vtab/cifar100",
775
+ "vtab/clevr_count_all",
776
+ "vtab/clevr_closest_object_distance",
777
+ "vtab/diabetic_retinopathy",
778
+ "vtab/dmlab",
779
+ "vtab/dsprites_label_orientation",
780
+ "vtab/dsprites_label_x_position",
781
+ "vtab/dtd",
782
+ "vtab/eurosat",
783
+ "vtab/kitti_closest_vehicle_distance",
784
+ "vtab/flowers",
785
+ "vtab/pets",
786
+ "vtab/pcam",
787
+ "vtab/resisc45",
788
+ "vtab/smallnorb_label_azimuth",
789
+ "vtab/smallnorb_label_elevation",
790
+ "vtab/svhn",
791
+ ],
792
+ "retrieval": [
793
+ "mscoco_captions",
794
+ "flickr8k",
795
+ "flickr30k",
796
+ ],
797
+ "imagenet_robustness": [
798
+ "imagenetv2",
799
+ "imagenet_sketch",
800
+ "imagenet-a",
801
+ "imagenet-r",
802
+ "objectnet",
803
+ ],
804
+ "sugar_crepe":[
805
+ "sugar_crepe/add_att",
806
+ "sugar_crepe/add_obj",
807
+ "sugar_crepe/replace_att",
808
+ "sugar_crepe/replace_obj",
809
+ "sugar_crepe/replace_rel",
810
+ "sugar_crepe/swap_att",
811
+ "sugar_crepe/swap_obj",
812
+ ]
813
+ }
814
+ # use by imagenet robustness datasets
815
+ all_imagenet_wordnet_ids = ['n01440764', 'n01443537', 'n01484850', 'n01491361', 'n01494475', 'n01496331', 'n01498041', 'n01514668', 'n01514859', 'n01518878', 'n01530575', 'n01531178', 'n01532829', 'n01534433', 'n01537544', 'n01558993', 'n01560419', 'n01580077', 'n01582220', 'n01592084', 'n01601694', 'n01608432', 'n01614925', 'n01616318', 'n01622779', 'n01629819', 'n01630670', 'n01631663', 'n01632458', 'n01632777', 'n01641577', 'n01644373', 'n01644900', 'n01664065', 'n01665541', 'n01667114', 'n01667778', 'n01669191', 'n01675722', 'n01677366', 'n01682714', 'n01685808', 'n01687978', 'n01688243', 'n01689811', 'n01692333', 'n01693334', 'n01694178', 'n01695060', 'n01697457', 'n01698640', 'n01704323', 'n01728572', 'n01728920', 'n01729322', 'n01729977', 'n01734418', 'n01735189', 'n01737021', 'n01739381', 'n01740131', 'n01742172', 'n01744401', 'n01748264', 'n01749939', 'n01751748', 'n01753488', 'n01755581', 'n01756291', 'n01768244', 'n01770081', 'n01770393', 'n01773157', 'n01773549', 'n01773797', 'n01774384', 'n01774750', 'n01775062', 'n01776313', 'n01784675', 'n01795545', 'n01796340', 'n01797886', 'n01798484', 'n01806143', 'n01806567', 'n01807496', 'n01817953', 'n01818515', 'n01819313', 'n01820546', 'n01824575', 'n01828970', 'n01829413', 'n01833805', 'n01843065', 'n01843383', 'n01847000', 'n01855032', 'n01855672', 'n01860187', 'n01871265', 'n01872401', 'n01873310', 'n01877812', 'n01882714', 'n01883070', 'n01910747', 'n01914609', 'n01917289', 'n01924916', 'n01930112', 'n01943899', 'n01944390', 'n01945685', 'n01950731', 'n01955084', 'n01968897', 'n01978287', 'n01978455', 'n01980166', 'n01981276', 'n01983481', 'n01984695', 'n01985128', 'n01986214', 'n01990800', 'n02002556', 'n02002724', 'n02006656', 'n02007558', 'n02009229', 'n02009912', 'n02011460', 'n02012849', 'n02013706', 'n02017213', 'n02018207', 'n02018795', 'n02025239', 'n02027492', 'n02028035', 'n02033041', 'n02037110', 'n02051845', 'n02056570', 'n02058221', 'n02066245', 'n02071294', 'n02074367', 'n02077923', 'n02085620', 'n02085782', 'n02085936', 'n02086079', 'n02086240', 'n02086646', 'n02086910', 'n02087046', 'n02087394', 'n02088094', 'n02088238', 'n02088364', 'n02088466', 'n02088632', 'n02089078', 'n02089867', 'n02089973', 'n02090379', 'n02090622', 'n02090721', 'n02091032', 'n02091134', 'n02091244', 'n02091467', 'n02091635', 'n02091831', 'n02092002', 'n02092339', 'n02093256', 'n02093428', 'n02093647', 'n02093754', 'n02093859', 'n02093991', 'n02094114', 'n02094258', 'n02094433', 'n02095314', 'n02095570', 'n02095889', 'n02096051', 'n02096177', 'n02096294', 'n02096437', 'n02096585', 'n02097047', 'n02097130', 'n02097209', 'n02097298', 'n02097474', 'n02097658', 'n02098105', 'n02098286', 'n02098413', 'n02099267', 'n02099429', 'n02099601', 'n02099712', 'n02099849', 'n02100236', 'n02100583', 'n02100735', 'n02100877', 'n02101006', 'n02101388', 'n02101556', 'n02102040', 'n02102177', 'n02102318', 'n02102480', 'n02102973', 'n02104029', 'n02104365', 'n02105056', 'n02105162', 'n02105251', 'n02105412', 'n02105505', 'n02105641', 'n02105855', 'n02106030', 'n02106166', 'n02106382', 'n02106550', 'n02106662', 'n02107142', 'n02107312', 'n02107574', 'n02107683', 'n02107908', 'n02108000', 'n02108089', 'n02108422', 'n02108551', 'n02108915', 'n02109047', 'n02109525', 'n02109961', 'n02110063', 'n02110185', 'n02110341', 'n02110627', 'n02110806', 'n02110958', 'n02111129', 'n02111277', 'n02111500', 'n02111889', 'n02112018', 'n02112137', 'n02112350', 'n02112706', 'n02113023', 'n02113186', 'n02113624', 'n02113712', 'n02113799', 'n02113978', 'n02114367', 'n02114548', 'n02114712', 'n02114855', 'n02115641', 'n02115913', 'n02116738', 'n02117135', 'n02119022', 'n02119789', 'n02120079', 'n02120505', 'n02123045', 'n02123159', 'n02123394', 'n02123597', 'n02124075', 'n02125311', 'n02127052', 'n02128385', 'n02128757', 'n02128925', 'n02129165', 'n02129604', 'n02130308', 'n02132136', 'n02133161', 'n02134084', 'n02134418', 'n02137549', 'n02138441', 'n02165105', 'n02165456', 'n02167151', 'n02168699', 'n02169497', 'n02172182', 'n02174001', 'n02177972', 'n02190166', 'n02206856', 'n02219486', 'n02226429', 'n02229544', 'n02231487', 'n02233338', 'n02236044', 'n02256656', 'n02259212', 'n02264363', 'n02268443', 'n02268853', 'n02276258', 'n02277742', 'n02279972', 'n02280649', 'n02281406', 'n02281787', 'n02317335', 'n02319095', 'n02321529', 'n02325366', 'n02326432', 'n02328150', 'n02342885', 'n02346627', 'n02356798', 'n02361337', 'n02363005', 'n02364673', 'n02389026', 'n02391049', 'n02395406', 'n02396427', 'n02397096', 'n02398521', 'n02403003', 'n02408429', 'n02410509', 'n02412080', 'n02415577', 'n02417914', 'n02422106', 'n02422699', 'n02423022', 'n02437312', 'n02437616', 'n02441942', 'n02442845', 'n02443114', 'n02443484', 'n02444819', 'n02445715', 'n02447366', 'n02454379', 'n02457408', 'n02480495', 'n02480855', 'n02481823', 'n02483362', 'n02483708', 'n02484975', 'n02486261', 'n02486410', 'n02487347', 'n02488291', 'n02488702', 'n02489166', 'n02490219', 'n02492035', 'n02492660', 'n02493509', 'n02493793', 'n02494079', 'n02497673', 'n02500267', 'n02504013', 'n02504458', 'n02509815', 'n02510455', 'n02514041', 'n02526121', 'n02536864', 'n02606052', 'n02607072', 'n02640242', 'n02641379', 'n02643566', 'n02655020', 'n02666196', 'n02667093', 'n02669723', 'n02672831', 'n02676566', 'n02687172', 'n02690373', 'n02692877', 'n02699494', 'n02701002', 'n02704792', 'n02708093', 'n02727426', 'n02730930', 'n02747177', 'n02749479', 'n02769748', 'n02776631', 'n02777292', 'n02782093', 'n02783161', 'n02786058', 'n02787622', 'n02788148', 'n02790996', 'n02791124', 'n02791270', 'n02793495', 'n02794156', 'n02795169', 'n02797295', 'n02799071', 'n02802426', 'n02804414', 'n02804610', 'n02807133', 'n02808304', 'n02808440', 'n02814533', 'n02814860', 'n02815834', 'n02817516', 'n02823428', 'n02823750', 'n02825657', 'n02834397', 'n02835271', 'n02837789', 'n02840245', 'n02841315', 'n02843684', 'n02859443', 'n02860847', 'n02865351', 'n02869837', 'n02870880', 'n02871525', 'n02877765', 'n02879718', 'n02883205', 'n02892201', 'n02892767', 'n02894605', 'n02895154', 'n02906734', 'n02909870', 'n02910353', 'n02916936', 'n02917067', 'n02927161', 'n02930766', 'n02939185', 'n02948072', 'n02950826', 'n02951358', 'n02951585', 'n02963159', 'n02965783', 'n02966193', 'n02966687', 'n02971356', 'n02974003', 'n02977058', 'n02978881', 'n02979186', 'n02980441', 'n02981792', 'n02988304', 'n02992211', 'n02992529', 'n02999410', 'n03000134', 'n03000247', 'n03000684', 'n03014705', 'n03016953', 'n03017168', 'n03018349', 'n03026506', 'n03028079', 'n03032252', 'n03041632', 'n03042490', 'n03045698', 'n03047690', 'n03062245', 'n03063599', 'n03063689', 'n03065424', 'n03075370', 'n03085013', 'n03089624', 'n03095699', 'n03100240', 'n03109150', 'n03110669', 'n03124043', 'n03124170', 'n03125729', 'n03126707', 'n03127747', 'n03127925', 'n03131574', 'n03133878', 'n03134739', 'n03141823', 'n03146219', 'n03160309', 'n03179701', 'n03180011', 'n03187595', 'n03188531', 'n03196217', 'n03197337', 'n03201208', 'n03207743', 'n03207941', 'n03208938', 'n03216828', 'n03218198', 'n03220513', 'n03223299', 'n03240683', 'n03249569', 'n03250847', 'n03255030', 'n03259280', 'n03271574', 'n03272010', 'n03272562', 'n03290653', 'n03291819', 'n03297495', 'n03314780', 'n03325584', 'n03337140', 'n03344393', 'n03345487', 'n03347037', 'n03355925', 'n03372029', 'n03376595', 'n03379051', 'n03384352', 'n03388043', 'n03388183', 'n03388549', 'n03393912', 'n03394916', 'n03400231', 'n03404251', 'n03417042', 'n03424325', 'n03425413', 'n03443371', 'n03444034', 'n03445777', 'n03445924', 'n03447447', 'n03447721', 'n03450230', 'n03452741', 'n03457902', 'n03459775', 'n03461385', 'n03467068', 'n03476684', 'n03476991', 'n03478589', 'n03481172', 'n03482405', 'n03483316', 'n03485407', 'n03485794', 'n03492542', 'n03494278', 'n03495258', 'n03496892', 'n03498962', 'n03527444', 'n03529860', 'n03530642', 'n03532672', 'n03534580', 'n03535780', 'n03538406', 'n03544143', 'n03584254', 'n03584829', 'n03590841', 'n03594734', 'n03594945', 'n03595614', 'n03598930', 'n03599486', 'n03602883', 'n03617480', 'n03623198', 'n03627232', 'n03630383', 'n03633091', 'n03637318', 'n03642806', 'n03649909', 'n03657121', 'n03658185', 'n03661043', 'n03662601', 'n03666591', 'n03670208', 'n03673027', 'n03676483', 'n03680355', 'n03690938', 'n03691459', 'n03692522', 'n03697007', 'n03706229', 'n03709823', 'n03710193', 'n03710637', 'n03710721', 'n03717622', 'n03720891', 'n03721384', 'n03724870', 'n03729826', 'n03733131', 'n03733281', 'n03733805', 'n03742115', 'n03743016', 'n03759954', 'n03761084', 'n03763968', 'n03764736', 'n03769881', 'n03770439', 'n03770679', 'n03773504', 'n03775071', 'n03775546', 'n03776460', 'n03777568', 'n03777754', 'n03781244', 'n03782006', 'n03785016', 'n03786901', 'n03787032', 'n03788195', 'n03788365', 'n03791053', 'n03792782', 'n03792972', 'n03793489', 'n03794056', 'n03796401', 'n03803284', 'n03804744', 'n03814639', 'n03814906', 'n03825788', 'n03832673', 'n03837869', 'n03838899', 'n03840681', 'n03841143', 'n03843555', 'n03854065', 'n03857828', 'n03866082', 'n03868242', 'n03868863', 'n03871628', 'n03873416', 'n03874293', 'n03874599', 'n03876231', 'n03877472', 'n03877845', 'n03884397', 'n03887697', 'n03888257', 'n03888605', 'n03891251', 'n03891332', 'n03895866', 'n03899768', 'n03902125', 'n03903868', 'n03908618', 'n03908714', 'n03916031', 'n03920288', 'n03924679', 'n03929660', 'n03929855', 'n03930313', 'n03930630', 'n03933933', 'n03935335', 'n03937543', 'n03938244', 'n03942813', 'n03944341', 'n03947888', 'n03950228', 'n03954731', 'n03956157', 'n03958227', 'n03961711', 'n03967562', 'n03970156', 'n03976467', 'n03976657', 'n03977966', 'n03980874', 'n03982430', 'n03983396', 'n03991062', 'n03992509', 'n03995372', 'n03998194', 'n04004767', 'n04005630', 'n04008634', 'n04009552', 'n04019541', 'n04023962', 'n04026417', 'n04033901', 'n04033995', 'n04037443', 'n04039381', 'n04040759', 'n04041544', 'n04044716', 'n04049303', 'n04065272', 'n04067472', 'n04069434', 'n04070727', 'n04074963', 'n04081281', 'n04086273', 'n04090263', 'n04099969', 'n04111531', 'n04116512', 'n04118538', 'n04118776', 'n04120489', 'n04125021', 'n04127249', 'n04131690', 'n04133789', 'n04136333', 'n04141076', 'n04141327', 'n04141975', 'n04146614', 'n04147183', 'n04149813', 'n04152593', 'n04153751', 'n04154565', 'n04162706', 'n04179913', 'n04192698', 'n04200800', 'n04201297', 'n04204238', 'n04204347', 'n04208210', 'n04209133', 'n04209239', 'n04228054', 'n04229816', 'n04235860', 'n04238763', 'n04239074', 'n04243546', 'n04251144', 'n04252077', 'n04252225', 'n04254120', 'n04254680', 'n04254777', 'n04258138', 'n04259630', 'n04263257', 'n04264628', 'n04265275', 'n04266014', 'n04270147', 'n04273569', 'n04275548', 'n04277352', 'n04285008', 'n04286575', 'n04296562', 'n04310018', 'n04311004', 'n04311174', 'n04317175', 'n04325704', 'n04326547', 'n04328186', 'n04330267', 'n04332243', 'n04335435', 'n04336792', 'n04344873', 'n04346328', 'n04347754', 'n04350905', 'n04355338', 'n04355933', 'n04356056', 'n04357314', 'n04366367', 'n04367480', 'n04370456', 'n04371430', 'n04371774', 'n04372370', 'n04376876', 'n04380533', 'n04389033', 'n04392985', 'n04398044', 'n04399382', 'n04404412', 'n04409515', 'n04417672', 'n04418357', 'n04423845', 'n04428191', 'n04429376', 'n04435653', 'n04442312', 'n04443257', 'n04447861', 'n04456115', 'n04458633', 'n04461696', 'n04462240', 'n04465501', 'n04467665', 'n04476259', 'n04479046', 'n04482393', 'n04483307', 'n04485082', 'n04486054', 'n04487081', 'n04487394', 'n04493381', 'n04501370', 'n04505470', 'n04507155', 'n04509417', 'n04515003', 'n04517823', 'n04522168', 'n04523525', 'n04525038', 'n04525305', 'n04532106', 'n04532670', 'n04536866', 'n04540053', 'n04542943', 'n04548280', 'n04548362', 'n04550184', 'n04552348', 'n04553703', 'n04554684', 'n04557648', 'n04560804', 'n04562935', 'n04579145', 'n04579432', 'n04584207', 'n04589890', 'n04590129', 'n04591157', 'n04591713', 'n04592741', 'n04596742', 'n04597913', 'n04599235', 'n04604644', 'n04606251', 'n04612504', 'n04613696', 'n06359193', 'n06596364', 'n06785654', 'n06794110', 'n06874185', 'n07248320', 'n07565083', 'n07579787', 'n07583066', 'n07584110', 'n07590611', 'n07613480', 'n07614500', 'n07615774', 'n07684084', 'n07693725', 'n07695742', 'n07697313', 'n07697537', 'n07711569', 'n07714571', 'n07714990', 'n07715103', 'n07716358', 'n07716906', 'n07717410', 'n07717556', 'n07718472', 'n07718747', 'n07720875', 'n07730033', 'n07734744', 'n07742313', 'n07745940', 'n07747607', 'n07749582', 'n07753113', 'n07753275', 'n07753592', 'n07754684', 'n07760859', 'n07768694', 'n07802026', 'n07831146', 'n07836838', 'n07860988', 'n07871810', 'n07873807', 'n07875152', 'n07880968', 'n07892512', 'n07920052', 'n07930864', 'n07932039', 'n09193705', 'n09229709', 'n09246464', 'n09256479', 'n09288635', 'n09332890', 'n09399592', 'n09421951', 'n09428293', 'n09468604', 'n09472597', 'n09835506', 'n10148035', 'n10565667', 'n11879895', 'n11939491', 'n12057211', 'n12144580', 'n12267677', 'n12620546', 'n12768682', 'n12985857', 'n12998815', 'n13037406', 'n13040303', 'n13044778', 'n13052670', 'n13054560', 'n13133613', 'n15075141']
816
+
817
+
CLIP_benchmark/clip_benchmark/datasets/caltech101.py ADDED
@@ -0,0 +1,243 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Code adapted from https://github.com/pytorch/vision/blob/main/torchvision/datasets/caltech.py
3
+ Modification of caltech101 from torchvision where the background class is not removed
4
+ Thanks to the authors of torchvision
5
+ """
6
+ from glob import glob
7
+ import os
8
+ import os.path
9
+ from typing import Any, Callable, List, Optional, Union, Tuple
10
+
11
+ from PIL import Image
12
+
13
+ from torchvision.datasets.utils import download_and_extract_archive, verify_str_arg
14
+ from torchvision.datasets.vision import VisionDataset
15
+
16
+
17
+ class Caltech101(VisionDataset):
18
+ """`Caltech 101 <http://www.vision.caltech.edu/Image_Datasets/Caltech101/>`_ Dataset.
19
+
20
+ .. warning::
21
+
22
+ This class needs `scipy <https://docs.scipy.org/doc/>`_ to load target files from `.mat` format.
23
+
24
+ Args:
25
+ root (string): Root directory of dataset where directory
26
+ ``caltech101`` exists or will be saved to if download is set to True.
27
+ target_type (string or list, optional): Type of target to use, ``category`` or
28
+ ``annotation``. Can also be a list to output a tuple with all specified
29
+ target types. ``category`` represents the target class, and
30
+ ``annotation`` is a list of points from a hand-generated outline.
31
+ Defaults to ``category``.
32
+ transform (callable, optional): A function/transform that takes in an PIL image
33
+ and returns a transformed version. E.g, ``transforms.RandomCrop``
34
+ target_transform (callable, optional): A function/transform that takes in the
35
+ target and transforms it.
36
+ download (bool, optional): If true, downloads the dataset from the internet and
37
+ puts it in root directory. If dataset is already downloaded, it is not
38
+ downloaded again.
39
+ """
40
+
41
+ def __init__(
42
+ self,
43
+ root: str,
44
+ target_type: Union[List[str], str] = "category",
45
+ transform: Optional[Callable] = None,
46
+ target_transform: Optional[Callable] = None,
47
+ download: bool = False,
48
+ ) -> None:
49
+ super().__init__(os.path.join(root, "caltech101"), transform=transform, target_transform=target_transform)
50
+ os.makedirs(self.root, exist_ok=True)
51
+ if isinstance(target_type, str):
52
+ target_type = [target_type]
53
+ self.target_type = [verify_str_arg(t, "target_type", ("category", "annotation")) for t in target_type]
54
+
55
+ if download:
56
+ self.download()
57
+
58
+ if not self._check_integrity():
59
+ raise RuntimeError("Dataset not found or corrupted. You can use download=True to download it")
60
+
61
+ self.categories = sorted(os.listdir(os.path.join(self.root, "101_ObjectCategories")))
62
+ #self.categories.remove("BACKGROUND_Google") # this is not a real class
63
+
64
+ # For some reason, the category names in "101_ObjectCategories" and
65
+ # "Annotations" do not always match. This is a manual map between the
66
+ # two. Defaults to using same name, since most names are fine.
67
+ name_map = {
68
+ "Faces": "Faces_2",
69
+ "Faces_easy": "Faces_3",
70
+ "Motorbikes": "Motorbikes_16",
71
+ "airplanes": "Airplanes_Side_2",
72
+ }
73
+ self.annotation_categories = list(map(lambda x: name_map[x] if x in name_map else x, self.categories))
74
+
75
+ self.index: List[int] = []
76
+ self.y = []
77
+ for (i, c) in enumerate(self.categories):
78
+ n = len(glob(os.path.join(self.root, "101_ObjectCategories", c, "*.jpg")))
79
+ self.index.extend(range(1, n + 1))
80
+ self.y.extend(n * [i])
81
+
82
+ def __getitem__(self, index: int) -> Tuple[Any, Any]:
83
+ """
84
+ Args:
85
+ index (int): Index
86
+
87
+ Returns:
88
+ tuple: (image, target) where the type of target specified by target_type.
89
+ """
90
+ import scipy.io
91
+
92
+ img = Image.open(
93
+ os.path.join(
94
+ self.root,
95
+ "101_ObjectCategories",
96
+ self.categories[self.y[index]],
97
+ f"image_{self.index[index]:04d}.jpg",
98
+ )
99
+ )
100
+
101
+ target: Any = []
102
+ for t in self.target_type:
103
+ if t == "category":
104
+ target.append(self.y[index])
105
+ elif t == "annotation":
106
+ data = scipy.io.loadmat(
107
+ os.path.join(
108
+ self.root,
109
+ "Annotations",
110
+ self.annotation_categories[self.y[index]],
111
+ f"annotation_{self.index[index]:04d}.mat",
112
+ )
113
+ )
114
+ target.append(data["obj_contour"])
115
+ target = tuple(target) if len(target) > 1 else target[0]
116
+
117
+ if self.transform is not None:
118
+ img = self.transform(img)
119
+
120
+ if self.target_transform is not None:
121
+ target = self.target_transform(target)
122
+
123
+ return img, target
124
+
125
+ def _check_integrity(self) -> bool:
126
+ # can be more robust and check hash of files
127
+ return os.path.exists(os.path.join(self.root, "101_ObjectCategories"))
128
+
129
+ def __len__(self) -> int:
130
+ return len(self.index)
131
+
132
+ def download(self) -> None:
133
+ if self._check_integrity():
134
+ print("Files already downloaded and verified")
135
+ return
136
+
137
+ download_and_extract_archive(
138
+ "https://drive.google.com/file/d/137RyRjvTBkBiIfeYBNZBtViDHQ6_Ewsp",
139
+ self.root,
140
+ filename="101_ObjectCategories.tar.gz",
141
+ md5="b224c7392d521a49829488ab0f1120d9",
142
+ )
143
+ download_and_extract_archive(
144
+ "https://drive.google.com/file/d/175kQy3UsZ0wUEHZjqkUDdNVssr7bgh_m",
145
+ self.root,
146
+ filename="Annotations.tar",
147
+ md5="6f83eeb1f24d99cab4eb377263132c91",
148
+ )
149
+
150
+ def extra_repr(self) -> str:
151
+ return "Target type: {target_type}".format(**self.__dict__)
152
+
153
+
154
+ class Caltech256(VisionDataset):
155
+ """`Caltech 256 <http://www.vision.caltech.edu/Image_Datasets/Caltech256/>`_ Dataset.
156
+
157
+ Args:
158
+ root (string): Root directory of dataset where directory
159
+ ``caltech256`` exists or will be saved to if download is set to True.
160
+ transform (callable, optional): A function/transform that takes in an PIL image
161
+ and returns a transformed version. E.g, ``transforms.RandomCrop``
162
+ target_transform (callable, optional): A function/transform that takes in the
163
+ target and transforms it.
164
+ download (bool, optional): If true, downloads the dataset from the internet and
165
+ puts it in root directory. If dataset is already downloaded, it is not
166
+ downloaded again.
167
+ """
168
+
169
+ def __init__(
170
+ self,
171
+ root: str,
172
+ transform: Optional[Callable] = None,
173
+ target_transform: Optional[Callable] = None,
174
+ download: bool = False,
175
+ ) -> None:
176
+ super().__init__(os.path.join(root, "caltech256"), transform=transform, target_transform=target_transform)
177
+ os.makedirs(self.root, exist_ok=True)
178
+
179
+ if download:
180
+ self.download()
181
+
182
+ if not self._check_integrity():
183
+ raise RuntimeError("Dataset not found or corrupted. You can use download=True to download it")
184
+
185
+ self.categories = sorted(os.listdir(os.path.join(self.root, "256_ObjectCategories")))
186
+ self.index: List[int] = []
187
+ self.y = []
188
+ for (i, c) in enumerate(self.categories):
189
+ n = len(
190
+ [
191
+ item
192
+ for item in os.listdir(os.path.join(self.root, "256_ObjectCategories", c))
193
+ if item.endswith(".jpg")
194
+ ]
195
+ )
196
+ self.index.extend(range(1, n + 1))
197
+ self.y.extend(n * [i])
198
+
199
+ def __getitem__(self, index: int) -> Tuple[Any, Any]:
200
+ """
201
+ Args:
202
+ index (int): Index
203
+
204
+ Returns:
205
+ tuple: (image, target) where target is index of the target class.
206
+ """
207
+ img = Image.open(
208
+ os.path.join(
209
+ self.root,
210
+ "256_ObjectCategories",
211
+ self.categories[self.y[index]],
212
+ f"{self.y[index] + 1:03d}_{self.index[index]:04d}.jpg",
213
+ )
214
+ )
215
+
216
+ target = self.y[index]
217
+
218
+ if self.transform is not None:
219
+ img = self.transform(img)
220
+
221
+ if self.target_transform is not None:
222
+ target = self.target_transform(target)
223
+
224
+ return img, target
225
+
226
+ def _check_integrity(self) -> bool:
227
+ # can be more robust and check hash of files
228
+ return os.path.exists(os.path.join(self.root, "256_ObjectCategories"))
229
+
230
+ def __len__(self) -> int:
231
+ return len(self.index)
232
+
233
+ def download(self) -> None:
234
+ if self._check_integrity():
235
+ print("Files already downloaded and verified")
236
+ return
237
+
238
+ download_and_extract_archive(
239
+ "https://drive.google.com/file/d/1r6o0pSROcV1_VwT4oSjA2FBUSCWGuxLK",
240
+ self.root,
241
+ filename="256_ObjectCategories.tar",
242
+ md5="67b4f42ca05d46448c6bb8ecd2220f6d",
243
+ )
CLIP_benchmark/clip_benchmark/datasets/cn_classnames.json ADDED
@@ -0,0 +1,1004 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "imagenet1k": [
3
+ "\u4e01\u9cb7",
4
+ "\u91d1\u9c7c",
5
+ "\u5927\u767d\u9ca8",
6
+ "\u864e\u9ca8",
7
+ "\u9524\u5934\u9ca8",
8
+ "\u7535\u9cd0",
9
+ "\u9ec4\u8c82\u9c7c",
10
+ "\u516c\u9e21",
11
+ "\u6bcd\u9e21",
12
+ "\u9e35\u9e1f",
13
+ "\u71d5\u96c0",
14
+ "\u91d1\u7fc5\u96c0",
15
+ "\u5bb6\u6731\u96c0",
16
+ "\u706f\u82af\u8349\u96c0",
17
+ "\u975b\u84dd\u96c0",
18
+ "\u84dd\u9e40",
19
+ "\u591c\u83ba",
20
+ "\u677e\u9e26",
21
+ "\u559c\u9e4a",
22
+ "\u5c71\u96c0",
23
+ "\u6cb3\u9e1f",
24
+ "\u9e22\uff08\u731b\u79bd\uff09",
25
+ "\u79c3\u5934\u9e70",
26
+ "\u79c3\u9e6b",
27
+ "\u5927\u7070\u732b\u5934\u9e70",
28
+ "\u6b27\u6d32\u706b\u877e\u8788",
29
+ "\u666e\u901a\u877e\u8788",
30
+ "\u6c34\u8725",
31
+ "\u6591\u70b9\u877e\u8788",
32
+ "\u877e\u8788",
33
+ "\u725b\u86d9",
34
+ "\u6811\u86d9",
35
+ "\u5c3e\u86d9",
36
+ "\u7ea2\u6d77\u9f9f",
37
+ "\u76ae\u9769\u9f9f",
38
+ "\u6ce5\u9f9f",
39
+ "\u6de1\u6c34\u9f9f",
40
+ "\u7bb1\u9f9f",
41
+ "\u5e26\u72b6\u58c1\u864e",
42
+ "\u666e\u901a\u9b23\u8725",
43
+ "\u7f8e\u56fd\u53d8\u8272\u9f99",
44
+ "\u97ad\u5c3e\u8725\u8734",
45
+ "\u98de\u9f99\u79d1\u8725\u8734",
46
+ "\u8936\u8fb9\u8725\u8734",
47
+ "\u9cc4\u9c7c\u8725\u8734",
48
+ "\u6bd2\u8725",
49
+ "\u7eff\u8725\u8734",
50
+ "\u975e\u6d32\u53d8\u8272\u9f99",
51
+ "\u79d1\u83ab\u591a\u8725\u8734",
52
+ "\u975e\u6d32\u9cc4",
53
+ "\u7f8e\u56fd\u9cc4\u9c7c",
54
+ "\u4e09\u89d2\u9f99",
55
+ "\u96f7\u86c7",
56
+ "\u73af\u86c7",
57
+ "\u5e0c\u814a\u86c7",
58
+ "\u7eff\u86c7",
59
+ "\u56fd\u738b\u86c7",
60
+ "\u889c\u5e26\u86c7",
61
+ "\u6c34\u86c7",
62
+ "\u85e4\u86c7",
63
+ "\u591c\u86c7",
64
+ "\u5927\u87d2\u86c7",
65
+ "\u5ca9\u77f3\u87d2\u86c7",
66
+ "\u5370\u5ea6\u773c\u955c\u86c7",
67
+ "\u7eff\u66fc\u5df4",
68
+ "\u6d77\u86c7",
69
+ "\u89d2\u8179\u86c7",
70
+ "\u83f1\u7eb9\u54cd\u5c3e\u86c7",
71
+ "\u89d2\u54cd\u5c3e\u86c7",
72
+ "\u4e09\u53f6\u866b",
73
+ "\u76f2\u8718\u86db",
74
+ "\u874e\u5b50",
75
+ "\u9ed1\u91d1\u82b1\u56ed\u8718\u86db",
76
+ "\u8c37\u4ed3\u8718\u86db",
77
+ "\u82b1\u56ed\u8718\u86db",
78
+ "\u9ed1\u5be1\u5987\u8718\u86db",
79
+ "\u72fc\u86db",
80
+ "\u72fc\u8718\u86db",
81
+ "\u58c1\u8671",
82
+ "\u8708\u86a3",
83
+ "\u9ed1\u677e\u9e21",
84
+ "\u677e\u9e21",
85
+ "\u62ab\u80a9\u9e21",
86
+ "\u8349\u539f\u9e21",
87
+ "\u5b54\u96c0",
88
+ "\u9e4c\u9e51",
89
+ "\u9e67\u9e2a",
90
+ "\u975e\u6d32\u7070\u9e66\u9e49",
91
+ "\u91d1\u521a\u9e66\u9e49",
92
+ "\u786b\u51a0\u9e66\u9e49",
93
+ "\u77ed\u5c3e\u9e66\u9e49",
94
+ "\u8910\u7fc5\u9e26\u9e43",
95
+ "\u98df\u8702\u9e1f\uff1b\u8702\u864e",
96
+ "\u7280\u9e1f",
97
+ "\u8702\u9e1f",
98
+ "\u9e5f\u4d15",
99
+ "\u5de8\u5634\u9e1f\uff1b\u5927\u5634\u9e1f",
100
+ "\u91ce\u9e2d",
101
+ "\u7ea2\u80f8\u79cb\u6c99\u9e2d",
102
+ "\u9e45",
103
+ "\u9ed1\u5929\u9e45",
104
+ "\u5927\u8c61",
105
+ "\u9488\u9f39\u9f20",
106
+ "\u9e2d\u5634\u517d",
107
+ "\u6c99\u888b\u9f20",
108
+ "\u8003\u62c9",
109
+ "\u888b\u718a",
110
+ "\u6c34\u6bcd",
111
+ "\u6d77\u8475",
112
+ "\u8111\u73ca\u745a",
113
+ "\u6241\u5f62\u866b\u6241\u866b",
114
+ "\u7ebf\u866b",
115
+ "\u6d77\u87ba",
116
+ "\u8717\u725b",
117
+ "\u9f3b\u6d95\u866b",
118
+ "\u6d77\u86de\u8753\uff1b\u6d77\u53c2",
119
+ "\u77f3\u9cd6",
120
+ "\u9e66\u9e49\u87ba",
121
+ "\u73cd\u5b9d\u87f9",
122
+ "\u77f3\u87f9",
123
+ "\u62db\u6f6e\u87f9",
124
+ "\u5e1d\u738b\u87f9",
125
+ "\u7f8e\u56fd\u9f99\u867e",
126
+ "\u5927\u87af\u867e",
127
+ "\u5c0f\u9f99\u867e",
128
+ "\u5bc4\u5c45\u87f9",
129
+ "\u7b49\u8db3\u76ee\u52a8\u7269\uff08\u660e\u867e\u548c\u8783\u87f9\u8fd1\u4eb2\uff09",
130
+ "\u767d\u9e73",
131
+ "\u9ed1\u9e73",
132
+ "\u9e6d",
133
+ "\u706b\u70c8\u9e1f",
134
+ "\u5c0f\u84dd\u9e6d",
135
+ "\u7f8e\u56fd\u9e6d",
136
+ "\u9ebb\u9e26",
137
+ "\u9e64",
138
+ "\u79e7\u9e64",
139
+ "\u6b27\u6d32\u6c34\u9e21",
140
+ "\u6cbc\u6cfd\u6ce5\u6bcd\u9e21",
141
+ "\u9e28",
142
+ "\u7ea2\u7ffb\u77f3\u9e6c",
143
+ "\u7ea2\u80cc\u9e6c",
144
+ "\u7ea2\u811a\u9e6c",
145
+ "\u534a\u8e7c\u9e6c",
146
+ "\u86ce\u9e6c",
147
+ "\u9e48\u9e55",
148
+ "\u56fd\u738b\u4f01\u9e45",
149
+ "\u4fe1\u5929\u7fc1",
150
+ "\u7070\u9cb8",
151
+ "\u6740\u4eba\u9cb8",
152
+ "\u6d77\u725b",
153
+ "\u6d77\u72ee",
154
+ "\u5409\u5a03\u5a03",
155
+ "\u65e5\u672c\u72c6\u72ac",
156
+ "\u9a6c\u5c14\u6d4e\u65af\u72ac",
157
+ "\u72ee\u5b50\u72d7",
158
+ "\u897f\u65bd\u72ac",
159
+ "\u5e03\u83b1\u5c3c\u59c6\u730e\u72ac",
160
+ "\u5df4\u6bd4\u72d7",
161
+ "\u73a9\u5177\u72ac",
162
+ "\u7f57\u5f97\u897f\u4e9a\u957f\u80cc\u730e\u72d7",
163
+ "\u963f\u5bcc\u6c57\u730e\u72ac",
164
+ "\u5df4\u5409\u5ea6\u730e\u72ac",
165
+ "\u6bd4\u683c\u72ac",
166
+ "\u4fa6\u63a2\u72ac",
167
+ "\u84dd\u8272\u5feb\u72d7",
168
+ "\u9ed1\u8910\u730e\u6d63\u718a\u72ac",
169
+ "\u6c83\u514b\u730e\u72ac",
170
+ "\u82f1\u56fd\u730e\u72d0\u72ac",
171
+ "\u7f8e\u6d32\u8d64\u72d7",
172
+ "\u4fc4\u7f57\u65af\u730e\u72fc\u72ac",
173
+ "\u7231\u5c14\u5170\u730e\u72fc\u72ac",
174
+ "\u610f\u5927\u5229\u7070\u72d7",
175
+ "\u60e0\u6bd4\u7279\u72ac",
176
+ "\u4f9d\u6bd4\u6c99\u730e\u72ac",
177
+ "\u632a\u5a01\u730e\u72ac",
178
+ "\u5965\u8fbe\u730e\u72ac",
179
+ "\u6c99\u514b\u72ac",
180
+ "\u82cf\u683c\u5170\u730e\u9e7f\u72ac",
181
+ "\u5a01\u739b\u730e\u72ac",
182
+ "\u65af\u5854\u798f\u5fb7\u90e1\u6597\u725b\u72ac",
183
+ "\u7f8e\u56fd\u65af\u5854\u798f\u5fb7\u90e1\u6897",
184
+ "\u8d1d\u5fb7\u7075\u987f\u6897",
185
+ "\u8fb9\u5883\u6897",
186
+ "\u51ef\u4e3d\u84dd\u6897",
187
+ "\u7231\u5c14\u5170\u6897",
188
+ "\u8bfa\u798f\u514b\u6897",
189
+ "\u8bfa\u7ef4\u5947\u6897",
190
+ "\u7ea6\u514b\u72ac\uff1b\u7ea6\u514b\u590f\u6897\u72ac",
191
+ "\u521a\u6bdb\u730e\u72d0\u6897",
192
+ "\u83b1\u514b\u5170\u6897",
193
+ "\u9521\u5229\u54c8\u59c6\u6897",
194
+ "\u827e\u5c14\u8c37\u72ac",
195
+ "\u51ef\u6069\u6897",
196
+ "\u6fb3\u5927\u5229\u4e9a\u6897",
197
+ "\u4e39\u8fea\u4e01\u8499\u6897",
198
+ "\u6ce2\u58eb\u987f\u6897",
199
+ "\u8ff7\u4f60\u96ea\u7eb3\u745e\u72ac",
200
+ "\u5de8\u578b\u96ea\u7eb3\u745e\u72ac",
201
+ "\u6807\u51c6\u96ea\u7eb3\u745e\u72ac",
202
+ "\u82cf\u683c\u5170\u6897\u72ac",
203
+ "\u897f\u85cf\u6897",
204
+ "\u4e1d\u6bdb\u6897",
205
+ "\u7231\u5c14\u5170\u8f6f\u6bdb\u6897\u72ac",
206
+ "\u897f\u9ad8\u5730\u767d\u6897",
207
+ "\u62c9\u8428\u963f\u666e\u7d22\u72ac",
208
+ "\u5e73\u6bdb\u5bfb\u56de\u72ac",
209
+ "\u5377\u6bdb\u5bfb\u56de\u72ac",
210
+ "\u91d1\u6bdb\u730e\u72ac",
211
+ "\u62c9\u5e03\u62c9\u591a\u730e\u72ac",
212
+ "\u4e5e\u6c99\u6bd4\u514b\u730e\u72ac",
213
+ "\u5fb7\u56fd\u77ed\u6bdb\u6307\u793a\u72ac",
214
+ "\u7ef4\u5179\u62c9\u72ac",
215
+ "\u82f1\u56fd\u585e\u7279\u72ac",
216
+ "\u7231\u5c14\u5170\u96ea\u8fbe\u72ac",
217
+ "\u6208\u767b\u96ea\u8fbe\u72ac",
218
+ "\u5e03\u5217\u5854\u5c3c\u72ac\u730e\u72ac",
219
+ "\u9ec4\u6bdb",
220
+ "\u82f1\u56fd\u53f2\u5bbe\u683c\u72ac",
221
+ "\u5a01\u5c14\u58eb\u53f2\u5bbe\u683c\u72ac",
222
+ "\u53ef\u5361\u72ac",
223
+ "\u8428\u585e\u514b\u65af\u730e\u72ac",
224
+ "\u7231\u5c14\u5170\u6c34\u730e\u72ac",
225
+ "\u54e5\u5a01\u65af\u72ac",
226
+ "\u8212\u67cf\u5947\u72ac",
227
+ "\u6bd4\u5229\u65f6\u7267\u7f8a\u72ac",
228
+ "\u9a6c\u91cc\u52aa\u963f\u72ac",
229
+ "\u4f2f\u745e\u72ac",
230
+ "\u51ef\u5c14\u76ae\u72ac",
231
+ "\u5308\u7259\u5229\u7267\u7f8a\u72ac",
232
+ "\u8001\u82f1\u56fd\u7267\u7f8a\u72ac",
233
+ "\u559c\u4e50\u8482\u7267\u7f8a\u72ac",
234
+ "\u7267\u7f8a\u72ac",
235
+ "\u8fb9\u5883\u7267\u7f8a\u72ac",
236
+ "\u6cd5\u5170\u5fb7\u65af\u7267\u725b\u72d7",
237
+ "\u7f57\u7279\u97e6\u5c14\u72ac",
238
+ "\u5fb7\u56fd\u7267\u7f8a\u72ac",
239
+ "\u591a\u4f2f\u66fc\u72ac",
240
+ "\u9e7f\u72ac\uff1b\u8ff7\u4f60\u675c\u5bbe\u72ac",
241
+ "\u5927\u745e\u58eb\u5c71\u5730\u72ac",
242
+ "\u4f2f\u6069\u5c71\u72ac",
243
+ "\u963f\u7b56\u5c14\u5c71\u72ac",
244
+ "\u6069\u7279\u5c14\u5e03\u8d6b\u5c71\u72ac",
245
+ "\u62f3\u5e08\u72d7",
246
+ "\u6597\u725b\u7352",
247
+ "\u85cf\u7352",
248
+ "\u6cd5\u56fd\u6597\u725b\u72ac",
249
+ "\u5927\u4e39\u72ac",
250
+ "\u5723\u4f2f\u7eb3\u5fb7\u72d7",
251
+ "\u7231\u65af\u57fa\u6469\u72ac",
252
+ "\u963f\u62c9\u65af\u52a0\u96ea\u6a47\u72ac",
253
+ "\u54c8\u58eb\u5947",
254
+ "\u8fbe\u5c14\u9a6c\u63d0\u4e9a",
255
+ "\u72ee\u6bdb\u72d7",
256
+ "\u5df4\u8f9b\u5409\u72d7",
257
+ "\u516b\u54e5\u72ac",
258
+ "\u83b1\u6602\u8d1d\u683c\u72d7",
259
+ "\u7ebd\u82ac\u5170\u72ac",
260
+ "\u5927\u767d\u718a\u72ac",
261
+ "\u8428\u6469\u8036\u72ac",
262
+ "\u535a\u7f8e\u72ac",
263
+ "\u677e\u72ee",
264
+ "\u51ef\u65af\u72ac",
265
+ "\u5e03\u9c81\u585e\u5c14\u683c\u6797\u82ac\u72ac",
266
+ "\u5f6d\u5e03\u6d1b\u514b\u5a01\u5c14\u58eb\u79d1\u57fa\u72ac",
267
+ "\u5a01\u5c14\u58eb\u67ef\u57fa\u72ac",
268
+ "\u73a9\u5177\u8d35\u5bbe\u72ac",
269
+ "\u8ff7\u4f60\u8d35\u5bbe\u72ac",
270
+ "\u6807\u51c6\u8d35\u5bbe\u72ac",
271
+ "\u58a8\u897f\u54e5\u65e0\u6bdb\u72ac",
272
+ "\u7070\u72fc",
273
+ "\u767d\u72fc",
274
+ "\u7ea2\u592a\u72fc",
275
+ "\u72fc",
276
+ "\u6fb3\u6d32\u91ce\u72d7",
277
+ "\u8c7a",
278
+ "\u975e\u6d32\u730e\u72ac",
279
+ "\u9b23\u72d7",
280
+ "\u7ea2\u72d0\u72f8",
281
+ "\u6c99\u72d0",
282
+ "\u5317\u6781\u72d0\u72f8",
283
+ "\u7070\u72d0\u72f8",
284
+ "\u864e\u6591\u732b",
285
+ "\u5c71\u732b",
286
+ "\u6ce2\u65af\u732b",
287
+ "\u66b9\u7f57\u732b",
288
+ "\u57c3\u53ca\u732b",
289
+ "\u7f8e\u6d32\u72ee",
290
+ "\u731e\u7301",
291
+ "\u8c79\u5b50",
292
+ "\u96ea\u8c79",
293
+ "\u7f8e\u6d32\u864e",
294
+ "\u72ee\u5b50",
295
+ "\u8001\u864e",
296
+ "\u730e\u8c79",
297
+ "\u68d5\u718a",
298
+ "\u7f8e\u6d32\u9ed1\u718a",
299
+ "\u51b0\u718a",
300
+ "\u61d2\u718a",
301
+ "\u7374",
302
+ "\u732b\u9f2c",
303
+ "\u864e\u7532\u866b",
304
+ "\u74e2\u866b",
305
+ "\u571f\u9cd6\u866b",
306
+ "\u5929\u725b",
307
+ "\u9f9f\u7532\u866b",
308
+ "\u7caa\u7532\u866b",
309
+ "\u7280\u725b\u7532\u866b",
310
+ "\u8c61\u7532",
311
+ "\u82cd\u8747",
312
+ "\u871c\u8702",
313
+ "\u8682\u8681",
314
+ "\u86b1\u8722",
315
+ "\u87cb\u87c0",
316
+ "\u7af9\u8282\u866b",
317
+ "\u87d1\u8782",
318
+ "\u87b3\u8782",
319
+ "\u8749",
320
+ "\u53f6\u8749",
321
+ "\u8349\u873b\u86c9",
322
+ "\u873b\u8713",
323
+ "\u8c46\u5a18",
324
+ "\u4f18\u7ea2\u86f1\u8776",
325
+ "\u5c0f\u73af\u8774\u8776",
326
+ "\u541b\u4e3b\u8774\u8776",
327
+ "\u83dc\u7c89\u8776",
328
+ "\u767d\u8774\u8776",
329
+ "\u7070\u8776",
330
+ "\u6d77\u661f",
331
+ "\u6d77\u80c6",
332
+ "\u6d77\u9ec4\u74dc\uff1b\u6d77\u53c2",
333
+ "\u91ce\u5154",
334
+ "\u5154",
335
+ "\u5b89\u54e5\u62c9\u5154",
336
+ "\u4ed3\u9f20",
337
+ "\u523a\u732c",
338
+ "\u9ed1\u677e\u9f20",
339
+ "\u571f\u62e8\u9f20",
340
+ "\u6d77\u72f8",
341
+ "\u8c5a\u9f20",
342
+ "\u6817\u8272\u9a6c",
343
+ "\u6591\u9a6c",
344
+ "\u732a",
345
+ "\u91ce\u732a",
346
+ "\u75a3\u732a",
347
+ "\u6cb3\u9a6c",
348
+ "\u725b",
349
+ "\u6c34\u725b",
350
+ "\u91ce\u725b",
351
+ "\u516c\u7f8a",
352
+ "\u5927\u89d2\u7f8a",
353
+ "\u5c71\u7f8a",
354
+ "\u72f7\u7f9a",
355
+ "\u9ed1\u6591\u7f9a",
356
+ "\u77aa\u7f9a",
357
+ "\u963f\u62c9\u4f2f\u5355\u5cf0\u9a86\u9a7c",
358
+ "\u9a86\u9a7c",
359
+ "\u9ec4\u9f20\u72fc",
360
+ "\u6c34\u8c82",
361
+ "\u81ed\u732b",
362
+ "\u9ed1\u8db3\u9f2c",
363
+ "\u6c34\u736d",
364
+ "\u81ed\u9f2c",
365
+ "\u737e",
366
+ "\u72b0\u72f3",
367
+ "\u6811\u61d2",
368
+ "\u7329\u7329",
369
+ "\u5927\u7329\u7329",
370
+ "\u9ed1\u7329\u7329",
371
+ "\u957f\u81c2\u733f",
372
+ "\u5408\u8dbe\u733f\u957f\u81c2\u733f",
373
+ "\u957f\u5c3e\u7334",
374
+ "\u8d64\u7334",
375
+ "\u72d2\u72d2",
376
+ "\u6052\u6cb3\u7334",
377
+ "\u767d\u5934\u53f6\u7334",
378
+ "\u75a3\u7334",
379
+ "\u957f\u9f3b\u7334",
380
+ "\u72e8\uff08\u7f8e\u6d32\u4ea7\u5c0f\u578b\u957f\u5c3e\u7334\uff09",
381
+ "\u5377\u5c3e\u7334",
382
+ "\u543c\u7334",
383
+ "\u4f36\u7334",
384
+ "\u8718\u86db\u7334",
385
+ "\u677e\u9f20\u7334",
386
+ "\u9a6c\u8fbe\u52a0\u65af\u52a0\u73af\u5c3e\u72d0\u7334",
387
+ "\u5927\u72d0\u7334",
388
+ "\u5370\u5ea6\u5927\u8c61",
389
+ "\u975e\u6d32\u8c61",
390
+ "\u5c0f\u718a\u732b",
391
+ "\u5927\u718a\u732b",
392
+ "\u6756\u9c7c",
393
+ "\u9cd7\u9c7c",
394
+ "\u94f6\u9c91",
395
+ "\u4e09\u8272\u523a\u8776\u9c7c",
396
+ "\u6d77\u8475\u9c7c",
397
+ "\u9c9f\u9c7c",
398
+ "\u96c0\u9cdd",
399
+ "\u72ee\u5b50\u9c7c",
400
+ "\u6cb3\u8c5a",
401
+ "\u7b97\u76d8",
402
+ "\u957f\u888d",
403
+ "\u5b66\u4f4d\u888d",
404
+ "\u624b\u98ce\u7434",
405
+ "\u539f\u58f0\u5409\u4ed6",
406
+ "\u822a\u7a7a\u6bcd\u8230",
407
+ "\u5ba2\u673a",
408
+ "\u98de\u8247",
409
+ "\u796d\u575b",
410
+ "\u6551\u62a4\u8f66",
411
+ "\u6c34\u9646\u4e24\u7528\u8f66",
412
+ "\u6a21\u62df\u65f6\u949f",
413
+ "\u8702\u623f",
414
+ "\u56f4\u88d9",
415
+ "\u5783\u573e\u6876",
416
+ "\u653b\u51fb\u6b65\u67aa",
417
+ "\u80cc\u5305",
418
+ "\u9762\u5305\u5e97",
419
+ "\u5e73\u8861\u6728",
420
+ "\u70ed\u6c14\u7403",
421
+ "\u5706\u73e0\u7b14",
422
+ "\u521b\u53ef\u8d34",
423
+ "\u73ed\u5353\u7434",
424
+ "\u680f\u6746",
425
+ "\u6760\u94c3",
426
+ "\u7406\u53d1\u5e08\u7684\u6905\u5b50",
427
+ "\u7406\u53d1\u5e97",
428
+ "\u7272\u53e3\u68da",
429
+ "\u6674\u96e8\u8868",
430
+ "\u5706\u7b52",
431
+ "\u56ed\u5730\u5c0f\u8f66",
432
+ "\u68d2\u7403",
433
+ "\u7bee\u7403",
434
+ "\u5a74\u513f\u5e8a",
435
+ "\u5df4\u677e\u7ba1",
436
+ "\u6e38\u6cf3\u5e3d",
437
+ "\u6c90\u6d74\u6bdb\u5dfe",
438
+ "\u6d74\u7f38",
439
+ "\u6c99\u6ee9\u8f66",
440
+ "\u706f\u5854",
441
+ "\u70e7\u676f",
442
+ "\u718a\u76ae\u9ad8\u5e3d",
443
+ "\u5564\u9152\u74f6",
444
+ "\u5564\u9152\u676f",
445
+ "\u949f\u5854",
446
+ "\uff08\u5c0f\u513f\u7528\u7684\uff09\u56f4\u5634",
447
+ "\u4e32\u8054\u81ea\u884c\u8f66",
448
+ "\u6bd4\u57fa\u5c3c",
449
+ "\u88c5\u8ba2\u518c",
450
+ "\u53cc\u7b52\u671b\u8fdc\u955c",
451
+ "\u9e1f\u820d",
452
+ "\u8239\u5e93",
453
+ "\u53cc\u4eba\u96ea\u6a47",
454
+ "\u9970\u6263\u5f0f\u9886\u5e26",
455
+ "\u9614\u8fb9\u5973\u5e3d",
456
+ "\u4e66\u6a71",
457
+ "\u4e66\u5e97",
458
+ "\u74f6\u76d6",
459
+ "\u5f13\u7bad",
460
+ "\u8774\u8776\u7ed3\u9886\u7ed3",
461
+ "\u94dc\u5236\u724c\u4f4d",
462
+ "\u5976\u7f69",
463
+ "\u9632\u6ce2\u5824",
464
+ "\u94e0\u7532",
465
+ "\u626b\u5e1a",
466
+ "\u6876",
467
+ "\u6263\u73af",
468
+ "\u9632\u5f39\u80cc\u5fc3",
469
+ "\u52a8\u8f66",
470
+ "\u8089\u94fa",
471
+ "\u51fa\u79df\u8f66",
472
+ "\u5927\u9505",
473
+ "\u8721\u70db",
474
+ "\u5927\u70ae",
475
+ "\u72ec\u6728\u821f",
476
+ "\u5f00\u74f6\u5668",
477
+ "\u5f00\u886b",
478
+ "\u8f66\u955c",
479
+ "\u65cb\u8f6c\u6728\u9a6c",
480
+ "\u6728\u5320\u7684\u5de5\u5177\u5305",
481
+ "\u7eb8\u7bb1",
482
+ "\u8f66\u8f6e",
483
+ "\u53d6\u6b3e\u673a",
484
+ "\u76d2\u5f0f\u5f55\u97f3\u5e26",
485
+ "\u5361\u5e26\u64ad\u653e\u5668",
486
+ "\u57ce\u5821",
487
+ "\u53cc\u4f53\u8239",
488
+ "CD\u64ad\u653e\u5668",
489
+ "\u5927\u63d0\u7434",
490
+ "\u79fb\u52a8\u7535\u8bdd",
491
+ "\u94c1\u94fe",
492
+ "\u56f4\u680f",
493
+ "\u94fe\u7532",
494
+ "\u7535\u952f",
495
+ "\u7bb1\u5b50",
496
+ "\u68b3\u5986\u53f0",
497
+ "\u7f16\u949f",
498
+ "\u4e2d\u56fd\u6a71\u67dc",
499
+ "\u5723\u8bde\u889c",
500
+ "\u6559\u5802",
501
+ "\u7535\u5f71\u9662",
502
+ "\u5207\u8089\u5200",
503
+ "\u60ac\u5d16\u5c4b",
504
+ "\u6597\u7bf7",
505
+ "\u6728\u5c50",
506
+ "\u9e21\u5c3e\u9152\u8c03\u9152\u5668",
507
+ "\u5496\u5561\u676f",
508
+ "\u5496\u5561\u58f6",
509
+ "\u87ba\u65cb\u7ed3\u6784\uff08\u697c\u68af\uff09",
510
+ "\u7ec4\u5408\u9501",
511
+ "\u7535\u8111\u952e\u76d8",
512
+ "\u7cd6\u679c",
513
+ "\u96c6\u88c5\u7bb1\u8239",
514
+ "\u655e\u7bf7\u8f66",
515
+ "\u74f6\u585e\u94bb",
516
+ "\u77ed\u53f7",
517
+ "\u725b\u4ed4\u9774",
518
+ "\u725b\u4ed4\u5e3d",
519
+ "\u6447\u7bee",
520
+ "\u8d77\u91cd\u673a",
521
+ "\u5934\u76d4",
522
+ "\u677f\u6761\u7bb1",
523
+ "\u5c0f\u513f\u5e8a",
524
+ "\u7802\u9505",
525
+ "\u69cc\u7403",
526
+ "\u62d0\u6756",
527
+ "\u80f8\u7532",
528
+ "\u5927\u575d",
529
+ "\u4e66\u684c",
530
+ "\u53f0\u5f0f\u7535\u8111",
531
+ "\u6709\u7ebf\u7535\u8bdd",
532
+ "\u5c3f\u5e03\u6e7f",
533
+ "\u6570\u5b57\u65f6\u949f",
534
+ "\u6570\u5b57\u624b\u8868",
535
+ "\u9910\u684c\u677f",
536
+ "\u62b9\u5e03",
537
+ "\u6d17\u7897\u673a",
538
+ "\u76d8\u5f0f\u5236\u52a8\u5668",
539
+ "\u7801\u5934",
540
+ "\u72d7\u62c9\u96ea\u6a47",
541
+ "\u5706\u9876",
542
+ "\u95e8\u57ab",
543
+ "\u94bb\u4e95\u5e73\u53f0",
544
+ "\u9f13",
545
+ "\u9f13\u69cc",
546
+ "\u54d1\u94c3",
547
+ "\u8377\u5170\u70e4\u7bb1",
548
+ "\u7535\u98ce\u6247",
549
+ "\u7535\u5409\u4ed6",
550
+ "\u7535\u529b\u673a\u8f66",
551
+ "\u7ec4\u5408\u7535\u89c6\u67dc",
552
+ "\u4fe1\u5c01",
553
+ "\u6d53\u7f29\u5496\u5561\u673a",
554
+ "\u6251\u9762\u7c89",
555
+ "\u5973\u7528\u957f\u56f4\u5dfe",
556
+ "\u6587\u4ef6",
557
+ "\u6d88\u9632\u8239",
558
+ "\u6d88\u9632\u8f66",
559
+ "\u706b\u7089\u680f",
560
+ "\u65d7\u6746",
561
+ "\u957f\u7b1b",
562
+ "\u6298\u53e0\u6905",
563
+ "\u6a44\u6984\u7403\u5934\u76d4",
564
+ "\u53c9\u8f66",
565
+ "\u55b7\u6cc9",
566
+ "\u94a2\u7b14",
567
+ "\u6709\u56db\u6839\u5e37\u67f1\u7684\u5e8a",
568
+ "\u8fd0\u8d27\u8f66\u53a2",
569
+ "\u5706\u53f7",
570
+ "\u714e\u9505",
571
+ "\u88d8\u76ae\u5927\u8863",
572
+ "\u5783\u573e\u8f66",
573
+ "\u9632\u6bd2\u9762\u5177",
574
+ "\u6c7d\u6cb9\u6cf5",
575
+ "\u9ad8\u811a\u676f",
576
+ "\u5361\u4e01\u8f66",
577
+ "\u9ad8\u5c14\u592b\u7403",
578
+ "\u9ad8\u5c14\u592b\u7403\u8f66",
579
+ "\u72ed\u957f\u5c0f\u8239",
580
+ "\u9523",
581
+ "\u793c\u670d",
582
+ "\u94a2\u7434",
583
+ "\u6e29\u5ba4",
584
+ "\u6563\u70ed\u5668\u683c\u6805",
585
+ "\u6742\u8d27\u5e97",
586
+ "\u65ad\u5934\u53f0",
587
+ "\u5c0f\u53d1\u5939",
588
+ "\u5934\u53d1\u55b7\u96fe",
589
+ "\u534a\u5c65\u5e26\u88c5\u7532\u8f66",
590
+ "\u9524\u5b50",
591
+ "\u5927\u7bee\u5b50",
592
+ "\u624b\u6447\u9f13\u98ce\u673a",
593
+ "\u624b\u63d0\u7535\u8111",
594
+ "\u624b\u5e15",
595
+ "\u786c\u76d8",
596
+ "\u53e3\u7434",
597
+ "\u7ad6\u7434",
598
+ "\u6536\u5272\u673a",
599
+ "\u65a7\u5934",
600
+ "\u624b\u67aa\u76ae\u5957",
601
+ "\u5bb6\u5ead\u5f71\u9662",
602
+ "\u8702\u7a9d",
603
+ "\u94a9\u722a",
604
+ "\u886c\u88d9",
605
+ "\u5355\u6760",
606
+ "\u9a6c\u8f66",
607
+ "\u6c99\u6f0f",
608
+ "iPod",
609
+ "\u71a8\u6597",
610
+ "\u5357\u74dc\u706f\u7b3c",
611
+ "\u725b\u4ed4\u88e4",
612
+ "\u5409\u666e\u8f66",
613
+ "T\u6064\u886b",
614
+ "\u62fc\u56fe",
615
+ "\u4eba\u529b\u8f66",
616
+ "\u64cd\u7eb5\u6746",
617
+ "\u548c\u670d",
618
+ "\u62a4\u819d",
619
+ "\u8774\u8776\u7ed3",
620
+ "\u5927\u8902",
621
+ "\u957f\u67c4\u52fa",
622
+ "\u706f\u7f69",
623
+ "\u7b14\u8bb0\u672c\u7535\u8111",
624
+ "\u5272\u8349\u673a",
625
+ "\u955c\u5934\u76d6",
626
+ "\u5f00\u4fe1\u5200\uff1b\u62c6\u4fe1\u5200",
627
+ "\u56fe\u4e66\u9986",
628
+ "\u6551\u751f\u8247",
629
+ "\u70b9\u706b\u5668",
630
+ "\u8c6a\u534e\u8f7f\u8f66",
631
+ "\u8fdc\u6d0b\u73ed\u8f6e",
632
+ "\u5507\u818f",
633
+ "\u5e73\u5e95\u4fbf\u978b",
634
+ "\u6d17\u5242",
635
+ "\u626c\u58f0\u5668",
636
+ "\u653e\u5927\u955c",
637
+ "\u952f\u6728\u5382",
638
+ "\u78c1\u7f57\u76d8",
639
+ "\u90ae\u888b",
640
+ "\u4fe1\u7bb1",
641
+ "\u5973\u6e38\u6cf3\u8863",
642
+ "\u6709\u80a9\u5e26\u6d74\u8863",
643
+ "\u7aa8\u4e95\u76d6",
644
+ "\u6c99\u7403\uff08\u4e00\u79cd\u6253\u51fb\u4e50\u5668\uff09",
645
+ "\u9a6c\u6797\u5df4\u6728\u7434",
646
+ "\u9762\u819c",
647
+ "\u706b\u67f4",
648
+ "\u82b1\u67f1",
649
+ "\u8ff7\u5bab",
650
+ "\u91cf\u676f",
651
+ "\u836f\u7bb1",
652
+ "\u5de8\u77f3",
653
+ "\u9ea6\u514b\u98ce",
654
+ "\u5fae\u6ce2\u7089",
655
+ "\u519b\u88c5",
656
+ "\u5976\u6876",
657
+ "\u8ff7\u4f60\u5df4\u58eb",
658
+ "\u8ff7\u4f60\u88d9",
659
+ "\u9762\u5305\u8f66\uff1b\u5c0f\u578b\u8d27\u8f66",
660
+ "\u5bfc\u5f39",
661
+ "\u8fde\u6307\u624b\u5957",
662
+ "\u6405\u62cc\u94b5",
663
+ "\u6d3b\u52a8\u623f\u5c4b\uff08\u7531\u6c7d\u8f66\u62d6\u62c9\u7684\uff09",
664
+ "\u798f\u7279T\u578b\u8f66",
665
+ "\u8c03\u5236\u89e3\u8c03\u5668\uff1b\u5149\u732b",
666
+ "\u4fee\u9053\u9662",
667
+ "\u663e\u793a\u5668",
668
+ "\u7535\u74f6\u8f66",
669
+ "\u7802\u6d46",
670
+ "\u5b66\u58eb",
671
+ "\u6e05\u771f\u5bfa",
672
+ "\u868a\u5e10",
673
+ "\u6469\u6258\u8f66",
674
+ "\u5c71\u5730\u81ea\u884c\u8f66",
675
+ "\u767b\u5c71\u5e10",
676
+ "\u9f20\u6807",
677
+ "\u6355\u9f20\u5668",
678
+ "\u642c\u5bb6\u8d27\u8f66",
679
+ "\u52a8\u7269\u7684\u53e3\u5957",
680
+ "\u91d1\u5c5e\u9489\u5b50",
681
+ "\u9888\u6258",
682
+ "\u9879\u94fe",
683
+ "\u4e73\u5934\uff08\u74f6\uff09",
684
+ "\u5e73\u677f\u7535\u8111",
685
+ "\u65b9\u5c16\u7891",
686
+ "\u53cc\u7c27\u7ba1",
687
+ "\u5c0f\u9e45\u7b1b\uff1b\u7403\u5f62\u7b1b(\u7ba1\u8eab\u692d\u5706\u5f62)",
688
+ "\u91cc\u7a0b\u8868",
689
+ "\u6ee4\u6cb9\u5668",
690
+ "\u98ce\u7434",
691
+ "\u793a\u6ce2\u5668",
692
+ "\u7f69\u88d9",
693
+ "\u725b\u8f66",
694
+ "\u6c27\u6c14\u9762\u7f69",
695
+ "\u5305\u88c5",
696
+ "\u8239\u6868",
697
+ "\u660e\u8f6e",
698
+ "\u6302\u9501",
699
+ "\u753b\u7b14",
700
+ "\u7761\u8863",
701
+ "\u5bab\u6bbf",
702
+ "\u6392\u7bab",
703
+ "\u7eb8\u5dfe",
704
+ "\u964d\u843d\u4f1e",
705
+ "\u53cc\u6760",
706
+ "\u516c\u56ed\u957f\u6905",
707
+ "\u505c\u8f66\u6536\u8d39\u8868",
708
+ "\u5ba2\u8f66",
709
+ "\u9732\u53f0",
710
+ "\u4ed8\u8d39\u7535\u8bdd",
711
+ "\u57fa\u5ea7",
712
+ "\u94c5\u7b14\u76d2",
713
+ "\u5377\u7b14\u5200",
714
+ "\u9999\u6c34\uff08\u74f6\uff09",
715
+ "\u57f9\u517b\u76bf",
716
+ "\u590d\u5370\u673a",
717
+ "\u62e8\u5f26\u7247",
718
+ "\u5c16\u9876\u5934\u76d4",
719
+ "\u7528\u5c16\u677f\u6761\u8fde\u6210\u7684\u5c16\u6869\u7bf1\u6805",
720
+ "\u76ae\u5361",
721
+ "\u6865\u58a9",
722
+ "\u5b58\u94b1\u7f50",
723
+ "\u836f\u74f6",
724
+ "\u6795\u5934",
725
+ "\u4e52\u4e53\u7403",
726
+ "\u98ce\u8f66",
727
+ "\u6d77\u76d7\u8239",
728
+ "\u6c34\u7f50",
729
+ "\u6728\u5de5\u5228",
730
+ "\u5929\u6587\u9986",
731
+ "\u5851\u6599\u888b",
732
+ "\u677f\u67b6",
733
+ "\u7281\u578b\u94f2\u96ea\u673a",
734
+ "\u624b\u538b\u76ae\u7897\u6cf5",
735
+ "\u5b9d\u4e3d\u6765\u76f8\u673a",
736
+ "\u7535\u7ebf\u6746",
737
+ "\u8b66\u8f66",
738
+ "\u96e8\u62ab",
739
+ "\u53f0\u7403\u684c",
740
+ "\u5145\u6c14\u996e\u6599\u74f6",
741
+ "\u82b1\u76c6",
742
+ "\u9676\u5de5\u65cb\u76d8",
743
+ "\u7535\u94bb",
744
+ "\u7948\u7977\u57ab",
745
+ "\u6253\u5370\u673a",
746
+ "\u76d1\u72f1",
747
+ "\u70ae\u5f39",
748
+ "\u6295\u5f71\u4eea",
749
+ "\u51b0\u7403",
750
+ "\u6c99\u5305",
751
+ "\u5c0f\u94b1\u888b\uff1b\u624b\u888b",
752
+ "\u7fbd\u7ba1\u7b14",
753
+ "\u88ab\u5b50",
754
+ "\u8d5b\u8f66",
755
+ "\u7403\u62cd",
756
+ "\u6563\u70ed\u5668",
757
+ "\u6536\u97f3\u673a",
758
+ "\u5c04\u7535\u671b\u8fdc\u955c",
759
+ "\u96e8\u6876",
760
+ "\u4f11\u95f2\u8f66",
761
+ "\u5377\u8f74",
762
+ "\u53cd\u5c04\u5f0f\u7167\u76f8\u673a",
763
+ "\u51b0\u7bb1",
764
+ "\u9065\u63a7\u5668",
765
+ "\u9910\u5385",
766
+ "\u5de6\u8f6e\u624b\u67aa",
767
+ "\u6b65\u67aa",
768
+ "\u6447\u6905",
769
+ "\u7535\u8f6c\u70e4\u8089\u67b6",
770
+ "\u6a61\u76ae",
771
+ "\u6a44\u6984\u7403",
772
+ "\u76f4\u5c3a",
773
+ "\u8dd1\u6b65\u978b",
774
+ "\u4fdd\u9669\u67dc",
775
+ "\u5b89\u5168\u522b\u9488",
776
+ "\u76d0\u74f6\uff08\u8c03\u5473\u7528\uff09",
777
+ "\u51c9\u978b",
778
+ "\u7eb1\u7b3c",
779
+ "\u8428\u514b\u65af\u7ba1",
780
+ "\u5251\u9798",
781
+ "\u79e4",
782
+ "\u6821\u8f66",
783
+ "\u5e06\u8239",
784
+ "\u8bb0\u5206\u724c",
785
+ "\u5c4f\u5e55",
786
+ "\u87ba\u4e1d",
787
+ "\u87ba\u4e1d\u5200",
788
+ "\u5b89\u5168\u5e26",
789
+ "\u7f1d\u7eab\u673a",
790
+ "\u76fe\u724c",
791
+ "\u76ae\u978b\u5e97",
792
+ "\u969c\u5b50",
793
+ "\u8d2d\u7269\u7bee",
794
+ "\u8d2d\u7269\u8f66",
795
+ "\u94c1\u9539",
796
+ "\u6d74\u5e3d",
797
+ "\u6d74\u5e18",
798
+ "\u6ed1\u96ea\u677f",
799
+ "\u6ed1\u96ea\u9762\u7f69",
800
+ "\u7761\u888b",
801
+ "\u6ed1\u5c3a",
802
+ "\u6ed1\u52a8\u95e8",
803
+ "\u89d2\u5b50\u8001\u864e\u673a",
804
+ "\u6f5c\u6c34\u901a\u6c14\u7ba1",
805
+ "\u6469\u6258\u96ea\u6a47\uff1b\u96ea\u5730\u673a\u52a8\u8f66",
806
+ "\u626b\u96ea\u673a",
807
+ "\u7682\u6db2\u5668",
808
+ "\u8db3\u7403",
809
+ "\u889c\u5b50",
810
+ "\u789f\u5f0f\u592a\u9633\u80fd",
811
+ "\u5bbd\u8fb9\u5e3d",
812
+ "\u6c64\u7897",
813
+ "\u7a7a\u683c\u952e",
814
+ "\u7a7a\u95f4\u52a0\u70ed\u5668",
815
+ "\u822a\u5929\u98de\u673a",
816
+ "\u9505\u94f2\uff1b\u505a\u996d\u7684\u94f2\u5b50",
817
+ "\u5feb\u8247",
818
+ "\u8718\u86db\u7f51",
819
+ "\u7eba\u9524\uff1b\u624b\u7eba\u7528\u7684\u7ed5\u7ebf\u6746",
820
+ "\u8dd1\u8f66",
821
+ "\u805a\u5149\u706f",
822
+ "\u821e\u53f0",
823
+ "\u84b8\u6c7d\u673a\u8f66",
824
+ "\u94a2\u62f1\u6865",
825
+ "\u94a2\u6eda\u7b52",
826
+ "\u542c\u8bca\u5668",
827
+ "\u5973\u7528\u62ab\u80a9",
828
+ "\u77f3\u5934\u5899",
829
+ "\u79d2\u8868",
830
+ "\u706b\u7089",
831
+ "\u8fc7\u6ee4\u5668",
832
+ "\u6709\u8f68\u7535\u8f66",
833
+ "\u62c5\u67b6",
834
+ "\u6c99\u53d1\u5e8a",
835
+ "\u4f5b\u5854",
836
+ "\u6f5c\u8247",
837
+ "\u5957\u88c5",
838
+ "\u65e5\u6677",
839
+ "\u592a\u9633\u955c",
840
+ "\u592a\u9633\u955c",
841
+ "\u9632\u6652\u971c",
842
+ "\u60ac\u7d22\u6865",
843
+ "\u62d6\u628a",
844
+ "\u8fd0\u52a8\u886b",
845
+ "\u6e38\u6cf3\u88e4",
846
+ "\u79cb\u5343",
847
+ "\u5f00\u5173",
848
+ "\u6ce8\u5c04\u5668\uff1b\u5438\u7ba1",
849
+ "\u53f0\u706f",
850
+ "\u5766\u514b",
851
+ "\u5f55\u97f3\u673a",
852
+ "\u8336\u58f6",
853
+ "\u6cf0\u8fea",
854
+ "\u7535\u89c6",
855
+ "\u7f51\u7403\uff1b\u6253\u7f51\u7403\u7684\u7403",
856
+ "\u8305\u8349",
857
+ "\u5e55\u5e03",
858
+ "\u9876\u9488",
859
+ "\u6253\u8c37\u673a\uff1b\u8131\u7c92\u673a",
860
+ "\u5b9d\u5ea7",
861
+ "\u74e6\u5c4b\u9876",
862
+ "\u70e4\u9762\u5305\u673a",
863
+ "\u70df\u8349\u5e97",
864
+ "\u9a6c\u6876",
865
+ "\u706b\u70ac",
866
+ "\u56fe\u817e\u67f1",
867
+ "\u62d6\u8f66\uff1b\u7275\u5f15\u8f66",
868
+ "\u73a9\u5177\u5e97",
869
+ "\u62d6\u62c9\u673a",
870
+ "\u534a\u6302\u6c7d\u8f66",
871
+ "\u6258\u76d8",
872
+ "\u98ce\u8863",
873
+ "\u4e09\u8f6e\u8f66",
874
+ "\u4e09\u4f53\u8239",
875
+ "\u4e09\u811a\u67b6",
876
+ "\u51ef\u65cb\u95e8",
877
+ "\u65e0\u8f68\u7535\u8f66",
878
+ "\u957f\u53f7",
879
+ "\u6d74\u76c6",
880
+ "\u65cb\u8f6c\u5f0f\u6805\u95e8",
881
+ "\u6253\u5b57\u673a\u952e\u76d8",
882
+ "\u4f1e",
883
+ "\u72ec\u8f6e\u8f66",
884
+ "\u76f4\u7acb\u5f0f\u94a2\u7434",
885
+ "\u5438\u5c18\u5668",
886
+ "\u82b1\u74f6\uff1b\u88c5\u9970\u74f6",
887
+ "\u62f1\u9876",
888
+ "\u5929\u9e45\u7ed2",
889
+ "\u81ea\u52a8\u552e\u8d27\u673a",
890
+ "\u6cd5\u8863\uff1b\u796d\u8863\uff1b\u796d\u670d",
891
+ "\u9ad8\u67b6\u6865",
892
+ "\u5c0f\u63d0\u7434",
893
+ "\u6392\u7403",
894
+ "\u677e\u997c\u673a",
895
+ "\u6302\u949f",
896
+ "\u94b1\u5305\uff1b\u94b1\u5939",
897
+ "\u8863\u67dc\u8863\u6a71",
898
+ "\u519b\u7528\u98de\u673a",
899
+ "\u6d17\u8138\u76c6",
900
+ "\u6d17\u8863\u673a",
901
+ "\u6c34\u74f6",
902
+ "\u6c34\u58f6",
903
+ "\u6c34\u5854",
904
+ "\u5a01\u58eb\u5fcc\u58f6",
905
+ "\u54e8\u5b50",
906
+ "\u5047\u53d1",
907
+ "\u7eb1\u7a97",
908
+ "\u767e\u53f6\u7a97",
909
+ "\u6e29\u838e\u9886\u5e26",
910
+ "\u8461\u8404\u9152\u74f6",
911
+ "\u98de\u673a\u7fc5\u8180",
912
+ "\u7092\u83dc\u9505",
913
+ "\u6728\u52fa\u5b50\uff1b\u6728\u5934\u52fa\u5b50",
914
+ "\u6bdb\u7ec7\u54c1",
915
+ "\u539f\u6728\u6805\u680f",
916
+ "\u6c89\u8239",
917
+ "\u53cc\u6845\u8239",
918
+ "\u8499\u53e4\u5305",
919
+ "\u7f51\u7ad9\uff1b\u7f51\u9875",
920
+ "\u6f2b\u753b",
921
+ "\u7eb5\u6a2a\u5b57\u8c1c",
922
+ "\u8def\u6807",
923
+ "\u4ea4\u901a\u4fe1\u53f7\u706f",
924
+ "\u9632\u5c18\u7f69",
925
+ "\u83dc\u5355",
926
+ "\u76d8\u5b50",
927
+ "\u58a8\u897f\u54e5\u9cc4\u68a8\u9171\uff1b\u58a8\u897f\u54e5\u725b\u6cb9\u679c\u9171",
928
+ "\u6e05\u7096\u8089\u6c64",
929
+ "\u706b\u9505",
930
+ "\u4e73\u8102\u86cb\u7cd5\uff1b\u82f1\u56fd\u751c\u70b9",
931
+ "\u51b0\u6dc7\u6dcb",
932
+ "\u51b0\u68cd\uff1b\u96ea\u7cd5",
933
+ "\u6cd5\u5f0f\u9762\u5305",
934
+ "\u767e\u5409\u997c",
935
+ "\u6912\u76d0\u8106\u997c",
936
+ "\u829d\u58eb\u6c49\u5821",
937
+ "\u70ed\u72d7",
938
+ "\u571f\u8c46\u6ce5",
939
+ "\u7ed3\u7403\u7518\u84dd",
940
+ "\u897f\u5170\u82b1\uff1b\u7eff\u83dc\u82b1",
941
+ "\u83dc\u82b1\uff1b\u82b1\u6930\u83dc",
942
+ "\u897f\u846b\u82a6",
943
+ "\u91d1\u4e1d\u74dc\uff1b\u610f\u9762\u5357\u74dc\uff1b\u9762\u6761\u74dc",
944
+ "\u7eff\u8272\u5c0f\u5357\u74dc\uff1b\u9752\u5357\u74dc",
945
+ "\u5357\u74dc",
946
+ "\u9ec4\u74dc",
947
+ "\u6d0b\u84df\uff1b\u7403\u84df",
948
+ "\u751c\u6912",
949
+ "\u523a\u68d8\u84df",
950
+ "\u8611\u83c7",
951
+ "\u7eff\u82f9\u679c",
952
+ "\u8349\u8393",
953
+ "\u6a58\u5b50",
954
+ "\u67e0\u6aac",
955
+ "\u65e0\u82b1\u679c",
956
+ "\u83e0\u841d",
957
+ "\u9999\u8549",
958
+ "\u83e0\u841d\u871c",
959
+ "\u756a\u8354\u679d",
960
+ "\u77f3\u69b4",
961
+ "\u5e72\u8349",
962
+ "\u57f9\u6839\u86cb\u9171\u610f\u5927\u5229\u9762",
963
+ "\u5de7\u514b\u529b\u9171",
964
+ "\u751f\u9762\uff1b\u9762\u56e2",
965
+ "\u745e\u58eb\u8089\u5305",
966
+ "\u62ab\u8428",
967
+ "\u9985\u997c",
968
+ "\u5377\u997c",
969
+ "\u7ea2\u8461\u8404\u9152",
970
+ "\u610f\u5f0f\u6d53\u7f29\u5496\u5561",
971
+ "\u676f\u5b50",
972
+ "\u86cb\u9152",
973
+ "\u9ad8\u5c71",
974
+ "\u6ce1\u6ce1",
975
+ "\u60ac\u5d16",
976
+ "\u73ca\u745a\u7901",
977
+ "\u95f4\u6b47\u6cc9\uff1b\u95f4\u65ad\u55b7\u53d1\u7684\u6e29\u6cc9",
978
+ "\u6e56\u8fb9",
979
+ "\u5cac\u89d2\uff1b\u6df1\u5165\u6d77\u4e2d\u7684\u72ed\u957f\u9ad8\u5730",
980
+ "\u6c99\u6d32",
981
+ "\u6c99\u6ee9",
982
+ "\u5ce1\u8c37",
983
+ "\u706b\u5c71",
984
+ "\u68d2\u7403\u8fd0\u52a8\u5458",
985
+ "\u65b0\u90ce",
986
+ "\u6f5c\u6c34\u5458",
987
+ "\u6cb9\u83dc",
988
+ "\u96cf\u83ca",
989
+ "\u9ec4\u8272\u6753\u5170",
990
+ "\u7389\u7c73",
991
+ "\u6a61\u5b50",
992
+ "\u73ab\u7470\u679c",
993
+ "\u4e03\u53f6\u6811\u679c\u5b9e",
994
+ "\u73ca\u745a\u83cc",
995
+ "\u6728\u8033",
996
+ "\u9e7f\u82b1\u83cc",
997
+ "\u81ed\u89d2\u83c7",
998
+ "\u5730\u661f",
999
+ "\u591a\u53f6\u5947\u679c\u83cc",
1000
+ "\u725b\u809d\u83cc",
1001
+ "\u7389\u7c73\u68d2\u5b50",
1002
+ "\u536b\u751f\u7eb8"
1003
+ ]
1004
+ }
CLIP_benchmark/clip_benchmark/datasets/cn_zeroshot_classification_templates.json ADDED
@@ -0,0 +1,84 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "imagenet1k": [
3
+ "{c}\u7684\u7167\u7247\u3002",
4
+ "\u8d28\u91cf\u5dee\u7684{c}\u7684\u7167\u7247\u3002",
5
+ "\u8bb8\u591a{c}\u7684\u7167\u7247\u3002",
6
+ "{c}\u7684\u96d5\u5851\u3002",
7
+ "\u96be\u4ee5\u770b\u5230{c}\u7684\u7167\u7247\u3002",
8
+ "{c}\u7684\u4f4e\u5206\u8fa8\u7387\u7167\u7247\u3002",
9
+ "{c}\u7684\u6e32\u67d3\u3002",
10
+ "\u6d82\u9e26{c}\u3002",
11
+ "{c}\u7684\u7cdf\u7cd5\u7167\u7247\u3002",
12
+ "{c}\u7684\u88c1\u526a\u7167\u7247\u3002",
13
+ "{c}\u7684\u7eb9\u8eab\u3002",
14
+ "{c}\u7684\u523a\u7ee3\u7167\u7247\u3002",
15
+ "\u5f88\u96be\u770b\u5230{c}\u7684\u7167\u7247\u3002",
16
+ "{c}\u7684\u660e\u4eae\u7167\u7247\u3002",
17
+ "\u4e00\u5f20\u5e72\u51c0\u7684{c}\u7684\u7167\u7247\u3002",
18
+ "\u4e00\u5f20\u5305\u542b{c}\u7684\u7167\u7247\u3002",
19
+ "{c}\u7684\u6df1\u8272\u7167\u7247\u3002",
20
+ "{c}\u7684\u624b\u7ed8\u753b\u3002",
21
+ "\u6211\u7684{c}\u7684\u7167\u7247\u3002",
22
+ "\u4e0d\u81ea\u7136\u7684{c}\u7684\u7167\u7247\u3002",
23
+ "\u4e00\u5f20\u9177\u7684{c}\u7684\u7167\u7247\u3002",
24
+ "{c}\u7684\u7279\u5199\u7167\u7247\u3002",
25
+ "{c}\u7684\u9ed1\u767d\u7167\u7247\u3002",
26
+ "\u4e00\u5e45{c}\u7684\u753b\u3002",
27
+ "\u4e00\u5e45{c}\u7684\u7ed8\u753b\u3002",
28
+ "\u4e00\u5f20{c}\u7684\u50cf\u7d20\u7167\u7247\u3002",
29
+ "{c}\u7684\u96d5\u50cf\u3002",
30
+ "\u4e00\u5f20{c}\u7684\u660e\u4eae\u7167\u7247\u3002",
31
+ "{c}\u7684\u88c1\u526a\u7167\u7247\u3002",
32
+ "\u4eba\u9020\u7684{c}\u7684\u7167\u7247\u3002",
33
+ "\u4e00\u5f20\u5173\u4e8e{c}\u7684\u7167\u7247\u3002",
34
+ "\u635f\u574f\u7684{c}\u7684jpeg\u7167\u7247\u3002",
35
+ "{c}\u7684\u6a21\u7cca\u7167\u7247\u3002",
36
+ "{c}\u7684\u76f8\u7247\u3002",
37
+ "\u4e00\u5f20{c}\u7684\u597d\u7167\u7247\u3002",
38
+ "{c}\u7684\u6e32\u67d3\u7167\u3002",
39
+ "\u89c6\u9891\u6e38\u620f\u4e2d\u7684{c}\u3002",
40
+ "\u4e00\u5f20{c}\u7684\u7167\u7247\u3002",
41
+ "{c}\u7684\u6d82\u9e26\u3002",
42
+ "{c}\u7684\u8fd1\u8ddd\u79bb\u7167\u7247\u3002",
43
+ "{c}\u7684\u6298\u7eb8\u3002",
44
+ "{c}\u5728\u89c6\u9891\u6e38\u620f\u4e2d\u3002",
45
+ "{c}\u7684\u8349\u56fe\u3002",
46
+ "{c}\u7684\u6d82\u9e26\u7167\u3002",
47
+ "{c}\u7684\u6298\u7eb8\u5f62\u72b6\u3002",
48
+ "\u4f4e\u5206\u8fa8\u7387\u7684{c}\u7684\u7167\u7247\u3002",
49
+ "\u73a9\u5177{c}\u3002",
50
+ "{c}\u7684\u526f\u672c\u3002",
51
+ "{c}\u7684\u5e72\u51c0\u7684\u7167\u7247\u3002",
52
+ "\u4e00\u5f20\u5927{c}\u7684\u7167\u7247\u3002",
53
+ "{c}\u7684\u91cd\u73b0\u3002",
54
+ "\u4e00\u5f20\u6f02\u4eae\u7684{c}\u7684\u7167\u7247\u3002",
55
+ "\u4e00\u5f20\u5947\u602a\u7684{c}\u7684\u7167\u7247\u3002",
56
+ "\u6a21\u7cca\u7684{c}\u7684\u7167\u7247\u3002",
57
+ "\u5361\u901a{c}\u3002",
58
+ "{c}\u7684\u827a\u672f\u4f5c\u54c1\u3002",
59
+ "{c}\u7684\u7d20\u63cf\u3002",
60
+ "\u523a\u7ee3{c}\u3002",
61
+ "{c}\u7684\u50cf\u7d20\u7167\u3002",
62
+ "{c}\u7684\u62cd\u7167\u3002",
63
+ "{c}\u7684\u635f\u574f\u7684\u7167\u7247\u3002",
64
+ "\u9ad8\u8d28\u91cf\u7684{c}\u7684\u7167\u7247\u3002",
65
+ "\u6bdb\u7ed2\u73a9\u5177{c}\u3002",
66
+ "\u6f02\u4eae\u7684{c}\u7684\u7167\u7247\u3002",
67
+ "\u5c0f{c}\u7684\u7167\u7247\u3002",
68
+ "\u7167\u7247\u662f\u5947\u602a\u7684{c}\u3002",
69
+ "\u6f2b\u753b{c}\u3002",
70
+ "{c}\u7684\u827a\u672f\u7167\u3002",
71
+ "{c}\u7684\u56fe\u5f62\u3002",
72
+ "\u5927{c}\u7684\u7167\u7247\u3002",
73
+ "\u9ed1\u767d\u7684{c}\u7684\u7167\u7247\u3002",
74
+ "{c}\u6bdb\u7ed2\u73a9\u5177\u3002",
75
+ "\u4e00\u5f20{c}\u7684\u6df1\u8272\u7167\u7247\u3002",
76
+ "{c}\u7684\u6444\u5f71\u56fe\u3002",
77
+ "{c}\u7684\u6d82\u9e26\u7167\u3002",
78
+ "\u73a9\u5177\u5f62\u72b6\u7684{c}\u3002",
79
+ "\u62cd\u4e86{c}\u7684\u7167\u7247\u3002",
80
+ "\u9177\u9177\u7684{c}\u7684\u7167\u7247\u3002",
81
+ "\u7167\u7247\u91cc\u7684\u5c0f{c}\u3002",
82
+ "{c}\u7684\u523a\u9752\u3002"
83
+ ]
84
+ }
CLIP_benchmark/clip_benchmark/datasets/cupl_prompts.json ADDED
The diff for this file is too large to render. See raw diff
 
CLIP_benchmark/clip_benchmark/datasets/en_classnames.json ADDED
@@ -0,0 +1,1701 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "flowers": [
3
+ "pink primrose",
4
+ "hard-leaved pocket orchid",
5
+ "canterbury bells",
6
+ "sweet pea",
7
+ "english marigold",
8
+ "tiger lily",
9
+ "moon orchid",
10
+ "bird of paradise",
11
+ "monkshood",
12
+ "globe thistle",
13
+ "snapdragon",
14
+ "colt's foot",
15
+ "king protea",
16
+ "spear thistle",
17
+ "yellow iris",
18
+ "globe flower",
19
+ "purple coneflower",
20
+ "peruvian lily",
21
+ "balloon flower",
22
+ "giant white arum lily",
23
+ "fire lily",
24
+ "pincushion flower",
25
+ "fritillary",
26
+ "red ginger",
27
+ "grape hyacinth",
28
+ "corn poppy",
29
+ "prince of wales feathers",
30
+ "stemless gentian",
31
+ "artichoke",
32
+ "sweet william",
33
+ "carnation",
34
+ "garden phlox",
35
+ "love in the mist",
36
+ "mexican aster",
37
+ "alpine sea holly",
38
+ "ruby-lipped cattleya",
39
+ "cape flower",
40
+ "great masterwort",
41
+ "siam tulip",
42
+ "lenten rose",
43
+ "barbeton daisy",
44
+ "daffodil",
45
+ "sword lily",
46
+ "poinsettia",
47
+ "bolero deep blue",
48
+ "wallflower",
49
+ "marigold",
50
+ "buttercup",
51
+ "oxeye daisy",
52
+ "common dandelion",
53
+ "petunia",
54
+ "wild pansy",
55
+ "primula",
56
+ "sunflower",
57
+ "pelargonium",
58
+ "bishop of llandaff",
59
+ "gaura",
60
+ "geranium",
61
+ "orange dahlia",
62
+ "pink and yellow dahlia",
63
+ "cautleya spicata",
64
+ "japanese anemone",
65
+ "black-eyed susan",
66
+ "silverbush",
67
+ "californian poppy",
68
+ "osteospermum",
69
+ "spring crocus",
70
+ "bearded iris",
71
+ "windflower",
72
+ "tree poppy",
73
+ "gazania",
74
+ "azalea",
75
+ "water lily",
76
+ "rose",
77
+ "thorn apple",
78
+ "morning glory",
79
+ "passion flower",
80
+ "lotus",
81
+ "toad lily",
82
+ "anthurium",
83
+ "frangipani",
84
+ "clematis",
85
+ "hibiscus",
86
+ "columbine",
87
+ "desert-rose",
88
+ "tree mallow",
89
+ "magnolia",
90
+ "cyclamen",
91
+ "watercress",
92
+ "canna lily",
93
+ "hippeastrum",
94
+ "bee balm",
95
+ "air plant",
96
+ "foxglove",
97
+ "bougainvillea",
98
+ "camellia",
99
+ "mallow",
100
+ "mexican petunia",
101
+ "bromelia",
102
+ "blanket flower",
103
+ "trumpet creeper",
104
+ "blackberry lily"
105
+ ],
106
+ "gtsrb": [
107
+ "red and white circle 20 kph speed limit",
108
+ "red and white circle 30 kph speed limit",
109
+ "red and white circle 50 kph speed limit",
110
+ "red and white circle 60 kph speed limit",
111
+ "red and white circle 70 kph speed limit",
112
+ "red and white circle 80 kph speed limit",
113
+ "end / de-restriction of 80 kph speed limit",
114
+ "red and white circle 100 kph speed limit",
115
+ "red and white circle 120 kph speed limit",
116
+ "red and white circle red car and black car no passing",
117
+ "red and white circle red truck and black car no passing",
118
+ "red and white triangle road intersection warning",
119
+ "white and yellow diamond priority road",
120
+ "red and white upside down triangle yield right-of-way",
121
+ "stop",
122
+ "empty red and white circle",
123
+ "red and white circle no truck entry",
124
+ "red circle with white horizonal stripe no entry",
125
+ "red and white triangle with exclamation mark warning",
126
+ "red and white triangle with black left curve approaching warning",
127
+ "red and white triangle with black right curve approaching warning",
128
+ "red and white triangle with black double curve approaching warning",
129
+ "red and white triangle rough / bumpy road warning",
130
+ "red and white triangle car skidding / slipping warning",
131
+ "red and white triangle with merging / narrow lanes warning",
132
+ "red and white triangle with person digging / construction / road work warning",
133
+ "red and white triangle with traffic light approaching warning",
134
+ "red and white triangle with person walking warning",
135
+ "red and white triangle with child and person walking warning",
136
+ "red and white triangle with bicyle warning",
137
+ "red and white triangle with snowflake / ice warning",
138
+ "red and white triangle with deer warning",
139
+ "white circle with gray strike bar no speed limit",
140
+ "blue circle with white right turn arrow mandatory",
141
+ "blue circle with white left turn arrow mandatory",
142
+ "blue circle with white forward arrow mandatory",
143
+ "blue circle with white forward or right turn arrow mandatory",
144
+ "blue circle with white forward or left turn arrow mandatory",
145
+ "blue circle with white keep right arrow mandatory",
146
+ "blue circle with white keep left arrow mandatory",
147
+ "blue circle with white arrows indicating a traffic circle",
148
+ "white circle with gray strike bar indicating no passing for cars has ended",
149
+ "white circle with gray strike bar indicating no passing for trucks has ended"
150
+ ],
151
+ "country211": [
152
+ "Andorra",
153
+ "United Arab Emirates",
154
+ "Afghanistan",
155
+ "Antigua and Barbuda",
156
+ "Anguilla",
157
+ "Albania",
158
+ "Armenia",
159
+ "Angola",
160
+ "Antarctica",
161
+ "Argentina",
162
+ "Austria",
163
+ "Australia",
164
+ "Aruba",
165
+ "Aland Islands",
166
+ "Azerbaijan",
167
+ "Bosnia and Herzegovina",
168
+ "Barbados",
169
+ "Bangladesh",
170
+ "Belgium",
171
+ "Burkina Faso",
172
+ "Bulgaria",
173
+ "Bahrain",
174
+ "Benin",
175
+ "Bermuda",
176
+ "Brunei Darussalam",
177
+ "Bolivia",
178
+ "Bonaire, Saint Eustatius and Saba",
179
+ "Brazil",
180
+ "Bahamas",
181
+ "Bhutan",
182
+ "Botswana",
183
+ "Belarus",
184
+ "Belize",
185
+ "Canada",
186
+ "DR Congo",
187
+ "Central African Republic",
188
+ "Switzerland",
189
+ "Cote d'Ivoire",
190
+ "Cook Islands",
191
+ "Chile",
192
+ "Cameroon",
193
+ "China",
194
+ "Colombia",
195
+ "Costa Rica",
196
+ "Cuba",
197
+ "Cabo Verde",
198
+ "Curacao",
199
+ "Cyprus",
200
+ "Czech Republic",
201
+ "Germany",
202
+ "Denmark",
203
+ "Dominica",
204
+ "Dominican Republic",
205
+ "Algeria",
206
+ "Ecuador",
207
+ "Estonia",
208
+ "Egypt",
209
+ "Spain",
210
+ "Ethiopia",
211
+ "Finland",
212
+ "Fiji",
213
+ "Falkland Islands",
214
+ "Faeroe Islands",
215
+ "France",
216
+ "Gabon",
217
+ "United Kingdom",
218
+ "Grenada",
219
+ "Georgia",
220
+ "French Guiana",
221
+ "Guernsey",
222
+ "Ghana",
223
+ "Gibraltar",
224
+ "Greenland",
225
+ "Gambia",
226
+ "Guadeloupe",
227
+ "Greece",
228
+ "South Georgia and South Sandwich Is.",
229
+ "Guatemala",
230
+ "Guam",
231
+ "Guyana",
232
+ "Hong Kong",
233
+ "Honduras",
234
+ "Croatia",
235
+ "Haiti",
236
+ "Hungary",
237
+ "Indonesia",
238
+ "Ireland",
239
+ "Israel",
240
+ "Isle of Man",
241
+ "India",
242
+ "Iraq",
243
+ "Iran",
244
+ "Iceland",
245
+ "Italy",
246
+ "Jersey",
247
+ "Jamaica",
248
+ "Jordan",
249
+ "Japan",
250
+ "Kenya",
251
+ "Kyrgyz Republic",
252
+ "Cambodia",
253
+ "St. Kitts and Nevis",
254
+ "North Korea",
255
+ "South Korea",
256
+ "Kuwait",
257
+ "Cayman Islands",
258
+ "Kazakhstan",
259
+ "Laos",
260
+ "Lebanon",
261
+ "St. Lucia",
262
+ "Liechtenstein",
263
+ "Sri Lanka",
264
+ "Liberia",
265
+ "Lithuania",
266
+ "Luxembourg",
267
+ "Latvia",
268
+ "Libya",
269
+ "Morocco",
270
+ "Monaco",
271
+ "Moldova",
272
+ "Montenegro",
273
+ "Saint-Martin",
274
+ "Madagascar",
275
+ "Macedonia",
276
+ "Mali",
277
+ "Myanmar",
278
+ "Mongolia",
279
+ "Macau",
280
+ "Martinique",
281
+ "Mauritania",
282
+ "Malta",
283
+ "Mauritius",
284
+ "Maldives",
285
+ "Malawi",
286
+ "Mexico",
287
+ "Malaysia",
288
+ "Mozambique",
289
+ "Namibia",
290
+ "New Caledonia",
291
+ "Nigeria",
292
+ "Nicaragua",
293
+ "Netherlands",
294
+ "Norway",
295
+ "Nepal",
296
+ "New Zealand",
297
+ "Oman",
298
+ "Panama",
299
+ "Peru",
300
+ "French Polynesia",
301
+ "Papua New Guinea",
302
+ "Philippines",
303
+ "Pakistan",
304
+ "Poland",
305
+ "Puerto Rico",
306
+ "Palestine",
307
+ "Portugal",
308
+ "Palau",
309
+ "Paraguay",
310
+ "Qatar",
311
+ "Reunion",
312
+ "Romania",
313
+ "Serbia",
314
+ "Russia",
315
+ "Rwanda",
316
+ "Saudi Arabia",
317
+ "Solomon Islands",
318
+ "Seychelles",
319
+ "Sudan",
320
+ "Sweden",
321
+ "Singapore",
322
+ "St. Helena",
323
+ "Slovenia",
324
+ "Svalbard and Jan Mayen Islands",
325
+ "Slovakia",
326
+ "Sierra Leone",
327
+ "San Marino",
328
+ "Senegal",
329
+ "Somalia",
330
+ "South Sudan",
331
+ "El Salvador",
332
+ "Sint Maarten",
333
+ "Syria",
334
+ "Eswatini",
335
+ "Togo",
336
+ "Thailand",
337
+ "Tajikistan",
338
+ "Timor-Leste",
339
+ "Turkmenistan",
340
+ "Tunisia",
341
+ "Tonga",
342
+ "Turkey",
343
+ "Trinidad and Tobago",
344
+ "Taiwan",
345
+ "Tanzania",
346
+ "Ukraine",
347
+ "Uganda",
348
+ "United States",
349
+ "Uruguay",
350
+ "Uzbekistan",
351
+ "Vatican",
352
+ "Venezuela",
353
+ "British Virgin Islands",
354
+ "United States Virgin Islands",
355
+ "Vietnam",
356
+ "Vanuatu",
357
+ "Samoa",
358
+ "Kosovo",
359
+ "Yemen",
360
+ "South Africa",
361
+ "Zambia",
362
+ "Zimbabwe"
363
+ ],
364
+ "eurosat": [
365
+ "annual crop land",
366
+ "forest",
367
+ "brushland or shrubland",
368
+ "highway or road",
369
+ "industrial buildings or commercial buildings",
370
+ "pasture land",
371
+ "permanent crop land",
372
+ "residential buildings or homes or apartments",
373
+ "river",
374
+ "lake or sea"
375
+ ],
376
+ "fer2013": [
377
+ "angry",
378
+ "disgusted",
379
+ "fearful",
380
+ "happy",
381
+ "neutral",
382
+ "sad",
383
+ "surprised"
384
+ ],
385
+ "caltech101": [
386
+ "background",
387
+ "off-center face",
388
+ "centered face",
389
+ "leopard",
390
+ "motorbike",
391
+ "accordion",
392
+ "airplane",
393
+ "anchor",
394
+ "ant",
395
+ "barrel",
396
+ "bass",
397
+ "beaver",
398
+ "binocular",
399
+ "bonsai",
400
+ "brain",
401
+ "brontosaurus",
402
+ "buddha",
403
+ "butterfly",
404
+ "camera",
405
+ "cannon",
406
+ "side of a car",
407
+ "ceiling fan",
408
+ "cellphone",
409
+ "chair",
410
+ "chandelier",
411
+ "body of a cougar cat",
412
+ "face of a cougar cat",
413
+ "crab",
414
+ "crayfish",
415
+ "crocodile",
416
+ "head of a crocodile",
417
+ "cup",
418
+ "dalmatian",
419
+ "dollar bill",
420
+ "dolphin",
421
+ "dragonfly",
422
+ "electric guitar",
423
+ "elephant",
424
+ "emu",
425
+ "euphonium",
426
+ "ewer",
427
+ "ferry",
428
+ "flamingo",
429
+ "head of a flamingo",
430
+ "garfield",
431
+ "gerenuk",
432
+ "gramophone",
433
+ "grand piano",
434
+ "hawksbill",
435
+ "headphone",
436
+ "hedgehog",
437
+ "helicopter",
438
+ "ibis",
439
+ "inline skate",
440
+ "joshua tree",
441
+ "kangaroo",
442
+ "ketch",
443
+ "lamp",
444
+ "laptop",
445
+ "llama",
446
+ "lobster",
447
+ "lotus",
448
+ "mandolin",
449
+ "mayfly",
450
+ "menorah",
451
+ "metronome",
452
+ "minaret",
453
+ "nautilus",
454
+ "octopus",
455
+ "okapi",
456
+ "pagoda",
457
+ "panda",
458
+ "pigeon",
459
+ "pizza",
460
+ "platypus",
461
+ "pyramid",
462
+ "revolver",
463
+ "rhino",
464
+ "rooster",
465
+ "saxophone",
466
+ "schooner",
467
+ "scissors",
468
+ "scorpion",
469
+ "sea horse",
470
+ "snoopy (cartoon beagle)",
471
+ "soccer ball",
472
+ "stapler",
473
+ "starfish",
474
+ "stegosaurus",
475
+ "stop sign",
476
+ "strawberry",
477
+ "sunflower",
478
+ "tick",
479
+ "trilobite",
480
+ "umbrella",
481
+ "watch",
482
+ "water lilly",
483
+ "wheelchair",
484
+ "wild cat",
485
+ "windsor chair",
486
+ "wrench",
487
+ "yin and yang symbol"
488
+ ],
489
+ "caltech101_vtab": [
490
+ "accordion",
491
+ "airplane",
492
+ "anchor",
493
+ "ant",
494
+ "background",
495
+ "barrel",
496
+ "bass",
497
+ "beaver",
498
+ "binocular",
499
+ "bonsai",
500
+ "brain",
501
+ "brontosaurus",
502
+ "buddha",
503
+ "butterfly",
504
+ "camera",
505
+ "cannon",
506
+ "side of a car",
507
+ "ceiling fan",
508
+ "cellphone",
509
+ "chair",
510
+ "chandelier",
511
+ "body of a cougar cat",
512
+ "face of a cougar cat",
513
+ "crab",
514
+ "crayfish",
515
+ "crocodile",
516
+ "head of a crocodile",
517
+ "cup",
518
+ "dalmatian",
519
+ "dollar bill",
520
+ "dolphin",
521
+ "dragonfly",
522
+ "electric guitar",
523
+ "elephant",
524
+ "emu",
525
+ "euphonium",
526
+ "ewer",
527
+ "off-center face",
528
+ "centered face",
529
+ "ferry",
530
+ "flamingo",
531
+ "head of a flamingo",
532
+ "garfield",
533
+ "gerenuk",
534
+ "gramophone",
535
+ "grand piano",
536
+ "hawksbill",
537
+ "headphone",
538
+ "hedgehog",
539
+ "helicopter",
540
+ "ibis",
541
+ "inline skate",
542
+ "joshua tree",
543
+ "kangaroo",
544
+ "ketch",
545
+ "lamp",
546
+ "laptop",
547
+ "leopard",
548
+ "llama",
549
+ "lobster",
550
+ "lotus",
551
+ "mandolin",
552
+ "mayfly",
553
+ "menorah",
554
+ "metronome",
555
+ "minaret",
556
+ "motorbike",
557
+ "nautilus",
558
+ "octopus",
559
+ "okapi",
560
+ "pagoda",
561
+ "panda",
562
+ "pigeon",
563
+ "pizza",
564
+ "platypus",
565
+ "pyramid",
566
+ "revolver",
567
+ "rhino",
568
+ "rooster",
569
+ "saxophone",
570
+ "schooner",
571
+ "scissors",
572
+ "scorpion",
573
+ "sea horse",
574
+ "snoopy (cartoon beagle)",
575
+ "soccer ball",
576
+ "stapler",
577
+ "starfish",
578
+ "stegosaurus",
579
+ "stop sign",
580
+ "strawberry",
581
+ "sunflower",
582
+ "tick",
583
+ "trilobite",
584
+ "umbrella",
585
+ "watch",
586
+ "water lilly",
587
+ "wheelchair",
588
+ "wild cat",
589
+ "windsor chair",
590
+ "wrench",
591
+ "yin and yang symbol"
592
+ ],
593
+ "imagenet1k": [
594
+ "tench",
595
+ "goldfish",
596
+ "great white shark",
597
+ "tiger shark",
598
+ "hammerhead shark",
599
+ "electric ray",
600
+ "stingray",
601
+ "rooster",
602
+ "hen",
603
+ "ostrich",
604
+ "brambling",
605
+ "goldfinch",
606
+ "house finch",
607
+ "junco",
608
+ "indigo bunting",
609
+ "American robin",
610
+ "bulbul",
611
+ "jay",
612
+ "magpie",
613
+ "chickadee",
614
+ "American dipper",
615
+ "kite (bird of prey)",
616
+ "bald eagle",
617
+ "vulture",
618
+ "great grey owl",
619
+ "fire salamander",
620
+ "smooth newt",
621
+ "newt",
622
+ "spotted salamander",
623
+ "axolotl",
624
+ "American bullfrog",
625
+ "tree frog",
626
+ "tailed frog",
627
+ "loggerhead sea turtle",
628
+ "leatherback sea turtle",
629
+ "mud turtle",
630
+ "terrapin",
631
+ "box turtle",
632
+ "banded gecko",
633
+ "green iguana",
634
+ "Carolina anole",
635
+ "desert grassland whiptail lizard",
636
+ "agama",
637
+ "frilled-necked lizard",
638
+ "alligator lizard",
639
+ "Gila monster",
640
+ "European green lizard",
641
+ "chameleon",
642
+ "Komodo dragon",
643
+ "Nile crocodile",
644
+ "American alligator",
645
+ "triceratops",
646
+ "worm snake",
647
+ "ring-necked snake",
648
+ "eastern hog-nosed snake",
649
+ "smooth green snake",
650
+ "kingsnake",
651
+ "garter snake",
652
+ "water snake",
653
+ "vine snake",
654
+ "night snake",
655
+ "boa constrictor",
656
+ "African rock python",
657
+ "Indian cobra",
658
+ "green mamba",
659
+ "sea snake",
660
+ "Saharan horned viper",
661
+ "eastern diamondback rattlesnake",
662
+ "sidewinder rattlesnake",
663
+ "trilobite",
664
+ "harvestman",
665
+ "scorpion",
666
+ "yellow garden spider",
667
+ "barn spider",
668
+ "European garden spider",
669
+ "southern black widow",
670
+ "tarantula",
671
+ "wolf spider",
672
+ "tick",
673
+ "centipede",
674
+ "black grouse",
675
+ "ptarmigan",
676
+ "ruffed grouse",
677
+ "prairie grouse",
678
+ "peafowl",
679
+ "quail",
680
+ "partridge",
681
+ "african grey parrot",
682
+ "macaw",
683
+ "sulphur-crested cockatoo",
684
+ "lorikeet",
685
+ "coucal",
686
+ "bee eater",
687
+ "hornbill",
688
+ "hummingbird",
689
+ "jacamar",
690
+ "toucan",
691
+ "duck",
692
+ "red-breasted merganser",
693
+ "goose",
694
+ "black swan",
695
+ "tusker",
696
+ "echidna",
697
+ "platypus",
698
+ "wallaby",
699
+ "koala",
700
+ "wombat",
701
+ "jellyfish",
702
+ "sea anemone",
703
+ "brain coral",
704
+ "flatworm",
705
+ "nematode",
706
+ "conch",
707
+ "snail",
708
+ "slug",
709
+ "sea slug",
710
+ "chiton",
711
+ "chambered nautilus",
712
+ "Dungeness crab",
713
+ "rock crab",
714
+ "fiddler crab",
715
+ "red king crab",
716
+ "American lobster",
717
+ "spiny lobster",
718
+ "crayfish",
719
+ "hermit crab",
720
+ "isopod",
721
+ "white stork",
722
+ "black stork",
723
+ "spoonbill",
724
+ "flamingo",
725
+ "little blue heron",
726
+ "great egret",
727
+ "bittern bird",
728
+ "crane bird",
729
+ "limpkin",
730
+ "common gallinule",
731
+ "American coot",
732
+ "bustard",
733
+ "ruddy turnstone",
734
+ "dunlin",
735
+ "common redshank",
736
+ "dowitcher",
737
+ "oystercatcher",
738
+ "pelican",
739
+ "king penguin",
740
+ "albatross",
741
+ "grey whale",
742
+ "killer whale",
743
+ "dugong",
744
+ "sea lion",
745
+ "Chihuahua",
746
+ "Japanese Chin",
747
+ "Maltese",
748
+ "Pekingese",
749
+ "Shih Tzu",
750
+ "King Charles Spaniel",
751
+ "Papillon",
752
+ "toy terrier",
753
+ "Rhodesian Ridgeback",
754
+ "Afghan Hound",
755
+ "Basset Hound",
756
+ "Beagle",
757
+ "Bloodhound",
758
+ "Bluetick Coonhound",
759
+ "Black and Tan Coonhound",
760
+ "Treeing Walker Coonhound",
761
+ "English foxhound",
762
+ "Redbone Coonhound",
763
+ "borzoi",
764
+ "Irish Wolfhound",
765
+ "Italian Greyhound",
766
+ "Whippet",
767
+ "Ibizan Hound",
768
+ "Norwegian Elkhound",
769
+ "Otterhound",
770
+ "Saluki",
771
+ "Scottish Deerhound",
772
+ "Weimaraner",
773
+ "Staffordshire Bull Terrier",
774
+ "American Staffordshire Terrier",
775
+ "Bedlington Terrier",
776
+ "Border Terrier",
777
+ "Kerry Blue Terrier",
778
+ "Irish Terrier",
779
+ "Norfolk Terrier",
780
+ "Norwich Terrier",
781
+ "Yorkshire Terrier",
782
+ "Wire Fox Terrier",
783
+ "Lakeland Terrier",
784
+ "Sealyham Terrier",
785
+ "Airedale Terrier",
786
+ "Cairn Terrier",
787
+ "Australian Terrier",
788
+ "Dandie Dinmont Terrier",
789
+ "Boston Terrier",
790
+ "Miniature Schnauzer",
791
+ "Giant Schnauzer",
792
+ "Standard Schnauzer",
793
+ "Scottish Terrier",
794
+ "Tibetan Terrier",
795
+ "Australian Silky Terrier",
796
+ "Soft-coated Wheaten Terrier",
797
+ "West Highland White Terrier",
798
+ "Lhasa Apso",
799
+ "Flat-Coated Retriever",
800
+ "Curly-coated Retriever",
801
+ "Golden Retriever",
802
+ "Labrador Retriever",
803
+ "Chesapeake Bay Retriever",
804
+ "German Shorthaired Pointer",
805
+ "Vizsla",
806
+ "English Setter",
807
+ "Irish Setter",
808
+ "Gordon Setter",
809
+ "Brittany dog",
810
+ "Clumber Spaniel",
811
+ "English Springer Spaniel",
812
+ "Welsh Springer Spaniel",
813
+ "Cocker Spaniel",
814
+ "Sussex Spaniel",
815
+ "Irish Water Spaniel",
816
+ "Kuvasz",
817
+ "Schipperke",
818
+ "Groenendael dog",
819
+ "Malinois",
820
+ "Briard",
821
+ "Australian Kelpie",
822
+ "Komondor",
823
+ "Old English Sheepdog",
824
+ "Shetland Sheepdog",
825
+ "collie",
826
+ "Border Collie",
827
+ "Bouvier des Flandres dog",
828
+ "Rottweiler",
829
+ "German Shepherd Dog",
830
+ "Dobermann",
831
+ "Miniature Pinscher",
832
+ "Greater Swiss Mountain Dog",
833
+ "Bernese Mountain Dog",
834
+ "Appenzeller Sennenhund",
835
+ "Entlebucher Sennenhund",
836
+ "Boxer",
837
+ "Bullmastiff",
838
+ "Tibetan Mastiff",
839
+ "French Bulldog",
840
+ "Great Dane",
841
+ "St. Bernard",
842
+ "husky",
843
+ "Alaskan Malamute",
844
+ "Siberian Husky",
845
+ "Dalmatian",
846
+ "Affenpinscher",
847
+ "Basenji",
848
+ "pug",
849
+ "Leonberger",
850
+ "Newfoundland dog",
851
+ "Great Pyrenees dog",
852
+ "Samoyed",
853
+ "Pomeranian",
854
+ "Chow Chow",
855
+ "Keeshond",
856
+ "brussels griffon",
857
+ "Pembroke Welsh Corgi",
858
+ "Cardigan Welsh Corgi",
859
+ "Toy Poodle",
860
+ "Miniature Poodle",
861
+ "Standard Poodle",
862
+ "Mexican hairless dog (xoloitzcuintli)",
863
+ "grey wolf",
864
+ "Alaskan tundra wolf",
865
+ "red wolf or maned wolf",
866
+ "coyote",
867
+ "dingo",
868
+ "dhole",
869
+ "African wild dog",
870
+ "hyena",
871
+ "red fox",
872
+ "kit fox",
873
+ "Arctic fox",
874
+ "grey fox",
875
+ "tabby cat",
876
+ "tiger cat",
877
+ "Persian cat",
878
+ "Siamese cat",
879
+ "Egyptian Mau",
880
+ "cougar",
881
+ "lynx",
882
+ "leopard",
883
+ "snow leopard",
884
+ "jaguar",
885
+ "lion",
886
+ "tiger",
887
+ "cheetah",
888
+ "brown bear",
889
+ "American black bear",
890
+ "polar bear",
891
+ "sloth bear",
892
+ "mongoose",
893
+ "meerkat",
894
+ "tiger beetle",
895
+ "ladybug",
896
+ "ground beetle",
897
+ "longhorn beetle",
898
+ "leaf beetle",
899
+ "dung beetle",
900
+ "rhinoceros beetle",
901
+ "weevil",
902
+ "fly",
903
+ "bee",
904
+ "ant",
905
+ "grasshopper",
906
+ "cricket insect",
907
+ "stick insect",
908
+ "cockroach",
909
+ "praying mantis",
910
+ "cicada",
911
+ "leafhopper",
912
+ "lacewing",
913
+ "dragonfly",
914
+ "damselfly",
915
+ "red admiral butterfly",
916
+ "ringlet butterfly",
917
+ "monarch butterfly",
918
+ "small white butterfly",
919
+ "sulphur butterfly",
920
+ "gossamer-winged butterfly",
921
+ "starfish",
922
+ "sea urchin",
923
+ "sea cucumber",
924
+ "cottontail rabbit",
925
+ "hare",
926
+ "Angora rabbit",
927
+ "hamster",
928
+ "porcupine",
929
+ "fox squirrel",
930
+ "marmot",
931
+ "beaver",
932
+ "guinea pig",
933
+ "common sorrel horse",
934
+ "zebra",
935
+ "pig",
936
+ "wild boar",
937
+ "warthog",
938
+ "hippopotamus",
939
+ "ox",
940
+ "water buffalo",
941
+ "bison",
942
+ "ram (adult male sheep)",
943
+ "bighorn sheep",
944
+ "Alpine ibex",
945
+ "hartebeest",
946
+ "impala (antelope)",
947
+ "gazelle",
948
+ "arabian camel",
949
+ "llama",
950
+ "weasel",
951
+ "mink",
952
+ "European polecat",
953
+ "black-footed ferret",
954
+ "otter",
955
+ "skunk",
956
+ "badger",
957
+ "armadillo",
958
+ "three-toed sloth",
959
+ "orangutan",
960
+ "gorilla",
961
+ "chimpanzee",
962
+ "gibbon",
963
+ "siamang",
964
+ "guenon",
965
+ "patas monkey",
966
+ "baboon",
967
+ "macaque",
968
+ "langur",
969
+ "black-and-white colobus",
970
+ "proboscis monkey",
971
+ "marmoset",
972
+ "white-headed capuchin",
973
+ "howler monkey",
974
+ "titi monkey",
975
+ "Geoffroy's spider monkey",
976
+ "common squirrel monkey",
977
+ "ring-tailed lemur",
978
+ "indri",
979
+ "Asian elephant",
980
+ "African bush elephant",
981
+ "red panda",
982
+ "giant panda",
983
+ "snoek fish",
984
+ "eel",
985
+ "silver salmon",
986
+ "rock beauty fish",
987
+ "clownfish",
988
+ "sturgeon",
989
+ "gar fish",
990
+ "lionfish",
991
+ "pufferfish",
992
+ "abacus",
993
+ "abaya",
994
+ "academic gown",
995
+ "accordion",
996
+ "acoustic guitar",
997
+ "aircraft carrier",
998
+ "airliner",
999
+ "airship",
1000
+ "altar",
1001
+ "ambulance",
1002
+ "amphibious vehicle",
1003
+ "analog clock",
1004
+ "apiary",
1005
+ "apron",
1006
+ "trash can",
1007
+ "assault rifle",
1008
+ "backpack",
1009
+ "bakery",
1010
+ "balance beam",
1011
+ "balloon",
1012
+ "ballpoint pen",
1013
+ "Band-Aid",
1014
+ "banjo",
1015
+ "baluster / handrail",
1016
+ "barbell",
1017
+ "barber chair",
1018
+ "barbershop",
1019
+ "barn",
1020
+ "barometer",
1021
+ "barrel",
1022
+ "wheelbarrow",
1023
+ "baseball",
1024
+ "basketball",
1025
+ "bassinet",
1026
+ "bassoon",
1027
+ "swimming cap",
1028
+ "bath towel",
1029
+ "bathtub",
1030
+ "station wagon",
1031
+ "lighthouse",
1032
+ "beaker",
1033
+ "military hat (bearskin or shako)",
1034
+ "beer bottle",
1035
+ "beer glass",
1036
+ "bell tower",
1037
+ "baby bib",
1038
+ "tandem bicycle",
1039
+ "bikini",
1040
+ "ring binder",
1041
+ "binoculars",
1042
+ "birdhouse",
1043
+ "boathouse",
1044
+ "bobsleigh",
1045
+ "bolo tie",
1046
+ "poke bonnet",
1047
+ "bookcase",
1048
+ "bookstore",
1049
+ "bottle cap",
1050
+ "hunting bow",
1051
+ "bow tie",
1052
+ "brass memorial plaque",
1053
+ "bra",
1054
+ "breakwater",
1055
+ "breastplate",
1056
+ "broom",
1057
+ "bucket",
1058
+ "buckle",
1059
+ "bulletproof vest",
1060
+ "high-speed train",
1061
+ "butcher shop",
1062
+ "taxicab",
1063
+ "cauldron",
1064
+ "candle",
1065
+ "cannon",
1066
+ "canoe",
1067
+ "can opener",
1068
+ "cardigan",
1069
+ "car mirror",
1070
+ "carousel",
1071
+ "tool kit",
1072
+ "cardboard box / carton",
1073
+ "car wheel",
1074
+ "automated teller machine",
1075
+ "cassette",
1076
+ "cassette player",
1077
+ "castle",
1078
+ "catamaran",
1079
+ "CD player",
1080
+ "cello",
1081
+ "mobile phone",
1082
+ "chain",
1083
+ "chain-link fence",
1084
+ "chain mail",
1085
+ "chainsaw",
1086
+ "storage chest",
1087
+ "chiffonier",
1088
+ "bell or wind chime",
1089
+ "china cabinet",
1090
+ "Christmas stocking",
1091
+ "church",
1092
+ "movie theater",
1093
+ "cleaver",
1094
+ "cliff dwelling",
1095
+ "cloak",
1096
+ "clogs",
1097
+ "cocktail shaker",
1098
+ "coffee mug",
1099
+ "coffeemaker",
1100
+ "spiral or coil",
1101
+ "combination lock",
1102
+ "computer keyboard",
1103
+ "candy store",
1104
+ "container ship",
1105
+ "convertible",
1106
+ "corkscrew",
1107
+ "cornet",
1108
+ "cowboy boot",
1109
+ "cowboy hat",
1110
+ "cradle",
1111
+ "construction crane",
1112
+ "crash helmet",
1113
+ "crate",
1114
+ "infant bed",
1115
+ "Crock Pot",
1116
+ "croquet ball",
1117
+ "crutch",
1118
+ "cuirass",
1119
+ "dam",
1120
+ "desk",
1121
+ "desktop computer",
1122
+ "rotary dial telephone",
1123
+ "diaper",
1124
+ "digital clock",
1125
+ "digital watch",
1126
+ "dining table",
1127
+ "dishcloth",
1128
+ "dishwasher",
1129
+ "disc brake",
1130
+ "dock",
1131
+ "dog sled",
1132
+ "dome",
1133
+ "doormat",
1134
+ "drilling rig",
1135
+ "drum",
1136
+ "drumstick",
1137
+ "dumbbell",
1138
+ "Dutch oven",
1139
+ "electric fan",
1140
+ "electric guitar",
1141
+ "electric locomotive",
1142
+ "entertainment center",
1143
+ "envelope",
1144
+ "espresso machine",
1145
+ "face powder",
1146
+ "feather boa",
1147
+ "filing cabinet",
1148
+ "fireboat",
1149
+ "fire truck",
1150
+ "fire screen",
1151
+ "flagpole",
1152
+ "flute",
1153
+ "folding chair",
1154
+ "football helmet",
1155
+ "forklift",
1156
+ "fountain",
1157
+ "fountain pen",
1158
+ "four-poster bed",
1159
+ "freight car",
1160
+ "French horn",
1161
+ "frying pan",
1162
+ "fur coat",
1163
+ "garbage truck",
1164
+ "gas mask or respirator",
1165
+ "gas pump",
1166
+ "goblet",
1167
+ "go-kart",
1168
+ "golf ball",
1169
+ "golf cart",
1170
+ "gondola",
1171
+ "gong",
1172
+ "gown",
1173
+ "grand piano",
1174
+ "greenhouse",
1175
+ "radiator grille",
1176
+ "grocery store",
1177
+ "guillotine",
1178
+ "hair clip",
1179
+ "hair spray",
1180
+ "half-track",
1181
+ "hammer",
1182
+ "hamper",
1183
+ "hair dryer",
1184
+ "hand-held computer",
1185
+ "handkerchief",
1186
+ "hard disk drive",
1187
+ "harmonica",
1188
+ "harp",
1189
+ "combine harvester",
1190
+ "hatchet",
1191
+ "holster",
1192
+ "home theater",
1193
+ "honeycomb",
1194
+ "hook",
1195
+ "hoop skirt",
1196
+ "gymnastic horizontal bar",
1197
+ "horse-drawn vehicle",
1198
+ "hourglass",
1199
+ "iPod",
1200
+ "clothes iron",
1201
+ "carved pumpkin",
1202
+ "jeans",
1203
+ "jeep",
1204
+ "T-shirt",
1205
+ "jigsaw puzzle",
1206
+ "rickshaw",
1207
+ "joystick",
1208
+ "kimono",
1209
+ "knee pad",
1210
+ "knot",
1211
+ "lab coat",
1212
+ "ladle",
1213
+ "lampshade",
1214
+ "laptop computer",
1215
+ "lawn mower",
1216
+ "lens cap",
1217
+ "letter opener",
1218
+ "library",
1219
+ "lifeboat",
1220
+ "lighter",
1221
+ "limousine",
1222
+ "ocean liner",
1223
+ "lipstick",
1224
+ "slip-on shoe",
1225
+ "lotion",
1226
+ "music speaker",
1227
+ "loupe magnifying glass",
1228
+ "sawmill",
1229
+ "magnetic compass",
1230
+ "messenger bag",
1231
+ "mailbox",
1232
+ "tights",
1233
+ "one-piece bathing suit",
1234
+ "manhole cover",
1235
+ "maraca",
1236
+ "marimba",
1237
+ "mask",
1238
+ "matchstick",
1239
+ "maypole",
1240
+ "maze",
1241
+ "measuring cup",
1242
+ "medicine cabinet",
1243
+ "megalith",
1244
+ "microphone",
1245
+ "microwave oven",
1246
+ "military uniform",
1247
+ "milk can",
1248
+ "minibus",
1249
+ "miniskirt",
1250
+ "minivan",
1251
+ "missile",
1252
+ "mitten",
1253
+ "mixing bowl",
1254
+ "mobile home",
1255
+ "ford model t",
1256
+ "modem",
1257
+ "monastery",
1258
+ "monitor",
1259
+ "moped",
1260
+ "mortar and pestle",
1261
+ "graduation cap",
1262
+ "mosque",
1263
+ "mosquito net",
1264
+ "vespa",
1265
+ "mountain bike",
1266
+ "tent",
1267
+ "computer mouse",
1268
+ "mousetrap",
1269
+ "moving van",
1270
+ "muzzle",
1271
+ "metal nail",
1272
+ "neck brace",
1273
+ "necklace",
1274
+ "baby pacifier",
1275
+ "notebook computer",
1276
+ "obelisk",
1277
+ "oboe",
1278
+ "ocarina",
1279
+ "odometer",
1280
+ "oil filter",
1281
+ "pipe organ",
1282
+ "oscilloscope",
1283
+ "overskirt",
1284
+ "bullock cart",
1285
+ "oxygen mask",
1286
+ "product packet / packaging",
1287
+ "paddle",
1288
+ "paddle wheel",
1289
+ "padlock",
1290
+ "paintbrush",
1291
+ "pajamas",
1292
+ "palace",
1293
+ "pan flute",
1294
+ "paper towel",
1295
+ "parachute",
1296
+ "parallel bars",
1297
+ "park bench",
1298
+ "parking meter",
1299
+ "railroad car",
1300
+ "patio",
1301
+ "payphone",
1302
+ "pedestal",
1303
+ "pencil case",
1304
+ "pencil sharpener",
1305
+ "perfume",
1306
+ "Petri dish",
1307
+ "photocopier",
1308
+ "plectrum",
1309
+ "Pickelhaube",
1310
+ "picket fence",
1311
+ "pickup truck",
1312
+ "pier",
1313
+ "piggy bank",
1314
+ "pill bottle",
1315
+ "pillow",
1316
+ "ping-pong ball",
1317
+ "pinwheel",
1318
+ "pirate ship",
1319
+ "drink pitcher",
1320
+ "block plane",
1321
+ "planetarium",
1322
+ "plastic bag",
1323
+ "plate rack",
1324
+ "farm plow",
1325
+ "plunger",
1326
+ "Polaroid camera",
1327
+ "pole",
1328
+ "police van",
1329
+ "poncho",
1330
+ "pool table",
1331
+ "soda bottle",
1332
+ "plant pot",
1333
+ "potter's wheel",
1334
+ "power drill",
1335
+ "prayer rug",
1336
+ "printer",
1337
+ "prison",
1338
+ "missile",
1339
+ "projector",
1340
+ "hockey puck",
1341
+ "punching bag",
1342
+ "purse",
1343
+ "quill",
1344
+ "quilt",
1345
+ "race car",
1346
+ "racket",
1347
+ "radiator",
1348
+ "radio",
1349
+ "radio telescope",
1350
+ "rain barrel",
1351
+ "recreational vehicle",
1352
+ "fishing casting reel",
1353
+ "reflex camera",
1354
+ "refrigerator",
1355
+ "remote control",
1356
+ "restaurant",
1357
+ "revolver",
1358
+ "rifle",
1359
+ "rocking chair",
1360
+ "rotisserie",
1361
+ "eraser",
1362
+ "rugby ball",
1363
+ "ruler measuring stick",
1364
+ "sneaker",
1365
+ "safe",
1366
+ "safety pin",
1367
+ "salt shaker",
1368
+ "sandal",
1369
+ "sarong",
1370
+ "saxophone",
1371
+ "scabbard",
1372
+ "weighing scale",
1373
+ "school bus",
1374
+ "schooner",
1375
+ "scoreboard",
1376
+ "CRT monitor",
1377
+ "screw",
1378
+ "screwdriver",
1379
+ "seat belt",
1380
+ "sewing machine",
1381
+ "shield",
1382
+ "shoe store",
1383
+ "shoji screen / room divider",
1384
+ "shopping basket",
1385
+ "shopping cart",
1386
+ "shovel",
1387
+ "shower cap",
1388
+ "shower curtain",
1389
+ "ski",
1390
+ "balaclava ski mask",
1391
+ "sleeping bag",
1392
+ "slide rule",
1393
+ "sliding door",
1394
+ "slot machine",
1395
+ "snorkel",
1396
+ "snowmobile",
1397
+ "snowplow",
1398
+ "soap dispenser",
1399
+ "soccer ball",
1400
+ "sock",
1401
+ "solar thermal collector",
1402
+ "sombrero",
1403
+ "soup bowl",
1404
+ "keyboard space bar",
1405
+ "space heater",
1406
+ "space shuttle",
1407
+ "spatula",
1408
+ "motorboat",
1409
+ "spider web",
1410
+ "spindle",
1411
+ "sports car",
1412
+ "spotlight",
1413
+ "stage",
1414
+ "steam locomotive",
1415
+ "through arch bridge",
1416
+ "steel drum",
1417
+ "stethoscope",
1418
+ "scarf",
1419
+ "stone wall",
1420
+ "stopwatch",
1421
+ "stove",
1422
+ "strainer",
1423
+ "tram",
1424
+ "stretcher",
1425
+ "couch",
1426
+ "stupa",
1427
+ "submarine",
1428
+ "suit",
1429
+ "sundial",
1430
+ "sunglasses",
1431
+ "sunglasses",
1432
+ "sunscreen",
1433
+ "suspension bridge",
1434
+ "mop",
1435
+ "sweatshirt",
1436
+ "swim trunks / shorts",
1437
+ "swing",
1438
+ "electrical switch",
1439
+ "syringe",
1440
+ "table lamp",
1441
+ "tank",
1442
+ "tape player",
1443
+ "teapot",
1444
+ "teddy bear",
1445
+ "television",
1446
+ "tennis ball",
1447
+ "thatched roof",
1448
+ "front curtain",
1449
+ "thimble",
1450
+ "threshing machine",
1451
+ "throne",
1452
+ "tile roof",
1453
+ "toaster",
1454
+ "tobacco shop",
1455
+ "toilet seat",
1456
+ "torch",
1457
+ "totem pole",
1458
+ "tow truck",
1459
+ "toy store",
1460
+ "tractor",
1461
+ "semi-trailer truck",
1462
+ "tray",
1463
+ "trench coat",
1464
+ "tricycle",
1465
+ "trimaran",
1466
+ "tripod",
1467
+ "triumphal arch",
1468
+ "trolleybus",
1469
+ "trombone",
1470
+ "hot tub",
1471
+ "turnstile",
1472
+ "typewriter keyboard",
1473
+ "umbrella",
1474
+ "unicycle",
1475
+ "upright piano",
1476
+ "vacuum cleaner",
1477
+ "vase",
1478
+ "vaulted or arched ceiling",
1479
+ "velvet fabric",
1480
+ "vending machine",
1481
+ "vestment",
1482
+ "viaduct",
1483
+ "violin",
1484
+ "volleyball",
1485
+ "waffle iron",
1486
+ "wall clock",
1487
+ "wallet",
1488
+ "wardrobe",
1489
+ "military aircraft",
1490
+ "sink",
1491
+ "washing machine",
1492
+ "water bottle",
1493
+ "water jug",
1494
+ "water tower",
1495
+ "whiskey jug",
1496
+ "whistle",
1497
+ "hair wig",
1498
+ "window screen",
1499
+ "window shade",
1500
+ "Windsor tie",
1501
+ "wine bottle",
1502
+ "airplane wing",
1503
+ "wok",
1504
+ "wooden spoon",
1505
+ "wool",
1506
+ "split-rail fence",
1507
+ "shipwreck",
1508
+ "sailboat",
1509
+ "yurt",
1510
+ "website",
1511
+ "comic book",
1512
+ "crossword",
1513
+ "traffic or street sign",
1514
+ "traffic light",
1515
+ "dust jacket",
1516
+ "menu",
1517
+ "plate",
1518
+ "guacamole",
1519
+ "consomme",
1520
+ "hot pot",
1521
+ "trifle",
1522
+ "ice cream",
1523
+ "popsicle",
1524
+ "baguette",
1525
+ "bagel",
1526
+ "pretzel",
1527
+ "cheeseburger",
1528
+ "hot dog",
1529
+ "mashed potatoes",
1530
+ "cabbage",
1531
+ "broccoli",
1532
+ "cauliflower",
1533
+ "zucchini",
1534
+ "spaghetti squash",
1535
+ "acorn squash",
1536
+ "butternut squash",
1537
+ "cucumber",
1538
+ "artichoke",
1539
+ "bell pepper",
1540
+ "cardoon",
1541
+ "mushroom",
1542
+ "Granny Smith apple",
1543
+ "strawberry",
1544
+ "orange",
1545
+ "lemon",
1546
+ "fig",
1547
+ "pineapple",
1548
+ "banana",
1549
+ "jackfruit",
1550
+ "cherimoya (custard apple)",
1551
+ "pomegranate",
1552
+ "hay",
1553
+ "carbonara",
1554
+ "chocolate syrup",
1555
+ "dough",
1556
+ "meatloaf",
1557
+ "pizza",
1558
+ "pot pie",
1559
+ "burrito",
1560
+ "red wine",
1561
+ "espresso",
1562
+ "tea cup",
1563
+ "eggnog",
1564
+ "mountain",
1565
+ "bubble",
1566
+ "cliff",
1567
+ "coral reef",
1568
+ "geyser",
1569
+ "lakeshore",
1570
+ "promontory",
1571
+ "sandbar",
1572
+ "beach",
1573
+ "valley",
1574
+ "volcano",
1575
+ "baseball player",
1576
+ "bridegroom",
1577
+ "scuba diver",
1578
+ "rapeseed",
1579
+ "daisy",
1580
+ "yellow lady's slipper",
1581
+ "corn",
1582
+ "acorn",
1583
+ "rose hip",
1584
+ "horse chestnut seed",
1585
+ "coral fungus",
1586
+ "agaric",
1587
+ "gyromitra",
1588
+ "stinkhorn mushroom",
1589
+ "earth star fungus",
1590
+ "hen of the woods mushroom",
1591
+ "bolete",
1592
+ "corn cob",
1593
+ "toilet paper"
1594
+ ],
1595
+ "clevr_count_all": [
1596
+ "three",
1597
+ "four",
1598
+ "five",
1599
+ "six",
1600
+ "seven",
1601
+ "eight",
1602
+ "nine",
1603
+ "ten"
1604
+ ],
1605
+ "clevr_closest_object_distance": [
1606
+ "very nearby",
1607
+ "nearby",
1608
+ "near",
1609
+ "",
1610
+ "distant",
1611
+ "very distant"
1612
+ ],
1613
+ "mnist": [
1614
+ "0",
1615
+ "1",
1616
+ "2",
1617
+ "3",
1618
+ "4",
1619
+ "5",
1620
+ "6",
1621
+ "7",
1622
+ "8",
1623
+ "9"
1624
+ ],
1625
+ "svhn": [
1626
+ "zero",
1627
+ "one",
1628
+ "two",
1629
+ "three",
1630
+ "four",
1631
+ "five",
1632
+ "six",
1633
+ "seven",
1634
+ "eight",
1635
+ "nine"
1636
+ ],
1637
+ "kitti_closest_vehicle_distance": [
1638
+ "a photo i took of a car on my left or right side.",
1639
+ "a photo i took with a car nearby.",
1640
+ "a photo i took with a car in the distance.",
1641
+ "a photo i took with no car."
1642
+ ],
1643
+ "dmlab": [
1644
+ "nearby apple/melon",
1645
+ "far apple/melon",
1646
+ "very far apple/melon",
1647
+ "nearby lemon",
1648
+ "far lemon",
1649
+ "very far lemon"
1650
+ ],
1651
+ "pets": [
1652
+ "Abyssinian",
1653
+ "American Bulldog",
1654
+ "American Pit Bull Terrier",
1655
+ "Basset Hound",
1656
+ "Beagle",
1657
+ "Bengal",
1658
+ "Birman",
1659
+ "Bombay",
1660
+ "Boxer",
1661
+ "British Shorthair",
1662
+ "Chihuahua",
1663
+ "Egyptian Mau",
1664
+ "English Cocker Spaniel",
1665
+ "English Setter",
1666
+ "German Shorthaired",
1667
+ "Great Pyrenees",
1668
+ "Havanese",
1669
+ "Japanese Chin",
1670
+ "Keeshond",
1671
+ "Leonberger",
1672
+ "Maine Coon",
1673
+ "Miniature Pinscher",
1674
+ "Newfoundland",
1675
+ "Persian",
1676
+ "Pomeranian",
1677
+ "Pug",
1678
+ "Ragdoll",
1679
+ "Russian Blue",
1680
+ "Saint Bernard",
1681
+ "Samoyed",
1682
+ "Scottish Terrier",
1683
+ "Shiba Inu",
1684
+ "Siamese",
1685
+ "Sphynx",
1686
+ "Staffordshire Bull Terrier",
1687
+ "Wheaten Terrier",
1688
+ "Yorkshire Terrier"
1689
+ ],
1690
+ "pcam": [
1691
+ "lymph node",
1692
+ "lymph node containing metastatic tumor tissue"
1693
+ ],
1694
+ "diabetic_retinopathy": [
1695
+ "no diabetic retinopathy",
1696
+ "mild diabetic retinopathy",
1697
+ "moderate diabetic retinopathy",
1698
+ "severe diabetic retinopathy",
1699
+ "proliferative diabetic retinopathy"
1700
+ ]
1701
+ }
CLIP_benchmark/clip_benchmark/datasets/en_zeroshot_classification_templates.json ADDED
@@ -0,0 +1,295 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cifar10": [
3
+ "a photo of a {c}.",
4
+ "a blurry photo of a {c}.",
5
+ "a black and white photo of a {c}.",
6
+ "a low contrast photo of a {c}.",
7
+ "a high contrast photo of a {c}.",
8
+ "a bad photo of a {c}.",
9
+ "a good photo of a {c}.",
10
+ "a photo of a small {c}.",
11
+ "a photo of a big {c}.",
12
+ "a photo of the {c}.",
13
+ "a blurry photo of the {c}.",
14
+ "a black and white photo of the {c}.",
15
+ "a low contrast photo of the {c}.",
16
+ "a high contrast photo of the {c}.",
17
+ "a bad photo of the {c}.",
18
+ "a good photo of the {c}.",
19
+ "a photo of the small {c}.",
20
+ "a photo of the big {c}."
21
+ ],
22
+ "cifar100": [
23
+ "a photo of a {c}.",
24
+ "a blurry photo of a {c}.",
25
+ "a black and white photo of a {c}.",
26
+ "a low contrast photo of a {c}.",
27
+ "a high contrast photo of a {c}.",
28
+ "a bad photo of a {c}.",
29
+ "a good photo of a {c}.",
30
+ "a photo of a small {c}.",
31
+ "a photo of a big {c}.",
32
+ "a photo of the {c}.",
33
+ "a blurry photo of the {c}.",
34
+ "a black and white photo of the {c}.",
35
+ "a low contrast photo of the {c}.",
36
+ "a high contrast photo of the {c}.",
37
+ "a bad photo of the {c}.",
38
+ "a good photo of the {c}.",
39
+ "a photo of the small {c}.",
40
+ "a photo of the big {c}."
41
+ ],
42
+ "imagenet1k": [
43
+ "a bad photo of a {c}.",
44
+ "a photo of many {c}.",
45
+ "a sculpture of a {c}.",
46
+ "a photo of the hard to see {c}.",
47
+ "a low resolution photo of the {c}.",
48
+ "a rendering of a {c}.",
49
+ "graffiti of a {c}.",
50
+ "a bad photo of the {c}.",
51
+ "a cropped photo of the {c}.",
52
+ "a tattoo of a {c}.",
53
+ "the embroidered {c}.",
54
+ "a photo of a hard to see {c}.",
55
+ "a bright photo of a {c}.",
56
+ "a photo of a clean {c}.",
57
+ "a photo of a dirty {c}.",
58
+ "a dark photo of the {c}.",
59
+ "a drawing of a {c}.",
60
+ "a photo of my {c}.",
61
+ "the plastic {c}.",
62
+ "a photo of the cool {c}.",
63
+ "a close-up photo of a {c}.",
64
+ "a black and white photo of the {c}.",
65
+ "a painting of the {c}.",
66
+ "a painting of a {c}.",
67
+ "a pixelated photo of the {c}.",
68
+ "a sculpture of the {c}.",
69
+ "a bright photo of the {c}.",
70
+ "a cropped photo of a {c}.",
71
+ "a plastic {c}.",
72
+ "a photo of the dirty {c}.",
73
+ "a jpeg corrupted photo of a {c}.",
74
+ "a blurry photo of the {c}.",
75
+ "a photo of the {c}.",
76
+ "a good photo of the {c}.",
77
+ "a rendering of the {c}.",
78
+ "a {c} in a video game.",
79
+ "a photo of one {c}.",
80
+ "a doodle of a {c}.",
81
+ "a close-up photo of the {c}.",
82
+ "a photo of a {c}.",
83
+ "the origami {c}.",
84
+ "the {c} in a video game.",
85
+ "a sketch of a {c}.",
86
+ "a doodle of the {c}.",
87
+ "a origami {c}.",
88
+ "a low resolution photo of a {c}.",
89
+ "the toy {c}.",
90
+ "a rendition of the {c}.",
91
+ "a photo of the clean {c}.",
92
+ "a photo of a large {c}.",
93
+ "a rendition of a {c}.",
94
+ "a photo of a nice {c}.",
95
+ "a photo of a weird {c}.",
96
+ "a blurry photo of a {c}.",
97
+ "a cartoon {c}.",
98
+ "art of a {c}.",
99
+ "a sketch of the {c}.",
100
+ "a embroidered {c}.",
101
+ "a pixelated photo of a {c}.",
102
+ "itap of the {c}.",
103
+ "a jpeg corrupted photo of the {c}.",
104
+ "a good photo of a {c}.",
105
+ "a plushie {c}.",
106
+ "a photo of the nice {c}.",
107
+ "a photo of the small {c}.",
108
+ "a photo of the weird {c}.",
109
+ "the cartoon {c}.",
110
+ "art of the {c}.",
111
+ "a drawing of the {c}.",
112
+ "a photo of the large {c}.",
113
+ "a black and white photo of a {c}.",
114
+ "the plushie {c}.",
115
+ "a dark photo of a {c}.",
116
+ "itap of a {c}.",
117
+ "graffiti of the {c}.",
118
+ "a toy {c}.",
119
+ "itap of my {c}.",
120
+ "a photo of a cool {c}.",
121
+ "a photo of a small {c}.",
122
+ "a tattoo of the {c}."
123
+ ],
124
+ "food101": [
125
+ "a photo of {c}, a type of food."
126
+ ],
127
+ "sun397": [
128
+ "a photo of a {c}.",
129
+ "a photo of the {c}."
130
+ ],
131
+ "cars": [
132
+ "a photo of a {c}.",
133
+ "a photo of the {c}.",
134
+ "a photo of my {c}.",
135
+ "i love my {c}!",
136
+ "a photo of my dirty {c}.",
137
+ "a photo of my clean {c}.",
138
+ "a photo of my new {c}.",
139
+ "a photo of my old {c}."
140
+ ],
141
+ "fgvc_aircraft": [
142
+ "a photo of a {c}, a type of aircraft.",
143
+ "a photo of the {c}, a type of aircraft."
144
+ ],
145
+ "dtd": [
146
+ "a photo of a {c} texture.",
147
+ "a photo of a {c} pattern.",
148
+ "a photo of a {c} thing.",
149
+ "a photo of a {c} object.",
150
+ "a photo of the {c} texture.",
151
+ "a photo of the {c} pattern.",
152
+ "a photo of the {c} thing.",
153
+ "a photo of the {c} object."
154
+ ],
155
+ "pets": [
156
+ "a photo of a {c}, a type of pet."
157
+ ],
158
+ "caltech101": [
159
+ "a photo of a {c}.",
160
+ "a painting of a {c}.",
161
+ "a plastic {c}.",
162
+ "a sculpture of a {c}.",
163
+ "a sketch of a {c}.",
164
+ "a tattoo of a {c}.",
165
+ "a toy {c}.",
166
+ "a rendition of a {c}.",
167
+ "a embroidered {c}.",
168
+ "a cartoon {c}.",
169
+ "a {c} in a video game.",
170
+ "a plushie {c}.",
171
+ "a origami {c}.",
172
+ "art of a {c}.",
173
+ "graffiti of a {c}.",
174
+ "a drawing of a {c}.",
175
+ "a doodle of a {c}.",
176
+ "a photo of the {c}.",
177
+ "a painting of the {c}.",
178
+ "the plastic {c}.",
179
+ "a sculpture of the {c}.",
180
+ "a sketch of the {c}.",
181
+ "a tattoo of the {c}.",
182
+ "the toy {c}.",
183
+ "a rendition of the {c}.",
184
+ "the embroidered {c}.",
185
+ "the cartoon {c}.",
186
+ "the {c} in a video game.",
187
+ "the plushie {c}.",
188
+ "the origami {c}.",
189
+ "art of the {c}.",
190
+ "graffiti of the {c}.",
191
+ "a drawing of the {c}.",
192
+ "a doodle of the {c}."
193
+ ],
194
+ "flowers": [
195
+ "a photo of a {c}, a type of flower."
196
+ ],
197
+ "mnist": [
198
+ "a photo of the number: \"{c}\"."
199
+ ],
200
+ "stl10": [
201
+ "a photo of a {c}.",
202
+ "a photo of the {c}."
203
+ ],
204
+ "eurosat": [
205
+ "a centered satellite photo of {c}.",
206
+ "a centered satellite photo of a {c}.",
207
+ "a centered satellite photo of the {c}."
208
+ ],
209
+ "gtsrb": [
210
+ "a zoomed in photo of a \"{c}\" traffic sign.",
211
+ "a centered photo of a \"{c}\" traffic sign.",
212
+ "a close up photo of a \"{c}\" traffic sign."
213
+ ],
214
+ "country211": [
215
+ "a photo i took in {c}.",
216
+ "a photo i took while visiting {c}.",
217
+ "a photo from my home country of {c}.",
218
+ "a photo from my visit to {c}.",
219
+ "a photo showing the country of {c}."
220
+ ],
221
+ "renderedsst2": [
222
+ "a {c} review of a movie."
223
+ ],
224
+ "voc2007": [
225
+ "a photo of a {c}."
226
+ ],
227
+ "voc2007_multilabel": [
228
+ "a photo of a {c}."
229
+ ],
230
+ "fer2013": [
231
+ "a photo of a {c} looking face.",
232
+ "a photo of a face showing the emotion: {c}.",
233
+ "a photo of a face looking {c}.",
234
+ "a face that looks {c}.",
235
+ "they look {c}.",
236
+ "look at how {c} they are."
237
+ ],
238
+ "clevr_count_all": [
239
+ "a picture of {c} objects"
240
+ ],
241
+ "clevr_closest_object_distance": [
242
+ "{c} shapes."
243
+ ],
244
+ "pcam": [
245
+ "a histopathology slide showing {c}",
246
+ "histopathology image of {c}"
247
+ ],
248
+ "svhn": [
249
+ "a photo of the number {c} written on a sign",
250
+ "an outdoor house number {c}",
251
+ "the number {c} in the center of the image",
252
+ "an outdoor number {c} writte on a sign",
253
+ "an outdoor number {c}",
254
+ "a centered image of the number {c}"
255
+ ],
256
+ "resisc45": [
257
+ "a sattelite image of {c}",
258
+ "an aerial view of {c}",
259
+ "a sattelite photo of {c}",
260
+ "{c} from above"
261
+ ],
262
+ "kitti_closest_vehicle_distance": [
263
+ "{c}"
264
+ ],
265
+ "smallnorb_label_azimuth": [
266
+ "an object rotated at {c}",
267
+ "something rotated at {c}",
268
+ "{c} rotation",
269
+ "something at a {c} angle"
270
+ ],
271
+ "smallnorb_label_elevation": [
272
+ "an object rotated at {c}",
273
+ "something rotated at {c}",
274
+ "{c} rotation",
275
+ "something at a {c} angle"
276
+ ],
277
+ "dsprites_label_x_position": [
278
+ "an object located at position {c}% on the horizontal axis"
279
+ ],
280
+ "dsprites_label_orientation": [
281
+ "an object rotated at {c}",
282
+ "something rotated at {c}",
283
+ "{c} rotation",
284
+ "something at a {c} angle"
285
+ ],
286
+ "dmlab": [
287
+ "{c}"
288
+ ],
289
+ "diabetic_retinopathy": [
290
+ "a retinal image with {c}"
291
+ ],
292
+ "dummy": [
293
+ "a photo of a {c}"
294
+ ]
295
+ }
CLIP_benchmark/clip_benchmark/datasets/flickr.py ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Adapted from https://github.com/pytorch/vision/blob/main/torchvision/datasets/flickr.py
3
+ Thanks to the authors of torchvision
4
+ """
5
+ from collections import defaultdict
6
+ import glob
7
+ import os
8
+ from collections import defaultdict
9
+ from html.parser import HTMLParser
10
+ from typing import Any, Callable, Dict, List, Optional, Tuple
11
+
12
+ from PIL import Image
13
+ from torchvision.datasets import VisionDataset
14
+
15
+ class Flickr(VisionDataset):
16
+
17
+ def __init__(
18
+ self,
19
+ root: str,
20
+ ann_file: str,
21
+ transform: Optional[Callable] = None,
22
+ target_transform: Optional[Callable] = None,
23
+ ) -> None:
24
+ super().__init__(root, transform=transform, target_transform=target_transform)
25
+ self.ann_file = os.path.expanduser(ann_file)
26
+ data = defaultdict(list)
27
+ with open(ann_file) as fd:
28
+ fd.readline()
29
+ for line in fd:
30
+ line = line.strip()
31
+ if line:
32
+ # some lines have comma in the caption, se we make sure we do the split correctly
33
+ img, caption = line.strip().split(".jpg,")
34
+ img = img + ".jpg"
35
+ data[img].append(caption)
36
+ self.data = list(data.items())
37
+
38
+ def __getitem__(self, index: int) -> Tuple[Any, Any]:
39
+ """
40
+ Args:
41
+ index (int): Index
42
+
43
+ Returns:
44
+ tuple: Tuple (image, target). target is a list of captions for the image.
45
+ """
46
+ img, captions = self.data[index]
47
+
48
+ # Image
49
+ img = Image.open(os.path.join(self.root, img)).convert("RGB")
50
+ if self.transform is not None:
51
+ img = self.transform(img)
52
+
53
+ # Captions
54
+ target = captions
55
+ if self.target_transform is not None:
56
+ target = self.target_transform(target)
57
+
58
+ return img, target
59
+
60
+
61
+ def __len__(self) -> int:
62
+ return len(self.data)
CLIP_benchmark/clip_benchmark/datasets/imagenetv2.py ADDED
@@ -0,0 +1,98 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Code from https://github.com/mlfoundations/wise-ft/blob/master/src/datasets/imagenetv2.py
3
+ Thanks to the authors of wise-ft
4
+ """
5
+ import pathlib
6
+ import tarfile
7
+ import requests
8
+ import shutil
9
+
10
+ from PIL import Image
11
+ from tqdm import tqdm
12
+ from torch.utils.data import Dataset, DataLoader
13
+ from torchvision.datasets import ImageFolder
14
+
15
+ URLS = {"matched-frequency" : "https://imagenetv2public.s3-us-west-2.amazonaws.com/imagenetv2-matched-frequency.tar.gz",
16
+ "threshold-0.7" : "https://imagenetv2public.s3-us-west-2.amazonaws.com/imagenetv2-threshold0.7.tar.gz",
17
+ "top-images": "https://imagenetv2public.s3-us-west-2.amazonaws.com/imagenetv2-top-images.tar.gz",
18
+ "val": "https://imagenetv2public.s3-us-west-2.amazonaws.com/imagenet_validation.tar.gz"}
19
+
20
+ FNAMES = {"matched-frequency" : "imagenetv2-matched-frequency-format-val",
21
+ "threshold-0.7" : "imagenetv2-threshold0.7-format-val",
22
+ "top-images": "imagenetv2-top-images-format-val",
23
+ "val": "imagenet_validation"}
24
+
25
+
26
+ V2_DATASET_SIZE = 10000
27
+ VAL_DATASET_SIZE = 50000
28
+
29
+ class ImageNetValDataset(Dataset):
30
+ def __init__(self, transform=None, location="."):
31
+ self.dataset_root = pathlib.Path(f"{location}/imagenet_validation/")
32
+ self.tar_root = pathlib.Path(f"{location}/imagenet_validation.tar.gz")
33
+ self.fnames = list(self.dataset_root.glob("**/*.JPEG"))
34
+ self.transform = transform
35
+ if not self.dataset_root.exists() or len(self.fnames) != VAL_DATASET_SIZE:
36
+ if not self.tar_root.exists():
37
+ print(f"Dataset imagenet-val not found on disk, downloading....")
38
+ response = requests.get(URLS["val"], stream=True)
39
+ total_size_in_bytes= int(response.headers.get('content-length', 0))
40
+ block_size = 1024 #1 Kibibyte
41
+ progress_bar = tqdm(total=total_size_in_bytes, unit='iB', unit_scale=True)
42
+ with open(self.tar_root, 'wb') as f:
43
+ for data in response.iter_content(block_size):
44
+ progress_bar.update(len(data))
45
+ f.write(data)
46
+ progress_bar.close()
47
+ if total_size_in_bytes != 0 and progress_bar.n != total_size_in_bytes:
48
+ assert False, f"Downloading from {URLS[variant]} failed"
49
+ print("Extracting....")
50
+ tarfile.open(self.tar_root).extractall(f"{location}")
51
+ shutil.move(f"{location}/{FNAMES['val']}", self.dataset_root)
52
+
53
+ self.dataset = ImageFolder(self.dataset_root)
54
+
55
+ def __len__(self):
56
+ return len(self.dataset)
57
+
58
+ def __getitem__(self, i):
59
+ img, label = self.dataset[i]
60
+ if self.transform is not None:
61
+ img = self.transform(img)
62
+ return img, label
63
+
64
+ class ImageNetV2Dataset(Dataset):
65
+ def __init__(self, variant="matched-frequency", transform=None, location="."):
66
+ self.dataset_root = pathlib.Path(f"{location}/ImageNetV2-{variant}/")
67
+ self.tar_root = pathlib.Path(f"{location}/ImageNetV2-{variant}.tar.gz")
68
+ self.fnames = list(self.dataset_root.glob("**/*.jpeg"))
69
+ self.transform = transform
70
+ assert variant in URLS, f"unknown V2 Variant: {variant}"
71
+ if not self.dataset_root.exists() or len(self.fnames) != V2_DATASET_SIZE:
72
+ if not self.tar_root.exists():
73
+ print(f"Dataset {variant} not found on disk, downloading....")
74
+ response = requests.get(URLS[variant], stream=True)
75
+ total_size_in_bytes= int(response.headers.get('content-length', 0))
76
+ block_size = 1024 #1 Kibibyte
77
+ progress_bar = tqdm(total=total_size_in_bytes, unit='iB', unit_scale=True)
78
+ with open(self.tar_root, 'wb') as f:
79
+ for data in response.iter_content(block_size):
80
+ progress_bar.update(len(data))
81
+ f.write(data)
82
+ progress_bar.close()
83
+ if total_size_in_bytes != 0 and progress_bar.n != total_size_in_bytes:
84
+ assert False, f"Downloading from {URLS[variant]} failed"
85
+ print("Extracting....")
86
+ tarfile.open(self.tar_root).extractall(f"{location}")
87
+ shutil.move(f"{location}/{FNAMES[variant]}", self.dataset_root)
88
+ self.fnames = list(self.dataset_root.glob("**/*.jpeg"))
89
+
90
+
91
+ def __len__(self):
92
+ return len(self.fnames)
93
+
94
+ def __getitem__(self, i):
95
+ img, label = Image.open(self.fnames[i]), int(self.fnames[i].parent.name)
96
+ if self.transform is not None:
97
+ img = self.transform(img)
98
+ return img, label
CLIP_benchmark/clip_benchmark/datasets/it_classnames.json ADDED
@@ -0,0 +1,1004 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "imagenet1k": [
3
+ "una tinca",
4
+ "un pesce rosso",
5
+ "un grande squalo bianco",
6
+ "uno squalo tigre",
7
+ "un pesce martello",
8
+ "un raggio elettrico",
9
+ "una pastinaca",
10
+ "un gallo",
11
+ "una gallina",
12
+ "uno struzzo",
13
+ "un rovo",
14
+ "un cardellino",
15
+ "un fringuello di casa",
16
+ "un giunco",
17
+ "uno zigolo indaco",
18
+ "un pettirosso",
19
+ "un bulbul",
20
+ "una ghiandaia",
21
+ "una gazza",
22
+ "una cinciallegra",
23
+ "un'ouzel d'acqua",
24
+ "un aquilone",
25
+ "un'aquila calva",
26
+ "un avvoltoio",
27
+ "un grande gufo grigio",
28
+ "una salamandra da fuoco europea",
29
+ "un tritone comune",
30
+ "a eft",
31
+ "una salamandra pezzata",
32
+ "un axolotl",
33
+ "una rana toro",
34
+ "una raganella",
35
+ "una rana con la coda",
36
+ "una testa di toro",
37
+ "una tartaruga di cuoio",
38
+ "una tartaruga di fango",
39
+ "una tartaruga",
40
+ "una tartaruga di scatola",
41
+ "un geco a bande",
42
+ "un'iguana comune",
43
+ "un camaleonte americano",
44
+ "una coda di frusta",
45
+ "a agama",
46
+ "una lucertola con le ali",
47
+ "una lucertola alligatore",
48
+ "un mostro di Gila",
49
+ "una lucertola verde",
50
+ "un camaleonte africano",
51
+ "un drago di Komodo",
52
+ "un coccodrillo africano",
53
+ "un alligatore americano",
54
+ "un triceratopo",
55
+ "un serpente di tuono",
56
+ "un serpente ringneck",
57
+ "un serpente hognose",
58
+ "un serpente verde",
59
+ "un serpente re",
60
+ "un serpente giarrettiera",
61
+ "un serpente d'acqua",
62
+ "un serpente a forma di vite",
63
+ "un serpente notturno",
64
+ "un boa constrictor",
65
+ "un pitone delle rocce",
66
+ "un cobra indiano",
67
+ "un mamba verde",
68
+ "un serpente di mare",
69
+ "una vipera cornuta",
70
+ "un diamondback",
71
+ "un sidewinder",
72
+ "un trilobite",
73
+ "un raccoglitore",
74
+ "uno scorpione",
75
+ "un ragno da giardino nero e oro",
76
+ "un ragno da fienile",
77
+ "un ragno del giardino",
78
+ "una vedova nera",
79
+ "una tarantola",
80
+ "un ragno lupo",
81
+ "un segno di spunta",
82
+ "un millepiedi",
83
+ "un fagiano di monte",
84
+ "pernice bianca",
85
+ "un gallo cedrone",
86
+ "un pollo della prateria",
87
+ "un pavone",
88
+ "una quaglia",
89
+ "una pernice",
90
+ "un grigio africano",
91
+ "un'ara",
92
+ "un cacatua dalla cresta sulfurea",
93
+ "un lorichetto",
94
+ "una coucal",
95
+ "un mangiatore di api",
96
+ "un bucerotide",
97
+ "un colibr\u00ec",
98
+ "un jacamar",
99
+ "un tucano",
100
+ "un drake",
101
+ "un merganser dal petto rosso",
102
+ "un'oca",
103
+ "un cigno nero",
104
+ "un tusker",
105
+ "un echidna",
106
+ "un ornitorinco",
107
+ "un wallaby",
108
+ "un koala",
109
+ "un vombato",
110
+ "una medusa",
111
+ "un anemone di mare",
112
+ "un corallo del cervello",
113
+ "un verme piatto",
114
+ "un nematode",
115
+ "uno strombo",
116
+ "una lumaca",
117
+ "una lumaca",
118
+ "una lumaca di mare",
119
+ "un chitone",
120
+ "un nautilus a camera",
121
+ "un granchio di Dungeness",
122
+ "un granchio di roccia",
123
+ "un granchio violinista",
124
+ "un granchio reale",
125
+ "un'aragosta americana",
126
+ "un'aragosta spinosa",
127
+ "un gambero di fiume",
128
+ "un paguro",
129
+ "un isopode",
130
+ "una cicogna bianca",
131
+ "una cicogna nera",
132
+ "una spatola",
133
+ "un fenicottero",
134
+ "un piccolo airone blu",
135
+ "una garzetta americana",
136
+ "un tarabuso",
137
+ "una gru",
138
+ "un moscerino",
139
+ "un gallinaccio europeo",
140
+ "una folaga americana",
141
+ "un'otarda",
142
+ "una pietra focaia rubiconda",
143
+ "un piovanello dal dorso rosso",
144
+ "una pettegola",
145
+ "una dowitcher",
146
+ "una beccaccia di mare",
147
+ "un pellicano",
148
+ "un pinguino reale",
149
+ "un albatros",
150
+ "una balena grigia",
151
+ "un'orca assassina",
152
+ "un dugongo",
153
+ "un leone marino",
154
+ "un chihuahua",
155
+ "uno spaniel giapponese",
156
+ "un cane maltese",
157
+ "un pechinese",
158
+ "uno Shih-Tzu",
159
+ "uno spaniel Blenheim",
160
+ "un papillon",
161
+ "un terrier giocattolo",
162
+ "un Rhodesian ridgeback",
163
+ "un segugio afgano",
164
+ "un bassotto",
165
+ "un beagle",
166
+ "un segugio",
167
+ "un bluetick",
168
+ "un coonhound nero e marrone",
169
+ "un segugio Walker",
170
+ "un foxhound inglese",
171
+ "un osso rosso",
172
+ "un borzoi",
173
+ "un cane lupo irlandese",
174
+ "un levriero italiano",
175
+ "un whippet",
176
+ "un segugio ibizenco",
177
+ "un elkhound norvegese",
178
+ "una lontra",
179
+ "un Saluki",
180
+ "un deerhound scozzese",
181
+ "un Weimaraner",
182
+ "uno Staffordshire bullterrier",
183
+ "un American Staffordshire terrier",
184
+ "un Bedlington terrier",
185
+ "un Border terrier",
186
+ "un Kerry blue terrier",
187
+ "un terrier irlandese",
188
+ "un Norfolk terrier",
189
+ "un Norwich terrier",
190
+ "uno Yorkshire terrier",
191
+ "un fox terrier a pelo corto",
192
+ "un Lakeland terrier",
193
+ "un Sealyham terrier",
194
+ "un Airedale",
195
+ "un tumulo",
196
+ "un terrier australiano",
197
+ "un Dandie Dinmont",
198
+ "un toro di Boston",
199
+ "uno schnauzer in miniatura",
200
+ "uno schnauzer gigante",
201
+ "uno schnauzer standard",
202
+ "un terrier scozzese",
203
+ "un terrier tibetano",
204
+ "un terrier di seta",
205
+ "un wheaten terrier a pelo morbido",
206
+ "un West Highland white terrier",
207
+ "a Lhasa",
208
+ "un flat-coated retriever",
209
+ "un retriever a pelo riccio",
210
+ "un golden retriever",
211
+ "un Labrador retriever",
212
+ "un Chesapeake Bay retriever",
213
+ "un pointer tedesco a pelo corto",
214
+ "un vizsla",
215
+ "un setter inglese",
216
+ "un setter irlandese",
217
+ "un setter Gordon",
218
+ "un Brittany spaniel",
219
+ "un idraulico",
220
+ "uno springer inglese",
221
+ "uno springer spaniel gallese",
222
+ "un cocker spaniel",
223
+ "uno spaniel del Sussex",
224
+ "uno spaniel d'acqua irlandese",
225
+ "un kuvasz",
226
+ "uno schipperke",
227
+ "una groenendael",
228
+ "un malinois",
229
+ "una briarda",
230
+ "un kelpie",
231
+ "un komondor",
232
+ "un vecchio cane da pastore inglese",
233
+ "un cane da pastore Shetland",
234
+ "un collie",
235
+ "un Border collie",
236
+ "un Bouvier des Flandres",
237
+ "un Rottweiler",
238
+ "un pastore tedesco",
239
+ "un dobermann",
240
+ "un pinscher in miniatura",
241
+ "un cane da montagna svizzero maggiore",
242
+ "un cane di montagna bernese",
243
+ "un Appenzeller",
244
+ "a EntleBucher",
245
+ "un pugile",
246
+ "un mastino toro",
247
+ "un mastino tibetano",
248
+ "un bulldog francese",
249
+ "un alano",
250
+ "un San Bernardo",
251
+ "un cane eschimese",
252
+ "un malamute",
253
+ "un Siberian husky",
254
+ "un dalmata",
255
+ "un affenpinscher",
256
+ "un basenji",
257
+ "un carlino",
258
+ "a Leonberg",
259
+ "un Terranova",
260
+ "un Grande Pireneo",
261
+ "un samoiedo",
262
+ "un Pomerania",
263
+ "un chow",
264
+ "un keeshond",
265
+ "un grifone di Brabancon",
266
+ "un Pembroke",
267
+ "un cardigan",
268
+ "un barboncino giocattolo",
269
+ "un barboncino in miniatura",
270
+ "un barboncino standard",
271
+ "un messicano senza capelli",
272
+ "un lupo di legno",
273
+ "un lupo bianco",
274
+ "un lupo rosso",
275
+ "un coyote",
276
+ "un dingo",
277
+ "un dhole",
278
+ "un cane da caccia africano",
279
+ "una iena",
280
+ "una volpe rossa",
281
+ "una volpe di kit",
282
+ "una volpe artica",
283
+ "una volpe grigia",
284
+ "un soriano",
285
+ "un gatto tigrato",
286
+ "un gatto persiano",
287
+ "un gatto siamese",
288
+ "un gatto egiziano",
289
+ "un puma",
290
+ "una lince",
291
+ "un leopardo",
292
+ "un leopardo delle nevi",
293
+ "un giaguaro",
294
+ "un leone",
295
+ "una tigre",
296
+ "un ghepardo",
297
+ "un orso bruno",
298
+ "un orso nero americano",
299
+ "un orso di ghiaccio",
300
+ "un orso bradipo",
301
+ "una mangusta",
302
+ "un suricato",
303
+ "uno scarabeo tigre",
304
+ "una coccinella",
305
+ "uno scarabeo di terra",
306
+ "uno scarabeo dalle lunghe corna",
307
+ "uno scarabeo delle foglie",
308
+ "uno scarabeo stercorario",
309
+ "uno scarabeo rinoceronte",
310
+ "un tonchio",
311
+ "una mosca",
312
+ "un'ape",
313
+ "una formica",
314
+ "una cavalletta",
315
+ "un grillo",
316
+ "un bastone da passeggio",
317
+ "uno scarafaggio",
318
+ "una mantide",
319
+ "una cicala",
320
+ "una cavalletta",
321
+ "un pizzo",
322
+ "una libellula",
323
+ "una damigella",
324
+ "un ammiraglio",
325
+ "un ricciolo",
326
+ "un monarca",
327
+ "una farfalla cavolo",
328
+ "una farfalla sulfurea",
329
+ "un licaenide",
330
+ "una stella marina",
331
+ "un riccio di mare",
332
+ "un cetriolo di mare",
333
+ "un coniglio di legno",
334
+ "una lepre",
335
+ "un angora",
336
+ "un criceto",
337
+ "un porcospino",
338
+ "uno scoiattolo volpe",
339
+ "una marmotta",
340
+ "un castoro",
341
+ "un porcellino d'India",
342
+ "un'acetosa",
343
+ "una zebra",
344
+ "un maiale",
345
+ "un cinghiale",
346
+ "un facocero",
347
+ "un ippopotamo",
348
+ "un bue",
349
+ "un bufalo d'acqua",
350
+ "un bisonte",
351
+ "ariete",
352
+ "un bighorn",
353
+ "uno stambecco",
354
+ "un alcefalo",
355
+ "un impala",
356
+ "una gazzella",
357
+ "un cammello arabo",
358
+ "un lama",
359
+ "una donnola",
360
+ "un visone",
361
+ "una puzzola",
362
+ "un furetto dai piedi neri",
363
+ "una lontra",
364
+ "una puzzola",
365
+ "un tasso",
366
+ "un armadillo",
367
+ "un bradipo a tre dita",
368
+ "un orango",
369
+ "un gorilla",
370
+ "uno scimpanz\u00e9",
371
+ "un gibbone",
372
+ "un siamang",
373
+ "un cercopiteco",
374
+ "un patas",
375
+ "un babbuino",
376
+ "un macaco",
377
+ "un langur",
378
+ "un colobo",
379
+ "una scimmia proboscide",
380
+ "uno uistit\u00ec",
381
+ "un cappuccino",
382
+ "una scimmia urlatrice",
383
+ "un titi",
384
+ "una scimmia ragno",
385
+ "una scimmia scoiattolo",
386
+ "un gatto del Madagascar",
387
+ "un indri",
388
+ "un elefante indiano",
389
+ "un elefante africano",
390
+ "un panda minore",
391
+ "un panda gigante",
392
+ "un barracuda",
393
+ "un'anguilla",
394
+ "un coho",
395
+ "una bellezza di roccia",
396
+ "un pesce anemone",
397
+ "uno storione",
398
+ "un capo d'abbigliamento",
399
+ "un pesce leone",
400
+ "un puffer",
401
+ "un abaco",
402
+ "un abaya",
403
+ "un abito accademico",
404
+ "una fisarmonica",
405
+ "una chitarra acustica",
406
+ "una portaerei",
407
+ "un aereo di linea",
408
+ "un dirigibile",
409
+ "un altare",
410
+ "un'ambulanza",
411
+ "un anfibio",
412
+ "un orologio analogico",
413
+ "un apiario",
414
+ "un grembiule",
415
+ "un frassino",
416
+ "un fucile d'assalto",
417
+ "uno zaino",
418
+ "un panificio",
419
+ "una trave di equilibrio",
420
+ "un pallone",
421
+ "una penna a sfera",
422
+ "un cerotto",
423
+ "un banjo",
424
+ "una balaustra",
425
+ "un bilanciere",
426
+ "una sedia da barbiere",
427
+ "un barbiere",
428
+ "un fienile",
429
+ "un barometro",
430
+ "un barile",
431
+ "un carretto",
432
+ "una palla da baseball",
433
+ "una pallacanestro",
434
+ "una culla",
435
+ "un fagotto",
436
+ "un berretto da bagno",
437
+ "un asciugamano da bagno",
438
+ "una vasca da bagno",
439
+ "un carro da spiaggia",
440
+ "un faro",
441
+ "un bicchiere",
442
+ "una pelle d'orso",
443
+ "una bottiglia di birra",
444
+ "un bicchiere di birra",
445
+ "un campanile",
446
+ "un bavaglino",
447
+ "una bicicletta costruita per due",
448
+ "un bikini",
449
+ "un raccoglitore",
450
+ "un binocolo",
451
+ "una casetta per uccelli",
452
+ "una rimessa per barche",
453
+ "un bob",
454
+ "una cravatta bolo",
455
+ "un cofano",
456
+ "una libreria",
457
+ "una libreria",
458
+ "un tappo di bottiglia",
459
+ "un arco",
460
+ "un papillon",
461
+ "un ottone",
462
+ "un reggiseno",
463
+ "un frangiflutti",
464
+ "una corazza",
465
+ "una scopa",
466
+ "un secchio",
467
+ "una fibbia",
468
+ "un giubbotto antiproiettile",
469
+ "un treno proiettile",
470
+ "una macelleria",
471
+ "un taxi",
472
+ "un calderone",
473
+ "una candela",
474
+ "un cannone",
475
+ "una canoa",
476
+ "un apriscatole",
477
+ "un cardigan",
478
+ "uno specchio per auto",
479
+ "una giostra",
480
+ "un kit da falegname",
481
+ "un cartone",
482
+ "una ruota di automobile",
483
+ "un bancomat",
484
+ "una cassetta",
485
+ "un lettore di cassette",
486
+ "un castello",
487
+ "un catamarano",
488
+ "un lettore CD",
489
+ "un violoncello",
490
+ "un telefono cellulare",
491
+ "una catena",
492
+ "una recinzione di rete metallica",
493
+ "una cotta di maglia",
494
+ "una motosega",
495
+ "un petto",
496
+ "una chiffoniera",
497
+ "una suoneria",
498
+ "una vetrina per porcellane",
499
+ "una calza di Natale",
500
+ "una chiesa",
501
+ "un cinema",
502
+ "una mannaia",
503
+ "una dimora sulla scogliera",
504
+ "un mantello",
505
+ "un intasamento",
506
+ "uno shaker da cocktail",
507
+ "una tazza da caff\u00e8",
508
+ "una caffettiera",
509
+ "una bobina",
510
+ "una serratura a combinazione",
511
+ "una tastiera di computer",
512
+ "una pasticceria",
513
+ "una nave container",
514
+ "una convertibile",
515
+ "un cavatappi",
516
+ "una cornetta",
517
+ "uno stivale da cowboy",
518
+ "un cappello da cowboy",
519
+ "una culla",
520
+ "una gru",
521
+ "un casco di protezione",
522
+ "una cassa",
523
+ "una culla",
524
+ "una pentola di coccio",
525
+ "una palla da croquet",
526
+ "una stampella",
527
+ "una corazza",
528
+ "una diga",
529
+ "una scrivania",
530
+ "un computer da tavolo",
531
+ "un telefono a selezione",
532
+ "un pannolino",
533
+ "un orologio digitale",
534
+ "un orologio digitale",
535
+ "un tavolo da pranzo",
536
+ "uno strofinaccio",
537
+ "una lavastoviglie",
538
+ "un freno a disco",
539
+ "un molo",
540
+ "una slitta trainata da cani",
541
+ "una cupola",
542
+ "uno zerbino",
543
+ "una piattaforma di perforazione",
544
+ "un tamburo",
545
+ "una bacchetta",
546
+ "un manubrio",
547
+ "un forno olandese",
548
+ "un ventilatore elettrico",
549
+ "una chitarra elettrica",
550
+ "una locomotiva elettrica",
551
+ "un centro di intrattenimento",
552
+ "una busta",
553
+ "una macchina per il caff\u00e8 espresso",
554
+ "una polvere per il viso",
555
+ "un boa di piume",
556
+ "un file",
557
+ "una barca antincendio",
558
+ "un'autopompa",
559
+ "uno schermo per il fuoco",
560
+ "un pennone",
561
+ "un flauto",
562
+ "una sedia pieghevole",
563
+ "un casco da calcio",
564
+ "un carrello elevatore",
565
+ "una fontana",
566
+ "una penna stilografica",
567
+ "un baldacchino",
568
+ "un vagone merci",
569
+ "un corno francese",
570
+ "una padella",
571
+ "una pelliccia",
572
+ "un camion della spazzatura",
573
+ "una maschera antigas",
574
+ "una pompa di benzina",
575
+ "un calice",
576
+ "un go-kart",
577
+ "una pallina da golf",
578
+ "un golfcart",
579
+ "una gondola",
580
+ "un gong",
581
+ "un abito",
582
+ "un pianoforte a coda",
583
+ "una serra",
584
+ "una griglia",
585
+ "un negozio di alimentari",
586
+ "una ghigliottina",
587
+ "uno scivolo per capelli",
588
+ "una lacca per capelli",
589
+ "una mezza traccia",
590
+ "un martello",
591
+ "un cesto regalo",
592
+ "un soffiatore a mano",
593
+ "un computer portatile",
594
+ "un fazzoletto",
595
+ "un disco rigido",
596
+ "un'armonica",
597
+ "un'arpa",
598
+ "una mietitrice",
599
+ "un'accetta",
600
+ "una fondina",
601
+ "un home theater",
602
+ "un nido d'ape",
603
+ "un gancio",
604
+ "una gonna a cerchio",
605
+ "una barra orizzontale",
606
+ "un carro di cavalli",
607
+ "una clessidra",
608
+ "un iPod",
609
+ "un ferro da stiro",
610
+ "una zucca",
611
+ "un jeans",
612
+ "una jeep",
613
+ "una maglia",
614
+ "un puzzle",
615
+ "a jinrikisha",
616
+ "un joystick",
617
+ "un kimono",
618
+ "una ginocchiera",
619
+ "un nodo",
620
+ "un camice da laboratorio",
621
+ "un mestolo",
622
+ "un paralume",
623
+ "un computer portatile",
624
+ "un tosaerba",
625
+ "un copriobiettivo",
626
+ "un tagliacarte",
627
+ "una biblioteca",
628
+ "una scialuppa di salvataggio",
629
+ "un accendino",
630
+ "una limousine",
631
+ "una fodera",
632
+ "un rossetto",
633
+ "un mocassino",
634
+ "una lozione",
635
+ "un altoparlante",
636
+ "una lente d'ingrandimento",
637
+ "una segheria",
638
+ "una bussola magnetica",
639
+ "una borsa della posta",
640
+ "una cassetta postale",
641
+ "un maillot",
642
+ "un maillot",
643
+ "un tombino",
644
+ "una maraca",
645
+ "una marimba",
646
+ "una maschera",
647
+ "un fiammifero",
648
+ "un palo di maggio",
649
+ "un labirinto",
650
+ "un misurino",
651
+ "una cassetta dei medicinali",
652
+ "un megalite",
653
+ "un microfono",
654
+ "un microonde",
655
+ "un'uniforme militare",
656
+ "una lattina di latte",
657
+ "un minibus",
658
+ "una minigonna",
659
+ "un minivan",
660
+ "un missile",
661
+ "un guanto",
662
+ "una ciotola di miscelazione",
663
+ "una casa mobile",
664
+ "un Modello T",
665
+ "un modem",
666
+ "un monastero",
667
+ "un monitor",
668
+ "un ciclomotore",
669
+ "un mortaio",
670
+ "una mortarboard",
671
+ "una moschea",
672
+ "una zanzariera",
673
+ "uno scooter",
674
+ "una bicicletta di montagna",
675
+ "una tenda di montagna",
676
+ "un topo",
677
+ "una trappola per topi",
678
+ "un furgone per traslochi",
679
+ "una museruola",
680
+ "un chiodo",
681
+ "un tutore per il collo",
682
+ "una collana",
683
+ "un capezzolo",
684
+ "un quaderno",
685
+ "un obelisco",
686
+ "un oboe",
687
+ "un'ocarina",
688
+ "un contachilometri",
689
+ "un filtro dell'olio",
690
+ "un organo",
691
+ "un oscilloscopio",
692
+ "una sopragonna",
693
+ "un carro da buoi",
694
+ "una maschera di ossigeno",
695
+ "un pacchetto",
696
+ "una pagaia",
697
+ "una ruota a pale",
698
+ "un lucchetto",
699
+ "un pennello",
700
+ "un pigiama",
701
+ "un palazzo",
702
+ "una panpipe",
703
+ "un tovagliolo di carta",
704
+ "un paracadute",
705
+ "una barra parallela",
706
+ "una panchina del parco",
707
+ "un parchimetro",
708
+ "un'autovettura",
709
+ "un patio",
710
+ "un telefono a pagamento",
711
+ "un piedistallo",
712
+ "una scatola di matite",
713
+ "un temperamatite",
714
+ "un profumo",
715
+ "una capsula di Petri",
716
+ "una fotocopiatrice",
717
+ "un grimaldello",
718
+ "un picconatore",
719
+ "una staccionata",
720
+ "un prelievo",
721
+ "un molo",
722
+ "un salvadanaio",
723
+ "una bottiglia di pillole",
724
+ "un cuscino",
725
+ "una pallina da ping-pong",
726
+ "una girandola",
727
+ "un pirata",
728
+ "un lanciatore",
729
+ "un aereo",
730
+ "un planetario",
731
+ "un sacchetto di plastica",
732
+ "un portapiatti",
733
+ "un aratro",
734
+ "uno stantuffo",
735
+ "una macchina fotografica Polaroid",
736
+ "un palo",
737
+ "un furgone della polizia",
738
+ "un poncho",
739
+ "un tavolo da biliardo",
740
+ "una bottiglia pop",
741
+ "una pentola",
742
+ "un tornio da vasaio",
743
+ "un trapano elettrico",
744
+ "un tappeto di preghiera",
745
+ "una stampante",
746
+ "una prigione",
747
+ "un proiettile",
748
+ "un proiettore",
749
+ "un disco",
750
+ "un sacco da boxe",
751
+ "una borsa",
752
+ "una penna d'oca",
753
+ "una trapunta",
754
+ "un corridore",
755
+ "una racchetta",
756
+ "un radiatore",
757
+ "una radio",
758
+ "un radiotelescopio",
759
+ "un barile per la pioggia",
760
+ "un veicolo ricreativo",
761
+ "una bobina",
762
+ "una macchina fotografica reflex",
763
+ "un frigorifero",
764
+ "un telecomando",
765
+ "un ristorante",
766
+ "un revolver",
767
+ "un fucile",
768
+ "una sedia a dondolo",
769
+ "un girarrosto",
770
+ "una gomma da cancellare",
771
+ "un pallone da rugby",
772
+ "una regola",
773
+ "una scarpa da corsa",
774
+ "una cassaforte",
775
+ "una spilla da balia",
776
+ "una saliera",
777
+ "un sandalo",
778
+ "un sarong",
779
+ "un sassofono",
780
+ "un fodero",
781
+ "una scala",
782
+ "uno scuolabus",
783
+ "una goletta",
784
+ "un tabellone segnapunti",
785
+ "uno schermo",
786
+ "una vite",
787
+ "un cacciavite",
788
+ "una cintura di sicurezza",
789
+ "una macchina da cucire",
790
+ "uno scudo",
791
+ "un negozio di scarpe",
792
+ "uno shoji",
793
+ "un cestino della spesa",
794
+ "un carrello della spesa",
795
+ "una pala",
796
+ "una cuffia da doccia",
797
+ "una tenda da doccia",
798
+ "uno sci",
799
+ "un passamontagna",
800
+ "un sacco a pelo",
801
+ "un regolo calcolatore",
802
+ "una porta scorrevole",
803
+ "una fessura",
804
+ "un boccaglio",
805
+ "una motoslitta",
806
+ "uno spazzaneve",
807
+ "un distributore di sapone",
808
+ "un pallone da calcio",
809
+ "un calzino",
810
+ "un piatto solare",
811
+ "un sombrero",
812
+ "una ciotola per la zuppa",
813
+ "una barra spaziatrice",
814
+ "una stufa per ambienti",
815
+ "una navetta spaziale",
816
+ "una spatola",
817
+ "un motoscafo",
818
+ "una ragnatela",
819
+ "un mandrino",
820
+ "un'auto sportiva",
821
+ "un riflettore",
822
+ "una fase",
823
+ "una locomotiva a vapore",
824
+ "un ponte ad arco in acciaio",
825
+ "un tamburo d'acciaio",
826
+ "uno stetoscopio",
827
+ "una stola",
828
+ "un muro di pietra",
829
+ "un cronometro",
830
+ "una stufa",
831
+ "un colino",
832
+ "un tram",
833
+ "una barella",
834
+ "un divano da studio",
835
+ "uno stupa",
836
+ "un sottomarino",
837
+ "un vestito",
838
+ "una meridiana",
839
+ "un occhiale da sole",
840
+ "occhiali da sole",
841
+ "una protezione solare",
842
+ "un ponte sospeso",
843
+ "un tampone",
844
+ "una felpa",
845
+ "un costume da bagno",
846
+ "un'altalena",
847
+ "un interruttore",
848
+ "una siringa",
849
+ "una lampada da tavolo",
850
+ "un carro armato",
851
+ "un lettore di nastri",
852
+ "una teiera",
853
+ "un orsacchiotto",
854
+ "una televisione",
855
+ "una palla da tennis",
856
+ "una paglia",
857
+ "un sipario teatrale",
858
+ "un ditale",
859
+ "una trebbiatrice",
860
+ "un trono",
861
+ "un tetto di tegole",
862
+ "un tostapane",
863
+ "un negozio di tabacco",
864
+ "un sedile del water",
865
+ "una torcia",
866
+ "un totem",
867
+ "un carro attrezzi",
868
+ "un negozio di giocattoli",
869
+ "un trattore",
870
+ "un camion con rimorchio",
871
+ "un vassoio",
872
+ "un trench",
873
+ "un triciclo",
874
+ "un trimarano",
875
+ "un treppiede",
876
+ "un arco di trionfo",
877
+ "un filobus",
878
+ "un trombone",
879
+ "una vasca da bagno",
880
+ "un tornello",
881
+ "una tastiera per macchina da scrivere",
882
+ "un ombrello",
883
+ "un monociclo",
884
+ "un montante",
885
+ "un vuoto",
886
+ "un vaso",
887
+ "una volta",
888
+ "un velluto",
889
+ "un distributore automatico",
890
+ "un paramento",
891
+ "un viadotto",
892
+ "un violino",
893
+ "una pallavolo",
894
+ "una piastra per cialde",
895
+ "un orologio da parete",
896
+ "un portafoglio",
897
+ "un armadio",
898
+ "un aereo da guerra",
899
+ "un lavandino",
900
+ "una rondella",
901
+ "una bottiglia d'acqua",
902
+ "una brocca d'acqua",
903
+ "una torre d'acqua",
904
+ "una brocca di whisky",
905
+ "un fischio",
906
+ "una parrucca",
907
+ "uno schermo per finestre",
908
+ "una tenda per finestre",
909
+ "una cravatta Windsor",
910
+ "una bottiglia di vino",
911
+ "un'ala",
912
+ "un wok",
913
+ "un cucchiaio di legno",
914
+ "una lana",
915
+ "un recinto di vermi",
916
+ "un relitto",
917
+ "uno yawl",
918
+ "una yurta",
919
+ "un sito web",
920
+ "un fumetto",
921
+ "un cruciverba",
922
+ "un cartello stradale",
923
+ "un semaforo",
924
+ "una giacca del libro",
925
+ "un menu",
926
+ "un piatto",
927
+ "un guacamole",
928
+ "un consomme",
929
+ "una pentola calda",
930
+ "un'inezia",
931
+ "un gelato",
932
+ "un ghiacciolo",
933
+ "una pagnotta francese",
934
+ "un bagel",
935
+ "un pretzel",
936
+ "un cheeseburger",
937
+ "un hotdog",
938
+ "un pur\u00e8 di patate",
939
+ "una testa di cavolo",
940
+ "un broccolo",
941
+ "un cavolfiore",
942
+ "una zucchina",
943
+ "una zucca per spaghetti",
944
+ "una zucca",
945
+ "una zucca butternut",
946
+ "un cetriolo",
947
+ "un carciofo",
948
+ "un peperone",
949
+ "un cardo",
950
+ "un fungo",
951
+ "una Granny Smith",
952
+ "una fragola",
953
+ "un'arancia",
954
+ "un limone",
955
+ "un fico",
956
+ "un ananas",
957
+ "una banana",
958
+ "un jackfruit",
959
+ "una mela custard",
960
+ "un melograno",
961
+ "un fieno",
962
+ "una carbonara",
963
+ "una salsa al cioccolato",
964
+ "un impasto",
965
+ "un polpettone",
966
+ "una pizza",
967
+ "una torta salata",
968
+ "un burrito",
969
+ "un vino rosso",
970
+ "un espresso",
971
+ "una tazza",
972
+ "uno zabaione",
973
+ "a alpe",
974
+ "una bolla",
975
+ "una scogliera",
976
+ "una barriera corallina",
977
+ "un geyser",
978
+ "un lago",
979
+ "un promontorio",
980
+ "un banco di sabbia",
981
+ "una riva del mare",
982
+ "una valle",
983
+ "un vulcano",
984
+ "un giocatore di pallone",
985
+ "uno sposo",
986
+ "un subacqueo",
987
+ "un seme di colza",
988
+ "una margherita",
989
+ "una pantofola gialla da donna",
990
+ "un mais",
991
+ "una ghianda",
992
+ "un'anca",
993
+ "un buckeye",
994
+ "un fungo corallino",
995
+ "un agarico",
996
+ "un gyromitra",
997
+ "una spina dorsale",
998
+ "una stella di terra",
999
+ "una gallina dei boschi",
1000
+ "un boleto",
1001
+ "un orecchio",
1002
+ "una carta igienica"
1003
+ ]
1004
+ }
CLIP_benchmark/clip_benchmark/datasets/it_zeroshot_classification_templates.json ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "imagenet1k": [
3
+ "una brutta foto di {c}",
4
+ "una scultura di {c}",
5
+ "una foto di {c} difficilmente visibile",
6
+ "una foto a bassa risoluzione di {c}",
7
+ "un rendering di {c}",
8
+ "graffiti di {c}",
9
+ "una pessima foto di {c}",
10
+ "una foto ritagliata di {c}",
11
+ "un tatuaggio di {c}",
12
+ "{c} ricamato",
13
+ "{c} ricamata",
14
+ "una foto luminosa di {c}",
15
+ "una foto di {c} pulito",
16
+ "una foto di {c} pulita",
17
+ "una foto di {c} sporco",
18
+ "una foto di {c} sporca",
19
+ "una foto di {c}\u00a0carino",
20
+ "una foto di {c} carina",
21
+ "una foto di {c} strano",
22
+ "una foto di {c} strana",
23
+ "una foto di {c} piccolo",
24
+ "una foto di {c} piccola",
25
+ "una foto di {c} largo",
26
+ "una foto di {c} larga",
27
+ "una foto di {c} grande",
28
+ "una foto scura di {c}",
29
+ "un disegno di {c}",
30
+ "{c} di plastica",
31
+ "una foto del {c} bella",
32
+ "una foto ravvicinata di {c}",
33
+ "una foto in bianco e nero di {c}",
34
+ "un dipinto di {c}",
35
+ "una foto sgranata di {c}",
36
+ "una foto ritagliata di {c}",
37
+ "una foto sfocata di {c}",
38
+ "una buona foto di {c}",
39
+ "una riproduzione di {c}",
40
+ "un rendering di {c}",
41
+ "{c} in un video gioco",
42
+ "uno scarabocchio di {c}",
43
+ "un origami di {c}",
44
+ "uno sketch di {c}",
45
+ "una bozza di {c}",
46
+ "una foto a bassa risoluzione di {c}",
47
+ "un giocattolo di {c}",
48
+ "una resa di {c}",
49
+ "{c} come cartone animato",
50
+ "un'opera di {c}",
51
+ "un peluche di {c}"
52
+ ]
53
+ }
CLIP_benchmark/clip_benchmark/datasets/jp_classnames.json ADDED
@@ -0,0 +1,1004 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "imagenet1k": [
3
+ "\u30c6\u30f3\u30c1",
4
+ "\u91d1\u9b5a",
5
+ "\u30db\u30db\u30b8\u30ed\u30b6\u30e1",
6
+ "\u30a4\u30bf\u30c1\u30b6\u30e1",
7
+ "\u30cf\u30f3\u30de\u30fc\u30d8\u30c3\u30c9",
8
+ "\u30b7\u30d3\u30ec\u30a8\u30a4",
9
+ "\u30a2\u30ab\u30a8\u30a4",
10
+ "\u30b3\u30c3\u30af",
11
+ "\u3081\u3093\u3069\u308a",
12
+ "\u30c0\u30c1\u30e7\u30a6",
13
+ "\u30a2\u30c8\u30ea",
14
+ "\u30b4\u30b7\u30ad\u30d2\u30ef",
15
+ "\u30cf\u30a6\u30b9\u30d5\u30a3\u30f3\u30c1",
16
+ "\u30e6\u30ad\u30d2\u30e1\u30c9\u30ea",
17
+ "\u30a4\u30f3\u30c7\u30a3\u30b4\u30db\u30aa\u30b8\u30ed",
18
+ "\u30ed\u30d3\u30f3",
19
+ "\u30d6\u30eb\u30d6\u30eb",
20
+ "\u30ab\u30b1\u30b9",
21
+ "\u30ab\u30b5\u30b5\u30ae",
22
+ "\u56db\u5341\u96c0",
23
+ "\u6c34\u30af\u30ed\u30a6\u30bf\u30c9\u30ea",
24
+ "\u51e7",
25
+ "\u767d\u982d\u30ef\u30b7",
26
+ "\u30cf\u30b2\u30ef\u30b7",
27
+ "\u30ab\u30e9\u30d5\u30c8\u30d5\u30af\u30ed\u30a6",
28
+ "\u6b27\u5dde\u30d5\u30a1\u30a4\u30a2\u30b5\u30e9\u30de\u30f3\u30c0\u30fc",
29
+ "\u5171\u901a\u30a4\u30e2\u30ea",
30
+ "\u30a4\u30e2\u30ea",
31
+ "\u30b5\u30f3\u30b7\u30e7\u30a6\u30a6\u30aa\u3092\u767a\u898b",
32
+ "\u30a2\u30db\u30ed\u30fc\u30c8\u30eb",
33
+ "\u30a6\u30b7\u30ac\u30a8\u30eb",
34
+ "\u30a2\u30de\u30ac\u30a8\u30eb",
35
+ "\u3064\u304b\u308c\u305f\u30ab\u30a8\u30eb",
36
+ "\u3068\u3093\u3061\u304d",
37
+ "\u30aa\u30b5\u30ac\u30e1",
38
+ "\u9f08",
39
+ "\u30c6\u30e9\u30d4\u30f3",
40
+ "\u30cf\u30b3\u30ac\u30e1",
41
+ "\u7e1e\u6a21\u69d8\u306e\u30e4\u30e2\u30ea",
42
+ "\u5171\u901a\u30a4\u30b0\u30a2\u30ca",
43
+ "\u30a2\u30e1\u30ea\u30ab\u30f3\u30ab\u30e1\u30ec\u30aa\u30f3",
44
+ "\u30a6\u30a3\u30c3\u30da\u30a4\u30eb",
45
+ "\u30a2\u30ac\u30de\u30c8\u30ab\u30b2",
46
+ "\u30d5\u30ea\u30eb\u30c8\u30ab\u30b2",
47
+ "\u30a2\u30ea\u30b2\u30fc\u30bf\u30fc\u30c8\u30ab\u30b2",
48
+ "\u30a2\u30e1\u30ea\u30ab\u30c9\u30af\u30c8\u30ab\u30b2",
49
+ "\u7dd1\u306e\u30c8\u30ab\u30b2",
50
+ "\u30a2\u30d5\u30ea\u30ab\u306e\u30ab\u30e1\u30ec\u30aa\u30f3",
51
+ "\u30b3\u30e2\u30c9\u30c9\u30e9\u30b4\u30f3",
52
+ "\u30a2\u30d5\u30ea\u30ab\u306e\u30ef\u30cb",
53
+ "\u30a2\u30e1\u30ea\u30ab\u30ef\u30cb",
54
+ "\u30c8\u30ea\u30b1\u30e9\u30c8\u30d7\u30b9",
55
+ "\u96f7\u306e\u30d8\u30d3",
56
+ "\u30ea\u30f3\u30b0\u30cd\u30c3\u30af\u30b9\u30cd\u30fc\u30af",
57
+ "\u30db\u30fc\u30ce\u30fc\u30b9\u30d8\u30d3",
58
+ "\u7dd1\u306e\u30d8\u30d3",
59
+ "\u30ad\u30f3\u30b0\u30b9\u30cd\u30fc\u30af",
60
+ "\u30ac\u30fc\u30bf\u30fc\u30b9\u30cd\u30fc\u30af",
61
+ "\u6c34\u86c7",
62
+ "\u3064\u308b\u30d8\u30d3",
63
+ "\u591c\u306e\u30d8\u30d3",
64
+ "\u30dc\u30a2\u30fb\u30b3\u30f3\u30b9\u30c8\u30ea\u30af\u30bf\u30fc",
65
+ "\u30ed\u30c3\u30af\u30d1\u30a4\u30bd\u30f3",
66
+ "\u30a4\u30f3\u30c9\u30b3\u30d6\u30e9",
67
+ "\u30b0\u30ea\u30fc\u30f3\u30de\u30f3\u30d0",
68
+ "\u30a6\u30df\u30d8\u30d3",
69
+ "\u30c4\u30ce\u30af\u30b5\u30ea\u30d8\u30d3",
70
+ "\u30c0\u30a4\u30e4",
71
+ "\u30b5\u30a4\u30c9\u30ef\u30a4\u30f3\u30c0\u30fc",
72
+ "\u4e09\u8449\u866b",
73
+ "\u5208\u308a\u5165\u308c\u4f5c\u696d\u8005",
74
+ "\u30b5\u30bd\u30ea",
75
+ "\u9ed2\u3068\u91d1\u306e\u5ead\u30af\u30e2",
76
+ "\u7d0d\u5c4b\u30af\u30e2",
77
+ "\u5ead\u30af\u30e2",
78
+ "\u30af\u30ed\u30b4\u30b1\u30b0\u30e2",
79
+ "\u30bf\u30e9\u30f3\u30c1\u30e5\u30e9",
80
+ "\u30aa\u30aa\u30ab\u30df\u306e\u30af\u30e2",
81
+ "\u30c0\u30cb",
82
+ "\u767e\u8db3",
83
+ "\u30af\u30ed\u30e9\u30a4\u30c1\u30e7\u30a6",
84
+ "\u96f7\u9ce5",
85
+ "\u3072\u3060\u3048\u308a\u306e\u4ed8\u3044\u305f\u30e9\u30a4\u30c1\u30e7\u30a6",
86
+ "\u8349\u539f\u30c1\u30ad\u30f3",
87
+ "\u5b54\u96c0",
88
+ "\u30a6\u30ba\u30e9",
89
+ "\u30e4\u30de\u30a6\u30ba\u30e9",
90
+ "\u30a2\u30d5\u30ea\u30ab\u306e\u7070\u8272",
91
+ "\u30b3\u30f3\u30b4\u30a6\u30a4\u30f3\u30b3",
92
+ "\u786b\u9ec4\u30c8\u30ad\u30aa\u30a6\u30e0",
93
+ "\u30a4\u30f3\u30b3",
94
+ "\u30d0\u30f3\u30b1\u30f3",
95
+ "\u8702\u98df\u3079\u308b\u4eba",
96
+ "\u30b5\u30a4\u30c1\u30e7\u30a6",
97
+ "\u30cf\u30c1\u30c9\u30ea",
98
+ "\u9310\u5634",
99
+ "\u30aa\u30aa\u30cf\u30b7",
100
+ "\u30c9\u30ec\u30a4\u30af",
101
+ "\u8d64\u30d6\u30ec\u30b9\u30c8\u30a2\u30a4\u30b5\u5c5e\u306e\u30ac\u30e2",
102
+ "\u30ac\u30c1\u30e7\u30a6",
103
+ "\u9ed2\u3044\u767d\u9ce5",
104
+ "\u30bf\u30b9\u30ab\u30fc\u30d3\u30fc\u30eb",
105
+ "\u30cf\u30ea\u30e2\u30b0\u30e9",
106
+ "\u30ab\u30e2\u30ce\u30cf\u30b7",
107
+ "\u30ef\u30e9\u30d3\u30fc",
108
+ "\u30b3\u30a2\u30e9",
109
+ "\u30a6\u30a9\u30f3\u30d0\u30c3\u30c8",
110
+ "\u30af\u30e9\u30b2",
111
+ "\u30a4\u30bd\u30ae\u30f3\u30c1\u30e3\u30af",
112
+ "\u8133\u30b5\u30f3\u30b4",
113
+ "\u6241\u5f62\u52d5\u7269",
114
+ "\u7dda\u866b",
115
+ "\u5dfb\u304d\u8c9d",
116
+ "\u30ab\u30bf\u30c4\u30e0\u30ea",
117
+ "\u30ca\u30e1\u30af\u30b8",
118
+ "\u30a6\u30df\u30a6\u30b7",
119
+ "\u30ad\u30c8\u30f3",
120
+ "\u30aa\u30a6\u30e0\u30ac\u30a4",
121
+ "\u30a2\u30e1\u30ea\u30ab\u30a4\u30c1\u30e7\u30a6\u30ac\u30cb",
122
+ "\u5ca9\u30ab\u30cb",
123
+ "\u30b7\u30aa\u30de\u30cd\u30ad",
124
+ "\u30bf\u30e9\u30d0\u30ac\u30cb",
125
+ "\u30a2\u30e1\u30ea\u30ab\u30f3\u30ed\u30d6\u30b9\u30bf\u30fc",
126
+ "\u4f0a\u52e2\u30a8\u30d3",
127
+ "\u30b6\u30ea\u30ac\u30cb",
128
+ "\u30e4\u30c9\u30ab\u30ea",
129
+ "\u7b49\u811a\u985e",
130
+ "\u30b3\u30a6\u30ce\u30c8\u30ea",
131
+ "\u30ca\u30d9\u30b3\u30a6",
132
+ "\u30d8\u30e9\u30b5\u30ae",
133
+ "\u30d5\u30e9\u30df\u30f3\u30b4",
134
+ "\u5c0f\u3055\u306a\u9752\u3044\u30b5\u30ae",
135
+ "\u30a2\u30e1\u30ea\u30ab\u30f3\u767d\u9dfa",
136
+ "\u306b\u304c\u308a",
137
+ "\u30af\u30ec\u30fc\u30f3",
138
+ "\u30c4\u30eb\u30e2\u30c9\u30ad\u79d1\u306e\u9ce5",
139
+ "\u30e8\u30fc\u30ed\u30d4\u30a2\u30f3\u6c34\u9ce5",
140
+ "\u30a2\u30e1\u30ea\u30ab\u30aa\u30aa\u30d0\u30f3",
141
+ "\u30ce\u30ac\u30f3",
142
+ "\u30ad\u30e7\u30a6\u30b8\u30e7\u30b7\u30ae",
143
+ "\u8d64\u62c5\u4fdd\u30b7\u30ae",
144
+ "\u30a2\u30ab\u30a2\u30b7\u30b7\u30ae",
145
+ "\u30aa\u30aa\u30cf\u30b7\u30b7\u30ae",
146
+ "\u30df\u30e4\u30b3\u30c9\u30ea",
147
+ "\u30da\u30ea\u30ab\u30f3",
148
+ "\u30ad\u30f3\u30b0\u30da\u30f3\u30ae\u30f3",
149
+ "\u30a2\u30eb\u30d0\u30c8\u30ed\u30b9",
150
+ "\u30b3\u30af\u30af\u30b8\u30e9",
151
+ "\u30b7\u30e3\u30c1",
152
+ "\u30b8\u30e5\u30b4\u30f3",
153
+ "\u30a2\u30b7\u30ab",
154
+ "\u30c1\u30ef\u30ef",
155
+ "\u72c6",
156
+ "\u30de\u30eb\u30c1\u30fc\u30ba\u72ac",
157
+ "\u72c6",
158
+ "\u30b7\u30fc\u30ba\u30fc\u3001\u30b7\u30fc\u30ba\u30fc",
159
+ "\u30d6\u30ec\u30ca\u30e0\u30b9\u30d1\u30cb\u30a8\u30eb",
160
+ "\u30d1\u30d4\u30e8\u30f3",
161
+ "\u30c8\u30a4\u30c6\u30ea\u30a2",
162
+ "\u30ed\u30fc\u30c7\u30b7\u30a2\u30f3\u30fb\u30ea\u30c3\u30b8\u30d0\u30c3\u30af",
163
+ "\u30a2\u30d5\u30ac\u30f3\u30cf\u30a6\u30f3\u30c9",
164
+ "\u30d0\u30bb\u30c3\u30c8\u72ac",
165
+ "\u30d3\u30fc\u30b0\u30eb",
166
+ "\u30d6\u30e9\u30c3\u30c9\u30cf\u30a6\u30f3\u30c9",
167
+ "\u30d6\u30eb\u30fc\u30c6\u30a3\u30c3\u30af",
168
+ "\u9ed2\u3068\u9ec4\u8910\u8272\u306e\u731f\u72ac",
169
+ "\u30a6\u30a9\u30fc\u30ab\u30fc\u30cf\u30a6\u30f3\u30c9",
170
+ "\u30a4\u30f3\u30b0\u30ea\u30c3\u30b7\u30e5\u30d5\u30a9\u30c3\u30af\u30b9\u30cf\u30a6\u30f3\u30c9",
171
+ "\u30ec\u30c3\u30c9\u30dc\u30fc\u30f3",
172
+ "\u30dc\u30eb\u30be\u30a4",
173
+ "\u30a2\u30a4\u30ea\u30c3\u30b7\u30e5\u30fb\u30a6\u30eb\u30d5\u30cf\u30a6\u30f3\u30c9",
174
+ "\u30a4\u30bf\u30ea\u30a2\u30f3\u30b0\u30ec\u30fc\u30cf\u30a6\u30f3\u30c9",
175
+ "\u30a6\u30a3\u30da\u30c3\u30c8",
176
+ "\u30a4\u30d3\u30b5\u30cf\u30a6\u30f3\u30c9",
177
+ "\u30ce\u30eb\u30a6\u30a7\u30fc\u30a8\u30eb\u30af\u30cf\u30a6\u30f3\u30c9",
178
+ "\u30aa\u30c3\u30bf\u30fc\u30cf\u30a6\u30f3\u30c9",
179
+ "\u30b5\u30eb\u30fc\u30ad",
180
+ "\u30b9\u30b3\u30c6\u30a3\u30c3\u30b7\u30e5\u30fb\u30c7\u30a3\u30a2\u30cf\u30a6\u30f3\u30c9",
181
+ "\u30ef\u30a4\u30de\u30e9\u30ca\u30fc",
182
+ "\u30b9\u30bf\u30d5\u30a9\u30fc\u30c9\u30b7\u30e3\u30fc\u30d6\u30eb\u30c6\u30ea\u30a2",
183
+ "\u30a2\u30e1\u30ea\u30ab\u30f3\u30fb\u30b9\u30bf\u30c3\u30d5\u30a9\u30fc\u30c9\u30b7\u30e3\u30fc\u30fb\u30c6\u30ea\u30a2",
184
+ "\u30d9\u30c9\u30ea\u30f3\u30c8\u30f3\u30c6\u30ea\u30a2",
185
+ "\u30dc\u30fc\u30c0\u30fc\u30c6\u30ea\u30a2",
186
+ "\u30b1\u30ea\u30fc\u30d6\u30eb\u30fc\u30c6\u30ea\u30a2",
187
+ "\u30a2\u30a4\u30ea\u30c3\u30b7\u30e5\u30c6\u30ea\u30a2",
188
+ "\u30ce\u30fc\u30d5\u30a9\u30fc\u30af\u30c6\u30ea\u30a2",
189
+ "\u30ce\u30fc\u30ea\u30c3\u30c1\u30fb\u30c6\u30ea\u30a2",
190
+ "\u30e8\u30fc\u30af\u30b7\u30e3\u30fc\u30c6\u30ea\u30a2",
191
+ "\u30ef\u30a4\u30e4\u30fc\u30d8\u30a2\u30fc\u30fb\u30d5\u30a9\u30c3\u30af\u30b9\u30c6\u30ea\u30a2",
192
+ "\u30ec\u30fc\u30af\u30e9\u30f3\u30c9\u30c6\u30ea\u30a2",
193
+ "\u30b7\u30fc\u30ea\u30fc\u30cf\u30e0\u30c6\u30ea\u30a2",
194
+ "\u30a8\u30a2\u30c7\u30fc\u30eb",
195
+ "\u30b1\u30eb\u30f3",
196
+ "\u30aa\u30fc\u30b9\u30c8\u30e9\u30ea\u30a2\u30c6\u30ea\u30a2",
197
+ "\u30c0\u30f3\u30c7\u30a3\u30c7\u30a3\u30f3\u30e2\u30f3\u30c8\u30c6\u30ea\u30a2",
198
+ "\u30dc\u30b9\u30c8\u30f3\u30d6\u30eb",
199
+ "\u30df\u30cb\u30c1\u30e5\u30a2\u30b7\u30e5\u30ca\u30a6\u30b6\u30fc",
200
+ "\u30b8\u30e3\u30a4\u30a2\u30f3\u30c8\u30b7\u30e5\u30ca\u30a6\u30b6\u30fc",
201
+ "\u30b9\u30bf\u30f3\u30c0\u30fc\u30c9\u30b7\u30e5\u30ca\u30a6\u30b6\u30fc",
202
+ "\u30b9\u30b3\u30c3\u30c1\u30c6\u30ea\u30a2",
203
+ "\u30c1\u30d9\u30bf\u30f3\u30c6\u30ea\u30a2",
204
+ "\u30b7\u30eb\u30ad\u30fc\u30c6\u30ea\u30a2",
205
+ "\u30bd\u30d5\u30c8\u30b3\u30fc\u30c6\u30c3\u30c9\u30fb\u30a6\u30a3\u30fc\u30c8\u30f3\u30fb\u30c6\u30ea\u30a2",
206
+ "\u30a6\u30a7\u30b9\u30c8\u30cf\u30a4\u30e9\u30f3\u30c9\u30db\u30ef\u30a4\u30c8\u30c6\u30ea\u30a2",
207
+ "\u30e9\u30b5",
208
+ "\u30d5\u30e9\u30c3\u30c8\u30b3\u30fc\u30c6\u30c3\u30c9\u30fb\u30ec\u30c8\u30ea\u30fc\u30d0\u30fc",
209
+ "\u30ab\u30fc\u30ea\u30fc\u30b3\u30fc\u30c6\u30a3\u30f3\u30b0\u3055\u308c\u305f\u30ec\u30c8\u30ea\u30fc\u30d0\u30fc",
210
+ "\u30b4\u30fc\u30eb\u30c7\u30f3\u30ec\u30c8\u30ea\u30d0\u30fc",
211
+ "\u30e9\u30d6\u30e9\u30c9\u30eb\u30fb\u30ec\u30c8\u30ea\u30fc\u30d0\u30fc\u72ac",
212
+ "\u30c1\u30a7\u30b5\u30d4\u30fc\u30af\u6e7e\u30ec\u30c8\u30ea\u30fc\u30d0\u30fc",
213
+ "\u30b8\u30e3\u30fc\u30de\u30f3\u30fb\u30b7\u30e7\u30fc\u30c8\u30d8\u30a2\u30fb\u30dd\u30a4\u30f3\u30bf",
214
+ "\u30d3\u30ba\u30e9",
215
+ "\u30a4\u30f3\u30b0\u30ea\u30c3\u30b7\u30e5\u30bb\u30c3\u30bf\u30fc",
216
+ "\u30a2\u30a4\u30ea\u30c3\u30b7\u30e5\u30bb\u30c3\u30bf\u30fc",
217
+ "\u30b4\u30fc\u30c9\u30f3\u30bb\u30c3\u30bf\u30fc",
218
+ "\u30d6\u30ea\u30bf\u30cb\u30fc\u30b9\u30d1\u30cb\u30a8\u30eb",
219
+ "\u30af\u30e9\u30f3\u30d0\u30fc",
220
+ "\u30a4\u30f3\u30b0\u30ea\u30c3\u30b7\u30e5\u30b9\u30d7\u30ea\u30f3\u30ac\u30fc",
221
+ "\u30a6\u30a7\u30eb\u30b7\u30e5\u30b9\u30d7\u30ea\u30f3\u30ac\u30fc\u30b9\u30d1\u30cb\u30a8\u30eb",
222
+ "\u30b3\u30c3\u30ab\u30fc\u30b9\u30d1\u30cb\u30a8\u30eb",
223
+ "\u30b5\u30bb\u30c3\u30af\u30b9\u30b9\u30d1\u30cb\u30a8\u30eb",
224
+ "\u30a2\u30a4\u30eb\u30e9\u30f3\u30c9\u306e\u30a6\u30a9\u30fc\u30bf\u30fc\u30b9\u30d1\u30cb\u30a8\u30eb",
225
+ "\u30af\u30d0\u30fc\u30b9\u72ac",
226
+ "\u30b9\u30ad\u30c3\u30d1\u30fc\u30ad\u30fc",
227
+ "\u30d9\u30eb\u30b8\u30a2\u30f3\u30fb\u30b7\u30a7\u30d1\u30fc\u30c9\u30fb\u30c9\u30c3\u30b0\u30fb\u30b0\u30ed\u30fc\u30cd\u30f3\u30c0\u30fc\u30eb",
228
+ "\u30de\u30ea\u30ce\u30a2",
229
+ "\u30d6\u30ea\u30a2\u30fc\u30eb",
230
+ "\u30b1\u30eb\u30d4\u30fc",
231
+ "\u30b3\u30e2\u30f3\u30c9\u30fc\u30eb",
232
+ "\u30aa\u30fc\u30eb\u30c9\u30a4\u30f3\u30b0\u30ea\u30c3\u30b7\u30e5\u30b7\u30fc\u30d7\u30c9\u30c3\u30b0",
233
+ "\u30b7\u30a7\u30c8\u30e9\u30f3\u30c9\u30b7\u30fc\u30d7\u30c9\u30c3\u30b0",
234
+ "\u30b3\u30ea\u30fc",
235
+ "\u30dc\u30fc\u30c0\u30fc\u30b3\u30ea\u30fc",
236
+ "\u30d6\u30fc\u30f4\u30a3\u30a8\u30fb\u30c7\u30fb\u30d5\u30e9\u30f3\u30c9\u30eb",
237
+ "\u30ed\u30c3\u30c8\u30ef\u30a4\u30e9\u30fc",
238
+ "\u30b8\u30e3\u30fc\u30de\u30f3\u30b7\u30a7\u30d1\u30fc\u30c9",
239
+ "\u30c9\u30fc\u30d9\u30eb\u30de\u30f3\u72ac",
240
+ "\u30df\u30cb\u30c1\u30e5\u30a2\u30d4\u30f3\u30b7\u30e3\u30fc",
241
+ "\u30b0\u30ec\u30fc\u30bf\u30fc\u30b9\u30a4\u30b9\u30de\u30a6\u30f3\u30c6\u30f3\u30c9\u30c3\u30b0",
242
+ "\u30d0\u30fc\u30cd\u30fc\u30ba\u30de\u30a6\u30f3\u30c6\u30f3\u30c9\u30c3\u30b0",
243
+ "\u30a2\u30c3\u30da\u30f3\u30c4\u30a7\u30eb",
244
+ "\u30a8\u30f3\u30c8\u30ec\u30d6\u30c3\u30b7\u30e3\u30fc",
245
+ "\u30dc\u30af\u30b5\u30fc",
246
+ "\u30d6\u30eb\u30de\u30b9\u30c1\u30d5",
247
+ "\u30c1\u30d9\u30c3\u30c8\u30de\u30b9\u30c1\u30d5",
248
+ "\u30d5\u30ec\u30f3\u30c1\u30d6\u30eb\u30c9\u30c3\u30b0",
249
+ "\u30b0\u30ec\u30fc\u30c8\u30c7\u30fc\u30f3",
250
+ "\u30bb\u30f3\u30c8\u30d0\u30fc\u30ca\u30fc\u30c9",
251
+ "\u30a8\u30b9\u30ad\u30e2\u30fc\u72ac",
252
+ "\u30de\u30e9\u30df\u30e5\u30fc\u30c8",
253
+ "\u30b7\u30d9\u30ea\u30a2\u30f3\u30cf\u30b9\u30ad\u30fc",
254
+ "\u30c0\u30eb\u30e1\u30b7\u30a2\u30f3",
255
+ "\u30a2\u30fc\u30d5\u30a7\u30f3\u30d4\u30f3\u30b7\u30e3\u30fc",
256
+ "\u30d0\u30bb\u30f3\u30b8\u30fc",
257
+ "\u30d1\u30b0",
258
+ "\u30ec\u30aa\u30f3\u30d0\u30fc\u30b0",
259
+ "\u30cb\u30e5\u30fc\u30d5\u30a1\u30f3\u30c9\u30e9\u30f3\u30c9\u5cf6",
260
+ "\u30b0\u30ec\u30fc\u30c8\u30d4\u30ec\u30cb\u30fc\u30ba",
261
+ "\u30b5\u30e2\u30a8\u30c9",
262
+ "\u30dd\u30e1\u30e9\u30cb\u30a2\u30f3",
263
+ "\u30c1\u30e3\u30a6",
264
+ "\u30ad\u30fc\u30b9\u30db\u30f3\u30c9",
265
+ "\u30d6\u30e9\u30d0\u30f3\u30bd\u30f3\u30b0\u30ea\u30d5\u30a9\u30f3",
266
+ "\u30da\u30f3\u30d6\u30ed\u30fc\u30af",
267
+ "\u30ab\u30fc\u30c7\u30a3\u30ac\u30f3",
268
+ "\u30c8\u30a4\u30d7\u30fc\u30c9\u30eb",
269
+ "\u30df\u30cb\u30c1\u30e5\u30a2\u30d7\u30fc\u30c9\u30eb",
270
+ "\u30b9\u30bf\u30f3\u30c0\u30fc\u30c9\u30d7\u30fc\u30c9\u30eb",
271
+ "\u30e1\u30ad\u30b7\u30ab\u30f3\u30fb\u30d8\u30a2\u30fc\u30ec\u30b9",
272
+ "\u30b7\u30f3\u30ea\u30f3\u30aa\u30aa\u30ab\u30df",
273
+ "\u767d\u3044\u30aa\u30aa\u30ab\u30df",
274
+ "\u30ec\u30c3\u30c9\u30a6\u30eb\u30d5",
275
+ "\u30b3\u30e8\u30fc\u30c6",
276
+ "\u30c7\u30a3\u30f3\u30b4",
277
+ "\u30c9\u30fc\u30eb",
278
+ "\u30ea\u30ab\u30aa\u30f3",
279
+ "\u30cf\u30a4\u30a8\u30ca",
280
+ "\u30a2\u30ab\u30ae\u30c4\u30cd",
281
+ "\u30ad\u30c3\u30c8\u30ad\u30c4\u30cd",
282
+ "\u30db\u30c3\u30ad\u30e7\u30af\u30ae\u30c4\u30cd",
283
+ "\u7070\u8272\u306e\u30ad\u30c4\u30cd",
284
+ "\u30bf\u30d3\u30fc",
285
+ "\u864e\u732b",
286
+ "\u30da\u30eb\u30b7\u30e3\u732b",
287
+ "\u30b7\u30e3\u30e0\u732b",
288
+ "\u30a8\u30b8\u30d7\u30c8\u306e\u732b",
289
+ "\u30af\u30fc\u30ac\u30fc",
290
+ "\u30aa\u30aa\u30e4\u30de\u30cd\u30b3",
291
+ "\u30d2\u30e7\u30a6",
292
+ "\u30e6\u30ad\u30d2\u30e7\u30a6",
293
+ "\u30b8\u30e3\u30ac\u30fc",
294
+ "\u30e9\u30a4\u30aa\u30f3",
295
+ "\u864e",
296
+ "\u30c1\u30fc\u30bf\u30fc",
297
+ "\u30d2\u30b0\u30de",
298
+ "\u30a2\u30e1\u30ea\u30ab\u30af\u30ed\u30af\u30de",
299
+ "\u6c37\u306e\u30af\u30de",
300
+ "\u30ca\u30de\u30b1\u30b0\u30de",
301
+ "\u30de\u30f3\u30b0\u30fc\u30b9",
302
+ "\u30df\u30fc\u30a2\u30ad\u30e3\u30c3\u30c8",
303
+ "\u30cf\u30f3\u30df\u30e7\u30a6",
304
+ "\u3066\u3093\u3068\u3046\u866b",
305
+ "\u30b0\u30e9\u30f3\u30c9\u30d3\u30fc\u30c8\u30eb",
306
+ "\u30ab\u30df\u30ad\u30ea\u30e0\u30b7",
307
+ "\u30cf\u30e0\u30b7",
308
+ "\u30d5\u30f3\u30b3\u30ed\u30ac\u30b7",
309
+ "\u30b5\u30a4\u30cf\u30e0\u30b7",
310
+ "\u30be\u30a6\u30e0\u30b7",
311
+ "\u30cf\u30a8",
312
+ "\u8702",
313
+ "\u87fb",
314
+ "\u30d0\u30c3\u30bf",
315
+ "\u30af\u30ea\u30b1\u30c3\u30c8",
316
+ "\u6756",
317
+ "\u30b4\u30ad\u30d6\u30ea",
318
+ "\u30ab\u30de\u30ad\u30ea",
319
+ "\u8749",
320
+ "\u30e8\u30b3\u30d0\u30a4",
321
+ "\u30af\u30b5\u30ab\u30b2\u30ed\u30a6",
322
+ "\u30c8\u30f3\u30dc",
323
+ "\u30a4\u30c8\u30c8\u30f3\u30dc",
324
+ "\u63d0\u7763",
325
+ "\u30ea\u30f3\u30b0\u30ec\u30c3\u30c8",
326
+ "\u541b\u4e3b",
327
+ "\u30e2\u30f3\u30b7\u30ed\u30c1\u30e7\u30a6",
328
+ "\u786b\u9ec4\u8776",
329
+ "\u30b7\u30b8\u30df\u30c1\u30e7\u30a6",
330
+ "\u30d2\u30c8\u30c7",
331
+ "\u3046\u306b",
332
+ "\u30ca\u30de\u30b3",
333
+ "\u6728\u306e\u30a6\u30b5\u30ae",
334
+ "\u91ce\u30a6\u30b5\u30ae",
335
+ "\u30a2\u30f3\u30b4\u30e9",
336
+ "\u30cf\u30e0\u30b9\u30bf\u30fc",
337
+ "\u30e4\u30de\u30a2\u30e9\u30b7",
338
+ "\u30ad\u30c4\u30cd\u30ea\u30b9",
339
+ "\u30de\u30fc\u30e2\u30c3\u30c8",
340
+ "\u30d3\u30fc\u30d0\u30fc",
341
+ "\u30e2\u30eb\u30e2\u30c3\u30c8",
342
+ "\u6817\u8272",
343
+ "\u30b7\u30de\u30a6\u30de",
344
+ "\u8c5a",
345
+ "\u30a4\u30ce\u30b7\u30b7",
346
+ "\u30a4\u30dc\u30a4\u30ce\u30b7\u30b7",
347
+ "\u30ab\u30d0",
348
+ "\u96c4\u725b",
349
+ "\u6c34\u725b",
350
+ "\u30d0\u30a4\u30bd\u30f3",
351
+ "\u30e9\u30e0",
352
+ "\u30d3\u30c3\u30b0\u30db\u30fc\u30f3",
353
+ "\u30a2\u30a4\u30d9\u30c3\u30af\u30b9",
354
+ "\u30cf\u30fc\u30c6\u30d3\u30fc\u30b9\u30c8",
355
+ "\u30a4\u30f3\u30d1\u30e9",
356
+ "\u30ac\u30bc\u30eb",
357
+ "\u30a2\u30e9\u30d3\u30a2\u30e9\u30af\u30c0",
358
+ "\u30e9\u30de",
359
+ "\u30a4\u30bf\u30c1",
360
+ "\u30df\u30f3\u30af",
361
+ "\u30b1\u30ca\u30ac\u30a4\u30bf\u30c1",
362
+ "\u30af\u30ed\u30a2\u30b7\u30a4\u30bf\u30c1",
363
+ "\u30ab\u30ef\u30a6\u30bd",
364
+ "\u30b9\u30ab\u30f3\u30af",
365
+ "\u72f8",
366
+ "\u30a2\u30eb\u30de\u30b8\u30ed",
367
+ "\u30df\u30e6\u30d3\u30ca\u30de\u30b1\u30e2\u30ce",
368
+ "\u30aa\u30e9\u30f3\u30a6\u30fc\u30bf\u30f3",
369
+ "\u30b4\u30ea\u30e9",
370
+ "\u30c1\u30f3\u30d1\u30f3\u30b8\u30fc",
371
+ "\u30c6\u30ca\u30ac\u30b6\u30eb",
372
+ "\u30d5\u30af\u30ed\u30c6\u30ca\u30ac\u30b6\u30eb",
373
+ "\u30aa\u30ca\u30ac\u30b6\u30eb",
374
+ "\u30d1\u30bf\u30b9",
375
+ "\u30d2\u30d2",
376
+ "\u30de\u30ab\u30af",
377
+ "\u30e4\u30bb\u30b6\u30eb",
378
+ "\u30b3\u30ed\u30d6\u30b9\u5c5e",
379
+ "\u30c6\u30f3\u30b0\u30b6\u30eb",
380
+ "\u30de\u30fc\u30e2\u30bb\u30c3\u30c8",
381
+ "\u30aa\u30de\u30ad\u30b6\u30eb",
382
+ "\u30db\u30a8\u30b6\u30eb",
383
+ "\u30c6\u30a3\u30c6\u30a3",
384
+ "\u30af\u30e2\u30b6\u30eb",
385
+ "\u30ea\u30b9\u30b6\u30eb",
386
+ "\u30de\u30c0\u30ac\u30b9\u30ab\u30eb\u732b",
387
+ "\u30a4\u30f3\u30c9\u30ea",
388
+ "\u30a4\u30f3\u30c9\u30be\u30a6",
389
+ "\u30a2\u30d5\u30ea\u30ab\u30be\u30a6",
390
+ "\u30ec\u30c3\u30b5\u30fc\u30d1\u30f3\u30c0",
391
+ "\u30b8\u30e3\u30a4\u30a2\u30f3\u30c8\u30d1\u30f3\u30c0",
392
+ "\u30d0\u30e9\u30af\u30fc\u30bf",
393
+ "\u30a6\u30ca\u30ae",
394
+ "\u30ae\u30f3\u30b6\u30b1",
395
+ "\u5ca9\u306e\u7f8e\u3057\u3055",
396
+ "\u30af\u30de\u30ce\u30df",
397
+ "\u30c1\u30e7\u30a6\u30b6\u30e1",
398
+ "\u30ac\u30fc",
399
+ "\u30df\u30ce\u30ab\u30b5\u30b4",
400
+ "\u30d5\u30b0",
401
+ "\u305d\u308d\u3070\u3093",
402
+ "\u30a2\u30d0\u30e4",
403
+ "\u30a2\u30ab\u30c7\u30df\u30c3\u30af\u30ac\u30a6\u30f3",
404
+ "\u30a2\u30b3\u30fc\u30c7\u30a3\u30aa\u30f3",
405
+ "\u30a2\u30b3\u30fc\u30b9\u30c6\u30a3\u30c3\u30af\u30ae\u30bf\u30fc",
406
+ "\u7a7a\u6bcd",
407
+ "\u65c5\u5ba2\u6a5f",
408
+ "\u98db\u884c\u8239",
409
+ "\u796d\u58c7",
410
+ "\u6551\u6025\u8eca",
411
+ "\u4e21\u751f\u985e",
412
+ "\u30a2\u30ca\u30ed\u30b0\u6642\u8a08",
413
+ "\u990a\u8702\u5834",
414
+ "\u30a8\u30d7\u30ed\u30f3",
415
+ "\u3054\u307f\u5165\u308c",
416
+ "\u30a2\u30b5\u30eb\u30c8\u30e9\u30a4\u30d5\u30eb",
417
+ "\u30d0\u30c3\u30af\u30d1\u30c3\u30af",
418
+ "\u30d9\u30fc\u30ab\u30ea\u30fc",
419
+ "\u5e73\u5747\u53f0",
420
+ "\u30d0\u30eb\u30fc\u30f3",
421
+ "\u30dc\u30fc\u30eb\u30da\u30f3",
422
+ "\u30d0\u30f3\u30c9\u30a8\u30a4\u30c9",
423
+ "\u30d0\u30f3\u30b8\u30e7\u30fc",
424
+ "\u30d0\u30cb\u30b9\u30bf\u30fc",
425
+ "\u30d0\u30fc\u30d9\u30eb",
426
+ "\u7406\u9aea\u5e97\u306e\u6905\u5b50",
427
+ "\u7406\u9aea\u5e97",
428
+ "\u7d0d\u5c4b",
429
+ "\u30d0\u30ed\u30e1\u30fc\u30bf\u30fc",
430
+ "\u30d0\u30ec\u30eb",
431
+ "\u30d0\u30ed\u30fc",
432
+ "\u91ce\u7403",
433
+ "\u30d0\u30b9\u30b1\u30c3\u30c8\u30dc\u30fc\u30eb",
434
+ "\u30d0\u30b7\u30cd\u30c3\u30c8",
435
+ "\u30d5\u30a1\u30b4\u30c3\u30c8",
436
+ "\u6c34\u6cf3\u5e3d",
437
+ "\u30d0\u30b9\u30bf\u30aa\u30eb",
438
+ "\u30d0\u30b9\u30bf\u30d6",
439
+ "\u30d3\u30fc\u30c1\u30ef\u30b4\u30f3",
440
+ "\u30d3\u30fc\u30b3\u30f3",
441
+ "\u30d3\u30fc\u30ab\u30fc",
442
+ "\u30d9\u30a2\u30b9\u30ad\u30f3",
443
+ "\u30d3\u30fc\u30eb\u74f6",
444
+ "\u30d3\u30fc\u30eb\u30b0\u30e9\u30b9",
445
+ "\u30d9\u30eb\u30b3\u30fc\u30c8",
446
+ "\u30d3\u30d6",
447
+ "\u81ea\u8ee2\u8eca",
448
+ "\u30d3\u30ad\u30cb",
449
+ "\u30d0\u30a4\u30f3\u30c0\u30fc",
450
+ "\u53cc\u773c\u93e1",
451
+ "\u5de3\u7bb1",
452
+ "\u30dc\u30fc\u30c8\u30cf\u30a6\u30b9",
453
+ "\u30dc\u30d6\u30b9\u30ec\u30fc",
454
+ "\u30eb\u30fc\u30d7\u30bf\u30a4",
455
+ "\u30dc\u30f3\u30cd\u30c3\u30c8",
456
+ "\u672c\u68da",
457
+ "\u66f8\u5e97",
458
+ "\u74f6\u306e\u30ad\u30e3\u30c3\u30d7",
459
+ "\u5f13",
460
+ "\u3061\u3087\u3046\u30cd\u30af\u30bf\u30a4",
461
+ "\u771f\u936e",
462
+ "\u30d6\u30e9\u30b8\u30e3\u30fc",
463
+ "\u9632\u6ce2\u5824",
464
+ "\u80f8\u5f53\u3066",
465
+ "\u307b\u3046\u304d",
466
+ "\u30d0\u30b1\u30c4",
467
+ "\u30d0\u30c3\u30af\u30eb",
468
+ "\u9632\u5f3e\u30c1\u30e7\u30c3\u30ad",
469
+ "\u65b0\u5e79\u7dda",
470
+ "\u7cbe\u8089\u5e97",
471
+ "\u30bf\u30af\u30b7\u30fc",
472
+ "\u5927\u91dc",
473
+ "\u30ad\u30e3\u30f3\u30c9\u30eb",
474
+ "\u5927\u7832",
475
+ "\u30ab\u30cc\u30fc",
476
+ "\u7f36\u5207\u308a",
477
+ "\u30ab\u30fc\u30c7\u30a3\u30ac\u30f3",
478
+ "\u8eca\u306e\u30df\u30e9\u30fc",
479
+ "\u56de\u8ee2\u6728\u99ac",
480
+ "\u5927\u5de5\u306e\u30ad\u30c3\u30c8",
481
+ "\u30ab\u30fc\u30c8\u30f3",
482
+ "\u8eca\u306e\u30db\u30a4\u30fc\u30eb",
483
+ "\u73fe\u91d1\u81ea\u52d5\u9810\u3051\u6255\u3044\u6a5f",
484
+ "\u30ab\u30bb\u30c3\u30c8",
485
+ "\u30ab\u30bb\u30c3\u30c8\u30fb\u30d7\u30ec\u30fc\u30e4\u30fc",
486
+ "\u57ce",
487
+ "\u30ab\u30bf\u30de\u30e9\u30f3",
488
+ "CD\u30d7\u30ec\u30fc\u30e4\u30fc",
489
+ "\u30c1\u30a7\u30ed",
490
+ "\u30b9\u30de\u30fc\u30c8\u30d5\u30a9\u30f3",
491
+ "\u9396",
492
+ "\u30c1\u30a7\u30fc\u30f3\u30ea\u30f3\u30af\u30d5\u30a7\u30f3\u30b9",
493
+ "\u30c1\u30a7\u30fc\u30f3\u30e1\u30fc\u30eb",
494
+ "\u30c1\u30a7\u30fc\u30f3\u30bd\u30fc",
495
+ "\u80f8",
496
+ "\u30b7\u30d5\u30a9\u30cb\u30a2",
497
+ "\u30c1\u30e3\u30a4\u30e0",
498
+ "\u4e2d\u56fd\u30ad\u30e3\u30d3\u30cd\u30c3\u30c8",
499
+ "\u30af\u30ea\u30b9\u30de\u30b9\u306e\u9774\u4e0b",
500
+ "\u6559\u4f1a",
501
+ "\u6620\u753b",
502
+ "\u30af\u30ea\u30fc\u30d0\u30fc",
503
+ "\u5d16\u306e\u4f4f\u5c45",
504
+ "\u30de\u30f3\u30c8",
505
+ "\u30af\u30ed\u30c3\u30b0",
506
+ "\u30ab\u30af\u30c6\u30eb\u30b7\u30a7\u30fc\u30ab\u30fc",
507
+ "\u30b3\u30fc\u30d2\u30fc\u30de\u30b0",
508
+ "\u30b3\u30fc\u30d2\u30fc\u30dd\u30c3\u30c8",
509
+ "\u30b3\u30a4\u30eb",
510
+ "\u30c0\u30a4\u30e4\u30eb\u9320",
511
+ "\u30b3\u30f3\u30d4\u30e5\u30fc\u30bf\u306e\u30ad\u30fc\u30dc\u30fc\u30c9",
512
+ "\u88fd\u83d3",
513
+ "\u30b3\u30f3\u30c6\u30ca\u8239",
514
+ "\u30b3\u30f3\u30d0\u30fc\u30c1\u30d6\u30eb",
515
+ "\u30b3\u30fc\u30af\u30b9\u30af\u30ea\u30e5\u30fc",
516
+ "\u30b3\u30eb\u30cd\u30c3\u30c8",
517
+ "\u30ab\u30a6\u30dc\u30fc\u30a4\u30d6\u30fc\u30c4",
518
+ "\u30ab\u30a6\u30dc\u30fc\u30a4\u30cf\u30c3\u30c8",
519
+ "\u30af\u30ec\u30fc\u30c9\u30eb",
520
+ "\u30af\u30ec\u30fc\u30f3",
521
+ "\u30af\u30e9\u30c3\u30b7\u30e5\u30d8\u30eb\u30e1\u30c3\u30c8",
522
+ "\u6728\u7bb1",
523
+ "\u30d9\u30d3\u30fc\u30d9\u30c3\u30c9",
524
+ "\u30af\u30ed\u30fc\u30af\u30dd\u30c3\u30c8",
525
+ "\u30af\u30ed\u30b1\u30c3\u30c8\u30dc\u30fc\u30eb",
526
+ "\u677e\u8449\u6756",
527
+ "\u80f8\u5f53\u3066",
528
+ "\u30c0\u30e0",
529
+ "\u673a",
530
+ "\u30c7\u30b9\u30af\u30c8\u30c3\u30d7\u30b3\u30f3\u30d4\u30e5\u30fc\u30bf\u30fc",
531
+ "\u30c0\u30a4\u30e4\u30eb\u96fb\u8a71",
532
+ "\u304a\u3080\u3064",
533
+ "\u30c7\u30b8\u30bf\u30eb\u6642\u8a08",
534
+ "\u30c7\u30b8\u30bf\u30eb\u8155\u6642\u8a08",
535
+ "\u30c0\u30a4\u30cb\u30f3\u30b0\u30c6\u30fc\u30d6\u30eb",
536
+ "\u610f\u6c17\u5730\u306a\u3057",
537
+ "\u98df\u5668\u6d17\u3044\u6a5f",
538
+ "\u30c7\u30a3\u30b9\u30af\u30d6\u30ec\u30fc\u30ad",
539
+ "\u30c9\u30c3\u30af",
540
+ "\u72ac\u305e\u308a",
541
+ "\u30c9\u30fc\u30e0",
542
+ "\u7384\u95a2\u30de\u30c3\u30c8",
543
+ "\u6398\u524a\u57fa\u5730",
544
+ "\u30c9\u30e9\u30e0",
545
+ "\u30c9\u30e9\u30e0\u30b9\u30c6\u30a3\u30c3\u30af",
546
+ "\u30c0\u30f3\u30d9\u30eb",
547
+ "\u30c0\u30c3\u30c1\u30aa\u30fc\u30d6\u30f3",
548
+ "\u6247\u98a8\u6a5f",
549
+ "\u30a8\u30ec\u30ad\u30ae\u30bf\u30fc",
550
+ "\u96fb\u6c17\u6a5f\u95a2\u8eca",
551
+ "\u5a2f\u697d\u65bd\u8a2d",
552
+ "\u5c01\u7b52",
553
+ "\u30a8\u30b9\u30d7\u30ec\u30c3\u30bd\u30de\u30b7\u30fc\u30f3",
554
+ "\u30d5\u30a7\u30fc\u30b9\u30d1\u30a6\u30c0\u30fc",
555
+ "\u30d5\u30a7\u30b6\u30fc\u30dc\u30a2",
556
+ "\u30d5\u30a1\u30a4\u30eb",
557
+ "\u6d88\u9632\u8247",
558
+ "\u6d88\u9632\u8eca",
559
+ "\u30d5\u30a1\u30a4\u30a2\u30fc\u30b9\u30af\u30ea\u30fc\u30f3",
560
+ "\u65d7\u7aff",
561
+ "\u30d5\u30eb\u30fc\u30c8",
562
+ "\u6298\u308a\u7573\u307f\u5f0f\u6905\u5b50",
563
+ "\u30d5\u30c3\u30c8\u30dc\u30fc\u30eb\u30d8\u30eb\u30e1\u30c3\u30c8",
564
+ "\u30d5\u30a9\u30fc\u30af\u30ea\u30d5\u30c8",
565
+ "\u5674\u6c34",
566
+ "\u4e07\u5e74\u7b46",
567
+ "\u56db\u67f1",
568
+ "\u8ca8\u8eca",
569
+ "\u30d5\u30ec\u30f3\u30c1\u30db\u30eb\u30f3",
570
+ "\u30d5\u30e9\u30a4\u30d1\u30f3",
571
+ "\u6bdb\u76ae\u306e\u30b3\u30fc\u30c8",
572
+ "\u3054\u307f\u53ce\u96c6\u8eca",
573
+ "\u30ac\u30b9\u30de\u30b9\u30af",
574
+ "\u30ac\u30bd\u30ea\u30f3\u30dd\u30f3\u30d7",
575
+ "\u30b4\u30d6\u30ec\u30c3\u30c8",
576
+ "\u30b4\u30fc\u30ab\u30fc\u30c8",
577
+ "\u30b4\u30eb\u30d5\u30dc\u30fc\u30eb",
578
+ "\u30b4\u30eb\u30d5\u30ab\u30fc\u30c8",
579
+ "\u30b4\u30f3\u30c9\u30e9",
580
+ "\u30b4\u30f3\u30b0",
581
+ "\u30ac\u30a6\u30f3",
582
+ "\u30b0\u30e9\u30f3\u30c9\u30d4\u30a2\u30ce",
583
+ "\u6e29\u5ba4",
584
+ "\u30b0\u30ea\u30eb",
585
+ "\u98df\u6599\u54c1\u5e97",
586
+ "\u30ae\u30ed\u30c1\u30f3",
587
+ "\u30d8\u30a2\u30b9\u30e9\u30a4\u30c9",
588
+ "\u30d8\u30a2\u30b9\u30d7\u30ec\u30fc",
589
+ "\u534a\u30c8\u30e9\u30c3\u30af",
590
+ "\u30cf\u30f3\u30de\u30fc",
591
+ "\u59a8\u3052\u307e\u3059",
592
+ "\u30cf\u30f3\u30c9\u30d6\u30ed\u30ef\u30fc",
593
+ "\u30bf\u30d6\u30ec\u30c3\u30c8",
594
+ "\u30cf\u30f3\u30ab\u30c1",
595
+ "\u30cf\u30fc\u30c9\u30c7\u30a3\u30b9\u30af",
596
+ "\u30cf\u30fc\u30e2\u30cb\u30ab",
597
+ "\u30cf\u30fc\u30d7",
598
+ "\u30cf\u30fc\u30d9\u30b9\u30bf",
599
+ "\u65a7",
600
+ "\u30db\u30eb\u30b9\u30bf\u30fc",
601
+ "\u30db\u30fc\u30e0\u30b7\u30a2\u30bf\u30fc",
602
+ "\u30cf\u30cb\u30ab\u30e0",
603
+ "\u30d5\u30c3\u30af",
604
+ "\u30d5\u30fc\u30d7\u30b9\u30ab\u30fc\u30c8",
605
+ "\u6c34\u5e73\u30d0\u30fc",
606
+ "\u99ac\u8eca",
607
+ "\u7802\u6642\u8a08",
608
+ "\u30a2\u30a4\u30d5\u30a9\u30fc\u30f3",
609
+ "\u9244",
610
+ "\u30b8\u30e3\u30c3\u30af\u30aa\u30fc\u30e9\u30f3\u30bf\u30f3",
611
+ "\u30b8\u30fc\u30f3\u30ba",
612
+ "\u30b8\u30fc\u30d7",
613
+ "\u30b8\u30e3\u30fc\u30b8\u30fc",
614
+ "\u30b8\u30b0\u30bd\u30fc\u30d1\u30ba\u30eb",
615
+ "\u4eba\u529b\u8eca",
616
+ "\u30b8\u30e7\u30a4\u30b9\u30c6\u30a3\u30c3\u30af",
617
+ "\u7740\u7269",
618
+ "\u819d\u30d1\u30c3\u30c9",
619
+ "\u7d50\u3073\u76ee",
620
+ "\u767d\u8863",
621
+ "\u3072\u3057\u3083\u304f",
622
+ "\u30e9\u30f3\u30d7\u306e\u304b\u3055",
623
+ "\u30ce\u30fc\u30c8\u30d1\u30bd\u30b3\u30f3",
624
+ "\u829d\u5208\u308a\u6a5f",
625
+ "\u30ec\u30f3\u30ba\u30ad\u30e3\u30c3\u30d7",
626
+ "\u30ec\u30bf\u30fc\u30aa\u30fc\u30d7\u30ca\u30fc",
627
+ "\u30e9\u30a4\u30d6\u30e9\u30ea",
628
+ "\u6551\u547d\u30dc\u30fc\u30c8",
629
+ "\u30e9\u30a4\u30bf\u30fc",
630
+ "\u30ea\u30e0\u30b8\u30f3",
631
+ "\u30e9\u30a4\u30ca\u30fc",
632
+ "\u53e3\u7d05",
633
+ "\u30ed\u30fc\u30d5\u30a1\u30fc",
634
+ "\u30ed\u30fc\u30b7\u30e7\u30f3",
635
+ "\u30b9\u30d4\u30fc\u30ab\u30fc",
636
+ "\u30eb\u30fc\u30da",
637
+ "\u88fd\u6750\u6240",
638
+ "\u78c1\u6c17\u30b3\u30f3\u30d1\u30b9",
639
+ "\u90f5\u888b",
640
+ "\u30e1\u30fc\u30eb\u30dc\u30c3\u30af\u30b9",
641
+ "\u30de\u30a4\u30e8",
642
+ "\u30de\u30a4\u30e8",
643
+ "\u30de\u30f3\u30db\u30fc\u30eb\u306e\u84cb",
644
+ "\u30de\u30e9\u30ab\u30b9",
645
+ "\u30de\u30ea\u30f3\u30d0",
646
+ "\u30de\u30b9\u30af",
647
+ "\u30de\u30c3\u30c1\u68d2",
648
+ "\u30e1\u30a4\u30dd\u30fc\u30eb",
649
+ "\u8ff7\u8def",
650
+ "\u8a08\u91cf\u30ab\u30c3\u30d7",
651
+ "\u85ac\u7bb1",
652
+ "\u5de8\u77f3",
653
+ "\u30de\u30a4\u30af",
654
+ "\u30de\u30a4\u30af\u30ed\u6ce2",
655
+ "\u8ecd\u670d",
656
+ "\u30df\u30eb\u30af\u7f36",
657
+ "\u30df\u30cb\u30d0\u30b9",
658
+ "\u30df\u30cb\u30b9\u30ab\u30fc\u30c8",
659
+ "\u30df\u30cb\u30d0\u30f3",
660
+ "\u30df\u30b5\u30a4\u30eb",
661
+ "\u30df\u30c8\u30f3",
662
+ "\u30df\u30ad\u30b7\u30f3\u30b0\u30dc\u30a6\u30eb",
663
+ "\u79fb\u52d5\u4f4f\u5b85",
664
+ "\u30e2\u30c7\u30ebT",
665
+ "\u30e2\u30c7\u30e0",
666
+ "\u4fee\u9053\u9662",
667
+ "\u30e2\u30cb\u30bf\u30fc",
668
+ "\u30e2\u30da\u30c3\u30c8",
669
+ "\u30e2\u30eb\u30bf\u30eb",
670
+ "\u30e2\u30eb\u30bf\u30eb\u30dc\u30fc\u30c9",
671
+ "\u30e2\u30b9\u30af",
672
+ "\u868a\u5e33",
673
+ "\u30b9\u30af\u30fc\u30bf\u30fc",
674
+ "\u30de\u30a6\u30f3\u30c6\u30f3\u30d0\u30a4\u30af",
675
+ "\u5c71\u306e\u30c6\u30f3\u30c8",
676
+ "\u30de\u30a6\u30b9",
677
+ "\u30cd\u30ba\u30df\u6355\u308a",
678
+ "\u5f15\u3063\u8d8a\u3057\u30c8\u30e9\u30c3\u30af",
679
+ "\u9283\u53e3",
680
+ "\u30cd\u30a4\u30eb",
681
+ "\u30cd\u30c3\u30af\u30d6\u30ec\u30fc\u30b9",
682
+ "\u30cd\u30c3\u30af\u30ec\u30b9",
683
+ "\u4e73\u9996",
684
+ "\u30ce\u30fc\u30c8",
685
+ "\u30aa\u30d9\u30ea\u30b9\u30af",
686
+ "\u30aa\u30fc\u30dc\u30a8",
687
+ "\u30aa\u30ab\u30ea\u30ca",
688
+ "\u30aa\u30c9\u30e1\u30fc\u30bf\u30fc",
689
+ "\u30aa\u30a4\u30eb\u30d5\u30a3\u30eb\u30bf\u30fc",
690
+ "\u5668\u5b98",
691
+ "\u30aa\u30b7\u30ed\u30b9\u30b3\u30fc\u30d7",
692
+ "\u30aa\u30fc\u30d0\u30fc\u30b9\u30ab\u30fc\u30c8",
693
+ "\u725b\u8eca",
694
+ "\u9178\u7d20\u30de\u30b9\u30af",
695
+ "\u30d1\u30b1\u30c3\u30c8",
696
+ "\u30d1\u30c9\u30eb",
697
+ "\u30d1\u30c9\u30eb\u30db\u30a4\u30fc\u30eb",
698
+ "\u5357\u4eac\u9320",
699
+ "\u7d75\u7b46",
700
+ "\u30d1\u30b8\u30e3\u30de",
701
+ "\u5bae\u6bbf",
702
+ "\u30d1\u30f3\u30d1\u30a4\u30d7",
703
+ "\u30da\u30fc\u30d1\u30fc\u30bf\u30aa\u30eb",
704
+ "\u30d1\u30e9\u30b7\u30e5\u30fc\u30c8",
705
+ "\u5e73\u884c\u68d2",
706
+ "\u516c\u5712\u306e\u30d9\u30f3\u30c1",
707
+ "\u30d1\u30fc\u30ad\u30f3\u30b0\u30e1\u30fc\u30bf\u30fc",
708
+ "\u4e57\u7528\u8eca",
709
+ "\u30d1\u30c6\u30a3\u30aa",
710
+ "\u6709\u6599\u96fb\u8a71",
711
+ "\u53f0\u5ea7",
712
+ "\u7b46\u7bb1",
713
+ "\u925b\u7b46\u524a\u308a",
714
+ "\u9999\u6c34",
715
+ "\u30da\u30c8\u30ea\u76bf",
716
+ "\u30b3\u30d4\u30fc\u6a5f",
717
+ "\u9078\u3076",
718
+ "\u30b9\u30d1\u30a4\u30af\u4ed8\u304d\u9244\u304b\u3076\u3068",
719
+ "\u676d\u67f5",
720
+ "\u62fe\u3046",
721
+ "\u685f\u6a4b",
722
+ "\u8caf\u91d1\u7bb1",
723
+ "\u9320\u5264\u74f6",
724
+ "\u6795",
725
+ "\u30d4\u30f3\u30dd\u30f3\u7403",
726
+ "\u98a8\u8eca",
727
+ "\u6d77\u8cca",
728
+ "\u30d4\u30c3\u30c1\u30e3\u30fc",
729
+ "\u98db\u884c\u6a5f",
730
+ "\u30d7\u30e9\u30cd\u30bf\u30ea\u30a6\u30e0",
731
+ "\u30d3\u30cb\u30fc\u30eb\u888b",
732
+ "\u76bf\u7acb\u3066",
733
+ "\u30d7\u30e9\u30a6",
734
+ "\u30d7\u30e9\u30f3\u30b8\u30e3\u30fc",
735
+ "\u30dd\u30e9\u30ed\u30a4\u30c9\u30ab\u30e1\u30e9",
736
+ "\u30dd\u30fc\u30eb",
737
+ "\u8b66\u5bdf\u8eca",
738
+ "\u30dd\u30f3\u30c1\u30e7",
739
+ "\u30d3\u30ea\u30e4\u30fc\u30c9\u53f0",
740
+ "\u30dd\u30c3\u30d7\u30fb\u30dc\u30c8\u30eb",
741
+ "\u30dd\u30c3\u30c8",
742
+ "\u308d\u304f\u308d",
743
+ "\u30d1\u30ef\u30fc\u30c9\u30ea\u30eb",
744
+ "\u793c\u62dd\u7528\u6577\u7269",
745
+ "\u30d7\u30ea\u30f3\u30bf",
746
+ "\u5211\u52d9\u6240",
747
+ "\u767a\u5c04\u4f53",
748
+ "\u30d7\u30ed\u30b8\u30a7\u30af\u30bf\u30fc",
749
+ "\u30d1\u30c3\u30af",
750
+ "\u30b5\u30f3\u30c9\u30d0\u30c3\u30b0",
751
+ "\u8ca1\u5e03",
752
+ "\u30af\u30a4\u30eb",
753
+ "\u30ad\u30eb\u30c8",
754
+ "\u30ec\u30fc\u30b5\u30fc",
755
+ "\u30e9\u30b1\u30c3\u30c8",
756
+ "\u30e9\u30b8\u30a8\u30fc\u30bf\u30fc",
757
+ "\u7121\u7dda",
758
+ "\u96fb\u6ce2\u671b\u9060\u93e1",
759
+ "\u5929\u6c34\u6876",
760
+ "RV\u8eca",
761
+ "\u30ea\u30fc\u30eb",
762
+ "\u30ec\u30d5\u30ec\u30c3\u30af\u30b9\u30ab\u30e1\u30e9",
763
+ "\u51b7\u8535\u5eab",
764
+ "\u30ea\u30e2\u30b3\u30f3",
765
+ "\u30ec\u30b9\u30c8\u30e9\u30f3",
766
+ "\u30ea\u30dc\u30eb\u30d0\u30fc",
767
+ "\u30e9\u30a4\u30d5\u30eb",
768
+ "\u30ed\u30c3\u30ad\u30f3\u30b0\u30c1\u30a7\u30a2",
769
+ "\u713c\u8089\u6599\u7406\u5e97",
770
+ "\u6d88\u3057\u30b4\u30e0",
771
+ "\u30e9\u30b0\u30d3\u30fc\u30dc\u30fc\u30eb",
772
+ "\u30eb\u30fc\u30eb",
773
+ "\u30e9\u30f3\u30cb\u30f3\u30b0\u30b7\u30e5\u30fc\u30ba",
774
+ "\u5b89\u5168",
775
+ "\u5b89\u5168\u30d4\u30f3",
776
+ "\u5869\u306e\u5165\u308c\u7269",
777
+ "\u30b5\u30f3\u30c0\u30eb",
778
+ "\u30b5\u30ed\u30f3",
779
+ "\u30b5\u30c3\u30af\u30b9",
780
+ "\u9798",
781
+ "\u898f\u6a21",
782
+ "\u30b9\u30af\u30fc\u30eb\u30d0\u30b9",
783
+ "\u30b9\u30af\u30fc\u30ca\u30fc",
784
+ "\u30b9\u30b3\u30a2\u30dc\u30fc\u30c9",
785
+ "\u753b\u9762",
786
+ "\u30b9\u30af\u30ea\u30e5\u30fc",
787
+ "\u30c9\u30e9\u30a4\u30d0\u30fc",
788
+ "\u30b7\u30fc\u30c8\u30d9\u30eb\u30c8",
789
+ "\u30df\u30b7\u30f3",
790
+ "\u30b7\u30fc\u30eb\u30c9",
791
+ "\u9774\u5c4b",
792
+ "\u969c\u5b50",
793
+ "\u8cb7\u3044\u7269\u304b\u3054",
794
+ "\u30b7\u30e7\u30c3\u30d4\u30f3\u30b0\u30ab\u30fc\u30c8",
795
+ "\u30b7\u30e3\u30d9\u30eb",
796
+ "\u30b7\u30e3\u30ef\u30fc\u30ad\u30e3\u30c3\u30d7",
797
+ "\u30b7\u30e3\u30ef\u30fc\u30ab\u30fc\u30c6\u30f3",
798
+ "\u30b9\u30ad\u30fc",
799
+ "\u30b9\u30ad\u30fc\u30de\u30b9\u30af",
800
+ "\u5bdd\u888b",
801
+ "\u8a08\u7b97\u5c3a",
802
+ "\u5f15\u304d\u6238",
803
+ "\u30b9\u30ed\u30c3\u30c8",
804
+ "\u30b9\u30ce\u30fc\u30b1\u30eb",
805
+ "\u30b9\u30ce\u30fc\u30e2\u30fc\u30d3\u30eb",
806
+ "\u9664\u96ea\u6a5f",
807
+ "\u30bd\u30fc\u30d7\u30c7\u30a3\u30b9\u30da\u30f3\u30b5\u30fc",
808
+ "\u30b5\u30c3\u30ab\u30fc\u30dc\u30fc\u30eb",
809
+ "\u9774\u4e0b",
810
+ "\u592a\u967d\u306e\u76bf",
811
+ "\u30bd\u30f3\u30d6\u30ec\u30ed",
812
+ "\u30b9\u30fc\u30d7\u76bf",
813
+ "\u30b9\u30da\u30fc\u30b9\u30ad\u30fc",
814
+ "\u30b9\u30da\u30fc\u30b9\u30d2\u30fc\u30bf\u30fc",
815
+ "\u30b9\u30da\u30fc\u30b9\u30b7\u30e3\u30c8\u30eb",
816
+ "\u3078\u3089",
817
+ "\u30b9\u30d4\u30fc\u30c9\u30dc\u30fc\u30c8",
818
+ "\u30af\u30e2\u306e\u5de3",
819
+ "\u30b9\u30d4\u30f3\u30c9\u30eb",
820
+ "\u30b9\u30dd\u30fc\u30c4\u30ab\u30fc",
821
+ "\u30b9\u30dd\u30c3\u30c8\u30e9\u30a4\u30c8",
822
+ "\u30b9\u30c6\u30fc\u30b8",
823
+ "\u84b8\u6c17\u6a5f\u95a2\u8eca",
824
+ "\u92fc\u30a2\u30fc\u30c1\u6a4b",
825
+ "\u30b9\u30c1\u30fc\u30eb\u30c9\u30e9\u30e0",
826
+ "\u8074\u8a3a\u5668",
827
+ "\u30b9\u30c8\u30fc\u30eb",
828
+ "\u77f3\u57a3",
829
+ "\u30b9\u30c8\u30c3\u30d7\u30a6\u30a9\u30c3\u30c1",
830
+ "\u30ec\u30f3\u30b8",
831
+ "\u30b9\u30c8\u30ec\u30fc\u30ca\u30fc",
832
+ "\u8def\u9762\u96fb\u8eca",
833
+ "\u30b9\u30c8\u30ec\u30c3\u30c1\u30e3\u30fc",
834
+ "\u30b9\u30bf\u30b8\u30aa\u30bd\u30d5\u30a1",
835
+ "\u4ecf\u820e\u5229\u5854",
836
+ "\u6f5c\u6c34\u8266",
837
+ "\u30b9\u30fc\u30c4",
838
+ "\u65e5\u6642\u8a08",
839
+ "\u30b5\u30f3\u30b0\u30e9\u30b9",
840
+ "\u30b5\u30f3\u30b0\u30e9\u30b9",
841
+ "\u65e5\u713c\u3051\u6b62\u3081\u5264",
842
+ "\u3064\u308a\u6a4b",
843
+ "\u7dbf\u68d2",
844
+ "\u30c8\u30ec\u30fc\u30ca\u30fc",
845
+ "\u6d77\u30d1\u30f3",
846
+ "\u30b9\u30a4\u30f3\u30b0",
847
+ "\u30b9\u30a4\u30c3\u30c1",
848
+ "\u6ce8\u5c04\u5668",
849
+ "\u96fb\u6c17\u30b9\u30bf\u30f3\u30c9",
850
+ "\u30bf\u30f3\u30af",
851
+ "\u30c6\u30fc\u30d7\u30d7\u30ec\u30fc\u30e4\u30fc",
852
+ "\u30c6\u30a3\u30fc\u30dd\u30c3\u30c8",
853
+ "\u30c6\u30c7\u30a3",
854
+ "\u30c6\u30ec\u30d3",
855
+ "\u30c6\u30cb\u30b9\u30dc\u30fc\u30eb",
856
+ "\u30b5\u30c3\u30c1",
857
+ "\u5287\u5834\u306e\u30ab\u30fc\u30c6\u30f3",
858
+ "\u6307\u306c\u304d",
859
+ "\u8131\u7a40\u6a5f",
860
+ "\u738b\u4f4d",
861
+ "\u74e6\u5c4b\u6839",
862
+ "\u30c8\u30fc\u30b9\u30bf\u30fc",
863
+ "\u30bf\u30d0\u30b3\u5c4b",
864
+ "\u4fbf\u5ea7",
865
+ "\u30c8\u30fc\u30c1",
866
+ "\u30c8\u30fc\u30c6\u30e0\u30dd\u30fc\u30eb",
867
+ "\u30ec\u30c3\u30ab\u30fc\u8eca",
868
+ "\u73a9\u5177\u5c4b",
869
+ "\u30c8\u30e9\u30af\u30bf\u30fc",
870
+ "\u30c8\u30ec\u30fc\u30e9\u30fc\u30c8\u30e9\u30c3\u30af",
871
+ "\u30c8\u30ec\u30a4",
872
+ "\u30c8\u30ec\u30f3\u30c1\u30b3\u30fc\u30c8",
873
+ "\u4e09\u8f2a\u8eca",
874
+ "\u4e09\u80f4\u8239",
875
+ "\u4e09\u811a",
876
+ "\u51f1\u65cb\u9580",
877
+ "\u30c8\u30ed\u30ea\u30fc\u30d0\u30b9",
878
+ "\u30c8\u30ed\u30f3\u30dc\u30fc\u30f3",
879
+ "\u30d0\u30b9\u30bf\u30d6",
880
+ "\u56de\u8ee2\u30c9\u30a2",
881
+ "\u30bf\u30a4\u30d7\u30e9\u30a4\u30bf\u30fc\u306e\u30ad\u30fc\u30dc\u30fc\u30c9",
882
+ "\u5098",
883
+ "\u4e00\u8f2a\u8eca",
884
+ "\u76f4\u7acb",
885
+ "\u771f\u7a7a",
886
+ "\u82b1\u74f6",
887
+ "\u30dc\u30fc\u30eb\u30c8",
888
+ "\u30d9\u30eb\u30d9\u30c3\u30c8",
889
+ "\u81ea\u52d5\u8ca9\u58f2\u6a5f",
890
+ "\u796d\u670d",
891
+ "\u9ad8\u67b6\u6a4b",
892
+ "\u30d0\u30a4\u30aa\u30ea\u30f3",
893
+ "\u30d0\u30ec\u30fc\u30dc\u30fc\u30eb",
894
+ "\u30ef\u30c3\u30d5\u30eb\u713c\u304d\u578b",
895
+ "\u58c1\u6642\u8a08",
896
+ "\u8ca1\u5e03",
897
+ "\u30ef\u30fc\u30c9\u30ed\u30fc\u30d6",
898
+ "\u6226\u95d8\u6a5f",
899
+ "\u6d17\u9762\u5668",
900
+ "\u30ef\u30c3\u30b7\u30e3\u30fc",
901
+ "\u6c34\u7b52",
902
+ "\u6c34\u5dee\u3057",
903
+ "\u7d66\u6c34\u5854",
904
+ "\u30a6\u30a4\u30b9\u30ad\u30fc\u30b8\u30e3\u30b0",
905
+ "\u30db\u30a4\u30c3\u30b9\u30eb",
906
+ "\u304b\u3064\u3089",
907
+ "\u7a93\u7db2\u6238",
908
+ "\u30d6\u30e9\u30a4\u30f3\u30c9",
909
+ "\u30a6\u30a3\u30f3\u30b6\u30fc\u30cd\u30af\u30bf\u30a4",
910
+ "\u30ef\u30a4\u30f3\u30dc\u30c8\u30eb",
911
+ "\u7ffc",
912
+ "\u4e2d\u83ef\u934b",
913
+ "\u6728\u88fd\u30b9\u30d7\u30fc\u30f3",
914
+ "\u30a6\u30fc\u30eb",
915
+ "\u30ef\u30fc\u30e0\u30d5\u30a7\u30f3\u30b9",
916
+ "\u96e3\u7834\u8239",
917
+ "\u30e8\u30fc\u30eb",
918
+ "\u30d1\u30aa",
919
+ "\u30b5\u30a4\u30c8",
920
+ "\u30b3\u30df\u30c3\u30af\u30d6\u30c3\u30af",
921
+ "\u30af\u30ed\u30b9\u30ef\u30fc\u30c9\u30d1\u30ba\u30eb",
922
+ "\u9053\u8def\u6a19\u8b58",
923
+ "\u4ea4\u901a\u4fe1\u53f7\u706f",
924
+ "\u30d6\u30c3\u30af\u30ab\u30d0\u30fc",
925
+ "\u30e1\u30cb\u30e5\u30fc",
926
+ "\u30d7\u30ec\u30fc\u30c8",
927
+ "\u30b0\u30a2\u30ab\u30e2\u30fc\u30ec",
928
+ "\u30b3\u30f3\u30bd\u30e1",
929
+ "\u30db\u30c3\u30c8\u30dd\u30c3\u30c8",
930
+ "\u30d1\u30d5\u30a7",
931
+ "\u30a2\u30a4\u30b9\u30af\u30ea\u30fc\u30e0",
932
+ "\u30a2\u30a4\u30b9\u30ad\u30e3\u30f3\u30c7\u30a3\u30fc",
933
+ "\u30d5\u30e9\u30f3\u30b9\u30d1\u30f3",
934
+ "\u30d9\u30fc\u30b0\u30eb",
935
+ "\u30d7\u30ec\u30c3\u30c4\u30a7\u30eb",
936
+ "\u30c1\u30fc\u30ba\u30d0\u30fc\u30ac\u30fc",
937
+ "\u30db\u30c3\u30c8\u30c9\u30c3\u30b0",
938
+ "\u30de\u30c3\u30b7\u30e5\u30dd\u30c6\u30c8",
939
+ "\u30ad\u30e3\u30d9\u30c4",
940
+ "\u30d6\u30ed\u30c3\u30b3\u30ea\u30fc",
941
+ "\u30ab\u30ea\u30d5\u30e9\u30ef\u30fc",
942
+ "\u30ba\u30c3\u30ad\u30fc\u30cb",
943
+ "\u305d\u3046\u3081\u3093\u304b\u307c\u3061\u3083",
944
+ "\u30c9\u30f3\u30b0\u30ea\u304b\u307c\u3061\u3083",
945
+ "\u30ab\u30dc\u30c1\u30e3",
946
+ "\u30ad\u30e5\u30a6\u30ea",
947
+ "\u30a2\u30fc\u30c6\u30a3\u30c1\u30e7\u30fc\u30af",
948
+ "\u30d4\u30fc\u30de\u30f3",
949
+ "\u30ab\u30eb\u30c9\u30f3",
950
+ "\u30ad\u30ce\u30b3",
951
+ "\u30ea\u30f3\u30b4",
952
+ "\u30a4\u30c1\u30b4",
953
+ "\u30aa\u30ec\u30f3\u30b8",
954
+ "\u30ec\u30e2\u30f3",
955
+ "\u30a4\u30c1\u30b8\u30af",
956
+ "\u30d1\u30a4\u30ca\u30c3\u30d7\u30eb",
957
+ "\u30d0\u30ca\u30ca",
958
+ "\u30d1\u30e9\u30df\u30c4",
959
+ "\u30ab\u30b9\u30bf\u30fc\u30c9\u30a2\u30c3\u30d7\u30eb",
960
+ "\u30b6\u30af\u30ed",
961
+ "\u5e72\u3057\u8349",
962
+ "\u30ab\u30eb\u30dc\u30ca\u30fc\u30e9",
963
+ "\u30c1\u30e7\u30b3\u30ec\u30fc\u30c8\u30bd\u30fc\u30b9",
964
+ "\u30d1\u30f3\u751f\u5730",
965
+ "\u30df\u30fc\u30c8\u30ed\u30fc\u30d5",
966
+ "\u30d4\u30b6",
967
+ "\u30dd\u30c3\u30c8\u30d1\u30a4",
968
+ "\u30d6\u30ea\u30c8\u30fc",
969
+ "\u8d64\u30ef\u30a4\u30f3",
970
+ "\u30a8\u30b9\u30d7\u30ec\u30c3\u30bd",
971
+ "\u30ab\u30c3\u30d7",
972
+ "\u30a8\u30c3\u30b0\u30ce\u30c3\u30b0",
973
+ "\u30a2\u30eb\u30d7\u30b9",
974
+ "\u30d0\u30d6\u30eb",
975
+ "\u5d16",
976
+ "\u30b5\u30f3\u30b4\u7901",
977
+ "\u9593\u6b20\u6cc9",
978
+ "\u6e56\u7554",
979
+ "\u5cac",
980
+ "\u7802\u5dde",
981
+ "\u6d77\u5cb8",
982
+ "\u8c37",
983
+ "\u706b\u5c71",
984
+ "\u91ce\u7403\u9078\u624b",
985
+ "\u65b0\u90ce",
986
+ "\u30b9\u30ad\u30e5\u30fc\u30d0\u30c0\u30a4\u30d0\u30fc",
987
+ "\u83dc\u7a2e",
988
+ "\u30c7\u30a4\u30b8\u30fc",
989
+ "\u862d",
990
+ "\u30c8\u30a6\u30e2\u30ed\u30b3\u30b7",
991
+ "\u30c9\u30f3\u30b0\u30ea",
992
+ "\u30d2\u30c3\u30d7",
993
+ "\u30c8\u30c1\u30ce\u30ad",
994
+ "\u30b5\u30f3\u30b4\u83cc",
995
+ "\u30cf\u30e9\u30bf\u30b1",
996
+ "\u30b7\u30e3\u30b0\u30de\u30a2\u30df\u30ac\u30b5\u30bf\u30b1",
997
+ "\u30b9\u30c3\u30dd\u30f3\u30bf\u30b1",
998
+ "\u30cf\u30e9\u30bf\u30b1",
999
+ "\u821e\u8338",
1000
+ "\u304d\u306e\u3053",
1001
+ "\u8033",
1002
+ "\u30c8\u30a4\u30ec\u30c3\u30c8\u30da\u30fc\u30d1\u30fc"
1003
+ ]
1004
+ }
CLIP_benchmark/clip_benchmark/datasets/jp_zeroshot_classification_templates.json ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "imagenet1k": [
3
+ "{c}\u306e\u60aa\u3044\u5199\u771f",
4
+ "\u591a\u304f\u306e{c}\u306e\u5199\u771f",
5
+ "{c}\u306e\u5f6b\u523b",
6
+ "\u898b\u3065\u3089\u3044{c}\u306e\u5199\u771f",
7
+ "{c}\u306e\u4f4e\u89e3\u50cf\u5ea6\u5199\u771f",
8
+ "{c}\u306e\u30ec\u30f3\u30c0\u30ea\u30f3\u30b0",
9
+ "{c}\u306e\u843d\u66f8\u304d",
10
+ "{c}\u306e\u30c8\u30ea\u30df\u30f3\u30b0\u5199\u771f",
11
+ "{c}\u306e\u30bf\u30c8\u30a5\u30fc",
12
+ "\u523a\u7e4d\u3055\u308c\u305f{c}",
13
+ "{c}\u306e\u660e\u308b\u3044\u5199\u771f",
14
+ "\u304d\u308c\u3044\u306a{c}\u306e\u5199\u771f",
15
+ "\u6c5a\u308c\u305f{c}\u306e\u5199\u771f",
16
+ "{c}\u306e\u6697\u3044\u5199\u771f",
17
+ "{c}\u306e\u7d75",
18
+ "\u79c1\u306e{c}\u306e\u5199\u771f",
19
+ "\u30d7\u30e9\u30b9\u30c1\u30c3\u30af\u88fd\u306e{c}",
20
+ "\u304b\u3063\u3053\u3044\u3044{c}\u306e\u5199\u771f",
21
+ "{c}\u306e\u30af\u30ed\u30fc\u30ba\u30a2\u30c3\u30d7\u5199\u771f",
22
+ "{c}\u306e\u767d\u9ed2\u5199\u771f",
23
+ "{c}\u306e\u30d4\u30af\u30bb\u30eb\u5199\u771f",
24
+ "jpeg\u3067\u52a0\u5de5\u3057\u305f{c}\u306e\u5199\u771f",
25
+ "{c}\u306e\u307c\u3084\u3051\u305f\u5199\u771f",
26
+ "{c}\u306e\u5199\u771f",
27
+ "{c}\u306e\u826f\u3044\u5199\u771f",
28
+ "\u30b2\u30fc\u30e0\u306b\u767b\u5834\u3059\u308b{c}",
29
+ "\u6298\u308a\u7d19\u3067\u4f5c\u3063\u305f{c}",
30
+ "{c}\u306e\u30b9\u30b1\u30c3\u30c1",
31
+ "\u304a\u3082\u3061\u3083\u306e{c}",
32
+ "{c}\u306e\u6f14\u51fa",
33
+ "\u5927\u304d\u306a{c}\u306e\u5199\u771f",
34
+ "\u7d20\u6575\u306a{c}\u306e\u5199\u771f",
35
+ "\u5947\u5999\u306a{c}\u306e\u5199\u771f",
36
+ "\u6f2b\u753b\u306e{c}",
37
+ "{c}\u306e\u82b8\u8853",
38
+ "{c}\u306e\u306c\u3044\u3050\u308b\u307f",
39
+ "\u5c0f\u3055\u306a{c}\u306e\u5199\u771f"
40
+ ]
41
+ }
CLIP_benchmark/clip_benchmark/datasets/kitti.py ADDED
@@ -0,0 +1,209 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2019 Google LLC.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ """Implements Kitti data class."""
17
+
18
+ from __future__ import absolute_import
19
+ from __future__ import division
20
+ from __future__ import print_function
21
+
22
+ import numpy as np
23
+ import task_adaptation.data.base as base
24
+ from task_adaptation.registry import Registry
25
+ import tensorflow.compat.v1 as tf
26
+ import tensorflow_datasets as tfds
27
+
28
+
29
+ def _count_all_pp(x):
30
+ """Count all objects."""
31
+ # Count distribution (thresholded at 15):
32
+
33
+ label = tf.math.minimum(tf.size(x["objects"]["type"]) - 1, 8)
34
+ return {"image": x["image"], "label": label}
35
+
36
+
37
+ def _count_vehicles_pp(x):
38
+ """Counting vehicles."""
39
+ # Label distribution:
40
+
41
+ vehicles = tf.where(x["objects"]["type"] < 3) # Car, Van, Truck.
42
+ # Cap at 3.
43
+ label = tf.math.minimum(tf.size(vehicles), 3)
44
+ return {"image": x["image"], "label": label}
45
+
46
+
47
+ def _count_left_pp(x):
48
+ """Count objects on the left hand side of the camera."""
49
+ # Count distribution (thresholded at 15):
50
+
51
+ # Location feature contains (x, y, z) in meters w.r.t. the camera.
52
+ objects_on_left = tf.where(x["objects"]["location"][:, 0] < 0)
53
+ label = tf.math.minimum(tf.size(objects_on_left), 8)
54
+ return {"image": x["image"], "label": label}
55
+
56
+
57
+ def _count_far_pp(x):
58
+ """Counts objects far from the camera."""
59
+ # Threshold removes ~half of the objects.
60
+ # Count distribution (thresholded at 15):
61
+
62
+ # Location feature contains (x, y, z) in meters w.r.t. the camera.
63
+ distant_objects = tf.where(x["objects"]["location"][:, 2] >= 25)
64
+ label = tf.math.minimum(tf.size(distant_objects), 8)
65
+ return {"image": x["image"], "label": label}
66
+
67
+
68
+ def _count_near_pp(x):
69
+ """Counts objects close to the camera."""
70
+ # Threshold removes ~half of the objects.
71
+ # Count distribution:
72
+
73
+ # Location feature contains (x, y, z) in meters w.r.t. the camera.
74
+ close_objects = tf.where(x["objects"]["location"][:, 2] < 25)
75
+ label = tf.math.minimum(tf.size(close_objects), 8)
76
+ return {"image": x["image"], "label": label}
77
+
78
+
79
+ def _closest_object_distance_pp(x):
80
+ """Predict the distance to the closest object."""
81
+ # Label distribution:
82
+
83
+ # Location feature contains (x, y, z) in meters w.r.t. the camera.
84
+ dist = tf.reduce_min(x["objects"]["location"][:, 2])
85
+ thrs = np.array([-100, 5.6, 8.4, 13.4, 23.4])
86
+ label = tf.reduce_max(tf.where((thrs - dist) < 0))
87
+ return {"image": x["image"], "label": label}
88
+
89
+
90
+ def _closest_vehicle_distance_pp(x):
91
+ """Predict the distance to the closest vehicle."""
92
+ # Label distribution:
93
+
94
+ # Location feature contains (x, y, z) in meters w.r.t. the camera.
95
+ vehicles = tf.where(x["objects"]["type"] < 3) # Car, Van, Truck.
96
+ vehicle_z = tf.gather(params=x["objects"]["location"][:, 2], indices=vehicles)
97
+ vehicle_z = tf.concat([vehicle_z, tf.constant([[1000.0]])], axis=0)
98
+ dist = tf.reduce_min(vehicle_z)
99
+ # Results in a uniform distribution over three distances, plus one class for
100
+ # "no vehicle".
101
+ thrs = np.array([-100.0, 8.0, 20.0, 999.0])
102
+ label = tf.reduce_max(tf.where((thrs - dist) < 0))
103
+ return {"image": x["image"], "label": label}
104
+
105
+
106
+ def _closest_object_x_location_pp(x):
107
+ """Predict the absolute x position of the closest object."""
108
+ # Label distribution:
109
+
110
+ # Location feature contains (x, y, z) in meters w.r.t. the camera.
111
+ idx = tf.math.argmin(x["objects"]["location"][:, 2])
112
+ xloc = x["objects"]["location"][idx, 0]
113
+ thrs = np.array([-100, -6.4, -3.5, 0.0, 3.3, 23.9])
114
+ label = tf.reduce_max(tf.where((thrs - xloc) < 0))
115
+ return {"image": x["image"], "label": label}
116
+
117
+
118
+ _TASK_DICT = {
119
+ "count_all": {
120
+ "preprocess_fn": _count_all_pp,
121
+ "num_classes": 16,
122
+ },
123
+ "count_left": {
124
+ "preprocess_fn": _count_left_pp,
125
+ "num_classes": 16,
126
+ },
127
+ "count_far": {
128
+ "preprocess_fn": _count_far_pp,
129
+ "num_classes": 16,
130
+ },
131
+ "count_near": {
132
+ "preprocess_fn": _count_near_pp,
133
+ "num_classes": 16,
134
+ },
135
+ "closest_object_distance": {
136
+ "preprocess_fn": _closest_object_distance_pp,
137
+ "num_classes": 5,
138
+ },
139
+ "closest_object_x_location": {
140
+ "preprocess_fn": _closest_object_x_location_pp,
141
+ "num_classes": 5,
142
+ },
143
+ "count_vehicles": {
144
+ "preprocess_fn": _count_vehicles_pp,
145
+ "num_classes": 4,
146
+ },
147
+ "closest_vehicle_distance": {
148
+ "preprocess_fn": _closest_vehicle_distance_pp,
149
+ "num_classes": 4,
150
+ },
151
+ }
152
+
153
+
154
+ @Registry.register("data.kitti", "class")
155
+ class KittiData(base.ImageTfdsData):
156
+ """Provides Kitti dataset.
157
+
158
+ Six tasks are supported:
159
+ 1. Count the number of objects.
160
+ 2. Count the number of objects on the left hand side of the camera.
161
+ 3. Count the number of objects in the foreground.
162
+ 4. Count the number of objects in the background.
163
+ 5. Predict the distance of the closest object.
164
+ 6. Predict the x-location (w.r.t. the camera) of the closest object.
165
+ """
166
+
167
+ def __init__(self, task, data_dir=None):
168
+
169
+ if task not in _TASK_DICT:
170
+ raise ValueError("Unknown task: %s" % task)
171
+
172
+ dataset_builder = tfds.builder("kitti:3.3.0", data_dir=data_dir)
173
+ dataset_builder.download_and_prepare()
174
+
175
+ tfds_splits = {
176
+ "train": "train",
177
+ "val": "validation",
178
+ "trainval": "train+validation",
179
+ "test": "test",
180
+ "train800": "train[:800]",
181
+ "val200": "validation[:200]",
182
+ "train800val200": "train[:800]+validation[:200]",
183
+ }
184
+
185
+ # Example counts are retrieved from the tensorflow dataset info.
186
+ train_count = dataset_builder.info.splits[tfds.Split.TRAIN].num_examples
187
+ val_count = dataset_builder.info.splits[tfds.Split.VALIDATION].num_examples
188
+ test_count = dataset_builder.info.splits[tfds.Split.TEST].num_examples
189
+ # Creates a dict with example counts for each split.
190
+ num_samples_splits = {
191
+ "train": train_count,
192
+ "val": val_count,
193
+ "trainval": train_count + val_count,
194
+ "test": test_count,
195
+ "train800": 800,
196
+ "val200": 200,
197
+ "train800val200": 1000,
198
+ }
199
+
200
+ task = _TASK_DICT[task]
201
+ base_preprocess_fn = task["preprocess_fn"]
202
+ super(KittiData, self).__init__(
203
+ dataset_builder=dataset_builder,
204
+ tfds_splits=tfds_splits,
205
+ num_samples_splits=num_samples_splits,
206
+ num_preprocessing_threads=400,
207
+ shuffle_buffer_size=10000,
208
+ base_preprocess_fn=base_preprocess_fn,
209
+ num_classes=task["num_classes"])
CLIP_benchmark/clip_benchmark/datasets/multilingual_mscoco.py ADDED
@@ -0,0 +1,91 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from subprocess import call
2
+ import os, json
3
+
4
+ from torchvision.datasets import VisionDataset
5
+ from PIL import Image
6
+
7
+
8
+ GITHUB_MAIN_ORIGINAL_ANNOTATION_PATH = 'https://github.com/mehdidc/retrieval_annotations/releases/download/1.0.0/coco_{}_karpathy.json'
9
+ GITHUB_MAIN_PATH = 'https://raw.githubusercontent.com/adobe-research/Cross-lingual-Test-Dataset-XTD10/main/XTD10/'
10
+ SUPPORTED_LANGUAGES = ['es', 'it', 'ko', 'pl', 'ru', 'tr', 'zh', 'en']
11
+
12
+ IMAGE_INDEX_FILE = 'mscoco-multilingual_index.json'
13
+ IMAGE_INDEX_FILE_DOWNLOAD_NAME = 'test_image_names.txt'
14
+
15
+ CAPTIONS_FILE_DOWNLOAD_NAME = 'test_1kcaptions_{}.txt'
16
+ CAPTIONS_FILE_NAME = 'multilingual_mscoco_captions-{}.json'
17
+
18
+ ORIGINAL_ANNOTATION_FILE_NAME = 'coco_{}_karpathy.json'
19
+
20
+
21
+
22
+ class Multilingual_MSCOCO(VisionDataset):
23
+
24
+ def __init__(self, root, ann_file, transform=None, target_transform=None):
25
+ super().__init__(root, transform=transform, target_transform=target_transform)
26
+ self.ann_file = os.path.expanduser(ann_file)
27
+ with open(ann_file, 'r') as fp:
28
+ data = json.load(fp)
29
+
30
+ self.data = [(img_path, txt) for img_path, txt in zip(data['image_paths'], data['annotations'])]
31
+
32
+ def __getitem__(self, index):
33
+ img, captions = self.data[index]
34
+
35
+ # Image
36
+ img = Image.open(os.path.join(self.root, img)).convert("RGB")
37
+ if self.transform is not None:
38
+ img = self.transform(img)
39
+
40
+ # Captions
41
+ target = [captions, ]
42
+ if self.target_transform is not None:
43
+ target = self.target_transform(target)
44
+
45
+ return img, target
46
+
47
+
48
+ def __len__(self) -> int:
49
+ return len(self.data)
50
+
51
+
52
+ def _get_downloadable_file(filename, download_url, is_json=True):
53
+ if (os.path.exists(filename) == False):
54
+ print("Downloading", download_url)
55
+ call("wget {} -O {}".format(download_url, filename), shell=True)
56
+ with open(filename, 'r') as fp:
57
+ if (is_json):
58
+ return json.load(fp)
59
+ return [line.strip() for line in fp.readlines()]
60
+
61
+
62
+ def create_annotation_file(root, lang_code):
63
+ print("Downloading multilingual_ms_coco index file")
64
+ download_path = os.path.join(GITHUB_MAIN_PATH, IMAGE_INDEX_FILE_DOWNLOAD_NAME)
65
+ target_images = _get_downloadable_file("multilingual_coco_images.txt", download_path, False)
66
+
67
+ print("Downloading multilingual_ms_coco captions:", lang_code)
68
+ download_path = os.path.join(GITHUB_MAIN_PATH, CAPTIONS_FILE_DOWNLOAD_NAME.format(lang_code))
69
+ target_captions = _get_downloadable_file('raw_multilingual_coco_captions_{}.txt'.format(lang_code), download_path, False)
70
+
71
+ number_of_missing_images = 0
72
+ valid_images, valid_annotations, valid_indicies = [], [], []
73
+ for i, (img, txt) in enumerate(zip(target_images, target_captions)):
74
+ # Create a new file name that includes the root split
75
+ root_split = 'val2014' if 'val' in img else 'train2014'
76
+ filename_with_root_split = "{}/{}".format(root_split, img)
77
+
78
+ if (os.path.exists(filename_with_root_split)):
79
+ print("Missing image file", img)
80
+ number_of_missing_images += 1
81
+ continue
82
+
83
+ valid_images.append(filename_with_root_split)
84
+ valid_annotations.append(txt)
85
+ valid_indicies.append(i)
86
+
87
+ if (number_of_missing_images > 0):
88
+ print("*** WARNING *** missing {} files.".format(number_of_missing_images))
89
+
90
+ with open(os.path.join(root, CAPTIONS_FILE_NAME.format(lang_code)), 'w') as fp:
91
+ json.dump({'image_paths': valid_images, 'annotations': valid_annotations, 'indicies': valid_indicies}, fp)
CLIP_benchmark/clip_benchmark/datasets/nllb_dist13b_prompts.json ADDED
The diff for this file is too large to render. See raw diff
 
CLIP_benchmark/clip_benchmark/datasets/objectnet.py ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Code adapted from https://github.com/mlfoundations/wise-ft/blob/master/src/datasets/objectnet.py
3
+ Thanks to the authors of wise-ft
4
+ """
5
+
6
+ import os
7
+ import json
8
+ from pathlib import Path
9
+ import PIL
10
+
11
+ import numpy as np
12
+
13
+ import torch
14
+ from torchvision import datasets
15
+ from torchvision.transforms import Compose
16
+
17
+ from pathlib import Path
18
+
19
+ def get_metadata(folder):
20
+ metadata = Path(folder)
21
+
22
+ with open(metadata / 'folder_to_objectnet_label.json', 'r') as f:
23
+ folder_map = json.load(f)
24
+ folder_map = {v: k for k, v in folder_map.items()}
25
+ with open(metadata / 'objectnet_to_imagenet_1k.json', 'r') as f:
26
+ objectnet_map = json.load(f)
27
+
28
+ with open(metadata / 'pytorch_to_imagenet_2012_id.json', 'r') as f:
29
+ pytorch_map = json.load(f)
30
+ pytorch_map = {v: k for k, v in pytorch_map.items()}
31
+
32
+ with open(metadata / 'imagenet_to_label_2012_v2', 'r') as f:
33
+ imagenet_map = {v.strip(): str(pytorch_map[i]) for i, v in enumerate(f)}
34
+
35
+ folder_to_ids, class_sublist = {}, []
36
+ classnames = []
37
+ for objectnet_name, imagenet_names in objectnet_map.items():
38
+ imagenet_names = imagenet_names.split('; ')
39
+ imagenet_ids = [int(imagenet_map[imagenet_name]) for imagenet_name in imagenet_names]
40
+ class_sublist.extend(imagenet_ids)
41
+ folder_to_ids[folder_map[objectnet_name]] = imagenet_ids
42
+
43
+ class_sublist = sorted(class_sublist)
44
+ class_sublist_mask = [(i in class_sublist) for i in range(1000)]
45
+ classname_map = {v: k for k, v in folder_map.items()}
46
+ return class_sublist, class_sublist_mask, folder_to_ids, classname_map
47
+
48
+ class ObjectNetDataset(datasets.ImageFolder):
49
+
50
+ def __init__(self, root, transform):
51
+ (self._class_sublist,
52
+ self.class_sublist_mask,
53
+ self.folders_to_ids,
54
+ self.classname_map) = get_metadata(root)
55
+ subdir = os.path.join(root, "objectnet-1.0", "images")
56
+ label_map = {name: idx for idx, name in enumerate(sorted(list(self.folders_to_ids.keys())))}
57
+ self.label_map = label_map
58
+ super().__init__(subdir, transform=transform)
59
+ self.samples = [
60
+ d for d in self.samples
61
+ if os.path.basename(os.path.dirname(d[0])) in self.label_map
62
+ ]
63
+ self.imgs = self.samples
64
+ self.classes = sorted(list(self.folders_to_ids.keys()))
65
+ self.classes = [self.classname_map[c].lower() for c in self.classes]
66
+
67
+ def __len__(self):
68
+ return len(self.samples)
69
+
70
+ def __getitem__(self, index):
71
+ path, target = self.samples[index]
72
+ sample = self.loader(path)
73
+ if self.transform is not None:
74
+ sample = self.transform(sample)
75
+ label = os.path.basename(os.path.dirname(path))
76
+ return sample, self.label_map[label]
CLIP_benchmark/clip_benchmark/datasets/sugar_crepe.py ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ from torch.utils.data import Dataset
3
+ from PIL import Image
4
+ import json
5
+ class SugarCrepe(Dataset):
6
+
7
+ def __init__(self, root, ann_file, transform=None):
8
+ self.root = root
9
+ self.ann = json.load(open(ann_file))
10
+ self.transform = transform
11
+
12
+ def __getitem__(self, idx):
13
+ data = self.ann[str(idx)]
14
+ img = Image.open(os.path.join(self.root, data['filename']))
15
+ if self.transform is not None:
16
+ img = self.transform(img)
17
+ caption = data['caption']
18
+ negative_caption = data['negative_caption']
19
+ return img, [caption, negative_caption]
20
+
21
+ def __len__(self):
22
+ return len(self.ann)
CLIP_benchmark/clip_benchmark/datasets/tfds.py ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from PIL import Image
3
+
4
+
5
+ def download_tfds_dataset(name, data_dir=None):
6
+ import tensorflow_datasets as tfds
7
+ import timm
8
+ builder = tfds.builder(name, data_dir=data_dir)
9
+ builder.download_and_prepare()
10
+
11
+ def disable_gpus_on_tensorflow():
12
+ import tensorflow as tf
13
+ tf.config.set_visible_devices([], 'GPU')
14
+
15
+ class VTABIterableDataset(torch.utils.data.IterableDataset):
16
+
17
+ def __init__(self, tfds_dataset, split="test", input_name="image", label_name="label", input_mode="RGB", transform=None, target_transform=None, classes=None):
18
+ self.tfds_dataset = tfds_dataset
19
+ self.input_name = input_name
20
+ self.label_name = label_name
21
+ self.transform = transform
22
+ self.target_transform = target_transform
23
+ self.input_mode = input_mode
24
+ self.num_examples = tfds_dataset.get_num_samples(split)
25
+ self.split = split
26
+ if classes is None:
27
+ self.classes = tfds_dataset._dataset_builder.info.features['label'].names
28
+ else:
29
+ self.classes = classes
30
+ def __iter__(self):
31
+ worker_info = torch.utils.data.get_worker_info()
32
+ iterator = self.tfds_dataset.get_tf_data(self.split, batch_size=1, epochs=1, for_eval=True)
33
+ if worker_info is not None:
34
+ iterator = iterator.shard(index=worker_info.id, num_shards=worker_info.num_workers)
35
+ nb = 0
36
+ for data in iterator:
37
+ inputs = (data[self.input_name].numpy())
38
+ labels = data[self.label_name].numpy()
39
+ for input, label in zip(inputs, labels):
40
+ input = Image.fromarray(input, mode=self.input_mode)
41
+ if self.transform is not None:
42
+ input = self.transform(input)
43
+ if self.target_transform is not None:
44
+ label = self.target_transform(label)
45
+ yield input, label
46
+
47
+ def __len__(self):
48
+ return self.num_examples
CLIP_benchmark/clip_benchmark/datasets/voc2007.py ADDED
@@ -0,0 +1,248 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Code from https://github.com/SsnL/dataset-distillation/blob/master/datasets/pascal_voc.py , thanks to the authors
2
+ """Dataset setting and data loader for PASCAL VOC 2007 as a classification task.
3
+
4
+ Modified from
5
+ https://github.com/Cadene/pretrained-models.pytorch/blob/56aa8c921819d14fb36d7248ab71e191b37cb146/pretrainedmodels/datasets/voc.py
6
+ """
7
+
8
+ import os
9
+ import os.path
10
+ import tarfile
11
+ import xml.etree.ElementTree as ET
12
+
13
+ import torch.utils.data as data
14
+ import torchvision
15
+ from PIL import Image
16
+ from urllib.parse import urlparse
17
+ import torch
18
+
19
+ object_categories = ['aeroplane', 'bicycle', 'bird', 'boat',
20
+ 'bottle', 'bus', 'car', 'cat', 'chair',
21
+ 'cow', 'diningtable', 'dog', 'horse',
22
+ 'motorbike', 'person', 'pottedplant',
23
+ 'sheep', 'sofa', 'train', 'tvmonitor']
24
+
25
+ category_to_idx = {c: i for i, c in enumerate(object_categories)}
26
+
27
+ urls = {
28
+ 'devkit': 'http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCdevkit_08-Jun-2007.tar',
29
+ 'trainval_2007': 'http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtrainval_06-Nov-2007.tar',
30
+ 'test_images_2007': 'http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtest_06-Nov-2007.tar',
31
+ 'test_anno_2007': 'http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtestnoimgs_06-Nov-2007.tar',
32
+ }
33
+
34
+
35
+ def download_url(url, path):
36
+ root, filename = os.path.split(path)
37
+ torchvision.datasets.utils.download_url(url, root=root, filename=filename, md5=None)
38
+
39
+
40
+ def download_voc2007(root):
41
+ path_devkit = os.path.join(root, 'VOCdevkit')
42
+ path_images = os.path.join(root, 'VOCdevkit', 'VOC2007', 'JPEGImages')
43
+ tmpdir = os.path.join(root, 'tmp')
44
+
45
+ # create directory
46
+ if not os.path.exists(root):
47
+ os.makedirs(root)
48
+
49
+ if not os.path.exists(path_devkit):
50
+
51
+ if not os.path.exists(tmpdir):
52
+ os.makedirs(tmpdir)
53
+
54
+ parts = urlparse(urls['devkit'])
55
+ filename = os.path.basename(parts.path)
56
+ cached_file = os.path.join(tmpdir, filename)
57
+
58
+ if not os.path.exists(cached_file):
59
+ download_url(urls['devkit'], cached_file)
60
+
61
+ # extract file
62
+ print('[dataset] Extracting tar file {file} to {path}'.format(file=cached_file, path=root))
63
+ cwd = os.getcwd()
64
+ tar = tarfile.open(cached_file, "r")
65
+ os.chdir(root)
66
+ tar.extractall()
67
+ tar.close()
68
+ os.chdir(cwd)
69
+ print('[dataset] Done!')
70
+
71
+ # train/val images/annotations
72
+ if not os.path.exists(path_images):
73
+
74
+ # download train/val images/annotations
75
+ parts = urlparse(urls['trainval_2007'])
76
+ filename = os.path.basename(parts.path)
77
+ cached_file = os.path.join(tmpdir, filename)
78
+
79
+ if not os.path.exists(cached_file):
80
+ download_url(urls['trainval_2007'], cached_file)
81
+
82
+ # extract file
83
+ print('[dataset] Extracting tar file {file} to {path}'.format(file=cached_file, path=root))
84
+ cwd = os.getcwd()
85
+ tar = tarfile.open(cached_file, "r")
86
+ os.chdir(root)
87
+ tar.extractall()
88
+ tar.close()
89
+ os.chdir(cwd)
90
+ print('[dataset] Done!')
91
+
92
+ # test annotations
93
+ test_anno = os.path.join(path_devkit, 'VOC2007/ImageSets/Main/aeroplane_test.txt')
94
+ if not os.path.exists(test_anno):
95
+
96
+ # download test annotations
97
+ parts = urlparse(urls['test_images_2007'])
98
+ filename = os.path.basename(parts.path)
99
+ cached_file = os.path.join(tmpdir, filename)
100
+
101
+ if not os.path.exists(cached_file):
102
+ download_url(urls['test_images_2007'], cached_file)
103
+
104
+ # extract file
105
+ print('[dataset] Extracting tar file {file} to {path}'.format(file=cached_file, path=root))
106
+ cwd = os.getcwd()
107
+ tar = tarfile.open(cached_file, "r")
108
+ os.chdir(root)
109
+ tar.extractall()
110
+ tar.close()
111
+ os.chdir(cwd)
112
+ print('[dataset] Done!')
113
+
114
+ # test images
115
+ test_image = os.path.join(path_devkit, 'VOC2007/JPEGImages/000001.jpg')
116
+ if not os.path.exists(test_image):
117
+
118
+ # download test images
119
+ parts = urlparse(urls['test_anno_2007'])
120
+ filename = os.path.basename(parts.path)
121
+ cached_file = os.path.join(tmpdir, filename)
122
+
123
+ if not os.path.exists(cached_file):
124
+ download_url(urls['test_anno_2007'], cached_file)
125
+
126
+ # extract file
127
+ print('[dataset] Extracting tar file {file} to {path}'.format(file=cached_file, path=root))
128
+ cwd = os.getcwd()
129
+ tar = tarfile.open(cached_file, "r")
130
+ os.chdir(root)
131
+ tar.extractall()
132
+ tar.close()
133
+ os.chdir(cwd)
134
+ print('[dataset] Done!')
135
+
136
+
137
+ def read_split(root, dataset, split):
138
+ base_path = os.path.join(root, 'VOCdevkit', dataset, 'ImageSets', 'Main')
139
+ filename = os.path.join(base_path, object_categories[0] + '_' + split + '.txt')
140
+
141
+ with open(filename, 'r') as f:
142
+ paths = []
143
+ for line in f.readlines():
144
+ line = line.strip().split()
145
+ if len(line) > 0:
146
+ assert len(line) == 2
147
+ paths.append(line[0])
148
+
149
+ return tuple(paths)
150
+
151
+
152
+ def read_bndbox(root, dataset, paths):
153
+ xml_base = os.path.join(root, 'VOCdevkit', dataset, 'Annotations')
154
+ instances = []
155
+ for path in paths:
156
+ xml = ET.parse(os.path.join(xml_base, path + '.xml'))
157
+ for obj in xml.findall('object'):
158
+ c = obj[0]
159
+ assert c.tag == 'name', c.tag
160
+ c = category_to_idx[c.text]
161
+ bndbox = obj.find('bndbox')
162
+ xmin = int(bndbox[0].text) # left
163
+ ymin = int(bndbox[1].text) # top
164
+ xmax = int(bndbox[2].text) # right
165
+ ymax = int(bndbox[3].text) # bottom
166
+ instances.append((path, (xmin, ymin, xmax, ymax), c))
167
+ return instances
168
+
169
+
170
+ class PASCALVoc2007(data.Dataset):
171
+ """
172
+ Multi-label classification problem for voc2007
173
+ labels are of one hot of shape (C,), denoting the presence/absence
174
+ of each class in each image, where C is the number of classes.
175
+ """
176
+ def __init__(self, root, set, transform=None, download=False, target_transform=None):
177
+ self.root = root
178
+ self.path_devkit = os.path.join(root, 'VOCdevkit')
179
+ self.path_images = os.path.join(root, 'VOCdevkit', 'VOC2007', 'JPEGImages')
180
+ self.transform = transform
181
+ self.target_transform = target_transform
182
+
183
+ # download dataset
184
+ if download:
185
+ download_voc2007(self.root)
186
+
187
+ paths = read_split(self.root, 'VOC2007', set)
188
+ bndboxes = read_bndbox(self.root, 'VOC2007', paths)
189
+ labels = torch.zeros(len(paths), len(object_categories))
190
+ path_index = {}
191
+ for i, p in enumerate(paths):
192
+ path_index[p] = i
193
+ for path, bbox, c in bndboxes:
194
+ labels[path_index[path], c] = 1
195
+ self.labels = labels
196
+ self.classes = object_categories
197
+ self.paths = paths
198
+
199
+ def __getitem__(self, index):
200
+ path = self.paths[index]
201
+ img = Image.open(os.path.join(self.path_images, path + '.jpg')).convert('RGB')
202
+ target = self.labels[index]
203
+ if self.transform is not None:
204
+ img = self.transform(img)
205
+ if self.target_transform is not None:
206
+ target = self.target_transform(target)
207
+ return img, target
208
+
209
+ def __len__(self):
210
+ return len(self.paths)
211
+
212
+ class PASCALVoc2007Cropped(data.Dataset):
213
+ """
214
+ voc2007 is originally object detection and multi-label.
215
+ In this version, we just convert it to single-label per image classification
216
+ problem by looping over bounding boxes in the dataset and cropping the relevant
217
+ object.
218
+ """
219
+ def __init__(self, root, set, transform=None, download=False, target_transform=None):
220
+ self.root = root
221
+ self.path_devkit = os.path.join(root, 'VOCdevkit')
222
+ self.path_images = os.path.join(root, 'VOCdevkit', 'VOC2007', 'JPEGImages')
223
+ self.transform = transform
224
+ self.target_transform = target_transform
225
+
226
+ # download dataset
227
+ if download:
228
+ download_voc2007(self.root)
229
+
230
+ paths = read_split(self.root, 'VOC2007', set)
231
+ self.bndboxes = read_bndbox(self.root, 'VOC2007', paths)
232
+ self.classes = object_categories
233
+
234
+ print('[dataset] VOC 2007 classification set=%s number of classes=%d number of bndboxes=%d' % (
235
+ set, len(self.classes), len(self.bndboxes)))
236
+
237
+ def __getitem__(self, index):
238
+ path, crop, target = self.bndboxes[index]
239
+ img = Image.open(os.path.join(self.path_images, path + '.jpg')).convert('RGB')
240
+ img = img.crop(crop)
241
+ if self.transform is not None:
242
+ img = self.transform(img)
243
+ if self.target_transform is not None:
244
+ target = self.target_transform(target)
245
+ return img, target
246
+
247
+ def __len__(self):
248
+ return len(self.bndboxes)
CLIP_benchmark/clip_benchmark/metrics/__init__.py ADDED
File without changes
CLIP_benchmark/clip_benchmark/metrics/captioning.py ADDED
@@ -0,0 +1,99 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ from open_clip import tokenize
3
+ from tqdm.auto import tqdm
4
+ from open_clip.tokenizer import _tokenizer
5
+
6
+
7
+ from pycocoevalcap.tokenizer.ptbtokenizer import PTBTokenizer
8
+ from pycocoevalcap.bleu.bleu import Bleu
9
+ from pycocoevalcap.meteor.meteor import Meteor
10
+ from pycocoevalcap.rouge.rouge import Rouge
11
+ from pycocoevalcap.cider.cider import Cider
12
+ from pycocoevalcap.spice.spice import Spice
13
+
14
+
15
+ """
16
+ Code adapted from https://github.com/salaniz/pycocoevalcap/blob/master/eval.py
17
+ Thanks to @salaniz for the code!
18
+ """
19
+ class COCOEvalCap:
20
+ def __init__(self, results):
21
+ self.evalImgs = []
22
+ self.eval = {}
23
+ self.imgToEval = {}
24
+ self.results = results
25
+ def evaluate(self):
26
+ gts = {}
27
+ res = {}
28
+ for imgId, r in enumerate(self.results):
29
+ gts[imgId] = r['true']
30
+ res[imgId] = r['gen']
31
+ # =================================================
32
+ # Set up scorers
33
+ # =================================================
34
+ print('tokenization...')
35
+ tokenizer = PTBTokenizer()
36
+ gts = tokenizer.tokenize(gts)
37
+ res = tokenizer.tokenize(res)
38
+
39
+ # =================================================
40
+ # Set up scorers
41
+ # =================================================
42
+ print('setting up scorers...')
43
+ scorers = [
44
+ (Bleu(4), ["Bleu_1", "Bleu_2", "Bleu_3", "Bleu_4"]),
45
+ (Meteor(),"METEOR"),
46
+ (Rouge(), "ROUGE_L"),
47
+ (Cider(), "CIDEr"),
48
+ (Spice(), "SPICE")
49
+ ]
50
+
51
+ # =================================================
52
+ # Compute scores
53
+ # =================================================
54
+ for scorer, method in scorers:
55
+ print('computing %s score...'%(scorer.method()))
56
+ score, scores = scorer.compute_score(gts, res)
57
+ if type(method) == list:
58
+ for sc, scs, m in zip(score, scores, method):
59
+ self.setEval(sc, m)
60
+ self.setImgToEvalImgs(scs, gts.keys(), m)
61
+ print("%s: %0.3f"%(m, sc))
62
+ else:
63
+ self.setEval(score, method)
64
+ self.setImgToEvalImgs(scores, gts.keys(), method)
65
+ print("%s: %0.3f"%(method, score))
66
+ self.setEvalImgs()
67
+
68
+ def setEval(self, score, method):
69
+ self.eval[method] = score
70
+
71
+ def setImgToEvalImgs(self, scores, imgIds, method):
72
+ for imgId, score in zip(imgIds, scores):
73
+ if not imgId in self.imgToEval:
74
+ self.imgToEval[imgId] = {}
75
+ self.imgToEval[imgId]["image_id"] = imgId
76
+ self.imgToEval[imgId][method] = score
77
+
78
+ def setEvalImgs(self):
79
+ self.evalImgs = [eval for imgId, eval in self.imgToEval.items()]
80
+
81
+ def evaluate(model, dataloader, batch_size, device, transform, train_dataloader=None, num_workers=None, amp=True, verbose=False):
82
+ results = []
83
+ image_id = 0
84
+ gt = []
85
+ for idx, (img, captions) in enumerate(tqdm(dataloader)):
86
+ out = model.generate(img.to(device))
87
+ decoded = [_tokenizer.decode(i).split("<end_of_text>")[0].replace("<start_of_text>", "").strip() for i in out.cpu().numpy()]
88
+ for pred, true in zip(decoded, captions):
89
+ true = [{'caption': t} for t in true]
90
+ pred = [{'caption': pred}]
91
+ results.append({"image_id":image_id, "gen":pred, "true": true})
92
+ image_id += 1
93
+ coco_eval = COCOEvalCap(results)
94
+ coco_eval.evaluate()
95
+ metrics = coco_eval.eval
96
+ # print output evaluation scores
97
+ for metric, score in metrics.items():
98
+ print(f'{metric}: {score:.3f}')
99
+ return metrics