File size: 430 Bytes
a636bcb
af2c647
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
IGNORE_TAGS = ["script", "style", "button"]
IGNORE_CLASSES = [
    "sidebar",
    "footer",
    "related",
    "comment",
    "topbar",
    # "menu",
    "offcanvas",
    "navbar",
    "post_side",
]

IGNORE_HOSTS = [
    "weibo.com",
    "hymson.com",
]

REQUESTS_HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36 Edg/111.0.1661.62",
}