1 import urllib.request
2 import urllib.parse
3
4
5 #拉钩网，反爬虫策略很好，请求头需要添加Referer（引用; 引用页; 推荐人; 参照页），请求对象需要data=，method='POST'
6 # url = 'https://baike.baidu.com/item/%E8%99%9A%E6%8B%9F%E7%8E%AF%E5%A2%83/6529153?fr=aladdin'
7 url = 'https://www.lagou.com/jobs/list_python?labelWords=&fromSearch=true&suginput='
8 headers = {
9 "User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36",
10 'Referer':'https://www.lagou.com/jobs/list_python?labelWords=&fromSearch=true&suginput='
11 }
12 data = {
13 'first':"true",
14 'pn':1,
15 'kd':'python'
16 }
17 #调用urllib.request中的Request类，创建一个req对象
18 req = urllib.request.Request(url,headers=headers,data=urllib.parse.urlencode(data).encode('utf-8'),method='POST')
19 #将创建的req对象传给urllib.request.urlopen
20 resp = urllib.request.urlopen(req)
21 print(resp.read().decode('utf-8'))
