爬取拉勾网某一词条的检索数据——异步加载

# URL地址栏不发生变化显示数据——异步加载

# 服务器先通过客户端访问的初始URL，返回一个cookie，然后再访问真实url
start_url = 'https://www.lagou.com/jobs/list_%E6%95%B0%E6%8D%AE%E5%88%86%E6%9E%90/p-city_0?&cl=false&fromSearch=true&labelWords=&suginput='

url = 'https://www.lagou.com/jobs/positionAjax.json?needAddtionalResult=false'

headers = {'User-Agent': "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36 SE 2.X MetaSr 1.0",
          'Connection': 'keep-alive',  # 会话保持
           'Referer': 'https://www.lagou.com/jobs/list_%E6%95%B0%E6%8D%AE%E5%88%86%E6%9E%90?labelWords=sug&fromSearch=true&suginput=%E6%95%B0%E6%8D%AE'
          }
data = {'first': 'true',
        'pn': '1',
        'kd': '数据分析'}
# response = requests.post(url, headers = headers, data = data)

# 创建一个session对象
session = requests.session()

# 用session对象先发送一次请求， 获得cookie保持
session.get(start_url, headers = headers)

# 再用一个带着cookies的session发送一个POST请求，这次是一次真正的访问
response = session.post(url, headers = headers, data = data, cookies = session.cookies)
print(response.status_code)
print(response.text)